diff --git a/.gitattributes b/.gitattributes index 05b1a132398..f9c3340cd2d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,3 +2,28 @@ **/ModuleBindings/** linguist-generated=true eol=lf /docs/llms/** linguist-generated=true /docs/llms/*-details.json linguist-generated=false +/tools/stack-bench/** text eol=lf +/tools/stack-bench/**/*.woff2 -text -diff +/tools/stack-bench/qualification-evidence/**/*.json -text whitespace=cr-at-eol +/tools/stack-bench/qualification-evidence/ecommerce-l3-navigation-20260927/** -text -whitespace +/tools/stack-bench/qualification-evidence/ecommerce-l3-readiness-20260927/** -text -whitespace +/tools/stack-bench/qualification-evidence/ecommerce-l3-module-layout-20260927/** -text -whitespace +/tools/stack-bench/qualification-evidence/ecommerce-l3-count-observation-20260927/** -text -whitespace +/tools/stack-bench/qualification-evidence/ecommerce-l3-catalog-support-20260927/** -text -whitespace +/tools/stack-bench/qualification-evidence/ecommerce-l3-readiness-final-20260927/** -text -whitespace +/tools/stack-bench/qualification-evidence/ecommerce-l3-entry-navigation-20260927/** -text -whitespace +/tools/stack-bench/qualification-evidence/ecommerce-l3-audit-fixes-ea80500/** -text -whitespace +/tools/stack-bench/qualification-evidence/ecommerce-l3-crash-order-20260927/** -text -whitespace +/tools/stack-bench/qualification-evidence/ecommerce-l3-signup-writes-20260928/** -text -whitespace +/tools/stack-bench/qualification-evidence/ecommerce-l3-restored-navigation-20260928/** -text -whitespace +/tools/stack-bench/qualification-evidence/ecommerce-l3-faceted-ranking-20260928/Dockerfile -text -whitespace +/tools/stack-bench/qualification-evidence/ecommerce-l3-checkout-readiness-20260928/** -text -whitespace +/tools/stack-bench/qualification-evidence/ecommerce-l3-click-classifier-20260928/** -text -whitespace +/tools/stack-bench/qualification-evidence/ecommerce-l3-probe-removal-20260928/** -text -whitespace + +# Preserve exact measured cookie-control evidence bytes. +tools/stack-bench/qualification-evidence/ecommerce-l3-cookie-session-20260928/** -text -whitespace + +# Preserve exact measured combined-control evidence bytes. +tools/stack-bench/qualification-evidence/ecommerce-l3-combined-20260929/** -text -whitespace +/tools/stack-bench/qualification-evidence/ecommerce-l3-opus55-c34ab8c8a7/** -text -whitespace diff --git a/.gitignore b/.gitignore index 186ed11a6a9..23d963c566e 100644 --- a/.gitignore +++ b/.gitignore @@ -276,6 +276,9 @@ nul # Any local file *.local +# Local working notes and temporary builds +/local-notes/ + # Generated when testing or publishing the TypeScript module test utilities. /crates/bindings-typescript/src/server/test-utils/portable-datastore-wasm/ /crates/bindings-typescript/src/server/test-utils/.portable-datastore-wasm-*/ diff --git a/codex-plugin/plugins/spacetimedb/skills/cli/SKILL.md b/codex-plugin/plugins/spacetimedb/skills/cli/SKILL.md index e7a51487fc1..2b7b0c63728 100644 --- a/codex-plugin/plugins/spacetimedb/skills/cli/SKILL.md +++ b/codex-plugin/plugins/spacetimedb/skills/cli/SKILL.md @@ -43,13 +43,15 @@ spacetime build --debug # faster iteration, slower runtime # Dev mode (auto-rebuild, auto-publish, generates bindings) spacetime dev -spacetime dev --client-lang typescript --module-bindings-path ./client/src/module_bindings +spacetime dev my-database --server local --yes --delete-data=never --client-lang typescript --module-bindings-path ./client/src/module_bindings # Generate client bindings spacetime generate --lang typescript|csharp|rust --out-dir ./bindings --module-path ./server spacetime generate --lang unrealcpp --uproject-dir ./MyGame --module-path ./server --unreal-module-name MyGame ``` +`dev` stays running and watches module changes. `--run "npm run dev"` starts a client command; `--server-only` omits it. `--module-path` selects the module directory when no publish targets exist in `spacetime.json`. Once targets exist, use their configured paths and omit that flag. Separate build/publish/generate commands remain useful for one-shot deployment. + ### Publishing & Deployment ```bash @@ -114,7 +116,7 @@ spacetime server add myserver --url https://my-spacetime.example.com # Set default server spacetime server set-default local -# Test connectivity +# Check connectivity spacetime server ping local # Start local instance @@ -172,10 +174,7 @@ spacetime server ping ``` ### "Schema conflict" -```bash -# Clear data and republish -spacetime publish my-db --delete-data=always --yes -``` +`--delete-data=never` rejects incompatible schema updates without clearing data. A compatible migration preserves existing data; `--delete-data=always` destroys it and is only appropriate for an intentional reset. ### "Build failed" ```bash diff --git a/codex-plugin/plugins/spacetimedb/skills/concepts/SKILL.md b/codex-plugin/plugins/spacetimedb/skills/concepts/SKILL.md index 8ea6e8f9cfb..e152e5d9e72 100644 --- a/codex-plugin/plugins/spacetimedb/skills/concepts/SKILL.md +++ b/codex-plugin/plugins/spacetimedb/skills/concepts/SKILL.md @@ -101,8 +101,8 @@ Lifecycle: Write → Compile → Publish (`spacetime publish`) → Hot-swap (rep ## Identity -- **Identity**: A long-lived, globally unique identifier for a user. -- **ConnectionId**: Identifies a specific client connection. +- **Identity**: A long-lived, globally unique identifier for a user, derived from the token's issuer and subject claims. The same token yields the same identity on every connection. +- **ConnectionId**: Identifies one client connection. A new connection gets a new connection ID; a disconnect ends the connection, not the identity. - Always use `ctx.sender` / `ctx.Sender` / `ctx.sender()` for authorization. SpacetimeDB works with many OIDC providers, including SpacetimeAuth (built-in), Auth0, Clerk, Keycloak, Google, and GitHub. diff --git a/codex-plugin/plugins/spacetimedb/skills/cpp-server/SKILL.md b/codex-plugin/plugins/spacetimedb/skills/cpp-server/SKILL.md index aebb82edf06..3facad75b0e 100644 --- a/codex-plugin/plugins/spacetimedb/skills/cpp-server/SKILL.md +++ b/codex-plugin/plugins/spacetimedb/skills/cpp-server/SKILL.md @@ -138,6 +138,8 @@ SPACETIMEDB_CLIENT_DISCONNECTED(on_disconnect, ReducerContext ctx) { } ``` +Connection hooks run once per connection. The same authenticated principal keeps the same identity (`ctx.sender()`) across connections, while each connection has its own connection ID (`ctx.connection_id()`). A disconnect ends one connection; it does not end the identity, which returns unchanged on the next connection with the same token. Use connection IDs for presence and other connection-scoped state. + ## Authentication & Timestamps ```cpp diff --git a/codex-plugin/plugins/spacetimedb/skills/csharp-server/SKILL.md b/codex-plugin/plugins/spacetimedb/skills/csharp-server/SKILL.md index 1d08ba89fe7..b1037f599c3 100644 --- a/codex-plugin/plugins/spacetimedb/skills/csharp-server/SKILL.md +++ b/codex-plugin/plugins/spacetimedb/skills/csharp-server/SKILL.md @@ -162,7 +162,9 @@ public static void OnConnect(ReducerContext ctx) { ... } public static void OnDisconnect(ReducerContext ctx) { ... } ``` -`ctx.ConnectionId` is `ConnectionId?`, including in connection lifecycle reducers. Check or unwrap it before storing it in a non-nullable column or passing it to an index accessor. +Connection hooks run once per connection. The same authenticated principal keeps the same identity (`ctx.Sender`) across connections, while each connection has its own connection ID (`ctx.ConnectionId`). A disconnect ends one connection; it does not end the identity, which returns unchanged on the next connection with the same token. Use connection IDs for presence and other connection-scoped state. + +`ctx.ConnectionId` is typed `ConnectionId?`. It is present inside connection lifecycle reducers and reducers invoked over a connection, and null in `Init` and scheduled reducers. Check or unwrap it before storing it in a non-nullable column or passing it to an index accessor. ## Views diff --git a/codex-plugin/plugins/spacetimedb/skills/rust-server/SKILL.md b/codex-plugin/plugins/spacetimedb/skills/rust-server/SKILL.md index 0e282242263..ec8e772c9c9 100644 --- a/codex-plugin/plugins/spacetimedb/skills/rust-server/SKILL.md +++ b/codex-plugin/plugins/spacetimedb/skills/rust-server/SKILL.md @@ -150,6 +150,8 @@ pub fn on_connect(ctx: &ReducerContext) { ... } pub fn on_disconnect(ctx: &ReducerContext) { ... } ``` +Connection hooks run once per connection. The same authenticated principal keeps the same identity (`ctx.sender()`) across connections, while each connection has its own connection ID (`ctx.connection_id()`). A disconnect ends one connection; it does not end the identity, which returns unchanged on the next connection with the same token. Use connection IDs for presence and other connection-scoped state. + The current connection ID is available through `ctx.connection_id()` (not a public field) and may be absent outside connection-scoped calls. ## Views diff --git a/codex-plugin/plugins/spacetimedb/skills/typescript-client/SKILL.md b/codex-plugin/plugins/spacetimedb/skills/typescript-client/SKILL.md index 4249daf15a1..29e0881d28f 100644 --- a/codex-plugin/plugins/spacetimedb/skills/typescript-client/SKILL.md +++ b/codex-plugin/plugins/spacetimedb/skills/typescript-client/SKILL.md @@ -152,7 +152,9 @@ Calls made while disconnected fail immediately. Pending reducer and procedure pr ## Gotchas -- **`useTable` rows are `readonly`.** Copy before sorting/mutating, or it fails to type-check: +- **Subscription rows have no presentation order.** A server view's array order does not + define client cache iteration order. `useTable` rows are `readonly`; a sorted copy can + express the application's display order: `const [rows] = useTable(tables.message); const sorted = [...rows].sort(...)`. -- **bigint in JSX.** ids/counts from `t.u64()`/`t.i64()` columns are `bigint`, which React - cannot render. Wrap it: `{Number(row.id)}` or `{String(count)}`. +- **64-bit display values.** `{String(row.id)}` preserves the full `bigint` value. + Conversion to `Number` can lose precision outside JavaScript's safe integer range. diff --git a/codex-plugin/plugins/spacetimedb/skills/typescript-server/SKILL.md b/codex-plugin/plugins/spacetimedb/skills/typescript-server/SKILL.md index 8563a5e9808..43cc00ef472 100644 --- a/codex-plugin/plugins/spacetimedb/skills/typescript-server/SKILL.md +++ b/codex-plugin/plugins/spacetimedb/skills/typescript-server/SKILL.md @@ -98,6 +98,11 @@ Every column is a `t` builder value: Modifiers: `.primaryKey()`, `.autoInc()`, `.unique()`, `.index('btree')`, `.default(value)`. +`.primaryKey()` and `.unique()` apply to one column. For uniqueness across +multiple columns, use a surrogate key, the multi-column index below, and a +reducer that rejects an existing index match before inserting. An index alone +does not enforce uniqueness. + Use `.default(value)` only for a newly appended migration-safe field. Do not put defaults on primary-key, unique, or auto-increment columns. Optional columns: `nickname: t.option(t.string())` @@ -130,7 +135,9 @@ export { default } from './schema'; // re-export the schema for the module ent ## Reducers -Reducers are created with `spacetimedb.reducer(...)`; the export name becomes the reducer name: +Reducers are created with `spacetimedb.reducer(...)`. An exported `signUp` +becomes `signUp` in generated clients and `sign_up` in `spacetime call` and +`describe`: ```typescript export const createEntity = spacetimedb.reducer( @@ -178,10 +185,27 @@ export const onConnect = spacetimedb.clientConnected((ctx) => { ... }); export const onDisconnect = spacetimedb.clientDisconnected((ctx) => { ... }); ``` -`ctx.connectionId` is `ConnectionId | null`, including in lifecycle contexts. Guard it before passing it to a helper or using it as a table key. +Connection hooks run once per connection. `ctx.connectionId` identifies that connection. +`ctx.sender` identifies the authenticated caller and stays the same when the client +reconnects with the same token. + +Use connection IDs for connection state, such as presence. A reload or network loss can +cause a disconnect. A disconnect alone does not mean the user signed out or their +application login expired. Keep connection cleanup separate from session revocation. + +`ctx.sender` is a SpacetimeDB identity, not necessarily an application account. Separate +identities can authenticate to the same application account. An application sign-out leaves +the identity unchanged, so data scoped to `ctx.sender` stays available to that client. + +`ctx.connectionId` is typed `ConnectionId | null`. It is present inside connection lifecycle hooks and reducers invoked over a connection, and `null` in `init` and scheduled reducers. Guard it before passing it to a helper or using it as a table key. ## Reducer Context API +Each reducer call runs in one database transaction. An error that escapes the +reducer rolls back its database changes. `ctx.sender` identifies the caller; +application roles and permissions are not inferred from that identity. Table +visibility and view filters control reads, not authorization to call reducers. + `ctx` is the only source of sender identity, time, and randomness; stdlib clocks and RNG are unavailable in modules. Let exported callbacks infer their context type. In helpers, use `ReducerCtx>`; do not annotate a context as `any`, because that erases table row types and can make `bigint` expressions infer as `number`. ```typescript @@ -282,9 +306,22 @@ const Shape = t.enum('Shape', { A client subscribing to a view receives only the rows it returns. Use a per-user view (keyed on `ctx.sender`) for per-viewer access control: deleting a row it depends on (e.g. a membership row) automatically drops the rows it was exposing from that client. +Use index accessors in views. Do not scan a whole table with `.iter()` when an +indexed lookup can select the required rows. `t.row(...)` and `t.object(...)` return schema builders, not TypeScript runtime row types. Let a view callback infer its result, or annotate a separately declared structural type such as `Array<{ sku: bigint; label: string }>`. A named output type must not reuse the generated PascalCase name of its view accessor (for example, reserve `DiscountedProduct` for a `discounted_product` view). +A view context is `ViewCtx` (and `AnonymousViewCtx`), both exported from +`spacetimedb/server`. It carries `sender`, a read-only `db`, and `from`; it is +not a `ReducerCtx`, so a helper shared between a reducer and a view must accept +either: + +```typescript +import type { ReducerCtx, ViewCtx, InferSchema } from 'spacetimedb/server'; +type S = InferSchema; +function stockOf(ctx: ReducerCtx | ViewCtx, itemId: bigint) { ... } +``` + Both `spacetimedb.view(...)` and `spacetimedb.anonymousView(...)` take three arguments: view options, the declared return schema, and the callback. ```typescript @@ -292,7 +329,7 @@ Both `spacetimedb.view(...)` and `spacetimedb.anonymousView(...)` take three arg export const activeUsers = spacetimedb.anonymousView( { name: 'active_users', public: true }, t.array(entity.rowType), - (ctx) => [...ctx.db.entity.iter()].filter(e => e.active) + (ctx) => [...ctx.db.entity.active.filter(true)] // active: t.bool().index('btree') ); // Per-user view (varies by ctx.sender): @@ -328,6 +365,9 @@ Procedural views read through `ctx.db` and return materialized values such as ar ## Client Visibility Filters +Client visibility filters are experimental and may change or be removed. Use views for access +control. `:sender` is the requesting client's identity. + ```typescript export const privateNoteFilter = spacetimedb.clientVisibilityFilter.sql( 'SELECT * FROM owned_row WHERE owner = :sender' diff --git a/crates/bindings-typescript/src/lib/query.ts b/crates/bindings-typescript/src/lib/query.ts index bb93b0e6ce3..0c17e3d145a 100644 --- a/crates/bindings-typescript/src/lib/query.ts +++ b/crates/bindings-typescript/src/lib/query.ts @@ -248,19 +248,21 @@ export type NamespacedQueryBuilder = * A runtime reference to a table. This materializes the RowExpr for us. * TODO: Maybe add the full SchemaDef to the type signature depending on how joins will work. */ -export type TableRef = Readonly<{ - type: 'table'; - sourceName: TableDef['sourceName']; - accessorName: string; - cols: RowExpr; - indexedCols: IndexedRowExpr; - tableDef: TableDef; +// Keep this named so TypeScript diagnostics show `TableRef` instead of its +// expanded structure. +export interface TableRef { + readonly type: 'table'; + readonly sourceName: TableDef['sourceName']; + readonly accessorName: string; + readonly cols: RowExpr; + readonly indexedCols: IndexedRowExpr; + readonly tableDef: TableDef; // Delegated UntypedTableDef properties for compatibility. - columns: TableDef['columns']; - indexes: TableDef['indexes']; - rowType: TableDef['rowType']; - constraints: any; -}>; + readonly columns: TableDef['columns']; + readonly indexes: TableDef['indexes']; + readonly rowType: TableDef['rowType']; + readonly constraints: any; +} class TableRefImpl implements TableRef, From diff --git a/crates/bindings-typescript/src/sdk/ws.ts b/crates/bindings-typescript/src/sdk/ws.ts index 96eb7e5fe51..daa0bb20ab0 100644 --- a/crates/bindings-typescript/src/sdk/ws.ts +++ b/crates/bindings-typescript/src/sdk/ws.ts @@ -95,12 +95,16 @@ export async function openWebSocket({ const WS = await resolveWS(); + // Endpoints resolve under the URI's path, so `ws://host/stdb` reaches `/stdb/v1/...`. + const base = new URL(url); + if (!base.pathname.endsWith('/')) base.pathname += '/'; + // We swap our original token to a shorter-lived token // to avoid sending the original via query params. let temporaryAuthToken: string | undefined; if (authToken) { headers.set('Authorization', `Bearer ${authToken}`); - const tokenUrl = new URL('v1/identity/websocket-token', url); + const tokenUrl = new URL('v1/identity/websocket-token', base); tokenUrl.protocol = url.protocol === 'wss:' ? 'https:' : 'http:'; const response = await fetch(tokenUrl, { method: 'POST', headers }); @@ -112,7 +116,7 @@ export async function openWebSocket({ } } - const databaseUrl = new URL(`v1/database/${nameOrAddress}/subscribe`, url); + const databaseUrl = new URL(`v1/database/${nameOrAddress}/subscribe`, base); if (temporaryAuthToken) { databaseUrl.searchParams.set('token', temporaryAuthToken); } diff --git a/crates/bindings-typescript/src/server/runtime.ts b/crates/bindings-typescript/src/server/runtime.ts index 0f4d8b377d9..1fc02e2b4c9 100644 --- a/crates/bindings-typescript/src/server/runtime.ts +++ b/crates/bindings-typescript/src/server/runtime.ts @@ -218,7 +218,19 @@ class AuthCtxImpl implements AuthCtx { }); } - /** If there is a connection id, look up the JWT payload from the system tables. */ + /** Create a context for an HTTP request, which has no connection but is not host-originated. */ + static httpRequest(): AuthCtx { + return new AuthCtxImpl({ + isInternal: false, + jwtSource: () => null, + senderIdentity: Identity.zero(), + }); + } + + /** + * If there is a connection id, look up the JWT payload from the system tables. + * A call without one was spawned by the database, such as `init` or a scheduled function. + */ static fromBackend( connectionId: ConnectionId | null, sender: Identity, @@ -226,7 +238,7 @@ class AuthCtxImpl implements AuthCtx { ): AuthCtx { if (connectionId === null) { return new AuthCtxImpl({ - isInternal: false, + isInternal: true, jwtSource: () => null, senderIdentity: sender, }); @@ -357,6 +369,13 @@ export const ReducerCtxImpl = class ReducerCtx< } } + /** A transaction inside an HTTP handler, which reports external authority. */ + static forHttpHandler(timestamp: Timestamp, dbView: DbView) { + const tx = new ReducerCtx(Identity.zero(), timestamp, null, dbView); + tx.#senderAuth = AuthCtxImpl.httpRequest(); + return tx; + } + get databaseIdentity() { return (this.#identity ??= new Identity(this.#backend.identity())); } @@ -887,12 +906,7 @@ class HandlerContextImpl withTx(body: (ctx: any) => T): T { const dispatches = this.#dispatches; return runWithTx(timestamp => { - const tx = new ReducerCtxImpl( - Identity.zero(), - timestamp, - null, - this.#dbView() - ); + const tx = ReducerCtxImpl.forHttpHandler(timestamp, this.#dbView()); if (dispatches.length > 0) { tx.as = buildAliasCtxMap(tx, dispatches, '') as any; } @@ -1011,10 +1025,8 @@ function buildHandlerAliasCtx( as: subAs, withTx(body: any) { return runWithTx((ts: Timestamp) => { - const tx = new ReducerCtxImpl( - Identity.zero(), + const tx = ReducerCtxImpl.forHttpHandler( ts, - null, (nsDb_ ??= buildDbViewForDispatch( dispatch, namePrefix diff --git a/crates/bindings-typescript/tests/sender_auth.test.ts b/crates/bindings-typescript/tests/sender_auth.test.ts new file mode 100644 index 00000000000..27445d17d94 --- /dev/null +++ b/crates/bindings-typescript/tests/sender_auth.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from 'vitest'; +import * as sys from 'spacetime:sys@2.0'; +import { schema } from '../src/server/schema'; +import { ReducerCtxImpl } from '../src/server/runtime'; +import { table } from '../src/lib/table'; +import { t } from '../src/lib/type_builders'; +import { Timestamp } from '../src/lib/timestamp'; + +describe('ctx.senderAuth', () => { + function senderAuthFor(connectionId: bigint) { + const items = table({ name: 'items' }, { id: t.u32().primaryKey() }); + const module = schema({ items }); + let isInternal: boolean | undefined; + const check = module.reducer(ctx => { + isInternal = ctx.senderAuth.isInternal; + }); + const hooks = (module as any)[(sys as any).moduleHooks]({ check }); + hooks.__call_reducer__( + 0, + 0n, + connectionId, + 0n, + new DataView(new ArrayBuffer(0)) + ); + return isInternal; + } + + it('is internal for a reducer the database spawns without a connection', () => { + expect(senderAuthFor(0n)).toBe(true); + }); + + it('is external for a reducer called over a connection', () => { + expect(senderAuthFor(7n)).toBe(false); + }); + + it('is external for a transaction inside an HTTP handler', () => { + const tx = ReducerCtxImpl.forHttpHandler(new Timestamp(0n), {} as never); + expect(tx.connectionId).toBeNull(); + expect(tx.senderAuth.isInternal).toBe(false); + expect(tx.senderAuth.hasJWT).toBe(false); + }); +}); diff --git a/crates/bindings-typescript/tests/table_ref_error_message.test.ts b/crates/bindings-typescript/tests/table_ref_error_message.test.ts new file mode 100644 index 00000000000..54274fd5455 --- /dev/null +++ b/crates/bindings-typescript/tests/table_ref_error_message.test.ts @@ -0,0 +1,78 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import * as ts from 'typescript'; +import { describe, expect, it } from 'vitest'; + +const bindingsRoot = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + '..' +); + +function runTypecheck(source: string) { + const tmpDir = mkdtempSync(path.join(tmpdir(), 'stdb-tableref-diag-')); + const reproPath = path.join(tmpDir, 'repro.ts'); + writeFileSync(reproPath, source); + + try { + const options: ts.CompilerOptions = { + target: ts.ScriptTarget.ESNext, + module: ts.ModuleKind.ESNext, + strict: true, + noEmit: true, + skipLibCheck: true, + forceConsistentCasingInFileNames: true, + allowImportingTsExtensions: true, + noImplicitAny: true, + moduleResolution: ts.ModuleResolutionKind.Bundler, + useDefineForClassFields: true, + verbatimModuleSyntax: true, + isolatedModules: true, + }; + + const host = ts.createCompilerHost(options); + const program = ts.createProgram( + [reproPath, path.join(bindingsRoot, 'src/server/sys.d.ts')], + options, + host + ); + const diagnostics = ts.getPreEmitDiagnostics(program); + return diagnostics.map(d => + ts.flattenDiagnosticMessageText(d.messageText, '\n') + ); + } finally { + rmSync(tmpDir, { recursive: true, force: true }); + } +} + +describe('TableRef diagnostics', () => { + const source = ` +import { t } from ${JSON.stringify(path.join(bindingsRoot, 'src/server/index.ts'))}; +import { table } from ${JSON.stringify(path.join(bindingsRoot, 'src/lib/table.ts'))}; +import { createTableRefFromDef } from ${JSON.stringify(path.join(bindingsRoot, 'src/lib/query.ts'))}; +import type { AllUnique } from ${JSON.stringify(path.join(bindingsRoot, 'src/lib/constraints.ts'))}; + +const cartItem = table( + { name: 'cart_item' }, + { id: t.u64().primaryKey().autoInc(), accountId: t.u64(), quantity: t.u32() } +); + +const ref = createTableRefFromDef(cartItem as any, 'cartItem'); +type Boom = AllUnique; +declare const b: Boom; +`; + + it('names the type instead of dumping its structure', () => { + const messages = runTypecheck(source); + const constraintError = messages.find(m => + m.includes("does not satisfy the constraint 'UntypedTableDef'") + ); + + expect(constraintError).toBeDefined(); + // The name, not the shape. + expect(constraintError).toContain('TableRef<'); + expect(constraintError).not.toContain('type: "table"'); + expect(constraintError).not.toContain('accessorName'); + }, 15000); +}); diff --git a/crates/bindings-typescript/tests/ws_url.test.ts b/crates/bindings-typescript/tests/ws_url.test.ts new file mode 100644 index 00000000000..38cd71760c3 --- /dev/null +++ b/crates/bindings-typescript/tests/ws_url.test.ts @@ -0,0 +1,49 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { openWebSocket } from '../src/sdk/ws'; + +describe('openWebSocket', () => { + afterEach(() => vi.unstubAllGlobals()); + + async function urlsFor(uri: string) { + const fetched: string[] = []; + const opened: string[] = []; + vi.stubGlobal('fetch', async (url: URL) => { + fetched.push(url.href); + return new Response(JSON.stringify({ token: 'short' }), { status: 200 }); + }); + vi.stubGlobal( + 'WebSocket', + class { + binaryType = ''; + constructor(url: string) { + opened.push(url); + } + } + ); + await openWebSocket({ + url: new URL(uri), + nameOrAddress: 'app', + wsProtocol: [], + authToken: 'stored', + compression: 'none', + lightMode: false, + }); + return { fetched, opened }; + } + + it('keeps a path prefix with or without a trailing slash', async () => { + for (const uri of ['ws://host/stdb', 'ws://host/stdb/']) { + const { fetched, opened } = await urlsFor(uri); + expect(fetched).toEqual(['http://host/stdb/v1/identity/websocket-token']); + expect(opened[0]).toMatch( + /^ws:\/\/host\/stdb\/v1\/database\/app\/subscribe\?/ + ); + } + }); + + it('is unchanged for an address without a path', async () => { + const { fetched, opened } = await urlsFor('wss://host'); + expect(fetched).toEqual(['https://host/v1/identity/websocket-token']); + expect(opened[0]).toMatch(/^wss:\/\/host\/v1\/database\/app\/subscribe\?/); + }); +}); diff --git a/crates/cli/build.rs b/crates/cli/build.rs index c5bd4303464..7e7d755cae5 100644 --- a/crates/cli/build.rs +++ b/crates/cli/build.rs @@ -6,6 +6,7 @@ use std::process::Command; use toml::Value; fn main() { + println!("cargo:rerun-if-env-changed=SPACETIMEDB_NIX_BUILD_GIT_COMMIT"); let git_hash = find_git_hash(); println!("cargo:rustc-env=GIT_HASH={git_hash}"); @@ -110,6 +111,7 @@ fn generate_template_files() { // Embed skill files from skills/*/SKILL.md let skills_dir = repo_root.join("skills"); + println!("cargo:rerun-if-changed={}", skills_dir.display()); let skill_names = discover_skill_names(&skills_dir); generated_code.push_str("pub fn get_skill(name: &str) -> Option<&'static str> {\n"); diff --git a/crates/cli/src/subcommands/dev.rs b/crates/cli/src/subcommands/dev.rs index d24344b6490..27faccf65cb 100644 --- a/crates/cli/src/subcommands/dev.rs +++ b/crates/cli/src/subcommands/dev.rs @@ -85,6 +85,12 @@ pub fn cli() -> Command { ) .arg(common_args::server().help("The nickname, host name or URL of the server to publish to")) .arg(common_args::yes()) + .arg( + Arg::new("ready-file") + .long("ready-file") + .value_parser(clap::value_parser!(PathBuf)) + .help("Write this file after the initial build cycle succeeds and file watching starts. This is a startup receipt, not ongoing health."), + ) .arg(common_args::clear_database()) .arg( Arg::new("template") @@ -150,6 +156,13 @@ struct DatabaseRow { } pub async fn exec(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::Error> { + if let Some(path) = args.get_one::("ready-file") { + match fs::remove_file(path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error).context("Failed to clear development readiness file"), + } + } let project_path = args.get_one::("project-path").unwrap(); let module_path_from_cli = args.get_one::("module-path"); let module_bindings_path = args.get_one::("module-bindings-path").unwrap(); @@ -760,7 +773,7 @@ pub async fn exec(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::E let loaded_config_dir = loaded_config.as_ref().map(|lc| lc.config_dir.clone()); generate_build_and_publish( - &config, + &mut config, &project_dir, loaded_config_dir.as_deref(), &spacetimedb_dir, @@ -861,6 +874,10 @@ pub async fn exec(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::E watcher.watch(watch_dir, RecursiveMode::Recursive)?; } + if let Some(path) = args.get_one::("ready-file") { + fs::write(path, "ready\n").context("Failed to write development readiness file")?; + } + let mut debounce_timer; loop { // Use recv_timeout so we can periodically check if the client process exited @@ -876,7 +893,7 @@ pub async fn exec(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::E println!("\n{}", "File change detected, rebuilding...".yellow()); match generate_build_and_publish( - &config, + &mut config, &project_dir, loaded_config_dir.as_deref(), &spacetimedb_dir, @@ -1024,7 +1041,7 @@ fn upsert_env_db_names_and_hosts(env_path: &Path, server_host_url: &str, databas #[allow(clippy::too_many_arguments)] async fn generate_build_and_publish( - config: &Config, + config: &mut Config, project_dir: &Path, config_dir: Option<&Path>, spacetimedb_dir: &Path, @@ -1181,7 +1198,8 @@ async fn generate_build_and_publish( publish_entry.insert("break-clients".to_string(), json!(true)); } - publish::exec_from_entry(config.clone(), publish_entry, config_dir, clear_database, yes).await?; + // Preserve a token created during publish for logs and later rebuilds. + publish::exec_from_entry(config, publish_entry, config_dir, clear_database, yes).await?; } println!("{}", "Published successfully!".green().bold()); @@ -2043,12 +2061,17 @@ mod tests { // Verify that --skip-publish and --skip-generate flags are registered let cmd = cli(); - let matches = cmd - .clone() - .get_matches_from(vec!["dev", "--skip-publish", "--skip-generate"]); + let matches = cmd.clone().get_matches_from(vec![ + "dev", + "--skip-publish", + "--skip-generate", + "--ready-file", + "ready", + ]); assert!(matches.get_flag("skip_publish")); assert!(matches.get_flag("skip_generate")); + assert_eq!(matches.get_one::("ready-file"), Some(&PathBuf::from("ready"))); } #[test] diff --git a/crates/cli/src/subcommands/publish.rs b/crates/cli/src/subcommands/publish.rs index b7311745d89..04ddaf5d62a 100644 --- a/crates/cli/src/subcommands/publish.rs +++ b/crates/cli/src/subcommands/publish.rs @@ -527,7 +527,7 @@ pub async fn exec_with_options( } pub async fn exec_from_entry( - mut config: Config, + config: &mut Config, entry: HashMap, config_dir: Option<&std::path::Path>, clear_database: ClearMode, @@ -543,7 +543,7 @@ pub async fn exec_from_entry( let yes = if force { YesFlags::all() } else { YesFlags::default() }; execute_publish_configs( - &mut config, + config, vec![command_config], true, config_dir, diff --git a/crates/codegen/build.rs b/crates/codegen/build.rs index cd3bca12faf..cff0d3ed9bf 100644 --- a/crates/codegen/build.rs +++ b/crates/codegen/build.rs @@ -3,6 +3,9 @@ use std::process::Command; // https://stackoverflow.com/questions/43753491/include-git-commit-hash-as-string-into-rust-program #[allow(clippy::disallowed_macros)] fn main() { + // Any rerun-if directive disables Cargo's default package scan; restore it. + println!("cargo:rerun-if-changed=."); + println!("cargo:rerun-if-env-changed=SPACETIMEDB_NIX_BUILD_GIT_COMMIT"); let git_hash = find_git_hash(); println!("cargo:rustc-env=GIT_HASH={git_hash}"); } diff --git a/crates/fs-utils/src/lib.rs b/crates/fs-utils/src/lib.rs index c4d1a6ba0c1..42e5a51ae25 100644 --- a/crates/fs-utils/src/lib.rs +++ b/crates/fs-utils/src/lib.rs @@ -35,12 +35,36 @@ pub fn atomic_write(file_path: &Path, data: String) -> anyhow::Result<()> { .write(true) .create_new(true) .open(&temp_path); - if let Ok(file) = opened { - temp_file = file; - break; + match opened { + Ok(file) => { + temp_file = file; + break; + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(error) => return Err(error.into()), } } temp_file.write_all(data.as_bytes())?; std::fs::rename(&temp_path, file_path)?; Ok(()) } + +#[cfg(test)] +mod tests { + use super::atomic_write; + + #[test] + fn atomic_write_replaces_contents_and_returns_open_errors() { + let dir = tempdir::TempDir::new("atomic-write").unwrap(); + let path = dir.path().join("config"); + atomic_write(&path, "before".into()).unwrap(); + atomic_write(&path, "after".into()).unwrap(); + assert_eq!(std::fs::read_to_string(&path).unwrap(), "after"); + + let error = atomic_write(&dir.path().join("missing/config"), "data".into()).unwrap_err(); + assert_eq!( + error.downcast_ref::().unwrap().kind(), + std::io::ErrorKind::NotFound + ); + } +} diff --git a/skills/cli/SKILL.md b/skills/cli/SKILL.md index e7a51487fc1..2b7b0c63728 100644 --- a/skills/cli/SKILL.md +++ b/skills/cli/SKILL.md @@ -43,13 +43,15 @@ spacetime build --debug # faster iteration, slower runtime # Dev mode (auto-rebuild, auto-publish, generates bindings) spacetime dev -spacetime dev --client-lang typescript --module-bindings-path ./client/src/module_bindings +spacetime dev my-database --server local --yes --delete-data=never --client-lang typescript --module-bindings-path ./client/src/module_bindings # Generate client bindings spacetime generate --lang typescript|csharp|rust --out-dir ./bindings --module-path ./server spacetime generate --lang unrealcpp --uproject-dir ./MyGame --module-path ./server --unreal-module-name MyGame ``` +`dev` stays running and watches module changes. `--run "npm run dev"` starts a client command; `--server-only` omits it. `--module-path` selects the module directory when no publish targets exist in `spacetime.json`. Once targets exist, use their configured paths and omit that flag. Separate build/publish/generate commands remain useful for one-shot deployment. + ### Publishing & Deployment ```bash @@ -114,7 +116,7 @@ spacetime server add myserver --url https://my-spacetime.example.com # Set default server spacetime server set-default local -# Test connectivity +# Check connectivity spacetime server ping local # Start local instance @@ -172,10 +174,7 @@ spacetime server ping ``` ### "Schema conflict" -```bash -# Clear data and republish -spacetime publish my-db --delete-data=always --yes -``` +`--delete-data=never` rejects incompatible schema updates without clearing data. A compatible migration preserves existing data; `--delete-data=always` destroys it and is only appropriate for an intentional reset. ### "Build failed" ```bash diff --git a/skills/concepts/SKILL.md b/skills/concepts/SKILL.md index 8ea6e8f9cfb..e152e5d9e72 100644 --- a/skills/concepts/SKILL.md +++ b/skills/concepts/SKILL.md @@ -101,8 +101,8 @@ Lifecycle: Write → Compile → Publish (`spacetime publish`) → Hot-swap (rep ## Identity -- **Identity**: A long-lived, globally unique identifier for a user. -- **ConnectionId**: Identifies a specific client connection. +- **Identity**: A long-lived, globally unique identifier for a user, derived from the token's issuer and subject claims. The same token yields the same identity on every connection. +- **ConnectionId**: Identifies one client connection. A new connection gets a new connection ID; a disconnect ends the connection, not the identity. - Always use `ctx.sender` / `ctx.Sender` / `ctx.sender()` for authorization. SpacetimeDB works with many OIDC providers, including SpacetimeAuth (built-in), Auth0, Clerk, Keycloak, Google, and GitHub. diff --git a/skills/cpp-server/SKILL.md b/skills/cpp-server/SKILL.md index aebb82edf06..6283f9535b5 100644 --- a/skills/cpp-server/SKILL.md +++ b/skills/cpp-server/SKILL.md @@ -41,6 +41,8 @@ Options: - `SPACETIMEDB_TABLE(Type, accessor, Public|Private)`: regular table - `SPACETIMEDB_TABLE(Type, accessor, Public|Private, true)`: event table +Only server-side code such as reducers reads a `Private` table. `Public` exposes all of a table's rows to every client through subscriptions. + Field constraints: - `FIELD_PrimaryKey(accessor, field)`: primary key - `FIELD_PrimaryKeyAutoInc(accessor, field)`: primary key with auto-increment (use 0 on insert) @@ -138,6 +140,8 @@ SPACETIMEDB_CLIENT_DISCONNECTED(on_disconnect, ReducerContext ctx) { } ``` +Connection hooks run once per connection. The same authenticated principal keeps the same identity (`ctx.sender()`) across connections, while each connection has its own connection ID (`ctx.connection_id()`). A disconnect ends one connection; it does not end the identity, which returns unchanged on the next connection with the same token. Use connection IDs for presence and other connection-scoped state. + ## Authentication & Timestamps ```cpp diff --git a/skills/csharp-server/SKILL.md b/skills/csharp-server/SKILL.md index 1d08ba89fe7..46eb79b048d 100644 --- a/skills/csharp-server/SKILL.md +++ b/skills/csharp-server/SKILL.md @@ -63,6 +63,8 @@ public partial struct Entity Options: `Accessor = "PascalCase"` (recommended), `Public = true`, `Scheduled = nameof(ReducerFn)`, `ScheduledAt = nameof(field)`, `Event = true` +Tables are private by default: only server-side code such as reducers and views reads them. `Public = true` exposes all of a table's rows to every client through subscriptions. + `ctx.Db` accessors use the `Accessor` name: `ctx.Db.Entity`, `ctx.Db.ScoreRecord`. ## Column Types @@ -162,7 +164,9 @@ public static void OnConnect(ReducerContext ctx) { ... } public static void OnDisconnect(ReducerContext ctx) { ... } ``` -`ctx.ConnectionId` is `ConnectionId?`, including in connection lifecycle reducers. Check or unwrap it before storing it in a non-nullable column or passing it to an index accessor. +Connection hooks run once per connection. The same authenticated principal keeps the same identity (`ctx.Sender`) across connections, while each connection has its own connection ID (`ctx.ConnectionId`). A disconnect ends one connection; it does not end the identity, which returns unchanged on the next connection with the same token. Use connection IDs for presence and other connection-scoped state. + +`ctx.ConnectionId` is typed `ConnectionId?`. It is present inside connection lifecycle reducers and reducers invoked over a connection, and null in `Init` and scheduled reducers. Check or unwrap it before storing it in a non-nullable column or passing it to an index accessor. ## Views diff --git a/skills/rust-server/SKILL.md b/skills/rust-server/SKILL.md index 0e282242263..60ca2938324 100644 --- a/skills/rust-server/SKILL.md +++ b/skills/rust-server/SKILL.md @@ -44,6 +44,8 @@ pub struct Entity { Options: `accessor = snake_case` (required), `public`, `scheduled(reducer_fn)`, `index(...)` +Tables are private by default: only server-side code such as reducers and views reads them. `public` exposes all of a table's rows to every client through subscriptions. + `ctx.db` accessors use the `accessor` name (snake_case). ## Column Types @@ -150,6 +152,8 @@ pub fn on_connect(ctx: &ReducerContext) { ... } pub fn on_disconnect(ctx: &ReducerContext) { ... } ``` +Connection hooks run once per connection. The same authenticated principal keeps the same identity (`ctx.sender()`) across connections, while each connection has its own connection ID (`ctx.connection_id()`). A disconnect ends one connection; it does not end the identity, which returns unchanged on the next connection with the same token. Use connection IDs for presence and other connection-scoped state. + The current connection ID is available through `ctx.connection_id()` (not a public field) and may be absent outside connection-scoped calls. ## Views diff --git a/skills/typescript-client/SKILL.md b/skills/typescript-client/SKILL.md index 4249daf15a1..29e0881d28f 100644 --- a/skills/typescript-client/SKILL.md +++ b/skills/typescript-client/SKILL.md @@ -152,7 +152,9 @@ Calls made while disconnected fail immediately. Pending reducer and procedure pr ## Gotchas -- **`useTable` rows are `readonly`.** Copy before sorting/mutating, or it fails to type-check: +- **Subscription rows have no presentation order.** A server view's array order does not + define client cache iteration order. `useTable` rows are `readonly`; a sorted copy can + express the application's display order: `const [rows] = useTable(tables.message); const sorted = [...rows].sort(...)`. -- **bigint in JSX.** ids/counts from `t.u64()`/`t.i64()` columns are `bigint`, which React - cannot render. Wrap it: `{Number(row.id)}` or `{String(count)}`. +- **64-bit display values.** `{String(row.id)}` preserves the full `bigint` value. + Conversion to `Number` can lose precision outside JavaScript's safe integer range. diff --git a/skills/typescript-server/SKILL.md b/skills/typescript-server/SKILL.md index 8563a5e9808..e618580ceef 100644 --- a/skills/typescript-server/SKILL.md +++ b/skills/typescript-server/SKILL.md @@ -74,6 +74,8 @@ const entity = table( Options: `name` (snake_case, recommended), `public: true`, `event: true`, `indexes: [...]` +Tables are private by default: only server-side code such as reducers and views reads them. `public: true` exposes all of a table's rows to every client through subscriptions. + `ctx.db` accessors are the keys passed to `schema({...})`, verbatim: `schema({ scoreRecord })` -> `ctx.db.scoreRecord`. Keep TypeScript identifiers and accessors camelCase. Use explicit `name: 'snake_case'` strings when you need a canonical database name that differs from the TypeScript identifier. ## Column Types @@ -98,6 +100,11 @@ Every column is a `t` builder value: Modifiers: `.primaryKey()`, `.autoInc()`, `.unique()`, `.index('btree')`, `.default(value)`. +`.primaryKey()` and `.unique()` apply to one column. For uniqueness across +multiple columns, use a surrogate key, the multi-column index below, and a +reducer that rejects an existing index match before inserting. An index alone +does not enforce uniqueness. + Use `.default(value)` only for a newly appended migration-safe field. Do not put defaults on primary-key, unique, or auto-increment columns. Optional columns: `nickname: t.option(t.string())` @@ -130,7 +137,9 @@ export { default } from './schema'; // re-export the schema for the module ent ## Reducers -Reducers are created with `spacetimedb.reducer(...)`; the export name becomes the reducer name: +Reducers are created with `spacetimedb.reducer(...)`. An exported `signUp` +becomes `signUp` in generated clients and `sign_up` in `spacetime call` and +`describe`: ```typescript export const createEntity = spacetimedb.reducer( @@ -178,10 +187,27 @@ export const onConnect = spacetimedb.clientConnected((ctx) => { ... }); export const onDisconnect = spacetimedb.clientDisconnected((ctx) => { ... }); ``` -`ctx.connectionId` is `ConnectionId | null`, including in lifecycle contexts. Guard it before passing it to a helper or using it as a table key. +Connection hooks run once per connection. `ctx.connectionId` identifies that connection. +`ctx.sender` identifies the authenticated caller and stays the same when the client +reconnects with the same token. + +Use connection IDs for connection state, such as presence. A reload or network loss can +cause a disconnect. A disconnect alone does not mean the user signed out or their +application login expired. Keep connection cleanup separate from session revocation. + +`ctx.sender` is a SpacetimeDB identity, not necessarily an application account. Separate +identities can authenticate to the same application account. An application sign-out leaves +the identity unchanged, so data scoped to `ctx.sender` stays available to that client. + +`ctx.connectionId` is typed `ConnectionId | null`. It is present inside connection lifecycle hooks and reducers invoked over a connection, and `null` in `init` and scheduled reducers. Guard it before passing it to a helper or using it as a table key. ## Reducer Context API +Each reducer call runs in one database transaction. An error that escapes the +reducer rolls back its database changes. `ctx.sender` identifies the caller; +application roles and permissions are not inferred from that identity. Table +visibility and view filters control reads, not authorization to call reducers. + `ctx` is the only source of sender identity, time, and randomness; stdlib clocks and RNG are unavailable in modules. Let exported callbacks infer their context type. In helpers, use `ReducerCtx>`; do not annotate a context as `any`, because that erases table row types and can make `bigint` expressions infer as `number`. ```typescript @@ -282,9 +308,22 @@ const Shape = t.enum('Shape', { A client subscribing to a view receives only the rows it returns. Use a per-user view (keyed on `ctx.sender`) for per-viewer access control: deleting a row it depends on (e.g. a membership row) automatically drops the rows it was exposing from that client. +Use index accessors in views. Do not scan a whole table with `.iter()` when an +indexed lookup can select the required rows. `t.row(...)` and `t.object(...)` return schema builders, not TypeScript runtime row types. Let a view callback infer its result, or annotate a separately declared structural type such as `Array<{ sku: bigint; label: string }>`. A named output type must not reuse the generated PascalCase name of its view accessor (for example, reserve `DiscountedProduct` for a `discounted_product` view). +A view context is `ViewCtx` (and `AnonymousViewCtx`), both exported from +`spacetimedb/server`. It carries `sender`, a read-only `db`, and `from`; it is +not a `ReducerCtx`, so a helper shared between a reducer and a view must accept +either: + +```typescript +import type { ReducerCtx, ViewCtx, InferSchema } from 'spacetimedb/server'; +type S = InferSchema; +function stockOf(ctx: ReducerCtx | ViewCtx, itemId: bigint) { ... } +``` + Both `spacetimedb.view(...)` and `spacetimedb.anonymousView(...)` take three arguments: view options, the declared return schema, and the callback. ```typescript @@ -292,7 +331,7 @@ Both `spacetimedb.view(...)` and `spacetimedb.anonymousView(...)` take three arg export const activeUsers = spacetimedb.anonymousView( { name: 'active_users', public: true }, t.array(entity.rowType), - (ctx) => [...ctx.db.entity.iter()].filter(e => e.active) + (ctx) => [...ctx.db.entity.active.filter(true)] // active: t.bool().index('btree') ); // Per-user view (varies by ctx.sender): @@ -328,6 +367,9 @@ Procedural views read through `ctx.db` and return materialized values such as ar ## Client Visibility Filters +Client visibility filters are experimental and may change or be removed. Use views for access +control. `:sender` is the requesting client's identity. + ```typescript export const privateNoteFilter = spacetimedb.clientVisibilityFilter.sql( 'SELECT * FROM owned_row WHERE owner = :sender' diff --git a/tools/stack-bench/.gitignore b/tools/stack-bench/.gitignore new file mode 100644 index 00000000000..a16e5569f30 --- /dev/null +++ b/tools/stack-bench/.gitignore @@ -0,0 +1,14 @@ +media/ +dist/ +local-notes/ +*.local.md +.loop-test/ +results/ +snapshot-l*/ +grader/.candidates/ +.spacetime-data/ +.spacetime-data.bak-*/ +grader/.mutation-report.json +tracks/*/overview.html +transcripts/ +operator.env diff --git a/tools/stack-bench/GETTING-STARTED.md b/tools/stack-bench/GETTING-STARTED.md new file mode 100644 index 00000000000..c536cc945a5 --- /dev/null +++ b/tools/stack-bench/GETTING-STARTED.md @@ -0,0 +1,56 @@ +# Start your first StackBench run + +## Requirements + +- A clean, normal Git clone of this repository. Release builds reject a changed + or worktree checkout. +- Docker running Linux containers for `linux/amd64`, with BuildKit and the + Compose plugin. +- Internet access for base images, npm packages, and the model provider. +- 4 CPUs, 8 GiB of memory, and 10 GiB of free storage allocated to Docker. The + first source build needs about 16 GiB of memory and 60 GiB of storage. + +Docker keeps the images and results on its configured disk; pick that disk in +Docker Desktop before the first build if the system drive is short of space. +Run one StackBench installation per Docker daemon; the demo shares its state +volume, container names, and ports. + +## Run the demo + +From the repository root: + +```sh +docker compose -f tools/stack-bench/appliance/demo.compose.yaml run --build --rm demo +``` + +The command builds the images, prepares the state volume, starts the dashboard, +and grades the supplied reference apps on three stacks. It makes no model calls +and needs no credentials. The first build includes the SpacetimeDB server and +SDK and can take a long time; leave the command running until it reports +completion or an error, then open [localhost:7331](http://localhost:7331). + +## Run a model + +1. Store a provider credential, following + [provider credentials](appliance/README.md#provider-credentials). +2. In the dashboard, open **New run** and fill in the form. The + **Production-quality app** setting is on by default and recorded with the + run; it is the production-quality line described in + [prompting](docs/prompting.md). In a `job prepare` selections file, the + field is `productionQuality`. +3. Select **Start run**, then open the run to follow it. Model work spends + provider credit. + +The CLI runs the same flow; see [jobs](appliance/README.md#jobs). + +## Read and keep results + +Open a run, then an attempt, to see its checks, logs, and transcript. Keeping +and exporting results is in +[results and cleanup](appliance/README.md#results-and-cleanup). + +## Stop and clean up + +The run's **Stop** control ends its active work and keeps its results. When no +run is active, stop the dashboard and cache containers in Docker Desktop. For +an interrupted run, follow the [recover-run skill](skills/recover-run/SKILL.md). diff --git a/tools/stack-bench/README.md b/tools/stack-bench/README.md new file mode 100644 index 00000000000..87d1e999b49 --- /dev/null +++ b/tools/stack-bench/README.md @@ -0,0 +1,41 @@ +# StackBench + +StackBench measures how well coding agents deliver production-quality features +by default. Each agent builds an online store from a feature list on one stack, +and StackBench scores how many of its features hold up under production +conditions. Every run happens in an isolated Docker sandbox, every check is +scripted, and the evidence behind every result is kept. + +The demo needs only Git and Docker. From the repository root: + +```sh +docker compose -f tools/stack-bench/appliance/demo.compose.yaml run --build --rm demo +``` + +[Getting started](GETTING-STARTED.md) explains the demo and the first model run. + +## What it covers + +- **Stacks:** PostgreSQL, MongoDB, Convex, Supabase, and SpacetimeDB. +- **Agents:** Claude Code, Codex, Grok Build, Antigravity, and any model on + OpenRouter. +- **Workload:** the [ecommerce track](tracks/ecommerce/LEVELS.md), a store with + a warehouse, built feature by feature through a feature graph. + +## How a run works + +1. A plan fixes everything about the run; see [plans](appliance/README.md#plans). +2. The agent builds the store in a sandbox from the + [prompt](docs/prompting.md). +3. StackBench restarts the store from its saved code, empties the database, and + runs every check: pages in a real browser, direct server requests, database + writes, restarts, and crashes. +4. [Grading](docs/grading.md) turns the check results into a score, with cost + beside it. + +## Documentation + +- [Getting started](GETTING-STARTED.md), then the + [appliance guide](appliance/README.md) to run campaigns. +- [Skills](skills/): procedures to run, recover, archive, qualify, and release. +- The [documentation index](docs/README.md) for how it works and how to change it. diff --git a/tools/stack-bench/appliance/Controller.Dockerfile b/tools/stack-bench/appliance/Controller.Dockerfile new file mode 100644 index 00000000000..e8b07ee6b70 --- /dev/null +++ b/tools/stack-bench/appliance/Controller.Dockerfile @@ -0,0 +1,124 @@ +# syntax=docker/dockerfile:1.7 + +FROM docker:29.6.2-cli@sha256:feb2d49bd65f274b3e4b4620beabe2f4691e5287e496da9fbc9830ed5f780676 AS docker-cli +FROM mcr.microsoft.com/playwright:v1.62.1-noble@sha256:c091b21d9fae78c76e85cd4356431e9b018402f172a214fc7d7a5e9a7e29d8ac AS source +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN apt-get update && apt-get install -y --no-install-recommends lsof=4.95.0-1build3 util-linux \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /workspace +# Git metadata is mounted only for trusted source export/verification. It is never +# copied into an image layer. Release builds require a clean normal Git checkout. +RUN --mount=type=bind,target=/checkout \ + git -c safe.directory=/checkout -C /checkout archive HEAD | tar -x -C /workspace +WORKDIR /workspace/tools/stack-bench +RUN corepack enable && pnpm --filter spacetimedb install --frozen-lockfile --ignore-scripts \ + && npm ci --ignore-scripts --no-audit --no-fund && npm run build \ + && node dist/src/references/reference-fixtures.js \ + && node dist/commands/check-calibration.js \ + && rm -rf /workspace/node_modules /workspace/crates/bindings-typescript/node_modules +# Normalize checkout text as Git does on Windows; explicit .gitattributes still apply. +RUN --mount=type=bind,target=/checkout \ + GIT_OPTIONAL_LOCKS=0 GIT_CONFIG_COUNT=2 GIT_CONFIG_KEY_0=safe.directory GIT_CONFIG_VALUE_0=/checkout \ + GIT_CONFIG_KEY_1=core.autocrlf GIT_CONFIG_VALUE_1=input \ + node --input-type=module -e 'import {writeFileSync} from "node:fs"; import {releaseSourceIdentity,binarySourceIdentity} from "./dist/src/releases/release-source.js"; writeFileSync("/workspace/stack-bench-source.json",JSON.stringify(releaseSourceIdentity("/checkout"))); writeFileSync("/workspace/stack-bench-binary-source.json",JSON.stringify(binarySourceIdentity("/checkout")));' + +FROM rust:1.93-slim-bookworm@sha256:8f8609d448e821fbc0e44241bc5ca4ce49663cc6306ff1a17f655a0e2a7cd084 AS binary-build +RUN apt-get update -qq && apt-get install -y --no-install-recommends \ + pkg-config libssl-dev build-essential clang cmake perl git curl python3 \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /workspace +COPY --from=source /workspace/ ./ +# The existing source-archive path in cli/build.rs avoids retaining Git metadata. +RUN --mount=type=cache,id=stack-bench-rust-target,target=/target \ + --mount=type=cache,id=stack-bench-cargo-registry,target=/usr/local/cargo/registry \ + --mount=type=cache,id=stack-bench-cargo-git,target=/usr/local/cargo/git,sharing=locked \ + --mount=type=cache,id=stack-bench-rustup,target=/usr/local/rustup,sharing=locked \ + export SPACETIMEDB_NIX_BUILD_GIT_COMMIT="$(python3 -c 'import json; print(json.load(open("stack-bench-source.json"))["revision"])')" \ + && CARGO_TARGET_DIR=/target cargo build --release --locked \ + -p spacetimedb-cli --bin spacetimedb-cli \ + -p spacetimedb-standalone --bin spacetimedb-standalone \ + && mkdir -p /binaries \ + && cp /target/release/spacetimedb-cli /target/release/spacetimedb-standalone /binaries/ + +FROM source AS stack-bench-build +COPY --from=binary-build /binaries/ ./container/bin/ +RUN node dist/container/binary-provenance.js record-snapshot \ + --root /workspace/tools/stack-bench --source-file /workspace/stack-bench-binary-source.json \ + && npm prune --omit=dev --ignore-scripts --no-audit --no-fund + +# build-linux-cli.sh uses this same build, rather than a second Rust recipe. +FROM scratch AS binary-export +COPY --from=stack-bench-build /workspace/tools/stack-bench/container/bin/ /bin/ +COPY --from=stack-bench-build /workspace/tools/stack-bench/container/spacetimedb-binaries.json /spacetimedb-binaries.json + +FROM source AS sdk-build +WORKDIR /workspace +RUN pnpm --filter spacetimedb install --frozen-lockfile --ignore-scripts \ + && pnpm --filter spacetimedb run build \ + && test -f crates/bindings-typescript/dist/server/index.d.ts \ + && test -f crates/bindings-typescript/dist/server/index.mjs + +FROM mcr.microsoft.com/playwright:v1.62.1-noble@sha256:c091b21d9fae78c76e85cd4356431e9b018402f172a214fc7d7a5e9a7e29d8ac + +ENV NODE_ENV=production \ + PLAYWRIGHT_BROWSERS_PATH=/ms-playwright + +COPY --from=docker-cli /usr/local/bin/docker /usr/local/bin/docker +COPY --from=docker-cli /usr/local/libexec/docker/cli-plugins /usr/local/libexec/docker/cli-plugins +ADD --checksum=sha256:4629c757b7618056f8ddd7e2625ae9fdd94c0372a65049520bc7d9df9efc7f71 \ + https://github.com/sigstore/cosign/releases/download/v3.1.3/cosign-linux-amd64 \ + /usr/local/bin/cosign + +RUN apt-get update \ + && apt-get install -y --no-install-recommends lsof=4.95.0-1build3 nftables util-linux \ + && chmod 0555 /usr/local/bin/cosign \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /opt/stack-bench +LABEL org.opencontainers.image.title="Stack Bench controller" + +COPY --from=stack-bench-build /workspace/tools/stack-bench/ ./ +COPY --from=stack-bench-build /workspace/stack-bench-source.json ./source-identity.json +RUN chmod 0555 /opt/stack-bench/dist/container/browser-pipe.js +COPY --from=source /workspace/skills/ /skills/ +COPY --from=source /workspace/crates/bindings-typescript/ /opt/stack-bench-embedded-deps/bindings-typescript/ +COPY --from=sdk-build /workspace/crates/bindings-typescript/dist/ /opt/stack-bench-embedded-deps/bindings-typescript/dist/ +COPY --from=source /workspace/licenses/BSL.txt /opt/stack-bench-embedded-deps/BSL.txt + +RUN node dist/container/binary-provenance.js verify \ + --root /opt/stack-bench --source-sha256 "$(node -p "require('./source-identity.json').binarySourceSha256")" \ + && test "$(node -p "require('playwright/package.json').version")" = "1.62.1" \ + && rm -rf tests dist/tests \ + && node --input-type=module -e 'import {AGENT_ADAPTER_REGISTRY,agentAdapterIdentity} from "./dist/src/agents/agent-adapters.js"; for (const id of AGENT_ADAPTER_REGISTRY.ids) agentAdapterIdentity(AGENT_ADAPTER_REGISTRY.get(id));' \ + && node --input-type=module -e 'import {resolveGuidanceProfile} from "./dist/src/campaigns/condition-compiler.js"; for (const id of ["neutral-dev","neutral-managed-dev"]) resolveGuidanceProfile(id,["mongodb","postgres","spacetime"]);' \ + && install -m 0555 container/bin/spacetimedb-cli \ + /opt/stack-bench-embedded-deps/spacetimedb-cli \ + && install -m 0555 container/bin/spacetimedb-standalone \ + /opt/stack-bench-embedded-deps/spacetimedb-standalone \ + && rm /opt/stack-bench-embedded-deps/bindings-typescript/LICENSE.txt \ + && mv /opt/stack-bench-embedded-deps/BSL.txt \ + /opt/stack-bench-embedded-deps/bindings-typescript/LICENSE.txt \ + && chmod 0444 /opt/stack-bench-embedded-deps/bindings-typescript/LICENSE.txt \ + && chmod 0555 /opt/stack-bench-embedded-deps/spacetimedb-cli \ + /opt/stack-bench-embedded-deps/spacetimedb-standalone \ + && cd /opt/stack-bench-embedded-deps/bindings-typescript \ + && pack_name="$(npm pack --pack-destination /opt/stack-bench-embedded-deps --silent)" \ + && mv "/opt/stack-bench-embedded-deps/$pack_name" /opt/stack-bench-embedded-deps/spacetimedb.tgz \ + && tar -tzf /opt/stack-bench-embedded-deps/spacetimedb.tgz | grep -Fxq package/dist/server/index.d.ts \ + && tar -tzf /opt/stack-bench-embedded-deps/spacetimedb.tgz | grep -Fxq package/dist/server/index.mjs \ + && cd /opt/stack-bench \ + && node dist/appliance/dependency-volume.js manifest \ + --source /opt/stack-bench-embedded-deps \ + --out /opt/stack-bench/dependency-manifest.json \ + && node dist/appliance/dependency-volume.js verify \ + --target /opt/stack-bench-embedded-deps \ + --manifest /opt/stack-bench/dependency-manifest.json \ + && rm -rf results .spacetime-data .loop-test + +# Direct launches use the image's pinned dependencies; Compose can override them. +ENV SPACETIME_BIN=/opt/stack-bench-embedded-deps/spacetimedb-cli \ + STDB_PACKAGE=/opt/stack-bench-embedded-deps/bindings-typescript + +ENTRYPOINT ["node", "/opt/stack-bench/dist/appliance/controller.js"] +CMD ["--help"] diff --git a/tools/stack-bench/appliance/Controller.Dockerfile.dockerignore b/tools/stack-bench/appliance/Controller.Dockerfile.dockerignore new file mode 100644 index 00000000000..6541c1bd7ac --- /dev/null +++ b/tools/stack-bench/appliance/Controller.Dockerfile.dockerignore @@ -0,0 +1,46 @@ +* +!tools +!tools/stack-bench +!tools/stack-bench/** +!skills +!skills/** +!crates +!crates/** +!licenses +!licenses/BSL.txt +!Cargo.toml +!Cargo.lock +!rust-toolchain.toml +!.cargo +!.cargo/** +!.git +!.git/** +!.gitattributes +!.gitignore +!templates +!templates/** +!package.json +!pnpm-lock.yaml +!pnpm-workspace.yaml + +**/node_modules +**/dist +**/target +tools/stack-bench/results +tools/stack-bench/.spacetime-data* +tools/stack-bench/.loop-test +tools/stack-bench/*.local.md +tools/stack-bench/**/*.local.md +tools/stack-bench/transcripts +tools/stack-bench/local-notes +tools/stack-bench/media +tools/stack-bench/snapshot-l* +tools/stack-bench/grader/.candidates +tools/stack-bench/grader/.mutation-report.json +tools/stack-bench/tracks/*/overview.html + +# These are generated by the Docker build, never host inputs. +tools/stack-bench/container/bin +tools/stack-bench/operator.env +.git/hooks +.git/logs diff --git a/tools/stack-bench/appliance/README.md b/tools/stack-bench/appliance/README.md new file mode 100644 index 00000000000..c9c395b85c1 --- /dev/null +++ b/tools/stack-bench/appliance/README.md @@ -0,0 +1,296 @@ +# StackBench appliance + +StackBench runs as one Docker appliance. Docker runs the controller, the coding +sessions, the databases, the grading browser, and a package cache. The host needs +Git and Docker and nothing else. + +Host requirements and the demo are in [Getting started](../GETTING-STARTED.md). +The controller manages containers through the Docker socket and keeps all state +in the `stack-bench-state` volume; its isolation is described in +[system design](../docs/system-design.md#appliance-isolation). + +## Set up for model runs + +Every command from here on runs from `tools/stack-bench` and needs +`operator.env`, the Compose environment file that `setup` writes. + +If you ran the demo, copy its saved setup: + +```sh +docker run --rm --mount type=volume,source=stack-bench-state,target=/state,readonly --entrypoint cat stack-bench-controller:local /state/controller-home/demo.env > operator.env +``` + +For a new installation without the demo, build the images and run setup from +the repository root: + +```sh +docker build --platform linux/amd64 -t stack-bench-build:local tools/stack-bench/container +docker build --platform linux/amd64 -f tools/stack-bench/appliance/Controller.Dockerfile -t stack-bench-controller:local . +docker run --rm --mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock stack-bench-controller:local setup > tools/stack-bench/operator.env +``` + +In Windows PowerShell 5, replace `> file` with `| Out-File -Encoding utf8 file`, +or the file is written as UTF-16. + +`setup` creates the volume, pins the two local images by content ID, pulls the +database images, installs the prepared plans, and keeps any existing plans and +credentials. `operator.env` is ignored by Git. + +`controller` below means this Compose prefix: + +```sh +docker compose --env-file operator.env -f appliance/docker-compose.yaml run --rm controller +``` + +### Provider credentials + +Secrets are read from the terminal and stored in the volume. They never appear +in a command line, the environment file, or a plan: + +```sh +docker run --rm -it --mount type=volume,source=stack-bench-state,target=/state stack-bench-controller:local set-secret +``` + +Then set `STACK_BENCH_AGENT_AUTH` in `operator.env` to the billing mode. +Each adapter has an API-key mode and an account (subscription) mode. There is +no fallback between them. + +| Adapter | Mode | `STACK_BENCH_AGENT_AUTH` | Secret name and source | +| --- | --- | --- | --- | +| `claude-code` | API key | `api-key` | `anthropic_api_key` | +| `claude-code` | Claude plan | `subscription-token` (default) | `claude_subscription_token` | +| `codex` | API key | `openai-api-key` | `openai_api_key` | +| `codex` | ChatGPT plan | `openai-account` | `codex_auth`: the `auth.json` written by `codex login` | +| `grok-build` | API key | `xai-api-key` | `xai_api_key` | +| `grok-build` | Grok plan | `grok-account` | `grok_auth`: `~/.grok/auth.json` written by `grok login --device-auth` | +| `antigravity` | API key | `gemini-api-key` | `gemini_api_key` | +| `antigravity` | Google plan | `agy-account` | `agy_auth`: see below | +| `openrouter` | API key | `openrouter-api-key` | `openrouter_api_key`; also set `STACK_BENCH_OPENROUTER_API_KEY_FILE` | + +Pipe a file into `set-secret` with `docker run -i` instead of `-it`. For a +Google plan, sign in to agy in its own volume, then store the token: + +```sh +docker run --rm -it -v stack-bench-agy-login:/root --entrypoint agy stack-bench-build:local +docker run --rm -v stack-bench-agy-login:/login:ro alpine cat /login/.gemini/antigravity-cli/antigravity-oauth-token | docker run --rm -i --mount type=volume,source=stack-bench-state,target=/state stack-bench-controller:local set-secret agy_auth +``` + +Account tokens: the controller renews Grok and Google sign-ins before and +during a session. It does not renew ChatGPT tokens; if one expires, run +`codex login` again and store the new file. Campaigns record the signed-in +account, not the token. + +OpenRouter needs a model with the Responses API and local tools. Each campaign +agent entry must fix `model`, `providerRoute`, and `maxOutputTokens`; the broker +uses that one route and refuses fallbacks. + +Cost is model usage priced at the plan's recorded rates from the token counts +the provider reports. It is a comparison cost, not an invoice. Provider +features whose charges the broker cannot price, such as hosted tools or image +input, are forwarded and marked unknown, and the run's cost becomes a lower +bound. The spending cap applies to priced spend only. + +### Named credential profiles + +Jobs can pick an account per adapter or per attempt instead of using the +environment. Set `STACK_BENCH_CREDENTIAL_PROFILES_FILE` to a JSON file in +controller storage: + +```json +{ + "claude-work": { "provider": "anthropic", "mode": "subscription-token", "secretFile": "/state/secrets/claude-work", "version": "v1" }, + "openai-api": { "provider": "openai", "mode": "api-key", "secretFile": "/state/secrets/openai-api", "version": "v1" } +} +``` + +`mode` is `api-key` or `subscription-token`, the same two modes as the table +above. A job references profiles by ID: a `default`, an `adapters` map, and an +`attempts` map keyed by attempt ID, with the most specific winning. Evidence +records the profile ID, version, provider, and mode, never the secret. Bump +`version` when you replace a secret, and never overwrite one that an active +attempt is using. + +## Validate the appliance + +```sh +docker compose --env-file operator.env -f appliance/docker-compose.yaml config --quiet +controller preflight --backend spacetime,postgres,mongodb --track ecommerce --levels 1 --run-index 0 --agent-adapter reference-fixture --guidance neutral +controller campaign trial plans/reference-check.json --out campaigns/reference-check +``` + +Preflight checks images, dependencies, ports, and storage without starting +anything. The trial grades the shipped reference app in real containers with no +model calls. Run both before collecting model results. + +## Plans + +A plan fixes everything a campaign needs: model, stacks, features, checks, +budgets, repetitions, parallelism, pricing, and the exact image IDs. Plans live +under `plans/` in the state volume. Setup installs `plans/paid-l1.json` from +[`campaign.paid-l1.json`](campaign.paid-l1.json); inspect it before launch: + +```sh +controller campaign show plans/paid-l1.json +``` + +Use a new plan ID and output directory for each comparison, and never edit a +running campaign's plan. A plan frozen on another machine binds that machine's +image IDs and will not run here. + +Plan fields that shape a dependency campaign: + +- `mode.workSelection`: how much ready work each session gets. `feature` (one), + `progressive` (every ready feature, the default), or `all-at-once`. +- `repair.selection` (`feature` or `batch`) and `repair.budget`: `total`, + `perFeature`, or `perDepth: { count, carry }`. `total: 0` means no repairs. +- `order`: `declared` (the catalog's order) or `shuffled` (a per-depth + permutation drawn from `ordering.seed` when the plan compiles). +- `mode.unchangedFailureLimit` (default 3): a feature whose repairs keep + producing the same failures stops after this many observations. +- `budgets.attemptTimeoutMinutes`: working time per attempt, excluding planned + depth pauses and waits for a grading slot. + +## Run a campaign + +```sh +controller campaign run plans/paid-l1.json --out campaigns/campaign-001 +controller campaign status campaigns/campaign-001 +controller campaign inspect campaigns/campaign-001 +controller campaign stop campaigns/campaign-001 +controller campaign resume plans/paid-l1.json --out campaigns/campaign-001 +controller campaign report campaigns/campaign-001 +``` + +`run` exits when every attempt is finished. `stop` ends active work and keeps +its evidence. `resume` continues pending attempts. `report` rebuilds +`report/report.html` from the evidence. The step-by-step version is the +[run-campaign skill](../skills/run-campaign/SKILL.md). + +### Capacity + +`parallelism` in the plan is the number of attempts that run at once. Each +attempt gets a coding container (2 CPUs, 4 GiB), a database container (1 CPU, +2 GiB), a browser container (1 CPU, 4 GiB), and a broker (256 MiB). The shared +package cache has 1 CPU and 2 GiB. These are caps, not reservations. + +With `STACK_BENCH_RUNNER_CAPACITY=dynamic`, the default, admission checks free +memory and CPU before each launch and reserves 2.5 GiB for the new attempt's +first minute. Dashboard runs and jobs wait when the host is busy. CLI `campaign +run`, `trial`, `resume`, and `extend` do not: they stop launching, let active +attempts finish, and exit 2. Set the variable to a number for a fixed quota. + +### Continue a campaign + +- A session waiting on a rate-limited provider: inspect it with + `campaign continuation-status --attempt --json`, fix the provider + account, then authorize one continuation with + `campaign continue-provider --attempt --request-id `. + The wait uses the attempt's time allowance. +- More time: `campaign grant-time --attempt --grant-id --minutes 120`. + It does not change the plan, the cost cap, or the repair allowance. A stopped + attempt can take a grant only at a completed-depth checkpoint; then + `campaign resume`. +- More repairs for a dependency campaign: + `campaign grant-repairs --attempt --grant-id --level --feature --repairs `, + then `campaign resume`. +- A pause between depths in one live execution: set `mode.pauseAfterDepth` in + the plan. Attempts wait after that depth; `campaign pause-status ` + shows them and `campaign continue-depth ` releases the whole cohort. The + controller must stay running; a controller loss ends the pause and needs + recovery. +- A new campaign seeded from a finished one's source at a given depth: + `campaign extend --from --depth 2 --out --prepare-only`, + review, then `campaign run`. Stack, model, repetition, guidance, and repair + settings must match the parent. The result is labelled a seeded continuation + and its cost excludes the parent's. + +### Regrade a saved attempt + +After a grader fix, grade a saved execution again without running the agent: + +```sh +controller run --grade-from campaigns//attempts//execution-1 --out regrades/ +``` + +Add `--check ` for each check to regrade, and `--grade-level +` for a dependency candidate. The regrade is separate evidence; it does not +replace the original run or count as a new build. + +## Jobs + +A job runs one whole campaign on one host with named credentials. Submit a file +like this: + +```json +{ "key": "release-42-l2", "planFile": "l2.json", + "credentials": { "adapters": { "claude-code": "claude-work", "codex": "openai-api" } }, + "hostId": "worker-east", "capacityPolicy": "wait" } +``` + +```sh +controller job submit submission.json +controller job status +controller job list --limit 50 +controller job work --host worker-east +controller job resume +controller job cancel +controller job worker --host worker-east --concurrency 2 +``` + +The same `key` with the same request returns the same job; with a different +request it fails. `work` claims and runs one job. `worker` keeps claiming queued +jobs for its host, up to `concurrency` campaigns at once, and drains on SIGTERM. +The `worker` Compose profile runs it with `STACK_BENCH_HOST_ID` and +`STACK_BENCH_JOB_CONCURRENCY`. `capacityPolicy` is `wait` or `fail`. + +Claims do not expire, because a worker that lost contact may still have paid +requests in flight. A dead worker leaves a claim to inspect; `job resume` can +continue it once campaign reconciliation proves cleanup. + +The dashboard exposes the same operations at `/api/jobs` and the run setup at +`/api/run-setup`, `/api/runs/prepare`, and `/api/runs`. The CLI equivalents are +`job options`, `job prepare `, and `job start `. +This is a local, single-host queue. It does not distribute one campaign's +attempts across hosts or share provider quotas. + +## Upload to StackBench + +Report a finished, frozen dependency campaign, then publish it to a file or to +the site: + +```sh +controller campaign report campaigns/campaign-001 +controller campaign publish campaigns/campaign-001 --suite ecommerce-1 --out upload.json +docker compose --env-file operator.env -f appliance/docker-compose.yaml run --rm -e STACKBENCH_UPLOAD_KEY controller campaign publish campaigns/campaign-001 --suite ecommerce-1 --to https://stackbench.org +``` + +`--exclude =` marks an attempt excluded; it is still +uploaded with the reason. The site needs a suite whose feature catalog matches +the campaign. Uploads are private until an admin publishes them. Active or +unfinished attempts block publication, and changed evidence needs a new report. + +## Results and cleanup + +Results stay in the `stack-bench-state` volume. Copy a campaign out and verify +it with the [archive-evidence skill](../skills/archive-evidence/SKILL.md) before +removing anything. `campaign export --out exports/` writes a smaller +research pack (report, CSV tables, hashed public artifacts) without source, +transcripts, or media. + +Cleanup removes only resources whose ownership it can prove. If it cannot, it +keeps the evidence and marks the run quarantined. Follow the +[recover-run skill](../skills/recover-run/SKILL.md); do not delete containers or +clear the state root by hand. + +Qualification, which proves the checks before results are published, is the +[qualify-checks skill](../skills/qualify-checks/SKILL.md). Building a signed +release is the [release skill](../skills/release/SKILL.md). + +## Dashboard + +```sh +docker compose --env-file operator.env -f appliance/docker-compose.yaml --profile dashboard up -d dashboard +``` + +It serves `localhost:7331` on the loopback interface only and reads the same +state as the CLI. See [dashboard/README.md](../dashboard/README.md). diff --git a/tools/stack-bench/appliance/campaign.demo.json b/tools/stack-bench/appliance/campaign.demo.json new file mode 100644 index 00000000000..d98f7a79840 --- /dev/null +++ b/tools/stack-bench/appliance/campaign.demo.json @@ -0,0 +1,150 @@ +{ + "schemaVersion": 8, + "kind": "campaign-manifest", + "id": "demo", + "version": "1.0.0", + "state": "draft", + "title": "Model-free three-stack demo: selected L1 checks", + "track": "ecommerce", + "mode": { + "id": "sequential" + }, + "repair": { + "selection": "batch", + "budget": { + "total": 0 + } + }, + "levels": [ + 1 + ], + "selection": { + "levels": [ + { + "level": 1, + "recipe": "ecommerce.sequential-l1", + "features": [ + "ecommerce.feature.accounts", + "ecommerce.feature.cart-checkout", + "ecommerce.feature.catalog-items", + "ecommerce.feature.catalog-discovery", + "ecommerce.feature.purchasing", + "ecommerce.feature.reviews", + "ecommerce.feature.warehouse-admin" + ], + "checks": [ + "ecommerce.feature.accounts.accounts.1a", + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b", + "ecommerce.spec.access-control.review-eligibility.108a", + "ecommerce.spec.access-control.review-eligibility.108b", + "ecommerce.spec.transactional-integrity.unique-review.6b", + "ecommerce.spec.access-control.warehouse-area-boundary.7a", + "ecommerce.feature.cart-checkout.cart.4d", + "ecommerce.spec.access-control.signed-out-purchase.3a", + "ecommerce.spec.concurrency-safety.stock-limit.3d" + ] + } + ] + }, + "stacks": [ + { + "id": "spacetime", + "adapterVersion": "1.4.0" + }, + { + "id": "postgres", + "adapterVersion": "1.6.0" + }, + { + "id": "mongodb", + "adapterVersion": "1.5.0" + } + ], + "agents": [ + { + "adapter": "reference-fixture", + "adapterVersion": "1.4.0", + "model": "reference-fixture" + } + ], + "conditions": [ + { + "id": "product-request", + "guidanceProfile": "neutral", + "repairPolicy": "scored-only", + "specifications": { + "levels": [ + { + "level": 1, + "requested": [], + "expected": [ + "ecommerce.spec.access-control", + "ecommerce.spec.concurrency-safety", + "ecommerce.spec.external-data-sync", + "ecommerce.spec.live-state", + "ecommerce.spec.state-durability", + "ecommerce.spec.transactional-integrity" + ], + "observed": [] + } + ] + } + } + ], + "repetitions": 1, + "parallelism": 3, + "ordering": { + "method": "balanced-rotation", + "seed": "demo-v1" + }, + "budgets": { + "attemptTimeoutMinutes": 20, + "maxCostUsdPerAttempt": null + }, + "attemptPolicy": { + "retries": 0, + "retryOn": [], + "excludeFromAnalysis": [ + "contaminated", + "harness_failure", + "inconclusive", + "ungraded", + "provider_failure" + ] + }, + "runtime": { + "releaseManifestSha256": null, + "controllerImage": null, + "buildImage": null, + "platform": "linux/amd64" + }, + "pricing": { + "currency": "USD", + "unit": "USD-per-million-tokens", + "capturedAt": "2026-08-12T00:00:00Z", + "source": "reference fixture adapter makes no billable provider calls", + "models": { + "reference-fixture": { + "input": 0, + "output": 0, + "cacheWrite5m": 0, + "cacheWrite1h": 0, + "cacheRead": 0 + } + } + }, + "analysis": { + "primaryMetric": "checkCompletionRate", + "secondaryMetrics": [ + "totalCostUsd", + "totalTokens", + "firstBuildScoreRate", + "totalDurationMs", + "invalidAttemptRate" + ], + "dispersion": "median-iqr", + "invalidAttempts": "report-separately", + "missingData": "no-imputation", + "comparisonUnit": "stack-agent-condition-recipe" + } +} diff --git a/tools/stack-bench/appliance/campaign.ecommerce-progression-reference.json b/tools/stack-bench/appliance/campaign.ecommerce-progression-reference.json new file mode 100644 index 00000000000..48e1ae4e3b5 --- /dev/null +++ b/tools/stack-bench/appliance/campaign.ecommerce-progression-reference.json @@ -0,0 +1,95 @@ +{ + "schemaVersion": 8, + "kind": "campaign-manifest", + "id": "ecommerce-progression-reference", + "version": "2.0.1", + "state": "draft", + "title": "Ecommerce progression reference pilot", + "track": "ecommerce", + "mode": { "id": "dependency", "workSelection": "progressive" }, + "repair": { "selection": "feature", "budget": { "total": 0 } }, + "levels": [1, 2, 3, 4, 5, 6], + "featureCatalog": "progression/ecommerce.json", + "selection": { + "levels": [ + { "level": 1, "recipe": "ecommerce.progression-catalog" }, + { "level": 2, "recipe": "ecommerce.progression-catalog" }, + { "level": 3, "recipe": "ecommerce.progression-catalog" }, + { "level": 4, "recipe": "ecommerce.progression-catalog" }, + { "level": 5, "recipe": "ecommerce.progression-catalog" }, + { "level": 6, "recipe": "ecommerce.progression-catalog" } + ] + }, + "stacks": [ + { "id": "mongodb", "adapterVersion": "1.5.0" }, + { "id": "postgres", "adapterVersion": "1.6.0" }, + { "id": "spacetime", "adapterVersion": "1.4.0" } + ], + "agents": [ + { + "adapter": "reference-fixture", + "adapterVersion": "1.4.0", + "model": "reference-fixture" + } + ], + "conditions": [ + { + "id": "reference-pilot", + "guidanceProfile": "neutral", + "repairPolicy": "scored-only" + } + ], + "repetitions": 1, + "parallelism": 1, + "ordering": { + "method": "balanced-rotation", + "seed": "ecommerce-progression-reference-1" + }, + "budgets": { + "attemptTimeoutMinutes": 180, + "maxCostUsdPerAttempt": null + }, + "attemptPolicy": { + "retries": 0, + "retryOn": [], + "excludeFromAnalysis": [ + "contaminated", + "harness_failure", + "inconclusive", + "ungraded" + ] + }, + "runtime": { + "releaseManifestSha256": null, + "controllerImage": null, + "buildImage": null, + "platform": "linux/amd64" + }, + "pricing": { + "currency": "USD", + "unit": "USD-per-million-tokens", + "capturedAt": "2026-08-25T00:00:00.000Z", + "source": "Reference fixtures make no provider calls.", + "models": { + "reference-fixture": { + "input": 0, + "output": 0, + "cacheWrite5m": 0, + "cacheWrite1h": 0, + "cacheRead": 0 + } + } + }, + "analysis": { + "primaryMetric": "finalScoreRate", + "secondaryMetrics": [ + "firstBuildScoreRate", + "totalDurationMs", + "invalidAttemptRate" + ], + "dispersion": "median-iqr", + "invalidAttempts": "report-separately", + "missingData": "no-imputation", + "comparisonUnit": "stack-agent-condition-recipe" + } +} diff --git a/tools/stack-bench/appliance/campaign.example.json b/tools/stack-bench/appliance/campaign.example.json new file mode 100644 index 00000000000..a812860a14f --- /dev/null +++ b/tools/stack-bench/appliance/campaign.example.json @@ -0,0 +1,141 @@ +{ + "schemaVersion": 8, + "kind": "campaign-manifest", + "id": "ecommerce-l1-reference-check", + "version": "2.0.0", + "state": "draft", + "title": "Ecommerce L1 model-free appliance check", + "track": "ecommerce", + "mode": { + "id": "sequential" + }, + "repair": { + "selection": "batch", + "budget": { + "total": 0 + } + }, + "levels": [ + 1 + ], + "selection": { + "levels": [ + { + "level": 1, + "recipe": "ecommerce.sequential-l1", + "features": [ + "ecommerce.feature.accounts", + "ecommerce.feature.cart-checkout", + "ecommerce.feature.catalog-items", + "ecommerce.feature.catalog-discovery", + "ecommerce.feature.purchasing", + "ecommerce.feature.reviews", + "ecommerce.feature.warehouse-admin" + ], + "checks": [] + } + ] + }, + "stacks": [ + { + "id": "spacetime", + "adapterVersion": "1.4.0" + }, + { + "id": "postgres", + "adapterVersion": "1.6.0" + }, + { + "id": "mongodb", + "adapterVersion": "1.5.0" + } + ], + "agents": [ + { + "adapter": "reference-fixture", + "adapterVersion": "1.4.0", + "model": "reference-fixture" + } + ], + "conditions": [ + { + "id": "product-request", + "productionQuality": true, + "guidanceProfile": "neutral", + "repairPolicy": "scored-only", + "specifications": { + "levels": [ + { + "level": 1, + "requested": [], + "expected": [ + "ecommerce.spec.access-control", + "ecommerce.spec.concurrency-safety", + "ecommerce.spec.external-data-sync", + "ecommerce.spec.live-state", + "ecommerce.spec.state-durability", + "ecommerce.spec.transactional-integrity" + ], + "observed": [] + } + ] + } + } + ], + "repetitions": 1, + "parallelism": 1, + "ordering": { + "method": "balanced-rotation", + "seed": "replace-before-measurement" + }, + "budgets": { + "attemptTimeoutMinutes": 240, + "maxCostUsdPerAttempt": null + }, + "attemptPolicy": { + "retries": 1, + "retryOn": [ + "provider_failure" + ], + "excludeFromAnalysis": [ + "contaminated", + "harness_failure", + "inconclusive", + "ungraded" + ] + }, + "runtime": { + "releaseManifestSha256": null, + "controllerImage": null, + "buildImage": null, + "platform": "linux/amd64" + }, + "pricing": { + "currency": "USD", + "unit": "USD-per-million-tokens", + "capturedAt": "2026-08-12T00:00:00.000Z", + "source": "reference fixture adapter makes no billable provider calls", + "models": { + "reference-fixture": { + "input": 0, + "output": 0, + "cacheWrite5m": 0, + "cacheWrite1h": 0, + "cacheRead": 0 + } + } + }, + "analysis": { + "primaryMetric": "firstBuildScoreRate", + "secondaryMetrics": [ + "finalScoreRate", + "totalCostUsd", + "totalDurationMs", + "invalidAttemptRate" + ], + "dispersion": "median-iqr", + "invalidAttempts": "report-separately", + "missingData": "no-imputation", + "comparisonUnit": "stack-agent-condition-recipe" + } +} diff --git a/tools/stack-bench/appliance/campaign.paid-l1-l3.json b/tools/stack-bench/appliance/campaign.paid-l1-l3.json new file mode 100644 index 00000000000..1f4edbc2995 --- /dev/null +++ b/tools/stack-bench/appliance/campaign.paid-l1-l3.json @@ -0,0 +1,217 @@ +{ + "schemaVersion": 8, + "kind": "campaign-manifest", + "id": "paid-l1-l3-pilot", + "version": "1.0.0", + "state": "draft", + "title": "Three-stack L1-L3 progression pilot", + "track": "ecommerce", + "mode": { + "id": "sequential" + }, + "repair": { + "selection": "batch", + "budget": { + "total": 6 + } + }, + "levels": [ + 1, + 2, + 3 + ], + "selection": { + "levels": [ + { + "level": 1, + "recipe": "ecommerce.sequential-l1", + "features": [ + "ecommerce.feature.accounts", + "ecommerce.feature.cart-checkout", + "ecommerce.feature.catalog-items", + "ecommerce.feature.catalog-discovery", + "ecommerce.feature.purchasing", + "ecommerce.feature.reviews", + "ecommerce.feature.warehouse-admin" + ], + "checks": [] + }, + { + "level": 2, + "recipe": "ecommerce.sequential-l2", + "features": [ + "ecommerce.feature.accounts", + "ecommerce.feature.cart-checkout", + "ecommerce.feature.catalog-items", + "ecommerce.feature.catalog-discovery", + "ecommerce.feature.purchasing", + "ecommerce.feature.reviews", + "ecommerce.feature.warehouse-admin", + "ecommerce.inventory-operations-features", + "ecommerce.operations-access-features", + "ecommerce.returns-pricing-features" + ], + "checks": [] + }, + { + "level": 3, + "recipe": "ecommerce.sequential-l3", + "features": [ + "ecommerce.feature.accounts", + "ecommerce.feature.cart-checkout", + "ecommerce.feature.catalog-items", + "ecommerce.feature.catalog-discovery", + "ecommerce.feature.purchasing", + "ecommerce.feature.reviews", + "ecommerce.feature.warehouse-admin", + "ecommerce.inventory-operations-features", + "ecommerce.operations-access-features", + "ecommerce.returns-pricing-features", + "ecommerce.l3.reservations-features", + "ecommerce.l3.scheduled-restocks-features", + "ecommerce.l3.order-delivery-features", + "ecommerce.l3.cart-expiration-features" + ], + "checks": [] + } + ] + }, + "stacks": [ + { + "id": "spacetime", + "adapterVersion": "1.4.0" + }, + { + "id": "postgres", + "adapterVersion": "1.6.0" + }, + { + "id": "mongodb", + "adapterVersion": "1.5.0" + } + ], + "agents": [ + { + "adapter": "claude-code", + "adapterVersion": "1.17.2", + "model": "claude-sonnet-5" + } + ], + "conditions": [ + { + "id": "product-request", + "productionQuality": true, + "guidanceProfile": "neutral", + "repairPolicy": "scored-only", + "specifications": { + "levels": [ + { + "level": 1, + "requested": [], + "expected": [ + "ecommerce.spec.access-control", + "ecommerce.spec.concurrency-safety", + "ecommerce.spec.external-data-sync", + "ecommerce.spec.live-state", + "ecommerce.spec.state-durability", + "ecommerce.spec.transactional-integrity" + ], + "observed": [] + }, + { + "level": 2, + "requested": [], + "expected": [ + "ecommerce.inventory-operations-specifications", + "ecommerce.operations-access-specifications", + "ecommerce.returns-pricing-specifications", + "ecommerce.spec.access-control", + "ecommerce.spec.concurrency-safety", + "ecommerce.spec.external-data-sync", + "ecommerce.spec.live-state", + "ecommerce.spec.state-durability", + "ecommerce.spec.transactional-integrity" + ], + "observed": [] + }, + { + "level": 3, + "requested": [], + "expected": [ + "ecommerce.inventory-operations-specifications", + "ecommerce.l3.deferred-access-specifications", + "ecommerce.l3.deferred-durability-specifications", + "ecommerce.l3.deferred-integrity-specifications", + "ecommerce.l3.server-time-specifications", + "ecommerce.operations-access-specifications", + "ecommerce.returns-pricing-specifications", + "ecommerce.spec.access-control", + "ecommerce.spec.concurrency-safety", + "ecommerce.spec.external-data-sync", + "ecommerce.spec.live-state", + "ecommerce.spec.state-durability", + "ecommerce.spec.transactional-integrity" + ], + "observed": [] + } + ] + } + } + ], + "repetitions": 1, + "parallelism": 3, + "ordering": { + "method": "balanced-rotation", + "seed": "paid-l1-l3-pilot-v1" + }, + "budgets": { + "attemptTimeoutMinutes": 120, + "maxCostUsdPerAttempt": 30 + }, + "attemptPolicy": { + "retries": 0, + "retryOn": [], + "excludeFromAnalysis": [ + "contaminated", + "harness_failure", + "inconclusive", + "ungraded", + "provider_failure" + ] + }, + "runtime": { + "releaseManifestSha256": null, + "controllerImage": null, + "buildImage": null, + "platform": "linux/amd64" + }, + "pricing": { + "currency": "USD", + "unit": "USD-per-million-tokens", + "capturedAt": "2026-09-05T03:44:22.451Z", + "source": "https://platform.claude.com/docs/en/about-claude/pricing — verified current Sonnet 5 standard rates; subscription usage is normalized API-equivalent token cost, not an invoice charge", + "models": { + "claude-sonnet-5": { + "input": 2, + "output": 10, + "cacheWrite5m": 2.5, + "cacheWrite1h": 4, + "cacheRead": 0.2 + } + } + }, + "analysis": { + "primaryMetric": "checkCompletionRate", + "secondaryMetrics": [ + "totalCostUsd", + "totalTokens", + "firstBuildScoreRate", + "totalDurationMs", + "invalidAttemptRate" + ], + "dispersion": "median-iqr", + "invalidAttempts": "report-separately", + "missingData": "no-imputation", + "comparisonUnit": "stack-agent-condition-recipe" + } +} diff --git a/tools/stack-bench/appliance/campaign.paid-l1.json b/tools/stack-bench/appliance/campaign.paid-l1.json new file mode 100644 index 00000000000..758d5a28bd7 --- /dev/null +++ b/tools/stack-bench/appliance/campaign.paid-l1.json @@ -0,0 +1,141 @@ +{ + "schemaVersion": 8, + "kind": "campaign-manifest", + "id": "paid-l1-demo", + "version": "1.0.0", + "state": "draft", + "title": "Three-stack L1 demo", + "track": "ecommerce", + "mode": { + "id": "sequential" + }, + "repair": { + "selection": "batch", + "budget": { + "total": 0 + } + }, + "levels": [ + 1 + ], + "selection": { + "levels": [ + { + "level": 1, + "recipe": "ecommerce.sequential-l1", + "features": [ + "ecommerce.feature.accounts", + "ecommerce.feature.cart-checkout", + "ecommerce.feature.catalog-items", + "ecommerce.feature.catalog-discovery", + "ecommerce.feature.purchasing", + "ecommerce.feature.reviews", + "ecommerce.feature.warehouse-admin" + ], + "checks": [] + } + ] + }, + "stacks": [ + { + "id": "spacetime", + "adapterVersion": "1.4.0" + }, + { + "id": "postgres", + "adapterVersion": "1.6.0" + }, + { + "id": "mongodb", + "adapterVersion": "1.5.0" + } + ], + "agents": [ + { + "adapter": "claude-code", + "adapterVersion": "1.17.2", + "model": "claude-sonnet-5" + } + ], + "conditions": [ + { + "id": "product-request", + "productionQuality": true, + "guidanceProfile": "neutral", + "repairPolicy": "scored-only", + "specifications": { + "levels": [ + { + "level": 1, + "requested": [], + "expected": [ + "ecommerce.spec.access-control", + "ecommerce.spec.concurrency-safety", + "ecommerce.spec.external-data-sync", + "ecommerce.spec.live-state", + "ecommerce.spec.state-durability", + "ecommerce.spec.transactional-integrity" + ], + "observed": [] + } + ] + } + } + ], + "repetitions": 1, + "parallelism": 3, + "ordering": { + "method": "balanced-rotation", + "seed": "paid-l1-demo-v1" + }, + "budgets": { + "attemptTimeoutMinutes": 60, + "maxCostUsdPerAttempt": 10 + }, + "attemptPolicy": { + "retries": 0, + "retryOn": [], + "excludeFromAnalysis": [ + "contaminated", + "harness_failure", + "inconclusive", + "ungraded", + "provider_failure" + ] + }, + "runtime": { + "releaseManifestSha256": null, + "controllerImage": null, + "buildImage": null, + "platform": "linux/amd64" + }, + "pricing": { + "currency": "USD", + "unit": "USD-per-million-tokens", + "capturedAt": "2026-09-05T03:44:22.451Z", + "source": "https://platform.claude.com/docs/en/about-claude/pricing — verified current Sonnet 5 standard rates; subscription usage is normalized API-equivalent token cost, not an invoice charge", + "models": { + "claude-sonnet-5": { + "input": 2, + "output": 10, + "cacheWrite5m": 2.5, + "cacheWrite1h": 4, + "cacheRead": 0.2 + } + } + }, + "analysis": { + "primaryMetric": "checkCompletionRate", + "secondaryMetrics": [ + "totalCostUsd", + "totalTokens", + "firstBuildScoreRate", + "totalDurationMs", + "invalidAttemptRate" + ], + "dispersion": "median-iqr", + "invalidAttempts": "report-separately", + "missingData": "no-imputation", + "comparisonUnit": "stack-agent-condition-recipe" + } +} diff --git a/tools/stack-bench/appliance/campaign.product-brief-reference.json b/tools/stack-bench/appliance/campaign.product-brief-reference.json new file mode 100644 index 00000000000..a8133b1feaa --- /dev/null +++ b/tools/stack-bench/appliance/campaign.product-brief-reference.json @@ -0,0 +1,115 @@ +{ + "schemaVersion": 8, + "kind": "campaign-manifest", + "id": "ecommerce-l1-product-brief-reference", + "version": "2.0.0", + "state": "draft", + "title": "Ecommerce L1 product brief and quality validation", + "track": "ecommerce", + "mode": { "id": "sequential" }, + "repair": { "selection": "batch", "budget": { "total": 0 } }, + "levels": [1], + "selection": { + "levels": [ + { + "level": 1, + "recipe": "ecommerce.sequential-l1", + "features": [ + "ecommerce.feature.accounts", + "ecommerce.feature.cart-checkout", + "ecommerce.feature.catalog-items", + "ecommerce.feature.catalog-discovery", + "ecommerce.feature.purchasing", + "ecommerce.feature.reviews", + "ecommerce.feature.warehouse-admin" + ], + "checks": [] + } + ] + }, + "stacks": [ + { "id": "spacetime", "adapterVersion": "1.4.0" }, + { "id": "postgres", "adapterVersion": "1.6.0" }, + { "id": "mongodb", "adapterVersion": "1.5.0" } + ], + "agents": [ + { + "adapter": "reference-fixture", + "adapterVersion": "1.4.0", + "model": "reference-fixture" + } + ], + "conditions": [ + { + "id": "product-brief-quality", + "guidanceProfile": "neutral", + "repairPolicy": "scored-only", + "specifications": { + "levels": [ + { + "level": 1, + "requested": [], + "expected": [ + "ecommerce.spec.access-control", + "ecommerce.spec.concurrency-safety", + "ecommerce.spec.external-data-sync", + "ecommerce.spec.live-state", + "ecommerce.spec.state-durability", + "ecommerce.spec.transactional-integrity" + ], + "observed": [] + } + ] + } + } + ], + "repetitions": 2, + "parallelism": 1, + "ordering": { + "method": "balanced-rotation", + "seed": "product-brief-quality-reference-1" + }, + "budgets": { + "attemptTimeoutMinutes": 60, + "maxCostUsdPerAttempt": null + }, + "attemptPolicy": { + "retries": 1, + "retryOn": ["harness_failure", "inconclusive"], + "excludeFromAnalysis": ["contaminated", "harness_failure", "inconclusive", "ungraded"] + }, + "runtime": { + "releaseManifestSha256": null, + "controllerImage": null, + "buildImage": null, + "platform": "linux/amd64" + }, + "pricing": { + "currency": "USD", + "unit": "USD-per-million-tokens", + "capturedAt": "2026-08-16T00:00:00.000Z", + "source": "reference fixture adapter makes no billable provider calls", + "models": { + "reference-fixture": { + "input": 0, + "output": 0, + "cacheWrite5m": 0, + "cacheWrite1h": 0, + "cacheRead": 0 + } + } + }, + "analysis": { + "primaryMetric": "firstBuildScoreRate", + "secondaryMetrics": [ + "finalScoreRate", + "totalCostUsd", + "totalDurationMs", + "invalidAttemptRate" + ], + "dispersion": "median-iqr", + "invalidAttempts": "report-separately", + "missingData": "no-imputation", + "comparisonUnit": "stack-agent-condition-recipe" + } +} diff --git a/tools/stack-bench/appliance/controller.ts b/tools/stack-bench/appliance/controller.ts new file mode 100644 index 00000000000..ec13d4ae8e0 --- /dev/null +++ b/tools/stack-bench/appliance/controller.ts @@ -0,0 +1,252 @@ +#!/usr/bin/env node + +import { spawn } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { resolveContainerImage } from '../src/runtime/container-image.js'; +import { parsePreflightArgs } from '../commands/preflight-cli.js'; +import { parseBenchArguments } from '../commands/bench-arguments.js'; +import { AGENT_ADAPTER_REGISTRY } from '../src/agents/agent-adapters.js'; +import { stateVolumeCommand } from './state-volume.js'; + +const RUNTIME_ROOT = join(STACK_BENCH_ROOT, 'dist'); + +const COMMANDS = Object.freeze({ + 'demo': [join(RUNTIME_ROOT, 'appliance', 'demo.js')], + 'init-deps': [join(RUNTIME_ROOT, 'appliance', 'dependency-volume.js'), 'init'], + 'verify-deps': [join(RUNTIME_ROOT, 'appliance', 'dependency-volume.js'), 'verify'], + 'preflight': [join(RUNTIME_ROOT, 'commands', 'preflight.js')], + 'qualify-reference': [join(RUNTIME_ROOT, 'src', 'references', 'reference-live.js')], + 'qualify-null': [join(RUNTIME_ROOT, 'commands', 'null-control.js')], + 'qualification': [join(RUNTIME_ROOT, 'commands', 'qualification-cli.js')], + 'pack-budget': [join(RUNTIME_ROOT, 'commands', 'pack-budget.js')], + 'job': [join(RUNTIME_ROOT, 'commands', 'job-cli.js')], + 'campaign': [join(RUNTIME_ROOT, 'commands', 'campaign-cli.js')], + 'dashboard': [join(RUNTIME_ROOT, 'dashboard', 'dashboard-server.js')], + 'repair': [join(RUNTIME_ROOT, 'commands', 'repair-cli.js')], + 'run': [join(RUNTIME_ROOT, 'commands', 'bench.js')], + 'verify-release': [join(RUNTIME_ROOT, 'src', 'releases', 'release-manifest.js'), 'verify'], + 'recover': [join(RUNTIME_ROOT, 'commands', 'recovery.js'), 'recover'], + 'recover-lease': [join(RUNTIME_ROOT, 'commands', 'recovery.js'), 'recover-lease'], +} satisfies Record); + +const COMMANDS_REQUIRING_AGENT_AUTH = new Set(['preflight', 'run']); + +export function controllerCommandRequiresAgentAuth(command: string | undefined, + args: string[] = []): boolean { + if (command === 'job' && ['prepare', 'start', 'resume', 'work', 'worker'].includes(args[0] ?? '')) return true; + if (command === 'run' && args.some(value => value === '--grade-from' || value.startsWith('--grade-from='))) { + return !parseBenchArguments([process.execPath, 'bench', ...args]).gradeFrom; + } + if (command === 'preflight' && args.length) { + const request = parsePreflightArgs([process.execPath, 'preflight', ...args]); + return AGENT_ADAPTER_REGISTRY.get(request.agentAdapter).costLimit !== 'non-billable'; + } + if (command && COMMANDS_REQUIRING_AGENT_AUTH.has(command)) return true; + return command === 'campaign' && ['run', 'resume', 'extend'].includes(args[0] ?? ''); +} + +export function controllerRuntimeEnvironment(source: NodeJS.ProcessEnv = process.env, + resolveImage = resolveContainerImage): NodeJS.ProcessEnv { + if (!source.STACK_BENCH_CONTROLLER_IMAGE) { + throw new Error('STACK_BENCH_CONTROLLER_IMAGE is required for runtime work'); + } + return { ...source, STACK_BENCH_CONTROLLER_IMAGE_ID: + resolveImage(source.STACK_BENCH_CONTROLLER_IMAGE).id }; +} + +export function controllerRuntimeCommand(args: string[], source: NodeJS.ProcessEnv = process.env) { + if (!source.STACK_BENCH_COMPOSE_FILE || !source.STACK_BENCH_STATE_ROOT + || !source.STACK_BENCH_CONTROLLER_IMAGE || !source.STACK_BENCH_BUILD_IMAGE) { + throw new Error('controller launch requires the setup environment and appliance Compose file'); + } + const ownership = randomUUID(); + const containerName = `stack-bench-controller-${ownership}`; + const ownershipLabel = `io.spacetimedb.stack-bench.controller-owner=${ownership}`; + const env: NodeJS.ProcessEnv = { ...controllerChildEnvironment(source, { requireAgentAuth: false }), + STACK_BENCH_BUILD_IMAGE: source.STACK_BENCH_BUILD_IMAGE }; + return { executable: 'docker', containerName, ownershipLabel, + args: ['compose', '-f', source.STACK_BENCH_COMPOSE_FILE, 'run', '--rm', '--no-deps', + '--name', containerName, '--label', ownershipLabel, 'controller', ...args], + env }; +} + +export interface ResolvedControllerCommand { + executable: string; + args: string[]; +} + +export function resolveControllerCommand(argv: string[]): ResolvedControllerCommand | null { + const [command, ...rest] = argv; + if (!command || command === '--help' || command === 'help') return null; + if (!Object.hasOwn(COMMANDS, command)) { + throw new Error(`unknown controller command ${JSON.stringify(command)}`); + } + return { executable: process.execPath, + args: [...COMMANDS[command as keyof typeof COMMANDS], ...rest] }; +} + +export function controllerChildEnvironment(source: NodeJS.ProcessEnv = process.env, + { requireAgentAuth = true }: { requireAgentAuth?: boolean } = {}): NodeJS.ProcessEnv { + const env = { ...source }; + const modes: Record = { + 'subscription-token': ['STACK_BENCH_CLAUDE_OAUTH_TOKEN_FILE', 'CLAUDE_CODE_OAUTH_TOKEN_FILE'], + 'api-key': ['STACK_BENCH_ANTHROPIC_API_KEY_FILE', 'ANTHROPIC_API_KEY_FILE'], + 'openrouter-api-key': ['STACK_BENCH_OPENROUTER_API_KEY_FILE', 'OPENROUTER_API_KEY_FILE'], + 'openai-api-key': ['STACK_BENCH_OPENAI_API_KEY_FILE', 'OPENAI_API_KEY_FILE'], + 'openai-account': ['STACK_BENCH_CODEX_AUTH_FILE', 'CODEX_AUTH_FILE'], + 'xai-api-key': ['STACK_BENCH_XAI_API_KEY_FILE', 'XAI_API_KEY_FILE'], + 'grok-account': ['STACK_BENCH_GROK_AUTH_FILE', 'GROK_AUTH_FILE'], + 'gemini-api-key': ['STACK_BENCH_GEMINI_API_KEY_FILE', 'GEMINI_API_KEY_FILE'], + 'agy-account': ['STACK_BENCH_AGY_AUTH_FILE', 'AGY_AUTH_FILE'], + }; + // Named jobs select per attempt. Keep the legacy default available without + // clearing credentials belonging to other providers. + if (source.STACK_BENCH_CREDENTIAL_PROFILES_FILE) { + const selected = modes[source.STACK_BENCH_AGENT_AUTH ?? 'subscription-token']; + if (selected && source[selected[0]]?.trim()) env[selected[1]] = source[selected[0]]!.trim(); + return env; + } + for (const [, variable] of Object.values(modes)) { + delete env[variable]; + delete env[variable.replace(/_FILE$/, '')]; + } + delete env.STACK_BENCH_AGENT_API_KEY; + if (!requireAgentAuth) return env; + const mode = source.STACK_BENCH_AGENT_AUTH ?? 'subscription-token'; + const selected = Object.hasOwn(modes, mode) ? modes[mode] : undefined; + if (!selected) throw new Error(`STACK_BENCH_AGENT_AUTH must be ${Object.keys(modes).join(' or ')}`); + const [sourceName, variable] = selected; + const path = source[sourceName]?.trim(); + if (!path) throw new Error(`${mode} auth requires ${sourceName}`); + env[variable] = path; + return env; +} + +interface SignalChild { + kill(signal: NodeJS.Signals): unknown; +} + +interface SignalSource { + on(signal: NodeJS.Signals, listener: () => void): unknown; + off(signal: NodeJS.Signals, listener: () => void): unknown; +} + +export function forwardControllerSignals(child: SignalChild, + source: SignalSource = process): () => void { + const signals: NodeJS.Signals[] = ['SIGINT', 'SIGTERM']; + const listeners = new Map void>(signals.map(signal => + [signal, () => { child.kill(signal); }])); + for (const [signal, listener] of listeners) source.on(signal, listener); + return () => { + for (const [signal, listener] of listeners) source.off(signal, listener); + }; +} + +function help(): void { + process.stdout.write('Stack Bench controller\n' + + '\n' + + 'Docker setup\n' + + ' setup prepare the state volume and print operator.env\n' + + ' set-secret read one secret from stdin into the volume mounted at /state\n' + + '\n' + + 'A campaign compares stacks by building the same product on each. Point\n' + + 'commands at the durable plans/ and campaigns/ directories.\n' + + '\n' + + 'Run a campaign\n' + + ' job options list workload and agent choices\n' + + ' job prepare review selections without running models\n' + + ' job start --host start a reviewed run\n' + + ' job submit submit an idempotent execution job\n' + + ' job work --host claim and execute one submitted job\n' + + ' job resume continue an interrupted job with its saved accounts and capacity policy\n' + + ' job worker --host --concurrency dispatch queued jobs automatically\n' + + ' job list|status |cancel inspect or cancel submitted work\n' + + ' preflight --backend --track --levels \n' + + ' verify the runner without creating an attempt\n' + + ' campaign validate compile a plan file and report what is wrong with it\n' + + ' campaign show print the compiled plan\n' + + ' campaign trial --out run the plan with a model-free agent\n' + + ' campaign run --out run the plan; run it again on the same to continue\n' + + ' campaign resume --out continue an interrupted dependency attempt from its saved state\n' + + ' campaign extend --from --depth --out continue a finished campaign deeper\n' + + ' campaign stop stop owned active work and retain its evidence\n' + + ' campaign pause-status inspect a planned between-depth hold\n' + + ' campaign continue-depth release the cohort at its planned depth boundary\n' + + ' campaign status [--full] what the campaign is doing now, from its saved state\n' + + ' campaign inspect every attempt, level, and check with its evidence\n' + + ' campaign report write the JSON and HTML report\n' + + ' campaign export --out write a partial copy with the report, CSV tables, and public artifacts\n' + + ' campaign publish --suite --to [--exclude =] upload a finished dependency campaign\n' + + ' use --out instead of --to to export the upload JSON\n' + + ' campaign audit check a finished reference campaign against its promises\n' + + ' campaign grant-repairs --attempt --grant-id --level --feature [--feature ...] --repairs add repair budget\n' + + ' campaign grant-time --attempt --grant-id --minutes add time\n' + + ' campaign continuation-status --attempt [--json] inspect a live provider wait\n' + + ' campaign continue-provider --attempt --request-id continue the waiting provider session once\n' + + ' campaign reconcile --out clean up after an interruption and prove it\n' + + ' campaign modes list the campaign modes this controller knows\n' + + ' dashboard [--port N] serve the local dashboard\n' + + '\n' + + 'One attempt outside a campaign\n' + + ' run --backend --track --levels --out [...] build and grade one attempt\n' + + ' run --grade-from --grade-level --check --out replay saved dependency source without model calls\n' + + ' repair status --level can a failed level continue?\n' + + ' repair grant --level --repairs add one repair budget\n' + + '\n' + + 'Qualify the grader\n' + + ' qualify-reference --backend --track --level grade the hand-built reference app, or its mutations\n' + + ' --mutation-workers N split the mutation run across 1 to 8 isolated workers\n' + + ' qualify-null --track --level prove an empty app scores nothing\n' + + ' qualification status --track --level which grading evidence is still missing\n' + + ' pack-budget recommend --track --level [--recipe ] --evidence [--evidence ...] --out derive pack limits from reference evidence\n' + + '\n' + + 'Recover and verify\n' + + ' recover retry cleanup for an interrupted attempt, or keep its quarantine\n' + + ' recover-lease --out recover when the attempt state was not kept\n' + + ' verify-release --root verify a candidate or signed release\n' + + ' init-deps | verify-deps create or verify the release dependency volume\n'); +} + +interface ChildOutcome { + code: number | null; + signal: NodeJS.Signals | null; +} + +async function main(argv: string[]): Promise { + const command = argv[2]; + if (command === 'setup' || command === 'set-secret') { + await stateVolumeCommand(command, argv.slice(3)); + return; + } + const resolved = resolveControllerCommand(argv.slice(2)); + if (!resolved) { help(); return; } + let env = controllerChildEnvironment(process.env, + { requireAgentAuth: controllerCommandRequiresAgentAuth(command, argv.slice(3)) }); + const runtime = ['preflight', 'run', 'qualify-reference', 'qualify-null', 'recover', 'recover-lease'] + .includes(command ?? '') || (command === 'campaign' + && ['run', 'trial', 'resume', 'extend', 'reconcile'].includes(argv[3] ?? '')) + || (command === 'job' && ['start', 'resume', 'work', 'worker'].includes(argv[3] ?? '')); + if (runtime) env = controllerRuntimeEnvironment(env); + const child = spawn(resolved.executable, resolved.args, { stdio: 'inherit', env }); + const stopForwardingSignals = forwardControllerSignals(child); + let outcome: ChildOutcome; + try { + outcome = await new Promise((resolveExit, reject) => { + child.once('error', reject); + child.once('exit', (code, signal) => { resolveExit({ code, signal }); }); + }); + } finally { stopForwardingSignals(); } + if (outcome.signal) process.kill(process.pid, outcome.signal); + process.exitCode = outcome.code ?? 1; +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + main(process.argv).catch((error: unknown) => { + console.error(`stack-bench-controller: ${error instanceof Error ? error.message : String(error)}`); + process.exitCode = 2; + }); +} diff --git a/tools/stack-bench/appliance/demo.compose.yaml b/tools/stack-bench/appliance/demo.compose.yaml new file mode 100644 index 00000000000..06267966444 --- /dev/null +++ b/tools/stack-bench/appliance/demo.compose.yaml @@ -0,0 +1,28 @@ +name: stack-bench-demo + +services: + build-image: + image: stack-bench-build:local + platform: linux/amd64 + build: + context: ../container + dockerfile: Dockerfile + entrypoint: ["/bin/true"] + demo: + image: stack-bench-controller:local + platform: linux/amd64 + build: + context: ../../.. + dockerfile: tools/stack-bench/appliance/Controller.Dockerfile + depends_on: + build-image: + condition: service_completed_successfully + init: true + command: ["demo"] + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - state:/state + +volumes: + state: + name: stack-bench-state diff --git a/tools/stack-bench/appliance/demo.ts b/tools/stack-bench/appliance/demo.ts new file mode 100644 index 00000000000..8ff0902403b --- /dev/null +++ b/tools/stack-bench/appliance/demo.ts @@ -0,0 +1,66 @@ +import { spawn } from 'node:child_process'; +import { chmodSync, existsSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { readCampaignState } from '../src/campaigns/campaign-scheduler.js'; +import { forwardControllerSignals } from './controller.js'; +import { prepareStateVolume } from './state-volume.js'; + +export function demoConfiguration(setup: string, source: NodeJS.ProcessEnv = process.env) { + const prepared: NodeJS.ProcessEnv = {}; + for (const line of setup.split('\n').filter(Boolean)) { + const separator = line.indexOf('='); + if (separator < 1) throw new Error('Invalid setup environment'); + prepared[line.slice(0, separator)] = line.slice(separator + 1); + } + const digest = prepared.STACK_BENCH_CONTROLLER_IMAGE?.match(/^(?:.*@)?sha256:([a-f0-9]{64})$/)?.[1]; + if (!digest) throw new Error('Demo requires a resolved controller image digest'); + prepared.STACK_BENCH_RELEASE_DEPS_VOLUME = `stack-bench-release-deps-${digest.slice(0, 12)}`; + const output = `campaigns/demo-${digest.slice(0, 12)}`; + const compose = ['compose', '-f', join(STACK_BENCH_ROOT, 'appliance/docker-compose.yaml')]; + return { env: { ...source, ...prepared }, output, + dashboard: [...compose, '--profile', 'dashboard', 'up', '-d', 'dashboard'], + campaign: [...compose, 'run', '--rm', '-T', '--name', `stack-bench-demo-${digest.slice(0, 12)}`, + 'controller', 'campaign', 'trial', 'plans/demo.json', '--out', output], + savedEnvironment: Object.entries(prepared) + .map(([key, value]) => `${key}=${value ?? ''}`).join('\n') + '\n', + }; +} + +async function docker(args: string[], env: NodeJS.ProcessEnv): Promise { + const child = spawn('docker', args, { env, stdio: 'inherit' }); + const stopForwarding = forwardControllerSignals(child); + try { + await new Promise((resolve, reject) => { + child.once('error', reject); + child.once('exit', (code, signal) => code === 0 ? resolve() + : reject(new Error(`Demo Docker command failed (${signal ?? code})`))); + }); + } finally { stopForwarding(); } +} + +export async function demoCommand(args: string[]): Promise { + if (args.length) throw new Error('demo accepts no arguments; set image references through the environment'); + const config = demoConfiguration(prepareStateVolume()); + const environmentPath = '/state/controller-home/demo.env'; + writeFileSync(environmentPath, config.savedEnvironment, { mode: 0o600 }); + chmodSync(environmentPath, 0o600); + await docker(config.dashboard, config.env); + console.log('Stack Bench dashboard: http://127.0.0.1:7331'); + const directory = join('/state/results', config.output); + if (existsSync(join(directory, 'state.json'))) { + const { state } = readCampaignState(directory); + console.log(`Existing demo campaign: ${state.status} (${config.output})`); + if (state.status !== 'completed') throw new Error(`Existing demo needs attention: ${state.status}`); + return; + } + await docker(config.campaign, config.env); +} + +if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) { + demoCommand(process.argv.slice(2)).catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/tools/stack-bench/appliance/dependency-volume.ts b/tools/stack-bench/appliance/dependency-volume.ts new file mode 100644 index 00000000000..e6f49e55a86 --- /dev/null +++ b/tools/stack-bench/appliance/dependency-volume.ts @@ -0,0 +1,165 @@ +#!/usr/bin/env node + +import { createHash } from 'node:crypto'; +import { + chmodSync, copyFileSync, existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, + renameSync, rmSync, writeFileSync, +} from 'node:fs'; +import { dirname, join, relative, resolve, sep } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parseArgs } from 'node:util'; + +const MARKER = '.stack-bench-release-deps.json'; + +export interface DependencyManifestFile { + path: string; + size: number; + mode: number; + sha256: string; +} + +export interface DependencyManifest { + schemaVersion: 1; + files: DependencyManifestFile[]; +} + +interface DependencyVerification { + manifestSha256: string; + files: number; +} + +interface DependencyInitialization extends DependencyVerification { + initialized: boolean; +} + +function sha256Bytes(bytes: string | NodeJS.ArrayBufferView): string { + return createHash('sha256').update(bytes).digest('hex'); +} + +function normalizedRelative(root: string, path: string): string { + const value = relative(root, path).split(sep).join('/'); + if (!value || value.startsWith('../') || value === '..') throw new Error(`path escapes dependency root: ${path}`); + return value; +} + +function walk(root: string, current = root): string[] { + const files: string[] = []; + for (const entry of readdirSync(current, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) { + const path = join(current, entry.name); + if (entry.isSymbolicLink()) throw new Error(`dependency tree cannot contain symlinks: ${normalizedRelative(root, path)}`); + if (entry.isDirectory()) files.push(...walk(root, path)); + else if (entry.isFile()) files.push(path); + else throw new Error(`dependency tree contains unsupported entry: ${normalizedRelative(root, path)}`); + } + return files; +} + +export function createDependencyManifest(root: string): DependencyManifest { + const absolute = resolve(root); + if (!existsSync(absolute) || !lstatSync(absolute).isDirectory()) { + throw new Error(`dependency source is not a directory: ${absolute}`); + } + const files = walk(absolute).map(path => { + const bytes = readFileSync(path); + return { path: normalizedRelative(absolute, path), size: bytes.length, + mode: lstatSync(path).mode & 0o777, sha256: sha256Bytes(bytes) }; + }); + if (!files.length) throw new Error('dependency source is empty'); + return { schemaVersion: 1, files }; +} + +export function manifestSha256(manifest: DependencyManifest): string { + return sha256Bytes(`${JSON.stringify(manifest)}\n`); +} + +export function verifyDependencyTree(root: string, manifest: DependencyManifest, + { allowMarker = false }: { allowMarker?: boolean } = {}): DependencyVerification { + if (!manifest || manifest.schemaVersion !== 1 || !Array.isArray(manifest.files) || !manifest.files.length) { + throw new Error('dependency manifest is invalid'); + } + const absolute = resolve(root); + const actual = createDependencyManifest(absolute); + if (allowMarker) actual.files = actual.files.filter(file => file.path !== MARKER); + if (JSON.stringify(actual.files) !== JSON.stringify(manifest.files)) { + throw new Error(`dependency tree does not match manifest ${manifestSha256(manifest)}`); + } + return { manifestSha256: manifestSha256(manifest), files: manifest.files.length }; +} + +export function initializeDependencyVolume({ source, target, manifest }: + { source: string; target: string; manifest: DependencyManifest }): DependencyInitialization { + const sourceRoot = resolve(source); + const targetRoot = resolve(target); + const verified = verifyDependencyTree(sourceRoot, manifest); + mkdirSync(targetRoot, { recursive: true, mode: 0o755 }); + const markerPath = join(targetRoot, MARKER); + const existing = readdirSync(targetRoot); + if (existing.length) { + if (!existsSync(markerPath)) throw new Error('dependency volume is non-empty but has no release marker'); + const marker = JSON.parse(readFileSync(markerPath, 'utf8')); + if (marker.schemaVersion !== 1 || marker.manifestSha256 !== verified.manifestSha256) { + throw new Error('dependency volume belongs to a different release'); + } + verifyDependencyTree(targetRoot, manifest, { allowMarker: true }); + return { ...verified, initialized: false }; + } + + const staging = join(targetRoot, `.staging-${process.pid}`); + mkdirSync(staging, { mode: 0o700 }); + try { + for (const file of manifest.files) { + const from = join(sourceRoot, ...file.path.split('/')); + const to = join(staging, ...file.path.split('/')); + mkdirSync(dirname(to), { recursive: true }); + copyFileSync(from, to); + chmodSync(to, file.mode); + } + for (const entry of readdirSync(staging)) renameSync(join(staging, entry), join(targetRoot, entry)); + rmSync(staging, { recursive: true, force: true }); + writeFileSync(markerPath, `${JSON.stringify({ schemaVersion: 1, + manifestSha256: verified.manifestSha256 })}\n`, { flag: 'wx', mode: 0o444 }); + verifyDependencyTree(targetRoot, manifest, { allowMarker: true }); + return { ...verified, initialized: true }; + } catch (error) { + rmSync(staging, { recursive: true, force: true }); + throw error; + } +} + +function main(argv: string[]): void { + const { values, positionals } = parseArgs({ args: argv.slice(2), allowPositionals: true, + options: { + source: { type: 'string', default: '/opt/stack-bench-embedded-deps' }, + target: { type: 'string', default: '/opt/stack-bench-release-deps' }, + manifest: { type: 'string', default: '/opt/stack-bench/dependency-manifest.json' }, + out: { type: 'string' }, + } }); + const [command] = positionals; + const source = values.source; + const target = values.target; + const manifestPath = values.manifest; + if (command === 'manifest') { + const output = values.out; + if (!output) throw new Error('manifest requires --out'); + writeFileSync(resolve(output), `${JSON.stringify(createDependencyManifest(source), null, 2)}\n`, { flag: 'wx' }); + return; + } + const manifest: DependencyManifest = JSON.parse(readFileSync(resolve(manifestPath), 'utf8')); + if (command === 'init') { + process.stdout.write(`${JSON.stringify(initializeDependencyVolume({ source, target, manifest }))}\n`); + return; + } + if (command === 'verify') { + process.stdout.write(`${JSON.stringify(verifyDependencyTree(target, manifest, { allowMarker: true }))}\n`); + return; + } + throw new Error('usage: dependency-volume manifest|init|verify [options]'); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + try { main(process.argv); } + catch (error) { + console.error(`dependency-volume: ${error instanceof Error ? error.message : String(error)}`); + process.exit(2); + } +} diff --git a/tools/stack-bench/appliance/docker-compose.yaml b/tools/stack-bench/appliance/docker-compose.yaml new file mode 100644 index 00000000000..95cdb5e848e --- /dev/null +++ b/tools/stack-bench/appliance/docker-compose.yaml @@ -0,0 +1,168 @@ +name: stack-bench-appliance + +services: + deps-init: + image: ${STACK_BENCH_CONTROLLER_IMAGE:?set STACK_BENCH_CONTROLLER_IMAGE to the manifest digest reference} + platform: linux/amd64 + command: ["init-deps"] + read_only: true + cap_drop: ["ALL"] + security_opt: ["no-new-privileges:true"] + volumes: + - type: volume + source: release-deps + target: /opt/stack-bench-release-deps + tmpfs: + - /tmp:size=64m,mode=1777 + + controller: + image: ${STACK_BENCH_CONTROLLER_IMAGE:?set STACK_BENCH_CONTROLLER_IMAGE to the manifest digest reference} + platform: linux/amd64 + init: true + network_mode: host + working_dir: ${STACK_BENCH_STATE_ROOT:?run controller setup}/results + read_only: true + cap_drop: ["ALL"] + security_opt: ["no-new-privileges:true"] + depends_on: + deps-init: + condition: service_completed_successfully + npm-cache: + condition: service_healthy + environment: + STACK_BENCH_CONTROLLER_IMAGE: ${STACK_BENCH_CONTROLLER_IMAGE:?set STACK_BENCH_CONTROLLER_IMAGE to the manifest digest reference} + STACK_BENCH_BUILD_IMAGE: ${STACK_BENCH_BUILD_IMAGE:?set STACK_BENCH_BUILD_IMAGE} + STACK_BENCH_STATE_ROOT: ${STACK_BENCH_STATE_ROOT:?run controller setup} + STACK_BENCH_NPM_REGISTRY: http://127.0.0.1:4873/ + STACK_BENCH_IMAGE: ${STACK_BENCH_BUILD_IMAGE:?set STACK_BENCH_BUILD_IMAGE to the manifest digest reference} + STACK_BENCH_RELEASE_MANIFEST: ${STACK_BENCH_RELEASE_MANIFEST:-} + STACK_BENCH_APPLIANCE: "1" + STACK_BENCH_RUNNER_CAPACITY: ${STACK_BENCH_RUNNER_CAPACITY:-} + STACK_BENCH_COMPOSE_FILE: /opt/stack-bench/appliance/docker-compose.yaml + STACK_BENCH_WORK_DIR: ${STACK_BENCH_STATE_ROOT:?run controller setup}/work + STACK_BENCH_RESULTS_DIR: ${STACK_BENCH_STATE_ROOT:?run controller setup}/results + STACK_BENCH_SUPERVISOR_DIR: ${STACK_BENCH_STATE_ROOT:?run controller setup}/controller-home/supervisor + STACK_BENCH_RUNTIME_DIR: ${STACK_BENCH_STATE_ROOT:?run controller setup}/controller-home/runtime + STACK_BENCH_RESOURCE_LOCK_DIR: ${STACK_BENCH_STATE_ROOT:?run controller setup}/controller-home/resource-locks + STACK_BENCH_RELEASE_DEPS_VOLUME: ${STACK_BENCH_RELEASE_DEPS_VOLUME:-stack-bench-release-deps} + STACK_BENCH_LINUX_CLI: /opt/stack-bench-release-deps/spacetimedb-cli + STDB_PACKAGE: /opt/stack-bench-release-deps/bindings-typescript + SPACETIME_BIN: /opt/stack-bench-release-deps/spacetimedb-cli + STACK_BENCH_CREDENTIAL_PROFILES_FILE: ${STACK_BENCH_CREDENTIAL_PROFILES_FILE:-} + STACK_BENCH_AGENT_AUTH: ${STACK_BENCH_AGENT_AUTH:-subscription-token} + STACK_BENCH_CLAUDE_OAUTH_TOKEN_FILE: ${STACK_BENCH_CLAUDE_OAUTH_TOKEN_FILE:-${STACK_BENCH_STATE_ROOT:?run controller setup}/secrets/claude_subscription_token} + STACK_BENCH_ANTHROPIC_API_KEY_FILE: ${STACK_BENCH_ANTHROPIC_API_KEY_FILE:-} + STACK_BENCH_OPENROUTER_API_KEY_FILE: ${STACK_BENCH_OPENROUTER_API_KEY_FILE:-} + STACK_BENCH_OPENAI_API_KEY_FILE: ${STACK_BENCH_OPENAI_API_KEY_FILE:-} + STACK_BENCH_CODEX_AUTH_FILE: ${STACK_BENCH_CODEX_AUTH_FILE:-} + STACK_BENCH_XAI_API_KEY_FILE: ${STACK_BENCH_XAI_API_KEY_FILE:-} + STACK_BENCH_GROK_AUTH_FILE: ${STACK_BENCH_GROK_AUTH_FILE:-} + STACK_BENCH_GEMINI_API_KEY_FILE: ${STACK_BENCH_GEMINI_API_KEY_FILE:-} + STACK_BENCH_AGY_AUTH_FILE: ${STACK_BENCH_AGY_AUTH_FILE:-} + HOME: ${STACK_BENCH_STATE_ROOT:?run controller setup}/controller-home + volumes: + - type: bind + source: /var/run/docker.sock + target: /var/run/docker.sock + - type: volume + source: state + target: ${STACK_BENCH_STATE_ROOT:?run controller setup} + - type: volume + source: release-deps + target: /opt/stack-bench-release-deps + read_only: true + tmpfs: + - /tmp:size=1g,mode=1777 + command: ["--help"] + logging: &bounded-logs + driver: json-file + options: + max-size: "10m" + max-file: "3" + + worker: + restart: on-failure:3 + extends: + service: controller + profiles: ["worker"] + # SIGTERM stops admission and drains claimed campaigns. Explicit job cancellation stops a run. + stop_grace_period: 24h + command: ["job", "worker", "--host", "${STACK_BENCH_HOST_ID:-}", "--concurrency", "${STACK_BENCH_JOB_CONCURRENCY:-}"] + + dashboard: + extends: + service: controller + profiles: ["dashboard"] + network_mode: bridge + ports: + - "127.0.0.1:7331:7331" + command: ["dashboard", "--host", "0.0.0.0", "--port", "7331", "--allow-container-bind"] + # Pull-through npm registry cache. Coding containers install through it, so a + # package reaches the public registry once per appliance, not once per run. + npm-cache: + image: verdaccio/verdaccio:6@sha256:09b403888c8f73ba9336d7fb3464622f64ea905a64bc46a87c847387c536d4dd + platform: linux/amd64 + container_name: stack-bench-npm-cache + cpus: 1 + mem_limit: 2g + pids_limit: 128 + logging: *bounded-logs + cap_drop: ["ALL"] + security_opt: ["no-new-privileges:true"] + ports: ["127.0.0.1:4873:4873"] + configs: + - source: npm-cache-config + target: /verdaccio/conf/config.yaml + volumes: + - npmcache:/verdaccio/storage + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:4873/-/ping"] + interval: 5s + timeout: 5s + retries: 12 + +configs: + npm-cache-config: + content: | + storage: /verdaccio/storage/data + plugins: /verdaccio/plugins + web: + enable: false + auth: + htpasswd: + file: /verdaccio/storage/htpasswd + max_users: -1 + uplinks: + npmjs: + url: https://registry.npmjs.org/ + cache: true + timeout: 60s + maxage: 10m + max_fails: 4 + fail_timeout: 2m + packages: + '@*/*': + access: $$all + publish: nobody + unpublish: nobody + proxy: npmjs + '**': + access: $$all + publish: nobody + unpublish: nobody + proxy: npmjs + server: + keepAliveTimeout: 60 + log: + type: stdout + format: pretty + level: warn + +volumes: + release-deps: + name: ${STACK_BENCH_RELEASE_DEPS_VOLUME:-stack-bench-release-deps} + state: + external: true + name: stack-bench-state + npmcache: + name: stack-bench-appliance-npmcache diff --git a/tools/stack-bench/appliance/operator.env.example b/tools/stack-bench/appliance/operator.env.example new file mode 100644 index 00000000000..e99dd6f9e37 --- /dev/null +++ b/tools/stack-bench/appliance/operator.env.example @@ -0,0 +1,46 @@ +# Prefer the controller setup command. It prints this file with the Docker +# volume mountpoint and immutable local image IDs already resolved. +STACK_BENCH_STATE_ROOT=/var/lib/docker/volumes/stack-bench-state/_data + +# Subscription billing is the default. Generate a dedicated long-lived Claude +# setup token, write only the token to this mode-0600 file, and never commit it. +STACK_BENCH_AGENT_AUTH=subscription-token +STACK_BENCH_CLAUDE_OAUTH_TOKEN_FILE=${STACK_BENCH_STATE_ROOT}/secrets/claude_subscription_token + +# To bill through an API key instead, set the mode to api-key and provide an +# absolute path below STACK_BENCH_STATE_ROOT containing only that key. +# STACK_BENCH_ANTHROPIC_API_KEY_FILE=${STACK_BENCH_STATE_ROOT}/secrets/anthropic_api_key + +# For a codex agent, select openai-api-key or openai-account explicitly. +# Account mode uses an unexpired Codex login access token, with no automatic refresh. +# STACK_BENCH_AGENT_AUTH=openai-api-key +# STACK_BENCH_OPENAI_API_KEY_FILE=${STACK_BENCH_STATE_ROOT}/secrets/openai_api_key +# STACK_BENCH_AGENT_AUTH=openai-account +# STACK_BENCH_CODEX_AUTH_FILE=${STACK_BENCH_STATE_ROOT}/secrets/codex_auth + +# For the grok-build agent, select xai-api-key or grok-account explicitly. +# Account mode uses a SuperGrok sign-in from `grok login --device-auth`, stored with set-secret grok_auth. +# The controller refreshes it before an attempt when it has too little time left. +# STACK_BENCH_AGENT_AUTH=grok-account +# STACK_BENCH_GROK_AUTH_FILE=${STACK_BENCH_STATE_ROOT}/secrets/grok_auth + +# For the antigravity agent, store a Gemini API key with set-secret gemini_api_key. +# STACK_BENCH_AGENT_AUTH=gemini-api-key +# STACK_BENCH_GEMINI_API_KEY_FILE=${STACK_BENCH_STATE_ROOT}/secrets/gemini_api_key +# Account mode uses a Google account sign-in from agy, stored with set-secret agy_auth. +# STACK_BENCH_AGENT_AUTH=agy-account +# STACK_BENCH_AGY_AUTH_FILE=${STACK_BENCH_STATE_ROOT}/secrets/agy_auth + +# For the openrouter adapter, store the key with set-secret openrouter_api_key. +# Freeze model, providerRoute, and maxOutputTokens in each campaign agent selection. +# STACK_BENCH_AGENT_AUTH=openrouter-api-key +# STACK_BENCH_OPENROUTER_API_KEY_FILE=${STACK_BENCH_STATE_ROOT}/secrets/openrouter_api_key + +# Setup emits immutable local sha256: values. Distributed releases use +# registry references ending in @sha256:<64 hex chars>. +STACK_BENCH_CONTROLLER_IMAGE=registry.example/stack-bench-controller@sha256:replace-with-release-digest +STACK_BENCH_BUILD_IMAGE=registry.example/stack-bench-build@sha256:replace-with-release-digest + +# Optional for internal campaigns and required for a distributed release +# campaign. The file must be below the appliance state root. +STACK_BENCH_RELEASE_MANIFEST=${STACK_BENCH_STATE_ROOT}/release/release.json diff --git a/tools/stack-bench/appliance/state-volume.ts b/tools/stack-bench/appliance/state-volume.ts new file mode 100644 index 00000000000..190916a5f48 --- /dev/null +++ b/tools/stack-bench/appliance/state-volume.ts @@ -0,0 +1,145 @@ +import { execFileSync } from 'node:child_process'; +import { chmodSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { readGrokLogin } from '../src/agents/grok-login.js'; +import { readAgyLogin } from '../src/agents/agy-login.js'; +import { resolveContainerImage } from '../src/runtime/container-image.js'; +import { DATABASE_IMAGES } from '../src/stacks/database-containers.js'; +import { stackReleaseImages } from '../src/stacks/stack-identities.js'; + +export const STATE_VOLUME = 'stack-bench-state'; +type Docker = (args: readonly string[]) => string; +const docker: Docker = args => execFileSync('docker', [...args], { + encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: args[0] === 'pull' ? 300_000 : 60_000, +}); + +/** The controller and Docker daemon see one native Linux path on every host OS. */ +export function prepareStateVolume(env: NodeJS.ProcessEnv = process.env, run: Docker = docker): string { + if (run(['version', '--format', '{{.Server.Os}}']).trim() !== 'linux') { + throw new Error('Stack Bench requires a Docker daemon running Linux containers'); + } + const inspect = (_command: string, args: readonly string[]) => run(args); + const controller = resolveContainerImage(env.STACK_BENCH_CONTROLLER_IMAGE + ?? 'stack-bench-controller:local', inspect).id; + const build = resolveContainerImage(env.STACK_BENCH_BUILD_IMAGE + ?? 'stack-bench-build:local', inspect).id; + for (const reference of [...Object.values(DATABASE_IMAGES), ...stackReleaseImages().map(image => image.reference)]) { + try { resolveContainerImage(reference, inspect); } + catch { + run(['pull', '--platform', 'linux/amd64', reference]); + resolveContainerImage(reference, inspect); + } + } + run(['volume', 'create', STATE_VOLUME]); + const root = run(['volume', 'inspect', '--format', '{{.Mountpoint}}', STATE_VOLUME]).trim(); + if (!/^\/[A-Za-z0-9_./-]+$/.test(root) || root.split('/').includes('..')) { + throw new Error('Docker state volume has an invalid Linux mountpoint'); + } + run(['run', '--rm', '--platform', 'linux/amd64', '--network', 'none', '--mount', + `type=volume,source=${STATE_VOLUME},target=${root}`, '--entrypoint', 'node', controller, '-e', + 'const fs=require("node:fs"),p=require("node:path");' + + 'const root=process.argv[1];' + + 'for(const name of ["work","results/plans","results/run-presets","secrets","controller-home"])' + + 'fs.mkdirSync(p.join(root,name),{recursive:true,mode:0o700});' + + 'for(const [source,name] of [["campaign.example.json","reference-check.json"],["campaign.ecommerce-progression-reference.json","ecommerce-progression.json"],["campaign.demo.json","demo.json"]]) {' + + 'const target=p.join(root,"results/plans",name);if(!fs.existsSync(target))' + + 'fs.copyFileSync(p.join("/opt/stack-bench/appliance",source),target,fs.constants.COPYFILE_EXCL);}' + + 'const demo=p.join(root,"results/plans/paid-l1.json");if(!fs.existsSync(demo)){' + + 'const plan=JSON.parse(fs.readFileSync("/opt/stack-bench/appliance/campaign.paid-l1.json","utf8"));' + + 'plan.runtime.controllerImage=process.argv[2];plan.runtime.buildImage=process.argv[3];plan.state="frozen";' + + 'fs.writeFileSync(demo,JSON.stringify(plan,null,2)+"\\n",{flag:"wx",mode:0o600});}' + + 'const paid=JSON.parse(fs.readFileSync("/opt/stack-bench/appliance/campaign.paid-l1-l3.json","utf8"));' + + 'for(const [source,id,title] of [["campaign.paid-l1-l3.json","ecommerce-sequential","Ecommerce — sequential levels"],' + + '["campaign.ecommerce-progression-reference.json","ecommerce-progressive","Ecommerce — progressive features"],' + + '["campaign.ecommerce-progression-reference.json","ecommerce-progressive-l3","Ecommerce — four stacks, progressive L1–L3"],' + + '["campaign.ecommerce-progression-reference.json","ecommerce-single-build","Ecommerce — single build"]]){' + + 'const target=p.join(root,"results/run-presets",id+".json");if(fs.existsSync(target))continue;' + + 'const d=JSON.parse(fs.readFileSync(p.join("/opt/stack-bench/appliance",source),"utf8"));' + + 'd.id=id;d.title=title;d.state="frozen";d.agents=paid.agents;d.pricing=paid.pricing;' + + 'if(id==="ecommerce-single-build")d.mode.workSelection="all-at-once";' + + 'if(id==="ecommerce-progressive-l3"){d.levels=d.levels.filter(n=>n<=3);d.selection.levels=d.selection.levels.filter(s=>s.level<=3);d.stacks.push({id:"convex",adapterVersion:"1.0.0"});}' + + 'd.runtime.controllerImage=process.argv[2];d.runtime.buildImage=process.argv[3];' + + 'd.budgets=paid.budgets;d.repair.budget={total:0};d.parallelism=d.stacks.length;' + + 'if(id==="ecommerce-progressive"||id==="ecommerce-progressive-l3"){d.budgets={attemptTimeoutMinutes:240,maxCostUsdPerAttempt:50};d.mode.retainPriorContracts=true;d.mode.unchangedFailureLimit=7;}' + + 'd.conditions=["neutral","neutral-no-sdk","neutral-dev","neutral-dev-no-sdk"].map(g=>({...d.conditions[0],id:g,guidanceProfile:g}));' + + 'fs.writeFileSync(target,JSON.stringify(d,null,2)+"\\n",{flag:"wx",mode:0o600});}', + root, controller, build]); + return [ + `STACK_BENCH_STATE_ROOT=${root}`, + `STACK_BENCH_CONTROLLER_IMAGE=${controller}`, + `STACK_BENCH_BUILD_IMAGE=${build}`, + 'STACK_BENCH_RUNNER_CAPACITY=dynamic', + 'STACK_BENCH_AGENT_AUTH=subscription-token', + `STACK_BENCH_CLAUDE_OAUTH_TOKEN_FILE=${root}/secrets/claude_subscription_token`, + `STACK_BENCH_ANTHROPIC_API_KEY_FILE=${root}/secrets/anthropic_api_key`, + `STACK_BENCH_OPENROUTER_API_KEY_FILE=${root}/secrets/openrouter_api_key`, + `STACK_BENCH_OPENAI_API_KEY_FILE=${root}/secrets/openai_api_key`, + `STACK_BENCH_CODEX_AUTH_FILE=${root}/secrets/codex_auth`, + `STACK_BENCH_GROK_AUTH_FILE=${root}/secrets/grok_auth`, + `STACK_BENCH_GEMINI_API_KEY_FILE=${root}/secrets/gemini_api_key`, + `STACK_BENCH_AGY_AUTH_FILE=${root}/secrets/agy_auth`, + 'STACK_BENCH_RELEASE_MANIFEST=', + '', + ].join('\n'); +} + +const SECRET_NAMES = ['claude_subscription_token', 'anthropic_api_key', 'openai_api_key', 'openrouter_api_key', + 'codex_auth', 'xai_api_key', 'grok_auth', 'zai_api_key', 'deepseek_api_key', 'moonshot_api_key', 'gemini_api_key', + 'agy_auth']; +function checkSecretName(name: string | undefined): void { + if (!SECRET_NAMES.includes(name ?? '')) throw new Error(`secret name must be ${SECRET_NAMES.join(', ')}`); +} + +export function writeStateSecret(name: string | undefined, input: string, + root = '/state'): void { + checkSecretName(name); + let value = input.trim(); + if (name === 'codex_auth' || name === 'grok_auth' || name === 'agy_auth') { + try { value = JSON.stringify(JSON.parse(value)); } + catch { throw new Error(`${name} must be valid JSON from the agent's account login`); } + if (name === 'grok_auth') readGrokLogin(value); + if (name === 'agy_auth') readAgyLogin(value); + } + if (!value || /[\r\n]/.test(value)) { + throw new Error('secret must be one non-empty line'); + } + mkdirSync(join(root, 'secrets'), { recursive: true, mode: 0o700 }); + chmodSync(join(root, 'secrets'), 0o700); + writeFileSync(join(root, 'secrets', name!), `${value}\n`, { mode: 0o600 }); + chmodSync(join(root, 'secrets', name!), 0o600); +} + +// At a terminal, ask for the one line and do not echo it; piped input is read whole. +function readSecretInput(name: string): Promise { + if (!process.stdin.isTTY) return Promise.resolve(readFileSync(0, 'utf8')); + process.stderr.write(`Paste the ${name} value and press Enter (it is not shown): `); + process.stdin.setRawMode(true); + return new Promise((resolve, reject) => { + let value = ''; + const finish = (settle: () => void) => { + process.stdin.setRawMode(false); + process.stdin.off('data', read).pause(); + process.stderr.write('\n'); + settle(); + }; + const read = (chunk: Buffer) => { + for (const character of chunk.toString('utf8')) { + if (character === '\r' || character === '\n') return finish(() => resolve(value)); + if (character === '\u0003') return finish(() => reject(new Error('set-secret cancelled'))); + value = character === '\u007f' || character === '\b' ? value.slice(0, -1) : value + character; + } + }; + process.stdin.on('data', read).resume(); + }); +} + +export async function stateVolumeCommand(command: string, args: string[]): Promise { + if (command === 'setup') { + if (args.length) throw new Error('setup accepts no arguments; configure image references through the environment'); + process.stdout.write(prepareStateVolume()); + } else { + if (args.length !== 1) throw new Error('set-secret requires exactly one secret name, such as set-secret gemini_api_key'); + checkSecretName(args[0]); + writeStateSecret(args[0], await readSecretInput(args[0]!)); + } +} diff --git a/tools/stack-bench/backends/convex.md b/tools/stack-bench/backends/convex.md new file mode 100644 index 00000000000..ce1b8a97c1b --- /dev/null +++ b/tools/stack-bench/backends/convex.md @@ -0,0 +1,20 @@ +# Backend: Convex + +Use a React client and native Convex TypeScript functions. Put the functions and +schema in `convex/`. The application mutations described below are exported from +`convex/api.ts`. Use the same native functions for the visible controls. + +## Deployment + +The local Convex deployment is already running. The environment supplies +`CONVEX_SELF_HOSTED_URL`, `CONVEX_SELF_HOSTED_ADMIN_KEY`, and `VITE_CONVEX_URL`. +Use these exact values at startup. Do not embed or save them in source or connect +to a hosted project. Keep the deployment key out of the web client and user sessions. +The native HTTP-action origin is `http://127.0.0.1:`. + +Create `/app/start.sh`. From a clean source checkout, it must install dependencies, +deploy the functions with `npx convex dev --once --typecheck disable`, build the web +application, and start it on `` without changing source files. +Deploy functions and initialize empty data even with +`APP_WARM_START=1`; reuse current dependencies when possible. Keep existing data +and accounts during subsequent starts, upgrades, and repairs. Leave the application running. diff --git a/tools/stack-bench/backends/minimal/convex.md b/tools/stack-bench/backends/minimal/convex.md new file mode 100644 index 00000000000..9c015518cb4 --- /dev/null +++ b/tools/stack-bench/backends/minimal/convex.md @@ -0,0 +1,22 @@ +# Convex + +Use Convex for application data and backend functions. Choose the client libraries, +architecture, and project structure. + +## Connection + +The local Convex service is already running. Read its URL from +`CONVEX_SELF_HOSTED_URL` and its deployment key from `CONVEX_SELF_HOSTED_ADMIN_KEY`. +The web client uses `VITE_CONVEX_URL`. These values are supplied in the process +environment and can change between launches. Do not embed or save them in source. +Keep the deployment key on the server; it is not an end-user credential. + +Use this deployment. Do not start another Convex server or connect to a hosted project. +The native HTTP-action origin is `http://127.0.0.1:`. +Serve the complete application on ``. + +Create `/app/start.sh`. From a clean source checkout, it must install dependencies, +deploy the Convex functions, build the web application, and start the complete application. +The script must not change source files. Deploy the functions and initialize empty application data even when +`APP_WARM_START=1`; that flag only permits reusing current dependencies. Preserve +existing application data and accounts. Leave the application running when work is complete. diff --git a/tools/stack-bench/backends/minimal/mongodb.md b/tools/stack-bench/backends/minimal/mongodb.md new file mode 100644 index 00000000000..a4a7e8d3c8a --- /dev/null +++ b/tools/stack-bench/backends/minimal/mongodb.md @@ -0,0 +1,21 @@ +# MongoDB + +Use MongoDB for the application data. Choose the libraries, architecture, and +project structure. + +## Connection + +| Setting | Value | +|---|---| +| `DATABASE_URL` | `` | +| Web application | `http://localhost:` | + +The MongoDB service is already running as a single-node replica set. Use the exact `DATABASE_URL`. Do not +start another MongoDB server, connect to another instance, or create another +database. Serve the complete application on ``. +Read `DATABASE_URL` from the process environment at startup. It can change between +launches; do not embed it in source or override it with a saved value. +Create `/app/start.sh`. From a clean source checkout, it must install +dependencies, build the complete application, and start it on ``. +The script must not change source files. Leave the application running when the +work is complete. diff --git a/tools/stack-bench/backends/minimal/postgres.md b/tools/stack-bench/backends/minimal/postgres.md new file mode 100644 index 00000000000..1a078206cf4 --- /dev/null +++ b/tools/stack-bench/backends/minimal/postgres.md @@ -0,0 +1,21 @@ +# PostgreSQL + +Use PostgreSQL for the application data. Choose the libraries, architecture, +and project structure. + +## Connection + +| Setting | Value | +|---|---| +| `DATABASE_URL` | `` | +| Web application | `http://localhost:` | + +The PostgreSQL service is already running. Use the exact `DATABASE_URL`. Do not +start another PostgreSQL server, connect to another instance, or create another +database. Serve the complete application on ``. +Read `DATABASE_URL` from the process environment at startup. It can change between +launches; do not embed it in source or override it with a saved value. +Create `/app/start.sh`. From a clean source checkout, it must install +dependencies, build the complete application, and start it on ``. +The script must not change source files. Leave the application running when the +work is complete. diff --git a/tools/stack-bench/backends/minimal/spacetime.md b/tools/stack-bench/backends/minimal/spacetime.md new file mode 100644 index 00000000000..508dbe353b5 --- /dev/null +++ b/tools/stack-bench/backends/minimal/spacetime.md @@ -0,0 +1,28 @@ +# SpacetimeDB + +Use SpacetimeDB for the application data. Put the TypeScript module in the +required directory below. Choose the schema, libraries, architecture, and the +rest of the project structure. + +## Connection + +Use the connection settings below. + +| Setting | Value | +|---|---| +| Server URI | `` | +| Module name | `` | +| SpacetimeDB CLI | `` | +| TypeScript SDK package | `` | +| Module source directory | `/app/backend/spacetimedb` | +| Web application | `http://localhost:` | + +Publish only the named module to the exact server URI. Local publish and +development commands must use `--yes`. Do not pipe confirmation input, publish +anonymously, or use the hosted service. Create `/app/start.sh`. From a clean +source checkout, it must install dependencies, build the complete application, +and start it on ``. The script must not change source files. Leave +the application running when the work is complete. + +The included TypeScript server and client skills provide SDK guidance. +CLI `--help` is available for command syntax. diff --git a/tools/stack-bench/backends/minimal/supabase.md b/tools/stack-bench/backends/minimal/supabase.md new file mode 100644 index 00000000000..4db863c8b11 --- /dev/null +++ b/tools/stack-bench/backends/minimal/supabase.md @@ -0,0 +1,34 @@ +# Supabase + +Use Supabase for application data, accounts, and backend functions. Choose the client +libraries, architecture, and project structure. + +## Connection + +The local Supabase service is already running with Auth, the data API (`/rest/v1`), +Realtime, Storage, and Edge Functions. The browser, this environment, and Edge Functions +all reach it at `SUPABASE_URL`. The web client uses `VITE_SUPABASE_URL` and +`VITE_SUPABASE_ANON_KEY`; `SUPABASE_ANON_KEY` holds the same public key. +`SUPABASE_SERVICE_ROLE_KEY` bypasses row-level security, and `SUPABASE_DB_URL` connects +to PostgreSQL as the `postgres` role. Keep both on the server; they are not end-user +credentials. These values are supplied in the process environment and can change +between launches. Do not embed or save them in source. + +Use this service. Do not start another Supabase server or connect to a hosted project. +Studio, the management API, email delivery, and OAuth providers are not available. +Auth identifies accounts by email address, compared without regard to case. Email +confirmation is off. Passwords must be at least 6 characters and at most 72 bytes. + +Edge Functions in `supabase/functions//index.ts` are served at +`$SUPABASE_URL/functions/v1/` and pick up file changes on the next request. They +run on Deno, accept `npm:` and `jsr:` imports, and receive `SUPABASE_URL`, +`SUPABASE_ANON_KEY`, `SUPABASE_SERVICE_ROLE_KEY`, and `SUPABASE_DB_URL`. +`psql` and Docker are not available; run SQL from Node through `SUPABASE_DB_URL`. +Serve the complete application on ``. + +Create `/app/start.sh`. From a clean source checkout, it must install dependencies, +apply the database schema, build the web application, and start the complete application. +The script must not change source files. Apply the schema and initialize empty application +data on every start, even when `APP_WARM_START=1`; that flag only permits reusing current +dependencies. Preserve existing application data and accounts. Leave the application +running when work is complete. diff --git a/tools/stack-bench/backends/model-free-stub.md b/tools/stack-bench/backends/model-free-stub.md new file mode 100644 index 00000000000..7a83d0e4bc2 --- /dev/null +++ b/tools/stack-bench/backends/model-free-stub.md @@ -0,0 +1,4 @@ +# Model-free service + +Use the supplied service. Leave the app running on the assigned client port +when the work is complete. diff --git a/tools/stack-bench/backends/mongodb.md b/tools/stack-bench/backends/mongodb.md new file mode 100644 index 00000000000..48ae7319b5f --- /dev/null +++ b/tools/stack-bench/backends/mongodb.md @@ -0,0 +1,50 @@ +# Backend: MongoDB + +An Express API server with Socket.io for live updates, Mongoose over MongoDB, +and a React client. + +## Layout + +``` +/ + server/ + package.json express, socket.io, mongoose, dotenv, tsx + .env DATABASE_URL and PORT + src/models.ts Mongoose schemas and models + src/index.ts Express routes, Socket.io handlers + client/ + package.json react, react-dom, vite, socket.io-client + vite.config.ts server.port , proxy /api and /socket.io to + index.html + src/main.tsx + src/App.tsx +``` + +## Deploy + +```bash +cd server && npm install && npm run dev # on +cd client && npm install && npm run dev # on +``` + +Mongoose creates collections on first write; there is no migration step. + +The server prints to the terminal running `npm run dev`; it restarts on save, +so a code change is live without redeploying. + +Keep existing application data when you change the schema. Do not drop +collections during upgrades or repairs. + +## Configuration + +| Setting | Value | +|---|---| +| `DATABASE_URL` | `` | +| API server port | `` | +| Client dev server | `` | + +Use this exact `DATABASE_URL`. Do not point at another MongoDB instance. +Read `DATABASE_URL` from the process environment at startup. It can change between +launches; do not embed it in source or override it with a saved value. + +The supplied database is a single-node replica set. diff --git a/tools/stack-bench/backends/postgres.md b/tools/stack-bench/backends/postgres.md new file mode 100644 index 00000000000..b605b9aeaf7 --- /dev/null +++ b/tools/stack-bench/backends/postgres.md @@ -0,0 +1,50 @@ +# Backend: PostgreSQL + +An Express API server with Socket.io for live updates, Drizzle ORM over +PostgreSQL, and a React client. + +## Layout + +``` +/ + server/ + package.json express, socket.io, drizzle-orm, pg, dotenv, tsx + .env DATABASE_URL and PORT + drizzle.config.ts + src/schema.ts Drizzle table definitions + src/index.ts Express routes, Socket.io handlers + client/ + package.json react, react-dom, vite, socket.io-client + vite.config.ts server.port , proxy /api and /socket.io to + index.html + src/main.tsx + src/App.tsx +``` + +## Deploy + +```bash +cd server && npm install && npx drizzle-kit push && npm run dev # on +cd client && npm install && npm run dev # on +``` + +Re-run `npx drizzle-kit push` after any schema change. + +The server prints to the terminal running `npm run dev`; it restarts on save, +so a code change is live without redeploying. + +Keep existing application data when you change the schema. Do not drop or +recreate tables during upgrades or repairs. + +## Configuration + +| Setting | Value | +|---|---| +| `DATABASE_URL` | `` | +| API server port | `` | +| Client dev server | `` | + +Use this exact `DATABASE_URL`. Do not point at another PostgreSQL instance and do +not create databases outside it. +Read `DATABASE_URL` from the process environment at startup. It can change between +launches; do not embed it in source or override it with a saved value. diff --git a/tools/stack-bench/backends/spacetime.md b/tools/stack-bench/backends/spacetime.md new file mode 100644 index 00000000000..250587f88fe --- /dev/null +++ b/tools/stack-bench/backends/spacetime.md @@ -0,0 +1,80 @@ +# Backend: SpacetimeDB + +The database runs your server logic. There is no separate API server and no ORM: +tables and reducers are a WASM module you publish, and the client subscribes to +tables and calls reducers over a live connection. + +## Layout + +``` +/ + backend/spacetimedb/ + package.json { "type": "module", dependencies: { "spacetimedb": "" }, + devDependencies: { "typescript": "~5.6.2" } } ← required; the build runs tsc from node_modules + tsconfig.json + src/schema.ts tables and indexes + src/index.ts reducers and lifecycle hooks + client/ + package.json react, react-dom, vite, and "spacetimedb": "" + vite.config.ts server.port must be + index.html + src/config.ts MODULE_NAME and SPACETIMEDB_URI + src/main.tsx React entry + src/App.tsx + src/module_bindings/ generated; never edit by hand +``` + +## Deploy + +Publish the module, then regenerate the client bindings from it: + +```bash + publish --module-path backend/spacetimedb -s --yes + generate --lang typescript --out-dir client/src/module_bindings --module-path backend/spacetimedb +``` + +**While iterating, run development mode instead of republishing by hand.** It +watches the module and automatically rebuilds, publishes, and regenerates the +client bindings on every save: + +```bash + dev --module-path backend/spacetimedb -s --yes +``` + +Leave it running in the background while you work. The manual commands below +are for one-off publishes and for the first deploy. + +Republish after any server change, and regenerate after any schema change. + +Keep existing application data when you change the schema. + +Always use `--yes` for local publish and development commands. It selects the +CLI's non-interactive authentication flow for the target server. Do not pipe +`y` into the command and do not publish anonymously. Use the same local +identity for every publish to the named module. + +Then start the client: + +```bash +cd client && npm install && npm run dev +``` + +` logs -s ` shows module output, including reducer errors. + +To inspect stored data while debugging: + +```bash + sql "SELECT * FROM item LIMIT 5" -s +``` + +## Configuration + +| Setting | Value | +|---|---| +| Server URI | `` | +| Module name | `` | +| Client dev server | `` | + +The SDK reference for writing modules and clients is in the skill documents +included with these instructions. Follow them for API specifics: import paths, +type builders, accessors and context typing. diff --git a/tools/stack-bench/backends/supabase.md b/tools/stack-bench/backends/supabase.md new file mode 100644 index 00000000000..17358bce56b --- /dev/null +++ b/tools/stack-bench/backends/supabase.md @@ -0,0 +1,37 @@ +# Backend: Supabase + +Use a React client with `@supabase/supabase-js`. Keep the database schema and +functions in SQL files under `supabase/migrations/`. The application operations +described below are PostgreSQL functions in the `public` schema. Use the same +functions for the visible controls. Put any Edge Functions in +`supabase/functions//index.ts`. + +## Deployment + +The local Supabase service is already running with Auth, the data API (`/rest/v1`), +Realtime, Storage, and Edge Functions. The environment supplies `SUPABASE_URL`, +`SUPABASE_ANON_KEY`, `SUPABASE_SERVICE_ROLE_KEY`, `SUPABASE_DB_URL`, +`VITE_SUPABASE_URL`, and `VITE_SUPABASE_ANON_KEY`. Use these exact values at startup. +Do not embed or save them in source or connect to a hosted project. The web client, +this environment, and Edge Functions all reach it at `SUPABASE_URL`. +Keep `SUPABASE_SERVICE_ROLE_KEY` and `SUPABASE_DB_URL` out of the web client and user +sessions; the service role key bypasses row-level security. + +Auth identifies accounts by email address, compared without regard to case. Email +confirmation is off. Passwords must be at least 6 characters and at most 72 bytes. Studio, the management API, +email delivery, and OAuth providers are not available. + +Edge Functions are served at `$SUPABASE_URL/functions/v1/` and pick up file +changes on the next request; there is no deploy step. They run on Deno, accept `npm:` +and `jsr:` imports, and receive `SUPABASE_URL`, `SUPABASE_ANON_KEY`, +`SUPABASE_SERVICE_ROLE_KEY`, and `SUPABASE_DB_URL`. + +`psql` and Docker are not available. Apply migrations with the `pg` package and +`SUPABASE_DB_URL`, which connects as the `postgres` role. + +Create `/app/start.sh`. From a clean source checkout, it must install dependencies, +apply the migrations, build the web application, and start it on `` without +changing source files. Apply migrations and initialize empty data on every start, even +with `APP_WARM_START=1`, so they must be safe to run again; reuse current dependencies +when possible. Keep existing data and accounts during subsequent starts, upgrades, and +repairs. Leave the application running. diff --git a/tools/stack-bench/backends/workflows/spacetime-dev.md b/tools/stack-bench/backends/workflows/spacetime-dev.md new file mode 100644 index 00000000000..7c281dd2dac --- /dev/null +++ b/tools/stack-bench/backends/workflows/spacetime-dev.md @@ -0,0 +1,44 @@ +--- +name: spacetime-dev +description: Use the SpacetimeDB development watcher while implementing an application. +--- + +# Development workflow + +Use `spacetime dev` while implementing and repairing the application. Keep one +watcher running for the assigned database. It builds module changes, publishes +them, and generates client bindings. Use the supplied CLI, server URI, database +name, and module directory. See the CLI skill for command syntax. + +Use the supplied server URL directly; do not register a server nickname or +change the CLI login. Use a project configuration with both publish and generate +targets. For example, in `/app/spacetime.json`, replacing the server, database, +and client directory with the supplied settings and your actual paths: + +```json +{ + "server": "http://SERVER:PORT", + "database": "DATABASE", + "module-path": "backend/spacetimedb", + "generate": [ + { "language": "typescript", "out-dir": "frontend/src/module_bindings" } + ] +} +``` + +From `/app`, run the supplied CLI with `dev --yes --delete-data=never +--server-only`. Start the web client separately. With these configured targets, +omit `--module-path`, `--project-path`, and `--module-bindings-path` flags. +Paths in this example are relative to the project directory. +Do not run competing publish commands or watchers. If bindings generation is skipped, +correct the generate target before continuing. + +Wait for the initial publish and bindings to succeed before opening the app. +After a module edit, check that the watcher published it successfully before +checking app behavior. Fix watcher errors; do not assume that the live module is current. +Restart the watcher if it exited or its configuration changed. + +Keep `/app/start.sh` able to build and start the complete application from a +clean source checkout without this development session. Stop the watcher before +checking that startup path. One-shot commands remain appropriate for that script +and for diagnosing a watcher failure. diff --git a/tools/stack-bench/backends/workflows/spacetime-managed-dev.md b/tools/stack-bench/backends/workflows/spacetime-managed-dev.md new file mode 100644 index 00000000000..3d51eb02bdd --- /dev/null +++ b/tools/stack-bench/backends/workflows/spacetime-managed-dev.md @@ -0,0 +1,24 @@ +--- +name: spacetime-managed-dev +description: Use the supplied command to manage the SpacetimeDB development watcher. +--- + +# Development workflow + +Development watcher support is available at `/deps/spacetime-dev`. It has not +started yet. Create the module and `/app/spacetime.json` first. Configure the +supplied server URL and database name, your `module-path`, and TypeScript +`generate` targets with their `out-dir` paths. Keep these paths inside `/app`. +This helper supports one database per application. See the CLI skill for configuration syntax. + +Run `/deps/spacetime-dev start`. It starts one watcher with data deletion disabled. +Repeated calls report the existing watcher. Run `/deps/spacetime-dev status` to +check startup, and read the reported log for build or publish errors. +"Starting" does not mean the initial publish has completed. "Running" confirms +the initial publish and bindings; later edits can still fail, so check the log. +Start the frontend separately. Do not start another watcher or competing publisher. + +Use `/deps/spacetime-dev stop` before changing configuration or checking a clean +startup, then `start` again when needed. An exited watcher is not restarted +automatically. Keep `/app/start.sh` able to build and start the complete application +without this development session. Container cleanup stops the watcher. diff --git a/tools/stack-bench/commands/agent.ts b/tools/stack-bench/commands/agent.ts new file mode 100644 index 00000000000..d409f000311 --- /dev/null +++ b/tools/stack-bench/commands/agent.ts @@ -0,0 +1,1001 @@ +#!/usr/bin/env node + +import { randomUUID } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import type { ExecFileSyncOptionsWithStringEncoding } from 'node:child_process'; +import { readFileSync, writeFileSync, mkdirSync, existsSync, realpathSync, + openSync, readSync, closeSync, readdirSync } from 'node:fs'; +import { join, dirname, resolve, relative, isAbsolute, sep } from 'node:path'; +import { homedir } from 'node:os'; +import { fileURLToPath } from 'node:url'; +import { parseArgs as parseNodeArgs } from 'node:util'; +import { loadTrack, levelPrompt, appendix, suitesFor, dbName, moduleName, portsFor, + DEFAULT_TRACK, TRACK_MANIFEST_FILE } from '../src/composition/tracks.js'; +import type { Track, TrackDefinition } from '../src/composition/tracks.js'; +import type { BackendLease } from '../src/runtime/backend-lease.js'; +import { resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { parseGuidanceMode, resolveDefaultGuidanceForStack, type GuidanceMode, + type ResolvedGuidanceDocument, type ResolvedSkills } + from '../src/campaigns/condition-compiler.js'; +import type { RecipeBinding, RecipeRequest } from '../src/composition/recipe-release.js'; +import { createBoundRecipeTaskRequest, resolveBoundRecipeTaskRequest } from '../src/composition/recipe-selection.js'; +import { agentVisibleContractText, assertAgentVisibleText } + from '../src/composition/agent-visible-contract.js'; +import { DEFAULT_SPACETIME_SERVER_URI, leaseFromEnv } from '../src/runtime/backend-lease.js'; +import { CODING_CONTAINER_APP_ROOT, CODING_CONTAINER_BUG_REPORT_FILE, + CODING_CONTAINER_RELEASE_DEPS_ROOT, CODING_CONTAINER_SPACETIME_CLI, + CODING_CONTAINER_SPACETIME_PACKAGE } + from '../src/runtime/coding-container-policy.js'; +import { resolveContainerImage } from '../src/runtime/container-image.js'; +import { hashDirectory, sessionProvenance, sha256 } from '../src/evidence/provenance.js'; +import type { StackRunPorts } from '../src/stacks/stack-adapter-contract.js'; +import { STACK_ADAPTER_REGISTRY } from '../src/stacks/stack-adapters.js'; +import { stackApplicationInterface } from '../src/stacks/stack-identities.js'; +import { attemptDatabaseUrl } from '../src/stacks/hosted-database-identity.js'; +import { requireLeasedDatabase, requireLeasedSpacetime } + from '../src/stacks/backend-reset-guard.js'; +import { DEFAULT_BUILD_IMAGE } from '../src/composition/product-config.js'; +import { dockerMountArguments } from '../src/runtime/container-mount.js'; +import { normalizePromptText, readAgentSkillDocuments, selectAgentSkills } from '../src/agents/agent-materials.js'; +import { codingSessionFailure, DEFAULT_THROTTLE_MAX_WAIT_MS, providerSessionFailure, + runCodingSessionWithRetries, PROVIDER_CONTINUATION_MESSAGE } from '../src/agents/coding-session-retry.js'; +import { captureNativeContinuation } from '../src/agents/provider-native-continuation.js'; +import { campaignProviderContinuationContext, persistCampaignProviderInvocation, + waitForCampaignProviderContinuation } from '../src/campaigns/campaign-provider-continuation.js'; +import type { CodingSessionRetryResult } from '../src/agents/coding-session-retry.js'; +import { AGENT_PROCESS_TIMEOUT_MS } from '../src/agents/coding-session-timeouts.js'; +import { toolPolicyFromEnvironment, toolPolicySwitch } from '../src/agents/tool-policy.js'; +import { assertNewOrEmptyDirectory } from '../src/runtime/path-safety.js'; +import { resolveContainerAuth } from '../container/container-auth.js'; +import { CODING_PROVIDERS, parseCodingProvider } from '../container/coding-providers.js'; +import { validateProviderRoute, validateProviderOutputLimit } from '../src/agents/agent-adapter-contract.js'; +import { PRICING_UNIT, validatePricingAuthority } + from '../src/evidence/pricing-authority.js'; +import type { PricingAuthority } from '../src/evidence/pricing-authority.js'; + +import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js'; +const REPO = resolve(ROOT, '..', '..'); +const CONTROL_COMMAND_TIMEOUT_MS = 120_000; +const DEFAULT_CODING_INTERRUPTION_RETRIES = 2; + +type UnknownRecord = Record; +interface PromptMaterials { + skillsText?: string; + requirementText?: string; + contractText?: string; + startingCatalog?: string; +} + +type RecipeTaskRequest = Parameters[1] & { + recipe?: Exclude; +}; + +type AgentMode = 'build' | 'upgrade' | 'fix' | 'resume'; + +interface AgentArgs { + provider: keyof typeof CODING_PROVIDERS; + providerRoute?: string; + maxOutputTokens?: number; + mode: AgentMode; + backend: string; + app: string; + level: number; + runIndex: number; + model: string; + guidance: GuidanceMode; + productionQuality?: boolean; + track: string; + pricing: Readonly | null; + guidanceDocument?: ResolvedGuidanceDocument; + credentialAliases?: Readonly>; + recipe?: string; + recipeTask?: RecipeTaskRequest; + maxBudgetUsd?: number; + skills?: string[]; + skillIdentity?: ResolvedSkills; + printPrompt?: boolean; +} + +interface ThinkingVolume { + blocks: number; + signatureBytes: number; + bytesPerBlock: number; +} + +interface SessionUsage { + input_tokens?: number; + output_tokens?: number; + cache_creation_input_tokens?: number; + cache_read_input_tokens?: number; +} + +const isRecord = (value: unknown): value is UnknownRecord => + value !== null && typeof value === 'object' && !Array.isArray(value); + +const stringValue = (value: unknown): string | null => typeof value === 'string' ? value : null; + +function stringArray(value: string, option: string): string[] { + const parsed: unknown = JSON.parse(value); + if (!Array.isArray(parsed) || parsed.some(item => typeof item !== 'string')) { + throw new Error(`${option} must be an array of strings`); + } + return parsed; +} + +function sessionUsage(value: unknown): SessionUsage { + return isRecord(value) ? { + input_tokens: typeof value.input_tokens === 'number' ? value.input_tokens : undefined, + output_tokens: typeof value.output_tokens === 'number' ? value.output_tokens : undefined, + cache_creation_input_tokens: typeof value.cache_creation_input_tokens === 'number' + ? value.cache_creation_input_tokens : undefined, + cache_read_input_tokens: typeof value.cache_read_input_tokens === 'number' + ? value.cache_read_input_tokens : undefined, + } : {}; +} + +// Use only the benchmark-owned SpacetimeDB host. +const STDB_URI = process.env.STACK_BENCH_STDB_URI ?? DEFAULT_SPACETIME_SERVER_URI; + +// Test the CLI and SDK from this checkout. +const LOCAL_CLI = join(REPO, 'target', 'release', 'spacetimedb-cli.exe'); +const STDB_BIN = process.env.SPACETIME_BIN ?? (existsSync(LOCAL_CLI) ? LOCAL_CLI : 'spacetime'); +const LOCAL_PKG = process.env.STDB_PACKAGE ?? join(REPO, 'crates', 'bindings-typescript'); + +const fwd = (path: string): string => path.split('\\').join('/'); + +// Keep the provider's default thinking budget unless an experiment selects one +// explicitly. The run records observed reasoning volume so default changes are +// visible in the evidence. +const THINKING_TOKENS = process.env.STACK_BENCH_THINKING ?? null; + +const EFFORT = process.env.STACK_BENCH_EFFORT ?? 'high'; + +// A campaign sets the tool policy for every attempt; a run outside one gets the default. +const TOOLS = toolPolicyFromEnvironment(process.env); + +const IMAGE = process.env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE; + +// Containers reach host services through this address. App ports remain local. +export function hostServiceAddress(env: NodeJS.ProcessEnv = process.env): string { + return env.STACK_BENCH_HOST_ALIAS + ?? (env.STACK_BENCH_APPLIANCE === '1' ? '127.0.0.1' : 'host.docker.internal'); +} + +const HOST_ADDR = hostServiceAddress(); +const hostUrl = (url: string): string => url.replace(/127\.0\.0\.1|localhost/g, HOST_ADDR); + +const C_BIN = CODING_CONTAINER_SPACETIME_CLI; + +// The container requires the Linux CLI from this checkout. +const LINUX_CLI = process.env.STACK_BENCH_LINUX_CLI + ?? join(ROOT, 'container', 'bin', 'spacetimedb-cli'); + +// The provider CLI keeps one JSONL transcript per session under its project +// directory for the application path. +function transcriptFile(appDir: string, sessionId: string): string | null { + const store = join(homedir(), '.claude', 'projects'); + if (!existsSync(store)) return null; + const want = resolve(appDir).replace(/[\\/:]/g, '-').toLowerCase(); + const dir = readdirSync(store).find(d => { + const n = d.toLowerCase(); + return n === want || n === want.replace(/^-+/, ''); + }); + const file = dir && join(store, dir, `${sessionId}.jsonl`); + return file && existsSync(file) ? file : null; +} + +// The model ids the provider actually served. The requested name is an alias +// that can resolve to different snapshots over time; the transcript records +// what answered each request. +function transcriptModels(appDir: string, sessionIds: readonly (string | null | undefined)[]): string[] { + const models = new Set(); + for (const sessionId of new Set(sessionIds.filter((id): id is string => Boolean(id)))) { + try { + const file = transcriptFile(appDir, sessionId); + if (!file) continue; + for (const line of readFileSync(file, 'utf8').split('\n')) { + if (!line.includes('"model"')) continue; + let record: unknown; + try { record = JSON.parse(line); } catch { continue; } + if (!isRecord(record) || !isRecord(record.message)) continue; + const model = stringValue(record.message.model); + if (model) models.add(model); + } + } catch { /* an unreadable transcript leaves the list shorter, never wrong */ } + } + return [...models].sort(); +} + +// The transcript exposes reasoning blocks and signature bytes, not reasoning tokens. +function thinkingVolume(appDir: string, sessionId: string | null | undefined): ThinkingVolume | null { + if (!sessionId) return null; + try { + const file = transcriptFile(appDir, sessionId); + if (!file) return null; + + let blocks = 0, bytes = 0; + for (const line of readFileSync(file, 'utf8').split('\n')) { + if (!line.includes('"thinking"')) continue; // cheap filter before parsing + let record: unknown; + try { record = JSON.parse(line); } catch { continue; } + if (!isRecord(record) || !isRecord(record.message) + || !Array.isArray(record.message.content)) continue; + for (const content of record.message.content) { + if (!isRecord(content) || content.type !== 'thinking') continue; + blocks++; + bytes += stringValue(content.signature)?.length ?? 0; + } + } + return { blocks, signatureBytes: bytes, + bytesPerBlock: blocks ? Math.round(bytes / blocks) : 0 }; + } catch { return null; } +} + +function combinedThinkingVolume(appDir: string, sessionIds: readonly (string | null | undefined)[]): ThinkingVolume | null { + const volumes: ThinkingVolume[] = [...new Set(sessionIds.filter((id): id is string => Boolean(id)))] + .map(id => thinkingVolume(appDir, id)).filter((item): item is ThinkingVolume => item !== null); + if (!volumes.length) return null; + const blocks = volumes.reduce((sum, item) => sum + item.blocks, 0); + const signatureBytes = volumes.reduce((sum, item) => sum + item.signatureBytes, 0); + return { blocks, signatureBytes, + bytesPerBlock: blocks ? Math.round(signatureBytes / blocks) : 0 }; +} + + +// Record the Linux CLI executed by the container. The host and container +// binaries can change independently and must not share an identity. +function linuxSpacetimeVersion(image: string): { commit: string | null; binarySha256: string | null; raw: string } { + try { + const releaseVolume = process.env.STACK_BENCH_RELEASE_DEPS_VOLUME?.trim() || null; + const mountArgs = releaseVolume + ? dockerMountArguments({ kind: 'volume', source: releaseVolume, + target: CODING_CONTAINER_RELEASE_DEPS_ROOT, readOnly: true }) + : ['-v', `${LINUX_CLI}:${CODING_CONTAINER_SPACETIME_CLI}:ro`]; + const entrypoint = releaseVolume + ? `${CODING_CONTAINER_RELEASE_DEPS_ROOT}/spacetimedb-cli` + : CODING_CONTAINER_SPACETIME_CLI; + const out = execFileSync('docker', + ['run', '--rm', ...mountArgs, '--entrypoint', entrypoint, image, '--version'], + { encoding: 'utf8', stdio: 'pipe', env: { ...process.env, MSYS_NO_PATHCONV: '1' }, + timeout: CONTROL_COMMAND_TIMEOUT_MS }); + const commit = out.match(/Commit:\s*([0-9a-f]+)/i)?.[1] ?? null; + return { commit, binarySha256: sha256(readFileSync(LINUX_CLI)), + raw: out.trim().split(/\r?\n/).slice(0, 2).join(' ') }; + } catch { return { commit: null, binarySha256: null, raw: 'unknown' }; } +} + +function bindingsIdentity(pkgDir: string): { package: string; sourceSha256: string | null; sourceFiles: number } { + try { + const p = JSON.parse(readFileSync(join(pkgDir, 'package.json'), 'utf8')); + const source = hashDirectory(pkgDir, { exclude: name => + /(^|\/)(node_modules|dist|target)(\/|$)/.test(name) }); + return { package: `${p.name}@${p.version}`, sourceSha256: source.sha256, + sourceFiles: source.files.length }; + } catch { return { package: 'unknown', sourceSha256: null, sourceFiles: 0 }; } +} + +// The CLI version inside the build image. Read by running it, not by trusting +// the tag: the image is pinned by ARG and a tag can be moved. +function imageCliVersion(image: string, executable: string): string { + try { + return execFileSync('docker', ['run', '--rm', '--entrypoint', executable, image, '--version'], + { encoding: 'utf8', stdio: 'pipe', env: { ...process.env, MSYS_NO_PATHCONV: '1' }, + timeout: CONTROL_COMMAND_TIMEOUT_MS }).trim(); + } catch { return 'unknown'; } +} + +function imageNodeVersion(image: string): string { + try { + return execFileSync('docker', ['run', '--rm', '--entrypoint', 'node', image, '--version'], + { encoding: 'utf8', stdio: 'pipe', env: { ...process.env, MSYS_NO_PATHCONV: '1' }, + timeout: CONTROL_COMMAND_TIMEOUT_MS }).trim(); + } catch { return 'unknown'; } +} + +function containerImage(name: string): { reference: string; imageId: string | null | undefined } { + try { + const out = execFileSync('docker', ['inspect', '-f', '{{.Config.Image}} {{.Image}}', name], + { encoding: 'utf8', stdio: 'pipe', timeout: CONTROL_COMMAND_TIMEOUT_MS }).trim(); + const [reference, imageId] = out.split(/\s+/, 2); + return { reference: reference ?? '', imageId }; + } catch { return { reference: 'unknown', imageId: null }; } +} + +// Record ambient provider configuration that can change model behaviour while +// replacing credential values with presence markers. +function ambientEnv(): Record { + const seen: Record = {}; + for (const [k, v] of Object.entries(process.env)) { + if (!/^(CLAUDE|ANTHROPIC|OPENAI|OPENROUTER|CODEX|MAX_THINKING|DISABLE_AUTOUPDATER|FORCE_PROMPT)/.test(k)) continue; + // Never record a credential, only that one was present. + seen[k] = /KEY|TOKEN|SECRET|AUTH/i.test(k) ? '' : v; + } + return seen; +} + +export function parseAgentArgs(argv: readonly string[]): AgentArgs { + const strings = ['provider', 'provider-route', 'max-output-tokens', 'mode', 'track', 'backend', 'level', 'app', 'run-index', 'model', + 'pricing-json', 'guidance', 'guidance-document-json', 'credential-aliases-json', + 'recipe', 'recipe-task-json', 'max-budget-usd', 'skills-json', + 'skill-identity-json'] as const; + const { values: rawValues } = parseNodeArgs({ args: [...argv.slice(2)], options: Object.fromEntries([ + ...strings.map(name => [name, { type: 'string' as const }]), + ['print-prompt', { type: 'boolean' as const }], + ['no-production-quality', { type: 'boolean' as const }], + ]), strict: true, allowPositionals: false }); + const values = rawValues as Partial> + & { 'print-prompt'?: boolean; 'no-production-quality'?: boolean }; + const mode = values.mode; + if (mode !== 'build' && mode !== 'upgrade' && mode !== 'fix' && mode !== 'resume') { + throw new Error('--mode must be build, upgrade, fix, or resume'); + } + const backend = values.backend; + const app = values.app; + if (!backend || !app) { + throw new Error('usage: node dist/commands/agent.js --mode build|upgrade|fix|resume ' + + '--backend --app [--level ]'); + } + const level = values.level === undefined ? 1 : Number(values.level); + if (!Number.isSafeInteger(level) || level < 1) { + throw new Error('--level must be a positive integer'); + } + const runIndex = values['run-index'] === undefined ? 0 : Number(values['run-index']); + if (!Number.isSafeInteger(runIndex) || runIndex < 0) { + throw new Error('--run-index must be a non-negative integer'); + } + const provider = parseCodingProvider(values.provider ?? 'anthropic'); + const codingProvider = CODING_PROVIDERS[provider]; + if (codingProvider.requiresBudget && !values.model) throw new Error(`--model is required for ${provider}`); + const providerRoute = validateProviderRoute(provider, values['provider-route']); + const maxOutputTokens = validateProviderOutputLimit(provider, + values['max-output-tokens'] === undefined ? undefined : Number(values['max-output-tokens'])); + const model = values.model ?? 'claude-sonnet-5'; + const maxBudgetUsd = values['max-budget-usd'] === undefined + ? undefined : Number(values['max-budget-usd']); + if (maxBudgetUsd !== undefined && (!Number.isFinite(maxBudgetUsd) || maxBudgetUsd <= 0)) { + throw new Error('--max-budget-usd must be a positive number'); + } + const skills = values['skills-json'] === undefined ? undefined + : stringArray(values['skills-json'], '--skills-json'); + let pricing = values['pricing-json'] === undefined + ? undefined : validatePricingAuthority(JSON.parse(values['pricing-json']), { at: '--pricing-json' }); + if (pricing === undefined && maxBudgetUsd !== undefined) { + const rates = CODING_PROVIDERS[provider].rates(model); + if (!rates) throw new Error(`no default pricing is recorded for model ${model}`); + pricing = validatePricingAuthority({ unit: PRICING_UNIT, rates }, + { at: 'default pricing' }); + } + return { provider, ...(providerRoute ? { providerRoute } : {}), mode, backend, app, level, runIndex, model, + ...(maxOutputTokens ? { maxOutputTokens } : {}), + guidance: parseGuidanceMode(values.guidance ?? 'prescribed'), + productionQuality: mode !== 'resume' && !values['no-production-quality'], + track: values.track ?? DEFAULT_TRACK, pricing: pricing ?? null, + ...(values['guidance-document-json'] ? { + guidanceDocument: JSON.parse(values['guidance-document-json']) as ResolvedGuidanceDocument, + } : {}), + ...(values['credential-aliases-json'] ? { + credentialAliases: JSON.parse(values['credential-aliases-json']) as Record, + } : {}), + ...(values.recipe ? { recipe: values.recipe } : {}), + ...(values['recipe-task-json'] ? { + recipeTask: JSON.parse(values['recipe-task-json']) as RecipeTaskRequest, + } : {}), + ...(maxBudgetUsd !== undefined ? { maxBudgetUsd } : {}), + ...(skills ? { skills } : {}), + ...(values['skill-identity-json'] ? { + skillIdentity: validateSkillIdentity(JSON.parse(values['skill-identity-json'])), + } : {}), + ...(values['print-prompt'] ? { printPrompt: true } : {}) }; +} + +const dbUrl = (backend: string, runIndex: number, dbPort: number | null, track: Track): string | null => { + const adapter = STACK_ADAPTER_REGISTRY.get(backend); + if (adapter.id === 'spacetime' || adapter.id === 'convex' || adapter.id === 'stub') return null; + if (!dbPort) throw new Error(`${backend} has no assigned database port`); + if (process.env.STACK_BENCH_APPLIANCE === '1' && process.env.STACK_BENCH_LEASE) { + const { lease } = leaseFromEnv(process.env, { backend, active: true }); + if (lease.resources.network) return attemptDatabaseUrl({ backend, + database: lease.resources.database!, ownershipToken: lease.ownershipToken }); + } + return adapter.agent.connectionUrl({ dbPort, database: dbName(track, runIndex), hostUrl }); +}; + +// Create the leased database before the app connects. A build clears its schema. +// A reset between suites preserves the schema required by the running app. +type DatabasePreparationLease = BackendLease; + +type DatabaseCommandOptions = Pick; + +type DatabaseCommandExecutor = (command: string, args: readonly string[], + options: DatabaseCommandOptions) => string; + +const databaseCommandExecutor: DatabaseCommandExecutor = (command, args, options) => + String(execFileSync(command, args, { ...options, encoding: 'utf8' })); + +interface DatabasePreparationOptions { + exec?: DatabaseCommandExecutor; + stdbBin?: string; + lease?: DatabasePreparationLease; +} + +export function ensureDatabase(backend: string, runIndex: number, dbPort: number | null, + track: Pick, wipe = false, + { exec = databaseCommandExecutor, stdbBin = STDB_BIN, lease: suppliedLease }: DatabasePreparationOptions = {}) { + const lease = suppliedLease ?? leaseFromEnv(process.env, { backend, active: true }).lease; + if (lease.runIndex !== runIndex || lease.track !== track.name) { + throw new Error(`backend lease ${lease.runId} belongs to ${lease.track}/run${lease.runIndex}, ` + + `not ${track.name}/run${runIndex}`); + } + const expectedName = dbName(track, runIndex); + const name = lease.resources.database ?? expectedName; + const input = { name, expectedName, wipe, exec, cli: stdbBin, + expectedServerUri: STDB_URI, expectedModule: moduleName(track, runIndex), dbPort }; + const adapter = STACK_ADAPTER_REGISTRY.get(backend); + if (adapter.id === 'postgres' || adapter.id === 'mongodb') { + return adapter.database.prepare({ ...input, lease: requireLeasedDatabase(lease) }); + } + if (adapter.id === 'spacetime') { + return adapter.database.prepare({ ...input, lease: requireLeasedSpacetime(lease) }); + } + if (!('prepare' in adapter.database)) { + // A platform lifecycle owns its database; start.sh deploys the application to it. + if (!lease.resources.serverUri || !lease.resources.container?.owned) { + throw new Error(`${backend} application setup requires its active owned deployment`); + } + return; + } + return adapter.database.prepare({ name }); +} + +// Prescribed guidance chooses an implementation stack. Neutral guidance gives +// only stack access facts and the selected API references. +export function readBackendGuidanceDocument( + document: ResolvedGuidanceDocument | undefined, + fallbackRelativePath: string, +): string { + if (typeof fallbackRelativePath !== 'string' || !fallbackRelativePath) { + throw new Error('backend guidance fallback path is required'); + } + if (document !== undefined) { + const fields = new Set(['path', 'sha256', 'bytes', 'applicationInterface']); + if (!document || typeof document !== 'object' || Array.isArray(document) + || Object.keys(document).some(field => !fields.has(field)) + || typeof document.path !== 'string' || !document.path || isAbsolute(document.path) + || document.path.includes('\\') + || !/^[a-f0-9]{64}$/.test(document.sha256) + || !Number.isSafeInteger(document.bytes) || document.bytes < 0 + || typeof document.applicationInterface !== 'string' + || !/^[a-z][a-z0-9-]*$/.test(document.applicationInterface)) { + throw new Error('campaign guidance document identity is invalid'); + } + } + const root = realpathSync(ROOT); + const candidate = resolve(root, document?.path ?? fallbackRelativePath); + const candidateRel = relative(root, candidate); + if (candidateRel === '..' || candidateRel.startsWith(`..${sep}`) || isAbsolute(candidateRel)) { + throw new Error('campaign guidance document escapes the Stack Bench root'); + } + const selectedPath = realpathSync(candidate); + const resolvedRel = relative(root, selectedPath); + if (resolvedRel === '..' || resolvedRel.startsWith(`..${sep}`) || isAbsolute(resolvedRel)) { + throw new Error('campaign guidance document resolves outside the Stack Bench root'); + } + const bytes = Buffer.from(normalizePromptText(readFileSync(selectedPath, 'utf8')), 'utf8'); + if (document && (sha256(bytes) !== document.sha256 || bytes.length !== document.bytes)) { + throw new Error(`campaign guidance document changed after compilation: ${document.path}`); + } + return bytes.toString('utf8'); +} + +function validateSkillIdentity(value: unknown): ResolvedSkills { + const fields = new Set(['ids', 'sha256', 'bytes']); + if (!isRecord(value) || Object.keys(value).some(field => !fields.has(field)) + || !Array.isArray(value.ids) || new Set(value.ids).size !== value.ids.length + || value.ids.some(id => typeof id !== 'string' || !/^[a-z][a-z0-9-]*$/.test(id)) + || typeof value.sha256 !== 'string' || !/^[a-f0-9]{64}$/.test(value.sha256) + || !Number.isSafeInteger(value.bytes) || Number(value.bytes) < 0) { + throw new Error('campaign skill identity is invalid'); + } + return { ids: value.ids as string[], sha256: value.sha256, bytes: Number(value.bytes) }; +} + +function backendDoc(args: AgentArgs, p: StackRunPorts, track: Track): string { + const defaultGuidance = resolveDefaultGuidanceForStack(args.guidance, args.backend); + let defaultPath = defaultGuidance?.documents[args.backend]?.path; + if (!defaultPath && args.guidance === 'neutral') { + throw new Error(`neutral guidance has no document for ${args.backend}`); + } + defaultPath ??= join('backends', `${args.backend}.md`); + const raw = readBackendGuidanceDocument(args.guidanceDocument, defaultPath); + return raw + .replaceAll('', String(p.vite)) + .replaceAll('', String(p.express ?? '')) + .replaceAll('', track.title) + .replaceAll('', moduleName(track, args.runIndex)) + .replaceAll('', p.dbPort ? dbUrl(args.backend, args.runIndex, p.dbPort, track) ?? '' : '') + .replaceAll('', hostUrl(STDB_URI)) + .replaceAll('', C_BIN) + .replaceAll('', `file:${CODING_CONTAINER_SPACETIME_PACKAGE}`); +} + +// Fail before a paid session when the selected container cannot run this checkout. +function containerBlocker(backend: string): string | null { + try { + execFileSync('docker', ['image', 'inspect', IMAGE], + { stdio: 'pipe', timeout: CONTROL_COMMAND_TIMEOUT_MS }); + } catch (error: unknown) { + const detail = error instanceof Error ? error.message.split('\n')[0] : String(error).split('\n')[0]; + return `cannot verify isolation image ${IMAGE}: ${detail} — ` + + `build it with docker build -t ${IMAGE} ${fwd(join(ROOT, 'container'))}`; + } + if (!STACK_ADAPTER_REGISTRY.get(backend).agent.linuxCliRequired) return null; + if (!existsSync(LINUX_CLI)) { + return `no Linux SpacetimeDB CLI at ${fwd(LINUX_CLI)} — ` + + 'bash tools/stack-bench/container/build-linux-cli.sh'; + } + // A file at this path must be a Linux executable, not the Windows build. + const magic = Buffer.alloc(4); + try { + const fd = openSync(LINUX_CLI, 'r'); + try { readSync(fd, magic, 0, 4, 0); } finally { closeSync(fd); } + } catch { + return `cannot read the Linux SpacetimeDB CLI at ${fwd(LINUX_CLI)}`; + } + if (magic.toString('binary') !== '\x7fELF') { + return `${fwd(LINUX_CLI)} is not a Linux binary; rebuild it with ` + + 'container/build-linux-cli.sh'; + } + return null; +} + +function decideIsolation(args: AgentArgs): { container: true; reason: null } { + const blocker = containerBlocker(args.backend); + if (!blocker) return { container: true, reason: null }; + console.error(`agent.js: isolated build unavailable: ${blocker}`); + console.error(' benchmark coding sessions require the isolation container'); + process.exit(2); +} + +// Pin every round to the build's recorded container topology. +function resolveIsolation(args: AgentArgs): { container: true; reason: null } { + const marker = resolve(args.app, '..', '.stack-bench-isolation'); + const backendMarker = resolve(args.app, '..', '.stack-bench-backend'); + + if (args.mode === 'build') { + const decided = decideIsolation(args); + if (!args.printPrompt) { + mkdirSync(dirname(marker), { recursive: true }); + writeFileSync(marker, 'container'); + } + return decided; + } + + if (existsSync(marker)) { + const pinned = readFileSync(marker, 'utf8').trim(); + if (pinned !== 'container') { + console.error(`agent.js: unsupported isolation marker ${JSON.stringify(pinned)}; expected "container"`); + process.exit(2); + } + const blocker = containerBlocker(args.backend); + if (blocker) { + console.error(`agent.js: this run's build ran in a container, but ${blocker}`); + console.error(' refusing to run this round in a different environment'); + process.exit(2); + } + return { container: true, reason: null }; + } + + // A backend marker without an isolation marker is ambiguous prior state. + if (existsSync(backendMarker)) { + console.error('agent.js: app has prior benchmark state but no isolation marker'); + console.error(' refusing to guess where earlier rounds ran; start a clean run'); + process.exit(2); + } + const decided = decideIsolation(args); + if (!args.printPrompt) { + mkdirSync(dirname(marker), { recursive: true }); + writeFileSync(marker, 'container'); + } + return decided; +} + +export function buildPrompt(args: AgentArgs, p: StackRunPorts, track: Track, + materials: PromptMaterials = {}): string { + const prompt = (lines: string[]): string => assertAgentVisibleText(lines.join('\n')); + const applicationInterface = args.guidanceDocument?.applicationInterface + ?? resolveDefaultGuidanceForStack(args.guidance, args.backend) + ?.documents[args.backend]?.applicationInterface; + if (applicationInterface !== stackApplicationInterface(args.backend)) { + throw new Error(`stack ${args.backend} guidance does not select its application interface`); + } + const common = [ + `Build the app in ${CODING_CONTAINER_APP_ROOT}.`, + // Published container ports require the app to bind to all interfaces. + '', + 'The web application must listen on 0.0.0.0, not localhost, so it is reachable ' + + 'outside its process.', + 'The environment can run /app/start.sh again with APP_WARM_START=1. ' + + 'When dependencies are current, reuse them instead of installing them again.', + 'Startup must work with an empty database by creating the supplied starting data and accounts. ' + + 'On an existing database, preserve current quantities, prices, and user data. ' + + 'This applies after upgrades and repairs too.', + '', + 'Chromium is installed at /usr/bin/chromium (CHROME_BIN). ' + + 'Use that executable with --no-sandbox in this isolated container; no browser download is needed. ' + + 'Puppeteer Core is installed at /opt/browser-tools/node_modules/puppeteer-core. ' + + 'In a .cjs script, use const puppeteer = require("/opt/browser-tools/node_modules/puppeteer-core"); ' + + 'then await puppeteer.launch({ executablePath: process.env.CHROME_BIN, args: ["--no-sandbox"] }).', + '', + '## Stack', + '', + agentVisibleContractText(backendDoc(args, p, track), args.credentialAliases, + applicationInterface), + ]; + const skills = materials.skillsText ?? readAgentSkillDocuments(ROOT, args.skills ?? []); + if (skills) common.push('', '## Selected API reference', '', skills); + + if (args.mode === 'resume') { + return prompt([ + 'Restore the existing application to a runnable state.', + '', + 'This is a saved application from an earlier completed run. Install its', + 'dependencies and start its existing database module, server, and web client', + 'as needed. Do not implement features or fix application behavior. Do not', + 'change source files. The saved source must remain byte-for-byte identical.', + '', + 'Output RESUME_COMPLETE when the existing app is running.', + '', + ...common, + ]); + } + + if (args.productionQuality) common.unshift('Build a production-quality application suitable for real users, not a prototype or demo.', ''); + const startingCatalog = materials.startingCatalog + ? ['', '## Starting catalog', '', args.mode === 'build' + ? 'Use exactly this starting data:' + : 'This is the original catalog baseline. Preserve its entity names and relationships. Do not reset current quantities, prices, or user data.', + '', '```json', materials.startingCatalog, '```'] : []; + + if (args.mode === 'fix') { + return prompt([ + 'Fix the reported application bugs.', + '', + `Read ${CODING_CONTAINER_BUG_REPORT_FILE} in the app directory. Each entry says what was expected`, + 'and what actually happened. Fix the app so the behaviour matches, redeploy,', + 'and make sure the dev server is running.', + '', + 'Change only what is needed. Do not alter behaviour that is already correct.', + '', + 'Output FIX_COMPLETE when done.', + '', + ...common, + '', + agentVisibleContractText(materials.requirementText ?? levelPrompt(track, args.level), + args.credentialAliases, applicationInterface), + ...startingCatalog, + '', + '## Application interface', + '', + agentVisibleContractText(materials.contractText ?? appendix(track, args.level), + args.credentialAliases, applicationInterface), + ]); + } + + const verb = args.mode === 'upgrade' + ? [ + 'Add the features below to the existing app.', + '', + 'Keep completed features working. Add only the current features below.', + ] + : [`Build the application described below and leave it running.`]; + + return prompt([ + ...verb, + '', + `After the web application is running, reply with ${args.mode === 'upgrade' + ? 'UPGRADE_COMPLETE' : 'DEPLOY_COMPLETE'}.`, + '', + ...common, + '', + agentVisibleContractText(materials.requirementText ?? levelPrompt(track, args.level), + args.credentialAliases, applicationInterface), + ...startingCatalog, + '', + '## Application interface', + '', + agentVisibleContractText(materials.contractText ?? appendix(track, args.level), + args.credentialAliases, applicationInterface), + ]); +} + +export function agentScenarioPaths(track: Track, level: number, + recipeBinding: RecipeBinding | null = null): string[] { + const execution = recipeBinding?.execution; + if (execution) return execution.map(entry => resolve(track.dir, entry.source ?? '')); + return suitesFor(track, level).map(suite => suite.spec); +} + +export function agentRecipeRequest(explicitRecipe: string | null = null, + recipeTask: RecipeTaskRequest | null = null): RecipeRequest | null { + const bound = recipeTask?.recipe; + if (!bound) return explicitRecipe; + if (explicitRecipe && explicitRecipe !== bound.id) { + throw new Error(`agent recipe ${explicitRecipe} does not match bound task ${bound.id}`); + } + return bound; +} + +// The coding container must not contain the controller or grading inputs. + +export function refreshCodingInvocationCredentials({ provider, apiKey, keyFile, + expectedMode, env = process.env }: { provider: keyof typeof CODING_PROVIDERS; + apiKey?: string; keyFile?: string; expectedMode: string | null; env?: NodeJS.ProcessEnv }) { + const credential = keyFile ? readFileSync(keyFile, 'utf8').trim() + : apiKey ?? env[CODING_PROVIDERS[provider].apiKeyEnvironment] ?? ''; + if (keyFile && !credential) throw new Error('selected API key file is empty'); + const auth = resolveContainerAuth({ provider, apiKey: credential, env, + credentialsPath: CODING_PROVIDERS[provider].credentialPath }); + if (expectedMode !== null && expectedMode !== auth.mode) { + throw new Error('provider billing mode changed during the coding action'); + } + return { mode: auth.mode, env: { ...env, STACK_BENCH_AGENT_API_KEY: credential } }; +} + +async function main() { + const args = parseAgentArgs(process.argv); + const track = loadTrack(args.track); + const p = portsFor(track, args.backend, args.runIndex); + const adapter = STACK_ADAPTER_REGISTRY.get(args.backend); + const defaultGuidance = resolveDefaultGuidanceForStack(args.guidance, args.backend); + args.credentialAliases ??= defaultGuidance?.credentialAliases ?? {}; + const profileSkills = defaultGuidance?.skills[args.backend]?.ids; + const defaultSkills = profileSkills ?? [...adapter.agent.defaultSkills]; + const selectedSkills = selectAgentSkills(defaultSkills, + args.skillIdentity?.ids ?? args.skills ?? null); + const skillsText = readAgentSkillDocuments(ROOT, selectedSkills); + if (args.skillIdentity && (sha256(skillsText) !== args.skillIdentity.sha256 + || Buffer.byteLength(skillsText) !== args.skillIdentity.bytes)) { + throw new Error('campaign skill material changed after compilation'); + } + const recipeBinding = resolveRecipeRelease(track, args.level, + agentRecipeRequest(args.recipe ?? null, args.recipeTask ?? null)); + if (args.recipeTask && !recipeBinding) { + throw new Error(`L${args.level} has no recipe release for the requested task`); + } + const selectedTask = recipeBinding + ? (args.recipeTask + ? resolveBoundRecipeTaskRequest(recipeBinding, args.recipeTask) + : createBoundRecipeTaskRequest(recipeBinding)) + : null; + const requirementText = selectedTask?.task.requirementText ?? levelPrompt(track, args.level); + const contractText = selectedTask?.task.contractText ?? appendix(track, args.level); + const startingCatalog = recipeBinding ? JSON.stringify({ + warehouses: recipeBinding.plan.fixture.warehouses, + items: recipeBinding.plan.fixture.items, + }, null, 2) : undefined; + + // Print the exact prompt without starting a session or changing the app. + if (args.printPrompt) { + process.stdout.write(buildPrompt(args, p, track, + { skillsText, requirementText, contractText, startingCatalog })); + return; + } + if (args.mode === 'build') { + assertNewOrEmptyDirectory(args.app, 'build application directory'); + } + resolveIsolation(args); + const imageIdentity = resolveContainerImage(IMAGE); + // Build wipes all backend state. Later rounds preserve it. + ensureDatabase(args.backend, args.runIndex, p.dbPort, track, args.mode === 'build'); + // Never erase a caller-supplied application tree. + mkdirSync(args.app, { recursive: true }); + writeFileSync(resolve(args.app, '..', '.stack-bench-backend'), args.backend); + + const prompt = buildPrompt(args, p, track, + { skillsText, requirementText, contractText, startingCatalog }); + const bugReportPath = join(args.app, CODING_CONTAINER_BUG_REPORT_FILE); + const bugReportText = args.mode === 'fix' && existsSync(bugReportPath) + ? readFileSync(bugReportPath, 'utf8') : null; + const provenance = sessionProvenance({ prompt, skillsText, contractText, bugReportText, + scenarioPaths: agentScenarioPaths(track, args.level, recipeBinding), + trackDir: track.dir, trackManifestPath: join(track.dir, TRACK_MANIFEST_FILE) }); + const started = Date.now(); + const retryLimitRaw = process.env.STACK_BENCH_CODING_INTERRUPTION_RETRIES + ?? String(DEFAULT_CODING_INTERRUPTION_RETRIES); + const retryLimit = Number(retryLimitRaw); + if (!Number.isInteger(retryLimit) || retryLimit < 0 || retryLimit > 3) { + throw new Error('STACK_BENCH_CODING_INTERRUPTION_RETRIES must be an integer from 0 to 3'); + } + // The throttle wait must fit inside the adapter deadline. + const throttleWaitRaw = process.env.STACK_BENCH_PROVIDER_THROTTLE_MAX_WAIT_MINUTES + ?? String(DEFAULT_THROTTLE_MAX_WAIT_MS / 60_000); + const throttleMaxWaitMinutes = Number(throttleWaitRaw); + if (!Number.isInteger(throttleMaxWaitMinutes) || throttleMaxWaitMinutes < 0 + || throttleMaxWaitMinutes > DEFAULT_THROTTLE_MAX_WAIT_MS / 60_000) { + throw new Error('STACK_BENCH_PROVIDER_THROTTLE_MAX_WAIT_MINUTES must be an integer from 0 to ' + + `${DEFAULT_THROTTLE_MAX_WAIT_MS / 60_000}`); + } + // Concurrent campaign slots must not wake and retry as one burst. This + // stable offset keeps retries reproducible while spreading them over 45s. + const throttleJitterMs = parseInt(sha256(Buffer.from( + `${args.backend}:${args.runIndex}:${args.level}:${args.mode}`)).slice(0, 8), 16) % 45_001; + const selectedKeyFile = process.env.STACK_BENCH_AGENT_API_KEY_FILE + ?? process.env.STACK_BENCH_API_KEY_FILE + ?? process.env[`${CODING_PROVIDERS[args.provider].apiKeyEnvironment}_FILE`]; + let selectedAuthMode: string | null = null; + const invocationEnvironment = (baseEnv: NodeJS.ProcessEnv = process.env): NodeJS.ProcessEnv => { + const refreshed = refreshCodingInvocationCredentials({ provider: args.provider, + keyFile: selectedKeyFile, expectedMode: selectedAuthMode, env: baseEnv }); + selectedAuthMode = refreshed.mode; + return refreshed.env; + }; + const actionId = randomUUID(); + const continuationContext = campaignProviderContinuationContext(); + const continuationIdentity = { actionId, mode: args.mode, level: args.level, model: args.model, + provider: args.provider, providerRoute: args.providerRoute, imageId: imageIdentity.id, provenance, pricing: args.pricing, + maxOutputTokens: args.maxOutputTokens, + guidance: args.guidance, skillIdentity: args.skillIdentity, + ...(args.productionQuality ? { productionQuality: true } : {}), + continuationMessage: PROVIDER_CONTINUATION_MESSAGE }; + let coding: CodingSessionRetryResult; + try { + // Send prompts through stdin to avoid the Windows command-line limit. + const cliEnv = { ...process.env, + // Absent unless deliberately overridden — see THINKING_TOKENS above. + ...(args.provider === 'anthropic' && THINKING_TOKENS + ? { MAX_THINKING_TOKENS: THINKING_TOKENS } + : {}), + // Keep the CLI fixed across the campaign. + ...(args.provider === 'anthropic' ? { DISABLE_AUTOUPDATER: '1', + // Pin cache lifetime so run order cannot change cost. + FORCE_PROMPT_CACHING_5M: '1' } : {}) }; + + coding = runCodingSessionWithRetries({ prompt, model: args.model, retryLimit, + maxBudgetUsd: args.maxBudgetUsd, + throttleMaxWaitMs: throttleMaxWaitMinutes * 60_000, + throttleJitterMs, + onInvocation: record => persistCampaignProviderInvocation({ + evidence: { ...continuationIdentity, ...record, billingMode: selectedAuthMode } }), + waitForProvider: continuationContext ? request => { + const nativeOptions = { appDir: args.app, provider: args.provider, + sessionId: request.sessionId, model: args.model, imageId: imageIdentity.id, env: process.env }; + const nativeIdentity = captureNativeContinuation(nativeOptions); + return waitForCampaignProviderContinuation({ + evidence: { ...continuationIdentity, ...request, nativeIdentity, billingMode: selectedAuthMode }, + validate: phase => { + const env = phase === 'continue' ? invocationEnvironment() : process.env; + if (captureNativeContinuation({ ...nativeOptions, env }) !== nativeIdentity) { + throw new Error('native provider session or runtime changed during provider wait'); + } + }, + }); + } : undefined, + invoke: ({ input, maxBudgetUsd, resumeSession, priorSessionUsage, recoverStoppedContainer }) => + execFileSync(process.execPath, [ + compiledEntrypoint('container', 'run-build.js'), + '--provider', args.provider, + ...(args.providerRoute ? ['--provider-route', args.providerRoute] : []), + ...(args.maxOutputTokens ? ['--max-output-tokens', String(args.maxOutputTokens)] : []), + '--app', args.app, + '--backend', args.backend, + '--image', imageIdentity.id, + '--effort', EFFORT, + '--model', args.model, + '--web-search', toolPolicySwitch(TOOLS.webSearch), + '--subagents', toolPolicySwitch(TOOLS.subagents), + ...(args.pricing ? ['--pricing-json', JSON.stringify(args.pricing)] : []), + '--completion-marker', args.mode === 'fix' ? 'FIX_COMPLETE' + : args.mode === 'upgrade' ? 'UPGRADE_COMPLETE' + : args.mode === 'resume' ? 'RESUME_COMPLETE' : 'DEPLOY_COMPLETE', + ...(maxBudgetUsd != null ? ['--max-budget-usd', String(maxBudgetUsd)] : []), + '--ports', [p.vite, p.express].filter(Boolean).join(','), + ...(resumeSession ? ['--resume-session', resumeSession] : []), + ...(priorSessionUsage ? ['--prior-session-usage', JSON.stringify(priorSessionUsage)] : []), + ...(recoverStoppedContainer ? ['--recover-stopped-container'] : []), + ], { input, encoding: 'utf8', maxBuffer: 256 * 1024 * 1024, + env: invocationEnvironment(cliEnv), + timeout: AGENT_PROCESS_TIMEOUT_MS }), + }); + } catch (err: unknown) { + coding = { raw: '', spawnError: codingSessionFailure(isRecord(err) ? err : {}), sessionResults: [], + interruptions: [], result: { total_cost_usd: 0, num_turns: 0, + usage: { input_tokens: 0, output_tokens: 0, cache_creation_input_tokens: 0, + cache_read_input_tokens: 0 }, stack_bench_cost_receipts: [] }, + throttle: { waits: 0, waitedMs: 0, maxWaitMs: throttleMaxWaitMinutes * 60_000, + jitterMs: throttleJitterMs } }; + } + + const { raw, spawnError, sessionResults, interruptions, result, throttle } = coding; + const noOutput = !result.session_id && !raw.trim(); + const failed = Boolean(spawnError || noOutput); + const providerFailure = providerSessionFailure(result); + const usage = sessionUsage(result.usage); + const input = usage.input_tokens ?? 0; + const output = usage.output_tokens ?? 0; + const cacheWrite = usage.cache_creation_input_tokens ?? 0; + const cacheRead = usage.cache_read_input_tokens ?? 0; + const turns = result.num_turns ?? 0; + + // Preserve the cost inputs needed to explain stack differences. + const setupMetadata = adapter.agent.setupMetadata({ + imageId: imageIdentity.id, + localPackage: LOCAL_PKG, + env: process.env, + helpers: { linuxSpacetimeVersion, bindingsIdentity, containerImage }, + }); + const out = { + ...(args.productionQuality ? { productionQuality: true } : {}), + appDir: args.app, + mode: args.mode, + level: args.level, + track: args.track, + backend: args.backend, + model: args.model, + guidance: args.guidance, + setup: { + provider: args.provider, + ...(args.providerRoute ? { providerRoute: args.providerRoute } : {}), + ...(args.maxOutputTokens ? { maxOutputTokens: args.maxOutputTokens } : {}), + thinkingTokens: args.provider !== 'anthropic' ? null : THINKING_TOKENS ? Number(THINKING_TOKENS) : 'cli default', + permissionMode: args.provider === 'anthropic' ? 'acceptEdits' : 'container-isolated', + effort: EFFORT, + tools: TOOLS, + skills: selectedSkills, + cacheTier: args.provider === 'anthropic' ? '5m' : 'provider-managed', + autoUpdater: 'disabled', + codingInterruptionRetries: { limit: retryLimit, + used: interruptions.filter(item => item.kind !== 'provider-throttled').length }, + providerThrottle: { maxWaitMinutes: throttleMaxWaitMinutes, + waits: throttle?.waits ?? 0, waitedMs: throttle?.waitedMs ?? 0, + jitterMs: throttle?.jitterMs ?? throttleJitterMs }, + cliVersion: imageCliVersion(imageIdentity.id, CODING_PROVIDERS[args.provider].executable), + isolation: { mode: 'container', image: imageIdentity.reference, + imageId: imageIdentity.id, hostAlias: HOST_ADDR }, + auth: selectedAuthMode ?? 'not-selected', + ...(isRecord(setupMetadata) ? setupMetadata : {}), + env: ambientEnv(), + node: { orchestrator: process.version, codingContainer: imageNodeVersion(imageIdentity.id) }, + platform: process.platform, + resources: result.stack_bench_resources ?? null, + }, + costUsd: Number((result.total_cost_usd ?? 0).toFixed(6)), + costReceipts: result.stack_bench_cost_receipts ?? [], + ...(result.stack_bench_unaccounted_invocations ? { unaccountedInvocations: result.stack_bench_unaccounted_invocations } : {}), + tokens: input + output + cacheWrite + cacheRead, + outputTokens: output, + usage: { input, output, cacheWrite, cacheRead }, + provenance, + turns, + promptBytes: Buffer.byteLength(prompt), + tokensPerTurn: turns ? Math.round((input + output + cacheWrite + cacheRead) / turns) : null, + thinking: args.provider === 'anthropic' + ? combinedThinkingVolume(args.app, sessionResults.map(item => item.session_id)) : null, + durationMs: Date.now() - started, + sessionId: result.session_id ?? null, + ok: !failed && result.is_error === false, + providerMetadata: { failureCode: failed + ? String(spawnError ?? '').startsWith('provider stayed throttled') + ? 'provider-throttle-exhausted' + : providerFailure?.code ?? (noOutput ? 'coding-session-no-output' : 'coding-session-failed') + : result.is_error === true ? 'provider-session-error' : null, + diagnostic: spawnError, + failure: failed ? { + providerStatus: result.api_error_status ?? null, + waitedMs: throttle?.waitedMs ?? 0, + waits: throttle?.waits ?? 0, + ...(result.stack_bench_provider_failure?.budget ? { budget: result.stack_bench_provider_failure.budget } : {}), + } : null, + interruptions, invocations: sessionResults.length, + providerWaits: coding.providerWaits ?? [], + terminalRecovery: isRecord(result) ? result.terminal_recovery ?? null : null, + credentialBroker: result.stack_bench_credential_broker ?? null, + sessionIds: [...new Set(sessionResults.map(item => item.session_id).filter(Boolean))], + models: args.provider === 'anthropic' + ? transcriptModels(args.app, sessionResults.map(item => item.session_id)) : [] }, + }; + console.log(JSON.stringify(out)); +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().catch(err => { console.error(err); process.exit(1); }); +} diff --git a/tools/stack-bench/commands/bench-arguments.ts b/tools/stack-bench/commands/bench-arguments.ts new file mode 100644 index 00000000000..e8398d53f23 --- /dev/null +++ b/tools/stack-bench/commands/bench-arguments.ts @@ -0,0 +1,355 @@ +import { dirname, resolve } from 'node:path'; +import { parseArgs } from 'node:util'; +import { readArtifact } from '../src/evidence/artifacts.js'; +import { DEFAULT_TRACK, RUN_INDEX_CAP } from '../src/composition/tracks.js'; +import { campaignProgressionOwner, validateCompiledCampaignPlan } from '../src/campaigns/campaign-compiler.js'; +import type { CampaignAttemptPlan, CampaignSelection } + from '../src/campaigns/campaign-compiler.js'; +import { readCampaignAdmission } + from '../src/campaigns/campaign-admission.js'; +import { compileProgressionInput, dependencyRuntimeDefinition, progressionLevels, + validateFeatureCatalogInput, validateProgressionInput } + from '../src/progression/progression-definition.js'; +import type { CompiledDependencyPolicyDefinition, CompiledProgressionDefinition, + ProgressionInput } from '../src/progression/progression-definition.js'; +import { validatePricingAuthority } from '../src/evidence/pricing-authority.js'; +import type { PricingAuthority } from '../src/evidence/pricing-authority.js'; +import { parseGuidanceMode } from '../src/campaigns/condition-compiler.js'; +import type { GuidanceMode } from '../src/campaigns/condition-compiler.js'; +import { validateCampaignExtensionSeed } from '../src/campaigns/campaign-scheduler.js'; +import type { CampaignExtensionSeed } from '../src/campaigns/campaign-scheduler.js'; +import { repairBudgetLimit } from '../src/progression/repair-plan.js'; + +type StudyCondition = CampaignAttemptPlan['condition']; +type UnknownRecord = Record; + +export interface BenchArguments { + backend?: string; + track: string; + levels: string; + levelsProvided: boolean; + levelList: number[]; + model: string | null; + providerRoute?: string; + maxOutputTokens?: number; + agentAdapter: string; + pricing?: PricingAuthority | null; + repairs: number; + maxBudgetUsd?: number; + runIndex: number; + out?: string; + app?: string; + url?: string; + media: boolean; + retainBackend?: boolean; + guidance: GuidanceMode; + productionQuality?: boolean; + guidanceDocument?: unknown; + condition?: StudyCondition; + /** Aliases the grader expects instead of the condition's; `{}` grades with the fixture credentials. */ + gradingCredentialAliases?: Record; + selectionRequest?: CampaignSelection; + taskMode?: string; + retainPriorContracts?: boolean; + pauseAfterDepth?: number; + packIds: string[]; + checkKeys: string[]; + featureIds: string[]; + requestedSpecifications: string[]; + expectedSpecifications: string[]; + observedSpecifications: string[]; + skills?: string[]; + apiKey?: string; + apiKeyFile?: string; + mutations?: string; + mutationShardIndex?: number; + mutationShardCount?: number; + mutationResumeFrom?: string; + mutationCheckpointOut?: string; + mutationBaselineBundle?: string; + expectedMutationCalibration?: unknown; + mutationMaxRuntimeMinutes: number; + referenceMutationOnly?: boolean; + seedFrom?: string; + seedThrough?: number; + progressionSeed?: CampaignExtensionSeed; + parentAttemptId?: string; + repairFrom?: string; + gradeFrom?: string; + gradeLevel?: number; + repairLevel?: number; + recipe?: string; + campaignFile?: string; + campaignAttemptId?: string; + campaignAdmissionId?: string; + progressionResumeFrom?: string; + experimentIdentity?: { id: string; version: string; sha256: string; state: string }; + runMode?: CampaignAttemptPlan['mode']; + featureCatalog?: ProgressionInput; + dependencyPolicy?: ProgressionInput; + progression?: ProgressionInput; + progressionOwner?: UnknownRecord; +} + +interface BenchCliOptions extends Partial { + pack?: string[]; + check?: string[]; + pricingJson?: unknown; + featureModule?: string[]; + expectSpec?: string[]; + expectedMutationCalibrationJson?: unknown; + progressionSeedJson?: unknown; +} + +function parseCli(argv: readonly string[]): BenchCliOptions { + const strings = ['backend', 'track', 'levels', 'campaign-file', 'campaign-attempt-id', + 'campaign-admission-id', 'progression-resume-from', 'recipe', 'model', 'provider-route', 'max-output-tokens', 'pricing-json', + 'repairs', 'max-budget-usd', 'run-index', 'out', 'app', 'url', + 'agent-adapter', 'guidance', 'task-mode', 'mutations', + 'mutation-shard-index', 'mutation-shard-count', 'mutation-resume-from', + 'mutation-checkpoint-out', 'mutation-baseline-bundle', + 'expected-mutation-calibration-json', 'mutation-max-runtime-minutes', 'seed-from', + 'seed-through', 'progression-seed-json', + 'parent-attempt-id', 'repair-from', 'repair-level', 'grade-from', 'grade-level'] as const; + const multiple = ['pack', 'check', 'feature-module', 'expect-spec'] as const; + const options = Object.fromEntries([ + ...strings.map(name => [name, { type: 'string' as const }]), + ...multiple.map(name => [name, { type: 'string' as const, multiple: true }]), + ...['no-media', 'retain-backend', 'reference-mutation-only', 'no-production-quality'].map(name => + [name, { type: 'boolean' as const }]), + ]); + const { values } = parseArgs({ args: [...argv.slice(2)], options, strict: true, + allowPositionals: false }); + const parsed: Record = {}; + for (const [key, value] of Object.entries(values)) { + parsed[key.replace(/-([a-z])/g, (_, letter: string) => letter.toUpperCase())] = value; + } + for (const key of ['pack', 'check', 'featureModule', 'expectSpec']) { + const value = parsed[key] as string[] | undefined; + if (value) parsed[key] = value.flatMap(item => item.split(',').filter(Boolean)); + } + for (const key of ['repairs', 'maxBudgetUsd', 'maxOutputTokens', 'mutationShardIndex', + 'mutationShardCount', 'mutationMaxRuntimeMinutes', 'repairLevel', 'gradeLevel', 'seedThrough']) { + if (typeof parsed[key] === 'string') parsed[key] = Number(parsed[key]); + } + if (typeof parsed.runIndex === 'string') parsed.runIndex = Number(parsed.runIndex); + for (const key of ['campaignFile', 'progressionResumeFrom', 'mutations', + 'mutationResumeFrom', 'mutationCheckpointOut', 'mutationBaselineBundle', 'repairFrom', 'gradeFrom']) { + if (typeof parsed[key] === 'string') parsed[key] = resolve(parsed[key]); + } + for (const key of ['pricingJson', 'expectedMutationCalibrationJson', 'progressionSeedJson']) { + if (typeof parsed[key] === 'string') parsed[key] = JSON.parse(parsed[key]); + } + if (typeof parsed.guidance === 'string') parsed.guidance = parseGuidanceMode(parsed.guidance); + if (parsed.noMedia === true) parsed.media = false; + delete parsed.noMedia; + if (parsed.noProductionQuality) parsed.productionQuality = false; + delete parsed.noProductionQuality; + return parsed as BenchCliOptions; +} + +export function parseBenchArguments(argv: readonly string[]): BenchArguments { + const args: BenchArguments = { model: null, agentAdapter: 'claude-code', + repairs: 10, runIndex: 0, levels: '1', levelsProvided: false, media: true, + levelList: [], guidance: 'prescribed', productionQuality: true, track: DEFAULT_TRACK, + packIds: [], checkKeys: [], featureIds: [], requestedSpecifications: [], + expectedSpecifications: [], observedSpecifications: [], + mutationMaxRuntimeMinutes: 60 }; + const { pack, check, pricingJson, featureModule, expectSpec, + expectedMutationCalibrationJson, progressionSeedJson, ...options } = parseCli(argv); + Object.assign(args, options); + if (args.gradeLevel !== undefined && (!args.gradeFrom + || !Number.isSafeInteger(args.gradeLevel) || args.gradeLevel < 1)) { + throw new Error('--grade-level requires --grade-from and a positive integer depth'); + } + for (const flag of ['--grade-from', '--grade-level']) { + if (argv.slice(2).filter(value => value.split('=', 1)[0] === flag).length > 1) { + throw new Error(`${flag} must be supplied only once`); + } + } + if (args.gradeFrom) { + const allowed = new Set(['--grade-from', '--grade-level', '--out', '--no-media', '--check']); + const forbidden = argv.slice(2).find(value => value.startsWith('--') + && !allowed.has(value.split('=', 1)[0]!)); + if (forbidden) throw new Error(`--grade-from cannot be combined with ${forbidden}`); + if (!args.out) throw new Error('--grade-from requires a separate --out directory'); + args.out = resolve(args.out); + args.repairs = 0; + } + if (pack) args.packIds = pack; + if (check) args.checkKeys = check; + if (pricingJson !== undefined) { + args.pricing = validatePricingAuthority(pricingJson, { at: '--pricing-json' }); + } + if (featureModule) args.featureIds = featureModule; + if (expectSpec) args.expectedSpecifications = expectSpec; + args.levelsProvided = options.levels !== undefined; + if (expectedMutationCalibrationJson !== undefined) { + args.expectedMutationCalibration = expectedMutationCalibrationJson; + } + if (progressionSeedJson !== undefined) { + args.progressionSeed = validateCampaignExtensionSeed(progressionSeedJson); + } + if ((args.mutationResumeFrom || args.mutationCheckpointOut || args.mutationBaselineBundle) + && !args.mutations) { + throw new Error('mutation control options require --mutations'); + } + if (args.expectedMutationCalibration && !args.mutations) { + throw new Error('--expected-mutation-calibration-json requires --mutations'); + } + if (!Number.isFinite(args.mutationMaxRuntimeMinutes) || args.mutationMaxRuntimeMinutes < 1 + || args.mutationMaxRuntimeMinutes > 120) { + throw new Error('--mutation-max-runtime-minutes must be from 1 through 120'); + } + if (args.referenceMutationOnly && (!args.mutations || args.agentAdapter !== 'reference-fixture' + || args.repairs !== 0 || !args.app || args.campaignFile)) { + throw new Error('--reference-mutation-only requires a mutation-bound reference fixture run'); + } + if (args.mutationBaselineBundle && !args.referenceMutationOnly) { + throw new Error('--mutation-baseline-bundle is an internal reference mutation option'); + } + if (args.repairFrom && (args.repairLevel === undefined + || !Number.isSafeInteger(args.repairLevel) || args.repairLevel < 1)) { + throw new Error('--repair-from requires --repair-level with a positive integer'); + } + if (args.campaignFile && !args.campaignAttemptId) { + throw new Error('--campaign-file requires --campaign-attempt-id'); + } + if (!args.campaignFile && (args.campaignAttemptId || args.campaignAdmissionId)) { + throw new Error('campaign binding requires --campaign-file'); + } + if (args.progressionResumeFrom && !args.campaignFile) { + throw new Error('--progression-resume-from requires a compiled campaign'); + } + if (args.seedThrough !== undefined && (!args.seedFrom || !args.campaignFile)) { + throw new Error('--seed-through requires --seed-from and a compiled campaign'); + } + if (args.seedThrough !== undefined && !args.progressionSeed) { + throw new Error('--seed-through requires --progression-seed-json'); + } + if (args.progressionSeed !== undefined && args.seedThrough === undefined) { + throw new Error('--progression-seed-json requires --seed-through'); + } + if (args.progressionSeed && args.progressionSeed.fromDepth !== args.seedThrough) { + throw new Error('--progression-seed-json does not match --seed-through'); + } + if (args.campaignFile) { + const allowed = new Set(['--campaign-file', '--campaign-attempt-id', + '--campaign-admission-id', '--progression-resume-from', '--run-index', '--out', + '--max-budget-usd', '--seed-from', '--seed-through', '--progression-seed-json']); + const override = argv.slice(2).find(value => value.startsWith('--') + && !allowed.has(value.split('=', 1)[0]!)); + if (override) throw new Error(`campaign attempts cannot override ${override}`); + bindCampaign(args); + } + if (!args.backend && !args.repairFrom && !args.gradeFrom) { + throw new Error('--backend is required unless --repair-from, --grade-from, or --campaign-file is supplied'); + } + if (args.progression) { + if (args.levelsProvided) throw new Error('--levels cannot be combined with progression input'); + args.progression = validateProgressionInput(args.progression); + args.levelList = progressionLevels(args.progression); + args.levels = `${args.levelList[0]}-${args.levelList.at(-1)}`; + const seedThrough = args.seedThrough; + if (seedThrough !== undefined && (!args.levelList.includes(seedThrough) + || !args.levelList.some(level => level > seedThrough))) { + throw new Error('--seed-through must precede another planned dependency depth'); + } + if (seedThrough !== undefined + && args.dependencyPolicy?.definition.workSelection !== 'progressive') { + throw new Error('--seed-through requires progressive dependency work selection'); + } + } else { + const [fromText, toText] = args.levels.split('-'); + const from = Number(fromText); + const to = toText === undefined ? from : Number(toText); + if (!Number.isSafeInteger(from) || from < 1 || !Number.isSafeInteger(to) || to < from) { + throw new Error('--levels must be one positive level or an ascending range'); + } + args.levelList = Array.from({ length: (to ?? from) - from + 1 }, (_, index) => from + index); + if (args.seedThrough !== undefined) { + throw new Error('--seed-through requires dependency mode'); + } + } + if (args.recipe && args.levelList.length !== 1) { + throw new Error('--recipe requires exactly one requested level'); + } + if (!Number.isSafeInteger(args.repairs) || args.repairs < 0) { + throw new Error('--repairs must be a non-negative safe integer'); + } + if (args.maxBudgetUsd !== undefined + && (!Number.isFinite(args.maxBudgetUsd) || args.maxBudgetUsd <= 0)) { + throw new Error('--max-budget-usd must be a positive number'); + } + if (!Number.isSafeInteger(args.runIndex) || args.runIndex < 0 || args.runIndex > RUN_INDEX_CAP) { + throw new Error(`--run-index must be an integer from 0 through ${RUN_INDEX_CAP}`); + } + if ((args.mutationShardIndex === undefined) !== (args.mutationShardCount === undefined)) { + throw new Error('--mutation-shard-index and --mutation-shard-count must be supplied together'); + } + return args; +} + +function bindCampaign(args: BenchArguments): void { + if (!args.campaignFile) throw new Error('campaign file is required'); + const artifact = readArtifact(args.campaignFile, { expectedKind: 'campaign_plan' }); + const plan = validateCompiledCampaignPlan(artifact.payload); + const attempt = plan.attempts.find(item => item.id === args.campaignAttemptId); + if (!attempt) throw new Error('--campaign-attempt-id is not in the compiled campaign plan'); + const plannedBudget = plan.definition.budgets.maxCostUsdPerAttempt; + if (args.maxBudgetUsd !== undefined + && (plannedBudget === null || args.maxBudgetUsd > plannedBudget)) { + throw new Error('--max-budget-usd exceeds the compiled campaign budget'); + } + args.backend = attempt.stack; + args.track = plan.definition.track; + args.model = attempt.model; + if (args.providerRoute !== undefined && args.providerRoute !== attempt.providerRoute) { + throw new Error('--provider-route differs from the compiled campaign'); + } + args.providerRoute = attempt.providerRoute; + if (args.maxOutputTokens !== undefined && args.maxOutputTokens !== attempt.maxOutputTokens) { + throw new Error('--max-output-tokens differs from the compiled campaign'); + } + args.maxOutputTokens = attempt.maxOutputTokens; + args.agentAdapter = attempt.agentAdapter; + args.pricing = validatePricingAuthority(attempt.pricing, { at: 'compiled campaign pricing' }); + args.guidance = parseGuidanceMode(attempt.guidance); + args.condition = structuredClone(attempt.condition); + args.productionQuality = attempt.condition.productionQuality === true; + args.skills = structuredClone(attempt.skills); + args.selectionRequest = structuredClone(plan.definition.selection); + args.guidanceDocument = structuredClone( + attempt.condition.guidance.documents[attempt.stack]); + args.packIds = structuredClone(plan.definition.selection.packs ?? []); + args.checkKeys = structuredClone(plan.definition.selection.checks ?? []); + args.repairs = attempt.mode.id === 'dependency' + ? 0 : repairBudgetLimit(plan.definition.repair); + args.maxBudgetUsd ??= plannedBudget ?? undefined; + args.parentAttemptId = attempt.id; + args.media = false; + args.levels = `${Math.min(...attempt.levels)}-${Math.max(...attempt.levels)}`; + args.experimentIdentity = { + id: plan.id, version: plan.version, sha256: plan.contentSha256, state: plan.state, + }; + if (args.campaignAdmissionId) { + const admission = readCampaignAdmission(dirname(args.campaignFile), + args.campaignAdmissionId, plan); + if (!admission.ok) throw new Error('campaign admission did not pass'); + } + args.runMode = structuredClone(attempt.mode); + if (plan.featureCatalog) { + args.featureCatalog = validateFeatureCatalogInput(plan.featureCatalog); + } + if (attempt.mode.id === 'dependency') { + args.pauseAfterDepth = attempt.mode.pauseAfterDepth; + args.retainPriorContracts = attempt.mode.retainPriorContracts === true; + if (!plan.dependencyPolicy || !args.featureCatalog) { + throw new Error('dependency campaign requires a feature catalog and dependency policy'); + } + args.dependencyPolicy = plan.dependencyPolicy; + args.progression = compileProgressionInput(dependencyRuntimeDefinition( + args.featureCatalog, args.dependencyPolicy)); + args.progressionOwner = { ...campaignProgressionOwner(plan, attempt) }; + } +} diff --git a/tools/stack-bench/commands/bench.ts b/tools/stack-bench/commands/bench.ts new file mode 100644 index 00000000000..8b9ca7c1e67 --- /dev/null +++ b/tools/stack-bench/commands/bench.ts @@ -0,0 +1,3173 @@ +#!/usr/bin/env node + +import { campaignProviderContinuationContext } from '../src/campaigns/campaign-provider-continuation.js'; +import { execFileSync } from 'node:child_process'; +import type { ExecFileSyncOptionsWithStringEncoding } from 'node:child_process'; +import { readFileSync, writeFileSync, mkdirSync, mkdtempSync, existsSync, copyFileSync, cpSync, rmSync, readdirSync, realpathSync, lstatSync } from 'node:fs'; +import { join, dirname, resolve, relative, sep, isAbsolute } from 'node:path'; +import { tmpdir } from 'node:os'; +import { pathToFileURL } from 'node:url'; +import { loadTrack, resultsName, portsFor, workDirFor, + moduleName, dbName, suitesFor } from '../src/composition/tracks.js'; +import { parseBenchArguments } from './bench-arguments.js'; +import type { BenchArguments } from './bench-arguments.js'; +import { packageRegistry, packageRegistryEnvironment } from '../src/runtime/package-registry.js'; +import { runBounded } from '../src/runtime/bounded-process.js'; +import { formatRepairProgress } from '../src/evidence/scoring.js'; +import { ARTIFACT_FILE, emptyArtifactIdentities, readArtifact, readArtifactPayload, + writeArtifact, writeRunJson, currentEngineIdentity } from '../src/evidence/artifacts.js'; +import { aggregateRunOutcome, classifyBundle, ladderMayAdvance, ladderMayContinue, + mutationControlEligible, runExitCode, runOutcomeKind } from '../src/evidence/outcomes.js'; +import { summarizeSessions } from '../src/evidence/session-metrics.js'; +import { hashDirectory, sha256 } from '../src/evidence/provenance.js'; +import { createBackendLease, newRunId, publicBackendLease, readBackendLease, + claimBackendResourcesWhenAvailable, backendResourceLockKeys, resourceLockScope, loopbackHttpUri, + GRADING_WAIT_RECEIPT_ENV, GRADING_WAIT_RECEIPT_FILE, readGradingWaits, recordedGradingWaitMs } from '../src/runtime/backend-lease.js'; +import { borrowCampaignReservation } + from '../src/campaigns/campaign-admission.js'; +import { captureApplicationDiagnostics } from '../src/runtime/backend-control.js'; +import type { RuntimeControlSpec } from '../src/runtime/backend-control.js'; +import { releaseBackendLease } from '../src/runtime/backend-teardown.js'; +import { resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { createAgentVisibleTaskRequest, createBoundRecipeTaskRequest } + from '../src/composition/recipe-selection.js'; +import { criterionEvidence, evidencePassed } from '../src/evidence/check-evidence.js'; +import { leasedDatabaseEnvironment, STACK_ADAPTER_REGISTRY } from '../src/stacks/stack-adapters.js'; +import { agentRecipeIdentity, agentRequestArgv, validateProviderRoute, validateProviderOutputLimit } from '../src/agents/agent-adapter-contract.js'; +import { agentSessionFailure, validateAgentResult } + from '../src/agents/agent-result-contract.js'; +import { AGENT_ADAPTER_REGISTRY, agentAdapterIdentity } from '../src/agents/agent-adapters.js'; +import { archiveTranscripts } from '../src/agents/transcript-archive.js'; +import { runPreflight } from '../src/runtime/preflight.js'; +import { checkpointSchema, recordRunCheckpoint } from '../src/evidence/run-checkpoints.js'; +import type { RunCheckpoint } from '../src/evidence/run-checkpoints.js'; +import { runCostEvidence } from '../src/evidence/cost-proof.js'; +import { redactCredentials } from '../src/evidence/diagnostic-sanitizer.js'; +import { DEFAULT_BUILD_IMAGE } from '../src/composition/product-config.js'; +import { SUPERVISOR_STATE_VERSION, writeRecoveryArtifact } from '../src/runtime/recovery.js'; +import { applyAgentCredential } from '../src/agents/agent-credentials.js'; +import { assertPlainAppSourceTree, hashAppSource, resetAppToSource, seedAppSource, snapshotAppSource } from '../src/runtime/source-snapshot.js'; +import { finalPackageEvidenceRequired, preserveFinalPackageEvidence, preserveLevelCheckpoint, + sourceBoundFirstBuildOutcome } from '../src/runtime/source-checkpoint.js'; +import { materializationAppFailure, materializeAcceptedSource, restoreRepairSource } + from '../src/runtime/source-materialization.js'; +import { compareRepairBaseline, createRepairGrant } from '../src/runtime/repair-grant.js'; +import { canonicalDefinitionJson } from '../src/composition/definition-plan.js'; +import { contractInterfaceNames } from '../src/composition/agent-visible-contract.js'; +import { clearPrivateGradingEvidence, privateGradingDirectory, levelGradeIsUsable, repairEvidenceDecision, + repairHistoryEntry, repairProgressState, repairRegressionDecision, + restorePrivateGradingEvidence } + from '../src/evidence/repair-evidence.js'; +import { mutationControlArgv, mutationControlTimeoutMs, pristineMutationBaselinePath } + from '../src/evidence/mutation-control.js'; +import type { MutationControlArgs } from '../src/evidence/mutation-control.js'; +import { progressionEngine } from '../src/progression/progression-engine.js'; +import { dependencyLevelRepairRecords, dependencyRepairBudget, dependencyRepairRecords, dependencyRepairStopReason } + from '../src/progression/dependency-mode.js'; +import { resolveProgressionRecipeAction, resolveProgressionRecipeLevelSelection, + resolveProgressionRepairTarget, validateProgressionCampaignLevelScope } + from '../src/progression/progression-recipe-selection.js'; +import { createLiveProgressionExecution, clearTimeContinuationBoundary, + commitTimeContinuationBoundary } + from '../src/progression/live-progression.js'; +import type { CampaignSelection } from '../src/campaigns/campaign-compiler.js'; +import { gradingRunTimeoutMs, selectedGradingSourceCount } + from '../src/runtime/grading-timeout.js'; +import { claudeRatesForModel } from '../src/evidence/claude-usage-cost.js'; +import { PRICING_UNIT, validatePricingAuthority } + from '../src/evidence/pricing-authority.js'; +import type { BoundRecipeTaskRequestResult } from '../src/composition/recipe-selection.js'; +import type { RecipeBinding } from '../src/composition/recipe-release.js'; +import type { RepairGrantResolution, RepairOutcome } from '../src/runtime/repair-grant.js'; +import type { AgentAdapter, AgentMode, AgentRequest } + from '../src/agents/agent-adapter-contract.js'; +import type { ValidatedAgentResult } from '../src/agents/agent-result-contract.js'; +import type { Track } from '../src/composition/tracks.js'; +import type { RunOutcome } from '../src/evidence/outcomes.js'; +import type { GradeBundlePayload, BenchmarkRunRecord, RunLevelRecord, + RunContinuation, RunRepairCandidate, RunSessionRecord, RunTotals } + from '../src/evidence/benchmark-run.js'; +import { depthPauseDue, readDepthPauseContext, waitAtDepthBoundary } from '../src/campaigns/campaign-depth-pause.js'; +import { addCostUsd, finalizeRunTotals, recordGradingWait, runSessionRecord } + from '../src/evidence/benchmark-run.js'; +import { formatLevelSummary } from '../src/evidence/evidence-presentation.js'; +import type { ProgressionAction } from '../src/progression/progression-engine.js'; +import type { ProgressionRepairRegression, ProgressionState } + from '../src/progression/progression-state.js'; +import type { ProgressionRecipeAction, ProgressionRecipeSelections } + from '../src/progression/progression-recipe-selection.js'; +import type { BackendLease } from '../src/runtime/backend-lease.js'; + +import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js'; +import { stackBenchResultsRoot } from '../src/runtime/operational-paths.js'; +import { runningContainerIdentity } from '../src/runtime/container-identity.js'; +const COMMAND_TIMEOUT_MS = 20 * 60_000; +const MAX_STALLED_REPAIRS = 3; + +type UnknownRecord = Record; +type ContaminationAudit = { kind: 'contaminated' | 'harness_failure'; evidence: string[]; + verdict: string }; +type LeakAuditEntry = { hits: Array<{ kind: string; path: string }> }; +type CommandFailure = Error & { stdout?: string | Buffer; stderr?: string | Buffer; + status?: number | null; signal?: NodeJS.Signals | null }; +type GradeOptions = { observation?: 'scored' | 'observed'; out?: string | null; + sourceSha256?: string | null; applicationFailure?: RunOutcome | null; + recipeTask?: GradeRecipeTask }; +type MutationControlResult = UnknownRecord & { ok: boolean; artifact?: string; + skipped?: boolean; processError?: string | null; outcome: RunOutcome | null }; +type RecipeTask = (BoundRecipeTaskRequestResult | ProgressionRecipeSelections['grader']) & { agentRequest?: UnknownRecord; + progressionAction?: ProgressionAction }; +type BenchArgs = BenchArguments & { + recipeTasks: Map; + recipeBindings: Map; + repairGrant?: RepairGrantResolution; + mutationImageId?: string; + spentBudgetUsd?: number; +}; +type ProgressionWorkRecipeAction = ProgressionRecipeSelections & { + action: Exclude; +}; +type FirstBuildRecord = { + score: number | null; + max: number | null; + regression: NonNullable['regression'] | null; + contractPass: boolean | null; + outcome: RunOutcome; + source: { sha256: string; files: number } | null; + missed: string[]; + observations?: UnknownRecord; +}; +type RepairStatus = 'not-needed' | 'corrected' | 'budget-exhausted' | 'incomplete' | 'ungraded'; +type ProgressionFailure = { kind?: string; reason?: string }; + +const object = (value: unknown): value is UnknownRecord => + value !== null && typeof value === 'object' && !Array.isArray(value); + +const errorMessage = (error: unknown): string => + redactCredentials(error instanceof Error ? error.message : String(error)); + +function commandFailure(error: unknown): CommandFailure { + if (error instanceof Error) return error; + throw error; +} + +function parseLeakAudit(value: string): LeakAuditEntry[] { + const parsed: unknown = JSON.parse(value); + if (!Array.isArray(parsed)) throw new Error('contamination audit output must be an array'); + return parsed.map((entry, index) => { + if (!object(entry) || !Array.isArray(entry.hits)) { + throw new Error(`contamination audit output[${index}] is invalid`); + } + const hits = entry.hits.map((hit, hitIndex) => { + if (!object(hit) || typeof hit.kind !== 'string' || typeof hit.path !== 'string') { + throw new Error(`contamination audit output[${index}].hits[${hitIndex}] is invalid`); + } + return { kind: hit.kind, path: hit.path }; + }); + return { hits }; + }); +} + +function stringArray(value: unknown, at: string): string[] { + if (!Array.isArray(value) || value.some(entry => typeof entry !== 'string')) { + throw new Error(`${at} must be an array of strings`); + } + return [...value]; +} + +function campaignSelection(value: unknown, at: string): CampaignSelection { + if (!object(value)) throw new Error(`${at} must be an object`); + const optionalStrings = (field: 'packs' | 'checks'): string[] | undefined => { + const entry = value[field]; + if (entry === undefined) return undefined; + return stringArray(entry, `${at}.${field}`); + }; + let levels: CampaignSelection['levels']; + if (value.levels !== undefined) { + if (!Array.isArray(value.levels)) throw new Error(`${at}.levels must be an array`); + levels = value.levels.map((entry, index) => { + if (!object(entry)) throw new Error(`${at}.levels[${index}] is invalid`); + const level = entry.level; + const recipe = entry.recipe; + if (typeof level !== 'number' || !Number.isSafeInteger(level) || typeof recipe !== 'string') { + throw new Error(`${at}.levels[${index}] is invalid`); + } + return { level, recipe, + ...(entry.features === undefined ? {} : { features: stringArray(entry.features, + `${at}.levels[${index}].features`) }), + ...(entry.checks === undefined ? {} : { checks: stringArray(entry.checks, + `${at}.levels[${index}].checks`) }) }; + }); + } + return { ...(optionalStrings('packs') === undefined ? {} : { packs: optionalStrings('packs') }), + ...(optionalStrings('checks') === undefined ? {} : { checks: optionalStrings('checks') }), + ...(levels === undefined ? {} : { levels }) }; +} + +function isProgressionWorkRecipeAction(value: ProgressionRecipeAction): + value is ProgressionWorkRecipeAction { + return value.action.type !== 'terminal'; +} + +function repairCheckKeys(value: ProgressionRecipeAction | null): string[] { + if (!value || !isProgressionWorkRecipeAction(value) || value.action.type !== 'repair') return []; + if (!object(value.action.prompt) || !Array.isArray(value.action.prompt.nodeIds) + || !object(value.action.grading) || !Array.isArray(value.action.grading.checks)) { + throw new Error('dependency repair action has invalid prompt or grading selections'); + } + const promptNodeIds = new Set(value.action.prompt.nodeIds.map(nodeId => { + if (typeof nodeId !== 'string' || !nodeId) { + throw new Error('dependency repair action has an invalid prompt node'); + } + return nodeId; + })); + const checks = value.action.grading.checks.flatMap(check => { + if (!object(check) || typeof check.id !== 'string' || !check.id + || typeof check.nodeId !== 'string' || !check.nodeId) { + throw new Error('dependency repair action has an invalid grading check'); + } + return promptNodeIds.has(check.nodeId) ? [check.id] : []; + }); + if (checks.length === 0) throw new Error('dependency repair action selects no repair checks'); + return checks; +} + +function repairReportArgs(value: ProgressionRecipeAction | null): string[] { + const checks = repairCheckKeys(value); + if (!value || !isProgressionWorkRecipeAction(value) || checks.length === 0) return []; + const interfaces = contractInterfaceNames(value.agent.task.contractText); + return ['--checks-json', JSON.stringify(checks), + '--controls-json', JSON.stringify(interfaces)]; +} + +function repairOwnerNodeIds(value: ProgressionRecipeAction | null): string[] { + if (!value || !isProgressionWorkRecipeAction(value) || value.action.type !== 'repair') return []; + return [...value.action.repair.nodeIds].sort(); +} + +function savedRepairRegression(state: ProgressionState | null, + selected: ProgressionRecipeAction | null): ProgressionRepairRegression | null { + const saved = state?.attempts.at(-1)?.repairRegression; + if (!saved) return null; + const owners = repairOwnerNodeIds(selected); + return JSON.stringify([...saved.ownerNodeIds].sort()) === JSON.stringify(owners) + ? structuredClone(saved) : null; +} + +function requireProgressionState(state: ProgressionState | null): ProgressionState { + if (!state) throw new Error('live dependency progression has no active state'); + return state; +} + +function requireContinuation(run: BenchmarkRunRecord): RunContinuation { + if (!run.continuation) throw new Error('repair continuation has no continuation record'); + return run.continuation; +} + +function requireRunTotals(run: BenchmarkRunRecord): RunTotals { + if (!run.totals) throw new Error('benchmark run totals are not available'); + return run.totals; +} + +function repairOutcome(outcome: RunOutcome): RepairOutcome { + return { kind: outcome.kind, appFailures: [...(outcome.appFailures ?? [])], + inconclusive: [...(outcome.inconclusive ?? [])], + harnessFailures: [...(outcome.harnessFailures ?? [])] }; +} + +function progressionFailure(outcome: RunOutcome): ProgressionFailure { + return { kind: outcome.kind, ...(outcome.reason === null || outcome.reason === undefined + ? {} : { reason: outcome.reason }) }; +} + +function featureCheckKeys(selected: ProgressionWorkRecipeAction, + state: ProgressionState): string[] { + if (!object(selected.action.prompt) || !Array.isArray(selected.action.prompt.nodeIds)) { + throw new Error('feature action has no prompt nodes'); + } + const selectedNodes = new Set(selected.action.prompt.nodeIds); + return state.definition.nodes + .filter(node => selectedNodes.has(node.id)) + .flatMap(node => node.gradingChecks + .filter(check => check.role === 'feature') + .map(check => check.id)); +} + +function bundlePassedChecks(bundle: GradeBundlePayload | null): Set { + return new Set(Object.values(bundle?.suites ?? {}).flatMap(suite => + (suite?.features ?? []).flatMap(feature => + (feature.criteria ?? []).flatMap(criterion => + typeof criterion.stableKey === 'string' + && evidencePassed(criterionEvidence(criterion)) ? [criterion.stableKey] : [])))); +} + +function featureCandidateAccepted(selected: ProgressionWorkRecipeAction, + state: ProgressionState, candidate: GradeBundlePayload | null): boolean { + if (!levelGradeIsUsable(classifyBundle(candidate))) return false; + const passed = bundlePassedChecks(candidate); + if (!featureCheckKeys(selected, state).every(check => passed.has(check))) return false; + return state.definition.nodes.every(node => node.gradingChecks.every(check => + state.nodes[node.id]?.checks[check.id] !== 'pass' || passed.has(check.id))); +} + +function featureActionNeedsCoding(selected: ProgressionWorkRecipeAction, + state: ProgressionState): boolean { + if (selected.action.type === 'repair') return true; + if (!object(selected.action.prompt) || !Array.isArray(selected.action.prompt.nodeIds)) { + throw new Error('feature action has no prompt nodes'); + } + return selected.action.prompt.nodeIds.some(nodeId => { + const node = state.nodes[String(nodeId)]; + return node?.status === 'active' + && Object.values(node.checks).every(outcome => outcome === null); + }); +} + +export function synchronizeProgressionSummary(run: Pick, + state: ProgressionState): void { + run.validation.ladder.completedLevels = [...new Set(state.attempts + .filter(attempt => attempt.outcome === 'conclusive').map(attempt => attempt.level))]; + for (const level of run.levels) { + const latest = state.attempts.findLast(attempt => attempt.level === level.level); + if (latest?.outcome === 'inconclusive') { + level.graded = false; + level.score = null; + level.max = null; + level.selection = latest.selectionSha256 ? { sha256: latest.selectionSha256 } : null; + if (level.outcome?.reason !== latest.reason + || ['passed', 'app_failure'].includes(level.outcome.kind)) { + const kind = latest.category === 'interrupted' ? 'ungraded' + : latest.category === 'inconclusive_evidence' ? 'inconclusive' + : runOutcomeKind(latest.category ?? 'harness_failure'); + level.outcome = { kind, phase: 'progression', reason: latest.reason ?? null, + appFailures: [], inconclusive: [], + harnessFailures: kind === 'harness_failure' ? [latest.reason ?? 'progression failed'] : [] }; + } + } + if (level.repair) { + const used = state.attempts.filter(attempt => attempt.level === level.level && attempt.repair).length; + level.repairs = used - (level.priorRepairs ?? 0); + level.repair.used = used; + level.repair.limit = Math.max(level.repair.limit, used); + level.repair.nodeRepairs = dependencyLevelRepairRecords(state, level.level); + if (level.cumulativeRepairs !== undefined) level.cumulativeRepairs = used; + if (latest?.outcome === 'inconclusive') level.repair.status = 'ungraded'; + else if (level.outcome.kind === 'passed') { + level.repair.status = used ? 'corrected' : 'not-needed'; + level.repair.stopReason = used ? 'passed' : 'not-needed'; + level.stalled = false; + } + } + } +} + +export function mergeFeatureLevelRecord(previous: RunLevelRecord | null, + current: RunLevelRecord): RunLevelRecord { + if (!previous) return current; + const buildSessions: RunSessionRecord[] = [ + ...(previous.buildSessions ?? []), + ...(current.buildSessions ?? []), + ]; + const repairSessions = [...(previous.repairSessions ?? []), ...(current.repairSessions ?? [])]; + const sessionTotals = summarizeSessions([...buildSessions, + ...(current.resumeSession ?? previous.resumeSession ? [current.resumeSession ?? previous.resumeSession!] : []), + ...repairSessions]); + const repairs = (previous.repairs ?? 0) + (current.repairs ?? 0); + const repair = current.repair ? { + ...current.repair, + limit: Math.max(previous.repair?.limit ?? 0, + (previous.repairs ?? 0) + current.repair.limit), + used: repairs, + } : previous.repair; + const merged: RunLevelRecord = { + ...previous, + ...current, + buildSessions, + buildCostUsd: addCostUsd(previous.buildCostUsd, current.buildCostUsd), + repairSessions, + repairCostUsd: addCostUsd(previous.repairCostUsd, current.repairCostUsd), + repairHistory: [...(previous.repairHistory ?? []), ...(current.repairHistory ?? [])], + repairs, + repair, + sessionTotals, + tokens: sessionTotals.tokens, + usage: sessionTotals.usage, + turns: sessionTotals.turns, + promptBytes: sessionTotals.promptBytes, + tokensPerTurn: sessionTotals.turns + ? Math.round(sessionTotals.tokens / sessionTotals.turns) : null, + thinking: sessionTotals.thinking, + costUsd: addCostUsd(previous.costUsd, current.costUsd), + durationSec: (previous.durationSec ?? 0) + (current.durationSec ?? 0), + }; + return merged; +} + +function mutationControlArgs(args: BenchArgs): MutationControlArgs { + if (!args.out || !args.mutations || !args.backend || !args.parentAttemptId) { + throw new Error('mutation control has incomplete run identity'); + } + return { levelList: args.levelList, out: args.out, recipe: args.recipe, + recipeTasks: args.recipeTasks, mutations: args.mutations, backend: args.backend, + track: args.track, runIndex: args.runIndex, parentAttemptId: args.parentAttemptId, + mutationShardIndex: args.mutationShardIndex, mutationShardCount: args.mutationShardCount, + mutationResumeFrom: args.mutationResumeFrom, mutationCheckpointOut: args.mutationCheckpointOut, + mutationBaselineBundle: args.mutationBaselineBundle, + expectedMutationCalibration: args.expectedMutationCalibration, + mutationMaxRuntimeMinutes: args.mutationMaxRuntimeMinutes, + mutationImageId: args.mutationImageId }; +} + +function mutationOutcome(value: unknown): RunOutcome | null { + if (value === null || value === undefined) return null; + if (!object(value)) { + throw new Error('mutation control artifact outcome is invalid'); + } + return { kind: runOutcomeKind(value.kind), + ...(typeof value.phase === 'string' ? { phase: value.phase } : {}), + ...(typeof value.reason === 'string' ? { reason: value.reason } : {}) }; +} + +function recipeRequestIdentity(value: unknown): { recipeSha256: string; selectionSha256: string; + taskPacks: unknown; taskSha256: string } { + if (!object(value) || !object(value.recipe) || !object(value.selection) || !object(value.task) + || typeof value.recipe.contentSha256 !== 'string' || typeof value.selection.sha256 !== 'string' + || typeof value.task.sha256 !== 'string') { + throw new Error('recipe task request has no complete identity'); + } + return { recipeSha256: value.recipe.contentSha256, selectionSha256: value.selection.sha256, + taskPacks: value.selection.taskPacks, taskSha256: value.task.sha256 }; +} + +// Match the database and registry endpoints supplied to the coding container. +// A matching port on another host is not owned by this run. +export function runAuditNetworkContext(track: Parameters[0], + args: { backend: string; runIndex: number }, lease: BackendLease): { ownEndpoints: string[]; isolatedLoopback: boolean } { + const ports = portsFor(track, args.backend, args.runIndex); + const databaseUrl = leasedDatabaseEnvironment(STACK_ADAPTER_REGISTRY.get(args.backend), { + database: lease.resources.database, networkMode: lease.resources.buildContainer?.networkMode, + lease, + }).DATABASE_URL; + const urls = [ports.vite, ports.express].filter((port): port is number => port !== null) + .map(port => `http://127.0.0.1:${port}`); + urls.push(...[databaseUrl, lease.resources.serverUri, packageRegistryEnvironment(packageRegistry(), + lease.resources.buildContainer?.networkMode, lease.resources.network).NPM_CONFIG_REGISTRY].filter((url): url is string => !!url)); + // Only endpoint authority leaves this process. Database credentials stay private. + // A loopback endpoint is also owned at each of the attempt's own network addresses, + // which is how a coding agent reaches its own application by bridge address. + const ownAddresses = lease.resources.network?.ownAddresses ?? []; + const ownEndpoints = [...new Set(urls.flatMap(value => { + const url = new URL(value); + const port = url.port || (url.protocol === 'https:' ? '443' : '80'); + const hosts = /^(?:127\.0\.0\.1|localhost|0\.0\.0\.0)$/.test(url.hostname) + ? [url.hostname, ...ownAddresses] : [url.hostname]; + return hosts.map(host => `${host}:${port}`); + }))]; + // The authenticated lease records the inspected coding container's namespace. + // A transcript's cwd or a Docker bridge alone does not prove isolation. + const network = lease.resources.network; + const build = lease.resources.buildContainer; + const isolatedLoopback = !!(network?.namespaceContainerId + && lease.resources.container?.owned && lease.resources.container.id === network.namespaceContainerId + && network.firewallSha256 && network.firewallInstalledAt + && build?.owned && build.networkMode === `container:${network.namespaceContainerId}`); + return { ownEndpoints, isolatedLoopback }; +} + +// Check contamination after every coding session. File-tool permissions do not +// govern shell reads, so the transcript audit remains a separate hard gate. +function auditContamination(appDir: string, network: ReturnType, + expectTranscripts: boolean): ContaminationAudit | null { + // A non-billable adapter runs no provider session and leaves no transcript; + // there is nothing to audit and nothing that could have been read. + if (!expectTranscripts) return null; + const args = [join(ROOT, 'dist', 'commands', 'leak-audit.js'), '--app', appDir, '--json', + '--own-endpoints', network.ownEndpoints.join(','), ...(network.isolatedLoopback ? ['--isolated-loopback'] : [])]; + let firstFailure: unknown = null; + for (let attempt = 1; attempt <= 2; attempt++) { + try { + const audit = sh('node', args, { stdio: 'pipe' }); + const entries = parseLeakAudit(audit); + if (entries.length === 0) { + return { kind: 'harness_failure', + evidence: ['no session transcript was found to audit'], + verdict: 'SCORES NOT USABLE — nothing verified this build stayed inside its directory.' }; + } + const escapes = entries.flatMap(entry => entry.hits); + const serious = escapes.filter(h => /GRADER|CONTRACT|BENCHMARK NOTES|PROMPTS|NETWORK|STACKBENCH SOURCE/.test(h.kind)); + if (firstFailure) { + console.error(` warning: contamination audit passed on retry after: ${auditFailureSummary(firstFailure)}`); + } + if (!serious.length) return null; + return { kind: 'contaminated', + evidence: [...new Set(serious.map(h => `${h.kind}: ${h.path.split('/').slice(-2).join('/')}`))].slice(0, 8), + verdict: 'SCORES NOT USABLE — the audit detected restricted file or network access attempts; see the evidence categories.' }; + } catch (error) { + firstFailure ??= error; + if (attempt === 2) { + // An audit that could not run is not a pass. Keep the process details so + // the failure can be repaired without another paid reproduction. + return { kind: 'harness_failure', + evidence: [`audit did not run after retry: ${auditFailureSummary(error)}`], + verdict: 'SCORES NOT USABLE — nothing verified this build stayed inside its directory.' }; + } + } + } + return null; +} + +export function auditFailureSummary(error: unknown): string { + const failure = object(error) ? error : {}; + const message = errorMessage(error).split(/\r?\n/)[0] ?? ''; + const stderrLines = String(failure.stderr ?? '').trim().split(/\r?\n/).filter(Boolean); + const stderr = stderrLines.find(line => /(?:error|eacces|permission denied|failed)/i.test(line)) + ?? stderrLines[0]; + const details = [ + Number.isInteger(failure.status) ? `exit ${String(failure.status)}` : null, + failure.signal ? `signal ${String(failure.signal)}` : null, + stderr ? `stderr: ${stderr}` : null, + ].filter(Boolean); + return details.length ? `${message} (${details.join('; ')})` : message; +} + +const sh = (cmd: string, args: readonly string[], + opts: Omit = {}): string => + execFileSync(cmd, [...args], { + encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, timeout: COMMAND_TIMEOUT_MS, ...opts, + }); + +let activeProcessCancellation: AbortController | null = null; +// Set once a run owns resources. The top-level rejection handler invokes this +// directly; relying only on process 'exit' made cleanup best-effort precisely +// when an awaited build rejected unexpectedly. +let emergencyTeardown: (() => void) | null = null; + +export function parseAgentProcessResult(stdout: string, stderr: string, processError: unknown, + request: AgentRequest): ValidatedAgentResult { + const resultLine = stdout.trim().split('\n').pop(); + let result: ValidatedAgentResult; + try { + if (!resultLine) throw new Error('agent returned no result line'); + result = validateAgentResult(JSON.parse(resultLine), request); + } catch (resultError) { + const stdoutTail = stdout.trim().slice(-2000) || ''; + const stderrTail = stderr.trim().slice(-4000) || ''; + const processDetail = processError ? `agent process failed: ${errorMessage(processError)}\n` : ''; + throw new Error(`${processDetail}agent returned an invalid result: ${errorMessage(resultError)}\n` + + `agent stdout tail:\n${stdoutTail}\nagent stderr tail:\n${stderrTail}`); + } + if (processError && result.ok) { + throw new Error(`agent process failed after reporting success: ${errorMessage(processError)}`); + } + return result; +} + +export async function runAgent( + args: BenchArgs, + adapter: AgentAdapter, + mode: AgentMode, + level: number, + appDir: string, +): Promise { + if (!args.backend || !args.model) throw new Error('agent run requires backend and model'); + const remainingBudget = args.maxBudgetUsd == null ? null + : addCostUsd(args.maxBudgetUsd, -(args.spentBudgetUsd ?? 0)); + if (remainingBudget !== null && remainingBudget <= 0) { + throw new Error(`Cost cap reached: attempt cost cap of ${args.maxBudgetUsd} was exhausted before ${mode} L${level}`); + } + if (remainingBudget !== null && adapter.costLimit === 'unsupported') { + throw new Error(`agent adapter ${adapter.id} cannot enforce --max-budget-usd`); + } + const recipeTask = args.recipeTasks.get(level)?.agentRequest + ?? args.recipeTasks.get(level)?.request ?? null; + const request: AgentRequest = { mode, level, app: appDir, backend: args.backend, track: args.track, + runIndex: args.runIndex, model: args.model, guidance: args.guidance, skills: args.skills, + productionQuality: args.productionQuality, + ...(adapter.usesStackSkills + ? { skillIdentity: args.condition?.guidance.skills[args.backend] } : {}), + recipe: agentRecipeIdentity(args.recipe, recipeTask), + guidanceDocument: args.guidanceDocument, + credentialAliases: args.condition?.guidance?.credentialAliases ?? {}, + recipeTask, pricing: args.pricing, providerRoute: args.providerRoute, maxOutputTokens: args.maxOutputTokens, + maxBudgetUsd: remainingBudget, adapterCostLimit: adapter.costLimit }; + const argv = agentRequestArgv(adapter, request); + if (args.apiKey && !adapter.apiKeyEnvironmentVariable) { + throw new Error(`agent adapter ${adapter.id} does not accept an API key`); + } + const env = { ...process.env }; + if (args.apiKeyFile) env.STACK_BENCH_AGENT_API_KEY_FILE = args.apiKeyFile; + if (args.apiKey) { + const credentialName = adapter.apiKeyEnvironmentVariable; + if (!credentialName) throw new Error(`agent adapter ${adapter.id} does not accept an API key`); + env[credentialName] = args.apiKey; + } + const supervised = campaignProviderContinuationContext(env) !== null; + const capture = mkdtempSync(join(dirname(appDir), '.agent-output-')); + const cancellation = new AbortController(); + activeProcessCancellation = cancellation; + try { + const processResult = await runBounded(process.execPath, argv, { + env, stdio: 'ignore', timeoutMs: supervised ? null : adapter.deadlineMs, + signal: cancellation.signal, + logs: { stdout: join(capture, 'stdout'), stderr: join(capture, 'stderr'), maxBytes: 64 * 1024 * 1024 }, + }); + const processError = processResult.error ?? (processResult.timedOut + ? new Error(`agent deadline exceeded after ${adapter.deadlineMs} ms`) + : processResult.cancelled ? new Error('agent process cancelled') + : !processResult.ok ? new Error(`agent exited ${processResult.code ?? processResult.signal}`) : null); + if (Object.values(processResult.logs ?? {}).some(log => log.truncated)) { + throw new Error('agent output exceeded the 64 MiB capture limit'); + } + const result = parseAgentProcessResult(readFileSync(join(capture, 'stdout'), 'utf8'), + readFileSync(join(capture, 'stderr'), 'utf8'), processError, request); + args.spentBudgetUsd = addCostUsd(args.spentBudgetUsd, result.costUsd); + return result; + } finally { + if (activeProcessCancellation === cancellation) activeProcessCancellation = null; + rmSync(capture, { recursive: true, force: true }); + } +} + +interface GradeCheck { + stableKey: string; + executionId?: string; + source?: string; + packId?: string; +} + +interface GradeRecipeTask { + request: UnknownRecord; + selection: { checks: readonly GradeCheck[] } + | { scoredChecks: readonly GradeCheck[]; observedChecks?: readonly GradeCheck[] }; +} + +/** The aliases grading expects: the run's own when set, else the condition's. */ +function gradingCredentialAliases(args: GradeArguments): Record | undefined { + return args.gradingCredentialAliases ?? args.condition?.guidance?.credentialAliases; +} + +function checksForGrade(task: GradeRecipeTask | undefined, observation: GradeOptions['observation']): + readonly GradeCheck[] { + if (!task) return []; + if ('scoredChecks' in task.selection) { + return observation === 'observed' + ? task.selection.observedChecks ?? [] : task.selection.scoredChecks; + } + return task.selection.checks; +} + +type GradeArguments = Pick & { + agentAdapter?: BenchArgs['agentAdapter']; + recipeTasks?: ReadonlyMap; + progression?: { identity: { policy?: string } }; + condition?: { guidance?: { credentialAliases?: Record } }; + gradingCredentialAliases?: Record; +}; + +export function gradeArgv( + args: GradeArguments, + appDir: string, + url: string, + label: string, + level: number, + track: Track, + parentAttemptId: string, + options: GradeOptions = {}, +): string[] { + const { observation = 'scored', out = null, sourceSha256 = null, + applicationFailure = null } = options; + if (!args.backend) throw new Error('grading requires a backend'); + const restartSpec = restartSpecFor(args, appDir, track); + const task = options.recipeTask ?? args.recipeTasks?.get(level); + return [compiledEntrypoint('commands', 'run-suite.js'), '--app', appDir, '--url', url, + '--backend', args.backend, '--label', label, '--level', String(level), + '--track', args.track, + '--run-index', String(args.runIndex), + '--parent-attempt-id', parentAttemptId, + '--observation', observation, + '--out', privateGradingDirectory(appDir, out), + ...(sourceSha256 ? ['--source-sha256', sourceSha256] : []), + ...(args.recipe ? ['--recipe', args.recipe] : []), + ...(task ? ['--recipe-task-json', JSON.stringify(task.request)] : []), + ...(gradingCredentialAliases(args) + ? ['--credential-aliases-json', JSON.stringify(gradingCredentialAliases(args))] : []), + ...(!applicationFailure && args.agentAdapter !== 'reference-fixture' + && STACK_ADAPTER_REGISTRY.get(args.backend).runPolicy.resetEnabled + ? ['--retry-inconclusive'] : []), + ...(applicationFailure + ? ['--application-failure-json', JSON.stringify(applicationFailure)] : []), + ...(observation === 'scored' && args.recipeTasks && !args.progression + ? ['--regression-checks-json', JSON.stringify([...args.recipeTasks.entries()] + .filter(([priorLevel]) => priorLevel < level) + .flatMap(([, priorTask]) => checksForGrade(priorTask, 'scored') + .map(check => check.stableKey)))] : []), + ...(args.media && observation === 'scored' ? [] : ['--no-media']), + ...(!STACK_ADAPTER_REGISTRY.get(args.backend).runPolicy.resetEnabled + ? ['--no-reset'] + : ['--restart-spec', JSON.stringify(restartSpec)])]; +} + +export function archiveCandidateGrade(appDir: string, outputDir: string, label: string): void { + const gradingDirectory = privateGradingDirectory(appDir); + if (!existsSync(gradingDirectory)) return; + cpSync(gradingDirectory, privateGradingDirectory(appDir, join(outputDir, 'candidate-grades', label)), { + recursive: true, + filter: source => !/[\\/]media([\\/]|$)/.test(source), + }); +} + +async function grade( + args: BenchArgs, + appDir: string, + url: string, + label: string, + level: number, + track: Track, + parentAttemptId: string, + options: GradeOptions = {}, +): Promise { + const out = privateGradingDirectory(appDir, options.out); + const source = hashAppSource(appDir); + const argv = gradeArgv(args, appDir, url, label, level, track, parentAttemptId, { + ...options, sourceSha256: options.sourceSha256 ?? source.sha256, + }); + const bundle = join(out, ARTIFACT_FILE.gradeBundle); + rmSync(bundle, { force: true }); + const task = options.recipeTask ?? args.recipeTasks?.get(level); + const currentChecks = checksForGrade(task, options.observation); + const regressionChecks = options.observation === 'observed' || args.progression + ? [] + : [...(args.recipeTasks?.entries() ?? [])] + .filter(([priorLevel]) => priorLevel < level) + .flatMap(([, priorTask]) => checksForGrade(priorTask, 'scored')); + const sourceCount = task + ? selectedGradingSourceCount(currentChecks, regressionChecks) + : suitesFor(track, level).length; + // A current bundle may still explain a scored failure, so the process result is not consulted. + // Suites wait for host grading slots inside the grader; those waits do not count against its time. + const receipt = process.env[GRADING_WAIT_RECEIPT_ENV]; + const startedAt = Date.now(); + const cancellation = new AbortController(); + activeProcessCancellation = cancellation; + try { + await runBounded(process.execPath, argv, { stdio: 'inherit', startedAt, signal: cancellation.signal, + timeoutMs: gradingRunTimeoutMs(sourceCount, args.recipeBindings.get(level)?.plan.packs ?? [], + [...currentChecks, ...regressionChecks]), + ...(receipt ? { pauseIntervals: () => readGradingWaits(receipt).filter(wait => wait.startedAt >= startedAt) } : {}) }); + } finally { + if (activeProcessCancellation === cancellation) activeProcessCancellation = null; + } + return existsSync(bundle) + ? readArtifactPayload(bundle, { expectedKind: 'grade_bundle' }) : null; +} + +function restartSpecFor(args: Pick, + appDir: string, track: Track): RuntimeControlSpec { + if (!args.backend) throw new Error('restart specification requires a backend'); + const port = portsFor(track, args.backend, args.runIndex).vite ?? null; + if (port == null) throw new Error(`stack ${args.backend} has no application port`); + return { backend: args.backend, app: appDir, port: Number(port), probe: '' }; +} + +function runMutationControl( + args: BenchArgs, + appDir: string, + url: string, + track: Track, + imageId: string | null, +): MutationControlResult { + if (!args.out || !args.mutations) throw new Error('mutation control requires output and manifest paths'); + const output = join(args.out, ARTIFACT_FILE.mutationControl); + if (!args.mutationResumeFrom || resolve(args.mutationResumeFrom) !== resolve(output)) { + rmSync(output, { force: true }); + } + if (imageId) args.mutationImageId = imageId; + else delete args.mutationImageId; + const argv = mutationControlArgv(mutationControlArgs(args), appDir, url, track); + let processError = null; + try { sh(process.execPath, argv, { + stdio: 'inherit', timeout: mutationControlTimeoutMs(args.mutationMaxRuntimeMinutes), + }); } + catch (error) { processError = errorMessage(error).split('\n')[0] ?? null; } + if (!existsSync(output)) { + return { ok: false, artifact: output, processError, + outcome: { kind: 'harness_failure', phase: 'mutation-control', + reason: processError ?? 'mutation runner produced no artifact' } }; + } + const artifact = readArtifactPayload(output, { expectedKind: 'mutation_control' }); + return { ok: artifact.ok === true && !processError, artifact: output, + processError, summary: artifact.summary ?? null, outcome: mutationOutcome(artifact.outcome), + results: artifact.results ?? [] }; +} + +function validateMutationInput(args: BenchArgs): void { + if (!args.mutations) return; + if (!args.app) throw new Error('--mutations requires an explicit pristine --app'); + const manifest = JSON.parse(readFileSync(args.mutations, 'utf8')); + if (!/^[a-f0-9]{64}$/.test(manifest.fixtureSha256 ?? '')) { + throw new Error('mutation manifest has no valid fixtureSha256'); + } + const fixture = hashDirectory(args.app); + if (fixture.sha256 !== manifest.fixtureSha256) { + throw new Error(`mutation manifest targets fixture ${manifest.fixtureSha256}, not ${fixture.sha256}`); + } +} + +export function inspectGradeSource(directory: string, + options: { level?: number; checkKeys?: string[] } = {}) { + const root = realpathSync(directory); + const runPath = join(root, ARTIFACT_FILE.run); + const parent = readArtifact(runPath, { expectedKind: 'benchmark_run' }); + const run = parent.payload; + if (!object(run.mode) + || !['sequential', 'dependency'].includes(String(run.mode.id)) || run.contaminated) { + throw new Error('--grade-from requires an uncontaminated sequential or dependency run'); + } + const dependency = run.mode.id === 'dependency'; + if (!parent.timestamps.completedAt) { + const recoveryPath = join(root, ARTIFACT_FILE.recovery); + if (!dependency || !existsSync(recoveryPath)) { + throw new Error('unfinished --grade-from requires a recovered dependency run and a completed candidate grade'); + } + const recovery = readArtifact<{ runId: string; backend: string; status: string; + cleanup: { succeeded: boolean; retained: boolean } }>(recoveryPath, { expectedKind: 'recovery' }); + if (recovery.attempt.parentId !== parent.id || recovery.payload.runId !== parent.id + || recovery.payload.backend !== run.backend || recovery.payload.status !== 'clean' + || recovery.payload.cleanup?.succeeded !== true || recovery.payload.cleanup.retained !== false) { + throw new Error('unfinished --grade-from recovery does not prove cleanup of its parent'); + } + } + if (dependency && (!Number.isSafeInteger(options.level) || options.level! < 1)) { + throw new Error('dependency --grade-from requires a positive --grade-level'); + } + if (dependency && !options.checkKeys?.length) throw new Error('dependency --grade-from requires explicit --check keys'); + if (!dependency && (options.level !== undefined || run.levels.length !== 1)) { + throw new Error('sequential --grade-from requires a single level and does not accept --grade-level'); + } + const levels = dependency ? run.levels.filter(item => item.level === options.level) : run.levels; + if (levels.length !== 1 || (dependency && run.mode.workSelection === 'feature')) { + throw new Error('saved replay requires exactly one unambiguous first-build candidate at the selected depth'); + } + const level = levels[0]!; + if (!parent.identities.agentAdapter?.id || !Number.isSafeInteger(run.backendLease?.runIndex) + || run.backendLease.runIndex < 0 || !run.runtime?.buildImage) { + throw new Error('saved run lacks its agent, runtime image, or run index'); + } + const serverUri = STACK_ADAPTER_REGISTRY.get(run.backend).orchestrator.serverUriVariable + ? loopbackHttpUri(run.backendLease.resources?.serverUri).origin : null; + const checkpoint = level.checkpoint; + const condition = run.condition as BenchArguments['condition']; + let declared = condition?.requested?.levels?.find(item => item.level === level.level); + if ((!dependency && !checkpoint) || !declared || declared.selection.schemaVersion !== 3 + || !declared.task.contractSha256 || !declared.task.requirementSha256 + || !Array.isArray(declared.selection.requested.features) + || !Array.isArray(declared.selection.scoredChecks)) { + throw new Error('saved run lacks a source checkpoint or a bound modular grading scope'); + } + const planned = new Set(declared.selection.scoredChecks.map(check => check.stableKey)); + const child = (name: string): string => { + const path = realpathSync(resolve(root, name)); + const rel = relative(root, path); + if (!rel || rel === '..' || rel.startsWith(`..${sep}`) || isAbsolute(rel)) throw new Error('saved checkpoint path escapes its execution'); + return path; + }; + const sourceDirectory = dependency ? `first-build-l${level.level}` : checkpoint!.directory; + const sourcePath = child(sourceDirectory); + assertPlainAppSourceTree(sourcePath); + const source = hashAppSource(sourcePath); + let candidate = null; + if (dependency) { + const gradePath = `first-build-l${level.level}-grading/bundle.json`; + const gradeFile = child(gradePath); + const grade = readArtifact(gradeFile, { expectedKind: 'grade_bundle' }); + const expected = level.firstBuild?.source; + if (!object(expected) || source.sha256 !== expected.sha256 || source.files.length !== expected.files + || grade.attempt.parentId !== parent.id || grade.payload.source?.sha256 !== source.sha256 + || grade.payload.backend !== run.backend || grade.payload.track !== run.track + || grade.payload.level !== level.level || grade.payload.observation !== 'scored' + || canonicalDefinitionJson(grade.identities.engine) !== canonicalDefinitionJson(parent.identities.engine) + || grade.identities.recipe?.id !== declared.recipe.id + || grade.identities.recipe?.sha256 !== declared.recipe.contentSha256 + || grade.payload.selection?.schemaVersion !== 3 + || grade.payload.selection.sha256 !== level.selection?.sha256 + // Inconclusive progression summaries retain only the selection hash. + || (Object.keys(level.selection).length !== 1 + && canonicalDefinitionJson(grade.payload.selection) !== canonicalDefinitionJson(level.selection)) + || !Array.isArray(grade.payload.selection.requested?.features) + || !Array.isArray(grade.payload.selection.scoredChecks)) { + throw new Error('saved first-build source or grade does not match its parent run and candidate scope'); + } + declared = { ...declared, selection: grade.payload.selection }; + candidate = { kind: 'first-build', directory: sourceDirectory, level: level.level, + grade: { path: gradePath, sha256: sha256(readFileSync(gradeFile)), id: grade.id, + identities: grade.identities }, outcome: level.firstBuild?.outcome ?? null }; + } else { + const saved = readArtifact<{ source: { directory: string; sha256: string; files: number }; + backend: string; track: string; level: number; selectionSha256: string }>(child(checkpoint!.artifact), + { expectedKind: 'source_checkpoint' }); + if (saved.attempt.parentId !== parent.id || saved.payload.backend !== run.backend + || saved.payload.track !== run.track || saved.payload.level !== level.level + || saved.payload.source.directory !== checkpoint!.directory + || saved.payload.source.sha256 !== checkpoint!.sha256 || saved.payload.source.files !== checkpoint!.files + || saved.payload.selectionSha256 !== declared.selection.sha256 + || source.sha256 !== checkpoint!.sha256 || source.files.length !== checkpoint!.files) { + throw new Error('saved source or checkpoint does not match its parent run'); + } + } + // A dependency replay may also run a check added since the run was planned. Resolving the + // current definition from the saved requested features still refuses checks outside that task. + if (options.checkKeys?.some(key => !declared.selection.scoredChecks!.some(check => check.stableKey === key) + && (!dependency || planned.has(key)))) { + throw new Error('regrade checks must belong to the original scored scope'); + } + const aliases = condition?.guidance.credentialAliases; + if (!aliases || Object.values(aliases).some(value => typeof value !== 'string')) { + throw new Error('saved run lacks valid grading credential aliases'); + } + return { root, parent, declared, dependency, candidate, serverUri, sourcePath, source: { sha256: source.sha256, files: source.files.length }, + runSha256: sha256(readFileSync(runPath)), aliases }; +} + +export function pendingRunSnapshot(run: BenchmarkRunRecord, pending: RunLevelRecord | null, + costComplete: boolean): BenchmarkRunRecord { + const snapshot = { ...run, levels: [...run.levels] }; + if (pending) { + const index = snapshot.levels.findIndex(level => level.level === pending.level); + const record = mergeFeatureLevelRecord(index < 0 ? null : snapshot.levels[index]!, pending); + if (index < 0) snapshot.levels.push(record); + else snapshot.levels[index] = record; + if (!['harness_failure', 'provider_failure'].includes(run.outcome?.kind ?? '')) snapshot.outcome = { kind: 'ungraded', phase: 'interrupted-level', + reason: 'level has not completed', appFailures: [], inconclusive: [], harnessFailures: [] }; + } + finalizeRunTotals(snapshot, Date.parse(run.startedAt), { costComplete }); + return snapshot; +} + +async function main() { + let pendingLevel: RunLevelRecord | null = null; + let sessionInFlight = false; + let runCostComplete = true; + const persistRun = (path: string, value: unknown) => { + const receipt = process.env[GRADING_WAIT_RECEIPT_ENV]; + const gradingWaitMs = receipt ? recordedGradingWaitMs(receipt) : 0; + if (gradingWaitMs > 0) recordGradingWait(value as BenchmarkRunRecord, gradingWaitMs); + if (pendingLevel || sessionInFlight) { + return writeRunJson(path, pendingRunSnapshot(value as BenchmarkRunRecord, + pendingLevel, runCostComplete && !sessionInFlight)); + } + return writeRunJson(path, value); + }; + const args: BenchArgs = { + ...parseBenchArguments(process.argv), + recipeTasks: new Map(), + recipeBindings: new Map(), + }; + const regrade = args.gradeFrom ? inspectGradeSource(args.gradeFrom, + { level: args.gradeLevel, checkKeys: args.checkKeys }) : null; + if (regrade) { + const { parent, declared, aliases } = regrade; + const requested = declared.selection.requested; + Object.assign(args, { backend: parent.payload.backend, track: parent.payload.track, + runIndex: parent.payload.backendLease.runIndex, + levels: String(declared.level), levelList: [declared.level], recipe: declared.recipe.id, + agentAdapter: parent.identities.agentAdapter!.id, model: parent.payload.model, + providerRoute: parent.payload.providerRoute, + maxOutputTokens: parent.payload.maxOutputTokens, + featureIds: requested.features, checkKeys: args.checkKeys.length ? args.checkKeys : requested.checks, + requestedSpecifications: requested.specifications?.requested ?? [], + expectedSpecifications: requested.specifications?.expected ?? [], + observedSpecifications: [], gradingCredentialAliases: aliases }); + if (declared.selection.observedChecks?.length) throw new Error('--grade-from does not support observed checks'); + const output = resolve(args.out!); + for (let path = output; dirname(path) !== path; path = dirname(path)) { + if (existsSync(path) && lstatSync(path).isSymbolicLink()) { + throw new Error('regrade output must not pass through a symbolic link'); + } + } + const overlaps = (base: string, target: string) => { + const rel = relative(base, target); + return !rel || (rel !== '..' && !rel.startsWith(`..${sep}`) && !isAbsolute(rel)); + }; + if (overlaps(regrade.root, output) || overlaps(output, regrade.root) + || (existsSync(output) && readdirSync(output).length)) { + throw new Error('--grade-from requires an empty output directory outside the original execution'); + } + if (process.env.STACK_BENCH_IMAGE && process.env.STACK_BENCH_IMAGE !== parent.payload.runtime.buildImage) { + throw new Error('regrade build image differs from the original run'); + } + process.env.STACK_BENCH_IMAGE = parent.payload.runtime.buildImage; + const serverUriVariable = STACK_ADAPTER_REGISTRY.get(parent.payload.backend).orchestrator.serverUriVariable; + if (regrade.serverUri && serverUriVariable) process.env[serverUriVariable] = regrade.serverUri; + } + let repairGrant = null; + if (args.repairFrom) { + const repairLevel = args.repairLevel; + if (typeof repairLevel !== 'number' || !Number.isSafeInteger(repairLevel) || repairLevel < 1) { + throw new Error('--repair-from requires a positive --repair-level'); + } + repairGrant = createRepairGrant(args.repairFrom, + { level: repairLevel, repairs: args.repairs }); + const config = repairGrant.configuration; + if (config.buildImage && process.env.STACK_BENCH_IMAGE + && config.buildImage !== process.env.STACK_BENCH_IMAGE) { + throw new Error('repair continuation build image differs from its parent run'); + } + if (config.buildImage) process.env.STACK_BENCH_IMAGE = config.buildImage; + Object.assign(args, { + backend: config.backend, + track: config.track, + recipe: config.recipe, + levels: String(config.level), + levelList: [config.level], + runIndex: config.runIndex, + agentAdapter: config.agentAdapter, + model: config.model, + providerRoute: config.providerRoute, + maxOutputTokens: config.maxOutputTokens, + guidance: config.guidance, + guidanceDocument: config.guidanceDocument, + productionQuality: config.condition?.productionQuality === true || (!config.condition && config.productionQuality === true), + condition: config.condition, + selectionRequest: campaignSelection(config.selectionRequest, 'repair configuration.selectionRequest'), + skills: config.skills, + packIds: [...(campaignSelection(config.selectionRequest, + 'repair configuration.selectionRequest').packs ?? [])], + checkKeys: [...(campaignSelection(config.selectionRequest, + 'repair configuration.selectionRequest').checks ?? [])], + featureIds: [], + requestedSpecifications: [], + expectedSpecifications: [], + observedSpecifications: [], + seedFrom: repairGrant.sourcePath, + url: config.url, + parentAttemptId: repairGrant.parent.id, + repairGrant, + }); + } + if (!args.backend) throw new Error('benchmark run requires a backend'); + const stackAdapter = STACK_ADAPTER_REGISTRY.get(args.backend); + const materializeCodingOutput = stackAdapter.id !== 'stub'; + const agentAdapter = AGENT_ADAPTER_REGISTRY.get(args.agentAdapter); + args.providerRoute = validateProviderRoute(agentAdapter.provider, args.providerRoute); + args.maxOutputTokens = validateProviderOutputLimit(agentAdapter.provider, args.maxOutputTokens); + // Credential aliases keep the fixture passwords out of an agent's prompt, + // so grading expects the aliases. A reference fixture is the fixture itself, + // seeded with the real credentials, and is graded with them. + if (!regrade && agentAdapter.gradesWithFixtureCredentials) args.gradingCredentialAliases = {}; + if (process.env.STACK_BENCH_APPLIANCE !== '1' && agentAdapter.costLimit !== 'non-billable') { + throw new Error(`agent adapter ${agentAdapter.id} requires the Docker appliance`); + } + if (repairGrant) { + const currentAgent = agentAdapterIdentity(agentAdapter); + const parentAgent = repairGrant.parentArtifact.identities.agentAdapter; + if (currentAgent.id !== parentAgent?.id || currentAgent.version !== parentAgent?.version + || currentAgent.sha256 !== parentAgent?.sha256) { + throw new Error('repair continuation agent adapter differs from its parent run'); + } + if (stackAdapter.id !== repairGrant.parentArtifact.identities.stackAdapter?.id + || stackAdapter.version !== repairGrant.parentArtifact.identities.stackAdapter?.version) { + throw new Error('repair continuation stack adapter differs from its parent run'); + } + } + if (!regrade) applyAgentCredential(args, agentAdapter); + args.model ??= agentAdapter.defaultModel; + if (!args.model) throw new Error(`agent adapter ${agentAdapter.id} has no default model`); + if (args.pricing !== undefined) { + args.pricing = validatePricingAuthority(args.pricing, { at: '--pricing-json' }); + } else if (args.maxBudgetUsd != null && agentAdapter.costLimit === 'native') { + const rates = claudeRatesForModel(args.model); + if (!rates) throw new Error(`no default pricing is recorded for model ${args.model}`); + args.pricing = validatePricingAuthority({ unit: PRICING_UNIT, rates }, + { at: 'default pricing' }); + } else { + args.pricing = null; + } + if (args.retainBackend && !stackAdapter.runPolicy.retainHostSupported) { + throw new Error(`stack adapter ${args.backend} does not support --retain-backend`); + } + const stackRuntime = stackAdapter.orchestrator.config( + { root: ROOT, env: process.env, helpers: { exists: existsSync } }); + Object.assign(process.env, stackRuntime.environment); + process.env.STACK_BENCH_NODE_BIN = process.platform === 'win32' ? 'node.exe' : process.execPath; + const track = loadTrack(args.track); + const auditsTranscripts = agentAdapter.costLimit !== 'non-billable'; + // Resolve the requested scope for every level before probing the sandbox, + // acquiring a backend lease or paying for a build. A pack that exists at L2 + // but not L1 is not a late grading surprise; it is an invalid run request. + args.selectionRequest ??= { packs: [...args.packIds], checks: [...args.checkKeys] }; + for (const level of args.levelList) { + const declared = args.condition?.requested?.levels?.find(entry => entry.level === level) ?? null; + const modularSelection = args.selectionRequest.levels?.find(entry => entry.level === level) ?? null; + if (declared?.selection?.schemaVersion === 3) { + const expected = args.featureCatalog + ? { level, recipe: declared.recipe.id } + : { level, recipe: declared.recipe.id, + features: declared.selection.requested.features, + checks: declared.selection.requested.checks }; + if (canonicalDefinitionJson(modularSelection) !== canonicalDefinitionJson(expected)) { + throw new Error(`campaign selection changed before L${level}`); + } + } else if (modularSelection) { + throw new Error(`campaign selection declares modular L${level} without a modular condition`); + } + const declaredRecipe = declared?.recipe.id ?? null; + const binding = resolveRecipeRelease(track, level, declaredRecipe ?? args.recipe); + if (!binding && (args.packIds.length || args.checkKeys.length)) { + throw new Error(`L${level} has no recipe release, so --pack/--check cannot be resolved`); + } + if (binding) { + args.recipeBindings.set(level, binding); + if (args.featureCatalog) { + validateProgressionCampaignLevelScope(binding, args.featureCatalog, declared, level); + } + const requested = declared?.selection?.requested; + const progressionSelection = args.featureCatalog + ? resolveProgressionRecipeLevelSelection(binding, args.featureCatalog, level, + { cumulative: Boolean(args.progression) }) : null; + const resolved = progressionSelection === null + ? createBoundRecipeTaskRequest(binding, requested?.features + ? { featureIds: requested.features, + requestedSpecifications: requested.specifications?.requested, + expectedSpecifications: requested.specifications?.expected, + observedSpecifications: requested.specifications?.observed, + checkKeys: requested.checks } + : regrade?.dependency ? { ...args, taskMode: regrade.declared.task.mode, + dependencyExpansion: regrade.declared.selection.requested.dependencyExpansion } : args) : null; + const grader = progressionSelection?.grader ?? resolved; + if (!grader) throw new Error(`L${level} has no recipe task request`); + if (args.condition && !declared) { + throw new Error(`study condition does not bind requested L${level}`); + } + const graderIdentity = recipeRequestIdentity(grader.request); + if (declared && (declared.recipe.contentSha256 !== graderIdentity.recipeSha256 + || declared.selection.sha256 !== graderIdentity.selectionSha256 + || JSON.stringify(declared.selection.taskPacks) !== JSON.stringify(graderIdentity.taskPacks) + || declared.task.sha256 !== graderIdentity.taskSha256)) { + throw new Error(`study condition requested scope changed before L${level}`); + } + if (progressionSelection) { + const progressionGrader = progressionSelection.grader; + args.recipeTasks.set(level, { + request: progressionGrader.request, + selection: progressionGrader.selection, + task: progressionGrader.task, + agentRequest: progressionSelection.agent.request, + }); + } else if (resolved) { + args.recipeTasks.set(level, { + ...resolved, + agentRequest: createAgentVisibleTaskRequest(binding, resolved), + }); + } + } + } + if (args.progression) { + const state = progressionEngine.initialize(args.progression.definition); + const declared = args.condition?.requested?.levels + ?.find(entry => entry.level === state.level) ?? null; + const binding = resolveRecipeRelease(track, state.level, + declared?.recipe.id ?? null); + if (!binding) throw new Error(`L${state.level} has no recipe release`); + resolveProgressionRecipeAction(binding, state); + if (!args.progressionOwner) { + throw new Error('live dependency progression requires an exact compiled campaign attempt'); + } + } + if (repairGrant) { + const expectedSelection = repairGrant.level.selection?.sha256 ?? null; + const repairTask = args.recipeTasks.get(repairGrant.level.level); + const resolvedSelection = repairTask ? recipeRequestIdentity(repairTask.request).selectionSha256 : null; + if (resolvedSelection !== expectedSelection) { + throw new Error('repair continuation test selection differs from its parent run'); + } + } + if (regrade) { + const task = args.recipeTasks.get(regrade.declared.level)!; + const keys = (checks: readonly { stableKey: string; points: number }[]) => + checks.map(check => `${check.stableKey}:${check.points}`).sort(); + const selected = 'scoredChecks' in task.selection ? task.selection.scoredChecks : task.selection.checks; + if (regrade.dependency) { + if (canonicalDefinitionJson(selected.map(check => check.stableKey).sort()) + !== canonicalDefinitionJson([...new Set(args.checkKeys)].sort())) { + throw new Error('dependency diagnostic changed the explicitly selected check scope'); + } + } else if (task.task.contractSha256 !== regrade.declared.task.contractSha256 + || task.task.requirementSha256 !== regrade.declared.task.requirementSha256 + || canonicalDefinitionJson(keys(selected)) + !== canonicalDefinitionJson(keys(regrade.declared.selection.scoredChecks!.filter(check => + !args.checkKeys.length || args.checkKeys.includes(check.stableKey))))) { + throw new Error('regrade changed the original product contract or scored check scope'); + } + } + if (!args.selectionRequest.levels && (JSON.stringify(args.selectionRequest.packs) !== JSON.stringify(args.packIds) + || JSON.stringify(args.selectionRequest.checks) !== JSON.stringify(args.checkKeys))) { + throw new Error('campaign pack/check selection changed before execution'); + } + // Caller-owned mutation inputs are pure request data. Reject them before + // checking credentials, Docker, ports, or any other ambient runner state so + // an invalid experiment can never be masked by an unrelated preflight error. + validateMutationInput(args); + // The deterministic adapter/stack is the model-free unit loop. Real runs + // prove the exact requested scope, engine, image, credentials, storage and + // ports before any paid coding session begins. + const performPreflight = (smoke = false) => { + const preflight = args.backend === 'stub' ? null : runPreflight({ + backends: [stackAdapter.id], track: args.track, levels: args.levels, + levelList: args.levelList, runIndex: args.runIndex, agentAdapter: args.agentAdapter, + providerRoute: args.providerRoute, + maxOutputTokens: args.maxOutputTokens, + modelFree: regrade !== null, + guidance: args.guidance, + recipe: args.recipe, + ...(args.condition?.requested ? { requestedScopes: [args.condition.requested] } : {}), + ...(args.featureCatalog ? { featureCatalog: args.featureCatalog } : {}), + ...(args.runMode ? { mode: args.runMode } : {}), + agentSkills: args.skills ?? null, + packIds: args.packIds, checkKeys: args.checkKeys, smoke, + ...(process.env.STACK_BENCH_SUPERVISOR_STATE + ? { supervisorState: process.env.STACK_BENCH_SUPERVISOR_STATE } : {}), + image: process.env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE, + resultsDir: resolve(args.out ?? stackBenchResultsRoot(ROOT)), + }, { ownedLease: { path: leasePath, runId }, env: args.apiKey && agentAdapter.apiKeyEnvironmentVariable + ? { ...process.env, [agentAdapter.apiKeyEnvironmentVariable]: '', + [`${agentAdapter.apiKeyEnvironmentVariable}_FILE`]: undefined } + : process.env }); + if (preflight) writeArtifact(join(outputDir, ARTIFACT_FILE.preflight), { + kind: 'preflight', id: `${runId}-preflight`, + attempt: { id: `${runId}-preflight`, parentId: runId }, + identities: emptyArtifactIdentities({ + agentAdapter: agentAdapterIdentity(agentAdapter), + stackAdapter: { id: stackAdapter.id, version: stackAdapter.version }, + }), + payload: preflight, + }); + if (preflight && !preflight.ok) { + const failures = preflight.checks.filter(check => check.status === 'fail'); + console.error('\nPREFLIGHT FAILED — no model session was started.'); + for (const failure of failures) { + console.error(` ${failure.id}: ${failure.summary}`); + if (failure.remediation) console.error(` hint: ${failure.remediation}`); + } + throw new Error('preflight failed; no model session was started'); + } + if (preflight) console.log(` preflight ... ${preflight.summary.passed} checks passed` + + `${preflight.summary.warnings ? `, ${preflight.summary.warnings} warning(s)` : ''}`); + }; + if (process.env.STACK_BENCH_APPLIANCE === '1') { + console.log(' sandbox ... coding container is isolated from the controller and grading files'); + } + const assignedPorts = portsFor(track, args.backend, args.runIndex); + let url = args.url ?? `http://localhost:${assignedPorts.vite}`; + const runDir = resultsName(track, args.backend, args.runIndex); + const runId = newRunId({ track: args.track, backend: args.backend, runIndex: args.runIndex }); + const artifactLabel = `${runDir}-${runId}`; + // Default results never reuse a directory. The stable backend/run name is a + // grouping directory only; every artifact beneath it belongs to one run id. + args.out ??= join(stackBenchResultsRoot(ROOT), runDir, runId); + if (!args.out) throw new Error('benchmark run has no results directory'); + const outputDir = args.out; + mkdirSync(args.out, { recursive: true }); + if (existsSync(join(args.out, ARTIFACT_FILE.run))) { + throw new Error(`refusing to reuse result directory containing ${ARTIFACT_FILE.run}: ${args.out}`); + } + + // Validate caller-owned source before acquiring a backend slot so a bad + // fixture cannot leave leased resources behind. + const ownWorkDir = !args.app; + const appDir = args.app ?? join(workDirFor(track, args.backend, args.runIndex, runId), 'app'); + if (args.app) mkdirSync(appDir, { recursive: true }); + privateGradingDirectory(appDir, join(outputDir, 'grading')); + if (args.repairGrant && url.startsWith('file:')) { + url = pathToFileURL(join(appDir, 'index.html')).href; + } + + // Bind destructive and lifecycle operations to exact resource identities and + // an ownership token. Targets come only from the lease, never generated code. + const runtimeRoot = resolve(process.env.STACK_BENCH_RUNTIME_DIR + ?? join(tmpdir(), 'stack-bench-runtime')); + const runtimeDir = join(runtimeRoot, runId); + const leasePath = join(runtimeDir, ARTIFACT_FILE.backendLease); + const preparedLease = stackAdapter.lease.prepare({ + track, + runIndex: args.runIndex, + runtimeDir, + serverUri: stackRuntime.lease.serverUri, + env: process.env, + helpers: { containerIdentity: runningContainerIdentity, dbName, moduleName }, + }); + const initialLease = createBackendLease({ + runId, + backend: args.backend, + track: args.track, + runIndex: args.runIndex, + ...preparedLease.lease, + }); + const lockScope = resourceLockScope(); + const lockKeys = backendResourceLockKeys(initialLease, assignedPorts, + [...preparedLease.lockKeys, ...(args.app ? [`workspace:${realpathSync(appDir)}`] : [])]); + let privateSupervisorStatePath = null; + try { + if ((args.campaignFile || args.campaignAdmissionId) && args.backend !== 'stub') { + const executionId = process.env.STACK_BENCH_CAMPAIGN_EXECUTION; + if (!executionId || !args.experimentIdentity || !args.campaignAdmissionId + || !borrowCampaignReservation({ env: process.env, + campaignSha256: args.experimentIdentity.sha256, admissionId: args.campaignAdmissionId, + executionId, output: resolve(outputDir), leasePath, lease: initialLease, keys: lockKeys })) { + throw new Error('campaign worker requires private resource delegation'); + } + } else { + await claimBackendResourcesWhenAvailable(leasePath, initialLease, { ...lockScope, keys: lockKeys }); + } + const supervisorState = process.env.STACK_BENCH_SUPERVISOR_STATE + ?? (process.env.STACK_BENCH_SUPERVISOR_DIR + ? join(resolve(process.env.STACK_BENCH_SUPERVISOR_DIR), `${runId}.json`) : null); + if (supervisorState) { + // Private handoff to an outer timeout supervisor. It contains the lease + // token, so create it once with owner-only permissions and never place it + // in the results tree. + privateSupervisorStatePath = resolve(supervisorState); + mkdirSync(dirname(privateSupervisorStatePath), { recursive: true, mode: 0o700 }); + writeFileSync(privateSupervisorStatePath, `${JSON.stringify({ + version: SUPERVISOR_STATE_VERSION, runId, backend: args.backend, runtimeDir, leasePath, + ownershipToken: initialLease.ownershipToken, output: resolve(args.out), + })}\n`, { flag: 'wx', mode: 0o600 }); + } + } catch (error) { + // A refused release keeps the lease: it is the only record of what still runs. + if (existsSync(leasePath) && releaseBackendLease(leasePath, initialLease.ownershipToken) + && !initialLease.campaignDelegation) rmSync(runtimeDir, { recursive: true, force: true }); + throw error; + } + process.env.STACK_BENCH_LEASE = leasePath; + process.env.STACK_BENCH_LEASE_TOKEN = initialLease.ownershipToken; + // Each suite grading takes a host grading slot and records any wait here; a supervisor may name the file. + process.env[GRADING_WAIT_RECEIPT_ENV] ??= join(resolve(args.out), GRADING_WAIT_RECEIPT_FILE); + const auditNetwork = () => auditsTranscripts ? runAuditNetworkContext(track, { backend: stackAdapter.id, runIndex: args.runIndex }, + readBackendLease(leasePath, { token: initialLease.ownershipToken, backend: args.backend, runId })) + : { ownEndpoints: [], isolatedLoopback: false }; + + if (process.platform === 'win32') { + // When Windows resolves `bash` through WSL, WSLENV must carry lease paths + // and tokens into lifecycle scripts with path translation. + const bridge = ['STACK_BENCH_LEASE/p', 'STACK_BENCH_LEASE_TOKEN', + 'STACK_BENCH_NODE_BIN', ...stackRuntime.windowsEnvironmentBridge]; + const existing = (process.env.WSLENV ?? '').split(':').filter(Boolean); + process.env.WSLENV = [...new Set([...existing, ...bridge])].join(':'); + } + + let tornDown = false; + let activeRun: BenchmarkRunRecord | null = null; + const recoveryPath = join(outputDir, ARTIFACT_FILE.recovery); + const writeLeaseEvidence = (knownLease: BackendLease | null = null) => { + const lease = knownLease ?? readBackendLease(leasePath, + { token: initialLease.ownershipToken, backend: args.backend, runId }); + const out = join(outputDir, ARTIFACT_FILE.backendLease); + const evidence = publicBackendLease(lease); + const id = `${runId}-backend-lease`; + writeArtifact(out, { + kind: 'backend_lease_evidence', id, + attempt: { id, parentId: runId }, + timestamps: { startedAt: evidence.createdAt, completedAt: new Date().toISOString() }, + identities: emptyArtifactIdentities({ stackAdapter: { id: args.backend } }), + payload: evidence, + }); + return evidence; + }; + const teardown = ({ reason = null, retainBackend = args.retainBackend }: + { reason?: string | null; retainBackend?: boolean } = {}) => { + if (tornDown) return; + activeProcessCancellation?.abort(); + activeProcessCancellation = null; + // Preserve restart failures before removing the only filesystem that holds + // their stderr. A 500 after restart is otherwise impossible to distinguish + // from an application defect, a dead dependency, or host pressure. + if (activeRun) { + try { + activeRun.backendDiagnostics = captureApplicationDiagnostics(join(outputDir, 'backend.log')); + } catch (error) { + activeRun.backendDiagnostics = { captured: false, + reason: errorMessage(error).split(/\r?\n/)[0] }; + } + } + let released = false; + let cleanupError: unknown = null; + try { + released = releaseBackendLease(leasePath, initialLease.ownershipToken, + { retainBackend }); + } catch (error) { cleanupError = error; } + let finalLease = initialLease; + try { + finalLease = readBackendLease(leasePath, + { token: initialLease.ownershipToken, backend: args.backend, runId }); + } catch (error) { cleanupError ??= error; released = false; } + const evidence = writeLeaseEvidence(finalLease); + writeRecoveryArtifact(recoveryPath, finalLease, { cleanupSucceeded: released, + retained: Boolean(retainBackend), + reason: cleanupError === null ? reason ?? (released ? null : 'authenticated cleanup refused') + : errorMessage(cleanupError) }); + if (activeRun) { + activeRun.backendLease = evidence; + activeRun.outcome ??= aggregateRunOutcome(activeRun.levels); + persistRun(join(outputDir, ARTIFACT_FILE.run), activeRun); + } + tornDown = released; + if (released && !retainBackend && !finalLease.campaignDelegation) { + rmSync(runtimeDir, { recursive: true, force: true }); + if (privateSupervisorStatePath) rmSync(privateSupervisorStatePath, { force: true }); + } + if (cleanupError) throw cleanupError; + if (!released) throw new Error(`backend teardown refused: listener no longer matches lease ${runId}`); + }; + emergencyTeardown = teardown; + + try { + performPreflight(); + stackAdapter.lifecycle.activate({ + leasePath, leaseToken: initialLease.ownershipToken, lease: initialLease, + ports: assignedPorts, app: appDir, + ...stackRuntime.lifecycle, + }); + performPreflight(true); + } catch (error) { + try { teardown({ reason: `backend activation failed: ${errorMessage(error)}`, retainBackend: false }); } + catch (cleanupError) { + console.error(` activation cleanup quarantined: ${errorMessage(cleanupError).split(/\r?\n/)[0]}`); + } + throw error; + } + + // Teardown stops only resources recorded in this run's lease. + const interrupt = (signal: NodeJS.Signals, exitCode: number) => { + console.log(`interrupted by ${signal} — stopping exact owned resources`); + try { teardown({ reason: `interrupted by ${signal}` }); } + catch (error) { console.error(` cleanup quarantined: ${errorMessage(error).split(/\r?\n/)[0]}`); } + process.exit(exitCode); + }; + process.on('SIGINT', () => interrupt('SIGINT', 130)); + process.on('SIGTERM', () => interrupt('SIGTERM', 143)); + process.on('exit', () => { + if (!tornDown) { + try { teardown(); } catch (error) { + console.error(` cleanup failed: ${errorMessage(error).split('\n')[0]}`); + } + } + }); + + // Seed source only; the upgrade session installs its own dependencies. + if (args.seedFrom) { + const from = resolve(args.seedFrom); + if (!existsSync(from)) { console.error(`--seed-from path does not exist: ${from}`); process.exit(2); } + seedAppSource(from, appDir); + if (args.progressionSeed) { + const seeded = hashAppSource(appDir); + if (seeded.sha256 !== args.progressionSeed.sourceSha256 + || seeded.files.length !== args.progressionSeed.sourceFiles) { + throw new Error('extension source does not match its recorded identity'); + } + } + console.log(args.repairGrant + ? ` restored L${args.levelList[0]} checkpoint from ${from} for a bounded repair continuation` + : ` seeded from ${from} — level ${args.levelList[0]} will UPGRADE it, not rebuild`); + } + + if (regrade) { + const startedAt = new Date().toISOString(); + let bundle: GradeBundlePayload | null = null; + let failure: string | null = null; + let diagnostics: unknown = null; + let cleanupFailure: unknown = null; + try { + seedAppSource(regrade.sourcePath, appDir); + // prepare-only owns the container but never invokes an agent or broker. + writeFileSync(join(dirname(appDir), '.stack-bench-isolation'), 'container'); + writeFileSync(join(dirname(appDir), '.stack-bench-backend'), args.backend); + sh(process.execPath, [compiledEntrypoint('container', 'run-build.js'), '--app', appDir, + '--backend', args.backend, '--image', process.env.STACK_BENCH_IMAGE!, + '--ports', [assignedPorts.vite, assignedPorts.express].filter(Boolean).join(','), '--prepare-only'], + { stdio: 'inherit', timeout: COMMAND_TIMEOUT_MS }); + const copiedSource = hashAppSource(appDir); + if (copiedSource.sha256 !== regrade.source.sha256 || copiedSource.files.length !== regrade.source.files) { + throw new Error('regrade source changed during preparation'); + } + await materializeAcceptedSource(regrade.sourcePath, appDir, restartSpecFor(args, appDir, track)); + bundle = await grade(args, appDir, url, `${args.backend}-regrade`, regrade.declared.level, + track, runId, { out: join(outputDir, 'grading'), sourceSha256: regrade.source.sha256 }); + if (!bundle || bundle.source?.sha256 !== regrade.source.sha256) { + throw new Error('regrade produced no matching source-bound grade bundle'); + } + if (hashAppSource(appDir).sha256 !== regrade.source.sha256) { + throw new Error('application source changed during regrading'); + } + } catch (error) { + failure = errorMessage(error); + if (object(error) && typeof error.startLog === 'string') { + writeFileSync(join(outputDir, 'application-start.log'), error.startLog); + } + throw error; + } finally { + try { diagnostics = captureApplicationDiagnostics(join(outputDir, 'backend.log')); } + catch (error) { diagnostics = { captured: false, reason: errorMessage(error) }; } + try { teardown({ reason: failure, retainBackend: false }); } + catch (error) { failure ??= errorMessage(error); cleanupFailure = error; } + finally { + writeFileSync(join(outputDir, 'regrade.json'), JSON.stringify({ schemaVersion: 1, + kind: 'saved-source-regrade', diagnosticOnly: true, + id: runId, startedAt, completedAt: new Date().toISOString(), + parent: { id: regrade.parent.id, runSha256: regrade.runSha256, + completedAt: regrade.parent.timestamps.completedAt, + engine: regrade.parent.identities.engine, source: regrade.source, + serverUri: regrade.serverUri, + selectionSha256: regrade.declared.selection.sha256, + recipe: regrade.declared.recipe, task: regrade.declared.task, + mode: regrade.parent.payload.mode, outcome: regrade.parent.payload.outcome, + candidate: regrade.candidate }, + engine: currentEngineIdentity(), buildImage: process.env.STACK_BENCH_IMAGE, + controllerImage: process.env.STACK_BENCH_CONTROLLER_IMAGE_ID, + dependencyVolume: process.env.STACK_BENCH_RELEASE_DEPS_VOLUME, + backend: args.backend, track: args.track, level: regrade.declared.level, + recipe: args.recipeTasks.get(regrade.declared.level)!.request.recipe, + task: args.recipeTasks.get(regrade.declared.level)!.request.task, + selectionSha256: args.recipeTasks.get(regrade.declared.level)!.selection.sha256, + selectedChecks: checksForGrade(args.recipeTasks.get(regrade.declared.level), 'scored') + .map(check => check.stableKey), + grading: bundle ? 'grading/bundle.json' : null, + gradingSha256: bundle ? sha256(readFileSync(join(outputDir, 'grading', 'bundle.json'))) : null, + outcome: bundle ? classifyBundle(bundle) : null, failure, diagnostics, + cleanupSucceeded: tornDown, + modelCalls: 0, additionalModelCostUsd: 0, + comparisonNote: regrade.dependency + ? 'Dependency first-build diagnostic; not terminal completion, a new build, or a replacement for the original outcome or spend.' + : 'Regrades the original saved app; not an independent build or a replacement for its model cost.' }, null, 2) + '\n', + { flag: 'wx' }); + if (tornDown && ownWorkDir) rmSync(dirname(appDir), { recursive: true, force: true }); + } + } + if (cleanupFailure) throw cleanupFailure; + console.log(`Saved-source regrade: ${classifyBundle(bundle).kind}; evidence ${outputDir}`); + process.exitCode = runExitCode(classifyBundle(bundle)); + return; + } + const started = Date.now(); + const run: BenchmarkRunRecord = { id: runId, + ...(args.repairGrant ? { kind: 'repair_continuation', + continuation: structuredClone(args.repairGrant.grant) } : {}), + startedAt: new Date(started).toISOString(), + parentAttemptId: args.parentAttemptId ?? null, + identities: emptyArtifactIdentities({ + experiment: args.experimentIdentity ?? null, + agentAdapter: agentAdapterIdentity(agentAdapter), + stackAdapter: { id: stackAdapter.id, version: stackAdapter.version }, + }), + mode: args.runMode ?? { id: args.progression ? 'dependency' : 'sequential' }, + track: args.track, backend: args.backend, model: args.model, + ...(args.providerRoute ? { providerRoute: args.providerRoute } : {}), + ...(args.maxOutputTokens ? { maxOutputTokens: args.maxOutputTokens } : {}), + pricing: args.pricing, + guidance: args.guidance, condition: args.condition ?? null, + ...(args.productionQuality && agentAdapter.provider ? { productionQuality: true } : {}), + skills: args.skills ?? [], + runtime: { buildImage: process.env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE, url }, + selectionRequest: args.selectionRequest, + featureCatalog: args.featureCatalog?.identity ?? null, + dependencyPolicy: args.dependencyPolicy?.identity ?? null, + ...(args.progressionOwner ? { progressionOwner: args.progressionOwner } : {}), + ...(args.progressionSeed ? { progressionSeed: { + fromDepth: args.progressionSeed.fromDepth, + sourceSha256: args.progressionSeed.sourceSha256, + sourceFiles: args.progressionSeed.sourceFiles, + parent: structuredClone(args.progressionSeed.parent), + validatedDepths: [], + } } : {}), + backendLease: publicBackendLease(readBackendLease(leasePath, + { token: initialLease.ownershipToken, backend: args.backend, runId })), + validation: { + ladder: { policy: args.progression ? args.progression.identity.policy : 'pass-before-next-level', + requestedLevels: [...args.levelList], + completedLevels: [], stoppedAfterLevel: null, blockedLevels: [] } }, levels: [] }; + activeRun = run; + + const progressionOwner = args.progression ? { + ...args.progressionOwner, + workspace: { appDirectory: 'source' }, + } : null; + const progressionExecution = args.progression ? createLiveProgressionExecution({ + progression: args.progression, + featureCatalogIdentity: args.featureCatalog?.identity, + dependencyPolicyIdentity: args.dependencyPolicy?.identity, + owner: progressionOwner, + statePath: join(args.out, ARTIFACT_FILE.progressionState), + runId, + outputDir: args.out, + appDir, + track: args.track, + backend: args.backend, + identities: run.identities, + recipeBindings: args.recipeBindings, + retainPriorContracts: args.retainPriorContracts ?? true, + resumeFrom: args.progressionResumeFrom ?? null, + getRunArtifact: () => { + persistRun(join(outputDir, ARTIFACT_FILE.run), run); + return readArtifact(join(outputDir, ARTIFACT_FILE.run)); + }, + onState: status => { + run.progressionStatus = status; + persistRun(join(outputDir, ARTIFACT_FILE.run), run); + }, + }) : null; + const progressionStart = progressionExecution?.initialize() ?? null; + if (progressionStart?.resumed) { + const prior = progressionStart.priorRun; + if (!prior) throw new Error('resumed dependency progression has no prior run artifact'); + const actionLevel = progressionStart.action.type === 'terminal' + ? Number.MAX_SAFE_INTEGER : progressionStart.action.level; + const inheritedLevels = (prior.payload.levels ?? []) + .filter(level => level.level < actionLevel).map(level => level.level); + run.levels = (prior.payload.levels ?? []) + .filter(level => inheritedLevels.includes(level.level)).map(level => structuredClone(level)); + run.validation.ladder.completedLevels = [...inheritedLevels]; + run.progressionResume = { + priorRunId: prior.id, + priorRunSha256: sha256(canonicalDefinitionJson(prior)), + stateSha256: progressionStart.stateSha256, + action: progressionStart.action.type === 'terminal' + ? { type: 'terminal' } + : { type: progressionStart.action.type, level: progressionStart.action.level }, + inheritedLevels, + priorTotals: prior.payload.totals ?? null, + }; + run.progressionStatus = progressionStart.status; + persistRun(join(args.out, ARTIFACT_FILE.run), run); + } + + const bindProgressionAction = (level: number): ProgressionRecipeAction | null => { + if (!progressionExecution) return null; + const selected = progressionExecution.bind(level); + if (!isProgressionWorkRecipeAction(selected)) return selected; + args.recipeTasks.set(level, { + request: selected.grader.request, + selection: selected.grader.selection, + task: selected.grader.task, + agentRequest: selected.agent.request, + progressionAction: selected.action, + }); + return selected; + }; + + const progressionBundles = new Map(); + const recordProgressionGrade = (input: Parameters['record']>[0]) => { + const next = progressionExecution?.record(input) ?? null; + const last = progressionExecution?.state?.attempts.at(-1) ?? null; + if (input.bundle && input.selected && isProgressionWorkRecipeAction(input.selected) + && last?.outcome === 'conclusive') { + progressionBundles.set(input.level, input.bundle as GradeBundlePayload); + } + return next; + }; + + const appendLevelRecord = (record: RunLevelRecord): void => { + pendingLevel = null; + if (args.dependencyPolicy?.definition.workSelection !== 'feature') { + run.levels.push(record); + return; + } + const index = run.levels.findIndex(candidate => candidate.level === record.level); + if (index < 0) run.levels.push(record); + else run.levels[index] = mergeFeatureLevelRecord(run.levels[index] ?? null, record); + }; + + const runAgentForLevel = async (mode: AgentMode, level: number, + onFailure?: () => Promise): Promise => { + try { + clearPrivateGradingEvidence(appDir); + pendingLevel ??= { level, graded: false, score: null, max: null, selection: null, + outcome: { kind: 'ungraded' }, buildSessions: [], repairSessions: [] }; + sessionInFlight = true; + persistRun(join(outputDir, ARTIFACT_FILE.run), run); + const result = await runAgent(args, agentAdapter, mode, level, appDir); + sessionInFlight = false; + const paid = runSessionRecord(result); + if (mode === 'fix') { + pendingLevel.repairSessions!.push(paid); + pendingLevel.repairCostUsd = addCostUsd(pendingLevel.repairCostUsd, paid.costUsd); + pendingLevel.repairs = (pendingLevel.repairs ?? 0) + 1; + } else if (mode === 'resume') { + pendingLevel.resumeSession = paid; + pendingLevel.resumeCostUsd = paid.costUsd; + } else { + pendingLevel.buildSessions!.push(paid); + pendingLevel.buildCostUsd = addCostUsd(pendingLevel.buildCostUsd, paid.costUsd); + } + pendingLevel.costUsd = addCostUsd(pendingLevel.buildCostUsd, pendingLevel.resumeCostUsd, + pendingLevel.repairCostUsd); + pendingLevel.sessionTotals = summarizeSessions([...pendingLevel.buildSessions!, + ...(pendingLevel.resumeSession ? [pendingLevel.resumeSession] : []), ...pendingLevel.repairSessions!]); + if (result.costComplete !== true) runCostComplete = false; + persistRun(join(outputDir, ARTIFACT_FILE.run), run); + return result; + } catch (error) { + await onFailure?.(); + const reason = errorMessage(error).split(/\r?\n/)[0] ?? 'agent execution failed'; + const capped = reason.startsWith('Cost cap reached:'); + run.outcome = { kind: capped ? 'provider_failure' : 'harness_failure', phase: capped ? 'cost-cap' : `agent-${mode}`, + reason, appFailures: [], inconclusive: [], harnessFailures: capped ? [] : [reason] }; + run.validation.ladder.stoppedAfterLevel = run.levels.at(-1)?.level ?? null; + run.validation.ladder.blockedLevels = args.levelList.filter(candidate => candidate >= level); + if (progressionExecution) { + recordProgressionGrade({ selected: progressionExecution.bind(level), bundle: null, level, + failure: progressionFailure(run.outcome) }); + run.progressionStatus = progressionExecution.status(); + synchronizeProgressionSummary(run, requireProgressionState(progressionExecution.state)); + } + finalizeRunTotals(run, started, { costComplete: false }); + run.completedAt = new Date().toISOString(); + persistRun(join(outputDir, ARTIFACT_FILE.run), run); + throw error; + } + }; + + // Stop before grading if a coding session read protected material or if the + // audit itself failed. Keep the paid session and exact cost in the run artifact even + // though no score may be used. + const abortUnusableSession = (whichSession: string, audit: ContaminationAudit, + levelRecord: UnknownRecord & { level: number }, + selected: ProgressionRecipeAction | null, completedRepair = false) => { + const reason = audit.evidence.join('; '); + const outcome: RunOutcome = { kind: audit.kind === 'harness_failure' ? 'harness_failure' : 'ungraded', + phase: 'contamination-audit', reason, + appFailures: [], inconclusive: [], + harnessFailures: audit.kind === 'harness_failure' ? [reason] : [] }; + run.contaminated = audit.kind === 'contaminated'; + run.contamination = { evidence: audit.evidence, verdict: audit.verdict, + detectedAt: whichSession }; + const record: RunLevelRecord = { ...levelRecord, error: reason, outcome, + level: levelRecord.level, graded: false, score: null, max: null, selection: null }; + appendLevelRecord(record); + if (progressionExecution) { + recordProgressionGrade({ selected, bundle: null, level: levelRecord.level, + failure: progressionFailure(outcome), + completedRepair }); + run.progressionStatus = progressionExecution!.status(); + synchronizeProgressionSummary(run, requireProgressionState(progressionExecution.state)); + } + run.validation.ladder.stoppedAfterLevel = run.levels.at(-2)?.level ?? null; + run.validation.ladder.blockedLevels = args.levelList + .filter(candidate => candidate >= levelRecord.level); + finalizeRunTotals(run, started, { costComplete: runCostComplete }); + run.outcome = outcome; + run.completedAt = new Date().toISOString(); + if (run.contaminated) { + console.log(`\n CONTAMINATED at ${whichSession}:`); + for (const evidence of audit.evidence) console.log(` ${evidence}`); + console.log(' Scores from this run must not be quoted.'); + } else { + console.log(`\n HARNESS FAILURE at ${whichSession}:`); + for (const evidence of audit.evidence) console.log(` ${evidence}`); + console.log(' The audit did not establish a usable result.'); + } + try { persistRun(join(outputDir, ARTIFACT_FILE.run), run); } catch { /* best effort */ } + try { archiveTranscripts(appDir, artifactLabel); } catch { /* best effort */ } + teardown(); + process.exit(4); + }; + + for (let levelIndex = 0; levelIndex < args.levelList.length; levelIndex += 1) { + const pauseDepth = args.pauseAfterDepth; + const activeState = progressionExecution?.state; + if (pauseDepth !== undefined && activeState?.phase === 'active' + && depthPauseDue(run, pauseDepth, activeState.level)) { + const context = readDepthPauseContext(); + if (!context || context.depth !== pauseDepth) throw new Error('planned depth pause has no controller authority'); + finalizeRunTotals(run, started, { costComplete: runCostComplete }); + persistRun(join(outputDir, ARTIFACT_FILE.run), run); + run.pausedDurationMs = await waitAtDepthBoundary(outputDir, appDir, context); + finalizeRunTotals(run, started, { costComplete: runCostComplete }); + persistRun(join(outputDir, ARTIFACT_FILE.run), run); + } + // A previous checkpoint cannot cover a newer session or partial grade. + clearTimeContinuationBoundary(outputDir); + const level = args.levelList[levelIndex]!; + const t0 = Date.now(); + const continuing = Boolean(args.repairGrant); + console.log(`\n================ ${args.backend} — level ${level} ================`); + + let progressionSelection = bindProgressionAction(level); + if (progressionSelection?.action.type === 'terminal') break; + if (args.dependencyPolicy?.definition.workSelection === 'all-at-once' + && progressionSelection?.action.level !== level) continue; + const applicationControl = materializeCodingOutput + ? restartSpecFor(args, appDir, track) : null; + const featureActionSequence = args.dependencyPolicy?.definition.workSelection === 'feature' + ? requireProgressionState(progressionExecution?.state ?? null).attempts.length + 1 : null; + const featureActionSuffix = featureActionSequence === null + ? '' : `-action${String(featureActionSequence).padStart(3, '0')}`; + // A clean-source start that fails voids a grade. Its launch log is the + // only account of why, so it stays beside the run. + const keepStartLog = (error: unknown, label: string): void => { + const startLog = error !== null && typeof error === 'object' && 'startLog' in error + ? error.startLog : null; + if (typeof startLog !== 'string' || !startLog) return; + writeFileSync(join(outputDir, `${label}-start.log`), `${startLog}\n`); + }; + const restoreFeatureAcceptedSource = async (resetDatabase = true): Promise => { + if (featureActionSequence === null) return; + const source = join(outputDir, 'source'); + try { + if (applicationControl) { + const restore = resetDatabase ? restoreRepairSource : materializeAcceptedSource; + await restore(source, appDir, applicationControl); + } else resetAppToSource(source, appDir); + } catch (error) { + console.log(` accepted feature restore failed: ${errorMessage(error)}`); + try { keepStartLog(error, `${args.backend}-l${level}${featureActionSuffix}-feature-restore`); } + catch (logError) { console.log(` could not preserve start log: ${errorMessage(logError)}`); } + // Failed coding/grading callers must still finalize their original evidence. + // The durable accepted source remains in outputDir even if local restore fails. + try { resetAppToSource(source, appDir); } + catch (restoreError) { console.log(` source restore failed: ${errorMessage(restoreError)}`); } + } + }; + if (featureActionSequence !== null && progressionSelection + && isProgressionWorkRecipeAction(progressionSelection) + && !featureActionNeedsCoding(progressionSelection, + requireProgressionState(progressionExecution?.state ?? null))) { + await restoreFeatureAcceptedSource(false); + const bundle = await grade(args, appDir, url, + `${args.backend}-l${level}${featureActionSuffix}-regrade`, level, track, runId); + const outcome = classifyBundle(bundle); + const repair = { status: levelGradeIsUsable(outcome) ? 'not-needed' as const : 'ungraded' as const, + limit: 0, used: 0, stopReason: 'accepted-source-regrade' }; + let next = recordProgressionGrade({ selected: progressionSelection, bundle, + level }); + let finalOutcome = outcome; + const currentState = requireProgressionState(progressionExecution?.state ?? null); + if (next?.type === 'build' && next.level === level + && !featureActionNeedsCoding(progressionSelection, currentState)) { + const reason = 'accepted source still has ungraded checks after regrade'; + finalOutcome = { kind: 'harness_failure', phase: 'feature-regrade', reason, + appFailures: [], inconclusive: [], harnessFailures: [reason] }; + next = recordProgressionGrade({ selected: progressionSelection, bundle: null, + level, failure: progressionFailure(finalOutcome) }); + } + const state = requireProgressionState(progressionExecution?.state ?? null); + const graded = levelGradeIsUsable(finalOutcome) + && state.attempts.at(-1)?.outcome === 'conclusive'; + let checkpoint = null; + if (graded && (state.phase === 'terminal' || state.level > level)) { + checkpoint = preserveLevelCheckpoint({ appDir, outputDir, runId, + identities: run.identities, track: args.track, backend: args.backend, level, + repair, outcome: finalOutcome, + selectionSha256: bundle?.selection?.sha256 ?? null }); + } + appendLevelRecord({ level, graded, + score: graded ? bundle?.totals?.score ?? null : null, + max: graded ? bundle?.totals?.max ?? null : null, + selection: graded ? bundle?.selection ?? null : null, + regression: bundle?.totals?.regression ?? null, + contractPass: bundle?.totals?.contractPass ?? null, + code: bundle?.code ?? null, + repair, + checkpoint, + sessionTotals: summarizeSessions([]), + costUsd: 0, + repairs: 0, + durationSec: Math.round((Date.now() - t0) / 1000), + outcome: finalOutcome }); + if (graded && (state.phase === 'terminal' || state.level > level) + && !run.validation.ladder.completedLevels.includes(level)) { + run.validation.ladder.completedLevels.push(level); + } + run.progressionStatus = progressionExecution!.status(); + persistRun(join(outputDir, ARTIFACT_FILE.run), run); + if (state.phase === 'terminal') break; + if (state.attempts.at(-1)?.outcome === 'inconclusive') { + run.validation.ladder.stoppedAfterLevel = level; + run.validation.ladder.blockedLevels = args.levelList.filter(candidate => candidate >= level); + break; + } + if (next?.type !== 'terminal' && next?.level === level) { + levelIndex -= 1; + continue; + } + if (next?.type !== 'terminal' && next && next.level < level) { + throw new Error(`dependency progression moved backward from L${level}`); + } + continue; + } + if (args.seedThrough !== undefined && level <= args.seedThrough) { + if (!progressionSelection || !isProgressionWorkRecipeAction(progressionSelection) + || progressionSelection.action.level !== level || !args.seedFrom || !run.progressionSeed) { + throw new Error(`extension cannot validate depth ${level}`); + } + let applicationFailure: RunOutcome | null = null; + if (applicationControl) { + try { + await materializeAcceptedSource(args.seedFrom, appDir, applicationControl); + } catch (error) { + applicationFailure = materializationAppFailure(error); + keepStartLog(error, `${args.backend}-extension-l${level}`); + } + } else { + resetAppToSource(args.seedFrom, appDir); + } + const bundle = await grade(args, appDir, url, `${args.backend}-extension-l${level}`, + level, track, runId, { applicationFailure }); + const outcome = applicationFailure ?? classifyBundle(bundle); + const next = recordProgressionGrade({ selected: progressionSelection, bundle, level }); + const progressionState = requireProgressionState(progressionExecution?.state ?? null); + const progressionAttempt = progressionState.attempts.at(-1) ?? null; + const graded = levelGradeIsUsable(outcome, progressionAttempt); + const passed = graded && outcome.kind === 'passed'; + const repair = { status: passed ? 'not-needed' as const : 'incomplete' as const, + limit: 0, used: 0, + stopReason: passed ? 'not-needed' : 'extension-validation-failed' }; + let checkpoint = null; + try { + checkpoint = preserveLevelCheckpoint({ appDir, outputDir: args.out, runId, + identities: run.identities, track: args.track, backend: args.backend, level, + repair, outcome, selectionSha256: bundle?.selection?.sha256 ?? null }); + } catch (error) { + throw new Error(`could not preserve extension depth ${level}: ${errorMessage(error)}`); + } + const source = hashAppSource(appDir); + appendLevelRecord({ level, graded, + score: graded ? bundle?.totals?.score ?? null : null, + max: graded ? bundle?.totals?.max ?? null : null, + selection: bundle?.selection ?? null, + baseline: { kind: 'extension-validation', source: { + sha256: source.sha256, files: source.files.length } }, + regression: bundle?.totals?.regression ?? null, + contractPass: bundle?.totals?.contractPass ?? null, + code: bundle?.code ?? null, + repair, + checkpoint, + sessionTotals: summarizeSessions([]), + costUsd: 0, + repairs: 0, + durationSec: Math.round((Date.now() - t0) / 1000), + outcome }); + if (progressionAttempt?.outcome === 'conclusive') { + if (!run.validation.ladder.completedLevels.includes(level)) { + run.validation.ladder.completedLevels.push(level); + } + } + run.progressionStatus = progressionExecution!.status(); + if (passed) run.progressionSeed.validatedDepths.push(level); + if (!passed) { + run.validation.ladder.stoppedAfterLevel = level > 1 ? level - 1 : null; + run.validation.ladder.blockedLevels = args.levelList.filter(candidate => candidate >= level); + persistRun(join(args.out, ARTIFACT_FILE.run), run); + break; + } + persistRun(join(args.out, ARTIFACT_FILE.run), run); + if (!next || next.type === 'terminal' || next.level === undefined || next.level <= level) { + throw new Error(`extension did not advance after depth ${level}`); + } + continue; + } + const resumedRepair = progressionStart?.resumed === true + && progressionStart.action.type === 'repair'; + // The interrupted repair was charged but never graded. Grade its preserved + // source before any coding session. + const resumedGrade = resumedRepair && progressionStart?.action.type === 'repair' + && progressionStart.action.repair.awaitingGrade === true; + const resumedRegression = resumedRepair + ? savedRepairRegression(progressionExecution?.state ?? null, progressionSelection) : null; + const resumedRegressionReport = resumedRegression + ? join(args.out, 'repair-reports', + `rejected-regression-l${level}${featureActionSuffix}-resume.md`) : null; + const priorRepairs = resumedRepair + ? progressionStart.priorRun?.payload.levels?.find(item => item.level === level) + ?.repair?.used ?? 0 + : args.progression ? 0 : run.levels.reduce((sum, item) => sum + (item.repairs ?? 0), 0); + const repairBudgetFor = (selected: ProgressionRecipeAction | null, + completedRepairs: number) => selected + && isProgressionWorkRecipeAction(selected) + ? dependencyRepairBudget(selected.action, completedRepairs) + : args.repairs; + const levelRepairNodeIds = new Set(progressionSelection?.action.repair.nodeIds ?? []); + let progressionRepairLimit = repairBudgetFor( + progressionSelection, priorRepairs); + const trackProgressionBudget = (selected: ProgressionRecipeAction | null, + completedRepairs: number) => { + if (!selected || !isProgressionWorkRecipeAction(selected)) return; + selected.action.repair.nodeIds.forEach(nodeId => levelRepairNodeIds.add(nodeId)); + progressionRepairLimit = Math.max( + progressionRepairLimit, repairBudgetFor(selected, completedRepairs)); + }; + if (resumedRepair && !resumedGrade) { + let reportFailure: string | null = null; + try { + if (resumedRegressionReport && resumedRegression) { + mkdirSync(dirname(resumedRegressionReport), { recursive: true }); + writeFileSync(resumedRegressionReport, resumedRegression.report); + } + sh('node', [join(ROOT, 'dist', 'commands', 'report-bugs.js'), '--app', appDir, + '--history-json', '[]', '--archive', join(args.out, 'repair-reports', + `bug-report-l${level}${featureActionSuffix}-resume.md`), + ...(resumedRegressionReport ? ['--prior-regression', resumedRegressionReport] : []), + ...repairReportArgs(progressionSelection)], + { stdio: 'pipe' }); + } catch (error) { + reportFailure = errorMessage(error).split(/\r?\n/)[0] + ?? 'repair report generation failed'; + } + if (reportFailure) { + const outcome: RunOutcome = { + kind: 'harness_failure', phase: 'repair-report', reason: reportFailure, + appFailures: [], inconclusive: [], harnessFailures: [reportFailure], + }; + recordProgressionGrade({ selected: progressionSelection, bundle: null, level, + failure: progressionFailure(outcome) }); + appendLevelRecord({ level, graded: false, score: null, max: null, + selection: null, error: reportFailure, outcome, + repair: { status: 'ungraded', limit: progressionRepairLimit, + used: priorRepairs, stopReason: 'repair-report' }, + repairCostUsd: 0, repairSessions: [], repairs: 0, priorRepairs, + cumulativeRepairs: priorRepairs, sessionTotals: summarizeSessions([]), + costUsd: 0, durationSec: Math.round((Date.now() - t0) / 1000) }); + run.validation.ladder.stoppedAfterLevel = level; + run.validation.ladder.blockedLevels = args.levelList.filter(candidate => candidate >= level); + persistRun(join(args.out, ARTIFACT_FILE.run), run); + break; + } + } + + const firstMode = resumedRepair ? 'fix' + : continuing ? 'resume' : args.seedFrom ? 'upgrade' : 'build'; + const build = resumedGrade ? null : await runAgentForLevel( + resumedRepair || run.levels.length === 0 ? firstMode : 'upgrade', level, + featureActionSequence === null ? undefined : restoreFeatureAcceptedSource); + // Only a resumed grade runs a level without a coding session. + const requireBuild = (): NonNullable => { + if (!build) throw new Error(`level ${level} has no coding session`); + return build; + }; + const buildFailure = build ? agentSessionFailure(build) : null; + const buildLeak = build && !buildFailure ? auditContamination(appDir, auditNetwork(), auditsTranscripts) : null; + if (buildLeak) { + const session = requireBuild(); + const buildSession = runSessionRecord(session, + resumedRepair ? priorRepairs + 1 : null); + const sessionTotals = summarizeSessions([buildSession]); + abortUnusableSession(`level ${level} ${firstMode}`, buildLeak, { + level, graded: false, score: null, max: null, selection: null, + ...(resumedRepair + ? { repairCostUsd: session.costUsd, repairSessions: [buildSession], repairs: 1, + priorRepairs, cumulativeRepairs: priorRepairs + 1 } + : continuing + ? { resumeCostUsd: session.costUsd, resumeSession: buildSession } + : { buildCostUsd: session.costUsd, buildSessions: [buildSession] }), + sessionTotals, costUsd: session.costUsd, durationMs: Date.now() - t0, + }, progressionSelection, resumedRepair); + } + // Record the session setup needed to compare runs. + if (build) run.setup ??= build.setup; + if (continuing) { + const session = requireBuild(); + requireContinuation(run).resumeSetup = { + sessionId: session.sessionId ?? null, + costUsd: session.costUsd, + durationMs: session.durationMs, + sourceVerified: false, + }; + } + // No session, no app. Grading an empty directory yields a real-looking zero + // that is a harness failure, not a result for this backend. + if (buildFailure) { + const session = requireBuild(); + await restoreFeatureAcceptedSource(); + console.log(` ABORTED: ${buildFailure.reason}. Details will be kept in ${join(args.out, ARTIFACT_FILE.run)}`); + const failedSession = runSessionRecord(session); + if (progressionExecution) { + recordProgressionGrade({ selected: progressionSelection, bundle: null, level, + failure: buildFailure }); + } + appendLevelRecord({ level, graded: false, score: null, max: null, + selection: null, error: buildFailure.reason, + outcome: buildFailure, + ...(continuing + ? { resumeSession: failedSession, resumeCostUsd: session.costUsd } + : { buildSessions: [failedSession], buildCostUsd: session.costUsd }), + sessionTotals: summarizeSessions([session]), + costUsd: session.costUsd, durationMs: Date.now() - t0 }); + break; + } + const gradeAcceptedSource = async (sourcePath: string, + label: string): Promise => { + let failure: RunOutcome | null = null; + if (applicationControl) { + try { + await materializeAcceptedSource(sourcePath, appDir, applicationControl); + } catch (error) { + failure = materializationAppFailure(error); + keepStartLog(error, label); + } + } else { + resetAppToSource(sourcePath, appDir); + } + return grade(args, appDir, url, label, level, track, runId, + { applicationFailure: failure }); + }; + const checkpointGrade = (phase: RunCheckpoint['phase'], measured: GradeBundlePayload, + sessions: RunSessionRecord[], accepted: boolean): void => { + if (!args.condition?.requested?.levels.length || !measured.source?.sha256 + || !measured.selection?.sha256) return; + const source = join(privateGradingDirectory(appDir), ARTIFACT_FILE.gradeBundle); + const evidence = readArtifactPayload(source, { expectedKind: 'grade_bundle' }); + if (evidence.source?.sha256 !== measured.source.sha256 + || evidence.selection?.sha256 !== measured.selection.sha256) { + throw new Error('grading checkpoint does not match the current source and selection'); + } + const path = `checkpoints/${(run.checkpoints?.length ?? 0) + 1}.json`; + mkdirSync(join(outputDir, 'checkpoints'), { recursive: true }); + copyFileSync(source, join(outputDir, path)); + const prior = progressionStart?.priorRun?.payload; + const priorCheckpoint = Array.isArray(prior?.checkpoints) + ? prior.checkpoints.map(value => checkpointSchema.parse(value)).findLast(checkpoint => checkpoint.accepted) + : undefined; + const prompt = progressionSelection?.action.prompt; + recordRunCheckpoint({ ...run, condition: args.condition, checkpoints: run.checkpoints ??= [] }, + { phase, level, bundle: evidence, + sourceSha256: measured.source.sha256, + evidence: { path, sha256: sha256(readFileSync(join(outputDir, path))) }, + extraSessions: sessions, accepted, + ...(prior ? { priorCost: runCostEvidence(prior), + initialChecks: priorCheckpoint?.checks } : {}), + workNodeIds: object(prompt) && Array.isArray(prompt.nodeIds) + ? prompt.nodeIds.filter((id): id is string => typeof id === 'string') : [] }); + persistRun(join(outputDir, ARTIFACT_FILE.run), run); + }; + const restoreAcceptedRepair = async (sourcePath: string, gradingPath: string, + completedRepair = true): Promise => { + try { + try { + if (applicationControl) await restoreRepairSource(sourcePath, appDir, applicationControl); + else resetAppToSource(sourcePath, appDir); + } finally { + restorePrivateGradingEvidence(appDir, gradingPath); + } + return true; + } catch (error) { + let reason = `could not restore the accepted repair source: ${errorMessage(error)}`; + const preserved = join(outputDir, `repair-rollback-l${level}${featureActionSuffix}-round${repairs}`); + try { + snapshotAppSource(sourcePath, join(preserved, 'source')); + if (existsSync(gradingPath)) cpSync(gradingPath, join(preserved, 'grading'), { recursive: true }); + keepStartLog(error, `${args.backend}-l${level}${featureActionSuffix}-rollback${repairs}`); + reason += `; accepted source and available grading evidence kept at ${preserved}`; + } catch (preserveError) { + reason += `; could not preserve the accepted rollback snapshot: ${errorMessage(preserveError)}`; + } + console.log(` ${reason}; stopping repairs`); + recordRepairHarnessFailure('repair-restore', reason, null, completedRepair); + return false; + } + }; + // Keep a repaired source whose grade did not finish beside the run, bound + // by hash, so a resume can grade exactly what the paid session produced. + const preserveRepairCandidate = (directory: string): RunRepairCandidate => { + const path = join(outputDir, directory); + rmSync(path, { recursive: true, force: true }); + const live = hashAppSource(appDir); + snapshotAppSource(appDir, path); + const preserved = hashDirectory(path); + if (live.sha256 !== preserved.sha256 || live.files.length !== preserved.files.length) { + throw new Error('preserved repair source differs from the live application source'); + } + return { directory, sha256: preserved.sha256, files: preserved.files.length }; + }; + if (resumedGrade) { + const candidate = progressionStart?.priorRun?.payload.levels + ?.find(item => item.level === level)?.repair?.candidate; + const candidateRoot = args.progressionResumeFrom ?? outputDir; + let candidateFailure: string | null = null; + if (!candidate || !/^[A-Za-z0-9._-]+$/.test(candidate.directory) + || !existsSync(join(candidateRoot, candidate.directory))) { + candidateFailure = 'the interrupted repair left no source to grade'; + } else { + const candidatePath = join(candidateRoot, candidate.directory); + const preserved = hashDirectory(candidatePath); + if (preserved.sha256 !== candidate.sha256 + || preserved.files.length !== candidate.files) { + candidateFailure = 'the interrupted repair source does not match its record'; + } else { + resetAppToSource(candidatePath, appDir); + console.log(` restored the interrupted repair source from ${candidatePath}`); + } + } + if (candidateFailure) { + const outcome: RunOutcome = { + kind: 'harness_failure', phase: 'repair-candidate', reason: candidateFailure, + appFailures: [], inconclusive: [], harnessFailures: [candidateFailure], + }; + recordProgressionGrade({ selected: progressionSelection, bundle: null, level, + failure: progressionFailure(outcome) }); + appendLevelRecord({ level, graded: false, score: null, max: null, + selection: null, error: candidateFailure, outcome, + repair: { status: 'ungraded', limit: progressionRepairLimit, + used: priorRepairs, stopReason: 'repair-candidate' }, + repairCostUsd: 0, repairSessions: [], repairs: 0, priorRepairs, + cumulativeRepairs: priorRepairs, sessionTotals: summarizeSessions([]), + costUsd: 0, durationSec: Math.round((Date.now() - t0) / 1000) }); + run.validation.ladder.stoppedAfterLevel = level; + run.validation.ladder.blockedLevels = args.levelList.filter(candidate => candidate >= level); + persistRun(join(args.out, ARTIFACT_FILE.run), run); + break; + } + } + if (continuing) { + // A resume may restore runtime state but cannot change checkpoint source. + const resumed = hashAppSource(appDir); + const repairGrant = args.repairGrant; + if (!repairGrant) throw new Error('repair continuation has no grant'); + if (resumed.sha256 !== repairGrant.checkpoint.payload.source.sha256 + || resumed.files.length !== repairGrant.checkpoint.payload.source.files) { + throw new Error('resume setup changed the parent checkpoint source'); + } + const continuation = requireContinuation(run); + if (!continuation.resumeSetup) throw new Error('repair continuation did not record resume setup'); + continuation.resumeSetup.sourceVerified = true; + } + if (args.referenceMutationOnly) { + const session = requireBuild(); + appendLevelRecord({ level, score: null, max: null, graded: false, contractPass: null, + selection: null, + outcome: { kind: 'ungraded', phase: 'reference-mutation-only', + reason: 'the parent qualification owns the full clean grade', + appFailures: [], inconclusive: [], harnessFailures: [] }, + buildSessions: [runSessionRecord(session)], + buildCostUsd: session.costUsd, sessionTotals: summarizeSessions([session]), + costUsd: session.costUsd, durationMs: Date.now() - t0 }); + break; + } + const firstBuildDirectory = continuing + ? `baseline-l${level}${featureActionSuffix}` + : `first-build-l${level}${featureActionSuffix}`; + const firstBuildPath = join(args.out, firstBuildDirectory); + let firstBuildSource = null; + let materializationOutcome: RunOutcome | null = null; + try { + const liveSource = hashAppSource(appDir); + snapshotAppSource(appDir, firstBuildPath); + const preservedSource = hashDirectory(firstBuildPath); + if (liveSource.sha256 !== preservedSource.sha256) { + throw new Error('preserved first-build source differs from the live application source'); + } + firstBuildSource = { sha256: liveSource.sha256, files: liveSource.files.length }; + if (applicationControl) { + await materializeAcceptedSource(firstBuildPath, appDir, applicationControl); + } + console.log(` kept the ${continuing ? 'continuation baseline' : 'unaided'} source at ${firstBuildPath}`); + } catch (error) { + if (firstBuildSource) { + materializationOutcome = materializationAppFailure(error); + keepStartLog(error, `${args.backend}-l${level}${featureActionSuffix}`); + } + console.log(materializationOutcome + ? ` application failure: ${materializationOutcome.reason}` + : ` harness failure: could not bind the first-build source: ${errorMessage(error).split('\n')[0]}`); + } + const firstBuildLabel = `${args.backend}-l${level}${featureActionSuffix}`; + let bundle = firstBuildSource + ? await grade(args, appDir, url, firstBuildLabel, level, track, runId, + { applicationFailure: materializationOutcome }) : null; + let reusableRepairEvidence: { + bundle: GradeBundlePayload; + results: string; + } | null = null; + + // Keep the unaided result separate from the result after repairs. + const firstBuild: FirstBuildRecord = { + score: bundle?.totals?.score ?? null, + max: bundle?.totals?.max ?? null, + regression: bundle?.totals?.regression ?? null, + contractPass: bundle?.totals?.contractPass ?? null, + outcome: materializationOutcome ?? sourceBoundFirstBuildOutcome(bundle, firstBuildSource), + source: firstBuildSource, + missed: Object.values(bundle?.suites ?? {}).flatMap(s => + (s?.features ?? []).flatMap(f => + (f.criteria ?? []).filter(c => !evidencePassed(criterionEvidence(c))) + .map(c => `${f.name}/${c.id}`))), + }; + + if (continuing) { + const repairGrant = args.repairGrant; + if (!repairGrant) throw new Error('repair continuation has no grant'); + if (firstBuild.score === null || firstBuild.max === null || firstBuildSource === null) { + throw new Error('repair continuation did not produce a source-bound baseline score'); + } + const reproduction = compareRepairBaseline(repairGrant.level, { + score: firstBuild.score, + max: firstBuild.max, + selectionSha256: bundle?.selection?.sha256 ?? null, + sourceSha256: firstBuildSource.sha256, + expectedSourceSha256: repairGrant.checkpoint.payload.source.sha256, + outcome: repairOutcome(firstBuild.outcome), + }); + requireContinuation(run).baseline = { + score: firstBuild.score, + max: firstBuild.max, + selectionSha256: bundle?.selection?.sha256 ?? null, + sourceSha256: firstBuildSource?.sha256 ?? null, + outcome: firstBuild.outcome, + ...reproduction, + }; + if (!reproduction.reproduced) { + const reason = `restored checkpoint did not reproduce its parent: ${reproduction.mismatches.join(', ')}`; + console.log(` CONTINUATION STOPPED: ${reason}`); + const failure = { kind: 'harness_failure', phase: 'continuation-baseline', reason, + appFailures: [], inconclusive: [], harnessFailures: [] }; + firstBuild.outcome = failure; + bundle = { ...bundle, outcome: failure }; + } + } + + if (bundle && firstBuildSource && levelGradeIsUsable(firstBuild.outcome)) { + const accepted = !(featureActionSequence !== null && progressionSelection + && isProgressionWorkRecipeAction(progressionSelection)) + || featureCandidateAccepted(progressionSelection, + requireProgressionState(progressionExecution?.state ?? null), bundle); + checkpointGrade(resumedRepair ? 'repair' : 'first-build', bundle, + build ? [runSessionRecord(build)] : [], accepted); + } + + const selectedObservedChecks = checksForGrade(args.recipeTasks.get(level), 'observed'); + if (!continuing && !resumedRepair && selectedObservedChecks.length) { + const observationOut = join(args.out, + `first-build-l${level}${featureActionSuffix}-observed`); + let observationBundle = null; + let observationOutcome; + if (!firstBuildSource) { + observationOutcome = { kind: 'harness_failure', phase: 'first-build-source', + reason: 'observed specifications require a source-bound first build' }; + } else if (!ladderMayContinue(firstBuild.outcome)) { + observationOutcome = { kind: 'ungraded', phase: 'first-build-observation', + reason: 'scored first-build grading did not establish a usable environment' }; + } else { + observationBundle = await grade(args, appDir, url, `${args.backend}-l${level}-observed`, level, + track, runId, { observation: 'observed', out: observationOut, + sourceSha256: firstBuildSource.sha256 }); + observationOutcome = classifyBundle(observationBundle); + } + firstBuild.observations = { + sourceSha256: firstBuildSource?.sha256 ?? null, + selectionSha256: args.recipeTasks.get(level)?.selection.sha256 ?? null, + selectedChecks: selectedObservedChecks.map(check => check.stableKey), + reportedChecks: observationBundle?.selection?.reportedChecks ?? [], + passedPoints: observationBundle?.totals?.score ?? null, + observedPoints: observationBundle?.totals?.max ?? null, + scoreContribution: false, + repairVisible: false, + artifact: observationBundle + ? `first-build-l${level}${featureActionSuffix}-observed/${ARTIFACT_FILE.gradeBundle}` : null, + outcome: observationOutcome, + }; + } + + let initialProgressionFailure: ProgressionFailure | null = null; + if (featureActionSequence !== null && progressionSelection + && isProgressionWorkRecipeAction(progressionSelection)) { + const candidateOutcome = classifyBundle(bundle); + archiveCandidateGrade(appDir, outputDir, `l${level}${featureActionSuffix}`); + if (!levelGradeIsUsable(candidateOutcome)) { + await restoreFeatureAcceptedSource(); + initialProgressionFailure = progressionFailure(candidateOutcome); + } else if (!featureCandidateAccepted(progressionSelection, + requireProgressionState(progressionExecution?.state ?? null), + bundle)) { + bundle = await gradeAcceptedSource(join(outputDir, 'source'), + `${args.backend}-l${level}${featureActionSuffix}-restored`, + ); + const restoredOutcome = classifyBundle(bundle); + if (!levelGradeIsUsable(restoredOutcome)) { + initialProgressionFailure = progressionFailure(restoredOutcome); + } + } + } + + // Preserve the first source and scored grading before repair overwrites the + // app. Observed evidence remains in its own source-bound result directory. + const acceptedGradingDirectory = continuing + ? `baseline-l${level}${featureActionSuffix}-grading` + : `first-build-l${level}${featureActionSuffix}-grading`; + try { + const gradingFrom = privateGradingDirectory(appDir); + if (existsSync(gradingFrom)) { + const gradingTo = join(args.out, acceptedGradingDirectory); + cpSync(gradingFrom, gradingTo, { + recursive: true, + filter: src => !/[\\/]media([\\/]|$)/.test(src), + }); + if (bundle) reusableRepairEvidence = { bundle, results: gradingTo }; + console.log(` kept the ${continuing ? 'continuation baseline' : 'unaided'} grading at ${join(args.out, acceptedGradingDirectory)}`); + } + } catch (e) { + // Never worth losing a run over: the score is already recorded. + console.log(` harness failure: could not keep the first build: ${errorMessage(e).split('\n')[0]}`); + } + + // A resumed grade settles a repair that was already charged; only a + // resumed coding session is a new repair. + let repairs = resumedRepair && !resumedGrade ? 1 : 0; + let repairCost = resumedRepair && build ? build.costUsd : 0; + const repairSessions = resumedRepair && build + ? [runSessionRecord(build, priorRepairs + 1)] : []; + const repairHistory: ReturnType[] = []; + let repairCandidate: RunRepairCandidate | null = null; + let priorRegressionReport = resumedRegressionReport; + let priorRegressionOwner = repairOwnerNodeIds(progressionSelection).join('\n') || null; + let regressed = false; + let repairStopReason: string | null = null; + let repairProgress = repairProgressState(null, bundle); + const pauseForRepeatedFindings = () => { + if (args.progression) return false; + repairProgress = repairProgressState(repairProgress, bundle); + if (repairProgress.stalledRounds < MAX_STALLED_REPAIRS) return false; + repairStopReason = 'repeated-findings'; + console.log(` pausing after ${repairProgress.stalledRounds} repairs ` + + 'with the same failed checks and no score gain'); + return true; + }; + const initialBundleOutcome = classifyBundle(bundle); + const initialProgressionAttempt = args.progression + ? requireProgressionState(progressionExecution?.state ?? null).attempts.at(-1) ?? null + : null; + const initialGradeUsable = levelGradeIsUsable(initialBundleOutcome, + initialProgressionAttempt); + if (!initialGradeUsable) { + repairStopReason = 'initial-grading-failed'; + console.log(' repairs skipped: the initial grade did not complete, so there are no reliable findings to fix'); + } + + let progressionNext = recordProgressionGrade({ + selected: progressionSelection, + bundle: initialProgressionFailure ? null : bundle, + level, + failure: initialProgressionFailure, + completedRepair: resumedRepair && !resumedGrade, + }); + // Never start a paid repair with nothing left to spend or while a charged + // repair still waits for its grade. + const progressionMayRepair = () => !args.progression + || (progressionNext?.type === 'repair' && progressionNext.repair.remaining > 0 + && progressionNext.repair.awaitingGrade !== true); + // One allowance for both modes: the engine's remaining repairs in + // dependency mode, the run-wide total less every repair so far otherwise. + const repairsRemaining = (): number => args.progression + ? (progressionNext?.type === 'repair' ? progressionNext.repair.remaining : 0) + : args.repairs - priorRepairs - repairs; + const mayRepair = (): boolean => args.progression + ? progressionMayRepair() : repairsRemaining() > 0; + const recordRepairProgression = ({ failure = null, repairRegression = null, + completedRepair = false }: { + failure?: ProgressionFailure | null; + repairRegression?: ProgressionRepairRegression | null; + completedRepair?: boolean; + } = {}) => { + progressionNext = recordProgressionGrade({ + selected: progressionSelection, + bundle: failure ? null : bundle, + level, + failure, + repairRegression, + completedRepair, + }); + return progressionMayRepair(); + }; + const recordRepairHarnessFailure = (phase: string, reason: string, + failedBundle: GradeBundlePayload | null = null, completedRepair = false): void => { + const failure: RunOutcome = { + kind: 'harness_failure', phase, reason, + appFailures: [], inconclusive: [], harnessFailures: [reason], + }; + bundle = failedBundle ? { ...failedBundle, outcome: failure } : { outcome: failure }; + repairStopReason = phase; + if (args.progression) recordRepairProgression({ + failure: progressionFailure(failure), completedRepair, + }); + const failedLevel = progressionExecution?.state?.attempts.at(-1)?.level; + const prior = run.levels.find(record => record.level === failedLevel); + if (prior) prior.outcome = failure; + }; + const restoreProgressionGrade = async (accepted: GradeBundlePayload | null, + label: string): Promise => { + const expected = progressionSelection && isProgressionWorkRecipeAction(progressionSelection) + ? progressionSelection.grader.selectionSha256 : null; + if (!expected || accepted?.selection?.sha256 === expected) { + bundle = accepted; + return true; + } + bundle = await grade(args, appDir, url, label, level, track, runId); + const outcome = classifyBundle(bundle); + if (levelGradeIsUsable(outcome)) return true; + recordRepairHarnessFailure('repair-restore-grading', + outcome.reason ?? 'restored source did not produce a reliable grade', bundle, true); + return false; + }; + const writeRepairReport = (results: string | null = null): + { status: 0 | 3 | 4 } | { status: 'failed'; reason: string } => { + try { + sh('node', [join(ROOT, 'dist', 'commands', 'report-bugs.js'), '--app', appDir, + ...(results ? ['--results', results] : []), + '--history-json', JSON.stringify(repairHistory), + '--archive', join(outputDir, 'repair-reports', + `bug-report-l${level}${featureActionSuffix}-round${repairs + 1}.md`), + ...(priorRegressionReport ? ['--prior-regression', priorRegressionReport] : []), + ...repairReportArgs(progressionSelection)], { stdio: 'pipe' }); + return { status: 0 }; + } catch (error) { + const failure = commandFailure(error); + if (failure.status === 3 || failure.status === 4) return { status: failure.status }; + return { status: 'failed', reason: errorMessage(failure).split(/\r?\n/)[0] + ?? 'repair report generation failed' }; + } + }; + const recordMissingRepairFeedback = (status: 3 | 4): void => { + repairStopReason = 'no-actionable-findings'; + if (!args.progression) return; + const reason = status === 3 + ? 'selected repair checks contain no failures' + : 'selected repair checks produced no actionable findings'; + const failure: RunOutcome = { + kind: 'harness_failure', phase: 'repair-report', reason, + appFailures: [], inconclusive: [], harnessFailures: [reason], + }; + bundle = { outcome: failure }; + recordRepairProgression({ failure: progressionFailure(failure) }); + }; + + // Hand back findings and let the agent fix, until clean or out of rounds. + while (levelGradeIsUsable(classifyBundle(bundle), args.progression + ? requireProgressionState(progressionExecution?.state ?? null).attempts.at(-1) ?? null + : null) && mayRepair()) { + let reportReady = false; + const acceptedBundle = bundle; + let repairBaselineBundle = bundle; + if (args.progression) { + progressionSelection = bindProgressionAction(level); + const repairOwner = repairOwnerNodeIds(progressionSelection).join('\n') || null; + if (repairOwner !== priorRegressionOwner) { + priorRegressionReport = null; + priorRegressionOwner = repairOwner; + } + trackProgressionBudget(progressionSelection, priorRepairs + repairs); + if (progressionSelection && isProgressionWorkRecipeAction(progressionSelection) + && bundle?.selection?.sha256 !== progressionSelection.grader.selectionSha256) { + const sequence = requireProgressionState(progressionExecution?.state ?? null).attempts.length + 1; + const targetChecks = repairCheckKeys(progressionSelection); + const sourceSha256 = hashAppSource(appDir).sha256; + const hasCurrentTargetEvidence = bundle?.source?.sha256 === sourceSha256 + && targetChecks.every(check => bundle?.selection?.reportedChecks?.includes(check)); + const reusable = reusableRepairEvidence?.bundle.source?.sha256 === sourceSha256 + && targetChecks.every(check => + reusableRepairEvidence?.bundle.selection?.reportedChecks?.includes(check)) + ? reusableRepairEvidence : null; + let repairResults = hasCurrentTargetEvidence ? null : reusable?.results ?? null; + if (reusable && !hasCurrentTargetEvidence) repairBaselineBundle = reusable.bundle; + if (!hasCurrentTargetEvidence && !reusable) { + const binding = args.recipeBindings.get(level); + if (!binding) throw new Error(`L${level} has no recipe binding`); + const targetTask = resolveProgressionRepairTarget(binding, + requireProgressionState(progressionExecution?.state ?? null)); + repairResults = join(outputDir, 'repair-grades', + `l${level}${featureActionSuffix}-round-${repairs + 1}`); + repairBaselineBundle = await grade(args, appDir, url, + `${args.backend}-l${level}-repair-target${sequence}`, + level, track, runId, { out: repairResults, recipeTask: targetTask }); + } + const refreshOutcome = classifyBundle(repairBaselineBundle); + const refreshUsable = levelGradeIsUsable(refreshOutcome); + if (!refreshUsable) { + recordRepairHarnessFailure('refresh-grading-failed', + refreshOutcome.reason ?? 'repair target did not produce a reliable grade', + repairBaselineBundle); + break; + } + const refreshReport = writeRepairReport(repairResults); + if (refreshReport.status === 'failed') { + recordRepairHarnessFailure('repair-report', refreshReport.reason); + break; + } + if (refreshReport.status === 3) { + bundle = await grade(args, appDir, url, + `${args.backend}-l${level}-repair-refresh${sequence}`, + level, track, runId); + if (!levelGradeIsUsable(classifyBundle(bundle))) { + recordRepairHarnessFailure('refresh-grading-failed', + classifyBundle(bundle).reason ?? 'repair refresh did not produce a reliable grade', + bundle); + break; + } + checkpointGrade('final', bundle!, + [...(build && !resumedRepair ? [runSessionRecord(build)] : []), ...repairSessions], true); + recordRepairProgression(); + continue; + } + if (refreshReport.status === 4) { + recordMissingRepairFeedback(refreshReport.status); + break; + } + reportReady = true; + } + } + const report = reportReady ? { status: 0 as const } : writeRepairReport(); + if (report.status === 'failed') { + recordRepairHarnessFailure('repair-report', report.reason); + break; + } + if (report.status !== 0) { + recordMissingRepairFeedback(report.status); + break; + } + + const before = repairBaselineBundle?.totals?.score ?? 0; + const beforeMax = repairBaselineBundle?.totals?.max ?? 0; + const beforeBundle = repairBaselineBundle; + // Keep the accepted source outside paths visible to the coding session. + const snapshot = join(tmpdir(), `stack-bench-snapshot-${args.backend}-${args.track}-run${args.runIndex}-l${level}`); + const gradingSnapshot = `${snapshot}-grading`; + const acceptedSource = hashAppSource(appDir); + snapshotAppSource(appDir, snapshot); + rmSync(gradingSnapshot, { recursive: true, force: true }); + if (existsSync(privateGradingDirectory(appDir))) { + cpSync(privateGradingDirectory(appDir), gradingSnapshot, { recursive: true }); + } + const cleanupRepairSnapshots = () => { + rmSync(snapshot, { recursive: true, force: true }); + rmSync(gradingSnapshot, { recursive: true, force: true }); + }; + try { + const displayedRepairBudget = args.progression + ? progressionRepairLimit + : args.repairs; + if (args.dependencyPolicy?.definition.repair.selection === 'feature' + && progressionSelection && isProgressionWorkRecipeAction(progressionSelection)) { + const [nodeId] = progressionSelection.action.repair.nodeIds; + const node = requireProgressionState(progressionExecution?.state ?? null).definition.nodes + .find(candidate => candidate.id === nodeId); + if (!nodeId || !node) { + throw new Error('feature repair has no selected feature'); + } + const used = requireProgressionState(progressionExecution?.state ?? null) + .nodes[nodeId]?.repairs.used ?? 0; + console.log(`--- feature repair ${used + 1}: ${node.title} ---`); + } else { + console.log(`--- repair ${priorRepairs + repairs + 1}/${displayedRepairBudget} ---`); + } + const fix = await runAgentForLevel('fix', level, + featureActionSequence === null ? undefined : restoreFeatureAcceptedSource); + repairCost += fix.costUsd; + repairSessions.push(runSessionRecord(fix, priorRepairs + repairs + 1)); + + const fixFailure = agentSessionFailure(fix); + if (fixFailure) { + if (featureActionSequence !== null) { + if (!await restoreAcceptedRepair(snapshot, gradingSnapshot, false)) break; + } + console.log(` coding session failed: ${fixFailure.reason}; stopping repairs`); + bundle = { outcome: fixFailure }; + repairHistory.push(repairHistoryEntry(repairs + 1, beforeBundle, bundle, + 'agent session failed')); + repairStopReason = 'agent-session-failure'; + recordRepairProgression({ failure: progressionFailure(fixFailure) }); + break; + } + repairs += 1; + + // Reject contaminated repairs before spending time on grading. + const fixLeak = auditContamination(appDir, auditNetwork(), auditsTranscripts); + if (fixLeak) { + const buildSession = build ? runSessionRecord(build) : null; + const sessions = resumedRepair || !buildSession + ? repairSessions : [buildSession, ...repairSessions]; + const sessionTotals = summarizeSessions(sessions); + cleanupRepairSnapshots(); + abortUnusableSession(`repair ${repairs}`, fixLeak, { + level, graded: false, score: null, max: null, + selection: bundle?.selection ?? null, + ...(resumedRepair + ? { resumedRepair: firstBuild } + : continuing + ? { baseline: firstBuild, resumeCostUsd: requireBuild().costUsd, + resumeSession: runSessionRecord(requireBuild()) } + : { firstBuild, buildCostUsd: requireBuild().costUsd, + buildSessions: [runSessionRecord(requireBuild())] }), + repairCostUsd: addCostUsd(repairCost), repairSessions, repairs, + ...(resumedRepair ? { priorRepairs, + cumulativeRepairs: priorRepairs + repairs } : {}), + repair: { status: 'ungraded', limit: displayedRepairBudget, + used: priorRepairs + repairs, + stopReason: fixLeak.kind === 'harness_failure' ? 'audit-failure' : 'contaminated' }, + sessionTotals, + costUsd: resumedRepair ? addCostUsd(repairCost) + : addCostUsd(requireBuild().costUsd, repairCost), + durationMs: Date.now() - t0, + }, progressionSelection, true); + } + if (hashAppSource(appDir).sha256 === acceptedSource.sha256) { + // A source hash does not cover installed dependencies or a live process. + if (!await restoreAcceptedRepair(snapshot, gradingSnapshot)) break; + const reason = 'repair made no source change'; + console.log(` ${reason}; ${args.progression + ? 'counting the failed attempt' + : 'pausing before another paid round'}`); + repairHistory.push(repairHistoryEntry(repairs, beforeBundle, beforeBundle, reason)); + if (args.progression) { + if (!await restoreProgressionGrade(acceptedBundle, + `${args.backend}-l${level}-unchanged${repairs}`)) break; + if (!recordRepairProgression({ completedRepair: true })) break; + continue; + } + repairStopReason = 'no-source-change'; + break; + } + const repairedSource = `${snapshot}-accepted`; + snapshotAppSource(appDir, repairedSource); + try { + bundle = await gradeAcceptedSource(repairedSource, `${args.backend}-l${level}-fix${repairs}`); + } finally { + rmSync(repairedSource, { recursive: true, force: true }); + } + + const repairedOutcome = classifyBundle(bundle); + if (!levelGradeIsUsable(repairedOutcome)) { + const reason = repairedOutcome.reason + ?? 'the repaired source did not produce a reliable grade'; + // The session is paid for and charged. Keep what it produced so a + // resume grades it instead of buying another repair; the feature's + // status waits for that grade. + const candidateDirectory = `repair-candidate-l${level}${featureActionSuffix}`; + let preserveFailure: string | null = null; + try { + repairCandidate = preserveRepairCandidate(candidateDirectory); + console.log(` repair grade failed: ${reason}; kept the repaired source at ` + + `${join(outputDir, candidateDirectory)} for grading on resume`); + } catch (error) { + preserveFailure = errorMessage(error).split(/\r?\n/)[0] + ?? 'could not keep the repaired source'; + console.log(` repair grade failed: ${reason}; ${preserveFailure}; restoring the accepted source`); + if (!await restoreAcceptedRepair(snapshot, gradingSnapshot)) break; + } + repairHistory.push(repairHistoryEntry(repairs, beforeBundle, bundle, + 'repair completed; grading failed')); + recordRepairHarnessFailure('repair-grading', + preserveFailure ? `${reason}; ${preserveFailure}` : reason, bundle, true); + break; + } + + if (featureActionSequence !== null && progressionSelection + && isProgressionWorkRecipeAction(progressionSelection) + && !featureCandidateAccepted(progressionSelection, + requireProgressionState(progressionExecution?.state ?? null), bundle)) { + const rejectedBundle = bundle; + if (rejectedBundle) checkpointGrade('repair', rejectedBundle, + [...(build && !resumedRepair ? [runSessionRecord(build)] : []), ...repairSessions], false); + archiveCandidateGrade(appDir, outputDir, `l${level}${featureActionSuffix}-repair${repairs}`); + if (!await restoreAcceptedRepair(snapshot, gradingSnapshot)) break; + if (!await restoreProgressionGrade(acceptedBundle, + `${args.backend}-l${level}${featureActionSuffix}-rollback${repairs}`)) break; + repairHistory.push(repairHistoryEntry(repairs, beforeBundle, rejectedBundle, + 'rolled back because the feature still failed or earlier behavior regressed')); + if (!recordRepairProgression({ completedRepair: true })) break; + if (pauseForRepeatedFindings()) break; + continue; + } + + const after = bundle?.totals?.score ?? 0; + const afterMax = bundle?.totals?.max ?? 0; + const repairedBundle = bundle; + // Lost or inconclusive evidence cannot hide a repair regression. + let decision = repairEvidenceDecision(beforeBundle, bundle); + const regressionDecision = repairRegressionDecision(acceptedBundle, bundle); + if (regressionDecision.action === 'rollback-regression') decision = regressionDecision; + if (bundle) checkpointGrade('repair', bundle, + [...(build && !resumedRepair ? [runSessionRecord(build)] : []), ...repairSessions], + !decision.action.startsWith('rollback-')); + const shared = decision.shared; + if (decision.action === 'keep-setup-repair') { + console.log(afterMax > 0 + ? ` application setup is now gradeable (${after}/${afterMax}); keeping this repair` + : ' application setup is still failing; keeping the attempted repair for the next round'); + repairHistory.push(repairHistoryEntry(repairs, beforeBundle, bundle, + afterMax > 0 + ? 'kept because the app became gradeable' + : 'kept to continue repairing application setup')); + if (!recordRepairProgression({ completedRepair: true })) break; + if (pauseForRepeatedFindings()) break; + continue; + } + if (decision.action === 'rollback-no-comparison') { + console.log(' no criteria were conclusively scored in both rounds; rolling back this fix'); + archiveCandidateGrade(appDir, outputDir, `l${level}${featureActionSuffix}-rejected${repairs}`); + if (!await restoreAcceptedRepair(snapshot, gradingSnapshot)) break; + if (!await restoreProgressionGrade(acceptedBundle, + `${args.backend}-l${level}-rollback${repairs}`)) break; + repairHistory.push(repairHistoryEntry(repairs, beforeBundle, repairedBundle, + 'rolled back because the result could not be compared')); + if (!recordRepairProgression({ completedRepair: true })) break; + if (pauseForRepeatedFindings()) break; + continue; + } + if (shared.points < Math.min(beforeMax, afterMax)) { + console.log(` comparing ${shared.points} point(s) across ${shared.count} criteria scored in both rounds` + + ` (${before}/${beforeMax} -> ${after}/${afterMax} overall)`); + } + if (decision.action === 'rollback-regression') { + if (shared.regressions.length) { + console.log(` broke ${shared.regressions.length} earlier passing check(s); rolling back this fix`); + } else if (shared.lostEvidence.length) { + console.log(` lost conclusive evidence for ${shared.lostEvidence.length} criterion/criteria; rolling back this fix`); + } else if (shared.definitionChanges.length) { + console.log(' rubric points changed between grades; rolling back this fix'); + } else { + console.log(` regressed (${shared.before} -> ${shared.after} on shared criteria); rolling back this fix`); + } + let repairRegression: ProgressionRepairRegression | null = null; + let regressionReportFailure: string | null = null; + try { + if (shared.regressions.length) { + const path = join(outputDir, 'repair-reports', + `rejected-regression-l${level}${featureActionSuffix}-round${repairs}.md`); + sh('node', [join(ROOT, 'dist', 'commands', 'report-bugs.js'), '--app', appDir, + '--out', path, '--checks-json', JSON.stringify(shared.regressions), + '--regression-context'], { stdio: 'pipe' }); + repairRegression = { + ownerNodeIds: repairOwnerNodeIds(progressionSelection), + report: readFileSync(path, 'utf8'), + }; + priorRegressionReport = path; + } + } catch (error) { + regressionReportFailure = errorMessage(error).split(/\r?\n/)[0] + ?? 'regression report generation failed'; + } + archiveCandidateGrade(appDir, outputDir, `l${level}${featureActionSuffix}-rejected${repairs}`); + if (!await restoreAcceptedRepair(snapshot, gradingSnapshot)) break; + if (regressionReportFailure) { + repairHistory.push(repairHistoryEntry(repairs, beforeBundle, bundle, + 'repair completed; regression reporting failed')); + recordRepairHarnessFailure('repair-regression-report', regressionReportFailure, + null, true); + break; + } + if (!await restoreProgressionGrade(acceptedBundle, + `${args.backend}-l${level}-rollback${repairs}`)) break; + regressed = true; + repairHistory.push(repairHistoryEntry(repairs, beforeBundle, repairedBundle, + 'rolled back because earlier behavior regressed')); + if (!recordRepairProgression({ repairRegression, completedRepair: true })) break; + if (pauseForRepeatedFindings()) break; + continue; + } + priorRegressionReport = null; + if (shared.after === shared.before) { + const remaining = displayedRepairBudget - priorRepairs - repairs; + console.log(` ${formatRepairProgress(shared, { before, beforeMax, after, afterMax })}; ` + + (remaining > 0 ? `${remaining} repair(s) remain` : 'repair budget exhausted')); + } + repairHistory.push(repairHistoryEntry(repairs, beforeBundle, bundle, + shared.after === shared.before ? 'kept with no score gain' : 'kept')); + if (!recordRepairProgression({ completedRepair: true })) break; + if (pauseForRepeatedFindings()) break; + } finally { + cleanupRepairSnapshots(); + } + } + + // Missing grade evidence is not a zero score. + const progressionState = progressionExecution + ? requireProgressionState(progressionExecution.state) : null; + const progressionAttempt = progressionState + ? progressionState.attempts.findLast(attempt => attempt.level === level) ?? null + : null; + const levelBundle = progressionAttempt?.outcome === 'inconclusive' + ? bundle : progressionBundles.get(level) ?? bundle; + const finalBundleOutcome = classifyBundle(levelBundle); + // Progression uses stricter evidence rules than a regular scored bundle. + // Store one answer when a selected check is not measured: the raw bundle + // remains available for diagnosis, but the level is not a usable grade. + const graded = levelGradeIsUsable(finalBundleOutcome, + args.progression ? progressionAttempt : null); + const finalTotals = graded ? levelBundle?.totals ?? null : null; + const nodeRepairs = progressionState + ? dependencyRepairRecords(progressionState, level, levelRepairNodeIds) + : null; + const repairLimit = progressionExecution + ? Math.max(priorRepairs + repairs, progressionRepairLimit) + : args.repairs; + const progressionStopReason = progressionNext?.type !== 'repair' + ? dependencyRepairStopReason(nodeRepairs ?? []) : null; + const repairBudgetExhausted = progressionExecution + ? finalBundleOutcome.kind === 'app_failure' && progressionStopReason !== null + && progressionStopReason !== 'repeated-findings' + : priorRepairs + repairs >= args.repairs; + repairStopReason ??= progressionStopReason; + const repairStatus: RepairStatus = repairStopReason === 'no-source-change' ? 'incomplete' + : !graded ? 'ungraded' + : finalBundleOutcome.kind === 'passed' ? (repairs > 0 ? 'corrected' : 'not-needed') + : repairBudgetExhausted ? 'budget-exhausted' : 'incomplete'; + const stopReasons: Record = { + 'not-needed': 'not-needed', + corrected: 'passed', + 'budget-exhausted': 'budget-exhausted', + incomplete: null, + ungraded: null, + }; + const stopReason = repairStopReason ?? stopReasons[repairStatus]; + const repair = { + status: repairStatus, + limit: repairLimit, + used: priorRepairs + repairs, + ...(!args.progression ? { stallLimitRounds: MAX_STALLED_REPAIRS } : {}), + stopReason, + ...(nodeRepairs ? { nodeRepairs } : {}), + ...(repairCandidate ? { candidate: repairCandidate } : {}), + }; + const latestProgressionAttempt = progressionState?.attempts.at(-1) ?? null; + const featureDepthContinues = featureActionSequence !== null + && progressionState?.phase === 'active' && progressionState.level === level; + if (continuing) { + const continuation = requireContinuation(run); + continuation.cumulativeRepairsAfter = continuation.cumulativeRepairsBefore + repairs; + } + let checkpoint = null; + if (graded && !featureDepthContinues + && (!latestProgressionAttempt || latestProgressionAttempt.level === level)) { + try { + checkpoint = preserveLevelCheckpoint({ + appDir, + outputDir: args.out, + runId, + identities: run.identities, + track: args.track, + backend: args.backend, + level, + repair, + outcome: finalBundleOutcome, + selectionSha256: levelBundle?.selection?.sha256 ?? null, + }); + console.log(` kept the L${level} source checkpoint at ${join(args.out, checkpoint.directory)}`); + } catch (error) { + console.log(` harness failure: could not keep the L${level} source checkpoint: ${errorMessage(error).split('\n')[0]}`); + } + } + if (!graded) { + console.log(` L${level}: GRADING DID NOT COMPLETE — no usable bundle. ` + + `Score is unknown, not zero; re-grade this level before using the run.`); + } + const buildSession = build ? runSessionRecord(build) : null; + const requireBuildSession = (): RunSessionRecord => { + if (!buildSession) throw new Error(`level ${level} has no coding session`); + return buildSession; + }; + const sessionTotals = summarizeSessions(resumedRepair || !buildSession ? repairSessions + : [buildSession, ...repairSessions]); + appendLevelRecord({ + level, + graded, + score: finalTotals?.score ?? null, + max: finalTotals?.max ?? null, + // Preserve earlier-level guarantees in the durable result. + regression: levelBundle?.totals?.regression ?? null, + selection: levelBundle?.selection ?? null, + ...(resumedRepair + ? { resumedRepair: firstBuild } + : continuing + ? { baseline: firstBuild, resumeCostUsd: requireBuild().costUsd, + resumeSession: requireBuildSession() } + : featureActionSequence !== null + ? { buildCostUsd: requireBuild().costUsd, buildSessions: [requireBuildSession()] } + : { firstBuild, buildCostUsd: requireBuild().costUsd, + buildSessions: [requireBuildSession()] }), + contractPass: levelBundle?.totals?.contractPass ?? null, + code: levelBundle?.code ?? null, + repairCostUsd: addCostUsd(repairCost), + repairSessions, + repairHistory, + sessionTotals, + tokens: sessionTotals.tokens, + usage: sessionTotals.usage, + turns: sessionTotals.turns, + promptBytes: sessionTotals.promptBytes, + tokensPerTurn: sessionTotals.turns + ? Math.round(sessionTotals.tokens / sessionTotals.turns) : null, + // Record actual reasoning because the provider default is not pinned. + thinking: sessionTotals.thinking, + repairs, + ...(resumedRepair ? { priorRepairs, + cumulativeRepairs: priorRepairs + repairs } : {}), + repair, + checkpoint, + // Keep the summary flag derived from the typed status so the two cannot drift. + stalled: repairStatus === 'budget-exhausted' + || ['repeated-findings', 'no-source-change'].includes(repairStopReason ?? ''), + regressed, + outcome: finalBundleOutcome, + durationSec: Math.round((Date.now() - t0) / 1000), + }); + if (!args.progression || requireProgressionState(progressionExecution?.state ?? null).attempts + .some(attempt => attempt.level === level && attempt.outcome === 'conclusive')) { + if (!featureDepthContinues && !run.validation.ladder.completedLevels.includes(level)) { + run.validation.ladder.completedLevels.push(level); + } + } + if (progressionState) synchronizeProgressionSummary(run, progressionState); + finalizeRunTotals(run, started, { costComplete: runCostComplete }); + run.outcome = aggregateRunOutcome(run.levels, progressionExecution?.state?.terminalOutcome); + persistRun(join(args.out, ARTIFACT_FILE.run), run); + if (progressionState && runCostComplete) { + commitTimeContinuationBoundary(outputDir, progressionState, level); + } + const blockedLevels = args.levelList.filter(candidate => candidate > level); + if (args.progression) { + const progressionState = requireProgressionState(progressionExecution?.state ?? null); + if (progressionState.phase === 'terminal') { + if (blockedLevels.length) run.validation.ladder.stoppedAfterLevel = level; + run.validation.ladder.blockedLevels = blockedLevels; + persistRun(join(args.out, ARTIFACT_FILE.run), run); + break; + } + if (progressionState.level <= level) { + if (progressionState.attempts.at(-1)?.outcome === 'inconclusive') { + run.validation.ladder.stoppedAfterLevel = level; + run.validation.ladder.blockedLevels = [level, ...blockedLevels]; + persistRun(join(args.out, ARTIFACT_FILE.run), run); + break; + } + if (featureActionSequence !== null && progressionState.level === level) { + levelIndex -= 1; + continue; + } + throw new Error(`dependency progression did not leave L${level} after its repair budget`); + } + continue; + } + if (blockedLevels.length && !ladderMayAdvance(finalBundleOutcome)) { + run.validation.ladder.stoppedAfterLevel = level; + run.validation.ladder.blockedLevels = blockedLevels; + persistRun(join(args.out, ARTIFACT_FILE.run), run); + console.log(` ladder paused after L${level}: L${level} must pass before ` + + `${blockedLevels.map(candidate => `L${candidate}`).join(', ')} can start`); + console.log(' inspect the failures, then explicitly grant more repairs or correct the benchmark'); + break; + } + } + + if (args.mutations) { + console.log(`\n================ ${args.backend} mutation control ================`); + const pristineOutcome = aggregateRunOutcome(run.levels, progressionExecution?.state?.terminalOutcome); + if (args.referenceMutationOnly || mutationControlEligible(pristineOutcome)) { + args.parentAttemptId = runId; + const baselineBundle = pristineMutationBaselinePath(args); + if (baselineBundle) args.mutationBaselineBundle = baselineBundle; + else delete args.mutationBaselineBundle; + run.mutationControl = runMutationControl(args, appDir, url, track, + run.setup?.isolation?.imageId ?? null); + } else { + console.log(` skipped: pristine outcome is ${pristineOutcome.kind}`); + run.mutationControl = { ok: false, skipped: true, + outcome: { kind: pristineOutcome.kind, phase: 'mutation-control-prerequisite', + reason: `pristine outcome is ${pristineOutcome.kind}` } }; + } + persistRun(join(args.out, ARTIFACT_FILE.run), run); + } + + // Record a final transcript audit in addition to the per-session hard gates. + // The same retry and diagnostic path is used at both gates. + let finalAuditFailure = null; + const finalAudit = auditContamination(appDir, auditNetwork(), auditsTranscripts); + if (!finalAudit) { + run.contaminated = false; + run.contamination = { evidence: 'no agent access to private benchmark files detected', + verdict: 'private-access audit passed' }; + } else if (finalAudit.kind === 'contaminated') { + run.contaminated = true; + run.contamination = { evidence: finalAudit.evidence, verdict: finalAudit.verdict }; + console.log('\n CONTAMINATED: restricted file or network access attempts were detected:'); + for (const evidence of finalAudit.evidence) console.log(` ${evidence}`); + console.log(' Scores from this run must not be quoted.'); + } else { + run.contaminated = false; + run.contamination = { evidence: finalAudit.evidence, verdict: finalAudit.verdict }; + const reason = finalAudit.evidence.join('; '); + finalAuditFailure = { kind: 'harness_failure', phase: 'contamination-audit', reason, + appFailures: [], inconclusive: [], harnessFailures: [reason] }; + console.log('\n HARNESS FAILURE: the contamination audit did not complete. Scores from this run must not be quoted.'); + } + + // Keep the transcript evidence outside the provider CLI's prunable store. + try { archiveTranscripts(appDir, artifactLabel); } + catch { console.log(' (transcript archiving failed — evidence is on a 30-day timer)'); } + + if (args.progression && progressionExecution?.state) { + synchronizeProgressionSummary(run, requireProgressionState(progressionExecution.state)); + } + run.outcome = finalAuditFailure ?? (args.referenceMutationOnly && run.mutationControl?.ok + ? { kind: 'passed', phase: 'mutation-control', reason: null, + appFailures: [], inconclusive: [], harnessFailures: [] } + : aggregateRunOutcome(run.levels, progressionExecution?.state?.terminalOutcome)); + if (args.mutations && !run.mutationControl?.ok && !run.mutationControl?.skipped) { + run.outcome = { kind: run.mutationControl?.outcome?.kind === 'incomplete' + ? 'incomplete' : 'harness_failure', phase: 'mutation-control', + reason: run.mutationControl?.outcome?.reason + ?? run.mutationControl?.processError + ?? 'one or more declared mutations were not cleanly caught', + appFailures: [], inconclusive: [] }; + } + + if (finalPackageEvidenceRequired(run.outcome, run.levels)) { + try { + preserveFinalPackageEvidence({ appDir, outputDir }); + console.log(` source kept at ${join(outputDir, 'source')}`); + console.log(` grading detail kept at ${join(outputDir, 'grading')}`); + } catch (error) { + const reason = errorMessage(error).split(/\r?\n/)[0] ?? 'evidence preservation failed'; + run.outcome = { kind: 'harness_failure', phase: 'evidence-preservation', reason, + appFailures: [], inconclusive: [], harnessFailures: [reason] }; + console.log(` stopped: ${reason}`); + } + } + + finalizeRunTotals(run, started, { costComplete: runCostComplete }); + if (args.repairGrant) { + const continuation = requireContinuation(run); + const totals = requireRunTotals(run); + continuation.cumulativeCostAfterUsd = addCostUsd(continuation.cumulativeCostBeforeUsd, totals.costUsd); + continuation.cumulativeDurationAfterSec = continuation.cumulativeDurationBeforeSec + totals.durationSec; + } + run.completedAt = new Date().toISOString(); + persistRun(join(args.out, ARTIFACT_FILE.run), run); + + console.log(`\n================ ${args.backend} summary ================`); + for (const l of run.levels) { + console.log(` ${formatLevelSummary(l)}`); + } + const totals = requireRunTotals(run); + console.log(` TOTAL ${totals.score}/${totals.max} ` + + `$${totals.costUsd} ${totals.repairs} repair(s) ${totals.durationSec}s`); + console.log(` ${join(outputDir, ARTIFACT_FILE.run)}`); + + teardown(); + + // Remove only the temporary directory created by this run. + if (ownWorkDir) { + try { + rmSync(dirname(appDir), { recursive: true, force: true }); + } catch (error) { + console.log(` could not remove work directory ${dirname(appDir)}: ${errorMessage(error)}`); + } + } + process.exitCode = runExitCode(run.outcome); +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + main().catch(error => { + console.error(redactCredentials(error instanceof Error ? error.stack ?? error.message : errorMessage(error))); + try { emergencyTeardown?.(); } + catch (cleanupError) { + console.error(`cleanup after failure also failed: ${errorMessage(cleanupError).split(/\r?\n/)[0]}`); + } + process.exitCode = 1; + }); +} diff --git a/tools/stack-bench/commands/campaign-cli.ts b/tools/stack-bench/commands/campaign-cli.ts new file mode 100644 index 00000000000..11b2faf2e8f --- /dev/null +++ b/tools/stack-bench/commands/campaign-cli.ts @@ -0,0 +1,433 @@ +#!/usr/bin/env node + +import { existsSync } from 'node:fs'; +import { basename, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { compileCampaignFile } from '../src/campaigns/campaign-compiler.js'; +import { CAMPAIGN_MODE_REGISTRY } from '../src/campaigns/campaign-mode.js'; +import { executeCampaign, inspectCampaign, reconcileCampaign } + from '../src/campaigns/campaign-runner.js'; +import { inspectCampaignSummary } from '../src/campaigns/campaign-inspection.js'; +import { exportCampaignReport, generateCampaignReport } from '../src/campaigns/campaign-report.js'; +import { publishCampaign } from '../src/campaigns/campaign-publish.js'; +import { grantCampaignDependencyRepairs } + from '../src/campaigns/campaign-progression-grant.js'; +import { requestCampaignTimeGrant } from '../src/campaigns/campaign-time-grant.js'; +import { readCampaignProviderContinuationStatus, requestCampaignProviderContinuation } + from '../src/campaigns/campaign-provider-continuation.js'; +import { auditProgressionReferenceCampaign, formatProgressionReferenceCampaignAudit } + from '../src/campaigns/progression-reference-campaign-audit.js'; +import type { ReferenceCampaignAudit } + from '../src/campaigns/progression-reference-campaign-audit.js'; +import { prepareCampaignExtension } from '../src/campaigns/campaign-extension.js'; +import { campaignDepthPauseStatus, continueCampaignDepth } from '../src/campaigns/campaign-depth-pause.js'; +import { statusWord } from '../src/evidence/status-words.js'; +import { readCampaignLock, requestCampaignCancellation } from '../src/campaigns/campaign-lock.js'; +import { SAFE_RESULT_NAME } from '../src/runtime/operational-paths.js'; + +interface CampaignSummaryPlan { + id: string; + version: string; + contentSha256: string; +} + +interface CampaignSummaryState { + status: string; + summary: unknown; + attempts: Array<{ + plan: { id: string }; + status: string; + executions: Array<{ + id: string; + outcome: unknown; + reason: string | null; + cleanupFailure?: string; + }>; + }>; +} + +interface ReferenceCampaignPlan { + attempts: Array<{ + mode?: { id?: string }; + agentAdapter?: string; + }>; +} + +interface ReferenceCampaignState { + status: string; +} + +interface ResumeCampaign { + plan: { + contentSha256: string; + definition: { mode?: { id?: string } }; + }; + state: { + status: string; + attempts: Array<{ executions: readonly unknown[] }>; + }; +} + +type ReferenceCampaignAuditFunction = (directory: string) => ReferenceCampaignAudit | null; + +export type CampaignArgs = + | { command: 'pause-status'; directory: string } + | { command: 'continue-depth'; directory: string } + | { command: 'continue-provider'; directory: string; attemptId: string; requestId: string } + | { command: 'continuation-status'; directory: string; attemptId: string; json: boolean } + | { command: 'grant-time'; directory: string; attemptId: string; grantId: string; minutes: number } + | { command: 'modes' } + | { command: 'validate'; path: string } + | { command: 'show'; path: string } + | { command: 'status'; directory: string; full: boolean } + | { command: 'inspect'; directory: string } + | { command: 'report'; directory: string } + | { command: 'export'; directory: string; output: string } + | { command: 'publish'; directory: string; suite: string; output?: string; to?: string; + exclusions: ReadonlyMap } + | { command: 'stop'; directory: string } + | { command: 'audit'; directory: string } + | { command: 'grant-repairs'; directory: string; attemptId: string; grantId: string; + level: number; nodeIds: string[]; repairs: number } + | { command: 'extend'; path: string; parentDirectory: string; fromDepth: number; + directory: string; prepareOnly: boolean } + | { command: 'trial'; path: string; directory: string } + | { command: 'run'; path: string; directory: string } + | { command: 'resume'; path: string; directory: string } + | { command: 'reconcile'; path: string; directory: string }; + +function isOneOf(value: string | undefined, + values: readonly T[]): value is T { + return value !== undefined && values.some(candidate => candidate === value); +} + +export function campaignStateSummary(plan: CampaignSummaryPlan, state: CampaignSummaryState) { + const failures = state.attempts.flatMap(attempt => { + const execution = attempt.executions.at(-1); + // A running execution whose cleanup failed awaits reconcile; show why. + const cleanupFailure = execution?.outcome === null ? execution.cleanupFailure : undefined; + if (!execution || (!cleanupFailure + && (execution.outcome === null || execution.outcome === 'passed'))) return []; + return [{ + attempt: attempt.plan.id, + status: statusWord(attempt.status), + execution: execution.id, + outcome: cleanupFailure ? 'cleanup failed' : statusWord(String(execution.outcome)), + reason: cleanupFailure ?? execution.reason, + }]; + }); + return { + campaign: { id: plan.id, version: plan.version, sha256: plan.contentSha256 }, + status: statusWord(state.status), + summary: state.summary, + failures, + }; +} + +export function auditCompletedReferenceCampaign(directory: string, plan: ReferenceCampaignPlan, + state: ReferenceCampaignState, { + audit = auditProgressionReferenceCampaign, +}: { audit?: ReferenceCampaignAuditFunction } = {}): ReferenceCampaignAudit | null { + const hasReferenceProgression = plan.attempts.some(attempt => + attempt.mode?.id === 'dependency' && attempt.agentAdapter === 'reference-fixture'); + return state.status === 'completed' && hasReferenceProgression ? audit(directory) : null; +} + +export function validateResumeCampaignState( + requested: { contentSha256: string }, existing: T): T { + if (requested.contentSha256 !== existing.plan.contentSha256) { + throw new Error('resume requires the exact campaign plan already stored in the output directory'); + } + if (existing.plan.definition.mode?.id !== 'dependency') { + throw new Error('resume is available only for dependency campaigns'); + } + const executions = existing.state.attempts.reduce((total, attempt) => + total + attempt.executions.length, 0); + if (existing.state.status !== 'prepared' || executions < 1) { + throw new Error('resume requires a dependency campaign with scheduled work'); + } + return existing; +} + +export function validateResumeCampaign(path: string, directory: string): ResumeCampaign { + return validateResumeCampaignState(compileCampaignFile(path), inspectCampaign(directory)); +} + +/** A new campaign directory must have a name the dashboard can open; existing ones continue. */ +function newCampaignDirectory(path: string): string { + const directory = resolve(path); + if (!existsSync(directory) && !SAFE_RESULT_NAME.test(basename(directory))) { + throw new Error(`campaign folder name "${basename(directory)}" is not supported: use 3-120 ` + + 'lowercase letters, digits, dots, or hyphens, starting with a letter or digit'); + } + return directory; +} + +export function parseCampaignArgs(argv: string[]): CampaignArgs { + const [command, path, ...rest] = argv.slice(2); + if ((command === 'pause-status' || command === 'continue-depth') && path && rest.length === 0) { + return { command, directory: resolve(path) }; + } + if ((command === 'continue-provider' || command === 'continuation-status') && path) { + const options = new Map(); + let json = false; + for (let i = 0; i < rest.length; i++) { + const flag = rest[i]!; + if (flag === '--json' && command === 'continuation-status' && !json) { json = true; continue; } + if (!['--attempt', ...(command === 'continue-provider' ? ['--request-id'] : [])].includes(flag) + || options.has(flag) || !rest[i + 1] || rest[i + 1]!.startsWith('--')) { + throw new Error('invalid provider continuation options'); + } + options.set(flag, rest[++i]!); + } + const attemptId = options.get('--attempt'); + if (!attemptId) throw new Error('provider continuation requires --attempt'); + if (command === 'continuation-status') return { command, directory: resolve(path), attemptId, json }; + const requestId = options.get('--request-id'); + if (!requestId) throw new Error('continue-provider requires --request-id'); + return { command, directory: resolve(path), attemptId, requestId }; + } + if (command === 'modes' && path === undefined) return { command }; + if (isOneOf(command, ['validate', 'show']) && path && rest.length === 0) { + return { command, path: resolve(path) }; + } + if (command === 'status' && path + && (rest.length === 0 || (rest.length === 1 && rest[0] === '--full'))) { + return { command, directory: resolve(path), full: rest.length === 1 }; + } + if (isOneOf(command, ['inspect', 'report', 'audit', 'stop']) && path && rest.length === 0) { + return { command, directory: resolve(path) }; + } + if (command === 'export' && path && rest.length === 2 && rest[0] === '--out' && rest[1]) { + return { command, directory: resolve(path), output: resolve(rest[1]) }; + } + if (command === 'publish') { + const options = new Map(); + const exclusions = new Map(); + for (let i = 0; i < rest.length; i += 2) { + const flag = rest[i]; const value = rest[i + 1]; + if (flag === '--exclude' && value && !value.startsWith('--')) { + // --exclude = + const at = value.indexOf('='); + if (at < 1 || exclusions.has(value.slice(0, at))) throw new Error('invalid publish options'); + exclusions.set(value.slice(0, at), value.slice(at + 1)); + continue; + } + if (!flag || !['--suite', '--out', '--to'].includes(flag) || !value + || value.startsWith('--') || options.has(flag)) throw new Error('invalid publish options'); + options.set(flag, value); + } + const suite = options.get('--suite'); + if (!path || !suite || options.has('--out') === options.has('--to')) { + throw new Error('publish requires --suite and exactly one of --out or --to'); + } + return { command, directory: resolve(path), suite, exclusions, + ...(options.has('--out') ? { output: resolve(options.get('--out')!) } : { to: options.get('--to')! }) }; + } + if (command === 'grant-time' && path) { + const options = new Map(); + for (let i = 0; i < rest.length; i += 2) { + const flag = rest[i]; const value = rest[i + 1]; + if (!flag || !['--attempt', '--grant-id', '--minutes'].includes(flag) + || !value || options.has(flag)) throw new Error('invalid grant-time options'); + options.set(flag, value); + } + const minutes = Number(options.get('--minutes')); + if (!options.get('--attempt') || !options.get('--grant-id') + || !Number.isSafeInteger(minutes * 60_000) || !Number.isInteger(minutes) || minutes <= 0) { + throw new Error('grant-time requires --attempt, --grant-id, --minutes '); + } + return { command, directory: resolve(path), attemptId: options.get('--attempt')!, + grantId: options.get('--grant-id')!, minutes }; + } + if (command === 'grant-repairs' && path) { + const values: { attemptId?: string; grantId?: string; level?: number; repairs?: number; + nodeIds: string[] } = { nodeIds: [] }; + const seen = new Set(); + for (let index = 0; index < rest.length; index += 2) { + const flag = rest[index]; + const value = rest[index + 1]; + if (flag === undefined || value === undefined + || !['--attempt', '--grant-id', '--level', '--feature', '--repairs'].includes(flag) + || (flag !== '--feature' && seen.has(flag))) { + throw new Error(`invalid or duplicate grant-repairs option ${String(flag)}`); + } + seen.add(flag); + if (flag === '--attempt') values.attemptId = value; + else if (flag === '--grant-id') values.grantId = value; + else if (flag === '--level') values.level = Number(value); + else if (flag === '--repairs') values.repairs = Number(value); + else values.nodeIds.push(value); + } + if (!values.attemptId || !values.grantId || typeof values.level !== 'number' + || !Number.isSafeInteger(values.level) || typeof values.repairs !== 'number' + || !Number.isSafeInteger(values.repairs) || values.nodeIds.length === 0) { + throw new Error('grant-repairs requires --attempt, --grant-id, --level, ' + + 'one or more --feature values, and --repairs'); + } + return { command, directory: resolve(path), attemptId: values.attemptId, + grantId: values.grantId, level: values.level, nodeIds: values.nodeIds, + repairs: values.repairs }; + } + if (command === 'extend' && path && (rest.length === 6 + || (rest.length === 7 && rest[6] === '--prepare-only')) + && rest[0] === '--from' && rest[2] === '--depth' && rest[4] === '--out') { + const fromDepth = Number(rest[3]); + if (!Number.isSafeInteger(fromDepth) || fromDepth < 1) { + throw new Error('extend --depth must be a positive integer'); + } + return { command, path: resolve(path), parentDirectory: resolve(rest[1]!), + fromDepth, directory: newCampaignDirectory(rest[5]!), prepareOnly: rest.length === 7 }; + } + if (isOneOf(command, ['trial', 'run', 'resume', 'reconcile']) + && path && rest.length === 2 && rest[0] === '--out') { + return { command, path: resolve(path), + directory: command === 'run' || command === 'trial' ? newCampaignDirectory(rest[1]!) : resolve(rest[1]!) }; + } + throw new Error('usage: campaign-cli.js modes | validate|show ' + + '| trial|run|resume|reconcile --out ' + + '| extend --from --depth --out [--prepare-only] ' + + '| status [--full] | inspect|report|audit|stop | export --out ' + + '| publish --suite (--to | --out ) ' + + '| grant-repairs --attempt --grant-id --level ' + + '--feature [--feature ...] --repairs ' + + '| grant-time --attempt --grant-id --minutes ' + + '| continue-provider --attempt --request-id ' + + '| continuation-status --attempt [--json] ' + + '| pause-status|continue-depth '); +} + +async function main() { + const args = parseCampaignArgs(process.argv); + if (args.command === 'pause-status' || args.command === 'continue-depth') { + console.log(JSON.stringify(args.command === 'pause-status' + ? campaignDepthPauseStatus(args.directory) : continueCampaignDepth(args.directory), null, 2)); + return; + } + if (args.command === 'modes') { + console.log(JSON.stringify(CAMPAIGN_MODE_REGISTRY.ids.map(value => { + const [id, version] = value.split('@'); + return { id, version }; + }), null, 2)); + return; + } + if (args.command === 'status') { + const campaign = inspectCampaign(args.directory, { requireCurrentInputs: false }); + console.log(JSON.stringify(args.full + ? campaign.state + : campaignStateSummary(campaign.plan, campaign.state), null, 2)); + return; + } + if (args.command === 'inspect') { + console.log(JSON.stringify(inspectCampaignSummary(args.directory), null, 2)); + return; + } + if (args.command === 'report') { + const generated = generateCampaignReport(args.directory); + console.log(`${generated.reportPath}\n${generated.htmlPath}\n${generated.report.contentSha256}`); + return; + } + if (args.command === 'export') { + console.log(exportCampaignReport(args.directory, args.output)); + return; + } + if (args.command === 'publish') { + console.log(JSON.stringify(await publishCampaign(args.directory, args), null, 2)); + return; + } + if (args.command === 'stop') { + const lock = readCampaignLock(args.directory); + if (!lock || !requestCampaignCancellation(args.directory, + { id: lock.campaignId, contentSha256: lock.campaignSha256 }, lock.ownershipMarkerSha256)) { + throw new Error('campaign has no current controller to stop'); + } + console.log('Stop requested. The controller will stop its children and release owned resources.'); + return; + } + if (args.command === 'audit') { + const report = auditProgressionReferenceCampaign(args.directory); + if (report === null) throw new Error('campaign has no dependency reference attempts to audit'); + console.log(formatProgressionReferenceCampaignAudit(report)); + if (!report.ok) process.exitCode = 1; + return; + } + if (args.command === 'continuation-status') { + const status = readCampaignProviderContinuationStatus(args.directory, args.attemptId); + console.log(args.json ? JSON.stringify(status, null, 2) + : status.eligible ? 'Waiting: eligible for provider continuation.' : `Ineligible: ${status.reason}`); + return; + } + if (args.command === 'continue-provider') { + console.log(JSON.stringify(requestCampaignProviderContinuation(args.directory, args), null, 2)); + return; + } + if (args.command === 'grant-time') { + console.log(JSON.stringify(requestCampaignTimeGrant(args.directory, { + attemptId: args.attemptId, grantId: args.grantId, minutes: args.minutes }), null, 2)); + return; + } + if (args.command === 'grant-repairs') { + console.log(JSON.stringify(grantCampaignDependencyRepairs(args.directory, { + attemptId: args.attemptId, + grantId: args.grantId, + level: args.level, + nodeIds: args.nodeIds, + repairs: args.repairs, + }), null, 2)); + return; + } + if (args.command === 'extend') { + prepareCampaignExtension(args.path, args.parentDirectory, args.directory, args.fromDepth); + if (args.prepareOnly) { + console.log(JSON.stringify({ status: 'prepared', directory: args.directory, + parentDirectory: args.parentDirectory, fromDepth: args.fromDepth }, null, 2)); + return; + } + const plan = compileCampaignFile(args.path); + const state = await executeCampaign(args.path, args.directory, { mode: 'frozen' }); + console.log(JSON.stringify(campaignStateSummary(plan, state), null, 2)); + if (state.status !== 'completed') process.exitCode = 1; + return; + } + const plan = compileCampaignFile(args.path); + if (args.command === 'reconcile') { + const state = reconcileCampaign(args.path, args.directory); + console.log(JSON.stringify(campaignStateSummary(plan, state), null, 2)); + return; + } + if (args.command === 'trial' || args.command === 'run' || args.command === 'resume') { + if (args.command === 'resume') validateResumeCampaign(args.path, args.directory); + const cancellation = new AbortController(); + const cancel = () => cancellation.abort(); + process.on('SIGINT', cancel); + process.on('SIGTERM', cancel); + let state; + try { + const executionMode = args.command === 'trial' + || (args.command === 'resume' && plan.state === 'draft') + ? 'model-free-trial' : 'frozen'; + state = await executeCampaign(args.path, args.directory, { + mode: executionMode, + signal: cancellation.signal, + }); + } finally { + process.off('SIGINT', cancel); + process.off('SIGTERM', cancel); + } + console.log(JSON.stringify(campaignStateSummary(plan, state), null, 2)); + const audit = auditCompletedReferenceCampaign(args.directory, plan, state); + if (audit !== null) console.log(formatProgressionReferenceCampaignAudit(audit)); + if (state.status !== 'completed' || audit?.ok === false) process.exitCode = 1; + return; + } + if (args.command === 'show') console.log(JSON.stringify(plan, null, 2)); + else console.log(`${plan.id}@${plan.version} ${plan.state}: ${plan.summary.attempts} attempts, ${plan.contentSha256}`); +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 2; + }); +} diff --git a/tools/stack-bench/commands/check-calibration.ts b/tools/stack-bench/commands/check-calibration.ts new file mode 100644 index 00000000000..e86f65c3635 --- /dev/null +++ b/tools/stack-bench/commands/check-calibration.ts @@ -0,0 +1,65 @@ +#!/usr/bin/env node + +import { existsSync, readFileSync, readdirSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { compileCalibrationDefinition, compileCalibrationFile } from '../src/composition/calibration-compiler.js'; +import { buildRecipeRelease } from '../src/composition/recipe-release.js'; +import { listTracks, TRACKS_DIR } from '../src/composition/tracks.js'; + +import { STACK_BENCH_ROOT as ROOT } from '../src/package-root.js'; + +export interface CalibrationCheckResult { + track: string; + id: string; + recipe: string; + controls: number; + stacks: number; + contentSha256: string; +} + +export function checkCalibrations( + { trackName = null }: { trackName?: string | null } = {}, +): CalibrationCheckResult[] { + const availableTracks = listTracks({ includeInternal: true }); + if (trackName && !availableTracks.includes(trackName)) { + throw new Error(`unknown calibration track ${trackName}`); + } + const tracks = trackName ? [trackName] : availableTracks; + const results: CalibrationCheckResult[] = []; + for (const name of tracks) { + const trackRoot = join(TRACKS_DIR, name); + const directory = join(trackRoot, 'composition', 'calibrations'); + if (!existsSync(directory)) continue; + for (const file of readdirSync(directory).filter(candidate => candidate.endsWith('.json')).sort()) { + const path = join(directory, file); + const source = `composition/calibrations/${file}`; + const input = JSON.parse(readFileSync(path, 'utf8')); + const definition = compileCalibrationDefinition(input, { source }); + const recipePath = resolve(dirname(path), definition.recipe.path); + const release = buildRecipeRelease(recipePath, { trackRoot }); + const plan = compileCalibrationFile(path, { trackRoot, stackBenchRoot: ROOT, release }); + results.push({ track: name, id: plan.id, + recipe: plan.recipe.id, controls: plan.controls.length, stacks: plan.qualification.stacks.length, + contentSha256: plan.contentSha256 }); + } + } + return results; +} + +function main() { + const results = checkCalibrations(); + for (const result of results) { + console.log(`${result.track}: ${result.id}; ` + + `${result.controls} controls, ${result.stacks} stacks, ${result.contentSha256.slice(0, 12)}`); + } +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { main(); } + catch (error: unknown) { + console.error(error instanceof Error ? error.stack ?? error.message : String(error)); + process.exitCode = 1; + } +} diff --git a/tools/stack-bench/commands/check-composition.ts b/tools/stack-bench/commands/check-composition.ts new file mode 100644 index 00000000000..e2ec804b47a --- /dev/null +++ b/tools/stack-bench/commands/check-composition.ts @@ -0,0 +1,78 @@ +import { existsSync, readFileSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { + compileFixtureDefinition, + compilePackDefinition, + compileRecipeFile, + compileRecipeSelectionFile, +} from '../src/composition/composition-compiler.js'; +import { TRACKS_DIR, listTracks } from '../src/composition/tracks.js'; + +function json(path: string): unknown { + try { return JSON.parse(readFileSync(path, 'utf8')); } + catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`cannot read composition source ${path}: ${message}`, { cause: error }); + } +} + +export interface CompositionSummary { + track: string; + packs: number; + fixtures: number; + recipes: number; + checks: number; + selections: number; +} + +export function checkCompositions( + { trackName = null }: { trackName?: string | null } = {}, +): CompositionSummary[] { + const names = trackName ? [trackName] : listTracks({ includeInternal: true }); + const summary = []; + for (const name of names) { + const trackRoot = join(TRACKS_DIR, name); + const root = join(trackRoot, 'composition'); + if (!existsSync(root)) { + if (trackName) throw new Error(`track ${name} has no composition directory`); + continue; + } + const packs = join(root, 'packs'); + const fixtures = join(root, 'fixtures'); + const recipes = join(root, 'recipes'); + const packFiles = readdirSync(packs).filter(file => file.endsWith('.json')).sort(); + const fixtureFiles = readdirSync(fixtures).filter(file => file.endsWith('.json')).sort(); + const recipeFiles = readdirSync(recipes).filter(file => file.endsWith('.json')).sort(); + if (!packFiles.length || !fixtureFiles.length || !recipeFiles.length) { + throw new Error(`track ${name} composition must contain packs, fixtures, and recipes`); + } + for (const file of packFiles) { + const path = join(packs, file); + compilePackDefinition(json(path), { source: path }); + } + for (const file of fixtureFiles) { + const path = join(fixtures, file); + compileFixtureDefinition(json(path), { source: path }); + } + const plans = recipeFiles.map(file => compileRecipeFile(join(recipes, file), { trackRoot })); + const selectionFiles = readdirSync(root).filter(file => file.endsWith('.json')).sort(); + const selections = selectionFiles.reduce((total, file) => total + + compileRecipeSelectionFile(join(root, file), { trackRoot }).entries.length, 0); + summary.push({ track: name, packs: packFiles.length, fixtures: fixtureFiles.length, + recipes: plans.length, checks: plans.reduce((total, plan) => total + plan.checks.length, 0), + selections }); + } + return summary; +} + +function main(): void { + const summary = checkCompositions(); + if (!summary.length) throw new Error('no composition sources found'); + for (const row of summary) { + console.log(`${row.track}: ${row.packs} packs, ${row.fixtures} fixtures, ${row.recipes} recipes, ${row.checks} selected checks, ${row.selections} recipe selections`); + } +} + +if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) main(); diff --git a/tools/stack-bench/commands/check-mutations.ts b/tools/stack-bench/commands/check-mutations.ts new file mode 100644 index 00000000000..042a6d9770b --- /dev/null +++ b/tools/stack-bench/commands/check-mutations.ts @@ -0,0 +1,81 @@ +#!/usr/bin/env node + +import { existsSync, readFileSync } from 'node:fs'; +import { parseArgs as parseNodeArgs } from 'node:util'; + +import { mutationFileEdits, resolveMutationFile, validateMutationDefinitions } + from '../src/evidence/mutation-analysis.js'; +import type { MutationDefinition } from '../src/evidence/mutation-analysis.js'; + +interface CliArgs { + app: string; + mutations: string; + quiet: boolean; +} + +interface MutationSpec { + anchoredTo?: unknown; + mutations?: MutationDefinition[]; +} + +function parseArgs(argv: string[]): CliArgs { + const { values: { app, mutations, quiet = false } } = parseNodeArgs({ args: argv.slice(2), + options: { app: { type: 'string' }, mutations: { type: 'string' }, quiet: { type: 'boolean' } } }); + if (!app || !mutations) { + console.error('Usage: node dist/commands/check-mutations.js --app --mutations '); + process.exit(2); + } + return { app, mutations, quiet }; +} + +const args = parseArgs(process.argv); +const parsed: unknown = JSON.parse(readFileSync(args.mutations, 'utf8')); +if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + throw new Error('mutation manifest must be an object'); +} +const spec = parsed as MutationSpec; +const say = (...message: unknown[]): void => { if (!args.quiet) console.log(...message); }; + +say(`mutations : ${args.mutations}`); +say(`app : ${args.app}`); +if (spec.anchoredTo) say(`anchored : ${String(spec.anchoredTo).split('.')[0]}`); +say(''); + +let bad = 0; +const definitions = validateMutationDefinitions(spec.mutations); +for (const issue of definitions.issues) { + console.log(` BAD MANIFEST ${issue.mutation ?? ''} -> ${issue.kind}`); + bad += 1; +} +for (const mutation of spec.mutations ?? []) { + const mutationId = String(mutation.id ?? ''); + for (const edit of mutationFileEdits(mutation)) { + let file: string; + try { file = resolveMutationFile(args.app, edit.file); } + catch { + console.log(` UNSAFE FILE ${mutationId} -> ${edit.file} escapes the app directory`); + bad += 1; + continue; + } + if (!existsSync(file)) { + console.log(` DEAD FILE ${mutationId} -> ${edit.file} does not exist in this app`); + bad += 1; + continue; + } + const source = readFileSync(file, 'utf8'); + const matches = source.split(edit.find).length - 1; + if (matches === 1) { + say(` ok ${mutationId} -> ${edit.file}`); + continue; + } + console.log(matches === 0 + ? ` DEAD ANCHOR ${mutationId} -> not found in ${edit.file}` + : ` AMBIGUOUS ${mutationId} -> matches ${matches}x in ${edit.file}; the edit would land in more than one place`); + bad += 1; + } +} + +console.log(bad + ? `\n${bad} problem(s) — these mutations cannot validate anything against this app.` + : '\nall anchors present and unique — this file can validate against this app.'); +process.exit(bad ? 1 : 0); diff --git a/tools/stack-bench/commands/check-scenarios.ts b/tools/stack-bench/commands/check-scenarios.ts new file mode 100644 index 00000000000..55fce70b454 --- /dev/null +++ b/tools/stack-bench/commands/check-scenarios.ts @@ -0,0 +1,317 @@ +#!/usr/bin/env node +// Check scenario action names, actors, UI hooks, and score totals without an app. + +import { existsSync, readFileSync, readdirSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parseArgs } from 'node:util'; + +import { ACTION_REGISTRY } from '../src/actions/action-catalog.js'; +import { compileRecipeFile, type CompiledOwnedTaskFragment, type CompiledRecipeRelease } + from '../src/composition/composition-compiler.js'; +import { compileScenarioDefinition, type CompiledStep } + from '../src/composition/definition-compiler.js'; +import { DEFAULT_TRACK, listTracks, loadTrack, type Track } + from '../src/composition/tracks.js'; + +interface ScenarioScope { + features: Map>; + contractOwners: Set; + requirementOwners: Set; + contractText: string; + requirementText: string; +} + +interface RecipeSource { + baseRecipe: string | null; + isolatesSelectedSources: boolean; +} + +type HooksByLevel = Map>; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function readJson(path: string): unknown { + return JSON.parse(readFileSync(path, 'utf8')) as unknown; +} + +function readRecipeSource(path: string): RecipeSource { + const source = readJson(path); + if (!isRecord(source)) throw new Error(`${path}: recipe must be an object`); + const task = source.task; + if (!isRecord(task)) throw new Error(`${path}: recipe task must be an object`); + const baseRecipe = task.baseRecipe; + let baseRecipePath: string | null = null; + if (baseRecipe !== undefined) { + if (!isRecord(baseRecipe) || typeof baseRecipe.path !== 'string') { + throw new Error(`${path}: task.baseRecipe.path must be a string`); + } + baseRecipePath = baseRecipe.path; + } + return { + baseRecipe: baseRecipePath, + isolatesSelectedSources: source.execution === 'all-selected-sources', + }; +} + +function contractHookIds(path: string): string[] { + const contract = readJson(path); + if (!isRecord(contract) || !Array.isArray(contract.hooks)) { + throw new Error(`${path}: hooks must be an array`); + } + return contract.hooks.map((hook, index) => { + if (!isRecord(hook) || typeof hook.id !== 'string') { + throw new Error(`${path}: hooks[${index}].id must be a string`); + } + return hook.id; + }); +} + +// Contract levels are cumulative. A level can use hooks introduced earlier. +function hooksByLevel(track: Track): HooksByLevel { + const perFile = new Map(); + for (const file of readdirSync(track.contracts).filter(name => /^\d\d-.*\.json$/.test(name))) { + perFile.set(file.slice(0, 2), contractHookIds(join(track.contracts, file))); + } + const byLevel: HooksByLevel = new Map(); + for (const level of perFile.keys()) { + const ids = [...perFile.entries()] + .filter(([candidate]) => candidate <= level) + .flatMap(([, hookIds]) => hookIds); + byLevel.set(level, new Set(ids)); + } + return byLevel; +} + +function ownedFragment( + fragment: CompiledOwnedTaskFragment, + owners: ReadonlySet, +): boolean { + return fragment.owners.some(owner => owners.has(owner)); +} + +function recipeScenarioScopes(track: Track, recipeFile: string): Map { + const recipeDir = join(track.dir, 'composition', 'recipes'); + const chain: CompiledRecipeRelease[] = []; + const seen = new Set(); + let currentFile: string | null = recipeFile; + let isolatesSelectedSources = false; + while (currentFile !== null) { + if (seen.has(currentFile)) throw new Error(`recipe base cycle at ${currentFile}`); + seen.add(currentFile); + const path = join(recipeDir, currentFile); + chain.push(compileRecipeFile(path, { trackRoot: track.dir })); + const source = readRecipeSource(path); + if (chain.length === 1) isolatesSelectedSources = source.isolatesSelectedSources; + currentFile = source.baseRecipe; + } + + const recipe = chain[0]; + if (recipe === undefined) throw new Error(`recipe chain is empty for ${recipeFile}`); + const packs = new Map(recipe.packs.map(pack => [pack.id, pack])); + const contracts = isolatesSelectedSources + ? recipe.recipe.task.contracts + : chain.flatMap(release => release.recipe.task.contracts); + const requirements = isolatesSelectedSources + ? recipe.recipe.task.requirements + : chain.flatMap(release => release.recipe.task.requirements); + const scopes = new Map(); + + const ownersFor = (check: CompiledRecipeRelease['checks'][number]): Set => { + const found = new Set([check.packId, ...(check.requiresFeatures ?? [])]); + const visit = (id: string): void => { + const pack = packs.get(id); + if (pack === undefined) return; + for (const reference of pack.requiresPacks) { + if (found.has(reference)) continue; + found.add(reference); + visit(reference); + } + }; + [...found].forEach(visit); + return found; + }; + + for (const check of recipe.checks) { + const source = check.source.replace(/^scenarios\//, ''); + const scope = scopes.get(source) ?? { + features: new Map>(), + contractOwners: new Set(), + requirementOwners: new Set(), + contractText: '', + requirementText: '', + }; + const criteria = scope.features.get(check.featureId) ?? new Set(); + criteria.add(check.criterionId); + scope.features.set(check.featureId, criteria); + for (const owner of ownersFor(check)) { + scope.contractOwners.add(owner); + scope.requirementOwners.add(owner); + } + scopes.set(source, scope); + } + + for (const scope of scopes.values()) { + const selectedContracts = isolatesSelectedSources + ? contracts.filter(fragment => ownedFragment(fragment, scope.contractOwners)) + : contracts; + const selectedRequirements = isolatesSelectedSources + ? requirements.filter(fragment => ownedFragment(fragment, scope.requirementOwners)) + : requirements; + scope.contractText = selectedContracts.map(fragment => fragment.text).join('\n'); + scope.requirementText = selectedRequirements.map(fragment => fragment.text).join('\n'); + } + return scopes; +} + +function normalizeText(text: string): string { + return text.replace(/\*\*/g, '').replace(/—/g, '-').toLowerCase().replace(/\s+/g, ' ').trim(); +} + +function promptFor(track: Track, level: string): string | null { + const dir = join(track.dir, 'prompts'); + if (!existsSync(dir)) return null; + const file = readdirSync(dir).find(name => name.startsWith(`${level}-`) && name.endsWith('.md')); + return file === undefined ? null : normalizeText(readFileSync(join(dir, file), 'utf8')); +} + +function referencedActors(step: CompiledStep): string[] { + return [step.do === 'expectCrashCheckout' ? undefined : step.from, step.fromActor] + .filter((actor): actor is string => actor !== undefined); +} + +export function referencedInterfaceValues(value: unknown, key: 'testid' | 'attribute'): string[] { + if (Array.isArray(value)) return value.flatMap(child => referencedInterfaceValues(child, key)); + if (!isRecord(value)) return []; + return Object.entries(value).flatMap(([name, child]) => name === key && typeof child === 'string' + ? [child] : referencedInterfaceValues(child, key)); +} + +function referencedTestIds(step: CompiledStep): string[] { + return referencedInterfaceValues(step, 'testid'); +} + +function main(args: readonly string[]): number { + const { values } = parseArgs({ args: [...args], options: { + track: { type: 'string' }, + recipe: { type: 'string' }, + }, strict: true, allowPositionals: false }); + const trackArg = values.track ?? null; + const recipeArg = values.recipe ?? null; + const availableTracks = listTracks(); + const trackNames = trackArg === null + ? (availableTracks.length > 0 ? availableTracks : [DEFAULT_TRACK]) + : [trackArg]; + if (recipeArg !== null && trackNames.length !== 1) { + throw new Error('--recipe requires one --track'); + } + + const knownActions = new Set(ACTION_REGISTRY.ids); + let problems = 0; + let unstatedWarnings = 0; + let staleStatementWarnings = 0; + const fail = (where: string, message: string): void => { + console.log(` ${where}: ${message}`); + problems += 1; + }; + + for (const name of trackNames) { + const track = loadTrack(name); + console.log(`# track: ${name}`); + const contracts = hooksByLevel(track); + const recipeScopes = recipeArg === null ? null : recipeScenarioScopes(track, recipeArg); + for (const file of readdirSync(track.scenarios).filter(candidate => candidate.endsWith('.json'))) { + const recipeScope = recipeScopes?.get(file); + if (recipeScopes !== null && recipeScope === undefined) continue; + const scenarioPath = join(track.scenarios, file); + let spec; + try { + spec = compileScenarioDefinition(readJson(scenarioPath), { source: scenarioPath }); + } catch (error: unknown) { + fail(file, error instanceof Error ? error.message : String(error)); + continue; + } + const level = String(spec.level).padStart(2, '0'); + const hooks = recipeScope === undefined ? (contracts.get(level) ?? null) : null; + + console.log(file); + const prompt = recipeScope === undefined + ? promptFor(track, level) + : normalizeText(recipeScope.requirementText); + for (const feature of spec.features) { + const selectedCriteria = recipeScope?.features.get(feature.id); + if (recipeScope !== undefined && selectedCriteria === undefined) continue; + const criteria = selectedCriteria === undefined + ? feature.criteria + : feature.criteria.filter(criterion => selectedCriteria.has(criterion.id)); + for (const criterion of criteria) { + if (criterion.statedBy !== undefined) { + if (recipeScope === undefined && prompt !== null + && !prompt.includes(normalizeText(criterion.statedBy))) { + staleStatementWarnings += 1; + console.log(` warn F${feature.id} ${criterion.id}: statedBy text is not in the level ${level} prompt`); + } + } else if (recipeScope === undefined && criterion.points > 0) { + unstatedWarnings += 1; + console.log(` warn F${feature.id} ${criterion.id}: carries ${criterion.points} point(s) with no statedBy - the requirement may be unstated`); + } + } + + const actors = new Set(feature.actors ?? []); + const steps = [...feature.setup, ...criteria.flatMap(criterion => criterion.steps)]; + const declared = (actor: string): boolean => actors.has(actor) + || [...actors].some(candidate => actor.startsWith(`${candidate}-`)); + for (const step of steps) { + const at = `F${feature.id} ${step.do}`; + if (!knownActions.has(step.do)) fail(at, `unknown step type "${step.do}"`); + if (step.actor !== undefined && actors.size > 0 && !declared(step.actor)) { + fail(at, `actor "${step.actor}" is not in the feature's actor list`); + } + for (const actor of referencedActors(step)) { + if (actors.size > 0 && !declared(actor)) { + fail(at, `actor "${actor}" is not in the feature's actor list`); + } + } + if (hooks !== null) { + for (const id of referencedTestIds(step)) { + if (!hooks.has(id)) fail(at, `testid "${id}" is not in the contract`); + } + } + if (recipeScope !== undefined) { + for (const id of referencedTestIds(step)) { + const declaredStaffRow = id.startsWith('staff-role-account-') && id.length > 'staff-role-account-'.length + && recipeScope.contractText.includes('`staff-role-account-`'); + if (!recipeScope.contractText.includes(`\`${id}\``) && !declaredStaffRow) { + fail(at, `testid "${id}" is not in the selected recipe contracts`); + } + } + for (const attribute of referencedInterfaceValues(step, 'attribute')) { + if (attribute.startsWith('data-') && !recipeScope.contractText.includes(`\`${attribute}\``)) { + fail(at, `attribute "${attribute}" is not in the selected recipe contracts`); + } + } + } + } + + const points = criteria.reduce((total, criterion) => total + criterion.points, 0); + if (recipeScope === undefined && feature.max !== undefined && points !== feature.max) { + fail(`F${feature.id}`, `criteria total ${points} but max says ${feature.max}`); + } + } + } + } + + const warnings = unstatedWarnings + staleStatementWarnings; + console.log(problems > 0 + ? `\n${problems} error(s); ${warnings} warning(s)` + : warnings > 0 + ? `\n0 errors; ${warnings} warning(s) (${unstatedWarnings} point-carrying criteria lack statedBy; ${staleStatementWarnings} statedBy references are outside level prompts)` + : '\n0 errors; 0 warnings'); + return problems > 0 ? 1 : 0; +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + process.exitCode = main(process.argv.slice(2)); +} diff --git a/tools/stack-bench/commands/composition-cli.ts b/tools/stack-bench/commands/composition-cli.ts new file mode 100644 index 00000000000..13411828c2c --- /dev/null +++ b/tools/stack-bench/commands/composition-cli.ts @@ -0,0 +1,315 @@ +#!/usr/bin/env node + +import { existsSync, readdirSync, realpathSync } from 'node:fs'; +import { join, relative, resolve, sep } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parseArgs } from 'node:util'; + +import { compilePackDefinition, compileRecipeFile, resolveTaskFragment } from '../src/composition/composition-compiler.js'; +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; +import { canonicalDefinitionJson, readDefinitionJson } + from '../src/composition/definition-plan.js'; +import { buildRecipeRelease } from '../src/composition/recipe-release.js'; +import { composeSelectedRecipeTask, selectRecipeRelease } from '../src/composition/recipe-selection.js'; +import { TRACKS_DIR } from '../src/composition/tracks.js'; +import type { CompiledPackDefinition, CompiledRecipePlan } from '../src/composition/composition-compiler.js'; +import type { RecipeRelease } from '../src/composition/recipe-release.js'; +import type { RecipeSelectionOptions, SelectedRecipeRelease } from '../src/composition/recipe-selection.js'; + +export { selectRecipeRelease } from '../src/composition/recipe-selection.js'; + +interface TrackRootOptions { + trackRoot: string; +} + +interface PackIndexEntry { + pack: CompiledPackDefinition; + path: string; +} + +interface CalibrationValue { + id: string; + recipe?: { id?: string; contentSha256?: string }; +} + +type RecipeOptions = TrackRootOptions & RecipeSelectionOptions; +type RecipeTaskKind = 'requirements' | 'contracts'; + +function contained(root: string, path: string, label: string): string { + const absoluteRoot = realpathSync(resolve(root)); + const candidate = resolve(path); + const lexical = relative(absoluteRoot, candidate); + if (lexical === '..' || lexical.startsWith(`..${sep}`)) throw new Error(`${label} escapes ${absoluteRoot}`); + if (!existsSync(candidate)) throw new Error(`${label} does not exist: ${candidate}`); + const absolute = realpathSync(candidate); + const physical = relative(absoluteRoot, absolute); + if (physical === '..' || physical.startsWith(`..${sep}`)) throw new Error(`${label} escapes ${absoluteRoot}`); + return absolute; +} + +function packIndex(trackRoot: string): Map { + const directory = join(trackRoot, 'composition', 'packs'); + const byRef = new Map(); + for (const name of readdirSync(directory).filter(file => file.endsWith('.json')).sort()) { + const path = join(directory, name); + const pack = compilePackDefinition(readDefinitionJson(path, 'pack'), { + source: relative(trackRoot, path).replaceAll('\\', '/'), + }); + if (byRef.has(pack.id)) throw new Error(`duplicate pack id ${pack.id}`); + byRef.set(pack.id, { pack, path: realpathSync(path) }); + } + for (const [ref, { pack }] of byRef) { + for (const dependency of [...pack.requiresPacks, ...pack.conflictsWith]) { + if (!byRef.has(dependency)) throw new Error(`${ref} references missing pack ${dependency}`); + } + } + return byRef; +} + +export function validatePackFile(path: string, options: Partial = {}) { + const { trackRoot } = options; + if (trackRoot === undefined) throw new Error('pack validation requires trackRoot'); + const root = realpathSync(resolve(trackRoot)); + const absolute = contained(join(root, 'composition'), path, 'pack path'); + const pack = compilePackDefinition(readDefinitionJson(absolute, 'pack'), { + source: relative(root, absolute).replaceAll('\\', '/'), + }); + const packs = packIndex(root); + const indexed = packs.get(pack.id); + if (!indexed || indexed.path !== absolute) throw new Error(`${pack.id} is not the indexed source ${absolute}`); + for (const ref of [...pack.requiresPacks, ...pack.conflictsWith]) { + if (!packs.has(ref)) throw new Error(`${pack.id} references missing pack ${ref}`); + } + const sourceCache = new Map(); + for (const kind of ['requirements', 'contracts'] satisfies RecipeTaskKind[]) { + for (const fragment of pack.task[kind]) { + resolveTaskFragment(fragment, { trackRoot: root, + source: `${relative(root, absolute).replaceAll('\\', '/')}.task.${kind}.${fragment.id}`, + sourceCache }); + } + } + const state = new Map(); + const visit = (ref: string, chain: string[] = []): void => { + if (state.get(ref) === 'done') return; + if (state.get(ref) === 'visiting') throw new Error(`pack dependency cycle: ${[...chain, ref].join(' -> ')}`); + state.set(ref, 'visiting'); + const entry = packs.get(ref); + if (!entry) throw new Error(`missing pack release ${ref}`); + for (const dependency of entry.pack.requiresPacks) { + if (!packs.has(dependency)) throw new Error(`${ref} references missing pack ${dependency}`); + visit(dependency, [...chain, ref]); + } + state.set(ref, 'done'); + }; + visit(pack.id); + let criteria = 0; + for (const check of pack.checks) { + const scenarioPath = contained(root, join(root, check.source), `${pack.id}.${check.id}.source`); + const scenario = compileScenarioDefinition(readDefinitionJson(scenarioPath, 'scenario'), { + source: relative(root, scenarioPath).replaceAll('\\', '/'), + }); + const feature = scenario.features.find(candidate => candidate.id === check.feature); + if (!feature) throw new Error(`${pack.id}.${check.id} references missing feature ${check.feature}`); + criteria += feature.criteria.length; + } + return { id: pack.id, path: absolute, + checkGroups: pack.checks.length, criteria, requiresPacks: pack.requiresPacks }; +} + +export function validateRecipeFile(path: string, options: Partial = {}): { + plan: CompiledRecipePlan; + release: RecipeRelease; +} { + const { trackRoot } = options; + if (trackRoot === undefined) throw new Error('recipe validation requires trackRoot'); + const absolute = contained(join(trackRoot, 'composition'), path, 'recipe path'); + const plan = compileRecipeFile(absolute, { trackRoot }); + const release = buildRecipeRelease(absolute, { trackRoot }); + return { plan, release }; +} + +export function showRecipeFile(path: string, options: RecipeOptions): SelectedRecipeRelease & { + builderTask: ReturnType & { note: string }; +} { + const compiled = validateRecipeFile(path, options); + const selected = selectRecipeRelease(compiled.release, options); + const builderTask = composeSelectedRecipeTask(compiled.plan, selected.selection); + return { + ...selected, + builderTask: { + ...builderTask, + note: 'Pack selection defines the requested task; a check-only filter narrows measurement inside it.', + }, + }; +} + +const same = (left: unknown, right: unknown): boolean => + canonicalDefinitionJson(left) === canonicalDefinitionJson(right); + +function meaningView(release: RecipeRelease) { + return { + track: release.track, + task: release.task, + checks: release.checkCatalog.map(({ stableKey, packId, checkGroupId, role, source, + featureId, criterionId, description }) => ({ stableKey, packId, checkGroupId, role, + source, featureId, criterionId, description })), + }; +} + +function scoringView(release: RecipeRelease) { + return { scoring: release.scoring, + checks: release.checkCatalog.map(({ stableKey, points }) => ({ stableKey, points })) }; +} + +function metadataView(release: RecipeRelease) { + return { id: release.id, title: release.title, + sequence: release.sequence, sourceManifestSha256: release.sourceManifestSha256 }; +} + +function matchingCalibrations(trackRoot: string, release: RecipeRelease): Array<{ + path: string; + value: CalibrationValue; +}> { + const directory = join(trackRoot, 'composition', 'calibrations'); + if (!existsSync(directory)) return []; + return readdirSync(directory).filter(name => name.endsWith('.json')).sort() + .map(name => ({ path: join(directory, name), + value: readDefinitionJson(join(directory, name), 'calibration') })) + .filter(({ value }) => value.recipe?.id === release.id + && value.recipe?.contentSha256 === release.contentSha256); +} + +export function diffRecipeFiles(fromPath: string, toPath: string, options: Partial = {}) { + const { trackRoot } = options; + if (trackRoot === undefined) throw new Error('recipe diff requires trackRoot'); + const from = validateRecipeFile(fromPath, { trackRoot }).release; + const to = validateRecipeFile(toPath, { trackRoot }).release; + const categories = { + meaning: !same(meaningView(from), meaningView(to)), + scoring: !same(scoringView(from), scoringView(to)), + fixtures: !same(from.components.fixture, to.components.fixture), + execution: from.executionSha256 !== to.executionSha256, + metadata: !same(metadataView(from), metadataView(to)), + }; + const recipeBindingChanged = from.id !== to.id + || from.meaningSha256 !== to.meaningSha256 || from.executionSha256 !== to.executionSha256 + || from.contentSha256 !== to.contentSha256; + const calibrations = matchingCalibrations(trackRoot, from).map(({ path, value }) => { + const invalidated = []; + if (recipeBindingChanged) invalidated.push('recipe binding'); + if (categories.fixtures) invalidated.push('fixture binding'); + if (categories.scoring) invalidated.push('zero-point control policy'); + if (categories.meaning || categories.scoring || categories.execution || categories.fixtures) { + invalidated.push('reference repetitions', 'mutation repetitions'); + } + if (categories.meaning || categories.scoring || categories.fixtures) invalidated.push('null repetitions'); + if (recipeBindingChanged) invalidated.push('selection binding'); + return { id: value.id, + path: relative(trackRoot, path).replaceAll('\\', '/'), invalidated: [...new Set(invalidated)] }; + }); + const fragmentDiff = (kind: RecipeTaskKind) => { + const before = new Map(from.task[kind].map(fragment => [fragment.id, fragment])); + const after = new Map(to.task[kind].map(fragment => [fragment.id, fragment])); + return { + added: [...after.keys()].filter(key => !before.has(key)).sort(), + removed: [...before.keys()].filter(key => !after.has(key)).sort(), + changed: [...after.keys()].filter(key => before.has(key) + && !same(before.get(key), after.get(key))).sort(), + }; + }; + return { + from: { id: from.id, meaningSha256: from.meaningSha256, + executionSha256: from.executionSha256, contentSha256: from.contentSha256 }, + to: { id: to.id, meaningSha256: to.meaningSha256, + executionSha256: to.executionSha256, contentSha256: to.contentSha256 }, + categories, + taskFragments: { + requirements: fragmentDiff('requirements'), + contracts: fragmentDiff('contracts'), + composedTaskChanged: from.task.composedSha256 !== to.task.composedSha256, + }, + calibrations, + }; +} + +type CliSubject = 'pack' | 'recipe'; +type CliCommand = 'validate' | 'show' | 'diff'; + +interface ParsedArgs extends RecipeSelectionOptions { + json: boolean; + positional: string[]; + packIds: string[]; + checkKeys: string[]; + track?: string; + trackRoot?: string; +} + +interface CliArgs extends ParsedArgs { + subject: CliSubject; + command: CliCommand; + paths: string[]; + trackRoot: string; +} + +function parse(argv: string[]): CliArgs { + const { positionals, values } = parseArgs({ args: argv.slice(2), allowPositionals: true, + options: { track: { type: 'string' }, 'track-root': { type: 'string' }, + pack: { type: 'string', multiple: true }, check: { type: 'string', multiple: true }, + json: { type: 'boolean' } } }); + const args: ParsedArgs = { json: values.json ?? false, positional: positionals, + packIds: (values.pack ?? []).flatMap(value => value.split(',').filter(Boolean)), + checkKeys: (values.check ?? []).flatMap(value => value.split(',').filter(Boolean)), + track: values.track, + trackRoot: values['track-root'] === undefined ? undefined : resolve(values['track-root']) }; + const [subject, command, ...paths] = args.positional; + if (subject !== 'pack' && subject !== 'recipe') { + throw new Error('usage: npm run pack -- validate --track | npm run recipe -- validate|show --track | npm run recipe -- diff --track '); + } + if (command !== 'validate' && command !== 'show' && command !== 'diff') { + throw new Error('usage: npm run pack -- validate --track | npm run recipe -- validate|show --track | npm run recipe -- diff --track '); + } + if (subject === 'pack' && command !== 'validate') throw new Error(`pack ${command} is not supported`); + if ((command === 'diff' ? paths.length !== 2 : paths.length !== 1)) throw new Error(`${subject} ${command} received the wrong number of paths`); + if (!args.trackRoot && !args.track) throw new Error('--track or --track-root is required'); + if ((args.packIds.length || args.checkKeys.length) && !(subject === 'recipe' && command === 'show')) { + throw new Error('--pack and --check are allowed only with recipe show'); + } + const trackRoot = args.trackRoot ?? join(TRACKS_DIR, args.track ?? ''); + return { ...args, subject, command, paths, trackRoot }; +} + +function main() { + const args = parse(process.argv); + const firstPath = args.paths[0]; + if (firstPath === undefined) throw new Error('command requires a source path'); + let result: object; + if (args.subject === 'pack') result = validatePackFile(firstPath, args); + else if (args.command === 'diff') { + const secondPath = args.paths[1]; + if (secondPath === undefined) throw new Error('recipe diff requires two source paths'); + result = diffRecipeFiles(firstPath, secondPath, args); + } else if (args.command === 'show') result = showRecipeFile(firstPath, args); + else { + const compiled = validateRecipeFile(firstPath, args); + result = { + id: compiled.release.id, + packs: compiled.release.components.packs.length, checks: compiled.release.checkCatalog.length, + points: compiled.release.checkCatalog.reduce((total, check) => total + check.points, 0), + meaningSha256: compiled.release.meaningSha256, + executionSha256: compiled.release.executionSha256, + contentSha256: compiled.release.contentSha256, + }; + } + if (args.json || args.command === 'show' || args.command === 'diff') console.log(JSON.stringify(result, null, 2)); + else if ('id' in result) { + console.log(`${String(result.id)}: valid`); + } else throw new Error('validation result has no release identity'); +} + +if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) { + try { main(); } + catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 2; + } +} diff --git a/tools/stack-bench/commands/container-smoke.ts b/tools/stack-bench/commands/container-smoke.ts new file mode 100644 index 00000000000..1c7c9702e23 --- /dev/null +++ b/tools/stack-bench/commands/container-smoke.ts @@ -0,0 +1,206 @@ +#!/usr/bin/env node +// Use only ephemeral resources owned by this smoke run. + +import { spawn, execFileSync } from 'node:child_process'; +import type { ChildProcess } from 'node:child_process'; +import { cpSync, existsSync, mkdirSync, mkdtempSync, rmSync } from 'node:fs'; +import { createServer } from 'node:net'; +import type { AddressInfo } from 'node:net'; +import { basename, join, resolve } from 'node:path'; +import { tmpdir } from 'node:os'; +import { setTimeout as delay } from 'node:timers/promises'; + +import { killTree, pidsOnPort, processIdentity } from '../src/runtime/platform.js'; +import { buildContainerName } from '../container/reconcile-build-container.js'; +import { createBackendLease, readBackendLease, writeBackendLease } from '../src/runtime/backend-lease.js'; +import { fetchStatus } from '../src/runtime/readiness.js'; +import { DEFAULT_BUILD_IMAGE } from '../src/composition/product-config.js'; +import { containerReachableSpacetimeUri } from '../src/runtime/spacetime-target.js'; +import { CODING_CONTAINER_APP_ROOT, CODING_CONTAINER_SPACETIME_CLI, + codingContainerAgentExecOptions } from '../src/runtime/coding-container-policy.js'; +import { ARTIFACT_FILE } from '../src/evidence/artifacts.js'; + +import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js'; +const REPO = resolve(ROOT, '..', '..'); +const IMAGE = process.env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE; +const CLI = process.env.SPACETIME_BIN ?? join(REPO, 'target', 'release', + process.platform === 'win32' ? 'spacetimedb-cli.exe' : 'spacetimedb-cli'); +const RUN_BUILD = compiledEntrypoint('container', 'run-build.js'); +const FIXTURE = join(ROOT, 'tests', 'fixtures', 'spacetime-module'); + +interface PreparedContainerIdentity { + containerName: string; + identity: string; + networkMode: string | null; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function parsePreparedContainerIdentity(text: string): PreparedContainerIdentity { + const value: unknown = JSON.parse(text.trim().split(/\r?\n/).pop() ?? ''); + if (!isRecord(value)) throw new Error('prepared container identity is invalid'); + const record = value; + if (typeof record.containerName !== 'string' || typeof record.identity !== 'string' + || (record.networkMode !== null && typeof record.networkMode !== 'string')) { + throw new Error('prepared container identity is invalid'); + } + return { containerName: record.containerName, identity: record.identity, networkMode: record.networkMode }; +} + +async function freePort() { + const server = createServer(); + await new Promise((ok, fail) => server.listen({ port: 0, host: '127.0.0.1' }, ok).once('error', fail)); + const address = server.address(); + if (!address || typeof address === 'string') throw new Error('could not allocate a TCP port'); + const port: AddressInfo['port'] = address.port; + await new Promise(ok => server.close(ok)); + return port; +} + +async function waitFor(check: () => boolean | Promise, timeoutMs: number, description: string): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (await check()) return; + await delay(250); + } + throw new Error(`timed out waiting for ${description}`); +} + +async function main() { + if (!existsSync(CLI)) throw new Error(`local SpacetimeDB CLI is missing: ${CLI}`); + execFileSync('docker', ['image', 'inspect', IMAGE], { stdio: 'pipe' }); + + const root = mkdtempSync(join(tmpdir(), 'stack-bench-container-smoke-')); + const app = join(root, 'app'); + const dataDir = join(root, 'spacetime-data'); + const port = await freePort(); + const uri = `http://127.0.0.1:${port}`; + const module = `stackbench-container-smoke-${process.pid}`; + const containerName = buildContainerName({ runId: basename(root), resources: {} }); + const leasePath = join(root, ARTIFACT_FILE.backendLease); + let host: ChildProcess | null = null; + let dev: ChildProcess | null = null; + let output = ''; + + try { + mkdirSync(app, { recursive: true }); + host = spawn(CLI, ['start', '--listen-addr', `127.0.0.1:${port}`, '--data-dir', dataDir], + { stdio: 'ignore', windowsHide: true }); + await waitFor(async () => { + const status = await fetchStatus(`${uri}/v1/ping`, { timeoutMs: 5000 }); + return status !== null && status >= 200 && status < 300; + }, 120_000, `dedicated SpacetimeDB host on :${port}`); + + const lease = createBackendLease({ runId: basename(root), backend: 'spacetime', + track: 'container-smoke', runIndex: 0, serverUri: uri, module, dataDir }); + lease.state = 'active'; + lease.resources.launchedProcess = host.pid ? processIdentity(host.pid) : null; + lease.resources.listenerProcesses = pidsOnPort(port).map(pid => processIdentity(pid)) + .filter((identity): identity is NonNullable => identity !== null); + writeBackendLease(leasePath, lease); + + const prepared = execFileSync(process.execPath, + [RUN_BUILD, '--app', app, '--backend', 'spacetime', '--image', IMAGE, '--prepare-only'], + { encoding: 'utf8', stdio: 'pipe', maxBuffer: 16 * 1024 * 1024, + env: { ...process.env, STACK_BENCH_LEASE: leasePath, + STACK_BENCH_LEASE_TOKEN: lease.ownershipToken, STACK_BENCH_STDB_URI: uri } }); + const identity = parsePreparedContainerIdentity(prepared); + if (identity.containerName !== containerName) { + throw new Error(`prepared unexpected container ${identity.containerName}`); + } + const leasedContainer = readBackendLease(leasePath, + { token: lease.ownershipToken, backend: 'spacetime', active: true }).resources.buildContainer; + if (!leasedContainer || identity.identity.split(' ')[0] !== leasedContainer.id) { + throw new Error('prepared container identity was not recorded in the backend lease'); + } + if (!leasedContainer.image || !/^sha256:[0-9a-f]{64}$/.test(leasedContainer.image)) { + throw new Error(`prepared container did not record an immutable image id: ${leasedContainer.image}`); + } + + cpSync(FIXTURE, join(app, 'spacetimedb'), { recursive: true }); + const agentExec = ['exec', ...codingContainerAgentExecOptions()]; + const browserDom = execFileSync('docker', [...agentExec, containerName, 'chromium', + '--headless', '--no-sandbox', '--disable-dev-shm-usage', '--dump-dom', + 'data:text/html,'], + { encoding: 'utf8', stdio: 'pipe', timeout: 30_000 }); + if (!browserDom.includes('42')) throw new Error('agent browser did not execute JavaScript'); + const cliAccess = execFileSync('docker', [...agentExec, containerName, 'sh', '-c', + `stat -c '%a %U %G' ${CODING_CONTAINER_SPACETIME_CLI}; test -x ${CODING_CONTAINER_SPACETIME_CLI}`], + { encoding: 'utf8', stdio: 'pipe' }); + if (!/^755 root root/m.test(cliAccess)) throw new Error(`unexpected CLI access: ${cliAccess.trim()}`); + execFileSync('docker', [...agentExec, containerName, 'sh', '-c', + `umask 000; cd ${CODING_CONTAINER_APP_ROOT}/spacetimedb && npm install --no-audit --no-fund`], + { stdio: 'pipe' }); + + const startedDev = spawn('docker', [...agentExec, '-i', containerName, 'sh', '-c', + `umask 000; cd ${CODING_CONTAINER_APP_ROOT}/spacetimedb && ${CODING_CONTAINER_SPACETIME_CLI} dev ${module} ` + + `--project-path ${CODING_CONTAINER_APP_ROOT}/spacetimedb --module-path . ` + + '--server-only --skip-generate ' + + `-s ${containerReachableSpacetimeUri({ resources: { serverUri: uri, + buildContainer: lease.resources.buildContainer } }, identity.networkMode)} -y`], + { stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true }); + dev = startedDev; + const collect = (chunk: Buffer): void => { output = (output + chunk.toString()).slice(-128 * 1024); }; + startedDev.stdout?.on('data', collect); + startedDev.stderr?.on('data', collect); + + await waitFor(() => { + if (/Published successfully!/.test(output)) return true; + if (startedDev.exitCode !== null) throw new Error(`spacetime dev exited ${startedDev.exitCode}:\n${output}`); + return false; + }, 240_000, 'containerized module publish'); + + const sql = execFileSync(CLI, ['sql', module, 'SELECT * FROM smoke_item', '-s', uri], + { encoding: 'utf8', stdio: 'pipe' }); + if (!/\bid\s*\|\s*value\b/.test(sql)) throw new Error(`SQL verification failed:\n${sql}`); + if (startedDev.exitCode !== null) throw new Error('spacetime dev did not remain alive as a watcher'); + + // Publishing and log streaming must retain one authenticated identity. A + // prior dev bug published with a token stored only in a Config clone, then + // directly logged in again for logs and received an authorization error. + await delay(2_000); + const logStreamingAuthorized = !/Log streaming error:.*not authorized/s.test(output); + console.log(JSON.stringify({ ok: true, image: IMAGE, container: identity.identity, + host: { uri, listenerPids: pidsOnPort(port) }, published: true, sqlVerified: true, + watcherAlive: true, leasedContainer: true, immutableImagePinned: true, + logStreamingAuthorized }, null, 2)); + if (!logStreamingAuthorized) { + throw new Error('`spacetime dev` published successfully but its log stream was not authorized'); + } + // The grader resets by republishing the same named database from this exact + // leased container. Prove that `-y` retained a reusable local identity, + // rather than merely proving that the first anonymous-looking publish ran. + execFileSync('docker', [...agentExec, containerName, 'sh', '-c', + 'for process in /proc/[0-9]*; do ' + + 'test "$(readlink "$process/exe" 2>/dev/null)" = /deps/.spacetimedb-cli ' + + '&& kill -TERM "${process##*/}" || true; done'], { stdio: 'pipe' }); + await waitFor(() => startedDev.exitCode !== null, 15_000, 'spacetime dev to stop before reset publish'); + const targetUri = containerReachableSpacetimeUri({ resources: { serverUri: uri, + buildContainer: lease.resources.buildContainer } }, identity.networkMode); + execFileSync('docker', [...agentExec, containerName, 'sh', '-c', + `umask 000; cd ${CODING_CONTAINER_APP_ROOT}/spacetimedb && ${CODING_CONTAINER_SPACETIME_CLI} publish ${module} ` + + `--module-path . -s ${targetUri} --delete-data -y`], + { stdio: 'pipe', timeout: 240_000 }); + const afterReset = execFileSync(CLI, ['sql', module, 'SELECT * FROM smoke_item', '-s', uri], + { encoding: 'utf8', stdio: 'pipe' }); + if (!/\bid\s*\|\s*value\b/.test(afterReset)) { + throw new Error(`SQL verification after reset publish failed:\n${afterReset}`); + } + console.log(JSON.stringify({ resetRepublished: true, resetSqlVerified: true })); + } finally { + if (dev && dev.exitCode === null) dev.kill('SIGTERM'); + try { execFileSync('docker', ['rm', '-f', containerName], { stdio: 'ignore' }); } catch { /* absent */ } + // The port was proven unused before this script started the host. Kill only + // listeners on that exact ephemeral port, then the wrapper if it remains. + for (const pid of pidsOnPort(port)) killTree(pid); + if (host && host.exitCode === null) killTree(host.pid); + rmSync(root, { recursive: true, force: true }); + } +} + +main().catch(error => { + console.error(error.stack ?? error.message); + process.exitCode = 1; +}); diff --git a/tools/stack-bench/commands/definition-snapshots.ts b/tools/stack-bench/commands/definition-snapshots.ts new file mode 100644 index 00000000000..9f227e19d30 --- /dev/null +++ b/tools/stack-bench/commands/definition-snapshots.ts @@ -0,0 +1,80 @@ +import { mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; +import { canonicalDefinitionJson, compileTrackPlan } from '../src/composition/definition-plan.js'; +import { listTracks } from '../src/composition/tracks.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; + +const SNAPSHOT_DIR = join(STACK_BENCH_ROOT, 'tests', 'snapshots', 'definitions'); +const ALL_ACTIONS = join(STACK_BENCH_ROOT, 'tests', 'fixtures', 'definitions', 'all-actions.json'); + +function atomicWrite(path: string, contents: string): void { + mkdirSync(dirname(path), { recursive: true }); + const temporary = `${path}.tmp-${process.pid}`; + writeFileSync(temporary, contents); + renameSync(temporary, path); +} + +interface DefinitionSnapshot { + name: string; + value: unknown; +} + +export function currentDefinitionSnapshots(): DefinitionSnapshot[] { + const entries: DefinitionSnapshot[] = listTracks({ includeInternal: true }).map(name => ({ + name: `${name}.snapshot.json`, + value: compileTrackPlan(name), + })); + entries.push({ + name: 'all-actions.snapshot.json', + value: compileScenarioDefinition(JSON.parse(readFileSync(ALL_ACTIONS, 'utf8')), { + source: ALL_ACTIONS, + }), + }); + return entries.sort((a, b) => a.name.localeCompare(b.name)); +} + +export interface DefinitionSnapshotResult { + checked: number; + changed: string[]; +} + +export function checkDefinitionSnapshots( + { update = false }: { update?: boolean } = {}, +): DefinitionSnapshotResult { + const entries = currentDefinitionSnapshots(); + const changed: string[] = []; + for (const entry of entries) { + const path = join(SNAPSHOT_DIR, entry.name); + const actual = canonicalDefinitionJson(entry.value); + let expected: string | null = null; + try { + expected = readFileSync(path, 'utf8').replaceAll('\r\n', '\n'); + } catch (error: unknown) { + if (!(error instanceof Error && 'code' in error && error.code === 'ENOENT')) throw error; + } + if (expected === actual) continue; + changed.push(entry.name); + if (update) atomicWrite(path, actual); + } + if (changed.length > 0 && !update) { + throw new Error( + `definition snapshot drift: ${changed.join(', ')}; inspect the semantic change, then run npm run check:definition-snapshots -- --update`, + ); + } + return { checked: entries.length, changed }; +} + +function main(): void { + const args = new Set(process.argv.slice(2)); + for (const arg of args) { + if (arg !== '--update') throw new Error(`unknown argument ${arg}`); + } + const result = checkDefinitionSnapshots({ update: args.has('--update') }); + console.log(`${result.checked} definition snapshots checked${ + result.changed.length > 0 ? `; ${result.changed.length} updated` : '; no drift'}`); +} + +if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) main(); diff --git a/tools/stack-bench/commands/fault-injection.ts b/tools/stack-bench/commands/fault-injection.ts new file mode 100644 index 00000000000..b8f056f3bcf --- /dev/null +++ b/tools/stack-bench/commands/fault-injection.ts @@ -0,0 +1,267 @@ +#!/usr/bin/env node +// Fault injection may remove only resources owned by its lease. + +import assert from 'node:assert/strict'; +import { execFileSync, spawn } from 'node:child_process'; +import type { ChildProcess } from 'node:child_process'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { createServer } from 'node:http'; +import type { Server } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { join, resolve } from 'node:path'; +import { tmpdir } from 'node:os'; +import { setTimeout as delay } from 'node:timers/promises'; + +import { createBackendLease, writeBackendLease } from '../src/runtime/backend-lease.js'; +import { killTree, pidsOnPort } from '../src/runtime/platform.js'; +import { recoverBackendLease } from '../src/runtime/recovery.js'; +import { buildContainerName } from '../container/reconcile-build-container.js'; +import { ARTIFACT_FILE, readArtifact, readArtifactPayload } from '../src/evidence/artifacts.js'; +import { DEFAULT_BUILD_IMAGE } from '../src/composition/product-config.js'; + +import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js'; +const REPO = resolve(ROOT, '..', '..'); +const IMAGE = process.env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE; +const CLI = process.env.SPACETIME_BIN ?? join(REPO, 'target', 'release', + process.platform === 'win32' ? 'spacetimedb-cli.exe' : 'spacetimedb-cli'); +const RUN_BUILD = compiledEntrypoint('container', 'run-build.js'); +interface ContainerIdentity { id: string; running: boolean; } +interface ExitResult { code: number | null; signal: NodeJS.Signals | null; } +interface FaultLeaseResources { + listenerProcesses: Array<{ pid: number; startMarker: string }>; + buildContainer: { id: string; image: string; running: boolean; removedAt?: string }; + locks: { releasedAt?: string }[]; +} +interface FaultLeaseEvidence { + runId: string; + state: string; + stoppedAt?: string; + releasedAt?: string; + resources: FaultLeaseResources; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +async function freePort() { + const server = createServer((_request, response) => response.end('foreign')); + await new Promise((ok, fail) => server.listen({ port: 0, host: '127.0.0.1' }, ok).once('error', fail)); + const address = server.address(); + if (!address || typeof address === 'string') throw new Error('could not allocate a TCP port'); + const port: AddressInfo['port'] = address.port; + await new Promise(ok => server.close(ok)); + return port; +} + +function inspectContainer(target: string): ContainerIdentity | null { + try { + const output = execFileSync('docker', ['inspect', '--format', + '{{.Id}} {{.State.Running}}', target], { encoding: 'utf8', stdio: 'pipe' }).trim(); + const [id, running] = output.split(/\s+/, 2); + if (!id) return null; + return { id, running: running === 'true' }; + } catch { return null; } +} + +function startContainer(name: string): ContainerIdentity { + const id = execFileSync('docker', ['run', '-d', '--init', '--name', name, + IMAGE, 'sleep', 'infinity'], { encoding: 'utf8', stdio: 'pipe' }).trim(); + assert.ok(id, `Docker did not return an id for ${name}`); + const container = inspectContainer(id); + if (!container) throw new Error(`Docker did not return a running container for ${name}`); + return container; +} + +function removeExactContainer(identity: ContainerIdentity | { id: string } | null): void { + if (!identity) return; + const current = inspectContainer(identity.id); + if (!current || current.id !== identity.id) return; + execFileSync('docker', ['rm', '-f', identity.id], { stdio: 'ignore' }); +} + +async function waitForExit(child: ChildProcess, timeoutMs: number): Promise { + return new Promise((resolveExit, reject) => { + const timeout = setTimeout(() => reject(new Error( + `benchmark runner did not exit after injected failure within ${timeoutMs}ms`)), timeoutMs); + child.once('exit', (code: number | null, signal: NodeJS.Signals | null) => { + clearTimeout(timeout); + resolveExit({ code, signal }); + }); + }); +} + +async function assertRefusesUnleasedCollision() { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-container-collision-')); + const app = join(root, 'app'); + const runId = `collision-${process.pid}`; + // Plant the foreign container under the exact name the launcher will claim. + const name = buildContainerName({ runId, resources: {} }); + const leasePath = join(root, ARTIFACT_FILE.backendLease); + let foreign = null; + try { + mkdirSync(app, { recursive: true }); + foreign = startContainer(name); + const lease = createBackendLease({ runId, backend: 'spacetime', + track: 'fault-injection', runIndex: 0, serverUri: 'http://127.0.0.1:1', + module: `collision-${process.pid}`, dataDir: join(root, 'data') }); + lease.state = 'active'; + writeBackendLease(leasePath, lease); + + let refused = false; + try { + execFileSync(process.execPath, + [RUN_BUILD, '--app', app, '--backend', 'spacetime', '--prepare-only'], + { stdio: 'pipe', env: { ...process.env, STACK_BENCH_LEASE: leasePath, + STACK_BENCH_LEASE_TOKEN: lease.ownershipToken } }); + } catch (error: unknown) { + const childError = error instanceof Error && isRecord(error) ? error : null; + refused = childError?.status === 3 + && /refusing to adopt existing unleased container/.test(String(childError?.stderr)); + } + assert.equal(refused, true, 'launcher did not explicitly refuse an unleased same-name container'); + assert.deepEqual(inspectContainer(foreign.id), foreign, + 'collision refusal changed or stopped the foreign container'); + } finally { + removeExactContainer(foreign); + rmSync(root, { recursive: true, force: true }); + } +} + +async function main() { + assert.ok(existsSync(CLI), `local SpacetimeDB CLI is missing: ${CLI}`); + execFileSync('docker', ['image', 'inspect', IMAGE], { stdio: 'pipe' }); + await assertRefusesUnleasedCollision(); + + const root = mkdtempSync(join(tmpdir(), 'stack-bench-fault-')); + const app = join(root, 'app'); + const out = join(root, 'out'); + const markerPath = join(app, '.fault-ready.json'); + const port = await freePort(); + const uri = `http://127.0.0.1:${port}`; + const foreignName = `stack-bench-foreign-${process.pid}-${Date.now()}`; + let foreignContainer: ContainerIdentity | null = null; + let foreignServer: Server | null = null; + let bench: ChildProcess | null = null; + let marker: { lease: { runId: string; state: string; resources: FaultLeaseResources }; leasePath: string; phase: string } | null = null; + let output = ''; + + try { + mkdirSync(app, { recursive: true }); + mkdirSync(out, { recursive: true }); + foreignContainer = startContainer(foreignName); + const startedForeignServer = createServer((_request, response) => response.end('foreign')); + foreignServer = startedForeignServer; + await new Promise((ok, fail) => startedForeignServer.listen({ port: 0, host: '127.0.0.1' }, ok).once('error', fail)); + const foreignAddress = startedForeignServer.address(); + if (!foreignAddress || typeof foreignAddress === 'string') throw new Error('could not allocate foreign TCP port'); + const foreignUri = `http://127.0.0.1:${foreignAddress.port}`; + + bench = spawn(process.execPath, + [compiledEntrypoint('commands', 'bench.js'), '--backend', 'spacetime', '--track', 'loop', + '--levels', '1', '--agent-adapter', 'fault-injection', '--app', app, '--out', out, + '--url', `file:///${app.replace(/\\/g, '/')}/index.html`], + { env: { ...process.env, STACK_BENCH_STDB_URI: uri, STACK_BENCH_IMAGE: IMAGE, + SPACETIME_BIN: CLI }, + stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true }); + const collect = (chunk: Buffer): void => { output = (output + chunk.toString()).slice(-256 * 1024); }; + bench.stdout?.on('data', collect); + bench.stderr?.on('data', collect); + const exited = await waitForExit(bench, 300_000); + assert.notEqual(exited.code, 0, 'injected coding-agent failure unexpectedly exited zero'); + assert.ok(existsSync(markerPath), `fault marker was not written before failure:\n${output}`); + const markerValue: unknown = JSON.parse(readFileSync(markerPath, 'utf8')); + if (!isRecord(markerValue) || !isRecord(markerValue.lease) || !isRecord(markerValue.lease.resources) + || typeof markerValue.leasePath !== 'string' || typeof markerValue.phase !== 'string' + || typeof markerValue.lease.runId !== 'string' || typeof markerValue.lease.state !== 'string') { + throw new Error('fault marker is invalid'); + } + const markerResources = markerValue.lease.resources; + if (!Array.isArray(markerResources.listenerProcesses) || !isRecord(markerResources.buildContainer) + || typeof markerResources.buildContainer.id !== 'string') throw new Error('fault marker resources are invalid'); + marker = { phase: markerValue.phase, leasePath: markerValue.leasePath, + lease: { runId: markerValue.lease.runId, state: markerValue.lease.state, + resources: { listenerProcesses: markerResources.listenerProcesses.filter((item): item is { + pid: number; startMarker: string } => isRecord(item) && typeof item.pid === 'number' + && typeof item.startMarker === 'string'), buildContainer: { + id: markerResources.buildContainer.id, image: String(markerResources.buildContainer.image ?? ''), + running: markerResources.buildContainer.running === true }, locks: [] } } }; + assert.equal(marker.phase, 'restart-stopped', + 'fault was not injected inside the backend restart window'); + assert.equal(marker.lease.state, 'restarting'); + assert.match(marker.lease.resources.buildContainer.image, /^sha256:[0-9a-f]{64}$/, + 'build container lease did not record an immutable image id'); + + const evidencePath = join(out, ARTIFACT_FILE.backendLease); + assert.ok(existsSync(evidencePath), `teardown did not preserve lease evidence:\n${output}`); + const evidence = readArtifactPayload(evidencePath, { expectedKind: 'backend_lease_evidence' }); + const preflight = readArtifact(join(out, ARTIFACT_FILE.preflight), + { expectedKind: 'preflight' }); + assert.equal(preflight.payload.ok, true, 'paid-run preflight did not pass'); + assert.equal(preflight.attempt.parentId, marker.lease.runId, + 'preflight evidence is not attached to the run it admitted'); + assert.equal(evidence.runId, marker.lease.runId); + assert.equal(evidence.state, 'released', 'benchmark lease did not reach its terminal state'); + assert.ok(evidence.stoppedAt, 'benchmark-owned SpacetimeDB host has no stop evidence'); + assert.ok(evidence.releasedAt, 'benchmark lease has no release evidence'); + assert.deepEqual(evidence.resources.listenerProcesses, []); + assert.equal(evidence.resources.buildContainer.running, false, + 'benchmark-owned build container was not marked removed'); + assert.ok(evidence.resources.buildContainer.removedAt); + assert.ok(evidence.resources.locks.every(lock => lock.releasedAt), + 'one or more resource locks were not released'); + assert.equal(inspectContainer(marker.lease.resources.buildContainer.id), null, + 'benchmark-owned build container survived fatal cleanup'); + assert.equal(pidsOnPort(port).length, 0, 'benchmark-owned listener survived fatal cleanup'); + assert.equal(existsSync(marker.leasePath), false, 'private runtime lease was not removed'); + + assert.equal((await fetch(foreignUri)).status, 200, + 'foreign listener was disturbed by benchmark cleanup'); + assert.deepEqual(inspectContainer(foreignContainer.id), foreignContainer, + 'foreign container was changed or removed by benchmark cleanup'); + + console.log(JSON.stringify({ ok: true, injectedAt: 'restart-stopped-before-replacement', + benchmarkHostStopped: true, benchmarkContainerRemoved: true, locksReleased: true, + privateLeaseRemoved: true, foreignListenerSurvived: true, + foreignContainerSurvived: true, unleasedCollisionRefused: true, + immutableImagePinned: true }, null, 2)); + } finally { + if (bench?.exitCode === null) { + killTree(bench.pid); + await delay(500); + } + if (!marker && existsSync(markerPath)) { + // The bench died before the marker was checked. Release its exact lease, + // including the build container, through authenticated recovery. + try { + const markerValue: unknown = JSON.parse(readFileSync(markerPath, 'utf8')); + if (isRecord(markerValue) && typeof markerValue.leasePath === 'string' + && existsSync(markerValue.leasePath)) { + recoverBackendLease(markerValue.leasePath, join(root, 'recovery')); + } + } catch (error) { + console.error(`fault lease recovery failed: ${error instanceof Error ? error.message : String(error)}`); + } + } + if (marker?.lease?.resources?.buildContainer) { + removeExactContainer(marker.lease.resources.buildContainer); + } + for (const identity of marker?.lease?.resources?.listenerProcesses ?? []) { + if (pidsOnPort(port).includes(String(identity.pid))) killTree(identity.pid); + } + if (foreignServer) { + const server = foreignServer; + // The verification fetch uses a keep-alive connection. Waiting on + // close() alone can hold CI open until Undici retires that socket. + server.closeAllConnections(); + await new Promise((ok, fail) => server.close(error => error ? fail(error) : ok())); + } + removeExactContainer(foreignContainer); + rmSync(root, { recursive: true, force: true }); + } +} + +main().catch(error => { + console.error(error.stack ?? error.message); + process.exitCode = 1; +}); diff --git a/tools/stack-bench/commands/job-cli.ts b/tools/stack-bench/commands/job-cli.ts new file mode 100644 index 00000000000..41f2caf9ed6 --- /dev/null +++ b/tools/stack-bench/commands/job-cli.ts @@ -0,0 +1,119 @@ +#!/usr/bin/env node +import { readFileSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parseArgs } from 'node:util'; +import { cancelExecutionJob, listExecutionJobs, readExecutionJob, + resumeOwnedExecutionJob, submitExecutionJob, workExecutionJob } from '../src/campaigns/execution-jobs.js'; +import { runExecutionWorker } from '../src/campaigns/execution-worker.js'; +import { prepareRun, runSetupCatalog, submitPreparedRun } from '../src/campaigns/run-setup.js'; +import { compileCampaignFile } from '../src/campaigns/campaign-compiler.js'; +import { executeCampaign, inspectCampaign } from '../src/campaigns/campaign-runner.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { stackBenchResultsRoot } from '../src/runtime/operational-paths.js'; +import { validateResumeCampaignState } from './campaign-cli.js'; + +// Every command also takes --results. +const COMMAND_OPTIONS: Record = { + options: [], prepare: [], start: ['host'], submit: [], list: ['after', 'limit'], + status: [], cancel: [], resume: [], work: ['host'], worker: ['host', 'concurrency'], +}; + +const EXECUTING = new Set(['start', 'work', 'resume']); + +class JobUsageError extends Error {} +const usage = (message: string) => new JobUsageError(message); + +/** Continue a job's interrupted campaign with the accounts and capacity policy the job saved. */ +export async function resumeExecutionJob(results: string, id: string, + { env = process.env, signal, execute = executeCampaign, inspect = inspectCampaign }: + { env?: NodeJS.ProcessEnv; signal?: AbortSignal; execute?: typeof executeCampaign; + inspect?: typeof inspectCampaign } = {}) { + const current = readExecutionJob(results, id); + if (current.status !== 'failed' && current.status !== 'running') { + throw new Error('resume requires a failed or reconciled interrupted job'); + } + const { job, campaignDirectory } = current; + const planFile = join(resolve(results), 'jobs', job.id, 'plan.json'); + const plan = compileCampaignFile(planFile); + validateResumeCampaignState(plan, inspect(campaignDirectory)); + return resumeOwnedExecutionJob(results, id, { env, signal, execute }); +} + +export async function jobCommand(argv: string[], env: NodeJS.ProcessEnv = process.env) { + let parsed; + try { + parsed = parseArgs({ args: argv, allowPositionals: true, options: { + results: { type: 'string' }, host: { type: 'string' }, after: { type: 'string' }, + limit: { type: 'string' }, concurrency: { type: 'string' }, + } }); + } catch (error) { throw usage(error instanceof Error ? error.message : String(error)); } + const { values, positionals } = parsed; + const [command, argument] = positionals; + if (argv[0] !== command) throw usage('put the job command before its options'); + if (positionals.length > 2) throw usage('unexpected job arguments'); + if (command && Object.hasOwn(COMMAND_OPTIONS, command)) { + const extra = Object.keys(values).find(option => option !== 'results' + && !COMMAND_OPTIONS[command]!.includes(option)); + if (extra) throw usage(`job ${command} does not take --${extra}`); + } + const results = values.results ? resolve(values.results) : stackBenchResultsRoot(STACK_BENCH_ROOT, env); + if (command === 'options' && !argument) return runSetupCatalog(results, env); + if (command === 'prepare' && argument) return prepareRun(results, + JSON.parse(readFileSync(argument === '-' ? 0 : argument, 'utf8')), env); + if (command === 'start' && argument) { + const host = values.host ?? env.STACK_BENCH_HOST_ID; + if (!host) throw usage('job start requires --host or STACK_BENCH_HOST_ID'); + const job = submitPreparedRun(results, JSON.parse(readFileSync(argument === '-' ? 0 : argument, 'utf8')), env); + // stdout carries only the final job status; the ID is available while the run blocks. + console.error(`job ${job.id} submitted; running campaign job-${job.id}`); + return jobCommand(['work', job.id, '--results', results, '--host', host], env); + } + if (command === 'submit' && argument) return submitExecutionJob(results, + JSON.parse(readFileSync(argument === '-' ? 0 : argument, 'utf8'))); + if (command === 'status' && argument) return readExecutionJob(results, argument); + if (command === 'cancel' && argument) { + cancelExecutionJob(results, argument); return readExecutionJob(results, argument); + } + if (command === 'list' && !argument) return listExecutionJobs(results, + { after: values.after, limit: values.limit === undefined ? undefined : Number(values.limit) }); + if ((command === 'work' && argument) || (command === 'worker' && !argument) + || (command === 'resume' && argument)) { + const host = values.host ?? env.STACK_BENCH_HOST_ID; + if (!host && command !== 'resume') throw usage('job work/worker requires --host or STACK_BENCH_HOST_ID'); + const controller = new AbortController(); + const stop = () => controller.abort(); + process.on('SIGTERM', stop); process.on('SIGINT', stop); + try { + if (command === 'resume') return await resumeExecutionJob(results, argument!, { env, signal: controller.signal }); + if (command === 'worker') { + await runExecutionWorker(results, host!, { env, signal: controller.signal, + concurrency: Number(values.concurrency) }); + return { status: 'stopped' as const }; + } + return await workExecutionJob(results, argument!, host!, { env, signal: controller.signal }); + } + finally { process.off('SIGTERM', stop); process.off('SIGINT', stop); } + } + throw usage('use job options, prepare , start --host , submit , list, status , cancel , resume , work --host , or worker --host --concurrency '); +} + +/** + * Prints one JSON document and returns the exit code: 0 when the command did what was + * asked, 1 when it failed (including a job that `start`, `work` or `resume` ran to a + * failure), 2 for a usage error. `status` and `cancel` report a failed job with 0. + */ +export async function runJobCli(argv: string[], env: NodeJS.ProcessEnv = process.env): Promise { + try { + const result = await jobCommand(argv, env); + console.log(JSON.stringify(result, null, 2)); + return EXECUTING.has(argv[0] ?? '') && 'status' in result && result.status === 'failed' ? 1 : 0; + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + return error instanceof JobUsageError ? 2 : 1; + } +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + void runJobCli(process.argv.slice(2)).then(code => { process.exitCode = code; }); +} diff --git a/tools/stack-bench/commands/leak-audit.ts b/tools/stack-bench/commands/leak-audit.ts new file mode 100644 index 00000000000..4c4f3cc4fd1 --- /dev/null +++ b/tools/stack-bench/commands/leak-audit.ts @@ -0,0 +1,411 @@ +#!/usr/bin/env node +// Use the recorded cwd as the app boundary; transcript folder names are not authority. + +import { readdirSync, readFileSync, existsSync } from 'node:fs'; +import { join, posix, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parseArgs } from 'node:util'; +import { CODING_CONTAINER_APP_ROOT } from '../src/runtime/coding-container-policy.js'; +import { transcriptDirectories } from '../src/agents/transcript-archive.js'; +import { codexTranscriptDirectory } from '../src/agents/codex-protocol.js'; +import { GROK_AUDIT_TOOLS, grokTranscriptDirectory } from '../src/agents/grok-protocol.js'; +import { ANTIGRAVITY_AUDIT_TOOLS, antigravityTranscriptDirectory } from '../src/agents/antigravity-protocol.js'; + +const norm = (value: unknown): string => String(value ?? '') + .replace(/\\/g, '/').replace(/^["']|["']$/g, '').toLowerCase(); + +// Ignore dependencies, build output, and this session's CLI task output. +const IGNORE = /node_modules|\.git[/\\]|package-lock\.json|\/dist\/|\.map$|[/\\]temp[/\\]claude[/\\].*[/\\]tasks[/\\]/; + +// Commands that pull file contents into context. +const READER = /(?:^|[;&|]\s*)(?:cat|head|tail|less|more|type|grep|rg|ack|find|ls\s+-\w*l|sed\s+-n|awk)\s+([^;&|]+)/g; + +// Network targets in a shell command: any URL, and a raw socket target. A +// A verified attempt namespace owns its loopback ports, including temporary +// test servers. Shared namespaces require exact endpoints. Internet targets +// are recorded, not judged. +const URL_TARGET = /https?:\/\/([^\s/'"`]+)/gi; +const SOCKET_TARGET = /(?:^|[;&|]\s*)(?:nc|ncat|netcat)\s+(?:-\S+\s+)*([\w.-]+)\s+(\d{2,5})\b/g; +const LOCAL_HOST = /^(?:127\.\d+\.\d+\.\d+|localhost|0\.0\.0\.0|\[::1\]|host\.docker\.internal|10\.\d+\.\d+\.\d+|172\.(?:1[6-9]|2\d|3[01])\.\d+\.\d+|192\.168\.\d+\.\d+)$/i; +const LOOPBACK_HOST = /^(?:127\.\d+\.\d+\.\d+|localhost|0\.0\.0\.0|\[::1\])$/i; + +export interface AuditNetworkContext { + ownEndpoints?: readonly string[]; + isolatedLoopback?: boolean; +} + +export interface NetworkTarget { + host: string; + port: number | null; +} + +export function networkTargetsFromBash(command: unknown): NetworkTarget[] { + const targets: NetworkTarget[] = []; + const text = String(command ?? ''); + for (const match of text.matchAll(URL_TARGET)) { + const authority = (match[1] ?? '').replace(/^[^@]*@/, ''); + const port = authority.match(/:(\d{1,5})$/)?.[1]; + targets.push({ host: authority.replace(/:\d{1,5}$/, ''), port: port ? Number(port) : null }); + } + for (const match of text.matchAll(SOCKET_TARGET)) { + targets.push({ host: match[1] ?? '', port: Number(match[2]) }); + } + return targets; +} + +const endpointHost = (host: string): string => /^(?:127\.0\.0\.1|0\.0\.0\.0|localhost|\[::1\])$/i.test(host) + ? 'localhost' : host.toLowerCase(); +const endpointKey = (target: NetworkTarget): string => `${endpointHost(target.host)}:${target.port}`; + +const networkKind = (target: NetworkTarget, ownEndpoints: ReadonlySet, isolatedLoopback: boolean): string | null => { + if (isolatedLoopback && LOOPBACK_HOST.test(target.host)) return null; + if (ownEndpoints.has(endpointKey(target))) return null; + if (!LOCAL_HOST.test(target.host)) return 'network (internet)'; + return 'NETWORK / OTHER RUN'; +}; + +const CLASSES: Array = [ + [/[/\\]stack-bench(?:[/\\]|$)/, 'GRADER / TEST SPECS'], + [/\.claude[/\\]projects.*memory|[/\\]memory[/\\].*\.md$/, 'BENCHMARK NOTES'], + [/scenarios[/\\].*\.json|grade\.(?:js|ts)|mutation|check-scenarios/, 'GRADER / TEST SPECS'], + [/contracts[/\\].*\.json|appendix-\d+\.md|walk\.(?:js|ts)|lint\.(?:js|ts)/, 'CONTRACT / LINTER'], + [/prompts[/\\]|test-plans[/\\]|GRADING|RUBRIC/, 'PROMPTS / RUBRIC'], + [/[/\\]skills[/\\]/, 'skill docs (intended)'], + [/backends[/\\].*\.md|CLAUDE\.md|README/, 'setup docs (intended)'], +]; +const classify = (path: string): string => CLASSES.find(([pattern]) => pattern.test(path))?.[1] + ?? 'other'; + +// StackBench's own files online: anything under tools/stack-bench in clockworklabs/SpacetimeDB, +// on any branch and through any host (GitHub, or a mirror such as jsDelivr or Sourcegraph), and +// the pull request that carries it. The rest of SpacetimeDB's source is fair to read. +export const STACKBENCH_SOURCE = 'STACKBENCH SOURCE'; +const STACKBENCH_PULL_REQUEST = '5766'; +const GITHUB_HOST = /^(?:www\.)?(?:github\.com|raw\.githubusercontent\.com|api\.github\.com|codeload\.github\.com|patch-diff\.githubusercontent\.com)$/i; +const REPOSITORY = /clockworklabs\/spacetimedb(?:\.git)?(?![\w.-])/; +const HARNESS_PATH = /(?:^|[^\w-])tools\/stack-bench(?![\w-])/; +const PULL_REQUEST = new RegExp(`(?:^|/)(?:pull|pulls|issues)/${STACKBENCH_PULL_REQUEST}(?![0-9])`); +const PULL_REQUEST_MENTION = new RegExp(`(?:\\bpulls?\\b|\\bpr\\b|#)\\s*/?\\s*${STACKBENCH_PULL_REQUEST}(?![0-9])`); +const ANY_URL = /https?:\/\/[^\s'"`<>)\]]+/gi; + +const decoded = (value: string): string => { + let text = value; + for (let round = 0; round < 3; round++) { + try { const next = decodeURIComponent(text); if (next === text) break; text = next; } catch { break; } + } + return text.toLowerCase(); +}; + +// The StackBench references in a shell command, a fetched URL, or a search: GitHub URLs to the +// harness path or the pull request, `gh` and git calls naming either, and a bare repository +// path to the harness. +export function stackBenchSourceReferences(text: unknown): string[] { + const value = String(text ?? ''); + const found = new Set(); + for (const match of value.matchAll(ANY_URL)) { + let url: URL; + try { url = new URL(match[0]); } catch { continue; } + const path = decoded(`${url.pathname}${url.search}`); + if (REPOSITORY.test(path) && (HARNESS_PATH.test(path) || (GITHUB_HOST.test(url.hostname) && PULL_REQUEST.test(path)))) { + found.add(match[0]); + } + } + const plain = decoded(value.replace(ANY_URL, ' ')); + if (REPOSITORY.test(plain) && HARNESS_PATH.test(plain)) found.add('clockworklabs/SpacetimeDB tools/stack-bench'); + // A search for the pull request, such as "clockworklabs SpacetimeDB pull 5766". + if (/\bspacetimedb\b/.test(plain) && PULL_REQUEST_MENTION.test(plain)) { + found.add(`clockworklabs/SpacetimeDB#${STACKBENCH_PULL_REQUEST}`); + } + if (/\bgh\s+(?:pr\s+(?:view|diff|checkout)|api)\b/.test(plain) && REPOSITORY.test(plain) + && new RegExp(`(?:\\s|/)${STACKBENCH_PULL_REQUEST}(?![0-9])`).test(plain)) { + found.add(`clockworklabs/SpacetimeDB#${STACKBENCH_PULL_REQUEST}`); + } + if (/\bgit\b/.test(plain) && new RegExp(`\\bpull/${STACKBENCH_PULL_REQUEST}/(?:head|merge)\\b`).test(plain)) { + found.add(`pull/${STACKBENCH_PULL_REQUEST}`); + } + return [...found]; +} + +// The agents' own web tools under their recorded names, audited for what they search and fetch. +const WEB_TOOLS = new Set(['WebFetch', 'WebSearch', 'web_fetch', 'web_search']); + +// The shallowest recorded cwd is the app boundary when --app is absent. +function sessionCwd(lines: string[]): string | null { + const seen = new Set(); + for (const l of lines) { + const m = l.match(/"cwd":"((?:[^"\\]|\\.)*)"/); + if (m?.[1]) seen.add(norm(m[1].replace(/\\\\/g, '/'))); + } + if (!seen.size) return null; + return [...seen].sort((a, b) => a.split('/').length - b.split('/').length || a.length - b.length)[0] + ?? null; +} + +export function pathsFromBash(command: unknown): string[] { + const out: string[] = []; + for (const match of String(command).matchAll(READER)) { + const argumentsText = match[1]; + if (!argumentsText) continue; + for (const tokRaw of argumentsText.split(/\s+/)) { + const t = tokRaw.replace(/^["']|["']$/g, ''); + if (!t || t.startsWith('-')) continue; + if (/[*?]/.test(t) || /\//.test(t) || /\\/.test(t) || /\.\w+$/.test(t)) out.push(t); + } + } + return out; +} + +// Count file-tool reads only after their result confirms success. +// Bash reads count unless the command fails. +interface AuditHit { + path: string; + via: string; + kind: string; + unresolved?: boolean; +} + +interface PendingRead { + paths: string[]; + network: Array<{ path: string; kind: string }>; + via: string; +} + +interface TranscriptAudit { + file: string; + cwd: string | null; + fileTool: number; + bashReads: number; + hits: AuditHit[]; + refused: AuditHit[]; +} + +interface AuditResult extends TranscriptAudit { + root: string; +} + +interface TranscriptContent { + type?: string; + name?: string; + id?: string; + tool_use_id?: string; + is_error?: boolean; + input?: { file_path?: string; path?: string; pattern?: string; command?: string; [key: string]: unknown }; +} + +// Feed both CLIs through the same path, network, and refusal checks. +// Grok Build reports its own tool names; audit them as the Claude tools they match. +function grokContent(part: TranscriptContent): TranscriptContent { + const tool = part.type === 'tool_use' ? GROK_AUDIT_TOOLS[part.name ?? ''] : undefined; + if (!tool) return part; + const input = part.input as Record | undefined; + return { ...part, name: tool.name, input: tool.path ? { ...part.input, file_path: input?.[tool.path] } : part.input }; +} + +// Antigravity reports each tool as a step that starts, then finishes or fails. +function antigravityContent(step: Record): TranscriptContent[] { + const info = step.tool_info as { name?: string; parameters?: Record; output?: string } | undefined; + const tool = step.step_type === 'tool' ? ANTIGRAVITY_AUDIT_TOOLS[info?.name ?? ''] : undefined; + if (!tool) return []; + const id = `${String(step.conversation_id)}:${String(step.step_index)}`; + if (step.state === 'ACTIVE') { + const parameters = info?.parameters ?? {}; + return [{ type: 'tool_use', id, name: tool.name, input: WEB_TOOLS.has(tool.name) ? { ...parameters } + : tool.path ? { file_path: parameters[tool.path] } : { command: parameters.CommandLine } }]; + } + return [{ type: 'tool_result', tool_use_id: id, is_error: step.state !== 'DONE' && !info?.output?.trim() }]; +} + +function transcriptContent(event: Record): TranscriptContent[] { + const message = event.message as { content?: TranscriptContent[] } | undefined; + if (Array.isArray(message?.content)) return message.content.map(grokContent); + if (event.event === 'step_update' && event.step_update && typeof event.step_update === 'object') { + return antigravityContent(event.step_update as Record); + } + if (event.type !== 'item.started' && event.type !== 'item.completed') return []; + const item = event.item as { id?: string; type?: string; command?: string; + aggregated_output?: string; exit_code?: number; status?: string; + changes?: { path: string }[] } | undefined; + if (!item?.id) return []; + const content: TranscriptContent[] = []; + if (item.type === 'web_search') { + // Codex records its hosted search, and each page it opens, as one item. + const { id: _id, type: _type, ...search } = item as Record; + content.push({ type: 'tool_use', id: item.id, name: 'WebSearch', input: search }); + } else if (item.type === 'command_execution') { + // Codex records the shell launcher around the command. + const command = (item.command ?? '').replace(/^(?:\/\S+\/)?(?:bash|sh|zsh)\s+-[a-z]*c\s+(['"])([\s\S]*)\1$/, '$2'); + content.push({ type: 'tool_use', id: item.id, name: 'Bash', input: { command } }); + } else if (item.type === 'file_change') { + for (const [index, change] of (item.changes ?? []).entries()) { + content.push({ type: 'tool_use', id: `${item.id}:${index}`, name: 'Edit', + input: { file_path: change.path } }); + } + } + if (event.type === 'item.completed') { + // Output can contain a successful read before a later command fails. + const blocked = item.status === 'failed' && !item.aggregated_output?.trim(); + for (const call of [...content]) content.push({ type: 'tool_result', + tool_use_id: call.id, is_error: blocked }); + } + return content; +} + +export function auditTranscript(file: string, boundary: string | null, + { ownEndpoints = [], isolatedLoopback = false }: AuditNetworkContext = {}): TranscriptAudit { + const endpoints = new Set(ownEndpoints.map(endpoint => { + const url = new URL(`http://${endpoint}`); + return endpointKey({ host: url.hostname, port: Number(url.port || 80) }); + })); + const lines = readFileSync(file, 'utf8').split('\n').filter(Boolean); + // Container transcripts use /app; --app is its host path. + const recorded = sessionCwd(lines); + const cwd = recorded === CODING_CONTAINER_APP_ROOT ? recorded : (boundary ?? recorded); + const hits: AuditHit[] = []; + const refused: AuditHit[] = []; + const pending = new Map(); + let fileTool = 0, bashReads = 0; + + for (const line of lines) { + let event: Record; + try { event = JSON.parse(line) as typeof event; } catch { continue; } + const c = transcriptContent(event); + for (const p of c) { + if (p.type === 'tool_result' && p.tool_use_id && pending.has(p.tool_use_id)) { + const completed = pending.get(p.tool_use_id ?? ''); + if (!completed) continue; + const { paths, network, via } = completed; + pending.delete(p.tool_use_id ?? ''); + const blocked = p.is_error === true; + for (const n of paths) (blocked ? refused : hits).push({ path: n, via, kind: classify(n) }); + for (const target of network) (blocked ? refused : hits).push({ ...target, via: `${via} network attempt` }); + continue; + } + if (p.type !== 'tool_use') continue; + const cand = []; + if (/^(Read|Grep|Glob|NotebookRead|Edit)$/.test(p.name ?? '')) { + fileTool++; + cand.push(p.input?.file_path ?? p.input?.path ?? p.input?.pattern ?? ''); + } else if (p.name === 'Bash') { + const found = pathsFromBash(p.input?.command ?? ''); + bashReads += found.length; + cand.push(...found); + } + const paths = []; + const network: Array<{ path: string; kind: string }> = []; + if (p.name === 'Bash') { + for (const target of networkTargetsFromBash(p.input?.command ?? '')) { + const kind = networkKind(target, endpoints, isolatedLoopback); + if (kind) network.push({ path: `${target.host}${target.port === null ? '' : `:${target.port}`}`, kind }); + } + } + if (p.name === 'Bash' || WEB_TOOLS.has(p.name ?? '')) { + const fetched = p.name === 'Bash' ? p.input?.command ?? '' : JSON.stringify(p.input ?? {}); + for (const reference of stackBenchSourceReferences(fetched)) network.push({ path: reference, kind: STACKBENCH_SOURCE }); + } + for (const raw of cand) { + let n = norm(raw); + if (!n || IGNORE.test(n)) continue; + // The CLI keeps auto-memory for the session's OWN project dir. A build + // A session may read its own memory, never another project's memory. + if (cwd && /[/\\]projects[/\\][^/\\]+[/\\]memory[/\\]/.test(n) + && n.includes(cwd.replace(/[\\/:]/g, '-'))) continue; + const absolute = /^[a-z]:/.test(n) || n.startsWith('/'); + if (!absolute && cwd) n = `${cwd}/${n.replace(/^\.\//, '')}`; + n = posix.normalize(n); + // A copy of StackBench's own source is never the app's, wherever it was put. + const privateHarnessPath = cwd + && (n === `${cwd}/stack-bench` || n.startsWith(`${cwd}/stack-bench/`) || HARNESS_PATH.test(n)); + if (!privateHarnessPath && !absolute && !cwd) continue; + if (!privateHarnessPath && cwd && (n === cwd || n.startsWith(`${cwd}/`))) continue; + paths.push(n); + } + if ((paths.length || network.length) && p.id && p.name) { + pending.set(p.id, { paths, network, via: p.name }); + } + } + } + // A call whose result never arrived (session cut short) is unresolved, and + // unresolved is not innocent: count it. + for (const { paths, network, via } of pending.values()) { + for (const n of paths) hits.push({ path: n, via, kind: classify(n), unresolved: true }); + for (const target of network) hits.push({ ...target, via: `${via} network attempt`, unresolved: true }); + } + + return { file, cwd, fileTool, bashReads, hits, refused }; +} + +function main(): void { + const { values } = parseArgs({ args: process.argv.slice(2), options: { + app: { type: 'string' }, json: { type: 'boolean' }, + 'own-endpoints': { type: 'string' }, + 'isolated-loopback': { type: 'boolean' }, + } }); + const ownEndpoints = (values['own-endpoints'] ?? '').split(',').filter(Boolean); + const requestedApp = values.app; + if (!requestedApp) throw new Error('--app is required'); + const roots = transcriptDirectories(requestedApp); + // The app directory is the boundary. Do not infer it from a transcript + // folder name. + const appBoundary = norm(resolve(requestedApp)); + const results: AuditResult[] = []; +for (const root of roots) { + if (!existsSync(root)) continue; + const stack = [root]; + while (stack.length) { + const d = stack.pop(); + if (!d) continue; + for (const e of readdirSync(d, { withFileTypes: true })) { + const p = join(d, e.name); + if (e.isDirectory()) { + // Codex, Grok and Antigravity native sessions are agent-writable; audit controller event logs only. + if (!/node_modules/.test(p) && root !== codexTranscriptDirectory(requestedApp) + && root !== grokTranscriptDirectory(requestedApp) + && root !== antigravityTranscriptDirectory(requestedApp)) stack.push(p); + continue; + } + if (!/\.jsonl$/.test(e.name)) continue; + // Include transcripts from the main session and its subagents. + if (!/transcript|^agent-|^[0-9a-f-]{36}\.jsonl$|\.events\.jsonl$/.test(e.name)) continue; + results.push({ ...auditTranscript(p, appBoundary, { ownEndpoints, + isolatedLoopback: values['isolated-loopback'] === true }), root }); + } + } +} + +if (values.json) { + console.log(JSON.stringify(results, null, 2)); + return; +} + +const label = (file: string): string => file.replace(/\\/g, '/') + .split('/').slice(-3).join('/').slice(0, 62); +console.log('\nBuilds that read outside their own directory'); +console.log('(counts BOTH file tools and Bash cat/grep/find; boundary = the session\'s own cwd)\n'); + +let clean = 0; +for (const r of results.sort((a, b) => b.hits.length - a.hits.length)) { + if (!r.cwd) { console.log(` ?? ${label(r.file)} — no cwd recorded, cannot judge`); continue; } + if (!r.hits.length) { + clean++; + // Blocked attempts are worth printing: they are the sandbox doing its job, + // and they say which paths a build still goes looking for. + if (r.refused?.length) { + const kinds = [...new Set(r.refused.map(h => h.kind))].join(', '); + console.log(` ${label(r.file)}\n clean — ${r.refused.length} attempt(s) BLOCKED by the sandbox (${kinds})`); + } + continue; + } + const byKind: Record = {}; + for (const h of r.hits) (byKind[h.kind] ??= []).push(h.path); + console.log(` ${label(r.file)}`); + console.log(` cwd: ...${r.cwd.slice(-52)} (${r.fileTool} file-tool, ${r.bashReads} bash reads)`); + for (const [k, v] of Object.entries(byKind).sort((a, b) => b[1].length - a[1].length)) { + const example = [...new Set(v)][0] ?? ''; + console.log(` ${String(v.length).padStart(3)}x ${k.padEnd(22)} ${example.split('/').slice(-2).join('/')}`); + } +} +console.log(`\n ${clean} transcript(s) read nothing outside their directory.`); +console.log(` ${results.length} transcript(s) examined.\n`); +} + +if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) main(); diff --git a/tools/stack-bench/commands/null-control.ts b/tools/stack-bench/commands/null-control.ts new file mode 100644 index 00000000000..1433548c8bb --- /dev/null +++ b/tools/stack-bench/commands/null-control.ts @@ -0,0 +1,286 @@ +#!/usr/bin/env node +// Grade the real validated production scenarios against a reachable app that +// implements nothing. Every point-bearing criterion must conclusively fail. + +import { execFile } from 'node:child_process'; +import { createServer } from 'node:http'; +import type { Server } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { mkdirSync, mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, join, relative, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parseArgs as parseNodeArgs } from 'node:util'; +import { chromium, type BrowserServer } from 'playwright'; +import { readArtifactPayload, writeRunJson } from '../src/evidence/artifacts.js'; +import { calibrationQualificationIdentity, calibrationQualificationRelease, + resolveCalibrationForRelease } from '../src/composition/calibration-compiler.js'; +import { qualificationScopeIdentity } from '../src/composition/qualification-scope.js'; +import { writeQualificationSnapshot } from '../src/composition/qualification-slices.js'; +import { analyseNullReports } from '../src/evidence/null-control-analysis.js'; +import { resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { createBoundRecipeTaskRequest, resolveRecipeSelection } from '../src/composition/recipe-selection.js'; +import { isDeclaredLevel, listTracks, loadTrack, suitesFor } from '../src/composition/tracks.js'; +import { controllerRunner } from '../src/runtime/runner-environment.js'; +import type { CalibrationPlan } from '../src/composition/calibration-compiler.js'; +import type { RecipeBinding } from '../src/composition/recipe-release.js'; +import type { Track } from '../src/composition/tracks.js'; + +import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js'; +const GRADE = compiledEntrypoint('grader', 'grade.js'); +const NULL_CONTROL_WORKERS = 4; + +interface NullControlArgs { + tracks: string[]; + level: number | null; + recipe?: string; + out?: string; + parentAttemptId?: string; + selectedChecks: string[]; +} + +export function parseNullControlArgs(argv: string[]): NullControlArgs { + const { values } = parseNodeArgs({ args: argv.slice(2), options: { + track: { type: 'string' }, level: { type: 'string' }, recipe: { type: 'string' }, + out: { type: 'string' }, 'parent-attempt-id': { type: 'string' }, + 'selected-check': { type: 'string', multiple: true }, + } }); + const args: NullControlArgs = { + tracks: values.track?.split(',').filter(Boolean) ?? listTracks(), + level: values.level === undefined ? null : Number(values.level), + recipe: values.recipe, out: values.out, parentAttemptId: values['parent-attempt-id'], + selectedChecks: values['selected-check'] ?? [], + }; + if (args.level !== null && (!Number.isInteger(args.level) || args.level < 1)) { + throw new Error('--level must be a positive integer'); + } + if (args.level !== null && args.tracks.length !== 1) { + throw new Error('--level requires exactly one --track'); + } + if (args.recipe && args.level === null) throw new Error('--recipe requires --level'); + if (args.selectedChecks.length && args.level === null) throw new Error('--selected-check requires --level'); + return args; +} + +function runGrade(argv: string[], timeoutMs = 300_000): Promise<{ stdout: string; stderr: string }> { + return new Promise((resolve, reject) => { + execFile(process.execPath, [GRADE, ...argv], { + encoding: 'utf8', maxBuffer: 32 * 1024 * 1024, timeout: timeoutMs, + }, (error, stdout, stderr) => { + if (error) { + error.message = `grader failed: ${error.message}\n${stdout}\n${stderr}`; + reject(error); + } else resolve({ stdout, stderr }); + }); + }); +} + +export function nullControlSuites(track: Track, selectedLevel: number | null = null, + binding: RecipeBinding | null = null) { + if (binding) { + if (selectedLevel === null) throw new Error('recipe-bound null control requires one level'); + if (!Array.isArray(binding.execution) || !binding.execution.length) { + throw new Error('recipe-bound null control requires a typed execution plan'); + } + const executionIds = new Set(); + const mappedKeys = new Set(); + const suites = binding.execution.map(execution => { + if (executionIds.has(execution.id)) { + throw new Error(`recipe-bound null control repeats execution ${execution.id}`); + } + executionIds.add(execution.id); + const checks = binding.release.checkCatalog.filter(check => check.executionId === execution.id); + if (!checks.length) { + throw new Error(`recipe-bound null control execution ${execution.id} maps no checks`); + } + for (const check of checks) { + if (mappedKeys.has(check.stableKey)) { + throw new Error(`recipe-bound null control maps check ${check.stableKey} more than once`); + } + mappedKeys.add(check.stableKey); + } + return { id: execution.id, spec: resolve(track.dir, execution.source ?? ''), + level: selectedLevel, checks }; + }); + const missing = binding.release.checkCatalog + .filter(check => !mappedKeys.has(check.stableKey)).map(check => check.stableKey); + if (missing.length) { + throw new Error(`recipe-bound null control leaves checks unmapped: ${missing.join(', ')}`); + } + return suites; + } + if (selectedLevel !== null && !isDeclaredLevel(track, selectedLevel)) { + throw new Error(`L${selectedLevel} is not declared for ${track.name}`); + } + const seen = new Set(); + const suites = []; + const levels = selectedLevel === null + ? Array.from({ length: track.validatedThrough }, (_, index) => index + 1) + : [selectedLevel]; + for (const level of levels) { + for (const suite of suitesFor(track, level)) { + if (seen.has(suite.spec)) continue; + seen.add(suite.spec); + suites.push({ ...suite, level }); + } + } + return suites; +} + +export function selectNullQualificationBinding(binding: RecipeBinding, calibration: CalibrationPlan): RecipeBinding { + const selected = calibrationQualificationRelease(calibration, binding.release, binding.execution); + return { ...binding, release: selected.release, execution: selected.execution }; +} + +export function createNullQualification(binding: RecipeBinding, calibration: CalibrationPlan, + selectedChecks: string[] = []) { + let selectedBinding = selectNullQualificationBinding(binding, calibration); + if (selectedChecks.length) { + const selected = calibrationQualificationRelease({ qualification: { checks: selectedChecks } }, + selectedBinding.release, selectedBinding.execution); + selectedBinding = { ...selectedBinding, ...selected }; + } + const selection = resolveRecipeSelection(selectedBinding.release, { + checkKeys: selectedBinding.release.checkCatalog.map(check => check.stableKey), + }); + return { + binding: selectedBinding, + calibration, + identity: calibrationQualificationIdentity(calibration), + selectionSha256: selection.sha256, + }; +} + +async function listen(server: Server): Promise { + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen({ port: 0, host: '127.0.0.1' }, () => resolve()); + }); + return (server.address() as AddressInfo).port; +} + +async function main() { + const args = parseNullControlArgs(process.argv); + const nullAttemptId = `null-control-${new Date().toISOString().replace(/[:.]/g, '-')}`; + const work = mkdtempSync(join(tmpdir(), 'stack-bench-null-')); + const app = join(work, 'app'); + const reportsDir = join(work, 'reports'); + mkdirSync(app, { recursive: true }); + mkdirSync(reportsDir, { recursive: true }); + + // Root navigation succeeds, proving the browser and server are healthy. All + // application/API behavior is absent: non-navigation requests get 404. + const server = createServer((request, response) => { + if (request.method === 'GET' && (request.url === '/' || request.headers.accept?.includes('text/html'))) { + response.writeHead(200, { 'content-type': 'text/html; charset=utf-8' }); + response.end('Null control'); + } else { + response.writeHead(404, { 'content-type': 'application/json' }); + response.end('{"error":"not implemented"}'); + } + }); + + const started = Date.now(); + const suiteReports = []; + let qualification: ReturnType | null = null; + let browserServer: BrowserServer | undefined; + try { + const port = await listen(server); + const url = `http://127.0.0.1:${port}`; + // This command owns the empty page; it has no generated app or attempt lease. + browserServer = await chromium.launchServer({ headless: true, host: '127.0.0.1' }); + const browserEndpoint = browserServer.wsEndpoint(); + for (const trackName of args.tracks) { + const track = loadTrack(trackName); + let binding: RecipeBinding | null = null; + let recipeTask: ReturnType['request'] | null = null; + if (args.level !== null) { + binding = resolveRecipeRelease(track, args.level, args.recipe); + if (!binding) throw new Error(`${trackName} L${args.level} has no recipe release`); + recipeTask = createBoundRecipeTaskRequest(binding, { + taskMode: binding.plan.recipe.task.mode === 'action' ? 'fresh' : undefined, + }).request; + const calibration = resolveCalibrationForRelease(binding.release, + { trackRoot: track.dir, stackBenchRoot: ROOT, alias: `L${args.level}` }); + if (!calibration) throw new Error(`${trackName} L${args.level} has no calibration`); + qualification = createNullQualification(binding, calibration, args.selectedChecks); + binding = qualification.binding; + } + const selectedSuites = nullControlSuites(track, args.level, binding); + const resolvedRecipe = binding?.release.id ?? args.recipe; + for (let index = 0; index < selectedSuites.length; index += NULL_CONTROL_WORKERS) { + const reports = await Promise.all(selectedSuites + .slice(index, index + NULL_CONTROL_WORKERS).map(async suite => { + const reportPath = join(reportsDir, + `${trackName}-l${suite.level}-${suite.id.replaceAll('@', '-')}.json`); + console.log(`${trackName} L${suite.level} ${suite.id} (${basename(suite.spec)})`); + await runGrade(['--url', url, '--level', String(suite.level), '--spec', suite.spec, + '--backend', 'postgres', '--track', trackName, '--app', app, '--out', reportPath, + '--null-control', + '--browser-ws-endpoint', browserEndpoint, + '--parent-attempt-id', nullAttemptId, + ...(resolvedRecipe ? ['--recipe', resolvedRecipe] : []), + ...(binding ? ['--expected-recipe-sha256', binding.release.contentSha256] : []), + ...(recipeTask ? ['--recipe-task-json', JSON.stringify(recipeTask)] : []), + ...(qualification ? ['--selection-sha256', qualification.selectionSha256] : []), + ...(('checks' in suite ? suite.checks : []) ?? []) + .flatMap(check => ['--selected-check', check.stableKey])]); + const report = readArtifactPayload(reportPath, { expectedKind: 'grade' }); + console.log(`${suite.id}: ${report.total}/${report.max}`); + return { track: trackName, level: suite.level, id: suite.id, + scenario: relative(track.dir, suite.spec).replaceAll('\\', '/'), report }; + })); + suiteReports.push(...reports); + } + } + + const analysis = analyseNullReports(suiteReports); + const artifact = { + id: nullAttemptId, + kind: 'null_control', + startedAt: new Date(started).toISOString(), + completedAt: new Date().toISOString(), + parentAttemptId: args.parentAttemptId ?? null, + identities: qualification ? { + recipe: { id: qualification.binding.release.id, + sha256: qualification.binding.release.contentSha256 }, + calibration: { id: qualification.identity.id, + sha256: qualification.identity.contentSha256 }, + } : undefined, + durationMs: Date.now() - started, + runner: controllerRunner(), + ...(qualification ? { qualificationScope: qualificationScopeIdentity({ + kind: 'null', release: qualification.binding.release, stackBenchRoot: ROOT, + }) } : {}), + tracks: args.tracks, + ...analysis, + }; + const outputPath = resolve(args.out ?? join(ROOT, 'results', `${artifact.id}.json`)); + writeRunJson(outputPath, artifact); + if (qualification) writeQualificationSnapshot(`${outputPath}.inputs.json`, + qualification.binding.recipePath, qualification.calibration, ROOT); + console.log(JSON.stringify({ + id: artifact.id, + kind: artifact.kind, + durationMs: artifact.durationMs, + tracks: artifact.tracks, + ok: artifact.ok, + summary: artifact.summary, + artifact: outputPath, + }, null, 2)); + if (!analysis.ok) process.exitCode = 1; + } finally { + try { await browserServer?.close(); } + finally { + await new Promise(resolve => server.close(resolve)); + rmSync(work, { recursive: true, force: true }); + } + } +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + main().catch(error => { + console.error(error.stack ?? error.message); + process.exitCode = 2; + }); +} diff --git a/tools/stack-bench/commands/pack-budget.ts b/tools/stack-bench/commands/pack-budget.ts new file mode 100644 index 00000000000..64976f5bee0 --- /dev/null +++ b/tools/stack-bench/commands/pack-budget.ts @@ -0,0 +1,81 @@ +#!/usr/bin/env node + +import { existsSync } from 'node:fs'; +import { dirname, relative, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parseArgs } from 'node:util'; + +import { artifactPayload, recipeArtifactIdentities, writeArtifact } from '../src/evidence/artifacts.js'; +import { resolveCalibrationForRelease } from '../src/composition/calibration-compiler.js'; +import { loadPackBudgetEvidence, PACK_BUDGET_POLICY, recommendPackBudgets } + from '../src/composition/pack-budget.js'; +import { resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { isDeclaredLevel, listTracks, loadTrack } from '../src/composition/tracks.js'; + +interface PackBudgetArgs { + command: 'recommend'; + track: string; + level: number; + evidence: string[]; + out: string; + recipe?: string; +} + +const USAGE = 'usage: pack-budget.js recommend --track --level ' + + '[--recipe ] --evidence [--evidence ...] ' + + '--out '; + +export function parsePackBudgetArgs(argv: string[]): PackBudgetArgs { + const [command, ...options] = argv.slice(2); + const { values } = parseArgs({ args: options, options: { + track: { type: 'string' }, + level: { type: 'string' }, + recipe: { type: 'string' }, + evidence: { type: 'string', multiple: true }, + out: { type: 'string' }, + } }); + const level = Number(values.level); + const evidence = (values.evidence ?? []).map(path => resolve(path)); + if (command !== 'recommend' || !values.track || !Number.isInteger(level) || level < 1 + || !evidence.length || !values.out) throw new Error(USAGE); + if (new Set(evidence).size !== evidence.length) throw new Error('--evidence paths must be unique'); + return { command, track: values.track, level, evidence, out: resolve(values.out), + ...(values.recipe ? { recipe: values.recipe } : {}) }; +} + +function main(): void { + const args = parsePackBudgetArgs(process.argv); + if (!listTracks().includes(args.track)) throw new Error(`unknown track ${args.track}`); + const track = loadTrack(args.track); + if (!isDeclaredLevel(track, args.level)) throw new Error(`L${args.level} is not declared for ${args.track}`); + const binding = resolveRecipeRelease(track, args.level, args.recipe); + if (!binding) throw new Error(`${args.track} L${args.level} has no recipe release`); + const calibration = resolveCalibrationForRelease(binding.release, { trackRoot: track.dir, alias: `L${args.level}` }); + if (!calibration) throw new Error(`${binding.release.id} has no calibration`); + const loaded = loadPackBudgetEvidence(args.evidence); + const result = recommendPackBudgets({ binding, calibration, evidence: loaded, trackRoot: track.dir }); + if (existsSync(args.out)) throw new Error(`refusing to replace existing budget measurement: ${args.out}`); + const id = `pack-budget-${args.track}-l${args.level}-${new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14)}`; + const artifact = writeArtifact(args.out, { kind: 'pack_budget_measurement', id, + identities: recipeArtifactIdentities(binding.release, { + calibration: { id: calibration.id, sha256: calibration.contentSha256 }, + }), + payload: { schemaVersion: 1, track: args.track, level: args.level, policy: PACK_BUDGET_POLICY, + runner: result.measuredRunner, + evidence: loaded.map(item => { + const stackAdapter = item.artifact.identities.stackAdapter; + if (!stackAdapter) throw new Error(`${item.path} has no stack adapter identity`); + return { path: relative(dirname(args.out), item.path).replaceAll('\\', '/'), + sha256: item.sha256, stack: stackAdapter.id }; + }), + samples: result.samples, recommendations: result.recommendations } }); + console.log(JSON.stringify(artifactPayload(artifact), null, 2)); +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + try { main(); } + catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 2; + } +} diff --git a/tools/stack-bench/commands/preflight-cli.ts b/tools/stack-bench/commands/preflight-cli.ts new file mode 100644 index 00000000000..c9b3a7c7fcd --- /dev/null +++ b/tools/stack-bench/commands/preflight-cli.ts @@ -0,0 +1,83 @@ +import { resolve } from 'node:path'; +import { parseArgs } from 'node:util'; + +import { DEFAULT_BUILD_IMAGE } from '../src/composition/product-config.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { stackBenchResultsRoot } from '../src/runtime/operational-paths.js'; +import type { PreflightReport, PreflightRequest } from '../src/runtime/preflight.js'; + +function splitList(value: unknown): string[] { + return String(value).split(',').map(item => item.trim()).filter(Boolean); +} + +export function parsePreflightArgs( + argv: string[], + { env = process.env }: { env?: NodeJS.ProcessEnv } = {}, +): PreflightRequest { + const { values } = parseArgs({ args: argv.slice(2), options: { + backend: { type: 'string', multiple: true }, + track: { type: 'string' }, + levels: { type: 'string' }, + recipe: { type: 'string' }, + 'run-index': { type: 'string' }, + parallelism: { type: 'string' }, + 'agent-adapter': { type: 'string' }, + 'provider-route': { type: 'string' }, + 'max-output-tokens': { type: 'string' }, + guidance: { type: 'string' }, + pack: { type: 'string', multiple: true }, + check: { type: 'string', multiple: true }, + 'results-dir': { type: 'string' }, + smoke: { type: 'boolean' }, + } }); + const request: PreflightRequest = { backends: [], track: 'ecommerce', levels: '1', levelList: [], + runIndex: 0, parallelism: 1, + agentAdapter: 'claude-code', guidance: 'prescribed', packIds: [], checkKeys: [], smoke: false, + image: env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE, + resultsDir: stackBenchResultsRoot(STACK_BENCH_ROOT, env) }; + request.backends = (values.backend ?? []).flatMap(splitList); + if (values.track !== undefined) request.track = values.track; + if (values.levels !== undefined) request.levels = values.levels; + if (values.recipe !== undefined) request.recipe = values.recipe; + if (values['run-index'] !== undefined) request.runIndex = Number(values['run-index']); + if (values.parallelism !== undefined) request.parallelism = Number(values.parallelism); + if (values['agent-adapter'] !== undefined) request.agentAdapter = values['agent-adapter']; + if (values['provider-route'] !== undefined) request.providerRoute = values['provider-route']; + if (values['max-output-tokens'] !== undefined) request.maxOutputTokens = Number(values['max-output-tokens']); + if (values.guidance !== undefined) request.guidance = values.guidance; + request.packIds = (values.pack ?? []).flatMap(splitList); + request.checkKeys = (values.check ?? []).flatMap(splitList); + if (values['results-dir'] !== undefined) request.resultsDir = resolve(values['results-dir']); + request.smoke = values.smoke ?? false; + if (!request.backends.length) throw new Error('--backend is required (comma-separated values are accepted)'); + if (request.guidance !== 'neutral' && request.guidance !== 'prescribed') { + throw new Error('--guidance must be neutral or prescribed'); + } + request.backends = [...new Set(request.backends)].sort(); + if (!Number.isInteger(request.runIndex) || request.runIndex < 0) { + throw new Error('--run-index must be a non-negative integer'); + } + if (!Number.isInteger(request.parallelism) || (request.parallelism ?? 0) < 1) { + throw new Error('--parallelism must be a positive integer'); + } + const match = String(request.levels).match(/^(\d+)(?:-(\d+))?$/); + if (!match || Number(match[2] ?? match[1]) < Number(match[1])) { + throw new Error('--levels must be N or N-M'); + } + request.levelList = Array.from({ length: Number(match[2] ?? match[1]) - Number(match[1]) + 1 }, + (_, index) => Number(match[1]) + index); + if (request.recipe && request.levelList.length !== 1) { + throw new Error('--recipe requires exactly one requested level'); + } + return request; +} + +export function printPreflightReport(report: PreflightReport): void { + console.log(`Stack Bench preflight: ${report.ok ? 'READY' : 'NOT READY'}`); + for (const check of report.checks) { + const mark = check.status === 'pass' ? 'PASS' : check.status === 'warn' ? 'WARN' : 'FAIL'; + console.log(` ${mark.padEnd(4)} ${check.id.padEnd(28)} ${check.summary}`); + if (check.remediation && check.status === 'fail') console.log(` hint: ${check.remediation}`); + } + console.log(`\n${report.summary.passed} passed, ${report.summary.failed} failed, ${report.summary.warnings} warnings`); +} diff --git a/tools/stack-bench/commands/preflight.ts b/tools/stack-bench/commands/preflight.ts new file mode 100644 index 00000000000..06d1d88fc31 --- /dev/null +++ b/tools/stack-bench/commands/preflight.ts @@ -0,0 +1,17 @@ +#!/usr/bin/env node + +import { parsePreflightArgs, printPreflightReport } from './preflight-cli.js'; +import { runPreflight } from '../src/runtime/preflight.js'; + +let request; +try { + request = parsePreflightArgs(process.argv); +} catch (error) { + console.error(`preflight: ${error instanceof Error ? error.message : String(error)}`); + console.error('Usage: stack-bench preflight --backend spacetime[,postgres,mongodb] [--track ecommerce] [--levels 1-2] [--smoke]'); + process.exit(2); +} + +const report = runPreflight(request); +printPreflightReport(report); +process.exitCode = report.ok ? 0 : 1; diff --git a/tools/stack-bench/commands/progression-graph.ts b/tools/stack-bench/commands/progression-graph.ts new file mode 100644 index 00000000000..82dc0a3be9e --- /dev/null +++ b/tools/stack-bench/commands/progression-graph.ts @@ -0,0 +1,21 @@ +import { dirname, join, resolve } from 'node:path'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { writeProgressionGraph } from '../src/progression/progression-graph.js'; + +interface ProgressionGraph { + nodes: unknown[]; + levels: number; +} + +const definitionPath = process.argv[2]; +if (!definitionPath) { + throw new Error('usage: progression-graph [html-path]'); +} +const resolvedDefinitionPath = resolve(definitionPath); +const graph: ProgressionGraph = writeProgressionGraph({ + definitionPath: resolvedDefinitionPath, + htmlPath: process.argv[3] ?? join(STACK_BENCH_ROOT, 'docs', 'dependency-graph.html'), + trackRoot: dirname(dirname(resolvedDefinitionPath)), +}); +console.log(`Rendered ${graph.nodes.length} nodes across ${graph.levels} levels.`); diff --git a/tools/stack-bench/commands/qualification-cli.ts b/tools/stack-bench/commands/qualification-cli.ts new file mode 100644 index 00000000000..97bd1e37909 --- /dev/null +++ b/tools/stack-bench/commands/qualification-cli.ts @@ -0,0 +1,253 @@ +#!/usr/bin/env node + +import { existsSync, readFileSync } from 'node:fs'; +import { basename, dirname, extname, join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parseArgs as parseNodeArgs } from 'node:util'; + +import { calibrationQualificationIdentity, resolveCalibrationForRelease } from '../src/composition/calibration-compiler.js'; +import { resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { isDeclaredLevel, listTracks, loadTrack } from '../src/composition/tracks.js'; +import { PACK_BUDGET_POLICY } from '../src/composition/pack-budget.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { stackBenchResultsRoot } from '../src/runtime/operational-paths.js'; +import { companionReferenceArtifactPath, MAX_MUTATION_WORKERS, validateMutationWorkerCount } + from '../src/references/reference-live.js'; +import type { CalibrationPlan } from '../src/composition/calibration-compiler.js'; +import type { RecipeBinding, RecipeRelease } from '../src/composition/recipe-release.js'; + +interface QualificationArgs { + command?: string; + track: string | null; + level: number | null; + recipe?: string; + mutationWorkers?: number; +} + +interface QualificationBlocker { + code: string; + path: string; + summary: string; +} + +export function parseQualificationArgs(argv: string[]): QualificationArgs { + const { positionals, values } = parseNodeArgs({ args: argv.slice(2), allowPositionals: true, + options: { track: { type: 'string' }, level: { type: 'string' }, recipe: { type: 'string' }, + 'mutation-workers': { type: 'string' } } }); + const args: QualificationArgs = { command: positionals[0], track: values.track ?? null, + level: values.level === undefined ? null : Number(values.level), + ...(values.recipe === undefined ? {} : { recipe: values.recipe }), + ...(values['mutation-workers'] === undefined ? {} : { + mutationWorkers: validateMutationWorkerCount(Number(values['mutation-workers'])) }) }; + if (args.command !== 'status' || typeof args.track !== 'string' || !args.track + || positionals.length !== 1 || args.level === null || !Number.isInteger(args.level) || args.level < 1) { + throw new Error('usage: node dist/commands/qualification-cli.js status --track --level ' + + '[--recipe ] [--mutation-workers ]'); + } + return args; +} + +function blocker(code: string, path: string, summary: string): QualificationBlocker { + return { code, path, summary }; +} + +function evidencePlan(calibration: CalibrationPlan) { + const stacks = [...calibration.qualification.stacks].sort(); + const evidence = []; + for (const stack of stacks) { + for (let repetition = 1; repetition <= calibration.qualification.referenceRepetitions; repetition += 1) { + evidence.push({ kind: 'reference', stack, repetition }); + } + for (let repetition = 1; repetition <= calibration.qualification.mutationRepetitions; repetition += 1) { + evidence.push({ kind: 'mutation', stack, repetition }); + } + } + for (let repetition = 1; repetition <= calibration.nullControl.repetitions; repetition += 1) { + evidence.push({ kind: 'null', stack: null, repetition }); + } + return evidence; +} + +export interface CalibrationMutationSelection { + mutations: Array<{ backend: string; path: string; targets: Array<{ id: string }> }>; +} + +export function mutationWorkerCount(calibration: CalibrationMutationSelection, stack: string, + readManifest: (path: string) => { mutations?: { id: string }[] } = path => + JSON.parse(readFileSync(resolve(STACK_BENCH_ROOT, path), 'utf8')) as { mutations?: { id: string }[] }, + requestedWorkers = MAX_MUTATION_WORKERS) { + validateMutationWorkerCount(requestedWorkers); + const entry = calibration.mutations.find(candidate => candidate.backend === stack); + if (!entry) return 1; + const manifest = readManifest(entry.path); + const selectedIds = new Set(entry.targets.map(target => target.id)); + const selectedMutations = (manifest.mutations ?? []).filter(mutation => + selectedIds.delete(mutation.id)); + if (selectedIds.size) { + throw new Error(`${stack} calibration selects missing mutations: ${[...selectedIds].sort().join(', ')}`); + } + return Math.min(requestedWorkers, Math.max(1, selectedMutations.length)); +} + +function mutationWorkerOption(calibration: CalibrationPlan, stack: string, requestedWorkers: number) { + const workers = mutationWorkerCount(calibration, stack, undefined, requestedWorkers); + return workers > 1 ? ` --mutation-workers ${workers}` : ''; +} + +function qualificationRunDirectory(artifactPath: string): string { + return join(dirname(artifactPath), `${basename(artifactPath, extname(artifactPath))}.runs`); +} + +function defectCheckCoverage(release: RecipeRelease, calibration: CalibrationPlan) { + const selected = calibration.qualification.checks + ? new Set(calibration.qualification.checks) : null; + const scored = release.checkCatalog.filter(check => check.points > 0 + && (selected === null || selected.has(check.stableKey))); + const scoredByKey = new Map(scored.map(check => [check.stableKey, check])); + const stacks = [...calibration.qualification.stacks].sort(); + return { + required: 'every scored check has an exact known-defect test on every supported stack', + totalChecks: scored.length, + totalPoints: scored.reduce((total, check) => total + check.points, 0), + stacks: stacks.map(stack => { + const covered = new Set(calibration.mutations + .filter(entry => entry.backend === stack) + .flatMap(entry => entry.targets.flatMap(target => target.stableKeys)) + .filter(key => scoredByKey.has(key))); + const missing = scored.filter(check => !covered.has(check.stableKey)); + return { + stack, + coveredChecks: covered.size, + coveredPoints: [...covered].reduce((total, key) => total + (scoredByKey.get(key)?.points ?? 0), 0), + missingChecks: missing.map(check => check.stableKey), + }; + }), + }; +} + +export function qualificationReadiness(trackName: string, level: number, recipe: string | null = null, + mutationWorkers = MAX_MUTATION_WORKERS) { + validateMutationWorkerCount(mutationWorkers); + if (!listTracks().includes(trackName)) throw new Error(`unknown qualification track ${trackName}`); + const track = loadTrack(trackName); + if (!isDeclaredLevel(track, level)) { + throw new Error(`L${level} is not declared for ${trackName}`); + } + const binding: RecipeBinding | null = resolveRecipeRelease(track, level, recipe); + if (!binding) throw new Error(`${trackName} L${level} has no recipe release`); + const calibration = resolveCalibrationForRelease(binding.release, + { trackRoot: track.dir, alias: `L${level}` }); + if (!calibration) { + throw new Error(`${binding.release.id} has no L${level} calibration`); + } + const identity = calibrationQualificationIdentity(calibration); + const qualificationLevel = Number(calibration.selection.alias.slice(1)); + const launchBlockers = []; + for (const pack of binding.plan.packs) { + if (pack.budget.status !== 'bounded') { + launchBlockers.push(blocker('pack_budget_unbounded', `packs.${pack.id}.budget`, + `${pack.id} needs a measured maxRuntimeMs before qualification`)); + } + } + + const requiredEvidence = evidencePlan(calibration); + const defectChecks = defectCheckCoverage(binding.release, calibration); + const recorded = new Set(calibration.qualification.evidence.map(entry => + `${entry.kind}:${entry.stack ?? ''}:${entry.repetition}`)); + const qualificationBlockers = [...launchBlockers]; + for (const coverage of defectChecks.stacks.filter(item => item.missingChecks.length > 0)) { + qualificationBlockers.push(blocker('defect_check_coverage_incomplete', + `defectChecks.${coverage.stack}`, + `${coverage.coveredChecks}/${defectChecks.totalChecks} scored checks have exact known-defect tests`)); + } + for (const item of requiredEvidence) { + const key = `${item.kind}:${item.stack ?? ''}:${item.repetition}`; + if (!recorded.has(key)) qualificationBlockers.push(blocker('evidence_missing', `evidence.${key}`, + `${key} has no hash-bound qualification artifact`)); + } + for (const stale of (calibration.qualificationStaleness ?? []) as { + kind: string; stack?: string; repetition: number; reason: string; + }[]) { + const key = `${stale.kind}:${stale.stack ?? ''}:${stale.repetition}`; + qualificationBlockers.push(blocker('qualification_evidence_stale', `evidence.${key}`, + `${key} must be regenerated: ${stale.reason}`)); + } + const output = join(stackBenchResultsRoot(STACK_BENCH_ROOT), 'qualification'); + const stacks = [...calibration.qualification.stacks].sort(); + const budgetEvidence = stacks.map(stack => + `${output}/budget-input/${trackName}-l${qualificationLevel}-${stack}.json`); + const budgetPreparationRequired = launchBlockers.some(item => item.code === 'pack_budget_unbounded'); + const recipeOption = ` --recipe ${binding.release.id}`; + const featureCatalog = calibration.qualification.featureCatalog; + const featureCatalogOption = featureCatalog + ? ` --feature-catalog ${featureCatalog.path}` : ''; + const combinedReferenceEvidence = calibration.qualification.referenceRepetitions + === calibration.qualification.mutationRepetitions; + const artifactStem = `${trackName}-l${qualificationLevel}-${binding.release.contentSha256.slice(0, 12)}`; + const artifactPaths = { + references: Object.fromEntries(stacks.map(stack => [stack, + `${output}/${artifactStem}-${stack}-reference.json`])), + mutations: Object.fromEntries(stacks.map(stack => [stack, + `${output}/${artifactStem}-${stack}-mutation.json`])), + null: `${output}/${artifactStem}-null.json`, + }; + const launchPaths = new Set([artifactPaths.null]); + for (const stack of stacks) { + const mutationPath = artifactPaths.mutations[stack]; + const referencePath = artifactPaths.references[stack]; + if (!mutationPath || !referencePath) throw new Error(`qualification path is missing for ${stack}`); + launchPaths.add(mutationPath); + launchPaths.add(qualificationRunDirectory(mutationPath)); + launchPaths.add(combinedReferenceEvidence + ? companionReferenceArtifactPath(mutationPath) : referencePath); + if (!combinedReferenceEvidence) { + launchPaths.add(qualificationRunDirectory(referencePath)); + } + } + for (const path of [...launchPaths].filter(existsSync).sort()) { + launchBlockers.push(blocker('qualification_output_exists', path, + 'qualification output already exists')); + } + return { + qualificationSchemaVersion: 1, + scope: { track: trackName, level, recipe: { id: binding.release.id, + contentSha256: binding.release.contentSha256 }, + calibration: { ...identity, contentSha256: calibration.contentSha256 }, + runner: calibration.qualification.runner ?? null }, + launch: { ok: launchBlockers.length === 0, blockers: launchBlockers }, + budgetPreparation: { + required: budgetPreparationRequired, + policy: PACK_BUDGET_POLICY, + commands: budgetPreparationRequired ? [ + ...stacks.map((stack, index) => + `qualify-reference --timing-only --backend ${stack} --track ${trackName} --level ${qualificationLevel}${recipeOption}${featureCatalogOption} --repetitions ${calibration.qualification.referenceRepetitions} --out ${budgetEvidence[index]}`), + `pack-budget recommend --track ${trackName} --level ${qualificationLevel}${recipeOption} ${budgetEvidence + .map(path => `--evidence ${path}`).join(' ')} --out ${output}/${trackName}-l${qualificationLevel}-pack-budgets.json`, + ] : [], + }, + requiredEvidence, + defectChecks, + artifactPaths, + commands: qualificationBlockers.length === 0 ? [] : [ + ...stacks.flatMap(stack => [ + ...(!combinedReferenceEvidence ? [ + `qualify-reference --backend ${stack} --track ${trackName} --level ${qualificationLevel}${recipeOption}${featureCatalogOption} --repetitions ${calibration.qualification.referenceRepetitions} --out ${artifactPaths.references[stack]}`, + ] : []), + `qualify-reference --backend ${stack} --track ${trackName} --level ${qualificationLevel}${recipeOption}${featureCatalogOption} --repetitions ${calibration.qualification.mutationRepetitions} --mutations --full-mutations${mutationWorkerOption(calibration, stack, mutationWorkers)} --out ${artifactPaths.mutations[stack]}`, + ]), + `qualify-null --track ${trackName} --level ${qualificationLevel}${recipeOption} --out ${artifactPaths.null}`, + ], + qualification: { ready: qualificationBlockers.length === 0, blockers: qualificationBlockers }, + }; +} + +function main() { + const args = parseQualificationArgs(process.argv); + if (!args.track || args.level === null) throw new Error('track and level are required'); + console.log(JSON.stringify(qualificationReadiness(args.track, args.level, args.recipe, args.mutationWorkers), null, 2)); +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + try { main(); } + catch (error: unknown) { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 2; } +} diff --git a/tools/stack-bench/commands/recovery.ts b/tools/stack-bench/commands/recovery.ts new file mode 100644 index 00000000000..c6ae1a72244 --- /dev/null +++ b/tools/stack-bench/commands/recovery.ts @@ -0,0 +1,28 @@ +#!/usr/bin/env node + +import { recoverBackendLease, recoverSupervisedRun } from '../src/runtime/recovery.js'; + +const [command, statePath, option, output] = process.argv.slice(2); +const supervisorRequest = command === 'recover' && statePath !== undefined && process.argv.length === 4; +const leaseRequest = command === 'recover-lease' && statePath !== undefined && option === '--out' + && output !== undefined && process.argv.length === 6; +if (!supervisorRequest && !leaseRequest) { + console.error('Usage:\n' + + ' stack-bench recover \n' + + ' stack-bench recover-lease --out '); + process.exit(2); +} + +try { + const result = leaseRequest + ? recoverBackendLease(statePath, output) + : recoverSupervisedRun(statePath); + if (result === null) { + console.log(`recovery: nothing to recover; supervisor state ${statePath} does not exist ` + + '(authenticated recovery removes it after proven cleanup)'); + } else console.log(JSON.stringify(result, null, 2)); + process.exitCode = result === null || result.ok ? 0 : 1; +} catch (error) { + console.error(`recovery: ${error instanceof Error ? error.message : String(error)}`); + process.exitCode = 2; +} diff --git a/tools/stack-bench/commands/repair-cli.ts b/tools/stack-bench/commands/repair-cli.ts new file mode 100644 index 00000000000..4b785fefc34 --- /dev/null +++ b/tools/stack-bench/commands/repair-cli.ts @@ -0,0 +1,222 @@ +#!/usr/bin/env node + +import { randomUUID } from 'node:crypto'; +import { existsSync, mkdirSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { acquireCampaignLock, releaseCampaignLock } from '../src/campaigns/campaign-lock.js'; +import { ARTIFACT_FILE, emptyArtifactIdentities, readArtifact, writeArtifact } + from '../src/evidence/artifacts.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { rescueSupervisedLease } from '../src/runtime/recovery.js'; +import { runBounded } from '../src/runtime/bounded-process.js'; +import type { BoundedProcessResult, RunBoundedOptions } + from '../src/runtime/bounded-process.js'; +import { createRepairGrant, inspectRepairParent } from '../src/runtime/repair-grant.js'; + +const BENCH = join(STACK_BENCH_ROOT, 'dist', 'commands', 'bench.js'); + +export interface RepairStatusArgs { + command: 'status'; + parent: string; + level: number; +} + +export interface RepairGrantArgs { + command: 'grant'; + parent: string; + level: number; + repairs: number; + maxBudgetUsd?: number; + timeoutMinutes: number; +} + +export type RepairArgs = RepairStatusArgs | RepairGrantArgs; + +export function parseRepairArgs(argv: string[]): RepairArgs { + const [command, parent, ...rest] = argv.slice(2); + if (command === 'status' && parent && rest.length === 2 && rest[0] === '--level') { + const level = Number(rest[1]); + if (!Number.isSafeInteger(level) || level < 1) throw new Error('--level must be a positive integer'); + return { command, parent: resolve(parent), level }; + } + if (command !== 'grant' || !parent) { + throw new Error('usage: repair status --level | repair grant --level --repairs [--max-budget-usd ] [--timeout-minutes ]'); + } + const values: { level?: number; repairs?: number; maxBudgetUsd?: number; + timeoutMinutes: number } = { timeoutMinutes: 120 }; + const seen = new Set(); + for (let index = 0; index < rest.length; index += 2) { + const flag = rest[index]; + if (!flag || !['--level', '--repairs', '--max-budget-usd', '--timeout-minutes'].includes(flag) + || index + 1 >= rest.length || seen.has(flag)) { + throw new Error(`invalid or duplicate repair option ${String(flag)}`); + } + seen.add(flag); + const value = Number(rest[index + 1]); + if (flag === '--level') values.level = value; + else if (flag === '--repairs') values.repairs = value; + else if (flag === '--max-budget-usd') values.maxBudgetUsd = value; + else values.timeoutMinutes = value; + } + const level = values.level; + if (level === undefined || !Number.isSafeInteger(level) || level < 1) { + throw new Error('--level must be a positive integer'); + } + const repairs = values.repairs; + if (repairs === undefined || !Number.isSafeInteger(repairs) || repairs < 1) { + throw new Error('--repairs must be a positive safe integer'); + } + if (values.maxBudgetUsd !== undefined + && (!Number.isFinite(values.maxBudgetUsd) || values.maxBudgetUsd <= 0)) { + throw new Error('--max-budget-usd must be a positive number'); + } + if (!Number.isFinite(values.timeoutMinutes) || values.timeoutMinutes < 10 + || values.timeoutMinutes > 480) { + throw new Error('--timeout-minutes must be from 10 through 480'); + } + return { command, parent: resolve(parent), level, + repairs, timeoutMinutes: values.timeoutMinutes, + ...(values.maxBudgetUsd === undefined ? {} : { maxBudgetUsd: values.maxBudgetUsd }) }; +} + +export function repairStatus(parent: string, level: number): Record { + try { + const inspected = inspectRepairParent(parent, level); + return { eligible: true, parentRunId: inspected.parent.id, level, + score: inspected.level.score, max: inspected.level.max, + used: inspected.cumulativeRepairsBefore, + checkpointSha256: inspected.checkpoint.payload.source.sha256 }; + } catch (error) { + return { eligible: false, level, + reason: error instanceof Error ? error.message : String(error) }; + } +} + +interface RepairExecutionDependencies { + execute?: (command: string, argv: string[], + options: RunBoundedOptions) => Promise; + rescue?: (path: string, output: string) => void; + uuid?: () => string; + env?: NodeJS.ProcessEnv; +} + +interface RepairContinuationPayload { + outcome?: unknown; + continuation?: { + parentRunId?: string; + repairsGranted?: number; + level?: number; + [key: string]: unknown; + }; + [key: string]: unknown; +} + +export async function executeRepairGrant(args: RepairGrantArgs, + { execute = runBounded, rescue = rescueSupervisedLease, uuid = randomUUID, + env = process.env }: RepairExecutionDependencies = {}) { + const resolved = createRepairGrant(args.parent, { level: args.level, repairs: args.repairs }); + const lock = acquireCampaignLock(join(resolved.root, '.repair-control'), { + id: `repair-l${args.level}`, + contentSha256: resolved.checkpoint.payload.source.sha256, + }); + const stamp = new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14); + const executionId = `grant-${stamp}-${uuid().replaceAll('-', '').slice(0, 12)}`; + const output = join(resolved.root, 'continuations', executionId); + const privateRoot = join(tmpdir(), 'stack-bench-repair-supervisors'); + const supervisorState = join(privateRoot, `${executionId}.json`); + let cleanupError: unknown = null; + try { + mkdirSync(output, { recursive: true }); + mkdirSync(privateRoot, { recursive: true, mode: 0o700 }); + const argv = [BENCH, + '--repair-from', resolved.root, + '--repair-level', String(args.level), + '--repairs', String(args.repairs), + '--out', output, + '--no-media']; + if (args.maxBudgetUsd !== undefined) { + argv.push('--max-budget-usd', String(args.maxBudgetUsd)); + } + const childEnv: NodeJS.ProcessEnv = { + ...env, + STACK_BENCH_SUPERVISOR_STATE: supervisorState, + }; + if (resolved.configuration.buildImage) { + childEnv.STACK_BENCH_IMAGE = resolved.configuration.buildImage; + } + const processResult = await execute(process.execPath, argv, { + cwd: STACK_BENCH_ROOT, + env: childEnv, + stdio: 'inherit', + timeoutMs: args.timeoutMinutes * 60_000, + logs: { stdout: join(output, 'process.stdout.log'), + stderr: join(output, 'process.stderr.log') }, + }); + cleanupError = null; + if (!processResult.ok && existsSync(supervisorState)) { + try { rescue(supervisorState, output); } + catch (error) { cleanupError = error; } + } + const streams = processResult.logs ? Object.fromEntries(Object.entries(processResult.logs) + .map(([name, value]) => [name, { ...value, path: `process.${name}.log` }])) : null; + writeArtifact(join(output, ARTIFACT_FILE.process), { + kind: 'repair_process', + id: `${executionId}-process`, + attempt: { id: `${executionId}-process`, parentId: resolved.parent.id }, + identities: emptyArtifactIdentities({ + agentAdapter: resolved.parentArtifact.identities.agentAdapter, + stackAdapter: resolved.parentArtifact.identities.stackAdapter, + }), + payload: { schemaVersion: 2, parentRunId: resolved.parent.id, + level: args.level, repairsGranted: args.repairs, + exitCode: processResult.code ?? null, signal: processResult.signal ?? null, + timedOut: processResult.timedOut, streams }, + }); + if (cleanupError) { + const detail = cleanupError instanceof Error ? cleanupError.message : String(cleanupError); + throw new Error(`repair continuation cleanup failed: ${detail}`); + } + const runPath = join(output, ARTIFACT_FILE.run); + if (!existsSync(runPath)) { + throw new Error(`repair continuation produced no run artifact${processResult.timedOut ? ' before its timeout' : ''}`); + } + const run = readArtifact(runPath, + { expectedKind: 'repair_continuation' }); + if (run.attempt.parentId !== resolved.parent.id + || run.payload.continuation?.parentRunId !== resolved.parent.id + || run.payload.continuation?.repairsGranted !== args.repairs + || run.payload.continuation?.level !== args.level) { + throw new Error('repair continuation result does not match its grant'); + } + return { output, process: processResult, run }; + } finally { + // A failed cleanup keeps the supervisor state as recover's authority. + if (!cleanupError) rmSync(supervisorState, { force: true }); + releaseCampaignLock(lock); + } +} + +async function main(): Promise { + const args = parseRepairArgs(process.argv); + if (args.command === 'status') { + const status = repairStatus(args.parent, args.level); + console.log(JSON.stringify(status, null, 2)); + if (status.eligible !== true) process.exitCode = 1; + return; + } + const result = await executeRepairGrant(args); + console.log(JSON.stringify({ output: result.output, id: result.run.id, + outcome: result.run.payload.outcome, + continuation: result.run.payload.continuation }, null, 2)); + if (!result.process.ok) process.exitCode = 1; +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 2; + }); +} diff --git a/tools/stack-bench/commands/repin.ts b/tools/stack-bench/commands/repin.ts new file mode 100644 index 00000000000..3115d840765 --- /dev/null +++ b/tools/stack-bench/commands/repin.ts @@ -0,0 +1,104 @@ +#!/usr/bin/env node + +import { createHash } from 'node:crypto'; +import { existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { buildRecipeRelease } from '../src/composition/recipe-release.js'; +import { listTracks, TRACKS_DIR } from '../src/composition/tracks.js'; +import { resolveFeatureCatalog } from '../src/progression/feature-catalog-selection.js'; +import { progressionLevels, selectFeatureCatalogLevels } from '../src/progression/progression-definition.js'; +import { inspectImportedReference, loadReferenceRegistry } from '../src/references/reference-fixtures.js'; +import { STACK_BENCH_ROOT as ROOT } from '../src/package-root.js'; +import { checkCalibrations } from './check-calibration.js'; + +type Json = Record; +const sha256 = (bytes: Buffer | string) => createHash('sha256').update(bytes).digest('hex'); +const readJson = (path: string): Json => JSON.parse(readFileSync(path, 'utf8')); + +// Write only a changed file, in the 2-space form these files already use. +function writeJson(path: string, value: unknown, changed: string[]): void { + const text = `${JSON.stringify(value, null, 2)}\n`; + if (readFileSync(path, 'utf8').replaceAll('\r\n', '\n') === text) return; + writeFileSync(path, text); + changed.push(path); +} + +function lockfiles(directory: string): string[] { + return readdirSync(directory, { withFileTypes: true }).flatMap(entry => entry.name === 'node_modules' ? [] + : entry.isDirectory() ? lockfiles(join(directory, entry.name)) + : entry.name === 'package-lock.json' ? [join(directory, entry.name)] : []); +} + +// Recompute every derived hash that sources pin, after the sources change. This +// never edits qualification evidence; check-calibration reports what a change invalidates. +export function repin(): string[] { + const changed: string[] = []; + // A release builds its local packages (resolved file:) itself, so a checksum + // pinned in a reference lockfile only breaks when that package changes. + for (const path of lockfiles(join(ROOT, 'reference-apps'))) { + const lock = readJson(path); + for (const entry of Object.values(lock.packages ?? {})) { + if (String(entry.resolved ?? '').startsWith('file:')) delete entry.integrity; + } + writeJson(path, lock, changed); + } + + const registryPath = join(ROOT, 'reference-apps', 'registry.json'); + const registry = readJson(registryPath); + for (const fixture of loadReferenceRegistry(registryPath).fixtures) { + if (!fixture.imported) continue; + const inspected = inspectImportedReference(fixture, { root: ROOT }); + if (!inspected.available || !inspected.sourceSha256) throw new Error(`${fixture.id}: ${inspected.failures.join('; ')}`); + registry.fixtures.find((entry: Json) => entry.id === fixture.id).imported.sourceSha256 = inspected.sourceSha256; + } + writeJson(registryPath, registry, changed); + const referenceHash = (id: string): string => { + const hash = registry.fixtures.find((entry: Json) => entry.id === id)?.imported?.sourceSha256; + if (!hash) throw new Error(`reference ${id} has no imported source hash`); + return hash; + }; + + for (const track of listTracks({ includeInternal: true })) { + const trackRoot = join(TRACKS_DIR, track); + const directory = join(trackRoot, 'composition', 'calibrations'); + if (!existsSync(directory)) continue; + for (const file of readdirSync(directory).filter(name => name.endsWith('.json')).sort()) { + const path = join(directory, file); + const calibration = readJson(path); + const release = buildRecipeRelease(resolve(dirname(path), calibration.recipe.path), { trackRoot }); + for (const key of ['meaningSha256', 'executionSha256', 'contentSha256']) calibration.recipe[key] = release[key]; + calibration.fixture.sourceSha256 = release.components.fixture.sha256; + calibration.selection.sha256 = sha256(readFileSync(resolve(trackRoot, calibration.selection.path))); + const catalog = calibration.qualification.featureCatalog; + if (catalog) { + const full = resolveFeatureCatalog(catalog.path, { dir: trackRoot, name: release.track }); + const level = Number(calibration.selection.alias.slice(1)); + catalog.contentSha256 = selectFeatureCatalogLevels(full, + progressionLevels(full).filter(candidate => candidate <= level)).identity.contentSha256; + } + for (const entry of calibration.references.entries) entry.sourceSha256 = referenceHash(entry.id); + for (const entry of calibration.mutations) { + const manifestPath = join(ROOT, entry.path); + const manifest = readJson(manifestPath); + manifest.fixtureSha256 = referenceHash(entry.referenceId); + writeJson(manifestPath, manifest, changed); + entry.sha256 = sha256(readFileSync(manifestPath)); + } + writeJson(path, calibration, changed); + } + } + checkCalibrations(); + return [...new Set(changed)]; +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { + const changed = repin(); + console.log(changed.length ? changed.map(path => `updated ${path.slice(ROOT.length + 1)}`).join('\n') : 'all pins current'); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} diff --git a/tools/stack-bench/commands/report-bugs.ts b/tools/stack-bench/commands/report-bugs.ts new file mode 100644 index 00000000000..b2ad2268d8a --- /dev/null +++ b/tools/stack-bench/commands/report-bugs.ts @@ -0,0 +1,398 @@ +#!/usr/bin/env node +import { privateGradingDirectory } from '../src/evidence/repair-evidence.js'; +// Turns grading results into a behavioral BUG_REPORT.md for the fix agent. +// +// Report behavior and typed observations, never implementation advice. A setup +// failure must not be described as a failure of the later criterion. + +import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parseArgs as parseNodeArgs } from 'node:util'; +import { findingStatus, renderRepairFinding } from '../src/actions/action-findings.js'; +import type { Finding } from '../src/actions/action-findings.js'; +import type { ActionEvidence } from '../src/actions/action-contract.js'; +import type { CheckEvidence } from '../src/evidence/check-evidence.js'; +import { sanitiseConsoleError, sanitiseDiagnostic } from '../src/evidence/diagnostic-sanitizer.js'; +import { ARTIFACT_FILE, readArtifactPayload } from '../src/evidence/artifacts.js'; +import { criterionEvidence, evidenceIsRepairable, validateCheckEvidence } from '../src/evidence/check-evidence.js'; +import { assertAgentVisibleText } from '../src/composition/agent-visible-contract.js'; +import { CODING_CONTAINER_BUG_REPORT_FILE, CODING_CONTAINER_START_SCRIPT } + from '../src/runtime/coding-container-policy.js'; + +interface RepairHistoryEntry { + round?: number; + beforeScore?: number; + beforeMax?: number; + afterScore?: number; + afterMax?: number; + result?: string; + remainingFailures?: string[]; +} + +interface ReportBugsArgs { + app: string; + results: string; + out: string; + archive?: string; + history: RepairHistoryEntry[]; + checks: string[] | null; + controls: string[] | null; + priorRegression: string | null; + regressionContext: boolean; +} + +interface ParsedArgs { + app?: string; + results?: string; + out?: string; + archive?: string; + history?: unknown; + checks?: unknown; + controls?: unknown; +} + +interface Criterion { + id?: string; + stableKey?: string; + desc?: string; + statedBy?: string; + points?: number; + evidence?: unknown; +} + +interface GradeFeature { + name?: string; + consoleErrors?: string[]; + criteria?: Criterion[]; + setupEvidence?: unknown; +} + +interface GradePayload { + features?: GradeFeature[]; +} + +interface ContractResult { + id: string; + status: string; + detail?: string; +} + +interface ContractLintPayload { + results?: ContractResult[]; +} + +interface GradeBundlePayload { + backend?: string; + outcome?: { kind?: string; phase?: string; reason?: string }; +} + +interface RepairBug { + area: string; + actor: string | null; + action: string | null; + expected: string | null; + observed: string; + consoleErrors: string[]; + contract: boolean; + context?: string[]; +} + +// Only completed public interactions, never raw action inputs or diagnostics. +// These observations explain where a sequence stopped without teaching a fix. +function observationContext(evidence: CheckEvidence): string[] { + const actions = evidence.actions.map(entry => ({ actor: entry.actor, + evidence: entry.evidence as ActionEvidence })); + const failureIndex = actions.findLastIndex(entry => entry.evidence.status === 'failed'); + const context: string[] = evidence.phase === 'setup' + ? ['Setup stopped before the named behavior was reached.'] : []; + if (failureIndex < 0) return context; + const completed: string[] = []; + const lifecycle: string[] = []; + for (const { actor, evidence: action } of actions.slice(0, failureIndex)) { + if (action.status !== 'passed') continue; + const observation = action.observation && typeof action.observation === 'object' + ? action.observation as Record : {}; + if (action.action.id === 'click' && typeof observation.clicked === 'string') { + completed.push(`${actor ? `${sanitiseDiagnostic(actor, 120)}: ` : ''}${sanitiseDiagnostic(observation.clicked, 120)}`); + } + if (action.action.id === 'callAction' && typeof observation.action === 'string' + && Number.isInteger(observation.status) && Number(observation.status) >= 100 && Number(observation.status) <= 599) { + completed.push(`${actor ? `${sanitiseDiagnostic(actor, 120)}: ` : ''}${sanitiseDiagnostic(observation.action, 120)} returned HTTP ${observation.status}`); + } + const account = ({ signIn: 'sign-in completed', signUp: 'account creation completed', + freshClient: 'fresh client opened' } as Record)[action.action.id]; + if (account) completed.push(`${actor ? `${sanitiseDiagnostic(actor, 120)}: ` : ''}${account}`); + const operations: Record = { reload: 'page reloaded', stopAppServer: 'application server stopped', + startAppServer: 'application server started', restartBackend: 'database runtime restarted' }; + const operation = operations[action.action.id]; + if (operation) lifecycle.push(operation); + } + if (completed.length) context.push(`Recent completed actions: ${completed.slice(-8).join(' → ')}.`); + if (lifecycle.length) context.push(`Completed lifecycle actions: ${[...new Set(lifecycle)].join('; ')}.`); + if (evidence.finding && ['control-missing', 'control-not-ready', 'control-blocked', 'control-unreadable', + 'choice-missing', 'page-timeout'].includes(evidence.finding.kind)) { + context.push('The sequence stopped at this control; later behavior was not observed.'); + } + if (evidence.finding?.kind === 'page-error') { + context.push('The sequence stopped at this action; later behavior was not observed.'); + } + if (evidence.finding && ['value-mismatch', 'number-mismatch'].includes(evidence.finding.kind)) { + context.push('The sequence stopped at this value check; later behavior was not observed.'); + } + return context; +} + +// The public verb for the step that failed. Control and action names are the +// agent's own vocabulary; nothing else about the step is repeated. +function failedAction(action: string | undefined, finding: Finding | null): string | null { + if (action === 'fill') { + return finding?.kind === 'choice-missing' ? 'Select the requested choice' : 'Enter the requested value'; + } + if (action === 'click') return 'Use the requested control'; + if (action === 'signIn') return 'Sign in'; + if (action === 'signUp') return 'Create the account'; + if (action === 'reload') return 'Reload the page'; + return null; +} + +// What the application did, from the finding alone. A failure without a +// finding (the feature's setup failed before this behavior was reached) +// says so and nothing more. +function observed(finding: Finding | null, phase: string): string { + if (finding?.kind === 'page-error') { + // Raw browser diagnostics can contain URLs, credentials and private probe text. + // Report only a recognized transport code, never the surrounding diagnostic. + const code = finding.fields.detail?.match(/\b(?:net::)?(ERR_CONNECTION_REFUSED|ERR_CONNECTION_RESET|ERR_CONNECTION_CLOSED|ERR_CONNECTION_TIMED_OUT|ERR_NAME_NOT_RESOLVED|ERR_ADDRESS_UNREACHABLE|ERR_EMPTY_RESPONSE|ERR_TIMED_OUT)\b/)?.[1]; + if (code) return `the browser request failed (${code})`; + } + if (finding) return renderRepairFinding(finding); + return phase === 'setup' + ? 'the application did not reach this behavior; an earlier step of the same feature failed' + : 'a failure was recorded without a detailed observation'; +} + +export function parseReportBugsArgs(argv: string[]): ReportBugsArgs { + const { values } = parseNodeArgs({ args: argv.slice(2), options: { + app: { type: 'string' }, results: { type: 'string' }, out: { type: 'string' }, + archive: { type: 'string' }, + 'history-json': { type: 'string' }, 'checks-json': { type: 'string' }, + 'controls-json': { type: 'string' }, + 'prior-regression': { type: 'string' }, + 'regression-context': { type: 'boolean' }, + } }); + const args: ParsedArgs = { app: values.app, results: values.results, out: values.out, + archive: values.archive, + history: values['history-json'] === undefined ? undefined : JSON.parse(values['history-json']), + checks: values['checks-json'] === undefined ? undefined : JSON.parse(values['checks-json']), + controls: values['controls-json'] === undefined ? undefined : JSON.parse(values['controls-json']) }; + if (!args.app) { + throw new Error('Usage: report-bugs --app [--out ]'); + } + args.results ??= privateGradingDirectory(args.app); + args.out ??= join(args.app, CODING_CONTAINER_BUG_REPORT_FILE); + args.history ??= []; + if (!Array.isArray(args.history)) throw new Error('--history-json must contain an array'); + args.checks ??= null; + if (args.checks !== null && (!Array.isArray(args.checks) + || args.checks.some(check => typeof check !== 'string' || !check) + || new Set(args.checks).size !== args.checks.length)) { + throw new Error('--checks-json must contain distinct non-empty strings'); + } + args.controls ??= null; + if (args.controls !== null && (!Array.isArray(args.controls) + || args.controls.some(control => typeof control !== 'string' || !control) + || new Set(args.controls).size !== args.controls.length)) { + throw new Error('--controls-json must contain distinct non-empty strings'); + } + return { app: args.app, results: args.results, out: args.out, archive: args.archive, + history: args.history as RepairHistoryEntry[], checks: args.checks as string[] | null, + controls: args.controls as string[] | null, + priorRegression: values['prior-regression'] ?? null, + regressionContext: values['regression-context'] ?? false }; +} + +function priorRegressionSection(path: string): string[] { + const details = assertAgentVisibleText(readFileSync(resolve(path), 'utf8')).trim() + .replace(/^### /gm, '#### ') + .replace(/^## /gm, '### '); + if (!details) throw new Error('prior regression report has no failure details'); + return [ + '## Previous repair regression', + '', + 'The previous repair was rolled back because it broke behavior that already worked.', + 'Keep this behavior working while you fix the current problems.', + '', + ...details.split(/\r?\n/), + '', + ]; +} + +export function createBugReport(args: ReportBugsArgs): number { + const resultsDir = resolve(args.results); + if (!existsSync(resultsDir)) throw new Error(`No grading results in ${resultsDir}`); + + const bugs: RepairBug[] = []; + const reportedSetups = new Set(); + const selectedChecks = args.checks === null ? null : new Set(args.checks); + const selectedControls = args.controls === null ? null : new Set(args.controls); + + for (const file of readdirSync(resultsDir).filter(name => /^grading-.*\.json$/.test(name))) { + const report = readArtifactPayload(join(resultsDir, file), { expectedKind: 'grade' }); + for (const feature of report.features ?? []) { + // Repairs receive only scored, typed application failures. + for (const criterion of feature.criteria ?? []) { + if (selectedChecks && (!criterion.stableKey + || !selectedChecks.has(criterion.stableKey))) continue; + if (!(Number(criterion.points) > 0)) continue; + const evidence = criterionEvidence(criterion); + if (!evidenceIsRepairable(evidence)) continue; + const failure = evidence.phase === 'setup' && feature.setupEvidence + ? validateCheckEvidence(feature.setupEvidence) : evidence; + if (!evidenceIsRepairable(failure) + || (failure.finding && findingStatus(failure.finding) !== 'failed')) continue; + if (evidence.phase === 'setup') { + // One failed setup is copied to each selected criterion it prevented. + // Use full evidence, not rendered prose, so distinct failures stay separate. + const key = JSON.stringify({ area: feature.name, failure }); + if (reportedSetups.has(key)) continue; + reportedSetups.add(key); + } + const actionEntry = failure.actions.findLast(entry => + entry.evidence !== null && typeof entry.evidence === 'object' + && (entry.evidence as { status?: string }).status === 'failed') ?? failure.actions.at(-1); + const actionId = actionEntry && typeof actionEntry.evidence === 'object' && actionEntry.evidence + ? String((actionEntry.evidence as { action?: { id?: string } }).action?.id ?? '') : undefined; + const expected = evidence.phase === 'setup' ? null + : (criterion.desc ?? criterion.statedBy ?? '').trim() || 'the requested behavior'; + bugs.push({ + area: sanitiseDiagnostic(feature.name, 120), + actor: sanitiseDiagnostic(actionEntry?.actor ?? failure.actor, 120) || null, + action: failedAction(actionId, failure.finding), + expected, + observed: observed(failure.finding, evidence.phase), + context: observationContext(failure), + consoleErrors: [...new Set((feature.consoleErrors ?? []) + .map(sanitiseConsoleError).filter(Boolean))].slice(0, 3), + contract: false, + }); + } + } + } + + // Contract failures are separate because the interface name is itself the public + // requirement here. Behavioral failures above must never expose one. + const lintPath = join(resultsDir, ARTIFACT_FILE.contractLint); + if (existsSync(lintPath)) { + const lint = readArtifactPayload(lintPath, { expectedKind: 'contract_lint' }); + for (const result of (lint.results ?? []).filter(item => item.status === 'FAIL' + && (!selectedControls || selectedControls.has(item.id)))) { + bugs.push({ + area: 'Application interface', + actor: null, + action: null, + expected: `A visible element for "${(result.detail ?? '').split('expected: ').pop()}" must use the "${result.id}" application interface`, + observed: sanitiseDiagnostic(result.detail + ?? `no visible element with id="${result.id}" was found after a clean reset`, 500), + consoleErrors: [], contract: true, + }); + } + } + + const bundlePath = join(resultsDir, ARTIFACT_FILE.gradeBundle); + if (existsSync(bundlePath)) { + const bundle = readArtifactPayload(bundlePath, { expectedKind: 'grade_bundle' }); + if (bundle.outcome?.kind === 'app_failure' && bundle.outcome.reason) { + const expectedByPhase: Record = { + 'database-provenance': `The app must use the ${bundle.backend} database and connection supplied for this run.`, + 'application-layout': 'The app must use a project layout that can be built, started, and reset repeatedly.', + 'application-restart': `The app must provide ${CODING_CONTAINER_START_SCRIPT}. From clean source, it must install dependencies, build, and start the complete application without changing source files.`, + }; + const expected = expectedByPhase[bundle.outcome.phase ?? ''] + ?? 'The app must start successfully in the supplied environment.'; + bugs.unshift({ + area: 'Application setup', + actor: null, + action: null, + expected, + observed: sanitiseDiagnostic(bundle.outcome.reason, 500), + consoleErrors: [], + contract: false, + }); + } + } + + if (bugs.length === 0) { + console.log('No failures — no bug report written.'); + return 3; + } + + const behavioral = bugs.filter(bug => !bug.contract); + const contractFailures = bugs.filter(bug => bug.contract); + const lines = args.regressionContext ? [] : [ + '# Bug Report', + '', + 'The application has these problems after a clean database reset and a fresh', + 'restart. Fix the behavior, then redeploy.', + 'Do not change behavior that is already correct. A result from existing local', + 'state does not replace the clean result below.', + '', + ]; + + if (!args.regressionContext && args.history.length) { + lines.push('## Earlier work', ''); + lines.push('Use the current source as the starting point. Preserve earlier fixes while', + 'addressing the remaining problems below.', ''); + } + + if (behavioral.length) { + lines.push('## Behavior', ''); + behavioral.forEach((bug, index) => { + lines.push(`### Bug ${index + 1}: ${bug.area}`, ''); + if (bug.actor) lines.push(`**Actor/session:** ${bug.actor}`, ''); + if (bug.action) lines.push(`**Failed action:** ${bug.action}`, ''); + if (bug.expected) lines.push(`**Expected:** ${bug.expected}`, ''); + lines.push(`**Actual:** ${bug.observed}`, ''); + if (bug.context?.length) lines.push(`**Observed context:** ${bug.context.join(' ')}`, ''); + if (bug.consoleErrors.length) { + lines.push('**Console or network errors:**', ''); + bug.consoleErrors.forEach(error => lines.push(`- \`${error}\``)); + lines.push(''); + } + }); + } + + if (contractFailures.length) { + lines.push('## Application interface', ''); + lines.push('These required elements were not available in the clean application state:', ''); + contractFailures.forEach(bug => { + lines.push(`- **Expected:** ${bug.expected}`); + lines.push(` **Actual:** ${bug.observed}`); + }); + lines.push(''); + } + + if (args.priorRegression) lines.push(...priorRegressionSection(args.priorRegression)); + + const reportText = assertAgentVisibleText(lines.join('\n')); + // The agent owns this directory: replace whatever is there rather than write through a link. + rmSync(args.out, { force: true }); + writeFileSync(args.out, reportText, { flag: 'wx' }); + if (args.archive) { + mkdirSync(dirname(args.archive), { recursive: true }); + writeFileSync(args.archive, reportText); + } + console.log(`Wrote ${bugs.length} bug(s) to ${args.out}`); + return 0; +} + +function main(): void { + try { + process.exitCode = createBugReport(parseReportBugsArgs(process.argv)); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 2; + } +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) main(); diff --git a/tools/stack-bench/commands/reset-backend.ts b/tools/stack-bench/commands/reset-backend.ts new file mode 100644 index 00000000000..1e08c6da017 --- /dev/null +++ b/tools/stack-bench/commands/reset-backend.ts @@ -0,0 +1,24 @@ +#!/usr/bin/env node + +import { GENERATED_APP_LAYOUT_EXIT_CODE, resetBackend } from '../src/stacks/backend-reset.js'; +import { GeneratedAppLayoutError } from '../src/runtime/spacetime-layout.js'; +import { redactCredentials } from '../src/evidence/diagnostic-sanitizer.js'; + +const [backend, app] = process.argv.slice(2); +if (!backend || !app) throw new Error('usage: node dist/commands/reset-backend.js '); + +Promise.resolve().then(() => resetBackend({ backend, app })).then(result => { + console.log(result); +}).catch(error => { + if (error instanceof GeneratedAppLayoutError || error?.code === 'generated_app_layout') { + console.error(`GENERATED_APP_LAYOUT: ${error.message}`); + process.exitCode = GENERATED_APP_LAYOUT_EXIT_CODE; + return; + } + const childOutput = [error?.stderr, error?.stdout] + .filter(value => value !== undefined && value !== null && String(value).trim()) + .map(value => String(value).trim()).join('\n'); + if (childOutput) console.error(redactCredentials(childOutput).slice(-2000)); + console.error(redactCredentials(error.stack ?? error.message)); + process.exitCode = 1; +}); diff --git a/tools/stack-bench/commands/run-suite.ts b/tools/stack-bench/commands/run-suite.ts new file mode 100644 index 00000000000..0b935363a78 --- /dev/null +++ b/tools/stack-bench/commands/run-suite.ts @@ -0,0 +1,1261 @@ +#!/usr/bin/env node +import { privateGradingDirectory } from '../src/evidence/repair-evidence.js'; + +import { execFile, execFileSync } from 'node:child_process'; +import type { ExecFileException, ExecFileSyncOptionsWithStringEncoding } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { performance } from 'node:perf_hooks'; +import { measurePhase, type PhaseTiming } from '../src/evidence/phase-timing.js'; +import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, rmSync, cpSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { parseArgs as parseNodeArgs, promisify } from 'node:util'; +import { chromium } from 'playwright'; +import { attemptBrowserLaunchOptions } from '../container/browser-pipe.js'; +import type { Browser, BrowserServer } from 'playwright'; +import { Actor } from '../grader/grade.js'; +import { ACTION_REGISTRY } from '../src/actions/action-catalog.js'; +import { ActionApplicationFailure, executeAction } from '../src/actions/action-contract.js'; +import { runApplicationNavigation } from '../src/actions/browser-navigation.js'; +import { stableElementSelector } from '../src/actions/element-selector.js'; +import { dbName, loadTrack, suitesFor, DEFAULT_TRACK } from '../src/composition/tracks.js'; +import { controlAppServer, parseRuntimeControlSpec } + from '../src/runtime/backend-control.js'; +import type { RuntimeControlSpec } from '../src/runtime/backend-control.js'; +import { ARTIFACT_FILE, readArtifactPayload, recipeArtifactIdentities, writeArtifact } + from '../src/evidence/artifacts.js'; +import { bundleRecipeRelease, resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { createBoundRecipeTaskRequest, resolveBoundRecipeTaskRequest } from '../src/composition/recipe-selection.js'; +import { contractInterfaceNames } from '../src/composition/agent-visible-contract.js'; +import { resolveCalibrationForRelease } from '../src/composition/calibration-compiler.js'; +import { criterionEvidence, evidencePassed } from '../src/evidence/check-evidence.js'; +import { renderEvidenceConsoleLine } from '../src/evidence/evidence-presentation.js'; +import { STACK_ADAPTER_REGISTRY } from '../src/stacks/stack-adapters.js'; +import { aggregatePackRuntime, exceededPackBudgets } from '../src/composition/pack-runtime.js'; +import { hashAppSource } from '../src/runtime/source-snapshot.js'; +import { GENERATED_APP_LAYOUT_EXIT_CODE } from '../src/stacks/backend-reset.js'; +import { acquireGradingSlot, readBackendLease } from '../src/runtime/backend-lease.js'; +import { redactCredentials } from '../src/evidence/diagnostic-sanitizer.js'; +import { canonicalDefinitionJson } from '../src/composition/definition-plan.js'; +import { sha256 } from '../src/evidence/provenance.js'; +import { GRADER_SOURCE_TIMEOUT_MS, GRADER_TIME_LIMIT_RESERVE_MS, gradingSourceTimeoutMs } from '../src/runtime/grading-timeout.js'; +import type { BackendLease, BackendLeaseExpectation } from '../src/runtime/backend-lease.js'; +import type { CheckEvidence } from '../src/evidence/check-evidence.js'; +import type { AggregatedPackRuntimeEvidence, PackRuntimeEvidence } from '../src/composition/pack-runtime.js'; +import { isModularRecipeTaskRequest } from '../src/composition/recipe-selection.js'; +import type { BoundRecipeTaskRequestResult, RecipeSelection } from '../src/composition/recipe-selection.js'; +import type { RecipeBinding, RecipeCheck } from '../src/composition/recipe-release.js'; +import type { Track, TrackSuite } from '../src/composition/tracks.js'; + +import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js'; +const RESET = compiledEntrypoint('commands', 'reset-backend.js'); + +type Observation = 'scored' | 'observed'; +type Selection = { + schemaVersion: number; + recipe: { id: string; contentSha256: string }; + requested: RecipeSelection['requested']; + sha256: string; + checks: RecipeCheck[]; + scoredPoints: number; + observedChecks?: Array; + observedPoints?: number; + evaluationSha256?: string; + regressionChecks?: Array; + regressionPoints?: number; + observation?: Observation; +}; +type DeclaredSuite = TrackSuite; +type Failure = Error & { stdout?: string; stderr?: string; status?: number | null; signal?: string | null; + code?: string }; +type FailureDetail = { message?: unknown; stderr?: unknown } | null; +type RecipeTaskArgument = { recipe: { id: string; contentSha256?: string } } & Record; +type RunArguments = { + app: string; + url: string; + backend: string; + label: string; + out: string; + level: string; + reset: boolean; + retryInconclusive: boolean; + media: boolean; + runIndex: number; + track: string; + observation: Observation; + recipe?: string; + recipeTask?: RecipeTaskArgument; + credentialAliases?: unknown; + regressionChecks: string[]; + sourceSha256?: string; + restartSpec?: RuntimeControlSpec; + applicationFailure?: ApplicationFailure; + parentAttemptId?: string; + databaseLease?: BackendLease | null; + browserWsEndpoint?: string; + selection?: Selection | null; + bundleArtifactId: string; +}; +type GradeCriterion = { id: string; stableKey?: string; serverCheck?: string; evidence?: CheckEvidence }; +type GradeFeature = { name: string; criteria: GradeCriterion[]; + cleanupEvidence?: { status?: string; failures: Array<{ stage: string }> } }; +type GradePayload = { total: number; max: number; features: GradeFeature[]; + cleanupEvidence?: { status?: string }; + selection?: { checks?: RecipeCheck[] }; packRuntime?: PackRuntimeEvidence }; +type LintPayload = { + pass: boolean; + counts: { pass: number; fail: number; blocked: number; scenario: number }; +}; +type RuntimeProvenance = { ok: boolean | null; verified: boolean; reason: string }; +type ApplicationProbeResult = { ok: boolean; detail: string | null; timedOut?: true }; +type ResetOutcome = { kind: string; phase: string; appFailures?: string[] }; +type ApplicationFailure = ResetOutcome & { kind: 'app_failure'; reason: string }; +type DatabaseProvenance = { ok: boolean; reason: string; url?: string }; +type GradeLeaseReader = typeof readBackendLease; +type MutationDirectoryEntry = { name: string; isDirectory(): boolean; isFile(): boolean }; +type MutationDirectoryReader = (path: string, options: { withFileTypes: true }) => readonly MutationDirectoryEntry[]; +type ProbeResponse = { ok: boolean; status: number }; +type ApplicationFetch = (url: string, init: { signal: AbortSignal }) => Promise; +type DatabaseProvenanceDefinition = Track['databaseProvenance']; +type DatabaseNameLease = { resources: { database?: string | null } }; +type ProvenanceWrite = { ok: true; marker: string } | { ok: false; marker: null; reason: string }; +type ApplicationFailureSelection = { checks: Array<{ executionId: string; points?: number; stableKey?: string }> }; +type ContractLintArguments = Pick; +type BundleSelection = Selection & { attemptedChecks: string[]; reportedChecks: string[]; + notRun: Array<{ stableKey: string; reason: string }> }; +type Bundle = { + definitionSchemaVersion: number; + recipeRelease: ReturnType; + calibration: { id: string; contentSha256: string } | null; + label: string; track: string; backend: string; url: string; app: string; level: number; + observation: Observation; source?: { sha256: string }; + suites: Record; + suiteRetries?: Record; + totals: Record; + selection: BundleSelection | null; + code?: ReturnType; + error?: string; + outcome?: { kind: string; phase: string; reason?: string; appFailures?: string[] }; + provenance?: DatabaseProvenance & { runtime?: RuntimeProvenance }; + packRuntime?: AggregatedPackRuntimeEvidence; + phaseTimings: PhaseTiming[]; +}; + +const isRecord = (value: unknown): value is Record => + value !== null && typeof value === 'object' && !Array.isArray(value); + +const parseObservation = (value: string): Observation => { + if (value === 'scored' || value === 'observed') return value; + throw new Error('--observation must be scored or observed'); +}; + +export function suitesForRecipe(track: Track, binding: RecipeBinding): DeclaredSuite[] { + if (!binding?.execution?.length) throw new Error('recipe has no typed execution plan'); + return binding.execution.map(entry => ({ + id: entry.id, + spec: resolve(track.dir, entry.source ?? ''), + ...(entry.ownership.kind === 'inherited' + ? { inherited: true, fromLevel: entry.ownership.fromLevel } + : {}), + })); +} + +export function childFailureDetail(failure: FailureDetail = null, stdout = '', limit = 600): string { + const processOutput = [failure?.stderr, stdout] + .filter(value => value !== undefined && value !== null && String(value).trim()) + .join('\n').trim(); + const diagnostic = processOutput || String(failure?.message ?? '').trim(); + const lines = diagnostic.split(/\r?\n/).map(line => line.trim()).filter(Boolean); + if (!lines.length) return ''; + const punctuationOnly = (line: string) => + [...line].every(character => '[]{},'.includes(character)); + const noise = (line: string) => line.startsWith('at ') || /^Node\.js v/.test(line) + || /^node:internal\//.test(line) || /^\^+$/.test(line) || punctuationOnly(line); + const cause = lines.find(line => !noise(line) && /(?:error|failed|timeout|closed|econn|killed)/i.test(line)) + ?? lines.find(line => !noise(line)) ?? lines[0]; + const selected = [cause, ...lines.slice(-4)].filter((line, index, all) => all.indexOf(line) === index); + return selected.join(' | ').slice(0, limit); +} + +export function resetFailureOutcome(error: unknown): ResetOutcome { + const failure = isRecord(error) ? error : {}; + return failure.status === GENERATED_APP_LAYOUT_EXIT_CODE + ? { kind: 'app_failure', phase: 'application-layout', + appFailures: ['application-layout'] } + : failure.code === 'generated_app_not_restartable' + ? { kind: 'app_failure', phase: 'application-restart', + appFailures: ['application-restart'] } + : { kind: 'harness_failure', phase: 'database-reset' }; +} + +// Passes measured before an app abort keep their points; inherited regression guards stay unscored. +export function applicationFailureTotals(selection: ApplicationFailureSelection | null | undefined, + declaredSuites: Array>, + passed: ReadonlySet): Record { + if (!selection?.checks?.length) return {}; + const inherited = new Set(declaredSuites.filter(suite => suite.inherited).map(suite => suite.id)); + const currentMax = selection.checks.filter(check => !inherited.has(check.executionId)) + .reduce((total, check) => total + Number(check.points ?? 0), 0); + const regressionMax = selection.checks.filter(check => inherited.has(check.executionId)) + .reduce((total, check) => total + Number(check.points ?? 0), 0); + const score = selection.checks.filter(check => !inherited.has(check.executionId) + && check.stableKey !== undefined && passed.has(check.stableKey)) + .reduce((total, check) => total + Number(check.points ?? 0), 0); + return { score, max: currentMax, dirty: false, contractPass: null, + regression: regressionMax ? { score: 0, max: regressionMax } : null }; +} + +export function clearPreviousGradeOutputs(output: string): void { + const generated = existsSync(output) ? readdirSync(output).filter(name => + /^grading-.+\.json$/.test(name) || /^grader-.+\.(?:stdout|stderr)\.log$/.test(name)) : []; + for (const name of [ARTIFACT_FILE.gradeBundle, ARTIFACT_FILE.contractLint, + ARTIFACT_FILE.actions, 'media', 'failure-media', + 'database-provenance', 'application-start.log', 'suite-retries', ...generated]) { + rmSync(join(output, name), { recursive: true, force: true }); + } +} + +function recordGraderChildResult(output: string, suiteId: string, + result: { stdout?: unknown; stderr?: unknown; failure?: Error | null }) { + const stdout = redactCredentials(String(result.stdout ?? '')); + const stderr = redactCredentials(String(result.stderr ?? '')); + const safeId = String(suiteId).replace(/[^A-Za-z0-9._-]/g, '_'); + const stdoutName = `grader-${safeId}.stdout.log`; + const stderrName = `grader-${safeId}.stderr.log`; + writeFileSync(join(output, stdoutName), stdout); + writeFileSync(join(output, stderrName), stderr); + const failure = result.failure ?? null; + if (failure) Object.assign(failure, { stdout, stderr }); + return { stdout, stderr, failure, stdoutName, stderrName }; +} + +const execFileAsync = promisify(execFile); + +export async function runGraderChild(argv: string[], output: string, suiteId: string, + timeout = COMMAND_TIMEOUT_MS) { + try { + const result = await execFileAsync(process.execPath, argv, { encoding: 'utf8', cwd: ROOT, + timeout, maxBuffer: 64 * 1024 * 1024 }); + return recordGraderChildResult(output, suiteId, result); + } catch (error) { + const failure = error instanceof Error ? error : new Error(String(error)); + const processFailure = error as ExecFileException & { stdout?: unknown; stderr?: unknown }; + return recordGraderChildResult(output, suiteId, { + stdout: processFailure.stdout, stderr: processFailure.stderr, failure, + }); + } +} + +function gradeLeaseInput(backend: string, env: NodeJS.ProcessEnv): { path: string; + expected: BackendLeaseExpectation } | null { + // A stack that can prove the application used its database grades against its run lease. + if (!('proveUse' in STACK_ADAPTER_REGISTRY.get(backend).database)) return null; + const path = String(env.STACK_BENCH_LEASE ?? '').trim(); + const token = String(env.STACK_BENCH_LEASE_TOKEN ?? '').trim(); + if (!path && !token) return null; + if (!path || !token) throw new Error('database grading requires both lease path and lease token'); + return { path, expected: { token, backend, active: true } }; +} + +export function databaseLeaseForGrading(backend: string, env = process.env, { + readLease = readBackendLease, +}: { readLease?: GradeLeaseReader } = {}) { + const input = gradeLeaseInput(backend, env); + if (!input) return null; + const lease = readLease(input.path, input.expected); + if (backend === 'spacetime') { + if (!lease.resources.module || !lease.resources.serverUri) { + throw new Error('active spacetime lease has no complete module target'); + } + return lease; + } + const container = String(lease.resources?.container?.name ?? '').trim(); + const containerId = String(lease.resources?.container?.id ?? '').trim(); + if (!container || !containerId) { + throw new Error(`active ${backend} lease has no complete database container identity`); + } + return lease; +} + +export function databaseNameForGrading(track: Pick, runIndex: number, + lease: DatabaseNameLease | null = null): string { + if (!lease) return dbName(track, runIndex); + const database = String(lease.resources?.database ?? '').trim(); + if (!database) throw new Error('active database lease has no database name'); + return database; +} + +function parseArgs(argv: string[]): RunArguments { + const { values } = parseNodeArgs({ args: argv.slice(2), options: { + app: { type: 'string' }, url: { type: 'string' }, backend: { type: 'string' }, + label: { type: 'string' }, out: { type: 'string' }, level: { type: 'string' }, + recipe: { type: 'string' }, 'recipe-task-json': { type: 'string' }, + 'credential-aliases-json': { type: 'string' }, 'regression-checks-json': { type: 'string' }, + observation: { type: 'string' }, 'source-sha256': { type: 'string' }, + 'no-media': { type: 'boolean' }, track: { type: 'string' }, + 'restart-spec': { type: 'string' }, 'application-failure-json': { type: 'string' }, + 'run-index': { type: 'string' }, 'no-reset': { type: 'boolean' }, + 'retry-inconclusive': { type: 'boolean' }, + 'parent-attempt-id': { type: 'string' }, + } }); + const a: RunArguments = { app: values.app ?? '', url: values.url ?? '', + backend: values.backend ?? '', label: values.label ?? '', out: values.out ?? '', + level: values.level ?? '1', reset: !(values['no-reset'] ?? false), + retryInconclusive: values['retry-inconclusive'] ?? false, + media: !(values['no-media'] ?? false), runIndex: Number(values['run-index'] ?? 0), + track: values.track ?? DEFAULT_TRACK, + observation: parseObservation(values.observation ?? 'scored'), + recipe: values.recipe, + recipeTask: values['recipe-task-json'] === undefined ? undefined : JSON.parse(values['recipe-task-json']), + credentialAliases: values['credential-aliases-json'] === undefined + ? undefined : JSON.parse(values['credential-aliases-json']), + regressionChecks: values['regression-checks-json'] === undefined + ? [] : JSON.parse(values['regression-checks-json']), + sourceSha256: values['source-sha256'], + restartSpec: values['restart-spec'] === undefined + ? undefined : parseRuntimeControlSpec(JSON.parse(values['restart-spec'])), + applicationFailure: values['application-failure-json'] === undefined + ? undefined : JSON.parse(values['application-failure-json']), + parentAttemptId: values['parent-attempt-id'], bundleArtifactId: '' }; + if (!a.app || !a.url || !a.backend || !a.label) { + console.error('Usage: node dist/commands/run-suite.js --app --url --backend --label [--out ] [--media] [--no-reset]'); + process.exit(2); + } + if (!['scored', 'observed'].includes(a.observation)) { + throw new Error('--observation must be scored or observed'); + } + if (a.observation === 'observed' && !/^[a-f0-9]{64}$/.test(a.sourceSha256 ?? '')) { + throw new Error('observed specifications require --source-sha256'); + } + if (a.sourceSha256 !== undefined && !/^[a-f0-9]{64}$/.test(a.sourceSha256)) { + throw new Error('--source-sha256 must be a SHA-256 digest'); + } + if (a.retryInconclusive && (!a.sourceSha256 + || (!a.reset && STACK_ADAPTER_REGISTRY.get(a.backend).runPolicy.resetEnabled))) { + throw new Error('suite recovery requires a frozen --source-sha256 and reset for stateful backends'); + } + if (a.applicationFailure && (a.applicationFailure.kind !== 'app_failure' + || typeof a.applicationFailure.phase !== 'string' || !a.applicationFailure.phase + || typeof a.applicationFailure.reason !== 'string' || !a.applicationFailure.reason)) { + throw new Error('--application-failure-json must describe an application failure'); + } + a.out = privateGradingDirectory(a.app, a.out); + if (!Array.isArray(a.regressionChecks) + || a.regressionChecks.some(key => typeof key !== 'string' || !key)) { + throw new Error('--regression-checks-json must contain stable check keys'); + } + return a; +} + +export function selectObservationScope(selectedTask: BoundRecipeTaskRequestResult | null, + observation: Observation = 'scored'): Selection | null { + if (observation === 'scored') return selectedTask?.selection ?? null; + if (observation !== 'observed') throw new Error(`unknown observation scope ${observation}`); + if (!selectedTask || !isModularRecipeTaskRequest(selectedTask)) { + throw new Error('observed specifications require a modular schema-3 task request'); + } + const selection = selectedTask.selection; + if (!selection.observedChecks.length) throw new Error('observed specification scope is empty'); + return { + ...selection, + observation: 'observed', + checks: selection.observedChecks, + scoredPoints: 0, + observedPoints: selection.observedChecks.reduce((total, check) => total + check.points, 0), + }; +} + +export function attachRegressionScope(selection: Selection | null, recipeBinding: RecipeBinding | null, + declaredSuites: DeclaredSuite[], stableKeys: string[] = []): Selection | null { + if (!stableKeys.length) return selection; + if (!selection || !recipeBinding?.release?.checkCatalog) { + throw new Error('regression checks require a recipe-bound scored selection'); + } + const uniqueKeys = [...new Set(stableKeys)]; + if (uniqueKeys.length !== stableKeys.length) throw new Error('regression checks contain duplicates'); + const currentKeys = new Set(selection.checks.map(check => check.stableKey)); + const catalog = new Map(recipeBinding.release.checkCatalog + .map(check => [check.stableKey, check])); + const inheritedSuites = new Set(declaredSuites.filter(suite => suite.inherited) + .map(suite => suite.id)); + const regressionChecks = uniqueKeys.map(key => { + if (currentKeys.has(key)) throw new Error(`regression check ${key} is already in the current score`); + const check = catalog.get(key); + if (!check) throw new Error(`regression check ${key} is absent from the cumulative recipe`); + if (!inheritedSuites.has(check.executionId)) { + throw new Error(`regression check ${key} does not belong to an inherited execution`); + } + return { ...check, treatment: check.treatment ?? 'regression' }; + }); + const evaluationDocument = { schemaVersion: 1, selectionSha256: selection.sha256, + regressionChecks: uniqueKeys.slice().sort() }; + return { + ...selection, + checks: [...selection.checks, ...regressionChecks], + regressionChecks: regressionChecks.map(check => ({ ...check, treatment: check.treatment ?? 'regression' })), + regressionPoints: regressionChecks.reduce((total, check) => total + check.points, 0), + evaluationSha256: sha256(Buffer.from(canonicalDefinitionJson(evaluationDocument))), + }; +} + +const COMMAND_TIMEOUT_MS = GRADER_SOURCE_TIMEOUT_MS; +const run = (cmd: string, args: readonly string[], opts: Omit = {}): string => + execFileSync(cmd, args, { + encoding: 'utf8', stdio: 'pipe', cwd: ROOT, timeout: COMMAND_TIMEOUT_MS, ...opts, + }); + +export async function verifyApplicationProbe(url: string, { + fetchImpl = fetch, timeoutMs = 5000, +}: { fetchImpl?: ApplicationFetch; timeoutMs?: number } = {}): Promise { + let response; + try { + response = await fetchImpl(url, { signal: AbortSignal.timeout(timeoutMs) }); + } catch (error) { + // A probe timeout measures nothing; only a refusal or HTTP error is app evidence. + return { ok: false, ...(error instanceof Error && error.name === 'TimeoutError' ? { timedOut: true } : {}), + detail: `application did not respond: ${error instanceof Error ? error.message : String(error)}` }; + } + if (!response.ok) { + return { ok: false, detail: `application returned HTTP ${response.status}` }; + } + return { ok: true, detail: null }; +} + +export async function waitForApplicationProbe(url: string, { + attempts = 9, intervalMs = 250, probeTimeoutMs = 1000, + probe = verifyApplicationProbe, sleepImpl = sleep, +}: { attempts?: number; intervalMs?: number; probeTimeoutMs?: number; + probe?: typeof verifyApplicationProbe; + sleepImpl?: (ms: number) => Promise } = {}): Promise { + if (!Number.isInteger(attempts) || attempts < 1) { + throw new Error('application probe attempts must be a positive integer'); + } + let result = null; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + result = await probe(url, { timeoutMs: probeTimeoutMs }); + if (result.ok || attempt === attempts) return result; + await sleepImpl(intervalMs); + } + return result ?? { ok: false, detail: 'application readiness probe did not run' }; +} + +// Confirm the app uses the database leased to this run. +export function checkDatabaseProvenance(args: Pick): DatabaseProvenance { + const adapter = STACK_ADAPTER_REGISTRY.get(args.backend); + // Source text cannot prove which database the running app uses. Leased runs + // prove it with an application write, including container-local endpoints. + if (args.databaseLease && 'proveUse' in adapter.database) { + return { ok: true, reason: 'leased database requires runtime marker verification' }; + } + const expected = adapter.ports.allocations().db; + if (!expected) return { ok: true, reason: 'no external database for this backend' }; + // Neutral guidance does not prescribe project layout. Search the app for the + // connection string instead of assuming it is in server/.env. + const urls: string[] = []; + let usesLeasedEnvironment = false; + const walk = (dir: string): void => { + if (!existsSync(dir)) return; + for (const e of readdirSync(dir, { withFileTypes: true })) { + if (/^(node_modules|dist|\.vite|\.git|module_bindings)$/.test(e.name)) continue; + const p = join(dir, e.name); + if (e.isDirectory()) { walk(p); continue; } + if (!/\.(env|ts|tsx|js|mjs|json|yaml|yml)$|^\.env/.test(e.name)) continue; + try { + const text = readFileSync(p, 'utf8'); + if (/process\.env(?:\.DATABASE_URL|\[['"]DATABASE_URL['"]\])/.test(text)) { + usesLeasedEnvironment = true; + } + urls.push(...adapter.agent.findDatabaseUrls({ text })); + } catch { /* unreadable file proves nothing */ } + } + }; + walk(args.app); + if (usesLeasedEnvironment) { + return { ok: true, url: 'process.env.DATABASE_URL', + reason: 'app reads the database URL supplied by its authenticated backend lease' }; + } + if (!urls.length) return { ok: false, + reason: 'app neither reads process.env.DATABASE_URL nor contains a database connection string' }; + const matchesExpectedPort = (value: string): boolean => { + try { return Number(new URL(value).port) === Number(expected); } + catch { return false; } + }; + const ok = urls.some(matchesExpectedPort); + return { ok, url: urls[0], + reason: ok ? 'ok' : `app targets ${urls[0]} but the database supplied for this run is on port ${expected}` }; +} + +export async function writeApplicationDatabaseMarker( + args: Pick, + definition: DatabaseProvenanceDefinition, + { browser: suppliedBrowser }: { browser?: Browser } = {}, +): Promise { + if (!definition) throw new Error('track does not define runtime database provenance'); + const marker = `sb${randomUUID().replaceAll('-', '').slice(0, 16)}`; + const browser = suppliedBrowser ?? (args.browserWsEndpoint + ? await chromium.connect(args.browserWsEndpoint) + : await chromium.launch({ headless: true, ...attemptBrowserLaunchOptions() })); + try { + const context = await browser.newContext(); + try { + const page = await context.newPage(); + page.setDefaultTimeout(8000); + const actor = new Actor('database-provenance', page, context); + await actor.ready; + await runApplicationNavigation(() => page.goto(args.url, { waitUntil: 'domcontentloaded', timeout: 20000 }), page); + const evidence = await executeAction(ACTION_REGISTRY, definition.browserAction, + { do: definition.browserAction, actor: actor.name, name: marker, exact: true }, { + capabilities: { + actors: new Map([[actor.name, actor]]), + 'browser-interaction': { defaultWithin: 8000, + roomName: (name: string) => name, scopedUser: (name: string) => name, + testId: stableElementSelector, + sleep: (ms: number, signal: AbortSignal) => sleep(ms, undefined, { signal }), + }, + }, + onAbort: () => context.close(), + }); + if (evidence.status === 'passed') return { ok: true, marker }; + if (evidence.status === 'failed') return { ok: false, marker: null, + reason: evidence.summary ?? 'application signup failed during database provenance' }; + throw new Error(`database provenance signup ${evidence.status}: ${evidence.summary}`); + } finally { await context.close(); } + } catch (error) { + if (error instanceof ActionApplicationFailure) return { ok: false, marker: null, reason: error.message }; + throw error; + } finally { if (!suppliedBrowser) await browser.close(); } +} + +export function databaseProvenanceFailure(error: unknown): { kind: string; phase: string; reason: string } { + return { kind: 'harness_failure', phase: 'database-provenance', + reason: `runtime database provenance failed: ${error instanceof Error ? error.message : String(error)}` }; +} + +// A successful browser/provider flow is not evidence of an application database write. +export async function verifyApplicationDatabaseMarker( + args: Pick, + definition: DatabaseProvenanceDefinition, + { write = writeApplicationDatabaseMarker, read = checkRuntimeDatabaseProvenance } = {}, +): Promise<{ write: ProvenanceWrite; runtime: RuntimeProvenance | null }> { + const result = await write(args, definition); + return { write: result, runtime: result.ok ? read(args, result.marker) : null }; +} + +export function checkRuntimeDatabaseProvenance(args: Pick, + marker: string | null = null): RuntimeProvenance { + const adapter = STACK_ADAPTER_REGISTRY.get(args.backend); + if (!('proveUse' in adapter.database)) { + return { ok: null, verified: false, + reason: 'exact runtime database marker proof is not implemented for this stack' }; + } + if (!args.databaseLease) { + return { ok: null, verified: false, + reason: 'standalone grading has no authenticated database lease' }; + } + if (typeof marker !== 'string' || !marker) { + return { ok: null, verified: false, + reason: 'the application action did not produce a database marker' }; + } + return adapter.database.proveUse({ lease: args.databaseLease, marker }); +} + +function isGradePayload(value: GradePayload | LintPayload | null | undefined): value is GradePayload { + return value !== null && value !== undefined && 'total' in value && 'max' in value; +} + +// Report the application size and direct runtime dependency count. +export function codeMetrics(args: Pick): { serverLoc: number; serverFiles: number; + totalLoc: number; totalFiles: number; runtimeDeps: number } { + // Minimal-guidance apps may place server code outside the conventional directory. + const adapter = STACK_ADAPTER_REGISTRY.get(args.backend); + const conventional = adapter.agent.serverDirectory; + const SERVER_DIR = existsSync(join(args.app, conventional)) ? conventional : '.'; + const walk = (dir: string, out: string[] = []): string[] => { + if (!existsSync(dir)) return out; + for (const e of readdirSync(dir, { withFileTypes: true })) { + if (/^(node_modules|dist|\.vite|module_bindings|drizzle)$/.test(e.name)) continue; + const p = join(dir, e.name); + if (e.isDirectory()) walk(p, out); + // Count supported source files; agent-planted links and FIFOs are not read. + else if (e.isFile() && /\.(ts|tsx|js|jsx|mjs|cjs)$/.test(e.name)) out.push(p); + } + return out; + }; + const count = (files: string[]): number => files.reduce((n, f) => n + readFileSync(f, 'utf8').split('\n').length, 0); + // With no conventional server directory, "server" is everything that is not + // the client — otherwise the fallback counts the client twice and serverLoc + // equals totalLoc, which reads as a much larger backend than was written. + const allFiles = walk(args.app); + const serverFiles = SERVER_DIR === '.' + ? allFiles.filter(f => !/[\\/]client[\\/]/.test(f)) + : walk(join(args.app, SERVER_DIR)); + + let deps = 0; + const packageFiles = new Set([ + resolve(args.app, 'package.json'), + resolve(args.app, SERVER_DIR, 'package.json'), + resolve(args.app, 'client/package.json'), + ]); + for (const p of packageFiles) { + if (!existsSync(p)) continue; + try { deps += Object.keys(JSON.parse(readFileSync(p, 'utf8')).dependencies ?? {}).length; } catch { /* ignore */ } + } + + return { + serverLoc: count(serverFiles), serverFiles: serverFiles.length, + totalLoc: count(allFiles), totalFiles: allFiles.length, + runtimeDeps: deps, + }; +} + +export function findMutationBackups(app: string, { readDir = readdirSync }: + { readDir?: MutationDirectoryReader } = {}): string[] { + const backups: string[] = []; + const walk = (dir: string): void => { + let entries; + try { + entries = readDir(dir, { withFileTypes: true }); + } catch (error) { + // Vite atomically replaces transient dependency directories while the + // app runs. They are not source and may vanish between parent and child + // reads; a missing directory cannot contain a mutation backup. + if (isRecord(error) && error.code === 'ENOENT') return; + throw error; + } + for (const entry of entries) { + if (/^(node_modules|dist|\.vite|\.git|module_bindings)$/.test(entry.name)) continue; + const path = join(dir, entry.name); + if (entry.isDirectory()) walk(path); + else if (entry.isFile() && entry.name.endsWith('.mutation-backup')) backups.push(path); + } + }; + walk(app); + return backups; +} + +function resetDatabase(args: RunArguments): { ok: boolean; detail: string | null; + outcome: { kind: string; phase: string; appFailures?: string[] } | null } { + process.stdout.write(' reset database ... '); + try { + run(process.execPath, [RESET, args.backend, args.app]); + console.log('ok'); + } catch (err) { + console.log('FAILED'); + const failure: Failure = err instanceof Error ? err : new Error(String(err)); + const detail = childFailureDetail(failure, failure.stdout); + console.log(` ${detail}`); + return { ok: false, detail, outcome: resetFailureOutcome(failure) }; + } + return { ok: true, detail: null, outcome: null }; +} + +export function contractLintArgv(args: ContractLintArguments, + selectedTask: BoundRecipeTaskRequestResult | null = null): string[] { + const interfaces = selectedTask ? contractInterfaceNames(selectedTask.task.contractText) : []; + const out = join(args.out, ARTIFACT_FILE.contractLint); + return [compiledEntrypoint('linter', 'lint.js'), '--url', args.url, '--level', args.level, + '--track', args.track, '--label', args.label, '--out', out, + '--parent-attempt-id', args.bundleArtifactId, + ...(args.credentialAliases + ? ['--credential-aliases-json', JSON.stringify(args.credentialAliases)] : []), + ...(selectedTask ? ['--selected-hooks'] : []), + ...interfaces.flatMap(id => ['--hook', id])]; +} + +function lint(args: RunArguments, selectedTask: BoundRecipeTaskRequestResult | null = null): LintPayload | null { + process.stdout.write(' contract lint ... '); + const out = join(args.out, ARTIFACT_FILE.contractLint); + rmSync(out, { force: true }); + let failure: unknown = null; + try { + run('node', contractLintArgv(args, selectedTask)); + } catch (error) { failure = error; /* hook failures still write a report */ } + if (!existsSync(out)) { + const output = failure && typeof failure === 'object' && 'stdout' in failure + ? String(failure.stdout ?? '') : undefined; + const detail = failure instanceof Error + ? childFailureDetail(failure, output) : null; + throw new Error(`contract lint produced no report${detail ? `: ${detail}` : ''}`); + } + const r = readArtifactPayload(out, { expectedKind: 'contract_lint' }); + console.log(r.pass + ? r.counts.pass > 0 + ? `PASS (${r.counts.pass} interfaces)` + : r.counts.scenario > 0 + ? `DEFERRED (${r.counts.scenario} interfaces checked during feature grading)` + : 'NO STANDALONE INTERFACES SELECTED' + : `FAIL (${r.counts.fail} failed, ${r.counts.blocked} blocked)`); + return r; +} + +export function suiteMayRetry(grade: GradePayload): boolean { + if (grade.cleanupEvidence?.status === 'harness_failure') return false; + if (grade.features.some(feature => feature.cleanupEvidence?.status === 'harness_failure' + || feature.cleanupEvidence?.failures.length)) return false; + const evidence = grade.features.flatMap(feature => feature.criteria.map(criterion => criterion.evidence)); + return evidence.some(value => value?.status === 'inconclusive' && value.retryable) + && evidence.every(value => value?.status === 'passed' + || (value?.status === 'inconclusive' && value.retryable)); +} + +async function gradeSuite(args: RunArguments, suite: DeclaredSuite, track: Track, + recipeBinding: RecipeBinding | null, selectedTask: BoundRecipeTaskRequestResult | null, + bundleArtifactId: string, selectedChecks: RecipeCheck[] = [], + { recordSelection = true, captureMedia = true, outputDirectory = args.out }: { + recordSelection?: boolean; captureMedia?: boolean; outputDirectory?: string; + } = {}): Promise { + process.stdout.write(` ${suite.id.padEnd(10)} ... `); + mkdirSync(outputDirectory, { recursive: true }); + const out = join(outputDirectory, `grading-${suite.id}.json`); + rmSync(out, { force: true }); + const argv = [compiledEntrypoint('grader', 'grade.js'), '--url', args.url, '--level', args.level, + '--label', `${args.label}-${suite.id}${outputDirectory === args.out ? '' : '-retry'}`, '--out', out]; + if (suite.spec) argv.push('--spec', suite.spec); + argv.push('--backend', args.backend, '--track', args.track); + if (recipeBinding) argv.push('--expected-recipe-sha256', recipeBinding.release.contentSha256); + const requestedRecipe = args.recipe ?? (args.recipeTask + ? args.recipeTask.recipe.id : null); + if (requestedRecipe) argv.push('--recipe', requestedRecipe); + if (selectedTask) argv.push('--recipe-task-json', JSON.stringify(selectedTask.request)); + for (const check of selectedChecks) argv.push('--selected-check', check.stableKey); + if (args.credentialAliases) { + argv.push('--credential-aliases-json', JSON.stringify(args.credentialAliases)); + } + if (recordSelection && args.selection?.sha256) { + argv.push('--selection-sha256', args.selection.evaluationSha256 ?? args.selection.sha256); + } + argv.push('--parent-attempt-id', bundleArtifactId); + // The out-of-band write goes straight to this run's database, with no + // app code in the loop; only the harness knows which one that is. + argv.push('--db-name', databaseNameForGrading(track, args.runIndex ?? 0, + args.databaseLease?.resources.database ? args.databaseLease : null)); + if (args.restartSpec) argv.push('--restart-spec', JSON.stringify(args.restartSpec)); + // The systems criteria run scripts the app itself ships (back-office writes), + // so the grader has to know where the app lives. + if (args.app) argv.push('--app', args.app); + if (captureMedia && args.media) argv.push('--media', join(outputDirectory, 'media'), '--trace'); + else if (captureMedia) argv.push('--failure-media', join(outputDirectory, 'failure-media')); + if (args.browserWsEndpoint) argv.push('--browser-ws-endpoint', args.browserWsEndpoint); + // The grader stops starting steps before the process limit, so an overrun is a + // recorded result for the running check instead of a lost source. + const timeoutMs = gradingSourceTimeoutMs(recipeBinding?.plan.packs ?? [], selectedChecks); + argv.push('--time-limit-ms', String(timeoutMs - GRADER_TIME_LIMIT_RESERVE_MS)); + const child = await runGraderChild(argv, outputDirectory, suite.id, timeoutMs); + const { stdout, failure } = child; + if (!existsSync(out)) { + console.log('NO REPORT'); + const detail = childFailureDetail(failure, stdout); + throw new Error(`grader produced no report for ${suite.id}${detail ? `: ${detail}` : ''}; ` + + `full diagnostics: ${child.stdoutName}, ${child.stderrName}`); + } + const r = readArtifactPayload(out, { expectedKind: 'grade' }); + if (failure) { + throw new Error(`grader process did not complete for ${suite.id}: ${childFailureDetail(failure, stdout)}; ` + + `full diagnostics: ${child.stdoutName}, ${child.stderrName}`); + } + if (selectedChecks.length) { + const expected = selectedChecks.map(check => check.stableKey).sort(); + const reported = (r.selection?.checks ?? []).map(check => check.stableKey).sort(); + if (JSON.stringify(reported) !== JSON.stringify(expected)) { + throw new Error(`grader report scope differs from requested suite scope for ${suite.id}`); + } + } + console.log(`${r.total}/${r.max}`); + for (const f of r.features) { + for (const c of f.criteria.filter(c => !evidencePassed(criterionEvidence(c)))) { + console.log(` ${renderEvidenceConsoleLine(criterionEvidence(c), `${f.name} / ${c.id}`, { + includeSummary: false, + })}`); + } + } + // Disclose passes that lack server-side confirmation. + const uiOnly = r.features.flatMap(f => + f.criteria.filter(c => evidencePassed(criterionEvidence(c)) && c.serverCheck === 'unverified') + .map(c => `${f.name}/${c.id}`)); + if (uiOnly.length) { + console.log(` note: ${uiOnly.length} criterion/criteria passed on interface behaviour only`); + for (const u of uiOnly) console.log(` ${u} — server-side check not runnable on this backend`); + } + return r; +} + +export async function closeSuiteBrowser(browser: Pick | null, + bundle: Pick, persist: () => unknown): Promise { + try { await browser?.close(); } + catch (error) { + bundle.error = `grader browser shutdown failed: ${error instanceof Error ? error.message : String(error)}`; + bundle.outcome = { kind: 'harness_failure', phase: 'grading-cleanup', reason: bundle.error }; + persist(); + throw error; + } +} + +export function preserveStartFailure(error: unknown, out: string): void { + if (isRecord(error) && typeof error.startLog === 'string' && error.startLog) { + writeFileSync(join(out, 'application-start.log'), redactCredentials(error.startLog) + '\n'); + } +} + +async function main() { + const startedAt = new Date().toISOString(); + const args = parseArgs(process.argv); + args.databaseLease = databaseLeaseForGrading(args.backend); + const track = loadTrack(args.track); + const recipeBinding = resolveRecipeRelease(track, Number(args.level), args.recipeTask?.recipe ?? args.recipe); + const selectedTask = recipeBinding + ? (args.recipeTask + ? resolveBoundRecipeTaskRequest(recipeBinding, args.recipeTask) + : createBoundRecipeTaskRequest(recipeBinding)) + : null; + let selection = selectObservationScope(selectedTask, args.observation); + if (args.sourceSha256) { + const source = hashAppSource(args.app); + if (source.sha256 !== args.sourceSha256) { + throw new Error('live application source differs from the source selected for grading'); + } + } + const declaredSuites = recipeBinding + ? suitesForRecipe(track, recipeBinding) + : suitesFor(track, Number(args.level)); + if (args.observation === 'scored') { + selection = attachRegressionScope(selection, recipeBinding, declaredSuites, + args.regressionChecks); + } else if (args.regressionChecks.length) { + throw new Error('observed grading cannot include regression checks'); + } + args.selection = selection; + if (selection) { + const suiteIds = new Set(declaredSuites.map(suite => suite.id)); + const unmapped = selection.checks.filter(check => !suiteIds.has(check.executionId)); + if (unmapped.length) { + throw new Error(`selected recipe checks do not map to a declared suite: ${ + unmapped.map(check => check.stableKey).join(', ')}`); + } + } + const calibration = resolveCalibrationForRelease(recipeBinding?.release ?? null, { + trackRoot: track.dir, + stackBenchRoot: ROOT, + alias: `L${args.level}`, + }); + const observationSuffix = args.observation === 'observed' ? '-observed' : ''; + const bundleArtifactId = `${args.parentAttemptId ?? args.label}-grade-bundle-l${args.level}${observationSuffix}`; + args.bundleArtifactId = bundleArtifactId; + mkdirSync(args.out, { recursive: true }); + // Remove all prior grade output before writing cumulative evidence. + clearPreviousGradeOutputs(args.out); + + console.log(`\n=== ${args.label} (${args.backend}) ===`); + console.log(` app: ${args.app}`); + console.log(` url: ${args.url}`); + if (recipeBinding && selection) { + console.log(` recipe: ${recipeBinding.alias} -> ${recipeBinding.release.id} ` + + `(${recipeBinding.release.contentSha256.slice(0, 12)})`); + console.log(args.observation === 'observed' + ? ` scope: ${selection.checks.length} observed check(s), ${selection.observedPoints} observed point(s), 0 score contribution` + : ` scope: ${selection.checks.length} check(s), ${selection.scoredPoints} point(s)`); + if (selection.requested.packs?.length) console.log(` packs: ${selection.requested.packs.join(', ')}`); + if (selection.requested.features?.length) { + console.log(` features: ${selection.requested.features.join(', ')}`); + } + if (selection.requested.checks.length) console.log(` extra checks: ${selection.requested.checks.join(', ')}`); + } + + const bundle: Bundle = { + definitionSchemaVersion: track.schemaVersion, + recipeRelease: bundleRecipeRelease(recipeBinding), + calibration: calibration ? { id: calibration.id, + contentSha256: calibration.contentSha256 } : null, + label: args.label, track: args.track, backend: args.backend, url: args.url, app: args.app, + level: Number(args.level), observation: args.observation, + ...(args.sourceSha256 ? { source: { sha256: args.sourceSha256 } } : {}), + suites: {}, totals: {}, phaseTimings: [], + selection: selection ? { ...selection, attemptedChecks: [], reportedChecks: [], notRun: [] } : null, + }; + const selectedPackIds = new Set(selection?.checks.map(check => check.packId) ?? []); + const selectedPackDefinitions = recipeBinding?.plan.packs + .filter(pack => selectedPackIds.has(pack.id)) ?? []; + const writeBundle = () => { + const writeStarted = performance.now(); + if (args.sourceSha256) { + const current = hashAppSource(args.app); + if (current.sha256 !== args.sourceSha256) { + bundle.error = 'application source changed while grading was in progress'; + bundle.outcome = { kind: 'harness_failure', phase: 'source-provenance', + reason: bundle.error }; + } + } + const result = writeArtifact(join(args.out, ARTIFACT_FILE.gradeBundle), { + kind: 'grade_bundle', + id: bundleArtifactId, + attempt: { id: bundleArtifactId, parentId: args.parentAttemptId ?? null }, + timestamps: { startedAt, completedAt: new Date().toISOString() }, + identities: recipeArtifactIdentities(recipeBinding?.release ?? null, { + calibration: calibration ? { id: calibration.id, + sha256: calibration.contentSha256 } : null, + stackAdapter: { id: args.backend }, + }), + payload: bundle, + }); + console.log(` evidence write and source verification ... ${(performance.now() - writeStarted).toFixed(1)}ms`); + return result; + }; + const recordApplicationAbort = () => { + const passed = new Set(Object.values(bundle.suites ?? {}).flatMap(suite => + (isGradePayload(suite) ? suite.features : []).flatMap(feature => feature.criteria ?? []) + .flatMap(criterion => criterion.evidence?.status === 'passed' && criterion.stableKey ? [criterion.stableKey] : []))); + bundle.totals = applicationFailureTotals(selection, declaredSuites, passed); + }; + const freshenFailureMessage = () => { + const detail = lastResetFailure ? `: ${lastResetFailure}` : ''; + if (lastResetOutcome?.phase !== 'application-readiness') { + return `database reset failed — scores would not be comparable${detail}`; + } + return lastResetOutcome.kind === 'harness_failure' + ? `application server stopped by the grader was not restored${detail}` + : `application did not become ready after database reset${detail}`; + }; + const markRemainingNotRun = (reason: string): void => { + if (!bundle.selection) return; + const accounted = new Set([ + ...bundle.selection.attemptedChecks, + ...bundle.selection.notRun.map(check => check.stableKey), + ]); + bundle.selection.notRun.push(...bundle.selection.checks + .filter(check => !accounted.has(check.stableKey)) + .map(check => ({ stableKey: check.stableKey, reason }))); + }; + + if (args.applicationFailure) { + bundle.error = args.applicationFailure.reason; + bundle.outcome = args.applicationFailure; + recordApplicationAbort(); + markRemainingNotRun(`run aborted: ${bundle.error}`); + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + process.exit(1); + } + + // Reset before each stateful check. + let lastResetFailure: string | null = null; + let lastResetOutcome: ResetOutcome = { kind: 'harness_failure', phase: 'database-reset' }; + // Set when a grader stopped the application server and could not start it + // again. Until the harness starts it, the app cannot be blamed for being + // unreachable. + let applicationLeftStopped = false; + let timingSuite: string | null = null; + const measure = (phase: string, work: () => T | Promise) => + measurePhase(bundle.phaseTimings, phase, timingSuite, work); + const freshen = async () => { + if (!args.reset) return true; + const requiresReseed = STACK_ADAPTER_REGISTRY.get(args.backend).reset.requiresReseed; + const restartSpec = args.restartSpec; + if (track.reseedOnReset && requiresReseed && !restartSpec) { + lastResetFailure = `track ${args.track} requires --restart-spec to initialize the app after reset`; + lastResetOutcome = { kind: 'harness_failure', phase: 'application-reset-control' }; + return false; + } + if (track.reseedOnReset && restartSpec && requiresReseed) { + process.stdout.write(' stop application ... '); + try { + await measure('stop', () => controlAppServer(restartSpec, 'stop')); + applicationLeftStopped = true; + console.log('ok'); + } catch (error) { + const failure: Failure = error instanceof Error ? error : new Error(String(error)); + lastResetFailure = childFailureDetail(failure); + lastResetOutcome = { kind: 'harness_failure', phase: 'application-reset-control' }; + console.log(`FAILED (${lastResetFailure})`); + return false; + } + } + const reset = await measure('reset', () => resetDatabase(args)); + lastResetFailure = reset.detail; + lastResetOutcome = reset.outcome ?? { kind: 'harness_failure', phase: 'database-reset' }; + if (!reset.ok) return false; + // Do not grade until the reset application is reachable. + const waitUntilReady = async () => { + const ready = await measure('readiness', () => waitForApplicationProbe(args.url)); + if (!ready.ok) { + lastResetFailure = ready.detail; + lastResetOutcome = applicationLeftStopped + ? { kind: 'harness_failure', phase: 'application-readiness' } + : ready.timedOut ? { kind: 'inconclusive', phase: 'application-readiness' } + : { kind: 'app_failure', phase: 'application-readiness', + appFailures: ['application-readiness'] }; + console.log(`FAILED (${ready.detail})`); + return false; + } + console.log('ok'); + return true; + }; + if (track.reseedOnReset && restartSpec && requiresReseed) { + process.stdout.write(' restart ... '); + // Judge restart success with the readiness probe. The restart command can + // leave a long-running server process behind, so the command also needs a deadline. + try { + // Do not give a background server an inherited pipe that keeps the + // synchronous restart command open. + await measure('start', () => controlAppServer(restartSpec, 'start')); + applicationLeftStopped = false; + } catch (err) { + preserveStartFailure(err, args.out); + const failure: Failure = err instanceof Error ? err : new Error(String(err)); + lastResetOutcome = resetFailureOutcome(failure); + const detail = ((failure.stderr || '') + (failure.stdout || '') + (failure.message || '')) + .toString().trim().split('\n').slice(-3).join(' | ').slice(0, 300); + lastResetFailure = detail || null; + console.log('FAILED (application did not restart)'); + console.log(` control: ${JSON.stringify(restartSpec)}`); + console.log(` ${detail}`); + return false; + } + return await waitUntilReady(); + } + process.stdout.write(' ready ... '); + return await waitUntilReady(); + }; + + bundle.code = codeMetrics(args); + console.log(` code ... ${bundle.code.serverLoc} server LOC in ${bundle.code.serverFiles} files, ` + + `${bundle.code.totalLoc} total LOC, ${bundle.code.runtimeDeps} runtime deps`); + + // Refuse source left modified by an interrupted mutation run. + const mutated = findMutationBackups(args.app); + if (mutated.length) { + bundle.error = `app still carries mutation backups (${mutated.join(', ')}) — its source is mutated, not the build under test`; + bundle.outcome = { kind: 'harness_failure', phase: 'mutation-cleanup', reason: bundle.error }; + markRemainingNotRun('run aborted because application source is still mutated'); + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + process.exit(1); + } + + const prov = checkDatabaseProvenance(args); + bundle.provenance = prov; + console.log(` database ... ${prov.ok ? prov.reason : `WRONG DATABASE — ${prov.reason}`}`); + if (!prov.ok) { + bundle.error = `app is not using the database supplied for this run: ${prov.reason}`; + bundle.outcome = { kind: 'app_failure', phase: 'database-provenance', reason: bundle.error, + appFailures: ['database-provenance'] }; + recordApplicationAbort(); + markRemainingNotRun('run aborted because database provenance was invalid'); + writeBundle(); + console.log('\nABORTED: results would not describe the benchmark environment.'); + process.exit(1); + } + + // Leased runs share the host's grading slots, taken per suite so a short grading need not wait out a + // long one. The stub backend holds no host resources. + const gradingSlot = () => args.backend === 'stub' || !process.env.STACK_BENCH_LEASE + ? Promise.resolve({ release() {} }) : acquireGradingSlot(); + if (args.observation === 'scored') { + const setupSlot = await gradingSlot(); + if (!(await freshen())) { + bundle.error = freshenFailureMessage(); + bundle.outcome = { ...lastResetOutcome, reason: bundle.error }; + if (bundle.outcome.kind === 'app_failure') recordApplicationAbort(); + markRemainingNotRun(`run aborted: ${bundle.error}`); + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + process.exit(1); + } + let runtime = checkRuntimeDatabaseProvenance(args); + let proofError = null; + let actionFailure: string | null = null; + const proof = track.databaseProvenance; + const supportsRuntimeProof = 'proveUse' in STACK_ADAPTER_REGISTRY.get(args.backend).database; + const requiresRuntimeProof = supportsRuntimeProof && args.databaseLease && args.reset; + if (requiresRuntimeProof && !proof) { + proofError = new Error(`${args.track} does not define a runtime database provenance check`); + } else if (requiresRuntimeProof && proof) { + try { + const result = await verifyApplicationDatabaseMarker(args, proof); + if (!result.write.ok) actionFailure = result.write.reason; + else if (result.runtime) runtime = result.runtime; + } catch (error) { + proofError = error; + } + + // The proof writes unique data through the application. Remove it before + // linting and scored grading so the proof cannot change the result. + if (!(await freshen())) { + bundle.error = freshenFailureMessage(); + bundle.outcome = { ...lastResetOutcome, reason: bundle.error }; + if (bundle.outcome.kind === 'app_failure') recordApplicationAbort(); + markRemainingNotRun(`run aborted: ${bundle.error}`); + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + process.exit(1); + } + } else if (supportsRuntimeProof && !args.reset) { + runtime = { ok: null, verified: false, + reason: 'runtime marker proof requires database reset to isolate its write' }; + } + + if (!proofError && !actionFailure && supportsRuntimeProof && args.databaseLease && !runtime.verified) { + proofError = new Error(`leased database identity was not verified: ${runtime.reason}`); + } + if (proofError) { + bundle.outcome = databaseProvenanceFailure(proofError); + bundle.error = bundle.outcome.reason; + markRemainingNotRun('run aborted because runtime database provenance could not be verified'); + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + process.exit(1); + } + + if (actionFailure) { + bundle.error = actionFailure; + bundle.outcome = { kind: 'app_failure', phase: 'database-provenance-action', + reason: actionFailure, appFailures: ['database-provenance-action'] }; + recordApplicationAbort(); + markRemainingNotRun('run aborted because the application database write failed'); + writeBundle(); + console.log(`\nABORTED: ${actionFailure}`); + process.exit(1); + } + + bundle.provenance.runtime = runtime; + console.log(` db runtime ... ${runtime.verified + ? runtime.ok ? runtime.reason : `WRONG DATABASE — ${runtime.reason}` + : runtime.reason}`); + if (runtime.ok === false) { + bundle.error = `app did not write its marker to the database supplied for this run: ${runtime.reason}`; + bundle.outcome = { kind: 'app_failure', phase: 'database-provenance', reason: bundle.error, + appFailures: ['database-provenance'] }; + recordApplicationAbort(); + markRemainingNotRun('run aborted because runtime database provenance failed'); + writeBundle(); + console.log('\nABORTED: application data came from outside the benchmark database.'); + process.exit(1); + } + try { + bundle.suites.lint = lint(args, selectedTask); + } catch (error) { + markRemainingNotRun('run aborted after contract lint failed to produce evidence'); + bundle.error = error instanceof Error ? error.message : String(error); + bundle.outcome = { kind: 'harness_failure', phase: 'contract-lint', reason: bundle.error }; + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + process.exit(1); + } + setupSlot.release(); + } + + // Keep current-level score separate from earlier guarantee regressions. + let total = 0, max = 0, regTotal = 0, regMax = 0; + const dirty = false; + let browserServer: BrowserServer | null = null; + let suiteSlot: { release(): void } | null = null; + try { + if (declaredSuites.some(suite => !selection + || selection.checks.some(check => check.executionId === suite.id))) { + browserServer = await chromium.launchServer({ headless: true, ...attemptBrowserLaunchOptions() }); + args.browserWsEndpoint = browserServer.wsEndpoint(); + } + for (const suite of declaredSuites) { + timingSuite = suite.id; + const selectedChecks = selection?.checks.filter(check => check.executionId === suite.id) ?? []; + if (selection && selectedChecks.length === 0) { + console.log(` ${suite.id.padEnd(10)} ... not selected`); + continue; + } + suiteSlot = await gradingSlot(); + if (!(await freshen())) { + bundle.error = freshenFailureMessage(); + console.log(` ${suite.id}: SKIPPED (${bundle.error})`); + markRemainingNotRun(`run aborted: ${bundle.error}`); + bundle.outcome = { ...lastResetOutcome, reason: bundle.error }; + if (bundle.outcome.kind === 'app_failure') recordApplicationAbort(); + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + throw new Error(bundle.error); + } + if (bundle.selection) { + bundle.selection.attemptedChecks.push(...selectedChecks.map(check => check.stableKey)); + } + let r; + try { + r = await measure('grader', () => gradeSuite(args, suite, track, recipeBinding, selectedTask, bundleArtifactId, selectedChecks)); + bundle.suites[suite.id] = r; + if (args.retryInconclusive && suiteMayRetry(r)) { + const recoveryDirectory = join(args.out, 'suite-retries', suite.id); + const initialDirectory = join(recoveryDirectory, 'initial'); + mkdirSync(initialDirectory, { recursive: true }); + for (const name of [`grading-${suite.id}.json`, `grader-${suite.id}.stdout.log`, `grader-${suite.id}.stderr.log`]) { + cpSync(join(args.out, name), join(initialDirectory, name)); + } + (bundle.suiteRetries ??= {})[suite.id] = r; + writeBundle(); + if (bundle.outcome?.kind === 'harness_failure') throw new Error(bundle.error); + console.log(` ${suite.id}: retrying this inconclusive suite once; completed suites are retained`); + if (!(await freshen())) throw new Error(`suite recovery could not restore fresh state: ${freshenFailureMessage()}`); + // Reset/start must not modify the source selected for either execution. + if (hashAppSource(args.app).sha256 !== args.sourceSha256) { + throw new Error('application source changed before suite recovery'); + } + r = await measure('grader', () => gradeSuite(args, suite, track, recipeBinding, selectedTask, + bundleArtifactId, selectedChecks, { outputDirectory: recoveryDirectory })); + // Keep the established final-grade paths for raw evidence consumers. + for (const name of [`grading-${suite.id}.json`, `grader-${suite.id}.stdout.log`, `grader-${suite.id}.stderr.log`, 'media', 'failure-media']) { + if (existsSync(join(recoveryDirectory, name))) cpSync(join(recoveryDirectory, name), join(args.out, name), { recursive: true }); + } + } + } catch (error) { + markRemainingNotRun(`run aborted after ${suite.id} grader failure`); + bundle.error = error instanceof Error ? error.message : String(error); + bundle.outcome = { kind: 'harness_failure', phase: `grade:${suite.id}`, reason: bundle.error }; + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + throw error; + } + suiteSlot.release(); + suiteSlot = null; + bundle.suites[suite.id] = r; + if (isGradePayload(r) && r.features.some(feature => feature.cleanupEvidence?.failures + .some(failure => failure.stage === 'application-restore'))) { + applicationLeftStopped = true; + } + if (bundle.selection) { + bundle.selection.reportedChecks.push(...selectedChecks.map(check => check.stableKey)); + } + if (selection) { + bundle.packRuntime = aggregatePackRuntime( + [...Object.values(bundle.suites).filter(isGradePayload), ...Object.values(bundle.suiteRetries ?? {})], + selectedPackDefinitions); + const exceeded = exceededPackBudgets(bundle.packRuntime); + if (exceeded.length) { + // Runtime budgets qualify references; generated apps still receive a complete grade. + console.log(` runtime ... ${exceeded.map(pack => + `${pack.id} ${pack.measuredRuntimeMs}ms > ${pack.budget.maxRuntimeMs}ms`) + .join(', ')} [recorded; grading continues]`); + } + } + if (suite.inherited) { regTotal += r.total; regMax += r.max; } + else { total += r.total; max += r.max; } + } + } finally { + suiteSlot?.release(); + args.browserWsEndpoint = undefined; + await closeSuiteBrowser(browserServer, bundle, writeBundle); + } + + bundle.totals = { + score: total, max, dirty, contractPass: isGradePayload(bundle.suites.lint) + ? null : bundle.suites.lint?.pass ?? null, + // null rather than 0/0 at L1, where there is nothing earlier to regress. + regression: regMax ? { score: regTotal, max: regMax } : null, + }; + writeBundle(); + + console.log(` ${'TOTAL'.padEnd(10)} ... ${total}/${max}${dirty ? ' [DIRTY]' : ''}`); + if (regMax) { + const kept = regTotal === regMax ? 'all earlier guarantees still hold' : `${regMax - regTotal} EARLIER GUARANTEE(S) LOST`; + console.log(` ${'REGRESSION'.padEnd(10)} ... ${regTotal}/${regMax} — ${kept}`); + } + console.log(` bundle: ${join(args.out, ARTIFACT_FILE.gradeBundle)}`); +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) main(); diff --git a/tools/stack-bench/commands/test-loop.ts b/tools/stack-bench/commands/test-loop.ts new file mode 100644 index 00000000000..4b573fd0ff8 --- /dev/null +++ b/tools/stack-bench/commands/test-loop.ts @@ -0,0 +1,312 @@ +#!/usr/bin/env node +import { privateGradingDirectory } from '../src/evidence/repair-evidence.js'; + +import { execFileSync, spawnSync } from 'node:child_process'; +import { readFileSync, existsSync, rmSync, mkdirSync, mkdtempSync, readdirSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { ARTIFACT_FILE, readArtifact, readArtifactPayload } from '../src/evidence/artifacts.js'; +import type { ArtifactIdentities } from '../src/evidence/artifacts.js'; +import type { CostRun, CostSession } from '../src/evidence/cost-proof.js'; +import type { PublicBackendLease } from '../src/runtime/backend-lease.js'; +import type { RepairLevel, RepairOutcome } from '../src/runtime/repair-grant.js'; +import type { LevelCheckpoint } from '../src/runtime/source-checkpoint.js'; +import { CODING_CONTAINER_BUG_REPORT_FILE } from '../src/runtime/coding-container-policy.js'; +import { CHECK_EVIDENCE_SCHEMA_VERSION } from '../src/evidence/check-evidence.js'; + +import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js'; +const WORK = mkdtempSync(join(tmpdir(), 'stack-bench-loop-')); +const APP = join(WORK, 'app'); +// A cold Playwright start plus two grades can exceed three minutes on Windows +// Docker hosts. The timeout is a deadlock guard, not a performance assertion. +const BENCH_TIMEOUT_MS = 300_000; + +interface LoopSession extends CostSession { + sessionId?: string; + tokens?: number; + turns?: number; + durationMs?: number; +} + +interface LoopLevel extends RepairLevel { + buildSessions?: LoopSession[]; + repairSessions?: LoopSession[]; + resumeSession?: LoopSession; + contractPass?: boolean; + stalled?: boolean; + code?: { totalLoc?: number }; + sessionTotals?: { sessions?: number; tokens?: number; turns?: number; durationMs?: number }; +} + +interface LoopRun extends CostRun { + id?: string; + levels?: LoopLevel[]; + outcome?: RepairOutcome; + artifactEnvelope?: { identities?: ArtifactIdentities }; + backendLease?: PublicBackendLease; + totals?: CostRun['totals'] & { max?: number; sessions?: number; tokens?: number; turns?: number; + modelDurationMs?: number; durationSec?: number }; +} + +interface GradeFeature { + id?: string; + setupEvidence?: { schemaVersion?: number; status?: string }; + criteria?: { evidence?: { schemaVersion?: number; status?: string; actions?: unknown[] } }[]; +} + +interface GradePayload { features?: GradeFeature[]; } +interface SourceCheckpointPayload { source: LevelCheckpoint; } +interface RepairContinuation { + baseline?: { reproduced?: boolean; score?: number; sourceSha256?: string }; + cumulativeRepairsBefore?: number; + cumulativeRepairsAfter?: number; + resumeSetup?: { sourceVerified?: boolean }; +} +interface RepairContinuationPayload extends LoopRun { continuation?: RepairContinuation; } + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function processOutput(error: unknown): string { + if (!isRecord(error)) return String(error); + return `${String(error.stdout ?? '')}${String(error.stderr ?? '')}`; +} + +// A failed assertion or interrupted CI job must not leave a fixture app that a +// later loop can mistake for its own output. +process.on('exit', () => rmSync(WORK, { recursive: true, force: true })); + +let failures = 0; +const check = (name: string, ok: boolean, detail = ''): void => { + console.log(` ${ok ? 'PASS' : 'FAIL'} ${name}${ok || !detail ? '' : ` — ${detail}`}`); + if (!ok) failures += 1; +}; + +function runBench(extra: string[] = []): string { + const argv = [compiledEntrypoint('commands', 'bench.js'), '--backend', 'stub', '--levels', '1', + '--agent-adapter', 'deterministic', + '--app', APP, '--out', WORK, + '--track', 'loop', + '--url', `file:///${join(APP, 'index.html').replace(/\\/g, '/')}`, ...extra]; + try { + return execFileSync('node', argv, { + encoding: 'utf8', + maxBuffer: 32 * 1024 * 1024, + timeout: BENCH_TIMEOUT_MS, + killSignal: 'SIGTERM', + }); + } catch (error: unknown) { return processOutput(error); } +} + +const invalidRounds = spawnSync('node', [compiledEntrypoint('commands', 'bench.js'), '--backend', 'stub', + '--repairs', '1.5'], + { encoding: 'utf8' }); +check('fractional correction budgets are rejected before a run starts', + invalidRounds.status !== 0 + && /--repairs must be a non-negative safe integer/.test(invalidRounds.stderr)); + +rmSync(WORK, { recursive: true, force: true }); +mkdirSync(APP, { recursive: true }); + +console.log('\nLoop test — one repair available'); +const out = runBench(['--repairs', '1']); +const runPath = join(WORK, ARTIFACT_FILE.run); + +check(`the benchmark run produced ${ARTIFACT_FILE.run}`, existsSync(runPath)); +if (!existsSync(runPath)) { + console.log(`\ncannot continue without ${ARTIFACT_FILE.run}`); + process.exit(1); +} + +const run = readArtifactPayload(runPath); +const level = run.levels?.[0]; +const evidenceDir = privateGradingDirectory(APP); +const bundleArtifact = readArtifact(join(evidenceDir, ARTIFACT_FILE.gradeBundle), + { expectedKind: 'grade_bundle' }); +const lintArtifact = readArtifact(join(evidenceDir, ARTIFACT_FILE.contractLint), + { expectedKind: 'contract_lint' }); +const gradeArtifact = readArtifact(join(evidenceDir, 'grading-features.json'), { expectedKind: 'grade' }); +const leaseArtifact = readArtifact(join(WORK, ARTIFACT_FILE.backendLease), + { expectedKind: 'backend_lease_evidence' }); +const checkpointArtifact = readArtifact(join(WORK, 'level-l1-checkpoint.json'), + { expectedKind: 'source_checkpoint' }); + +check('recorded exactly one level', run.levels?.length === 1); +check('run and level carry structured outcomes', + typeof run.outcome?.kind === 'string' && run.outcome.kind === level?.outcome?.kind, + `run=${run.outcome?.kind} level=${level?.outcome?.kind}`); +check('artifacts carry the producing run id', typeof run.id === 'string' && run.id.length > 10); +check('run envelope identifies engine, agent adapter, and stack adapter', + /^[a-f0-9]{64}$/.test(run.artifactEnvelope?.identities?.engine?.sha256 ?? '') + && /^[a-f0-9]{64}$/.test(run.artifactEnvelope?.identities?.agentAdapter?.sha256 ?? '') + && run.artifactEnvelope?.identities?.stackAdapter?.id === 'stub'); +check('bundle is a child of the run', bundleArtifact.attempt.parentId === run.id, + JSON.stringify(bundleArtifact.attempt)); +check('public lease evidence is a child of the run', leaseArtifact.attempt.parentId === run.id, + JSON.stringify(leaseArtifact.attempt)); +check('level source checkpoint is hash-bound and linked to the run', + checkpointArtifact.attempt.parentId === run.id + && level?.checkpoint?.artifact === 'level-l1-checkpoint.json' + && level.checkpoint.sha256 === checkpointArtifact.payload.source.sha256 + && /^[a-f0-9]{64}$/.test(level.checkpoint.sha256) + && existsSync(join(WORK, level.checkpoint.directory)), + JSON.stringify(level?.checkpoint)); +check('lint and grade evidence are children of the bundle', + [lintArtifact, gradeArtifact] + .every(artifact => artifact.attempt.parentId === bundleArtifact.attempt.id)); +check('suite did not emit an unsolicited action probe', + !existsSync(join(evidenceDir, ARTIFACT_FILE.actions)) + && !Object.hasOwn(bundleArtifact.payload, 'actions')); +const gradedFeatures = gradeArtifact.payload?.features ?? []; +check('grade artifacts retain typed setup, criterion, and action evidence', + gradedFeatures.length > 0 + && gradedFeatures.every(feature => feature.setupEvidence?.schemaVersion === CHECK_EVIDENCE_SCHEMA_VERSION + && (feature.criteria ?? []).every(criterion => criterion.evidence?.schemaVersion === CHECK_EVIDENCE_SCHEMA_VERSION + && Array.isArray(criterion.evidence.actions))), + JSON.stringify(gradedFeatures.map(feature => ({ id: feature.id, + setup: feature.setupEvidence?.status, + criteria: feature.criteria?.map(criterion => criterion.evidence?.status) })))); +const publicJson = [runPath, join(WORK, ARTIFACT_FILE.backendLease), + join(evidenceDir, ARTIFACT_FILE.gradeBundle), join(evidenceDir, ARTIFACT_FILE.contractLint), + join(evidenceDir, 'grading-features.json')] + .map(path => readFileSync(path, 'utf8')).join('\n'); +check('public envelopes contain no secret or lease-token fields', + !/"(?:apiKey|leaseToken|ownershipToken|password|secret)"\s*:/i.test(publicJson)); +check('backend lease was released', + ['released', 'stopped'].includes(run.backendLease?.state ?? '') + && (run.backendLease?.resources?.locks?.every(lock => lock.releasedAt) ?? false), + JSON.stringify(run.backendLease?.state)); +check('a repair ran', level?.repairs === 1, `repairs=${level?.repairs}`); +check('successful repair is explicit', level?.repair?.status === 'corrected' + && level.repair.limit === 1 && level.repair.used === 1 + && level.repair.stopReason === 'passed', + JSON.stringify(level?.repair)); +const reportPath = join(APP, CODING_CONTAINER_BUG_REPORT_FILE); +const reportExists = existsSync(reportPath); +check('the bug report was written', reportExists); +// Behavioural findings must never reveal how they were detected, or a fix can +// target the check instead of the app. Missing-control findings are exempt: +// there the element id is the requirement. +const report = reportExists ? readFileSync(reportPath, 'utf8') : ''; +const behaviourSection = report.split('## Application interface')[0] ?? ''; +check('behavioural findings do not leak selectors or timings', + !/data-(?:role|testid)|locator|within \d+ms/.test(behaviourSection)); +check('missing interfaces are reported separately', /## Application interface/.test(report)); +check('build and fix costs are both recorded', + (level?.buildCostUsd ?? 0) > 0 && (level?.repairCostUsd ?? 0) > 0, + `build=${level?.buildCostUsd} fix=${level?.repairCostUsd}`); +check('build and fix sessions remain individually auditable', + level?.buildSessions?.length === 1 + && level.buildSessions[0]?.sessionId === 'stub-build' + && level?.repairSessions?.length === 1 + && level.repairSessions[0]?.sessionId === 'stub-fix', + JSON.stringify({ builds: level?.buildSessions, fixes: level?.repairSessions })); +check('level session totals include the build and fix', + level?.sessionTotals?.sessions === 2 + && level.sessionTotals.tokens === 2000 + && level.sessionTotals.turns === 5 + && level.sessionTotals.durationMs === 100, + JSON.stringify(level?.sessionTotals)); +check('grading produced a score out of a maximum', Number.isInteger(level?.score) && (level?.max ?? 0) > 0, + `${level?.score}/${level?.max}`); +check('code metrics captured', Boolean(level?.code) && typeof level?.code?.totalLoc === 'number', + JSON.stringify(level?.code)); +check('totals aggregate the levels', run.totals?.max === level?.max); +check('run totals aggregate every model session', + run.totals?.sessions === 2 && run.totals.tokens === 2000 + && run.totals.turns === 5 && run.totals.modelDurationMs === 100, + JSON.stringify(run.totals)); +check('wall time recorded', (run.totals?.durationSec ?? -1) >= 0); +check('the fix improved the contract lint', + /APPLICATION CONTRACT FAIL[\s\S]*APPLICATION CONTRACT PASS/.test(out) + || level?.contractPass === true, + 'expected the broken fixture to fail the lint and the fixed one to pass'); + +console.log('\nLoop test — zero repairs allowed'); +rmSync(WORK, { recursive: true, force: true }); +mkdirSync(APP, { recursive: true }); +runBench(['--repairs', '0']); +const capped = readArtifactPayload(runPath); +check('no fix ran when the cap is zero', capped.levels?.[0]?.repairs === 0); +check('no bug report was written when no fix is allowed', + !existsSync(join(APP, CODING_CONTAINER_BUG_REPORT_FILE))); + +console.log('\nLoop test - flat corrections exhaust their declared budget'); +rmSync(WORK, { recursive: true, force: true }); +mkdirSync(APP, { recursive: true }); +runBench(['--repairs', '2', '--model', 'deterministic-stall']); +const exhausted = readArtifactPayload(runPath); +const exhaustedLevel = exhausted.levels?.[0]; +check('both correction rounds ran after the first flat result', exhaustedLevel?.repairs === 2, + `repairs=${exhaustedLevel?.repairs}`); +check('an unresolved app records budget exhaustion', exhaustedLevel?.repair?.status === 'budget-exhausted' + && exhaustedLevel.repair.limit === 2 && exhaustedLevel.repair.used === 2 + && exhaustedLevel.repair.stopReason === 'budget-exhausted' + && exhaustedLevel.stalled === true && exhausted.outcome?.kind === 'app_failure', + JSON.stringify({ repair: exhaustedLevel?.repair, outcome: exhausted.outcome })); + +console.log('\nLoop test - a later finite grant continues the exact exhausted source'); +rmSync(WORK, { recursive: true, force: true }); +mkdirSync(APP, { recursive: true }); +runBench(['--repairs', '2', '--model', 'deterministic-deferred']); +const parentBefore = readFileSync(runPath, 'utf8'); +const deferred = readArtifactPayload(runPath); +check('the deferred parent exhausted its original two-round budget', + deferred.levels?.[0]?.repair?.status === 'budget-exhausted' + && deferred.levels[0].repair.used === 2, + JSON.stringify(deferred.levels?.[0]?.repair)); +let continuationOutput = ''; +try { + continuationOutput = execFileSync('node', [join(ROOT, 'dist', 'commands', 'repair-cli.js'), 'grant', WORK, + '--level', '1', '--repairs', '2', '--timeout-minutes', '10'], { + encoding: 'utf8', maxBuffer: 32 * 1024 * 1024, timeout: BENCH_TIMEOUT_MS, + }); +} catch (error: unknown) { + continuationOutput = processOutput(error); +} +const continuationRoot = join(WORK, 'continuations'); +const continuationDirectories = existsSync(continuationRoot) + ? readdirSync(continuationRoot, { withFileTypes: true }).filter(entry => entry.isDirectory()) : []; +const continuationDirectory = continuationDirectories.length === 1 + ? join(continuationRoot, continuationDirectories[0]?.name ?? '') : null; +const continuationPath = continuationDirectory + ? join(continuationDirectory, ARTIFACT_FILE.run) : null; +check('repair grant produced one linked continuation', + continuationPath !== null && existsSync(continuationPath), continuationOutput.slice(-2000)); +if (continuationDirectory && continuationPath && existsSync(continuationPath)) { + const continuationArtifact = readArtifact(continuationPath, { expectedKind: 'repair_continuation' }); + const continuation = continuationArtifact.payload; + const continuedLevel = continuation.levels?.[0]; + const continuationDetails = continuation.continuation; + const deferredLevel = deferred.levels?.[0]; + check('continuation reproduced the exact failed baseline before spending a repair', + continuationDetails?.baseline?.reproduced === true + && continuationDetails.baseline.score === deferredLevel?.score + && continuationDetails.baseline.sourceSha256 === deferredLevel?.checkpoint?.sha256, + JSON.stringify(continuationDetails?.baseline)); + check('continuation reached correctness inside its finite added budget', + continuation.outcome?.kind === 'passed' + && continuedLevel?.repair?.status === 'corrected' + && continuedLevel.repair.used === 1 + && continuationDetails?.cumulativeRepairsBefore === 2 + && continuationDetails?.cumulativeRepairsAfter === 3, + JSON.stringify({ repair: continuedLevel?.repair, continuation: continuationDetails })); + check('resume setup is visible, separately costed, and does not consume a repair', + continuationDetails?.resumeSetup?.sourceVerified === true + && continuedLevel?.resumeSession?.sessionId === 'stub-resume' + && (continuedLevel?.resumeCostUsd ?? 0) > 0 + && continuedLevel.repairs === 1, + JSON.stringify({ setup: continuationDetails?.resumeSetup, + resume: continuedLevel?.resumeSession, fixes: continuedLevel?.repairs })); + check('continuation process outcome is retained as a typed child artifact', + readArtifact(join(continuationDirectory, ARTIFACT_FILE.process), + { expectedKind: 'repair_process' }).attempt.parentId === deferred.id); +} +check('grant left the original run artifact byte-for-byte unchanged', + readFileSync(runPath, 'utf8') === parentBefore); + +rmSync(WORK, { recursive: true, force: true }); +console.log(`\n${failures === 0 ? 'loop OK' : `${failures} check(s) failed`}`); +process.exit(failures === 0 ? 0 : 1); diff --git a/tools/stack-bench/conditions/catalog.json b/tools/stack-bench/conditions/catalog.json new file mode 100644 index 00000000000..c88e06cf670 --- /dev/null +++ b/tools/stack-bench/conditions/catalog.json @@ -0,0 +1,16 @@ +{ + "schemaVersion": 1, + "kind": "study-condition-catalog", + "guidanceProfiles": { + "model-free-stub": "guidance/model-free-stub.json", + "prescribed": "guidance/prescribed.json", + "neutral": "guidance/neutral.json", + "neutral-dev": "guidance/neutral-dev.json", + "neutral-dev-no-sdk": "guidance/neutral-dev-no-sdk.json", + "neutral-managed-dev": "guidance/neutral-managed-dev.json", + "neutral-no-sdk": "guidance/neutral-no-sdk.json" + }, + "repairPolicies": { + "scored-only": "repairs/scored-only.json" + } +} diff --git a/tools/stack-bench/conditions/guidance/model-free-stub.json b/tools/stack-bench/conditions/guidance/model-free-stub.json new file mode 100644 index 00000000000..ffd536b8c02 --- /dev/null +++ b/tools/stack-bench/conditions/guidance/model-free-stub.json @@ -0,0 +1,20 @@ +{ + "schemaVersion": 1, + "kind": "backend-guidance-profile", + "id": "model-free-stub", + "mode": "neutral", + "material": { + "accessFacts": true, + "apiReference": false, + "designAdvice": false + }, + "documents": { + "stub": "backends/model-free-stub.md" + }, + "applicationInterfaces": { + "stub": "http" + }, + "skills": { + "stub": [] + } +} diff --git a/tools/stack-bench/conditions/guidance/neutral-dev-no-sdk.json b/tools/stack-bench/conditions/guidance/neutral-dev-no-sdk.json new file mode 100644 index 00000000000..7e65395a17f --- /dev/null +++ b/tools/stack-bench/conditions/guidance/neutral-dev-no-sdk.json @@ -0,0 +1,39 @@ +{ + "schemaVersion": 1, + "kind": "backend-guidance-profile", + "id": "neutral-dev-no-sdk", + "mode": "neutral", + "material": { + "accessFacts": true, + "apiReference": true, + "designAdvice": true + }, + "documents": { + "mongodb": "backends/minimal/mongodb.md", + "postgres": "backends/minimal/postgres.md", + "spacetime": "backends/minimal/spacetime.md", + "convex": "backends/minimal/convex.md", + "supabase": "backends/minimal/supabase.md" + }, + "applicationInterfaces": { + "mongodb": "http", + "postgres": "http", + "spacetime": "reducer", + "convex": "convex", + "supabase": "supabase" + }, + "skills": { + "mongodb": [], + "postgres": [], + "spacetime": [ + "spacetime-dev" + ], + "convex": [], + "supabase": [] + }, + "credentialAliases": { + "stackbench-admin-2026": "store-admin-2026", + "stackbench-customer-2026": "store-customer-2026", + "stackbench-staff-2026": "store-staff-2026" + } +} diff --git a/tools/stack-bench/conditions/guidance/neutral-dev.json b/tools/stack-bench/conditions/guidance/neutral-dev.json new file mode 100644 index 00000000000..3d94f92bdc9 --- /dev/null +++ b/tools/stack-bench/conditions/guidance/neutral-dev.json @@ -0,0 +1,42 @@ +{ + "schemaVersion": 1, + "kind": "backend-guidance-profile", + "id": "neutral-dev", + "mode": "neutral", + "material": { + "accessFacts": true, + "apiReference": true, + "designAdvice": true + }, + "documents": { + "mongodb": "backends/minimal/mongodb.md", + "postgres": "backends/minimal/postgres.md", + "spacetime": "backends/minimal/spacetime.md", + "convex": "backends/minimal/convex.md", + "supabase": "backends/minimal/supabase.md" + }, + "applicationInterfaces": { + "mongodb": "http", + "postgres": "http", + "spacetime": "reducer", + "convex": "convex", + "supabase": "supabase" + }, + "skills": { + "mongodb": [], + "postgres": [], + "spacetime": [ + "typescript-server", + "typescript-client", + "cli", + "spacetime-dev" + ], + "convex": [], + "supabase": [] + }, + "credentialAliases": { + "stackbench-admin-2026": "store-admin-2026", + "stackbench-customer-2026": "store-customer-2026", + "stackbench-staff-2026": "store-staff-2026" + } +} diff --git a/tools/stack-bench/conditions/guidance/neutral-managed-dev.json b/tools/stack-bench/conditions/guidance/neutral-managed-dev.json new file mode 100644 index 00000000000..17c99c04966 --- /dev/null +++ b/tools/stack-bench/conditions/guidance/neutral-managed-dev.json @@ -0,0 +1,42 @@ +{ + "schemaVersion": 1, + "kind": "backend-guidance-profile", + "id": "neutral-managed-dev", + "mode": "neutral", + "material": { + "accessFacts": true, + "apiReference": true, + "designAdvice": true + }, + "documents": { + "mongodb": "backends/minimal/mongodb.md", + "postgres": "backends/minimal/postgres.md", + "spacetime": "backends/minimal/spacetime.md", + "convex": "backends/minimal/convex.md", + "supabase": "backends/minimal/supabase.md" + }, + "applicationInterfaces": { + "mongodb": "http", + "postgres": "http", + "spacetime": "reducer", + "convex": "convex", + "supabase": "supabase" + }, + "skills": { + "mongodb": [], + "postgres": [], + "spacetime": [ + "typescript-server", + "typescript-client", + "cli", + "spacetime-managed-dev" + ], + "convex": [], + "supabase": [] + }, + "credentialAliases": { + "stackbench-admin-2026": "store-admin-2026", + "stackbench-customer-2026": "store-customer-2026", + "stackbench-staff-2026": "store-staff-2026" + } +} diff --git a/tools/stack-bench/conditions/guidance/neutral-no-sdk.json b/tools/stack-bench/conditions/guidance/neutral-no-sdk.json new file mode 100644 index 00000000000..6818f5add45 --- /dev/null +++ b/tools/stack-bench/conditions/guidance/neutral-no-sdk.json @@ -0,0 +1,37 @@ +{ + "schemaVersion": 1, + "kind": "backend-guidance-profile", + "id": "neutral-no-sdk", + "mode": "neutral", + "material": { + "accessFacts": true, + "apiReference": true, + "designAdvice": true + }, + "documents": { + "mongodb": "backends/minimal/mongodb.md", + "postgres": "backends/minimal/postgres.md", + "spacetime": "backends/minimal/spacetime.md", + "convex": "backends/minimal/convex.md", + "supabase": "backends/minimal/supabase.md" + }, + "applicationInterfaces": { + "mongodb": "http", + "postgres": "http", + "spacetime": "reducer", + "convex": "convex", + "supabase": "supabase" + }, + "skills": { + "mongodb": [], + "postgres": [], + "spacetime": [], + "convex": [], + "supabase": [] + }, + "credentialAliases": { + "stackbench-admin-2026": "store-admin-2026", + "stackbench-customer-2026": "store-customer-2026", + "stackbench-staff-2026": "store-staff-2026" + } +} diff --git a/tools/stack-bench/conditions/guidance/neutral.json b/tools/stack-bench/conditions/guidance/neutral.json new file mode 100644 index 00000000000..a6adbbfb2a1 --- /dev/null +++ b/tools/stack-bench/conditions/guidance/neutral.json @@ -0,0 +1,41 @@ +{ + "schemaVersion": 1, + "kind": "backend-guidance-profile", + "id": "neutral", + "mode": "neutral", + "material": { + "accessFacts": true, + "apiReference": true, + "designAdvice": true + }, + "documents": { + "mongodb": "backends/minimal/mongodb.md", + "postgres": "backends/minimal/postgres.md", + "spacetime": "backends/minimal/spacetime.md", + "convex": "backends/minimal/convex.md", + "supabase": "backends/minimal/supabase.md" + }, + "applicationInterfaces": { + "mongodb": "http", + "postgres": "http", + "spacetime": "reducer", + "convex": "convex", + "supabase": "supabase" + }, + "skills": { + "mongodb": [], + "postgres": [], + "spacetime": [ + "typescript-server", + "typescript-client", + "cli" + ], + "convex": [], + "supabase": [] + }, + "credentialAliases": { + "stackbench-admin-2026": "store-admin-2026", + "stackbench-customer-2026": "store-customer-2026", + "stackbench-staff-2026": "store-staff-2026" + } +} diff --git a/tools/stack-bench/conditions/guidance/prescribed.json b/tools/stack-bench/conditions/guidance/prescribed.json new file mode 100644 index 00000000000..891c526c62b --- /dev/null +++ b/tools/stack-bench/conditions/guidance/prescribed.json @@ -0,0 +1,41 @@ +{ + "schemaVersion": 1, + "kind": "backend-guidance-profile", + "id": "prescribed", + "mode": "prescribed", + "material": { + "accessFacts": true, + "apiReference": true, + "designAdvice": true + }, + "documents": { + "mongodb": "backends/mongodb.md", + "postgres": "backends/postgres.md", + "spacetime": "backends/spacetime.md", + "convex": "backends/convex.md", + "supabase": "backends/supabase.md" + }, + "applicationInterfaces": { + "mongodb": "http", + "postgres": "http", + "spacetime": "reducer", + "convex": "convex", + "supabase": "supabase" + }, + "skills": { + "mongodb": [], + "postgres": [], + "spacetime": [ + "typescript-server", + "typescript-client", + "cli" + ], + "convex": [], + "supabase": [] + }, + "credentialAliases": { + "stackbench-admin-2026": "store-admin-2026", + "stackbench-customer-2026": "store-customer-2026", + "stackbench-staff-2026": "store-staff-2026" + } +} diff --git a/tools/stack-bench/conditions/repairs/scored-only.json b/tools/stack-bench/conditions/repairs/scored-only.json new file mode 100644 index 00000000000..e3fc69b24af --- /dev/null +++ b/tools/stack-bench/conditions/repairs/scored-only.json @@ -0,0 +1,8 @@ +{ + "schemaVersion": 1, + "kind": "repair-policy", + "id": "scored-only", + "scoredEvidence": true, + "observedEvidence": false, + "scenarioValues": "failed-observations" +} diff --git a/tools/stack-bench/container/Dockerfile b/tools/stack-bench/container/Dockerfile new file mode 100644 index 00000000000..9a0065e6da5 --- /dev/null +++ b/tools/stack-bench/container/Dockerfile @@ -0,0 +1,64 @@ +# The image a generated app is built in. +# +# The generated app must not see the harness, grader, or test definitions. +# +# No harness, grader, or test definition is copied in or mounted at run time. +# An adapter can mount only its selected stack artifacts, read-only. +# Keep the readable tag, but bind the base to an exact manifest. Campaigns use +# the digest of the completed image, so every attempt runs the same artifact. +FROM node:22-slim@sha256:f86be15afa9a8277608e141ce2a8aa55d3d9c40845921b8511f4fb7897be2554 + +# git: builds initialise repositories and some tooling shells out to it. +# curl: readiness probes against the app's own dev server. +# ca-certificates: TLS for npm and the API. +# procps: the build starts and stops its own dev servers. +# lsof: `kill-port` locates a listener with lsof on Linux, and finds nothing +# without it — it then prints "Process on port N killed" and exits 0 while the +# server keeps running. Every durability and deploy-window test would pass +# without restarting anything, which is worse than failing. None of lsof, fuser, +# ss or netstat is present in node:22-slim. +RUN apt-get update && apt-get install -y --no-install-recommends \ + git curl ca-certificates procps lsof chromium util-linux \ + && rm -rf /var/lib/apt/lists/* + +# Browser testing uses the system browser; no downloads into the agent home are needed. +ENV CHROME_BIN=/usr/bin/chromium +COPY browser-tools/package*.json /opt/browser-tools/ +RUN npm ci --prefix /opt/browser-tools --omit=dev --ignore-scripts \ + && node -e "require('/opt/browser-tools/node_modules/puppeteer-core')" + +# Pinned, and the auto-updater disabled: a CLI that updates itself mid-series +# changes the thing under test between one backend and the next. Override at +# build time to move deliberately rather than by drift. +ARG CLAUDE_VERSION=2.1.284 +ARG CODEX_VERSION=0.159.3 +ENV DISABLE_AUTOUPDATER=1 +RUN npm install -g @anthropic-ai/claude-code@${CLAUDE_VERSION} \ + && claude --version +RUN npm install -g @openai/codex@${CODEX_VERSION} && codex --version +# xAI's installer does not verify the download, so the binary is checked here. +ARG GROK_VERSION=1.0.46 +ARG GROK_SHA256=41626a53292324140b92556b9d42ff5542e3dcd04aff85eafb8689dd4adb44fc +ENV GROK_DISABLE_AUTOUPDATER=1 +RUN curl -fsSL "https://x.ai/cli/grok-${GROK_VERSION}-linux-x86_64.gz" | gzip -dc > /usr/local/bin/grok \ + && echo "${GROK_SHA256} /usr/local/bin/grok" | sha256sum -c - \ + && chmod 0755 /usr/local/bin/grok && grok --version +# The release Google's installer resolves for linux_x64, checked against its manifest's SHA-512. +ARG AGY_URL=https://storage.googleapis.com/antigravity-public/antigravity-cli/1.3.2-5813501495738368/linux-x64/cli_linux_x64.tar.gz +ARG AGY_SHA512=8dc2bb8471cfa6326e29e2896add767ad2d1f3714304b70fbae720d68af011182e7afde43964a7a3f54c604ab72c4fad772c8595608b059cfc582fb6798a25d2 +RUN curl -fsSL "${AGY_URL}" -o /tmp/agy.tar.gz \ + && echo "${AGY_SHA512} /tmp/agy.tar.gz" | sha512sum -c - \ + && tar -xzf /tmp/agy.tar.gz -C /tmp antigravity && install -m 0755 /tmp/antigravity /usr/local/bin/agy \ + && rm /tmp/agy.tar.gz /tmp/antigravity && agy --version + +# The coding session can change the app and its own temporary state. It does +# not run as root, so it cannot change the system or harness control files. +RUN useradd --uid 10001 --create-home --shell /bin/bash developer \ + && chmod 0700 /home/developer + +# The app under construction. Everything the build writes lives here, and the +# host mounts its own work directory over it. +WORKDIR /app + +# No ENTRYPOINT: the run-build command supplies the whole command so the prompt can go +# in on stdin exactly as it does on the host. diff --git a/tools/stack-bench/container/agy-renewal.ts b/tools/stack-bench/container/agy-renewal.ts new file mode 100644 index 00000000000..bbe5078f539 --- /dev/null +++ b/tools/stack-bench/container/agy-renewal.ts @@ -0,0 +1,33 @@ +import { execFile } from 'node:child_process'; +import { randomBytes } from 'node:crypto'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; + +const RENEWAL_TIMEOUT_MS = 2 * 60_000; + +// Google's CLI renews a sign-in before any command, but only once its token has expired. +// This runs it on a private copy of the sign-in marked expired, in its own container +// outside every agent's sandbox, and returns the file it leaves. The token it replaces +// stays valid until its own expiry. `directory` must be at the same path on the Docker host. +export function agyRenewal(image: string, directory: string): (text: string) => Promise { + return async text => { + const home = mkdtempSync(join(directory, 'stack-bench-agy-renewal-')); + try { + const state = join(home, '.gemini', 'antigravity-cli'); + mkdirSync(state, { recursive: true, mode: 0o700 }); + const login = JSON.parse(text) as { token: { expiry: string } }; + login.token.expiry = '2000-01-01T00:00:00Z'; + writeFileSync(join(state, 'antigravity-oauth-token'), JSON.stringify(login), { mode: 0o600 }); + writeFileSync(join(state, 'settings.json'), '{"enableTelemetry":false}\n', { mode: 0o600 }); + // The model list names the account's plan; none of the output is kept. + try { + await promisify(execFile)('docker', ['run', '--rm', '--name', `sb-agy-renewal-${randomBytes(6).toString('hex')}`, + '--user', `${process.getuid?.() ?? 0}:${process.getgid?.() ?? 0}`, '--cap-drop', 'ALL', + '--security-opt', 'no-new-privileges:true', '--mount', `type=bind,src=${home},dst=/agy`, '-e', 'HOME=/agy', + '-w', '/agy', image, 'agy', 'models'], { timeout: RENEWAL_TIMEOUT_MS, maxBuffer: 16 * 1024 * 1024 }); + } catch { throw new Error('agy could not renew the Antigravity sign-in; sign in with agy again'); } + return readFileSync(join(state, 'antigravity-oauth-token'), 'utf8'); + } finally { rmSync(home, { recursive: true, force: true }); } + }; +} diff --git a/tools/stack-bench/container/binary-provenance.ts b/tools/stack-bench/container/binary-provenance.ts new file mode 100644 index 00000000000..6fa39d099be --- /dev/null +++ b/tools/stack-bench/container/binary-provenance.ts @@ -0,0 +1,186 @@ +#!/usr/bin/env node + +import { createHash } from 'node:crypto'; +import { existsSync, readFileSync, statSync, writeFileSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { SOURCE_IDENTITY_SCHEME } from '../src/releases/release-source.js'; +import { STACK_BENCH_RUNNER_PLATFORM } from '../src/runtime/runner-environment.js'; + +export const RUST_BUILDER_IMAGE = + 'rust:1.93-slim-bookworm@sha256:8f8609d448e821fbc0e44241bc5ca4ce49663cc6306ff1a17f655a0e2a7cd084'; +export const BINARY_NAMES = Object.freeze(['spacetimedb-cli', 'spacetimedb-standalone']); +const PROVENANCE_NAME = 'spacetimedb-binaries.json'; + +interface BinarySourceIdentity { + identityScheme: typeof SOURCE_IDENTITY_SCHEME; + revision: string; + sha256: string; + files: number; +} + +interface BinaryRecord { + sha256: string; + size: number; +} + +interface BinaryProvenance { + schemaVersion: 2; + platform: typeof STACK_BENCH_RUNNER_PLATFORM; + builderImage: string; + source: BinarySourceIdentity; + binaries: Record; +} + +function sha256File(path: string): string { + return createHash('sha256').update(readFileSync(path)).digest('hex'); +} + +function binaryPath(stackBenchRoot: string, name: string): string { + return join(stackBenchRoot, 'container', 'bin', name); +} + +function provenancePath(stackBenchRoot: string): string { + return join(stackBenchRoot, 'container', PROVENANCE_NAME); +} + +function assertSha256(value: unknown, label: string): asserts value is string { + if (typeof value !== 'string' || !/^[a-f0-9]{64}$/.test(value)) { + throw new Error(`${label} must be a SHA-256 digest`); + } +} + +function inspectBinary(path: string, name: string): BinaryRecord { + if (!existsSync(path)) { + throw new Error(`${name} is absent; run tools/stack-bench/container/build-linux-cli.sh`); + } + const stat = statSync(path); + if (!stat.isFile() || stat.size < 4) throw new Error(`${name} is not a non-empty file`); + const magic = readFileSync(path).subarray(0, 4); + if (!magic.equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46]))) { + throw new Error(`${name} is not a Linux ELF binary`); + } + return { sha256: sha256File(path), size: stat.size }; +} + +export function createBinaryProvenance(stackBenchRoot: string, + source: BinarySourceIdentity): BinaryProvenance { + if (source?.identityScheme !== SOURCE_IDENTITY_SCHEME) { + throw new Error('binary source identity scheme is unsupported'); + } + assertSha256(source?.sha256, 'binary source identity'); + if (!/^[a-f0-9]{40}(?:[a-f0-9]{24})?$/.test(source?.revision ?? '')) { + throw new Error('binary source revision must be an exact commit id'); + } + if (!Number.isSafeInteger(source?.files) || source.files < 1) { + throw new Error('binary source file count must be a positive integer'); + } + const binaries: Record = {}; + for (const name of BINARY_NAMES) binaries[name] = inspectBinary(binaryPath(stackBenchRoot, name), name); + return { + schemaVersion: 2, + platform: STACK_BENCH_RUNNER_PLATFORM, + builderImage: RUST_BUILDER_IMAGE, + source: { identityScheme: source.identityScheme, + revision: source.revision, sha256: source.sha256, files: source.files }, + binaries, + }; +} + +export function assertBinarySourceUnchanged(before: BinarySourceIdentity, + after: BinarySourceIdentity): void { + if (before?.identityScheme !== after?.identityScheme + || before?.revision !== after?.revision || before?.sha256 !== after?.sha256 + || before?.files !== after?.files) { + throw new Error('binary source changed during the build'); + } +} + +function readProvenance(stackBenchRoot: string): BinaryProvenance { + const path = provenancePath(stackBenchRoot); + if (!existsSync(path)) { + throw new Error(`${PROVENANCE_NAME} is absent; run tools/stack-bench/container/build-linux-cli.sh`); + } + let manifest: BinaryProvenance & { status?: string }; + try { manifest = JSON.parse(readFileSync(path, 'utf8')); } + catch (error) { + throw new Error(`${PROVENANCE_NAME} is not valid JSON: ${error instanceof Error + ? error.message : String(error)}`); + } + if (manifest.status === 'unbuilt') { + throw new Error(`${PROVENANCE_NAME} has no verified binaries; run tools/stack-bench/container/build-linux-cli.sh`); + } + return manifest; +} + +export function verifyBinaryProvenance(stackBenchRoot: string, + { sourceSha256 }: { sourceSha256: string }): BinaryProvenance { + assertSha256(sourceSha256, 'expected binary source identity'); + const manifest = readProvenance(stackBenchRoot); + if (manifest.schemaVersion !== 2) throw new Error('unsupported binary provenance schema'); + if (manifest.platform !== STACK_BENCH_RUNNER_PLATFORM) { + throw new Error(`binary provenance platform must be ${STACK_BENCH_RUNNER_PLATFORM}`); + } + if (manifest.builderImage !== RUST_BUILDER_IMAGE) { + throw new Error('binary provenance does not use the pinned Rust builder image'); + } + assertSha256(manifest.source?.sha256, 'recorded binary source identity'); + if (manifest.source.identityScheme !== SOURCE_IDENTITY_SCHEME) { + throw new Error('recorded binary source identity scheme is unsupported'); + } + if (manifest.source.sha256 !== sourceSha256) { + throw new Error('SpacetimeDB binaries do not match the selected release source'); + } + if (!/^[a-f0-9]{40}(?:[a-f0-9]{24})?$/.test(manifest.source?.revision ?? '')) { + throw new Error('recorded binary source revision is invalid'); + } + if (!Number.isSafeInteger(manifest.source?.files) || manifest.source.files < 1) { + throw new Error('recorded binary source file count is invalid'); + } + for (const name of BINARY_NAMES) { + const expected = manifest.binaries?.[name]; + assertSha256(expected?.sha256, `${name} recorded checksum`); + if (!Number.isSafeInteger(expected.size) || expected.size < 4) { + throw new Error(`${name} recorded size is invalid`); + } + const actual = inspectBinary(binaryPath(stackBenchRoot, name), name); + if (actual.size !== expected.size) throw new Error(`${name} size does not match provenance`); + if (actual.sha256 !== expected.sha256) throw new Error(`${name} checksum does not match provenance`); + } + return manifest; +} + +function option(args: string[], name: string): string { + const index = args.indexOf(name); + const value = index === -1 ? undefined : args[index + 1]; + if (!value) throw new Error(`${name} is required`); + return value; +} + +function main(): void { + const [command, ...args] = process.argv.slice(2); + if (command === 'record-snapshot') { + // The Docker source stage records this identity before compiling the same + // immutable Git archive. This path needs no host-built binary or Git metadata. + const root = resolve(option(args, '--root')); + const source = JSON.parse(readFileSync(resolve(option(args, '--source-file')), 'utf8')); + writeFileSync(provenancePath(root), `${JSON.stringify(createBinaryProvenance(root, source), null, 2)}\n`); + return; + } + if (command === 'verify') { + const stackBenchRoot = resolve(option(args, '--root')); + verifyBinaryProvenance(stackBenchRoot, { sourceSha256: option(args, '--source-sha256') }); + console.log('verified SpacetimeDB CLI and standalone binary provenance'); + return; + } + throw new Error('Usage: binary-provenance record-snapshot --root PATH --source-file PATH | verify --root PATH --source-sha256 SHA256'); +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + try { main(); } + catch (error) { + console.error(`binary provenance failed: ${error instanceof Error ? error.message : String(error)}`); + process.exit(1); + } +} diff --git a/tools/stack-bench/container/broker-protocols.ts b/tools/stack-bench/container/broker-protocols.ts new file mode 100644 index 00000000000..83d6710a5b7 --- /dev/null +++ b/tools/stack-bench/container/broker-protocols.ts @@ -0,0 +1,691 @@ +import type { IncomingMessage, OutgoingHttpHeaders } from 'node:http'; +import { brotliDecompressSync, gunzipSync, inflateSync } from 'node:zlib'; +import { createParser } from 'eventsource-parser'; +import { ANTHROPIC_UPSTREAMS, firstUnpricedReason } from './credential-broker-accounting.js'; +import type { BrokerConfig, UnpricedReason } from './credential-broker-accounting.js'; +import type { ProviderFailure } from '../src/agents/provider-failure.js'; +import { ANTIGRAVITY_ACCOUNT_CONTROL_PATHS, ANTIGRAVITY_ACCOUNT_HOST, ANTIGRAVITY_ACCOUNT_LOOKUP, ANTIGRAVITY_ACCOUNT_MODELS, + ANTIGRAVITY_ACCOUNT_HELPERS, ANTIGRAVITY_TITLE_MODEL, ANTIGRAVITY_TITLE_PROMPT, antigravityTools } + from '../src/agents/antigravity-protocol.js'; + +const MAX_REQUEST_BYTES = 32 * 1024 * 1024; +// Account Responses does not promise max_output_tokens. Reserve the documented +// maximum for each known model; an unknown bound must fail before a paid request. +// https://developers.openai.com/api/docs/models/gpt-6-sol +const VERIFIED_ACCOUNT_OUTPUT_LIMITS: Readonly> = { + 'gpt-5.3-codex': 128_000, 'gpt-5.4': 128_000, 'gpt-5.4-2026-03-05': 128_000, + 'gpt-5.6-sol': 128_000, 'gpt-6-astra': 128_000, 'gpt-6-sol': 128_000, +}; +type JsonRecord = Record; +function isRecord(value: unknown): value is JsonRecord { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} +function isNumber(value: unknown): value is number { return typeof value === 'number'; } +function fail(message: string): never { throw new Error(`credential broker: ${message}`); } + +function upstreamHeaders(request: IncomingMessage, config: BrokerConfig): OutgoingHttpHeaders { + const headers: OutgoingHttpHeaders = { ...request.headers }; + delete headers.host; + // Request identity encoding so accounting and the client read the same bytes. + delete headers['accept-encoding']; + delete headers.authorization; + delete headers['proxy-authorization']; + delete headers['x-api-key']; + if (config.mode === 'api-key') headers['x-api-key'] = config.credential; + else headers.authorization = `Bearer ${config.credential}`; + return headers; +} + +function parseProviderRequest(body: Buffer, path: string, config: BrokerConfig): JsonRecord { + let payload: unknown; + try { payload = JSON.parse(body.toString('utf8')); } + catch { fail('request body must be valid JSON'); } + if (!isRecord(payload)) { + fail('request body must be an object'); + } + if (payload.model !== config.model) fail('request model does not match the selected model'); + if (path === '/v1/messages' + && (!isNumber(payload.max_tokens) || !Number.isInteger(payload.max_tokens) || payload.max_tokens < 1 + || payload.max_tokens > config.maxOutputTokens)) { + fail(`max_tokens must be from 1 through ${config.maxOutputTokens}`); + } + return payload; +} + +const declared = (value: unknown): boolean => + value !== undefined && value !== null && !(Array.isArray(value) && value.length === 0); + +// Token rates cannot price MCP connectors, containers, or a non-standard speed, +// and the response cannot prove they added no charge. Server tools are priced +// only when the response reports no use. Client tools carry no type or 'custom'. +function anthropicRequestPricing(payload: JsonRecord): RequestPricing { + const serverTools = payload.tools !== undefined && (!Array.isArray(payload.tools) + || payload.tools.some(tool => !isRecord(tool) || (tool.type !== undefined && tool.type !== 'custom'))); + const unpriced = firstUnpricedReason( + hasUnpricedInput([payload.messages, payload.system]) ? 'unpriced-input' : null, + payload.service_tier !== undefined && !['auto', 'standard_only'].includes(String(payload.service_tier)) ? 'service-tier' : null, + payload.speed !== undefined && payload.speed !== 'standard' ? 'speed' : null, + declared(payload.mcp_servers) ? 'mcp' : null, + declared(payload.container) ? 'container' : null); + return { unpriced, requiresUsage: serverTools ? 'server-tool' : null, + bounded: !serverTools && unpriced === null }; +} + +// Server tools run inside the provider and return these content blocks. +const serverBlock = (value: unknown): boolean => isRecord(value) && typeof value.type === 'string' + && (value.type === 'server_tool_use' || value.type === 'mcp_tool_use' || value.type.endsWith('_tool_result')); + +// The web searches a response reports, which the provider charges per search. +function anthropicWebSearches(usages: JsonRecord[]): number { + return Math.max(0, ...usages.map(usage => isRecord(usage.server_tool_use) + && Number.isSafeInteger(usage.server_tool_use.web_search_requests) ? usage.server_tool_use.web_search_requests as number : 0)); +} + +function anthropicResponseUnpriced(usages: JsonRecord[], serverBlocks: boolean): UnpricedReason | null { + const serverToolUse = serverBlocks || usages.some(usage => isRecord(usage.server_tool_use) + && Object.values(usage.server_tool_use).some(count => typeof count === 'number' && count > 0)); + const serviceTier = usages.some(usage => usage.service_tier !== undefined && usage.service_tier !== null + && usage.service_tier !== 'standard'); + return firstUnpricedReason(serverToolUse ? 'server-tool' : null, serviceTier ? 'service-tier' : null); +} + +function decodedResponseBody(body: Buffer, contentEncoding: string | string[] | undefined): Buffer { + const encodings = String(contentEncoding ?? '') + .split(',').map(value => value.trim().toLowerCase()).filter(Boolean); + let decoded = body; + for (const encoding of encodings.reverse()) { + if (encoding === 'identity') continue; + const options = { maxOutputLength: MAX_REQUEST_BYTES }; + if (encoding === 'gzip' || encoding === 'x-gzip') decoded = gunzipSync(decoded, options); + else if (encoding === 'deflate') decoded = inflateSync(decoded, options); + else if (encoding === 'br') decoded = brotliDecompressSync(decoded, options); + else throw new Error(`unsupported response encoding ${encoding}`); + } + return decoded; +} + +function responseUsage(body: Buffer, contentEncoding: string | string[] | undefined = undefined): JsonRecord | null { + const values: JsonRecord[] = []; + const tokenCount = (value: unknown): boolean => typeof value === 'number' + && Number.isSafeInteger(value) && value >= 0; + const completeUsage = (value: unknown): boolean => isRecord(value) + && tokenCount(value.input_tokens) && tokenCount(value.output_tokens); + let serverBlocks = false; + const add = (value: unknown): void => { + if (!isRecord(value)) return; + if (isRecord(value.usage)) values.push(value.usage); + if (isRecord(value.message) && isRecord(value.message.usage)) values.push(value.message.usage); + for (const blocks of [value.content, isRecord(value.message) ? value.message.content : undefined]) { + if (Array.isArray(blocks) && blocks.some(serverBlock)) serverBlocks = true; + } + if (serverBlock(value.content_block)) serverBlocks = true; + }; + let text: string; + try { text = decodedResponseBody(body, contentEncoding).toString('utf8'); } + catch { return null; } + try { + const message: unknown = JSON.parse(text); + if (!isRecord(message) || !completeUsage(message.usage)) return null; + add(message); + } catch { + let sawError = false; + let sawInitialUsage = false; + let sawFinalUsage = false; + let sawMessageStop = false; + let parseError = false; + const parser = createParser({ + maxBufferSize: MAX_REQUEST_BYTES, + onError: () => { parseError = true; }, + onEvent: ({ data }) => { + if (!data || data === '[DONE]') return; + try { + const event = JSON.parse(data); + if (isRecord(event) && event.type === 'error') sawError = true; + if (isRecord(event) && event.type === 'message_start' && isRecord(event.message)) { + sawInitialUsage = completeUsage(event.message.usage); + } + if (isRecord(event) && event.type === 'message_delta' && isRecord(event.usage)) { + sawFinalUsage = tokenCount(event.usage.output_tokens); + } + if (isRecord(event) && event.type === 'message_stop') sawMessageStop = true; + add(event); + } catch { /* Ignore non-JSON event data. */ } + }, + }); + try { parser.feed(`${text}\n\n`); } + catch { parseError = true; } + if (parseError || sawError || !sawInitialUsage || !sawFinalUsage || !sawMessageStop) return null; + } + // Missing or malformed usage must use the broker's estimate, not an exact zero. + // Delta input/cache counts and message cache fields may be absent or null. + if (values.some(value => ['input_tokens', 'output_tokens', 'cache_read_input_tokens', 'cache_creation_input_tokens'] + .some(field => value[field] != null && !tokenCount(value[field])) + || (value.cache_creation != null && (!isRecord(value.cache_creation) + || !tokenCount(value.cache_creation.ephemeral_5m_input_tokens) + || !tokenCount(value.cache_creation.ephemeral_1h_input_tokens))))) return null; + const number = (field: string): number => Math.max(0, ...values.map(value => Number(value[field]) || 0)); + const cacheWrite = (field: string): number => Math.max(0, ...values.map(value => + isRecord(value.cache_creation) ? Number(value.cache_creation[field]) || 0 : 0)); + const cacheWrite5m = cacheWrite('ephemeral_5m_input_tokens'); + const cacheWrite1h = cacheWrite('ephemeral_1h_input_tokens'); + const flatCacheWrite = number('cache_creation_input_tokens'); + return { + input_tokens: number('input_tokens'), + output_tokens: number('output_tokens'), + cache_read_input_tokens: number('cache_read_input_tokens'), + cache_creation: { + ephemeral_5m_input_tokens: cacheWrite5m + cacheWrite1h > 0 ? cacheWrite5m : flatCacheWrite, + ephemeral_1h_input_tokens: cacheWrite1h, + }, + unpriced_reason: anthropicResponseUnpriced(values, serverBlocks), + web_search_calls: anthropicWebSearches(values), + }; +} + + +// How far the broker's token rates and reservation cover a forwarded request. +export interface RequestPricing { + // A declared feature whose charge the rates cannot price. + unpriced: UnpricedReason | null; + // Pricing needs valid response usage (including evidence of unused server tools). + requiresUsage: UnpricedReason | null; + // False when server-side input or tools can exceed the request's reservation. + bounded: boolean; +} + +interface BrokerProtocol { + hostname: string; + allowedPaths: Set; + // GET paths forwarded with the credential and never billed: account and configuration reads. + readPaths?: Set; + // POST paths forwarded the same way, for a service whose account reads are POSTs. + controlPaths?: Set; + // An account check the CLI sends to another host through its HTTPS proxy. + accountLookup?: { hostname: string; path: string }; + upstreamPath(path: string): string; + billable(path: string): boolean; + headers(request: IncomingMessage): OutgoingHttpHeaders; + parseRequest(body: Buffer, path: string): JsonRecord; + requestPricing(payload: JsonRecord): RequestPricing; + inputTokenAdjustment?(payload: JsonRecord): number; + outputLimit(payload: JsonRecord): number; + responseUsage(body: Buffer, encoding?: string | string[]): JsonRecord | null; + // A call the CLI makes beside the session's own work, such as titling it. + sideCall?(path: string, payload: JsonRecord): boolean; + responseRejection?(body: Buffer, encoding?: string | string[]): Pick | null; +} + +/** + * A successful stream can carry the provider's refusal instead of a response, for example when the + * model has no capacity. A refusal before any output is a rejection, not a broken transport. + */ +function responsesRejection(body: Buffer, encoding?: string | string[]): Pick | null { + let rejection: Pick | null = null; + let produced = false; + const accept = (value: unknown): void => { + if (!isRecord(value) || rejection) return; + if (typeof value.type === 'string' && value.type.startsWith('response.output_')) produced = true; + if (value.type !== 'error' && value.type !== 'response.failed') return; + const response = isRecord(value.response) ? value.response : null; + const error = [response?.error, value.error, value].find(isRecord) ?? {}; + const code = typeof error.code === 'string' && /^[a-zA-Z0-9_.-]{1,100}$/.test(error.code) ? error.code : null; + // Only the code is kept; the message is read to recognise a capacity refusal and then dropped. + const busy = /capacity|overloaded|rate.?limit/i.test(`${code ?? ''} ${typeof error.message === 'string' ? error.message : ''}`); + rejection = { category: busy ? 'rate-limit' : 'request', code }; + }; + try { + const text = decodedResponseBody(body, encoding).toString('utf8'); + try { accept(JSON.parse(text)); } + catch { + const parser = createParser({ maxBufferSize: MAX_REQUEST_BYTES, + onEvent: ({ data }) => { try { accept(JSON.parse(data)); } catch { /* not an event object */ } } }); + parser.feed(`${text} + +`); + } + } catch { return null; } + return produced ? null : rejection; +} + +// A hosted web search appears in a Responses output as one `web_search_call` item per search, +// streamed as it finishes and listed again in the final response. +const WEB_SEARCH_TOOL = /^web_search/; +const webSearchCall = (item: unknown): item is JsonRecord => isRecord(item) && item.type === 'web_search_call'; + +function responsesUsage(body: Buffer, encoding?: string | string[], config?: BrokerConfig): JsonRecord | null { + let response: JsonRecord | null = null; + let failed = false; + let metadata: JsonRecord | null = null; + const searches = new Set(); + let streamedUnnamed = 0; + const accept = (value: unknown): void => { + if (!isRecord(value)) return; + if (value.type === 'response.output_item.done' && webSearchCall(value.item)) { + if (typeof value.item.id === 'string') searches.add(value.item.id); + else streamedUnnamed += 1; + } + if (isRecord(value.openrouter_metadata)) metadata = value.openrouter_metadata; + if (value.type === 'error' || value.type === 'response.failed') failed = true; + if ((value.type === 'response.completed' || value.type === 'response.incomplete' + || (config?.provider === 'openrouter' && value.type === 'response.done')) && isRecord(value.response)) { + response = value.response; + } else if (value.object === 'response' && (value.status === 'completed' || value.status === 'incomplete')) { + response = value; + } + }; + try { + const text = decodedResponseBody(body, encoding).toString('utf8'); + try { accept(JSON.parse(text)); } + catch { + const parser = createParser({ maxBufferSize: MAX_REQUEST_BYTES, + onError: () => { failed = true; }, + onEvent: ({ data }) => { + if (data === '[DONE]') return; + try { accept(JSON.parse(data)); } catch { failed = true; } + }, + }); + parser.feed(`${text}\n\n`); + } + } catch { return null; } + const usage = (response as JsonRecord | null)?.usage; + if (failed || !isRecord(usage)) return null; + const items = (response as JsonRecord | null)?.output; + const listed = Array.isArray(items) ? items.filter(webSearchCall) : []; + for (const item of listed) if (typeof item.id === 'string') searches.add(item.id); + const webSearchCalls = searches.size + + Math.max(streamedUnnamed, listed.filter(item => typeof item.id !== 'string').length); + const input = usage.input_tokens; + const output = usage.output_tokens; + const cached = isRecord(usage.input_tokens_details) ? usage.input_tokens_details.cached_tokens : 0; + if (![input, output, cached].every(value => typeof value === 'number' && Number.isSafeInteger(value) && value >= 0) + || (cached as number) > (input as number)) return null; + // Searches are charged per call on top of tokens; OpenRouter's reported cost includes them. + const normalized = { input_tokens: (input as number) - (cached as number), output_tokens: output, + cache_read_input_tokens: cached, cache_creation_input_tokens: 0, web_search_calls: webSearchCalls, + unpriced_reason: webSearchCalls > 0 && config?.provider !== 'openrouter' ? 'hosted-tool' : null }; + if (config?.provider !== 'openrouter') return normalized; + const final = response as JsonRecord | null; + const route = (final?.openrouter_metadata ?? metadata) as JsonRecord | null; + const selected = isRecord(route?.endpoints) && Array.isArray(route.endpoints.available) + ? route.endpoints.available.filter(endpoint => isRecord(endpoint) && endpoint.selected === true) : []; + if (typeof usage.cost !== 'number' || !Number.isFinite(usage.cost) || usage.cost < 0 + || final?.model !== config.model || !route || route.requested !== config.model + || route.strategy !== 'direct' || route.is_byok !== false || route.attempt !== 1 + || (route.pipeline !== undefined && (!Array.isArray(route.pipeline) || route.pipeline.length !== 0)) + || selected.length !== 1 || !isRecord(selected[0]) || selected[0].model !== config.model + || typeof selected[0].provider !== 'string' || !selected[0].provider || selected[0].provider.length > 128) return null; + return { ...normalized, provider_reported_cost_usd: usage.cost, upstream_provider: selected[0].provider }; +} + + +function hasUnpricedInput(value: unknown): boolean { + if (Array.isArray(value)) return value.some(hasUnpricedInput); + if (!isRecord(value)) return false; + if (['input_file', 'item_reference'].includes(String(value.type))) return true; + if (isRecord(value.source) && ['url', 'file'].includes(String(value.source.type))) return true; + return Object.values(value).some(hasUnpricedInput); +} + +function images(value: unknown): JsonRecord[] { + if (Array.isArray(value)) return value.flatMap(images); + if (!isRecord(value)) return []; + if (value.type === 'input_image') return [value]; + return Object.values(value).flatMap(images); +} + +// Only inline images on these models have a verified token bound. +function inlineImageUrl(image: JsonRecord): string | null { + return typeof image.image_url === 'string' + && /^data:image\/(png|jpeg|webp|gif);base64,[A-Za-z0-9+/]+={0,2}$/.test(image.image_url) + && image.file_id === undefined ? image.image_url : null; +} + +function boundedImageUrl(image: JsonRecord, model: string): string | null { + return ['gpt-6-astra', 'gpt-5.6-sol', 'gpt-5.6-terra', 'gpt-5.6-luna', 'gpt-5.4', 'gpt-5.4-2026-03-05'].includes(model) + ? inlineImageUrl(image) : null; +} + +// Provider receipts price actual tokens; this only bounds the reservation. +export function imageTokenAdjustment(value: unknown, model: string): number { + return images(value).reduce((sum, image) => { + const url = boundedImageUrl(image, model); + // OpenAI vision: 30,000 patches maximum x 1.2 tokens, plus rounding. + // https://developers.openai.com/api/docs/guides/images-vision + // Replace base64 text bytes rather than charging for both representations. + // An unbounded inline image is not text. Its visual tokens are unknown before usage; + // do not reserve its base64 bytes as if they were priced text tokens. + const inline = inlineImageUrl(image); + return url === null ? sum - (inline === null ? 0 : Buffer.byteLength(inline, 'utf8')) + : sum + 36_001 - Buffer.byteLength(url, 'utf8'); + }, 0); +} + +// A declared web search is priced when the response shows it unused. +const declaresWebSearch = (payload: JsonRecord): boolean => Array.isArray(payload.tools) + && payload.tools.some(tool => isRecord(tool) && WEB_SEARCH_TOOL.test(String(tool.type))); + +// Responses usage covers tokens only. A requested tier, hosted tools, files, +// remote images, non-text output, and stored context can add charges or +// input that the broker neither prices nor reserves. +function responsesRequestFeature(payload: JsonRecord): UnpricedReason | null { + return firstUnpricedReason( + payload.service_tier !== undefined && payload.service_tier !== 'default' && payload.service_tier !== 'auto' + ? 'service-tier' : null, + payload.tools !== undefined && (!Array.isArray(payload.tools) + || payload.tools.some(tool => !isRecord(tool) + || (!['function', 'custom'].includes(String(tool.type)) && !WEB_SEARCH_TOOL.test(String(tool.type))))) + ? 'hosted-tool' : null, + hasUnpricedInput(payload.input) || images(payload.input).some(image => inlineImageUrl(image) === null) + ? 'unpriced-input' : null, + payload.image_config !== undefined || payload.audio !== undefined + || (payload.modalities !== undefined && (!Array.isArray(payload.modalities) + || payload.modalities.some(modality => modality !== 'text'))) ? 'unpriced-output' : null, + payload.prompt || payload.previous_response_id || payload.conversation ? 'stored-context' : null); +} + +export function brokerHostname({ provider, mode, upstream }: Pick): string { + if (upstream) return ANTHROPIC_UPSTREAMS[upstream].hostname; + if (!provider || provider === 'anthropic') return 'api.anthropic.com'; + if (provider === 'openrouter') return 'openrouter.ai'; + if (provider === 'google') return mode === 'subscription-token' ? ANTIGRAVITY_ACCOUNT_HOST : 'generativelanguage.googleapis.com'; + if (provider === 'xai') return mode === 'subscription-token' ? 'cli-chat-proxy.grok.com' : 'api.x.ai'; + return mode === 'subscription-token' ? 'chatgpt.com' : 'api.openai.com'; +} + +export function brokerProtocol(config: BrokerConfig): BrokerProtocol { + if (!config.provider || config.provider === 'anthropic') return { + hostname: brokerHostname(config), + allowedPaths: new Set(['/v1/messages', '/v1/messages/count_tokens']), + upstreamPath: path => (config.upstream ? ANTHROPIC_UPSTREAMS[config.upstream].pathPrefix : '') + path, + billable: path => path === '/v1/messages', + headers: request => { + const headers = upstreamHeaders(request, config); + // The makers' endpoints take the key as a bearer token, as their Claude Code guides set it. + if (config.upstream) { + delete headers['x-api-key']; + headers.authorization = `Bearer ${config.credential}`; + } + return headers; + }, + parseRequest: (body, path) => parseProviderRequest(body, path, config), + requestPricing: anthropicRequestPricing, + outputLimit: payload => payload.max_tokens as number, + responseUsage, + }; + if (config.provider === 'xai') return grokProtocol(config); + if (config.provider === 'google') return config.mode === 'subscription-token' ? googleAccountProtocol(config) : googleProtocol(config); + const router = config.provider === 'openrouter'; + if (router && (!/^[a-z0-9._-]+\/[a-zA-Z0-9._-]+$/.test(config.model) + || config.model.startsWith('openrouter/') || /(?:^|[-/])latest$/.test(config.model))) { + fail('OpenRouter requires one explicit model without routing variants'); + } + const account = config.mode === 'subscription-token'; + if (account && !config.accountId) fail('OpenAI account identity is required'); + // API requests enforce their configured cap; account requests reserve the + // verified model maximum because the account endpoint may ignore that cap. + const outputLimit = account + ? Object.hasOwn(VERIFIED_ACCOUNT_OUTPUT_LIMITS, config.model) + ? VERIFIED_ACCOUNT_OUTPUT_LIMITS[config.model] : config.explicitOutputLimit ? config.maxOutputTokens : undefined + : config.maxOutputTokens; + if (!outputLimit) fail('OpenAI account model has no verified output-token bound'); + return { + hostname: brokerHostname(config), + allowedPaths: new Set(['/v1/responses']), + upstreamPath: () => router ? '/api/v1/responses' : account ? '/backend-api/codex/responses' : '/v1/responses', + billable: () => true, + headers: request => { + // OpenRouter routing/auth headers are trusted configuration, not agent input. + const headers = router ? { 'content-type': 'application/json', accept: 'text/event-stream', + 'x-openrouter-metadata': 'enabled' } as OutgoingHttpHeaders : upstreamHeaders(request, config); + delete headers['x-api-key']; + for (const name of ['chatgpt-account-id', 'openai-organization', 'openai-project']) delete headers[name]; + headers.authorization = `Bearer ${config.credential}`; + if (account) headers['chatgpt-account-id'] = config.accountId; + return headers; + }, + parseRequest: body => { + const payload = JSON.parse(body.toString('utf8')); + if (!isRecord(payload) || payload.model !== config.model) fail('request model does not match'); + if (router && ['provider', 'models', 'route', 'plugins', 'transforms', 'preset', 'user', 'session_id', 'trace', 'debug'].some(key => key in payload)) { + fail('OpenRouter routing and transforms are owned by the broker'); + } + const requested = payload.max_output_tokens; + if (requested !== undefined && (!Number.isSafeInteger(requested) || (requested as number) < 1 + || (requested as number) > outputLimit)) fail('invalid max_output_tokens'); + if (!account) { + payload.max_output_tokens = requested ?? outputLimit; + // An explicitly requested tier is forwarded and recorded as unpriced. + if (payload.service_tier === undefined || payload.service_tier === 'auto') payload.service_tier = 'default'; + } + if (router) { + const rates = config.pricingRates!; + payload.provider = { only: [config.providerRoute], order: [config.providerRoute], + allow_fallbacks: false, require_parameters: true, + max_price: { prompt: rates.input, completion: rates.output, request: 0 } }; + payload.plugins = []; + payload.transforms = []; + payload.store = false; + } + return payload; + }, + // OpenRouter's reported cost includes every feature; only the reservation is affected. + requestPricing: payload => { + const feature = responsesRequestFeature(payload); + const unboundedImage = images(payload.input).some(image => boundedImageUrl(image, config.model) === null); + // Vision input tokens are included in Responses usage. A missing model-specific + // reservation bound does not make complete usage unpriced. Without usage, + // keep the unknown input flag; never treat the partial reservation as a ceiling. + // https://developers.openai.com/api/docs/guides/images-vision#calculating-costs + const search = declaresWebSearch(payload); + return { unpriced: router ? null : feature, + requiresUsage: router ? null : firstUnpricedReason(search ? 'hosted-tool' : null, unboundedImage ? 'unpriced-input' : null), + bounded: feature === null && !unboundedImage && !search }; + }, + inputTokenAdjustment: payload => imageTokenAdjustment(payload.input, config.model), + outputLimit: payload => account ? outputLimit : payload.max_output_tokens as number, + responseUsage: (body, encoding) => responsesUsage(body, encoding, config), + responseRejection: responsesRejection, + }; +} + +// Grok Build titles every session with this model and a 100-token cap, whatever model +// the session runs on. That one call is accepted and priced at the plan's rates. +const GROK_TITLE_MODEL = 'grok-4.6'; +const GROK_TITLE_OUTPUT_TOKENS = 100; + +// The reads Grok Build makes against its session service before and between model calls. +const GROK_READ_PATHS = new Set(['/v1/login-config', '/v1/user', '/v1/settings', '/v1/models', '/v1/bundle/archive', + '/v1/subagents/bundle', '/v1/feedback/config']); + +// A SuperGrok sign-in reaches the CLI chat proxy, which also needs the CLI's token marker; +// an API key reaches the public API. Both speak the Responses API. The CLI's model +// calls name no output cap: an API request gets the plan's, and an account request +// goes as sent with the plan's cap reserved, as Codex account requests do. +function grokProtocol(config: BrokerConfig): BrokerProtocol { + const account = config.mode === 'subscription-token'; + return { + hostname: brokerHostname(config), + allowedPaths: new Set(['/v1/responses']), + ...(account ? { readPaths: GROK_READ_PATHS } : {}), + upstreamPath: path => path, + billable: path => path === '/v1/responses', + headers: request => { + const headers = upstreamHeaders(request, config); + delete headers['x-api-key']; + delete headers['x-xai-token-auth']; + headers.authorization = `Bearer ${config.credential}`; + if (account) headers['x-xai-token-auth'] = 'xai-grok-cli'; + return headers; + }, + parseRequest: body => { + const payload = JSON.parse(body.toString('utf8')); + if (!isRecord(payload)) fail('request model does not match'); + const requested = payload.max_output_tokens; + const title = payload.model === GROK_TITLE_MODEL && Number.isSafeInteger(requested) + && (requested as number) <= GROK_TITLE_OUTPUT_TOKENS; + if (payload.model !== config.model && !title) fail('request model does not match'); + if (requested !== undefined && (!Number.isSafeInteger(requested) || (requested as number) < 1 + || (requested as number) > config.maxOutputTokens)) fail('invalid max_output_tokens'); + if (!account) payload.max_output_tokens = requested ?? config.maxOutputTokens; + return payload; + }, + requestPricing: payload => { + const feature = responsesRequestFeature(payload); + const search = declaresWebSearch(payload); + return { unpriced: feature, requiresUsage: search ? 'hosted-tool' : null, bounded: feature === null && !search }; + }, + outputLimit: payload => (payload.max_output_tokens as number | undefined) ?? config.maxOutputTokens, + responseUsage: (body, encoding) => responsesUsage(body, encoding, config), + responseRejection: responsesRejection, + }; +} + +const GEMINI_WEB_TOOLS = ['googleSearch', 'googleSearchRetrieval', 'urlContext']; + +// A Gemini generateContent request, checked and bounded the same way on both routes. The +// CLI makes its own helper calls on a small model, such as titling the conversation: a +// helper call carries only its known prompt, and no tools. +function geminiRequest(request: JsonRecord, config: BrokerConfig, helperPrompt: string | null): JsonRecord { + if (helperPrompt !== null && (!JSON.stringify(request.systemInstruction ?? '').includes(helperPrompt) + || request.tools !== undefined)) fail('request model does not match the selected model'); + const generation = isRecord(request.generationConfig) ? request.generationConfig : {}; + const limit = generation.maxOutputTokens; + if (limit !== undefined && (!Number.isSafeInteger(limit) || (limit as number) < 1 + || (limit as number) > config.maxOutputTokens)) fail('invalid maxOutputTokens'); + request.generationConfig = { ...generation, maxOutputTokens: limit ?? config.maxOutputTokens }; + // The CLI cannot remove a tool, so only the file, shell and task functions, and those the tool + // policy turns on, reach the model. + // Google's hosted search and URL tools are web search too. + const allowed = antigravityTools(config.tools); + if (Array.isArray(request.tools)) request.tools = request.tools.map(tool => { + if (!isRecord(tool)) return tool; + const kept = Object.fromEntries(Object.entries(tool) + .filter(([key]) => config.tools?.webSearch || !GEMINI_WEB_TOOLS.includes(key))); + return Array.isArray(kept.functionDeclarations) ? { ...kept, functionDeclarations: kept.functionDeclarations + .filter(declaration => isRecord(declaration) && allowed.includes(String(declaration.name))) } : kept; + }).filter(tool => !isRecord(tool) || Object.keys(tool).length > 0); + return request; +} + +// Grounding with Google Search is priced when the response shows no search. +function geminiPricing(request: JsonRecord): RequestPricing { + const search = Array.isArray(request.tools) && request.tools.some(tool => isRecord(tool) && tool.googleSearch !== undefined); + const unpriced = firstUnpricedReason( + Array.isArray(request.tools) && request.tools.some(tool => !isRecord(tool) + || Object.keys(tool).some(key => key !== 'functionDeclarations' && key !== 'googleSearch')) ? 'hosted-tool' : null, + request.cachedContent !== undefined ? 'stored-context' : null, + JSON.stringify(request.contents ?? []).includes('"fileData"') ? 'unpriced-input' : null, + request.serviceTier !== undefined && request.serviceTier !== 'standard' ? 'service-tier' : null); + return { unpriced, requiresUsage: search ? 'hosted-tool' : null, bounded: unpriced === null && !search }; +} + +function geminiOutputLimit(request: JsonRecord): number { + const generation = isRecord(request.generationConfig) ? request.generationConfig : {}; + const thinking = isRecord(generation.thinkingConfig) ? generation.thinkingConfig.thinkingBudget : 0; + return (generation.maxOutputTokens as number) + (typeof thinking === 'number' && thinking > 0 ? thinking : 0); +} + +function jsonObject(body: Buffer): JsonRecord { + const payload: unknown = JSON.parse(body.toString('utf8')); + return isRecord(payload) ? payload : fail('request body must be an object'); +} + +// The Gemini API as the Antigravity CLI calls it with an API key; the model is in the path. +// The title call is forwarded and priced at the selected model's rates. +function googleProtocol(config: BrokerConfig): BrokerProtocol { + const call = (model: string) => `/v1beta/models/${model}:streamGenerateContent`; + const title = call(ANTIGRAVITY_TITLE_MODEL); + return { + hostname: brokerHostname(config), + allowedPaths: new Set([call(config.model), title]), + upstreamPath: path => path, + billable: () => true, + headers: request => { + const headers = upstreamHeaders(request, config); + delete headers['x-api-key']; + delete headers.authorization; + headers['x-goog-api-key'] = config.credential; + return headers; + }, + parseRequest: (body, path) => geminiRequest(jsonObject(body), config, path === title ? ANTIGRAVITY_TITLE_PROMPT : null), + requestPricing: geminiPricing, + outputLimit: geminiOutputLimit, + responseUsage: googleUsage, + sideCall: path => path === title, + }; +} + +// Google's Code Assist service as the Antigravity CLI calls it with a Google account +// sign-in. The model is named in a wrapper around the Gemini request, and the CLI reads +// the account's plan, models and settings through POSTs that are forwarded unbilled. +// The account's own model names stand for the API models a plan selects. +function googleAccountProtocol(config: BrokerConfig): BrokerProtocol { + const models = (Object.hasOwn(ANTIGRAVITY_ACCOUNT_MODELS, config.model) ? ANTIGRAVITY_ACCOUNT_MODELS[config.model] : null) + ?? fail('no Antigravity account model is known for the selected model'); + const helperPrompt = (payload: JsonRecord): string | null => typeof payload.model === 'string' + && Object.hasOwn(ANTIGRAVITY_ACCOUNT_HELPERS, payload.model) ? ANTIGRAVITY_ACCOUNT_HELPERS[payload.model]! : null; + return { + hostname: brokerHostname(config), + allowedPaths: new Set(['/v1internal:streamGenerateContent']), + controlPaths: new Set(ANTIGRAVITY_ACCOUNT_CONTROL_PATHS), + accountLookup: ANTIGRAVITY_ACCOUNT_LOOKUP, + upstreamPath: path => path, + billable: () => true, + headers: request => upstreamHeaders(request, config), + parseRequest: body => { + const payload = jsonObject(body); + const helper = helperPrompt(payload); + if ((!models.includes(String(payload.model)) && helper === null) || !isRecord(payload.request)) { + fail('request model does not match the selected model'); + } + payload.request = geminiRequest(payload.request, config, helper); + return payload; + }, + requestPricing: payload => geminiPricing(payload.request as JsonRecord), + outputLimit: payload => geminiOutputLimit(payload.request as JsonRecord), + responseUsage: googleUsage, + sideCall: (_path, payload) => helperPrompt(payload) !== null, + }; +} + +// The last streamed chunk carries the call's usage, wrapped in `response` by Code Assist. +// Prompt tokens include cache reads; thinking is billed as output. +// A grounded response names the searches it ran in its grounding metadata. +function googleUsage(body: Buffer, encoding?: string | string[]): JsonRecord | null { + let usage: JsonRecord | null = null; + let failed = false; + let grounded = false; + const queries = new Set(); + const accept = (value: unknown): void => { + for (const event of Array.isArray(value) ? value : [value]) { + if (!isRecord(event)) continue; + const item = isRecord(event.response) ? event.response : event; + if (isRecord(event.error) || isRecord(item.error)) failed = true; + if (isRecord(item.usageMetadata)) usage = item.usageMetadata; + for (const candidate of Array.isArray(item.candidates) ? item.candidates : []) { + if (!isRecord(candidate) || !isRecord(candidate.groundingMetadata)) continue; + grounded = true; + const asked = candidate.groundingMetadata.webSearchQueries; + if (Array.isArray(asked)) for (const query of asked) queries.add(String(query)); + } + } + }; + try { + const text = decodedResponseBody(body, encoding).toString('utf8'); + try { accept(JSON.parse(text)); } + catch { + const parser = createParser({ maxBufferSize: MAX_REQUEST_BYTES, onError: () => { failed = true; }, + onEvent: ({ data }) => { try { accept(JSON.parse(data)); } catch { failed = true; } } }); + parser.feed(`${text}\n\n`); + } + } catch { return null; } + const final = usage as JsonRecord | null; + if (failed || !final) return null; + const prompt = final.promptTokenCount, cached = final.cachedContentTokenCount ?? 0; + const output = final.candidatesTokenCount ?? 0, thinking = final.thoughtsTokenCount ?? 0; + if (![prompt, cached, output, thinking].every(value => typeof value === 'number' && Number.isSafeInteger(value) && value >= 0) + || (cached as number) > (prompt as number)) return null; + return { input_tokens: (prompt as number) - (cached as number), output_tokens: (output as number) + (thinking as number), + cache_read_input_tokens: cached, cache_creation_input_tokens: 0, web_search_calls: queries.size, + unpriced_reason: grounded ? 'hosted-tool' : null }; +} diff --git a/tools/stack-bench/container/browser-network-proxy.ts b/tools/stack-bench/container/browser-network-proxy.ts new file mode 100644 index 00000000000..e41494d7e11 --- /dev/null +++ b/tools/stack-bench/container/browser-network-proxy.ts @@ -0,0 +1,297 @@ +#!/usr/bin/env node + +// Forwarding-only proxy for one interruptible browser context. It runs where the +// browser runs (inside the attempt's browser container in the appliance) and is +// controlled only over stdin/stdout: no control, status, or debug route listens. +// Commands, one JSON object per line: {id, cmd: 'config', user, pass}, then +// {id, cmd: 'cut' | 'restore' | 'dispose'}. EOF or a malformed command closes +// the listener and every owned socket. +import { createServer, get as httpGet, request as httpRequest } from 'node:http'; +import type { IncomingHttpHeaders, Server, ServerResponse } from 'node:http'; +import { connect } from 'node:net'; +import type { Socket } from 'node:net'; +import { resolve } from 'node:path'; +import { createInterface } from 'node:readline'; +import { pathToFileURL } from 'node:url'; +import { randomUUID } from 'node:crypto'; +import { StringDecoder } from 'node:string_decoder'; +import { brotliDecompressSync, gunzipSync, inflateSync, constants } from 'node:zlib'; + +// Hop-by-hop and proxy headers never reach the application. +const HOP = new Set(['proxy-authorization', 'proxy-connection', 'connection', 'keep-alive', + 'transfer-encoding', 'te', 'trailer', 'upgrade']); +const endToEnd = (headers: IncomingHttpHeaders) => + Object.fromEntries(Object.entries(headers).filter(([key]) => !HOP.has(key))); + +// A Vite reload socket carries the token its own client module embeds. +export function viteToken(authority: string, token: string): Promise { + return new Promise(done => { + let settled = false; + const finish = (valid: boolean) => { + if (settled) return; + settled = true; + clearTimeout(timer); + request.destroy(); + done(valid); + }; + const request = httpGet(`http://${authority}/@vite/client`, response => { + let body = ''; + response.setEncoding('utf8'); + response.on('data', chunk => { + body += chunk; + if (body.length > 2_000_000) finish(false); + }); + response.on('end', () => finish(response.statusCode === 200 && body.includes(`const wsToken = ${JSON.stringify(token)}`))); + }); + const timer = setTimeout(() => finish(false), 5000); + request.on('error', () => finish(false)); + }); +} + +export function startNetworkProxy(reply: (value: object) => void, exit: (code: number) => void) { + let server: Server | undefined, expected = '', cut = false, port = 0; + const limit = 8 * 1024 * 1024; + let observeHttp = false, incomplete = false, wireBytes = 0, wireChunks = 0, retainedBytes = 0, metadataBytes = 0; + const bodies = new Set(); + const pendingHttp = new Set(); + const records: { id: string; url: string; method: string; status: number; contentType: string; + body: string; complete: boolean; encoding: string; chunks: Buffer[] | null; bytes: number }[] = []; + const decoded = (record: typeof records[number]): string => { + if (!record.chunks) return record.body; + const wire = Buffer.concat(record.chunks, record.bytes); + if (!wire.length) return ''; + try { + const options = { maxOutputLength: limit, + finishFlush: record.complete ? constants.Z_FINISH : constants.Z_SYNC_FLUSH }; + const bytes = record.encoding === '' || record.encoding === 'identity' ? wire + : record.encoding === 'gzip' ? gunzipSync(wire, options) + : record.encoding === 'deflate' ? inflateSync(wire, options) + : record.encoding === 'br' ? brotliDecompressSync(wire, { maxOutputLength: limit, + finishFlush: record.complete ? constants.BROTLI_OPERATION_FINISH : constants.BROTLI_OPERATION_FLUSH }) + : null; + if (!bytes) { incomplete = true; return ''; } + const decoder = new StringDecoder('utf8'); + return decoder.write(bytes) + (record.complete ? decoder.end() : ''); + } catch { incomplete = true; return ''; } + }; + const retain = (record: typeof records[number]) => { + if (!record.chunks) return; + const body = decoded(record), size = Buffer.byteLength(body); + if (!bodies.has(body)) { + if (retainedBytes + size > limit) incomplete = true; + else { bodies.add(body); retainedBytes += size; record.body = body; } + } + wireBytes -= record.bytes; + wireChunks -= record.chunks.length; + record.chunks = null; + }; + const httpSnapshot = () => { + let size = retainedBytes; + const seen = new Set(bodies); + const snapshot = records.map(record => { + let body = record.body; + if (record.chunks) { + body = decoded(record); + if (seen.has(body)) body = ''; + else if (size + Buffer.byteLength(body) > limit) { incomplete = true; body = ''; } + else { seen.add(body); size += Buffer.byteLength(body); } + } + return { id: record.id, url: record.url, method: record.method, status: record.status, + contentType: record.contentType, body, complete: record.complete }; + }); + return { records: snapshot, incomplete, pending: pendingHttp.size }; + }; + const owned = new Map(); // socket -> the Vite token it carries, if tooling + const pending = new Map>(); + // Where each browser-side connection goes, as host:port plus path (never the query). + const targets = new Map(); + const own = (socket: Socket, target?: string) => { + if (target) targets.set(socket, target); + if (!owned.has(socket)) { + owned.set(socket, null); + socket.on('close', () => { owned.delete(socket); targets.delete(socket); pending.delete(socket); }); + socket.on('error', () => {}); + } + }; + const shutdown = (code: number) => { + for (const socket of owned.keys()) socket.destroy(); + server?.close(); + exit(code); + }; + // Requests aimed back at this listener would loop. + const loops = (host: string, target: number) => target === port && /^(127\.|localhost$|\[?::1\]?$)/.test(host); + + const listen = () => new Promise(ready => { + server = createServer((req, res) => { + if (req.headers['proxy-authorization'] !== expected) { + res.writeHead(407, { 'Proxy-Authenticate': 'Basic realm="stack-bench"', Connection: 'close' }).end(); + return; + } + if (cut) { req.socket.destroy(); return; } + let target: URL; + try { target = new URL(req.url ?? ''); } catch { res.writeHead(400).end(); return; } + if (target.protocol !== 'http:' || loops(target.hostname, Number(target.port || 80))) { + res.writeHead(400).end(); + return; + } + if (observeHttp) { + pendingHttp.add(res); + const settled = () => pendingHttp.delete(res); + res.once('finish', settled); + res.once('close', settled); + res.once('error', settled); + } + own(req.socket, `${target.hostname}:${target.port || 80}${target.pathname}`); + const upstream = httpRequest(target, { method: req.method, headers: endToEnd(req.headers) }, answer => { + const headers = endToEnd(answer.headers); + if (observeHttp) { + const record = { id: randomUUID(), url: target.href, method: req.method ?? 'GET', + status: answer.statusCode ?? 502, contentType: answer.headers['content-type'] ?? '', + body: '', complete: false, encoding: (answer.headers['content-encoding'] ?? '').trim().toLowerCase(), + chunks: [] as Buffer[] | null, bytes: 0 }; + // Native EventSource is observed without waiting for stream closure. + // The browser observer separately rejects fetch-based SSE capture. + if (/text\/event-stream/i.test(record.contentType)) pendingHttp.delete(res); + metadataBytes += Buffer.byteLength(JSON.stringify({ ...record, chunks: undefined })); + // Overwrite any application-supplied receipt. An unretained ID cannot + // be matched to a successful snapshot by the observer. + headers['x-stack-bench-capture'] = record.id; + if (metadataBytes > limit) incomplete = true; + else { + records.push(record); + // Match the existing body observer: binary images remain uninspected, + // while scripts, styles and even mislabeled HTML fonts are retained. + const textual = /(application\/(json|[^;]+\+json|x-ndjson|(?:x-)?(?:java|ecma)script)|text\/(plain|html|css|(?:java|ecma)script))/i.test(record.contentType); + const write = res.write.bind(res); + res.write = (chunk, encodingOrCallback?, callback?) => { + const live = !res.destroyed && !res.writableEnded; + const forwarded = Reflect.apply(write, res, [chunk, encodingOrCallback, callback]) as boolean; + if (live && textual && record.chunks) { + const bytes = typeof chunk === 'string' ? Buffer.from(chunk, + typeof encodingOrCallback === 'string' ? encodingOrCallback : 'utf8') : Buffer.from(chunk); + const available = Math.max(0, limit - wireBytes); + // Bound Buffer/array overhead as well as payload bytes when a + // response deliberately writes many one-byte chunks. + if (bytes.length > available || wireChunks >= limit / 64) incomplete = true; + if (bytes.length && available && wireChunks < limit / 64) { + const copy = Buffer.from(bytes.subarray(0, available)); + record.chunks.push(copy); record.bytes += copy.length; wireBytes += copy.length; wireChunks++; + } + } + return forwarded; + }; + res.on('finish', () => { record.complete = answer.complete; retain(record); }); + res.on('close', () => retain(record)); + } + } + res.writeHead(answer.statusCode ?? 502, headers); + answer.pipe(res); + }); + upstream.on('socket', socket => own(socket)); + upstream.on('error', () => res.destroy()); + req.pipe(upstream); + }); + server.headersTimeout = 10_000; + server.maxHeadersCount = 100; + server.on('connect', (req, client: Socket, head: Buffer) => { + client.on('error', () => {}); + if (req.headers['proxy-authorization'] !== expected) { + client.end('HTTP/1.1 407 Proxy Authentication Required\r\nProxy-Authenticate: Basic realm="stack-bench"\r\n' + + 'Content-Length: 0\r\nConnection: close\r\n\r\n'); + return; + } + if (cut) { client.destroy(); return; } + const match = /^([^:\s[\]]+|\[[0-9a-fA-F:.]+\]):(\d{1,5})$/.exec(req.url ?? ''); + const host = match?.[1]?.replace(/^\[|\]$/g, '') ?? '', targetPort = Number(match?.[2]); + if (!match || targetPort < 1 || targetPort > 65535 || loops(host, targetPort)) { + client.end('HTTP/1.1 400 Bad Request\r\n\r\n'); + return; + } + own(client, req.url); + const upstream = connect(targetPort, host); + own(upstream); + upstream.on('error', () => client.destroy()); + upstream.on('connect', () => { + client.write('HTTP/1.1 200 Connection Established\r\n\r\n'); + // Inspect a copy. Forwarding cannot wait for the asynchronous Vite check. + let prefix = '', inspecting = true; + const inspect = (chunk: Buffer) => { + if (!inspecting) return; + prefix += chunk.toString('latin1', 0, Math.min(chunk.length, 4096 - prefix.length)); + const end = prefix.indexOf('\r\n'); + if (end < 0 && prefix.length < 4096) return; + inspecting = false; + const line = end < 0 ? '' : prefix.slice(0, end); + const path = /^GET (\S+) HTTP\/1\.1$/.exec(line)?.[1]; + if (path?.startsWith('/')) targets.set(client, `${req.url}${path.split('?')[0]}`); + const token = path?.startsWith('/') ? new URL(path, 'http://tunnel').searchParams.get('token') : null; + if (token) { + const lookup = viteToken(req.url!, token).then(tooling => { + if (tooling && !client.destroyed && !upstream.destroyed) { + owned.set(client, token); owned.set(upstream, token); + } + }).finally(() => { pending.delete(client); pending.delete(upstream); }); + pending.set(client, lookup); pending.set(upstream, lookup); + } + }; + client.on('data', inspect); + if (head.length) { inspect(head); upstream.write(head); } + client.pipe(upstream); + upstream.pipe(client); + }); + }); + server.listen(0, '127.0.0.1', () => { port = (server!.address() as { port: number }).port; ready(port); }); + }); + + let queue = Promise.resolve(); + const handle = async (line: string) => { + let command: { id?: unknown; cmd?: unknown; user?: unknown; pass?: unknown; observeHttp?: unknown }; + try { command = JSON.parse(line); } catch { return shutdown(2); } + const { id, cmd } = command ?? {}; + if (cmd === 'config' && !server && typeof command.user === 'string' && typeof command.pass === 'string') { + observeHttp = command.observeHttp === true; + expected = `Basic ${Buffer.from(`${command.user}:${command.pass}`).toString('base64')}`; + return reply({ id, ok: true, port: await listen() }); + } + if (!server) return shutdown(2); + if (cmd === 'httpSnapshot' && observeHttp) return reply({ id, ok: true, ...httpSnapshot() }); + if (cmd === 'cut') { + // Refuse first, then close, so nothing races through the cut. + cut = true; + const tooling = new Set(), closed: string[] = []; + const awaiting = [...pending.keys()]; + for (const socket of awaiting) socket.pause(); + for (const [socket, token] of owned) { + if (pending.has(socket)) continue; + if (token) { tooling.add(token); continue; } + socket.destroy(); + const target = targets.get(socket); + if (target) closed.push(target); + } + await Promise.all(new Set(awaiting.map(socket => pending.get(socket)).filter(p => p !== undefined))); + for (const socket of awaiting) { + if (socket.destroyed) continue; + const token = owned.get(socket); + if (token) { tooling.add(token); socket.resume(); continue; } + socket.destroy(); + const target = targets.get(socket); + if (target) closed.push(target); + } + return reply({ id, ok: true, closed, tooling: [...tooling] }); + } + if (cmd === 'restore') { cut = false; return reply({ id, ok: true }); } + if (cmd === 'dispose') { reply({ id, ok: true }); return shutdown(0); } + return shutdown(2); + }; + return (line: string) => { queue = queue.then(() => handle(line)); }; +} + +function main(): void { + const receive = startNetworkProxy(value => process.stdout.write(`${JSON.stringify(value)}\n`), + code => process.exit(code)); + const input = createInterface({ input: process.stdin }); + input.on('line', receive); + input.on('close', () => receive('dispose-on-eof')); +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) main(); diff --git a/tools/stack-bench/container/browser-pipe.ts b/tools/stack-bench/container/browser-pipe.ts new file mode 100644 index 00000000000..84ef33fc8fa --- /dev/null +++ b/tools/stack-bench/container/browser-pipe.ts @@ -0,0 +1,63 @@ +#!/usr/bin/env node + +import { spawn } from 'node:child_process'; +import { Socket } from 'node:net'; +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { chromium } from 'playwright'; +import type { LaunchOptions } from 'playwright'; +import { compiledEntrypoint } from '../src/package-root.js'; +import { readBackendLease } from '../src/runtime/backend-lease.js'; + +export function browserContainer(): string | null { + const path = process.env.STACK_BENCH_LEASE; + if (!path) { + if (process.env.STACK_BENCH_APPLIANCE === '1') throw new Error('browser requires a private attempt lease'); + return null; + } + const token = process.env.STACK_BENCH_LEASE_TOKEN; + if (!token) throw new Error('browser requires the attempt ownership token'); + const lease = readBackendLease(path, { token }); + if (lease.backend === 'stub') return null; + const container = lease.resources.browserContainer; + if (!container?.owned || container.running === false || !/^[a-f0-9]{64}$/.test(container.id)) { + throw new Error('browser requires the exact running attempt browser container'); + } + return container.id; +} + +export function attemptBrowserLaunchOptions(): LaunchOptions { + return browserContainer() ? { + executablePath: compiledEntrypoint('container', 'browser-pipe.js'), + // The owned browser has private shared memory; do not fill /tmp with IPC buffers. + ignoreDefaultArgs: ['--disable-dev-shm-usage'], + } : {}; +} + +// Playwright uses fd 3/4. Docker carries these bytes over stdin/stdout, so no +// browser control socket is reachable from the generated app's network. +function main(): void { + const id = browserContainer(); + if (!id) throw new Error('browser pipe requires an isolated attempt'); + const child = spawn('docker', ['exec', '-i', id, 'sh', '-c', + 'exec 3<&0 4>&1 1>&2; exec "$@"', 'sh', chromium.executablePath(), ...process.argv.slice(2)], + { stdio: ['pipe', 'pipe', 'inherit'] }); + const input = new Socket({ fd: 3, readable: true, writable: false }); + const output = new Socket({ fd: 4, readable: false, writable: true }); + input.pipe(child.stdin); + child.stdout.pipe(output); + const close = () => { input.destroy(); child.stdin.end(); }; + process.on('SIGTERM', close); + process.on('SIGINT', close); + child.stdin.on('error', error => { + if ('code' in error && error.code === 'EPIPE') close(); + else throw error; + }); + child.on('error', error => { console.error(error.message); process.exit(1); }); + child.on('exit', () => { input.destroy(); child.stdin.destroy(); }); + // fd 3 can still have a pending read while Playwright waits for this process + // to exit. Docker's close event means all browser output has been forwarded. + child.on('close', code => { process.exit(code ?? 1); }); +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) main(); diff --git a/tools/stack-bench/container/browser-tools/package-lock.json b/tools/stack-bench/container/browser-tools/package-lock.json new file mode 100644 index 00000000000..cd5ab8d8f7e --- /dev/null +++ b/tools/stack-bench/container/browser-tools/package-lock.json @@ -0,0 +1,328 @@ +{ + "name": "stack-bench-browser-tools", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "stack-bench-browser-tools", + "dependencies": { + "puppeteer-core": "25.10.0" + } + }, + "node_modules/@puppeteer/browsers": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-3.2.2.tgz", + "integrity": "sha512-q2BU4YfO9h/Wt7IcWPcggpOOqLk2Tbs1hDwolvKZrweRjy751OJBKMN9zO5bfD0pzU7X/tvKw/exQds4pM/LOg==", + "license": "Apache-2.0", + "dependencies": { + "modern-tar": "^0.8.4", + "yargs": "^18.0.0" + }, + "bin": { + "browsers": "lib/main-cli.js" + }, + "engines": { + "node": ">=22.12.0" + }, + "peerDependencies": { + "proxy-agent": ">=8.0.1", + "yauzl": "^2.10.0 || ^3.4.0" + }, + "peerDependenciesMeta": { + "proxy-agent": { + "optional": true + }, + "yauzl": { + "optional": true + } + } + }, + "node_modules/ansi-regex": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.3.0.tgz", + "integrity": "sha512-WpDfL7NO6j7tH88IDBNVdUJxDh9nmCteAVW9dsep846XdwF4naCBK+/tGLX3KJgcpgMRXCFlTM2hKGoK9FsdrQ==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/chromium-bidi": { + "version": "17.0.2", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-17.0.2.tgz", + "integrity": "sha512-5v9GQFhTktFvotn/OFNJBmKLKRAb6n9r0bVCwf7sHgWc3/JryK0bj1nn93L3pHFrfgcsu6Be6EWsDi+1XHTGDg==", + "license": "Apache-2.0", + "dependencies": { + "mitt": "^3.0.1", + "zod": "^3.24.1" + }, + "engines": { + "node": ">=20.19.0 <22.0.0 || >=22.12.0" + }, + "peerDependencies": { + "devtools-protocol": "*" + } + }, + "node_modules/cliui": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-9.0.1.tgz", + "integrity": "sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==", + "license": "ISC", + "dependencies": { + "string-width": "^7.2.0", + "strip-ansi": "^7.1.0", + "wrap-ansi": "^9.0.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/cliui/node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/devtools-protocol": { + "version": "0.0.1666840", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1666840.tgz", + "integrity": "sha512-gCcO42XCHKEs7Ag0S7aGYsnJ7hlgrO3qderYqeiY0Eqk+0GFfuvT13IA0hHreJTa2KCdDVyGMeOhdMNmrrTjVg==", + "license": "BSD-3-Clause" + }, + "node_modules/emoji-regex": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz", + "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", + "license": "MIT" + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-east-asian-width": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.6.0.tgz", + "integrity": "sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mitt": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", + "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==", + "license": "MIT" + }, + "node_modules/modern-tar": { + "version": "0.8.5", + "resolved": "https://registry.npmjs.org/modern-tar/-/modern-tar-0.8.5.tgz", + "integrity": "sha512-snEhs+6G5Tjd4I7tLCDOaoln2RgE0bD19RzEKgvgK2hZ5VKy3MpLhLTZ2fWpXSTg4K2cyPwp+VHATFJhxfnOeA==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/puppeteer-core": { + "version": "25.10.0", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-25.10.0.tgz", + "integrity": "sha512-Hy5eMQshOEMil4JUUx03h5pw1HYkYCso1RG/gcpPlFSd4cYPOcopxcXEAxpLPOkOPJb9LIJtwxuj66bSdvknFg==", + "license": "Apache-2.0", + "dependencies": { + "@puppeteer/browsers": "3.2.2", + "chromium-bidi": "17.0.2", + "devtools-protocol": "0.0.1666840", + "typed-query-selector": "^2.12.2", + "webdriver-bidi-protocol": "0.4.3", + "ws": "^8.21.3" + }, + "engines": { + "node": ">=22.12.0" + } + }, + "node_modules/string-width": { + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.2.2.tgz", + "integrity": "sha512-GaPUh5gfdrYzqeVNZvUfT23vYYxXzKYidUcnMtJg/3rxRV63EFZy3k6xfKlmfeJD0176lnUV/Usr3XcwSvFzpg==", + "license": "MIT", + "dependencies": { + "get-east-asian-width": "^1.5.0", + "strip-ansi": "^7.1.2" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/typed-query-selector": { + "version": "2.12.2", + "resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.2.tgz", + "integrity": "sha512-EOPFbyIub4ngnEdqi2yOcNeDLaX/0jcE1JoAXQDDMIthap7FoN795lc/SHfIq2d416VufXpM8z/lD+WRm2gfOQ==", + "license": "MIT" + }, + "node_modules/webdriver-bidi-protocol": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.4.3.tgz", + "integrity": "sha512-uuN0goWfxP22B7J/uAgBpOYNPttC+XVseYE+rSY5+rQ+YBeVz/VORw8WbmLVcqW78zNg5A4qnjNXYUWR3il2ig==", + "license": "Apache-2.0" + }, + "node_modules/wrap-ansi": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.2.tgz", + "integrity": "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.1", + "string-width": "^7.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ws": { + "version": "8.21.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz", + "integrity": "sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "18.1.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-18.1.0.tgz", + "integrity": "sha512-2rAgRKu54VsHkqI0/tYkmluGXHD4KW7yZoycuqDQ15QOTnc2VVfy0nN/1eMhnQLO00A+dwtK20xuCnc1YGeUyg==", + "license": "MIT", + "dependencies": { + "cliui": "^9.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "string-width": "^8.2.1", + "y18n": "^5.0.5", + "yargs-parser": "^22.0.0" + }, + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=23" + } + }, + "node_modules/yargs-parser": { + "version": "22.0.0", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-22.0.0.tgz", + "integrity": "sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==", + "license": "ISC", + "engines": { + "node": "^20.19.0 || ^22.12.0 || >=23" + } + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/tools/stack-bench/container/browser-tools/package.json b/tools/stack-bench/container/browser-tools/package.json new file mode 100644 index 00000000000..5d7eeecaa9c --- /dev/null +++ b/tools/stack-bench/container/browser-tools/package.json @@ -0,0 +1 @@ +{"name":"stack-bench-browser-tools","private":true,"dependencies":{"puppeteer-core":"25.10.0"}} diff --git a/tools/stack-bench/container/build-container-inspection.ts b/tools/stack-bench/container/build-container-inspection.ts new file mode 100644 index 00000000000..22605d0a3f5 --- /dev/null +++ b/tools/stack-bench/container/build-container-inspection.ts @@ -0,0 +1,228 @@ +import { spawnSync } from 'node:child_process'; +import type { SpawnSyncOptionsWithStringEncoding, SpawnSyncReturns } from 'node:child_process'; +import { resolve } from 'node:path'; + +import { LEGACY_SUBSCRIPTION_TOKEN_TARGET } from './container-auth.js'; +import type { ContainerMount } from '../src/runtime/container-mount.js'; +import { redactCredentials } from '../src/evidence/diagnostic-sanitizer.js'; + +type InspectedMount = { + type: string; + source: string; + name: string | null; + destination: string; + readOnly: boolean; +}; + +export type InspectedBuildContainer = { + id: string; + image: string; + running: boolean; + networkMode: string | null; + readonlyRootfs: boolean; + tmpfs: Record; + capAdd: string[]; + capDrop: string[]; + securityOpt: string[]; + pidsLimit: number | null; + nanoCpus: number | null; + memoryBytes: number | null; + memorySwapBytes: number | null; + mounts: InspectedMount[]; + unsafeCredentialExposure: boolean; +}; + +type DockerExecute = (command: string, args: readonly string[], + options: SpawnSyncOptionsWithStringEncoding) => SpawnSyncReturns; + +type JsonRecord = Record; + +function isRecord(value: unknown): value is JsonRecord { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function stringArray(value: unknown): string[] { + return Array.isArray(value) ? value.filter((item): item is string => typeof item === 'string') : []; +} + +function numberOrNull(value: unknown): number | null { + return typeof value === 'number' ? value : null; +} + +function dockerDetail(result: SpawnSyncReturns): string { + return String(result.stderr || result.stdout || result.error?.message || `exit ${result.status}`).trim(); +} + +export function waitForBuildContainerReady(id: string, readyFile: string, description: string, { + env = process.env, + execute = spawnSync as DockerExecute, +}: { env?: NodeJS.ProcessEnv; execute?: DockerExecute } = {}): void { + const options = { encoding: 'utf8' as const, env, timeout: 10_000 }; + try { + const deadline = Date.now() + 90_000; + while (Date.now() < deadline) { + const probe = execute('docker', ['exec', id, 'test', '-f', readyFile], options); + if (probe.status === 0) return; + const inspection = execute('docker', ['inspect', '--format', '{{json .State}}', id], options); + if (inspection.status !== 0) throw new Error(`cannot inspect build container ${id}: ${dockerDetail(inspection)}`); + const state = JSON.parse(inspection.stdout) as { Status: string; ExitCode: number; OOMKilled: boolean }; + if (state.Status === 'exited' || state.Status === 'dead') { + throw new Error(`build container ${id} ${state.Status} before ${description}; ` + + `exit code ${state.ExitCode}; OOMKilled=${state.OOMKilled}`); + } + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 500); + } + throw new Error(`timed out waiting for ${description}`); + } catch (error) { + const logs = execute('docker', ['logs', '--tail', '100', id], options); + throw new Error(redactCredentials(`${error instanceof Error ? error.message : String(error)}\n` + + `${logs.stdout || ''}${logs.stderr || ''}`)); + } +} + +export function parseCgroupResources(value: string) { + const count = (name: string, key?: string): number | null => { + const section = (value.split(`[${name}]`)[1]?.split('[')[0] ?? '').replaceAll('\r', ''); + const match = section.match(new RegExp(key ? `^${key} (\\d+)$` : '^(\\d+)$', 'm')); + if (!match) return null; + const parsed = Number(match[1]); + return Number.isSafeInteger(parsed) ? parsed : null; + }; + return { + buildContainerMemory: { + currentBytes: count('memory.current'), + peakBytes: count('memory.peak'), + limitBytes: count('memory.max'), + oomEvents: count('memory.events', 'oom'), + oomKillEvents: count('memory.events', 'oom_kill'), + }, + buildContainerPids: { + current: count('pids.current'), + peak: count('pids.peak'), + limit: count('pids.max'), + limitEvents: count('pids.events', 'max'), + }, + }; +} + +export function inspectBuildContainer(name: string, { + env = process.env, + timeoutMs = 120_000, + execute = spawnSync as DockerExecute, +}: { env?: NodeJS.ProcessEnv; timeoutMs?: number; execute?: DockerExecute } = {}): InspectedBuildContainer | null { + const result = execute('docker', ['inspect', name], { encoding: 'utf8', env, timeout: timeoutMs }); + if (result.status !== 0) { + const detail = dockerDetail(result); + if (/no such (?:object|container)/i.test(detail)) return null; + throw new Error(`cannot inspect build container ${name}: ${detail}`); + } + + let parsed: unknown; + try { parsed = JSON.parse(result.stdout); } + catch (error) { + throw new Error(`Docker returned invalid inspection JSON for ${name}: ${error instanceof Error + ? error.message : String(error)}`); + } + if (!Array.isArray(parsed) || !isRecord(parsed[0])) { + throw new Error(`Docker returned an invalid container inspection for ${name}`); + } + + const inspected = parsed[0]; + const mounts = Array.isArray(inspected.Mounts) ? inspected.Mounts.filter(isRecord) : []; + const config = isRecord(inspected.Config) ? inspected.Config : {}; + const hostConfig = isRecord(inspected.HostConfig) ? inspected.HostConfig : {}; + const state = isRecord(inspected.State) ? inspected.State : {}; + const sensitiveTargets = new Set([LEGACY_SUBSCRIPTION_TOKEN_TARGET, '/root/.claude/.credentials.json', + '/root/.codex/auth.json', '/home/developer/.codex/auth.json']); + const capabilities = (values: unknown): string[] => stringArray(values).map(value => value.replace(/^CAP_/, '')); + const tmpfs = isRecord(hostConfig.Tmpfs) + ? Object.fromEntries(Object.entries(hostConfig.Tmpfs) + .filter((entry): entry is [string, string] => typeof entry[1] === 'string')) : {}; + + return { + id: String(inspected.Id), + image: String(inspected.Image), + running: state.Running === true, + networkMode: typeof hostConfig.NetworkMode === 'string' ? hostConfig.NetworkMode : null, + readonlyRootfs: hostConfig.ReadonlyRootfs === true, + tmpfs, + capAdd: capabilities(hostConfig.CapAdd), + capDrop: capabilities(hostConfig.CapDrop), + securityOpt: stringArray(hostConfig.SecurityOpt).map(option => option.replace(/:true$/, '')), + pidsLimit: numberOrNull(hostConfig.PidsLimit), + nanoCpus: numberOrNull(hostConfig.NanoCpus), + memoryBytes: numberOrNull(hostConfig.Memory), + memorySwapBytes: numberOrNull(hostConfig.MemorySwap), + mounts: mounts.map(mount => ({ + type: String(mount.Type), + source: String(mount.Source), + name: typeof mount.Name === 'string' ? mount.Name : null, + destination: String(mount.Destination), + readOnly: mount.RW !== true, + })), + unsafeCredentialExposure: mounts.some(mount => sensitiveTargets.has(String(mount.Destination))) + || stringArray(config.Env).some(value => /^(?:ANTHROPIC_API_KEY|CLAUDE_CODE_OAUTH_TOKEN|OPENAI_API_KEY|OPENROUTER_API_KEY|CODEX_AUTH_FILE)=/.test(value)), + }; +} + +export function sameHostPath(left: string, right: string, + platform: NodeJS.Platform = process.platform): boolean { + const normalize = (value: string): string => resolve(value).replaceAll('\\', '/'); + const normalizedLeft = normalize(left); + const normalizedRight = normalize(right); + return platform === 'win32' + ? normalizedLeft.toLowerCase() === normalizedRight.toLowerCase() + : normalizedLeft === normalizedRight; +} + +export function parsePublishedPorts(value: string | undefined): string[] { + if (!value) return []; + const ports = value.split(',').map(port => port.trim()).filter(Boolean); + if (ports.some(port => !/^\d+$/.test(port) || Number(port) < 1 || Number(port) > 65_535)) { + throw new Error('--ports must contain integers from 1 through 65535'); + } + if (new Set(ports).size !== ports.length) throw new Error('--ports must not contain duplicates'); + return ports; +} + +export function hasRequiredBuildContainerIsolation(container: InspectedBuildContainer, { + expectedMounts, + requiredTmpfs, + requiredCapabilities, + pidsLimit, + cpuCount, + memoryBytes, + memorySwapBytes, + image, +}: { + expectedMounts: ContainerMount[]; + requiredTmpfs: Readonly>; + requiredCapabilities: readonly string[]; + pidsLimit: number; + cpuCount: number; + memoryBytes: number; + memorySwapBytes: number; + image: string; +}): boolean { + const mountsMatch = container.mounts.length === expectedMounts.length + && expectedMounts.every(expected => container.mounts.some(actual => + actual.type === (expected.kind ?? 'bind') + && actual.destination === expected.target + && actual.readOnly === expected.readOnly + && (expected.kind === 'volume' + ? actual.name === expected.source + : sameHostPath(actual.source, expected.source)))); + return container.readonlyRootfs + && Object.entries(requiredTmpfs).every(([path, options]) => container.tmpfs[path] === options) + && Object.keys(container.tmpfs).length === Object.keys(requiredTmpfs).length + && requiredCapabilities.every(capability => container.capAdd.includes(capability)) + && container.capAdd.length === requiredCapabilities.length + && container.capDrop.includes('ALL') + && container.securityOpt.includes('no-new-privileges') + && container.pidsLimit === pidsLimit + && container.nanoCpus === cpuCount * 1_000_000_000 + && container.memoryBytes === memoryBytes + && container.memorySwapBytes === memorySwapBytes + && container.image === image + && mountsMatch; +} diff --git a/tools/stack-bench/container/build-linux-cli.sh b/tools/stack-bench/container/build-linux-cli.sh new file mode 100644 index 00000000000..8ce82bb007f --- /dev/null +++ b/tools/stack-bench/container/build-linux-cli.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Export native binaries through the same Docker build used by the appliance. +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO="$(cd "$HERE/../../.." && pwd)" +MSYS_NO_PATHCONV=1 docker build --platform linux/amd64 \ + --file "$REPO/tools/stack-bench/appliance/Controller.Dockerfile" \ + --target binary-export --output "type=local,dest=$HERE" "$REPO" diff --git a/tools/stack-bench/container/claude-transcript-reader.ts b/tools/stack-bench/container/claude-transcript-reader.ts new file mode 100644 index 00000000000..604c3c3d610 --- /dev/null +++ b/tools/stack-bench/container/claude-transcript-reader.ts @@ -0,0 +1,54 @@ +import { execFileSync } from 'node:child_process'; +import { isAbsolute, join, relative, sep } from 'node:path'; +import type { ClaudeTranscriptReader } from '../src/agents/claude-terminal-recovery.js'; +import { CODING_CONTAINER_AGENT, codingContainerAgentExecOptions } + from '../src/runtime/coding-container-policy.js'; + +// Read as the transcript owner. Claude creates private files while the controller +// has no DAC override; the final transcript handback cannot serve a live reader. +export const CONTAINER_CLAUDE_TRANSCRIPT_READ = ` +const fs = require('node:fs'), path = require('node:path'); +const [root, name, offset, count] = process.argv.slice(1); +if (name === '') { + const files = fs.readdirSync(root, { recursive: true, withFileTypes: true }) + .filter(entry => entry.isFile() && entry.name.endsWith('.jsonl')) + .map(entry => path.join(entry.parentPath, entry.name)) + .map(file => [path.relative(root, file), fs.statSync(file).size, fs.statSync(file).mtimeMs]); + process.stdout.write(JSON.stringify(files)); +} else { + const file = path.resolve(root, name), resolvedRoot = fs.realpathSync(root); + if (!file.endsWith('.jsonl') || !fs.realpathSync(file).startsWith(resolvedRoot + path.sep)) { + throw new Error('transcript is outside the attempt directory'); + } + const start = Number(offset), length = Number(count); + if (!Number.isSafeInteger(start) || start < 0 || !Number.isSafeInteger(length) || length < 0) { + throw new Error('invalid transcript range'); + } + const buffer = Buffer.alloc(length), fd = fs.openSync(file, 'r'); + try { process.stdout.write(buffer.subarray(0, fs.readSync(fd, buffer, 0, length, start))); } + finally { fs.closeSync(fd); } +} +`; + +export function containerClaudeTranscriptReader(containerId: string, directory: string, + env: NodeJS.ProcessEnv): ClaudeTranscriptReader { + if (!/^[a-f0-9]{64}$/.test(containerId)) throw new Error('transcript reader requires an exact container ID'); + const root = `${CODING_CONTAINER_AGENT.home}/.claude/projects/-app`; + const read = (name: string, start = 0, length = 0): Buffer => execFileSync('docker', [ + 'exec', ...codingContainerAgentExecOptions(), containerId, 'node', '-e', + CONTAINER_CLAUDE_TRANSCRIPT_READ, root, name, String(start), String(length), + ], { env, timeout: 5_000, maxBuffer: 256 * 1024 * 1024 }); + return { + snapshot() { + const entries: [string, number][] = JSON.parse(read('').toString('utf8')); + return new Map(entries.map(([name, size]) => [join(directory, name), size])); + }, + read(path, start, length) { + const name = relative(directory, path); + if (!name || isAbsolute(name) || name.split(sep).includes('..')) { + throw new Error('transcript is outside the attempt directory'); + } + return read(name.split(sep).join('/'), start, length); + }, + }; +} diff --git a/tools/stack-bench/container/coding-providers.ts b/tools/stack-bench/container/coding-providers.ts new file mode 100644 index 00000000000..3a65d49c746 --- /dev/null +++ b/tools/stack-bench/container/coding-providers.ts @@ -0,0 +1,185 @@ +import { appendFileSync, existsSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { codexArguments, codexTranscriptDirectory, parseCodexResult, runCodexProcess } + from '../src/agents/codex-protocol.js'; +import type { CodexUsage } from '../src/agents/codex-protocol.js'; +import { claudeRatesForModel } from '../src/evidence/claude-usage-cost.js'; +import { runTranscriptAwareProcess } from '../src/agents/claude-terminal-recovery.js'; +import type { PricingRates } from '../src/evidence/pricing-authority.js'; +import { unavailableToolsReason } from '../src/agents/tool-policy.js'; +import type { ToolPolicy } from '../src/agents/tool-policy.js'; +import { containerClaudeTranscriptReader } from './claude-transcript-reader.js'; +import { CODING_CONTAINER_AGENT, CODING_CONTAINER_APP_ROOT } from '../src/runtime/coding-container-policy.js'; +import { validateAntigravityNativeSession, validateClaudeNativeSession, validateCodexNativeSession, validateGrokNativeSession } + from '../src/agents/native-session-validation.js'; +import { grokArguments, grokTranscriptDirectory, parseGrokResult } from '../src/agents/grok-protocol.js'; +import { ANTIGRAVITY_LAUNCHER, antigravityArguments, antigravityFinishedAnswer, antigravityTranscriptDirectory, + parseAntigravityResult } + from '../src/agents/antigravity-protocol.js'; + +type Invocation = { model: string; effort: string; baseUrl: string; resumeSession: string | null; + maxBudgetUsd: string | null; tools: ToolPolicy }; +type ProcessOptions = Parameters[0] & { + projects: string; containerId: string; marker: string; model: string; + pricingRates: PricingRates | null; resumeSession: string | null; +}; +interface CodingProvider { + requiresBudget: boolean; + executable: string; + apiKeyEnvironment: string; + credentialPath?: string; + containerTranscripts: string; + tokenEnvironment: string; + environment(baseUrl: string, mode: 'api-key' | 'subscription-token'): string[]; + projects(appDir: string): string; + rates(model: string): PricingRates | null; + args(options: Invocation): string[]; + run(options: ProcessOptions): ReturnType; + result(stdout: string, appDir: string, invocationToken: string, prior: CodexUsage | null): Record | null; + validateContinuation(directory: string, sessionId: string, model: string): void; +} + +const codexProvider: CodingProvider = { + requiresBudget: true, + executable: 'codex', apiKeyEnvironment: 'OPENAI_API_KEY', + containerTranscripts: `${CODING_CONTAINER_AGENT.home}/.codex/sessions`, + tokenEnvironment: 'MODEL_PROXY_TOKEN', + environment: () => [`CODEX_HOME=${CODING_CONTAINER_AGENT.home}/.codex`], + projects: appDir => join(codexTranscriptDirectory(appDir), 'sessions'), + rates: () => null, + args: codexArguments, + run: runCodexProcess, + validateContinuation: validateCodexNativeSession, + result: (stdout, appDir, invocationToken, prior) => { + const result = parseCodexResult(stdout, prior); + const sessionId = result.session_id; + const eventFile = typeof sessionId === 'string' && /^[0-9a-f-]{36}$/i.test(sessionId) + ? `${sessionId}.events.jsonl` : `interrupted-${invocationToken}.events.jsonl`; + const path = join(codexTranscriptDirectory(appDir), eventFile); + const header = existsSync(path) ? '' : `${JSON.stringify({ type: 'stack_bench_context', cwd: '/app' })}\n`; + appendFileSync(path, `${header}${stdout}\n`, { mode: 0o600 }); + return result; + }, +}; + +// Grok signs in through its external auth command with the broker's session token, +// which the broker exchanges for the real SuperGrok token. The CLI never sees it. +const GROK_SIGN_IN = `printf '{"access_token":"%s","expires_in":86400,"issuer":"https://auth.x.ai"}' "$GROK_BROKER_TOKEN"`; +const grokProvider: CodingProvider = { + requiresBudget: true, + executable: 'sh', apiKeyEnvironment: 'XAI_API_KEY', + containerTranscripts: `${CODING_CONTAINER_AGENT.home}/.grok/sessions`, + tokenEnvironment: 'GROK_BROKER_TOKEN', + // Telemetry and session uploads would reach the broker as refused requests. The + // turn summary is a display call over the whole turn that the CLI exits without + // awaiting, so a long session leaves it unpriced. + environment: baseUrl => [`GROK_CLI_CHAT_PROXY_BASE_URL=${baseUrl}/v1`, `GROK_AUTH_PROVIDER_COMMAND=${GROK_SIGN_IN}`, + 'GROK_DISABLE_AUTOUPDATER=1', 'GROK_TELEMETRY_ENABLED=0', 'GROK_TELEMETRY_TRACE_UPLOAD=0', 'DISABLE_TELEMETRY=1', + 'GROK_TURN_SUMMARY=0'], + projects: appDir => join(grokTranscriptDirectory(appDir), 'sessions'), + rates: () => null, + // Sign-in prints the account's email address; none of its output may reach evidence. + args: options => ['-c', 'grok login < /dev/null > /dev/null 2>&1 || { echo "Grok sign-in failed" >&2; exit 1; }; exec grok "$@"', + 'grok', ...grokArguments(options)], + run: runCodexProcess, + validateContinuation: validateGrokNativeSession, + // The controller keeps the stream as the session's audited transcript, beside the + // agent-writable native sessions. + result: (stdout, appDir, invocationToken) => { + const result = parseGrokResult(stdout); + const name = typeof result.session_id === 'string' ? result.session_id : `interrupted-${invocationToken}`; + appendFileSync(join(grokTranscriptDirectory(appDir), `${name}.events.jsonl`), `${stdout}\n`, { mode: 0o600 }); + return result; + }, +}; + +// The CLI sends the broker's session token as its Gemini key or account token, and the +// broker sends the real one to Google. Its whole state directory persists between sessions +// so a session can resume. +const googleProvider: CodingProvider = { + requiresBudget: true, + executable: 'sh', apiKeyEnvironment: 'GEMINI_API_KEY', + containerTranscripts: `${CODING_CONTAINER_AGENT.home}/.gemini/antigravity-cli`, + tokenEnvironment: 'GEMINI_API_KEY', + environment: (baseUrl, mode) => mode === 'subscription-token' ? [`CLOUD_CODE_URL=${baseUrl}`, `HTTPS_PROXY=${baseUrl}`] + : [`GOOGLE_GEMINI_BASE_URL=${baseUrl}`], + projects: appDir => join(antigravityTranscriptDirectory(appDir), 'state'), + rates: () => null, + args: options => ['-c', ANTIGRAVITY_LAUNCHER, 'agy', ...antigravityArguments(options)], + run: options => { + const startedAt = Date.now(); + return runCodexProcess({ ...options, + finishedAnswer: () => antigravityFinishedAnswer(options.projects, options.marker, startedAt) }); + }, + validateContinuation: validateAntigravityNativeSession, + // The controller keeps the stream as the session's audited transcript, beside the + // agent-writable native state. + result: (stdout, appDir, invocationToken, prior) => { + const result = parseAntigravityResult(stdout, prior); + const name = typeof result.session_id === 'string' ? result.session_id : `interrupted-${invocationToken}`; + appendFileSync(join(antigravityTranscriptDirectory(appDir), `${name}.events.jsonl`), `${stdout}\n`, { mode: 0o600 }); + return result; + }, +}; + +export const CODING_PROVIDERS = { + anthropic: { + requiresBudget: false, + executable: 'claude', apiKeyEnvironment: 'ANTHROPIC_API_KEY', + credentialPath: join(homedir(), '.claude', '.credentials.json'), + containerTranscripts: `${CODING_CONTAINER_AGENT.home}/.claude/projects/-app`, + tokenEnvironment: 'ANTHROPIC_AUTH_TOKEN', + environment: baseUrl => [`ANTHROPIC_BASE_URL=${baseUrl}`, 'DISABLE_AUTOUPDATER=1', 'FORCE_PROMPT_CACHING_5M=1'], + projects: appDir => join(homedir(), '.claude', 'projects', + resolve(appDir).replace(/[\\/:]/g, '-').toLowerCase()), + rates: claudeRatesForModel, + validateContinuation: validateClaudeNativeSession, + // `--bare` already leaves Claude without web or subagent tools, so a policy turning + // either on is refused before it reaches here. + args: ({ model, effort, maxBudgetUsd, resumeSession, tools }) => { + const unavailable = unavailableToolsReason('anthropic', tools); + if (unavailable) throw new Error(unavailable); + return [ + '--print', '--output-format', 'json', + // Isolate the session from project memory, plugins, and integrations. + '--bare', + '--permission-mode', 'acceptEdits', + '--settings', JSON.stringify({ permissions: { allow: ['Bash'] } }), + '--effort', effort, + '--model', model, + ...(maxBudgetUsd !== null ? ['--max-budget-usd', maxBudgetUsd] : []), + // The app is the only directory a session may reach; inside the container + // that is all there is, but the flag is kept so host and container runs are + // configured identically. + '--add-dir', CODING_CONTAINER_APP_ROOT, + ...(resumeSession !== null ? ['--resume', resumeSession] : []), + ]; + }, + run: options => { + const transcriptReader = containerClaudeTranscriptReader(options.containerId, options.projects, options.env); + return runTranscriptAwareProcess({ ...options, transcriptDirectory: options.projects, + transcriptReader, transcriptSnapshot: transcriptReader.snapshot(), pollMs: 1_000 }); + }, + result: stdout => { + try { return JSON.parse(stdout); } + catch { + for (const line of stdout.split(/\r?\n/).reverse()) { + try { return JSON.parse(line); } catch { /* Keep looking. */ } + } + } + return null; + }, + }, + openai: codexProvider, + openrouter: { ...codexProvider, apiKeyEnvironment: 'OPENROUTER_API_KEY' }, + xai: grokProvider, + google: googleProvider, +} satisfies Record; + +export type CodingProviderId = keyof typeof CODING_PROVIDERS; + +export function parseCodingProvider(value: string): CodingProviderId { + if (!Object.hasOwn(CODING_PROVIDERS, value)) throw new Error(`unsupported coding provider: ${value}`); + return value as CodingProviderId; +} diff --git a/tools/stack-bench/container/container-auth.ts b/tools/stack-bench/container/container-auth.ts new file mode 100644 index 00000000000..57a26877387 --- /dev/null +++ b/tools/stack-bench/container/container-auth.ts @@ -0,0 +1,122 @@ +import { readPinnedExecutionCredential } from '../src/agents/credential-profiles.js'; +import { existsSync, readFileSync } from 'node:fs'; +import type { PathLike } from 'node:fs'; +import { isAbsolute, resolve } from 'node:path'; +import { readGrokLogin } from '../src/agents/grok-login.js'; +import { readAgyLogin } from '../src/agents/agy-login.js'; +import type { AnthropicUpstream } from './credential-broker-accounting.js'; + +export const SUBSCRIPTION_TOKEN_ENVIRONMENT = 'CLAUDE_CODE_OAUTH_TOKEN'; +export const LEGACY_SUBSCRIPTION_TOKEN_TARGET = '/run/secrets/claude-code-oauth-token'; + +export type ContainerAuth = { + provider?: 'anthropic' | 'openai' | 'openrouter' | 'xai' | 'google'; + upstream?: AnthropicUpstream; + accountId?: string; + mode: 'api-key' | 'subscription-token'; + credential: string; +}; + +type ReadTextFile = (path: PathLike | number, encoding: BufferEncoding) => string; + +export interface ResolveContainerAuthOptions { + provider?: 'anthropic' | 'openai' | 'openrouter' | 'xai' | 'google'; + apiKey?: string; + env?: NodeJS.ProcessEnv; + credentialsPath?: string; + exists?: (path: PathLike) => boolean; + read?: ReadTextFile; +} + +export function resolveContainerAuth({ provider = 'anthropic', apiKey = '', env = process.env, credentialsPath, + exists = existsSync, read = readFileSync as ReadTextFile }: ResolveContainerAuthOptions = {}): ContainerAuth { + const pinned = readPinnedExecutionCredential(env); + if (pinned) { + if (pinned.assignment.provider !== provider) throw new Error('Pinned credential provider does not match invocation'); + apiKey = pinned.assignment.mode === 'api-key' ? pinned.secret : ''; + // Resolve the broker credential from the same bytes that passed its pin check. + // Never reopen a file that an operator can replace between validation and use. + read = path => { + if (String(path) !== pinned.secretFile) throw new Error('Pinned credential file does not match invocation'); + return pinned.secret; + }; + } + if (provider === 'xai') { + const authFile = env.GROK_AUTH_FILE?.trim(); + if (apiKey && authFile) throw new Error('use only one of xAI API-key and Grok sign-in authentication'); + if (apiKey) return { provider, mode: 'api-key', credential: apiKey }; + if (!authFile || !isAbsolute(authFile)) throw new Error('xAI requires an API key or an absolute GROK_AUTH_FILE'); + if (!exists(authFile)) throw new Error('GROK_AUTH_FILE does not exist'); + // run-build refreshes the sign-in before taking this snapshot for a session. + return { provider, mode: 'subscription-token', credential: readGrokLogin(read(authFile, 'utf8')).token }; + } + if (provider === 'google') { + const authFile = env.AGY_AUTH_FILE?.trim(); + if (apiKey && authFile) throw new Error('use only one of Gemini API-key and Google account authentication'); + if (apiKey) return { provider, mode: 'api-key', credential: apiKey }; + if (!authFile || !isAbsolute(authFile)) throw new Error('Google requires a Gemini API key or an absolute AGY_AUTH_FILE'); + if (!exists(authFile)) throw new Error('AGY_AUTH_FILE does not exist'); + // run-build renews the sign-in before the session and while it runs. + return { provider, mode: 'subscription-token', credential: readAgyLogin(read(authFile, 'utf8')).token }; + } + if (provider === 'openrouter') { + if (!apiKey) throw new Error('OpenRouter requires an API key'); + return { provider, mode: 'api-key', credential: apiKey }; + } + if (provider === 'openai') { + const authFile = env.CODEX_AUTH_FILE?.trim(); + if (apiKey && authFile) throw new Error('use only one of OpenAI API-key and account authentication'); + if (apiKey) return { provider, mode: 'api-key', credential: apiKey }; + if (!authFile) throw new Error('OpenAI requires an API key or an explicit CODEX_AUTH_FILE'); + if (!isAbsolute(authFile)) throw new Error('CODEX_AUTH_FILE must be an absolute path'); + if (!exists(authFile)) throw new Error('CODEX_AUTH_FILE does not exist'); + let auth: { auth_mode?: string; OPENAI_API_KEY?: unknown; + tokens?: { access_token?: unknown; account_id?: unknown } }; + try { auth = JSON.parse(read(authFile, 'utf8')); } + catch { throw new Error('CODEX_AUTH_FILE must contain valid Codex login JSON'); } + if (!auth || auth.OPENAI_API_KEY || auth.auth_mode !== 'chatgpt' + || typeof auth.tokens?.access_token !== 'string' || !auth.tokens.access_token.trim() + || typeof auth.tokens.account_id !== 'string' || !auth.tokens.account_id.trim()) { + throw new Error('CODEX_AUTH_FILE must contain ChatGPT account login tokens, not an API key'); + } + let expiry: unknown; + try { expiry = JSON.parse(Buffer.from(auth.tokens.access_token.split('.')[1]!, 'base64url').toString()).exp; } + catch { throw new Error('Codex account access token has no valid expiry; log in again'); } + if (typeof expiry !== 'number' || !Number.isFinite(expiry) || expiry * 1000 <= Date.now()) { + throw new Error('Codex account access token is expired; log in again and replace CODEX_AUTH_FILE'); + } + // Each broker uses an access-token snapshot. It never rotates shared refresh tokens. + return { provider, mode: 'subscription-token', credential: auth.tokens.access_token, + accountId: auth.tokens.account_id }; + } + const token = String(env[SUBSCRIPTION_TOKEN_ENVIRONMENT] ?? '').trim(); + const tokenFileValue = String(env[`${SUBSCRIPTION_TOKEN_ENVIRONMENT}_FILE`] ?? '').trim(); + if (token && tokenFileValue) { + throw new Error(`use only one of ${SUBSCRIPTION_TOKEN_ENVIRONMENT} and ` + + `${SUBSCRIPTION_TOKEN_ENVIRONMENT}_FILE`); + } + if (apiKey && (token || tokenFileValue)) { + throw new Error('use only one of API-key and subscription-token authentication'); + } + const upstream = pinned?.assignment.upstream; + if (apiKey) return { mode: 'api-key', credential: apiKey, ...(upstream ? { upstream } : {}) }; + if (token) return { mode: 'subscription-token', credential: token }; + if (tokenFileValue) { + if (!isAbsolute(tokenFileValue)) { + throw new Error(`${SUBSCRIPTION_TOKEN_ENVIRONMENT}_FILE must be an absolute path`); + } + const source = resolve(tokenFileValue); + if (!exists(source)) throw new Error(`subscription token file does not exist: ${source}`); + const credential = String(read(source, 'utf8')).trim(); + if (!credential) { + throw new Error(`subscription token file is empty: ${source}`); + } + return { mode: 'subscription-token', credential }; + } + if (credentialsPath && exists(credentialsPath)) { + throw new Error('rotating Claude credential files cannot be isolated from generated shell commands; ' + + 'select an API key or CLAUDE_CODE_OAUTH_TOKEN_FILE'); + } + throw new Error(`no API key, ${SUBSCRIPTION_TOKEN_ENVIRONMENT}, ` + + `${SUBSCRIPTION_TOKEN_ENVIRONMENT}_FILE, or credentials file is available`); +} diff --git a/tools/stack-bench/container/credential-broker-accounting.ts b/tools/stack-bench/container/credential-broker-accounting.ts new file mode 100644 index 00000000000..910bc15b523 --- /dev/null +++ b/tools/stack-bench/container/credential-broker-accounting.ts @@ -0,0 +1,457 @@ +import type { ProviderFailure } from '../src/agents/provider-failure.js'; +import { randomBytes } from 'node:crypto'; +import { readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs'; +import { z } from 'zod'; + +import { normalizeClaudeUsage, priceClaudeUsage } from '../src/evidence/claude-usage-cost.js'; +import type { ClaudeUsage } from '../src/evidence/claude-usage-cost.js'; +import { validatePricingRates as validateSharedPricingRates } from '../src/evidence/pricing-authority.js'; +import { formatZodError } from '../src/zod-error.js'; +import type { ToolPolicy } from '../src/agents/tool-policy.js'; + +export const BROKER_LEDGER_SCHEMA_VERSION = 5; +// Why a billable request was charged its cost ceiling instead of priced from +// the provider's usage: a 2xx response without complete usage (an aborted or +// errored stream, an oversized body), a response that broke off, or an +// upstream connection that failed. The ceiling makes spend an upper bound. +export const ESTIMATE_REASONS = ['no-usage', 'response-aborted', 'upstream-error'] as const; +export type EstimateReason = typeof ESTIMATE_REASONS[number]; +export type EstimateCounts = Record; +export const noEstimates = (): EstimateCounts => ({ 'no-usage': 0, 'response-aborted': 0, 'upstream-error': 0 }); +// Why a forwarded request may cost more than the broker's token rates price: +// a server tool the response reports using, a non-standard service tier or +// speed, an MCP connector or container, a hosted tool, input or output the +// rates do not cover, or server-side stored context. The priced figure is then +// a lower bound. Each request counts under its first reason in this order. +export const UNPRICED_REASONS = ['server-tool', 'service-tier', 'speed', 'mcp', 'container', + 'hosted-tool', 'unpriced-input', 'unpriced-output', 'stored-context'] as const; +export type UnpricedReason = typeof UNPRICED_REASONS[number]; +export type UnpricedCounts = Record; +export const noUnpriced = (): UnpricedCounts => Object.fromEntries( + UNPRICED_REASONS.map(reason => [reason, 0])) as UnpricedCounts; +export const firstUnpricedReason = (...reasons: Array): UnpricedReason | null => + UNPRICED_REASONS.find(reason => reasons.includes(reason)) ?? null; +export const MAX_BROKER_OUTPUT_TOKENS = 128_000; +export const CLAUDE_USAGE_FIELDS = ['input', 'output', 'cacheRead', 'cacheWrite5m', 'cacheWrite1h'] as const; +const COST_TOLERANCE_USD = 0.0001; + +type JsonRecord = Record; +export type BrokerMode = 'api-key' | 'subscription-token'; +export type PricingRates = ReturnType; + +// Model makers that serve their own models behind an Anthropic Messages endpoint. +export const ANTHROPIC_UPSTREAMS = { + zai: { hostname: 'api.z.ai', pathPrefix: '/api/anthropic' }, + deepseek: { hostname: 'api.deepseek.com', pathPrefix: '/anthropic' }, + moonshot: { hostname: 'api.moonshot.ai', pathPrefix: '/anthropic' }, +} as const; +export type AnthropicUpstream = keyof typeof ANTHROPIC_UPSTREAMS; +export const ANTHROPIC_UPSTREAM_IDS = Object.keys(ANTHROPIC_UPSTREAMS) as [AnthropicUpstream, ...AnthropicUpstream[]]; + +export type BrokerConfig = { + provider?: 'anthropic' | 'openai' | 'openrouter' | 'xai' | 'google'; + upstream?: AnthropicUpstream; + accountId?: string; + providerRoute?: string; + mode: BrokerMode; + credential: string; + // A file the controller rewrites with a renewed credential; read before each request. + credentialPath?: string; + // The per-session certificate for a protocol's account lookup host. + tlsKeyPath?: string; + tlsCertPath?: string; + sessionToken: string; + readyPath?: string; + parentPid?: number; + heartbeatPath?: string; + expiresAt?: number; + listenHost?: '127.0.0.1' | '0.0.0.0'; + ledgerPath?: string; + model: string; + maxOutputTokens: number; + explicitOutputLimit?: boolean; + maxBudgetUsd?: number | null; + pricingRates?: PricingRates; + // The agent's tool policy, for a protocol that removes tools itself. Without one only the + // file and shell tools remain. + tools?: ToolPolicy; +}; + +export type BrokerLedger = { + provider?: 'openrouter'; + providerRoute?: string; + providerReportedCostUsd?: number; + upstreamProviders?: string[]; + providerIntegrityError?: string; + providerFailure?: ProviderFailure | null; + schemaVersion: number; + model: string; + maxBudgetUsd: number | null; + acceptedRequests: number; + billableRequests: number; + completedBillableRequests: number; + estimatedBillableRequests: number; + estimatedByReason: EstimateCounts; + unpricedBillableRequests: number; + unpricedByReason: UnpricedCounts; + // Web searches the provider's responses report; each is charged on top of tokens. Absent + // from ledgers written before searches were counted. + webSearchCalls?: number; + spentUsd: number; + reservedUsd: number; + usage: ClaudeUsage; + complete: boolean; + updatedAt: string; +}; + +// `costUsd` is what the broker charged: exact provider usage priced at the +// receipt's rates, plus the cost ceiling of every estimated request. With +// estimated requests it is an upper bound and `calculatedCostUsd`, priced from +// the exact usage alone, a lower bound. With unpriced requests the provider may +// have charged more than `costUsd`, so the session's spend is unknown. +export interface CredentialBrokerReceipt { + costSource?: 'provider-reported'; + provider?: 'openrouter'; + providerRoute?: string; + providerReportedCostUsd?: number; + upstreamProviders?: string[]; + schemaVersion: 4; + source: 'credential-broker'; + model: string; + maxBudgetUsd: number; + costUsd: number; + cliCostUsd: number | null; + calculatedCostUsd: number | null; + usage: ClaudeUsage | null; + pricingRates: PricingRates | null; + exact: boolean; + estimatedRequests: number; + estimatedByReason: EstimateCounts; + unpricedRequests: number; + unpricedByReason: UnpricedCounts; + webSearchCalls: number; + complete: boolean; + reconciled: boolean; + error: string | null; +} + +export interface CredentialBrokerResult extends JsonRecord { + total_cost_usd: number; + usage?: ReturnType; + stack_bench_cost_receipt: CredentialBrokerReceipt; +} + +const positiveFinite = z.number().finite().positive(); +const nonNegativeFinite = z.number().finite().nonnegative(); +const nonNegativeSafeInteger = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER); +const usageSchema = z.strictObject({ + input: nonNegativeSafeInteger, + output: nonNegativeSafeInteger, + cacheRead: nonNegativeSafeInteger, + cacheWrite5m: nonNegativeSafeInteger, + cacheWrite1h: nonNegativeSafeInteger, +}); +const brokerConfigSchema = z.strictObject({ + provider: z.enum(['anthropic', 'openai', 'openrouter', 'xai', 'google']).optional(), + accountId: z.string().min(1).optional(), + providerRoute: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9._/-]{0,127}$/).optional(), + upstream: z.enum(ANTHROPIC_UPSTREAM_IDS).optional(), + mode: z.enum(['api-key', 'subscription-token']), + credential: z.string().min(16), + credentialPath: z.string().min(1).optional(), + tlsKeyPath: z.string().min(1).optional(), + tlsCertPath: z.string().min(1).optional(), + sessionToken: z.string().min(16), + readyPath: z.string().min(1).optional(), + parentPid: z.number().int().positive().max(Number.MAX_SAFE_INTEGER).optional(), + heartbeatPath: z.string().min(1).optional(), + expiresAt: positiveFinite.optional(), + listenHost: z.enum(['127.0.0.1', '0.0.0.0']).optional(), + ledgerPath: z.string().min(1).optional(), + model: z.string().min(1), + maxOutputTokens: z.number().int().min(1).max(MAX_BROKER_OUTPUT_TOKENS), + explicitOutputLimit: z.boolean().optional(), + maxBudgetUsd: positiveFinite.nullable().optional(), + pricingRates: z.unknown().optional(), + tools: z.strictObject({ webSearch: z.boolean(), subagents: z.boolean() }).optional(), +}).superRefine((value, context) => { + if (value.provider === 'openrouter' && (value.mode !== 'api-key' || !value.providerRoute || value.maxBudgetUsd == null)) { + context.addIssue({ code: 'custom', message: 'OpenRouter requires API-key auth, providerRoute, and a spend budget' }); + } + if (value.upstream !== undefined && ((value.provider ?? 'anthropic') !== 'anthropic' || value.mode !== 'api-key')) { + context.addIssue({ code: 'custom', path: ['upstream'], message: 'only an Anthropic API key names an upstream' }); + } + if (value.provider !== 'openrouter' && value.providerRoute !== undefined) { + context.addIssue({ code: 'custom', path: ['providerRoute'], message: 'only OpenRouter uses providerRoute' }); + } + if (value.expiresAt !== undefined && value.expiresAt <= Date.now()) { + context.addIssue({ code: 'custom', path: ['expiresAt'], message: 'must be in the future' }); + } +}); +const brokerLedgerSchema = z.strictObject({ + provider: z.literal('openrouter').optional(), + providerRoute: z.string().min(1).optional(), + providerReportedCostUsd: nonNegativeFinite.optional(), + upstreamProviders: z.array(z.string().min(1).max(128)).optional(), + providerIntegrityError: z.string().max(200).optional(), + providerFailure: z.strictObject({ + category: z.enum(['rate-limit', 'unreachable', 'quota', 'authentication', 'transport', 'request', 'broker-budget']), + status: z.number().int().min(100).max(599).nullable(), + code: z.string().regex(/^[a-zA-Z0-9_.-]{1,100}$/).nullable(), + budget: z.strictObject({ + maxBudgetUsd: positiveFinite, + spentUsd: nonNegativeFinite, + estimatedSpendUsd: nonNegativeFinite, + reservedUsd: nonNegativeFinite, + requestCeilingUsd: nonNegativeFinite, + }).optional(), + }).nullable().optional(), + // Version 4 ledgers predate unpriced requests; the broker then refused them. + schemaVersion: z.union([z.literal(4), z.literal(BROKER_LEDGER_SCHEMA_VERSION)]), + model: z.string().min(1), + maxBudgetUsd: positiveFinite.nullable(), + acceptedRequests: nonNegativeSafeInteger, + billableRequests: nonNegativeSafeInteger, + completedBillableRequests: nonNegativeSafeInteger, + estimatedBillableRequests: nonNegativeSafeInteger, + estimatedByReason: z.strictObject({ + 'no-usage': nonNegativeSafeInteger, + 'response-aborted': nonNegativeSafeInteger, + 'upstream-error': nonNegativeSafeInteger, + }), + unpricedBillableRequests: nonNegativeSafeInteger.optional(), + unpricedByReason: z.strictObject(Object.fromEntries(UNPRICED_REASONS.map(reason => + [reason, nonNegativeSafeInteger])) as Record).optional(), + webSearchCalls: nonNegativeSafeInteger.optional(), + spentUsd: nonNegativeFinite, + reservedUsd: nonNegativeFinite, + usage: usageSchema, + complete: z.boolean(), + updatedAt: z.string().refine(value => !Number.isNaN(Date.parse(value)), 'must be a timestamp'), +}); + +function isRecord(value: unknown): value is JsonRecord { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function isNumber(value: unknown): value is number { + return typeof value === 'number'; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function fail(message: string): never { + throw new Error(`credential broker: ${message}`); +} + +export function validatePricingRates(value: unknown): PricingRates { + try { return validateSharedPricingRates(value, { at: 'pricingRates' }); } + catch (error) { return fail(errorMessage(error)); } +} + +export function priceNormalizedClaudeUsage(usage: ClaudeUsage, rates: PricingRates): number { + return priceClaudeUsage({ + input_tokens: usage.input, + output_tokens: usage.output, + cache_read_input_tokens: usage.cacheRead, + cache_creation: { + ephemeral_5m_input_tokens: usage.cacheWrite5m, + ephemeral_1h_input_tokens: usage.cacheWrite1h, + }, + }, rates); +} + +function rawUsage(usage: ClaudeUsage): JsonRecord { + return { + input_tokens: usage.input, + output_tokens: usage.output, + cache_read_input_tokens: usage.cacheRead, + cache_creation_input_tokens: usage.cacheWrite5m + usage.cacheWrite1h, + cache_creation: { + ephemeral_5m_input_tokens: usage.cacheWrite5m, + ephemeral_1h_input_tokens: usage.cacheWrite1h, + }, + }; +} + +function brokerCoversCliUsage(broker: ClaudeUsage, cli: ClaudeUsage): boolean { + return broker.input >= cli.input + && broker.output >= cli.output + && broker.cacheRead >= cli.cacheRead + && broker.cacheWrite5m + broker.cacheWrite1h >= cli.cacheWrite5m + cli.cacheWrite1h; +} + +export function validateBrokerConfig(value: unknown): BrokerConfig { + const parsed = brokerConfigSchema.safeParse(value); + if (!parsed.success) fail(formatZodError(parsed.error, 'configuration')); + const { pricingRates, ...config } = parsed.data; + return config.maxBudgetUsd === null || config.maxBudgetUsd === undefined + ? config + : { ...config, pricingRates: validatePricingRates(pricingRates) }; +} + +function validateLedger(value: unknown, + { model = null, maxBudgetUsd = undefined }: { model?: string | null; maxBudgetUsd?: number | null } = {}): BrokerLedger { + const parsed = brokerLedgerSchema.safeParse(value); + if (!parsed.success) fail(formatZodError(parsed.error, 'spend ledger')); + const { unpricedBillableRequests, unpricedByReason, ...fields } = parsed.data; + if (fields.schemaVersion === BROKER_LEDGER_SCHEMA_VERSION + ? unpricedBillableRequests === undefined || unpricedByReason === undefined + : (unpricedBillableRequests ?? 0) !== 0 + || UNPRICED_REASONS.some(reason => (unpricedByReason?.[reason] ?? 0) !== 0)) { + fail('spend ledger unpriced request counts are invalid'); + } + const ledger: BrokerLedger = { ...fields, unpricedBillableRequests: unpricedBillableRequests ?? 0, + unpricedByReason: unpricedByReason ?? noUnpriced() }; + if (model !== null && ledger.model !== model) fail('spend ledger model does not match'); + if (maxBudgetUsd !== undefined && ledger.maxBudgetUsd !== maxBudgetUsd) { + fail('spend ledger budget does not match'); + } + if (ledger.completedBillableRequests > ledger.billableRequests) { + fail('spend ledger completed request count is invalid'); + } + if (ledger.estimatedBillableRequests > ledger.completedBillableRequests) { + fail('spend ledger estimated request count is invalid'); + } + const reasons = ESTIMATE_REASONS.reduce((sum, reason) => sum + ledger.estimatedByReason[reason], 0); + if (reasons !== ledger.estimatedBillableRequests) fail('spend ledger estimate reasons do not add up'); + if (ledger.unpricedBillableRequests > ledger.completedBillableRequests) { + fail('spend ledger unpriced request count is invalid'); + } + const unpriced = UNPRICED_REASONS.reduce((sum, reason) => sum + ledger.unpricedByReason[reason], 0); + if (unpriced !== ledger.unpricedBillableRequests) fail('spend ledger unpriced reasons do not add up'); + const complete = ledger.reservedUsd === 0 + && ledger.completedBillableRequests === ledger.billableRequests; + if (ledger.complete !== complete) fail('spend ledger completion state is invalid'); + if (ledger.provider === 'openrouter' && (!ledger.providerRoute || ledger.providerReportedCostUsd === undefined + || !ledger.upstreamProviders || ledger.providerReportedCostUsd > ledger.spentUsd + COST_TOLERANCE_USD)) { + fail('OpenRouter spend ledger lacks valid cost and routing provenance'); + } + return ledger; +} + +export function writeCredentialBrokerLedger(path: string | undefined, value: unknown): void { + if (!path) return; + const ledger = validateLedger(value); + if (ledger.schemaVersion !== BROKER_LEDGER_SCHEMA_VERSION) fail('spend ledger schema version is not current'); + const temporary = `${path}.${process.pid}.${randomBytes(8).toString('hex')}.tmp`; + writeFileSync(temporary, `${JSON.stringify(ledger)}\n`, { flag: 'wx', mode: 0o600 }); + try { renameSync(temporary, path); } + catch (error) { rmSync(temporary, { force: true }); throw error; } +} + +export function readCredentialBrokerLedger(path: string, + expected: { model?: string | null; maxBudgetUsd?: number | null } = {}): BrokerLedger { + return validateLedger(JSON.parse(readFileSync(path, 'utf8')), expected); +} + +export function reconcileCredentialBrokerReceipt({ ledger, cliResult, model, maxBudgetUsd, + pricingRates, provider = 'anthropic', brokerDiagnostics = null, toleranceUsd = COST_TOLERANCE_USD }: { + provider?: 'anthropic' | 'openai' | 'openrouter' | 'xai' | 'google'; + ledger: unknown; cliResult: unknown; model: unknown; maxBudgetUsd: unknown; pricingRates: unknown; + brokerDiagnostics?: unknown; toleranceUsd?: number; +}): { ok: boolean; result: CredentialBrokerResult; receipt: CredentialBrokerReceipt } { + if (typeof model !== 'string' || !model) fail('receipt model is invalid'); + if (!isNumber(maxBudgetUsd) || !Number.isFinite(maxBudgetUsd) || maxBudgetUsd <= 0) fail('receipt budget is invalid'); + if (!Number.isFinite(toleranceUsd) || toleranceUsd < 0) fail('receipt tolerance is invalid'); + const receiptBudget = maxBudgetUsd; + let verifiedLedger: BrokerLedger | null = null; + let verifiedRates: PricingRates | null = null; + let usage: ClaudeUsage | null = null; + let cliUsage: ClaudeUsage | null = null; + let calculatedCostUsd: number | null = null; + let issue: string | null = null; + try { verifiedLedger = validateLedger(ledger, { model, maxBudgetUsd: receiptBudget }); } + catch (error) { issue = errorMessage(error); } + if (!issue && verifiedLedger?.complete !== true) issue = 'credential broker spend ledger is incomplete'; + const estimatedRequests = verifiedLedger?.estimatedBillableRequests ?? 0; + const unpricedRequests = verifiedLedger?.unpricedBillableRequests ?? 0; + // Token totals are exact without estimates; unpriced charges come on top. + const tokensExact = verifiedLedger !== null && estimatedRequests === 0; + const exact = tokensExact && unpricedRequests === 0; + try { verifiedRates = validatePricingRates(pricingRates); } + catch (error) { if (!issue) issue = errorMessage(error); } + try { cliUsage = normalizeClaudeUsage(isRecord(cliResult) ? cliResult.usage : undefined); } + catch (error) { if (!issue) issue = errorMessage(error); } + if (verifiedLedger) usage = structuredClone(verifiedLedger.usage); + if (!issue && tokensExact && cliUsage && usage && !brokerCoversCliUsage(usage, cliUsage)) { + issue = 'credential broker usage is lower than CLI usage totals'; + } + if (!issue && provider === 'openrouter' && verifiedLedger?.provider !== 'openrouter') issue = 'OpenRouter spend ledger lacks provenance'; + if (!issue && verifiedLedger?.providerIntegrityError) issue = verifiedLedger.providerIntegrityError; + try { if (provider !== 'openrouter' && verifiedRates && usage) calculatedCostUsd = priceNormalizedClaudeUsage(usage, verifiedRates); } + catch (error) { if (!issue) issue = errorMessage(error); } + // Server-side input of an unpriced request can exceed its reservation, so + // its priced spend is never clipped to the budget. + const brokerCost = verifiedLedger + ? provider === 'openrouter' || unpricedRequests > 0 ? verifiedLedger.spentUsd + verifiedLedger.reservedUsd + : Math.min(receiptBudget, verifiedLedger.spentUsd + verifiedLedger.reservedUsd) : receiptBudget; + // Estimated requests contribute ceilings, not observed tokens. Require those + // ceilings to cover every usage component seen by either recorder. + if (!issue && provider !== 'openrouter' && !tokensExact && verifiedRates && usage && cliUsage) { + const observedUsage = { ...usage }; + for (const field of CLAUDE_USAGE_FIELDS) { + observedUsage[field] = Math.max(usage[field], cliUsage[field]); + } + if (priceNormalizedClaudeUsage(observedUsage, verifiedRates) - brokerCost > toleranceUsd) { + issue = 'observed usage-priced spend exceeds credential broker spend ceiling'; + } + } + const cliCost = Number(isRecord(cliResult) ? cliResult.total_cost_usd : undefined); + if (!issue && (provider === 'anthropic' || (isRecord(cliResult) && cliResult.total_cost_usd !== undefined)) + && (!Number.isFinite(cliCost) || cliCost < 0)) { + issue = 'coding session did not return a usable cost receipt'; + } + // Exact spend must price back to the broker's figure. Estimated requests + // add their ceilings on top of the priced usage, so the priced usage can + // only fall below the broker's figure, never above it. A CLI cost above the + // broker's figure is kept as `cliCostUsd`; it can include unpriced fees. + if (!issue && calculatedCostUsd !== null && tokensExact && Math.abs(calculatedCostUsd - brokerCost) > toleranceUsd) { + issue = `usage-priced spend $${calculatedCostUsd.toFixed(6)} does not match credential broker spend $${brokerCost.toFixed(6)}`; + } + if (!issue && provider === 'openrouter' && brokerCost > receiptBudget + toleranceUsd) { + issue = 'OpenRouter reported spend exceeds the session budget'; + } + const receipt: CredentialBrokerReceipt = { + ...(provider === 'openrouter' ? { costSource: 'provider-reported' as const, provider, + providerRoute: verifiedLedger?.providerRoute, providerReportedCostUsd: verifiedLedger?.providerReportedCostUsd, + upstreamProviders: verifiedLedger?.upstreamProviders } : {}), + schemaVersion: 4, + source: 'credential-broker', + model, + maxBudgetUsd: receiptBudget, + costUsd: Number(brokerCost.toFixed(6)), + cliCostUsd: Number.isFinite(cliCost) && cliCost >= 0 ? Number(cliCost.toFixed(6)) : null, + calculatedCostUsd: calculatedCostUsd === null ? null : Number(calculatedCostUsd.toFixed(6)), + usage, + pricingRates: verifiedRates, + exact, + estimatedRequests, + estimatedByReason: verifiedLedger ? structuredClone(verifiedLedger.estimatedByReason) : noEstimates(), + unpricedRequests, + unpricedByReason: verifiedLedger ? structuredClone(verifiedLedger.unpricedByReason) : noUnpriced(), + webSearchCalls: verifiedLedger?.webSearchCalls ?? 0, + complete: verifiedLedger?.complete === true, + reconciled: issue === null, + error: issue, + }; + const result: CredentialBrokerResult = { + ...(isRecord(cliResult) + ? structuredClone(cliResult) : { type: 'result', is_error: true, result: '' }), + total_cost_usd: receipt.costUsd, + stack_bench_cost_receipt: receipt, + }; + delete result.stack_bench_provider_failure; + if (verifiedLedger?.providerFailure) result.stack_bench_provider_failure = verifiedLedger.providerFailure; + if (usage) result.usage = rawUsage(usage); + if (brokerDiagnostics) result.stack_bench_credential_broker = structuredClone(brokerDiagnostics); + if (issue) { + result.is_error = true; + result.terminal_reason = 'cost_receipt_error'; + result.result = [typeof result.result === 'string' ? result.result.trim() : '', issue] + .filter(Boolean).join('\n'); + } + return { ok: issue === null, result, receipt }; +} diff --git a/tools/stack-bench/container/credential-broker-process.ts b/tools/stack-bench/container/credential-broker-process.ts new file mode 100644 index 00000000000..821d1e13170 --- /dev/null +++ b/tools/stack-bench/container/credential-broker-process.ts @@ -0,0 +1,449 @@ +import { spawn, spawnSync } from 'node:child_process'; +import type { ChildProcess } from 'node:child_process'; +import { randomBytes } from 'node:crypto'; +import { chmodSync, existsSync, mkdtempSync, readFileSync, renameSync, rmSync, utimesSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { isAbsolute, join } from 'node:path'; +import { setTimeout as wait } from 'node:timers/promises'; + +import type { ContainerAuth } from './container-auth.js'; +import { MAX_BROKER_OUTPUT_TOKENS, readCredentialBrokerLedger, validateBrokerConfig } + from './credential-broker-accounting.js'; +import type { BrokerLedger, PricingRates } from './credential-broker-accounting.js'; +import { brokerProtocol } from './broker-protocols.js'; +import { compiledEntrypoint } from '../src/package-root.js'; +import { killTree } from '../src/runtime/platform.js'; +import { ATTEMPT_CREATION_LABEL } from '../src/runtime/container-identity.js'; +import { BROKER_CONTAINER_RESOURCE_LIMITS } from '../src/composition/product-config.js'; +import type { ToolPolicy } from '../src/agents/tool-policy.js'; + +const BROKER_DRAIN_TIMEOUT_MS = 30_000; +const BROKER_DRAIN_POLL_MS = 100; +const BROKER_STDERR_LIMIT_BYTES = 16 * 1024; +const BROKER_STOP_GRACE_MS = 2_000; +const BROKER_STOP_FORCE_MS = 2_000; + +type JsonRecord = Record; +type Alive = (pid: number | undefined) => boolean; + +export type BrokerProcessState = { + exitCode: number | null; + signal: NodeJS.Signals | null; + exitedAt: string | null; + stderrTail: string; + stderrPending: string; + stderrTruncated: boolean; +}; + +export type BrokerError = { type: string; phase: string; message: string }; + +export type BrokerDiagnostics = { + schemaVersion: number; + endpointKind: string; + child: { pid: number | undefined | null; exitCode: number | null; signal: NodeJS.Signals | null; + exitedAt: string | null; stderrTail: string | null; stderrTruncated: boolean }; + drain: { timeoutMs: number; elapsedMs: number; timedOut: boolean; reason: string | null; + terminationRequested: boolean } | null; + termination: { gracefulRequested: boolean; forceRequested: boolean; exited: boolean; + gracefulTimeoutMs: number; forceTimeoutMs: number } | null; + ledger: BrokerLedger | null; + errors: BrokerError[]; +}; + +export interface CredentialBrokerChild { + pid?: number; + exitCode?: number | null; + signalCode?: NodeJS.Signals | null; + kill?: (signal?: NodeJS.Signals | number) => boolean; +} + +export interface CredentialBrokerHandle { + child: CredentialBrokerChild; + root: string; + ledgerPath: string; + model: string; + maxBudgetUsd: number | null; + sessionToken?: string; + baseUrl?: string; + listenHost?: string; + endpointKind?: string; + processState?: Partial; + diagnosticSecrets?: string[]; + finalDiagnostics?: BrokerDiagnostics | null; + finalLedger?: BrokerLedger | null; + container?: CredentialBrokerContainer; + heartbeat?: NodeJS.Timeout; +} + +export type CredentialBrokerContainer = { + name: string; id: string; image: string; owned: true; networkMode: string; +}; + +export type CredentialBrokerDockerOptions = { + imageId: string; + networkContainerId: string; + // The caller persists this intent before creating any resource. + name: string; + creationToken: string; + // This private path must be mounted at the identical path on the Docker host. + privateDirectory: string; + onCreated: (container: CredentialBrokerContainer) => void; +}; + +function brokerDocker(args: string[]): string { + const result = spawnSync('docker', args, { encoding: 'utf8', timeout: 10_000, windowsHide: true }); + if (result.status !== 0) throw new Error(`credential broker Docker ${args[0]} failed: ` + + (result.stderr || result.error?.message || `exit ${result.status}`).trim()); + return result.stdout.trim(); +} + +function signalBrokerContainer(container: CredentialBrokerContainer, signal: 'TERM' | 'KILL'): void { + brokerDocker(['kill', '--signal', signal, container.id]); +} + +export interface CredentialBroker extends CredentialBrokerHandle { + child: ChildProcess; + sessionToken: string; + baseUrl: string; + listenHost: string; + endpointKind: string; + processState: BrokerProcessState; + diagnosticSecrets: string[]; + finalDiagnostics: BrokerDiagnostics | null; + finalLedger: BrokerLedger | null; + // The certificate authority a CLI trusts for the broker's account lookup, if the protocol has one. + certificateAuthority: string | null; + // Hands the running broker a renewed credential for its next request. + replaceCredential: (credential: string) => void; +} + +function isRecord(value: unknown): value is JsonRecord { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function fail(message: string): never { + throw new Error(`credential broker: ${message}`); +} + +function redactDiagnosticText(value: unknown, + broker: CredentialBrokerHandle | string[] | null | undefined): string { + let result = String(value ?? ''); + const secrets = Array.isArray(broker) ? broker : broker?.diagnosticSecrets ?? []; + for (const secret of secrets) { + if (typeof secret === 'string' && secret) result = result.replaceAll(secret, '[REDACTED]'); + } + return result; +} + +function appendDiagnosticStderr(state: BrokerProcessState, chunk: string, secrets: string[], flush = false): void { + const raw = state.stderrPending + chunk; + let redacted = redactDiagnosticText(raw, secrets); + let pendingLength = 0; + if (!flush) { + for (const secret of secrets) { + for (let length = Math.min(secret.length - 1, redacted.length); length > pendingLength; length -= 1) { + if (redacted.endsWith(secret.slice(0, length))) { pendingLength = length; break; } + } + } + } else if (raw && secrets.some(secret => secret.startsWith(raw))) redacted = '[REDACTED]'; + state.stderrPending = pendingLength ? redacted.slice(-pendingLength) : ''; + const safe = pendingLength ? redacted.slice(0, -pendingLength) : redacted; + const next = state.stderrTail + safe; + if (Buffer.byteLength(next) > BROKER_STDERR_LIMIT_BYTES) { + state.stderrTruncated = true; + state.stderrTail = Buffer.from(next).subarray(-BROKER_STDERR_LIMIT_BYTES).toString('utf8'); + } else state.stderrTail = next; +} + +// A certificate for one host, signed by an authority made for this broker alone. The +// authority's key is deleted once it has signed, so it can sign nothing else. +export function lookupCertificate(root: string, hostname: string): { keyPath: string; certPath: string; authority: string } { + const path = (name: string) => join(root, name); + const openssl = (args: string[]) => { + const result = spawnSync('openssl', args, { encoding: 'utf8', windowsHide: true }); + if (result.status !== 0) fail(`account lookup certificate failed: ${(result.stderr || result.error?.message || '').trim()}`); + }; + const newKey = ['-newkey', 'ec', '-pkeyopt', 'ec_paramgen_curve:P-256', '-nodes']; + try { + openssl(['req', '-x509', ...newKey, '-keyout', path('authority-key.pem'), '-out', path('authority.pem'), + '-days', '1', '-subj', '/CN=Credential broker']); + openssl(['req', ...newKey, '-keyout', path('lookup-key.pem'), '-out', path('lookup.csr'), '-subj', `/CN=${hostname}`]); + writeFileSync(path('lookup.ext'), `subjectAltName=DNS:${hostname}\nextendedKeyUsage=serverAuth\n`, { mode: 0o600 }); + openssl(['x509', '-req', '-in', path('lookup.csr'), '-CA', path('authority.pem'), '-CAkey', path('authority-key.pem'), + '-set_serial', '1', '-days', '1', '-extfile', path('lookup.ext'), '-out', path('lookup.pem')]); + } finally { for (const name of ['authority-key.pem', 'lookup.csr', 'lookup.ext']) rmSync(path(name), { force: true }); } + return { keyPath: path('lookup-key.pem'), certPath: path('lookup.pem'), authority: readFileSync(path('authority.pem'), 'utf8') }; +} + +export async function startCredentialBroker(selectedAuth: ContainerAuth, { networkMode, deadlineMs, + model, providerRoute, maxOutputTokens, maxBudgetUsd = null, pricingRates = null, renewable = false, tools, + env = process.env, docker }: { networkMode: string; deadlineMs: number; model: string; renewable?: boolean; + tools?: ToolPolicy; + maxOutputTokens?: number; maxBudgetUsd?: number | null; pricingRates?: PricingRates | null; + providerRoute?: string; + env?: NodeJS.ProcessEnv; docker?: CredentialBrokerDockerOptions }): Promise { + if (!['bridge', 'host'].includes(networkMode) + && (!docker || networkMode !== `container:${docker.networkContainerId}`)) fail('network mode is invalid'); + if (!Number.isFinite(deadlineMs) || deadlineMs <= 0) fail('deadline is invalid'); + const credential = selectedAuth.credential.trim(); + if (!credential) fail('selected authentication has no broker credential'); + if (docker) { + if (process.platform !== 'linux' || !isAbsolute(docker.privateDirectory)) { + fail('Docker broker requires a Linux controller and an absolute shared private directory'); + } + if (!/^sha256:[a-f0-9]{64}$/.test(docker.imageId) + || !/^[a-f0-9]{64}$/.test(docker.networkContainerId) + || !/^[a-z0-9][a-z0-9_.-]{0,127}$/.test(docker.name) + || !/^[a-f0-9]{32,64}$/.test(docker.creationToken)) fail('Docker broker identity is invalid'); + } + const root = mkdtempSync(join(docker?.privateDirectory ?? tmpdir(), 'stack-bench-credential-broker-')); + let child: ChildProcess | null = null; + let container: CredentialBrokerContainer | undefined; + const processState: BrokerProcessState = { exitCode: null, signal: null, exitedAt: null, + stderrTail: '', stderrPending: '', stderrTruncated: false }; + try { + chmodSync(root, 0o700); + const configPath = join(root, 'config.json'); + const readyPath = join(root, 'ready.json'); + const ledgerPath = join(root, 'spend-ledger.json'); + const heartbeatPath = join(root, 'heartbeat'); + const credentialPath = join(root, 'credential'); + if (renewable) writeFileSync(credentialPath, credential, { mode: 0o600 }); + if (docker) writeFileSync(heartbeatPath, '', { mode: 0o600 }); + const sessionToken = randomBytes(32).toString('hex'); + const listenHost = docker || networkMode === 'host' ? '127.0.0.1' : '0.0.0.0'; + const settings = { provider: selectedAuth.provider, upstream: selectedAuth.upstream, + providerRoute, accountId: selectedAuth.accountId, + mode: selectedAuth.mode, credential, ...(renewable ? { credentialPath } : {}), sessionToken, readyPath, + ...(docker ? { heartbeatPath } : { parentPid: process.pid }), + expiresAt: Date.now() + deadlineMs + 60_000, listenHost, ledgerPath, + model, maxOutputTokens: maxOutputTokens ?? MAX_BROKER_OUTPUT_TOKENS, + ...(maxOutputTokens !== undefined ? { explicitOutputLimit: true } : {}), maxBudgetUsd, pricingRates, + ...(tools ? { tools } : {}) }; + const lookup = brokerProtocol(validateBrokerConfig(settings)).accountLookup; + const certificate = lookup ? lookupCertificate(root, lookup.hostname) : null; + const config = validateBrokerConfig({ ...settings, + ...(certificate ? { tlsKeyPath: certificate.keyPath, tlsCertPath: certificate.certPath } : {}) }); + writeFileSync(configPath, `${JSON.stringify(config)}\n`, { flag: 'wx', mode: 0o600 }); + if (docker) { + const network = `container:${docker.networkContainerId}`; + const id = brokerDocker(['create', '--name', docker.name, + '--label', `${ATTEMPT_CREATION_LABEL}=${docker.creationToken}`, + '--network', network, '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges:true', + '--read-only', '--pids-limit', String(BROKER_CONTAINER_RESOURCE_LIMITS.pids), + '--memory', String(BROKER_CONTAINER_RESOURCE_LIMITS.memoryBytes), + '--memory-swap', String(BROKER_CONTAINER_RESOURCE_LIMITS.memoryBytes), + '--mount', `type=bind,src=${root},dst=${root}`, + '--entrypoint', 'node', docker.imageId, + '/opt/stack-bench/dist/container/credential-broker.js', '--config', configPath]); + if (!/^[a-f0-9]{64}$/.test(id)) fail('Docker broker did not return a container ID'); + container = { id, name: docker.name, image: docker.imageId, owned: true, networkMode: network }; + docker.onCreated(container); + } + child = spawn(container ? 'docker' : process.execPath, + container ? ['start', '--attach', container.id] + : [compiledEntrypoint('container', 'credential-broker.js'), '--config', configPath], { + stdio: ['ignore', 'ignore', 'pipe'], + windowsHide: true, + env: Object.fromEntries(['PATH', 'Path', 'SystemRoot', 'WINDIR', 'SSL_CERT_FILE', + 'NODE_EXTRA_CA_CERTS', 'HTTPS_PROXY', 'HTTP_PROXY'] + .filter(name => env[name] !== undefined).map(name => [name, env[name]])), + }); + const diagnosticSecrets = [credential, sessionToken]; + child.stderr?.on('data', (chunk: Buffer) => appendDiagnosticStderr( + processState, chunk.toString('utf8'), diagnosticSecrets)); + child.once('exit', (code: number | null, signal: NodeJS.Signals | null) => { + appendDiagnosticStderr(processState, '', diagnosticSecrets, true); + processState.exitCode = code; + processState.signal = signal; + processState.exitedAt = new Date().toISOString(); + }); + let spawnError: Error | null = null; + child.once('error', (error: Error) => { spawnError = error; }); + const readyDeadline = Date.now() + 10_000; + while (!spawnError && child.exitCode === null && !existsSync(readyPath) && Date.now() < readyDeadline) { + await wait(100); + } + if (spawnError) throw spawnError; + if (!existsSync(readyPath)) throw new Error(`credential broker did not become ready${processState.stderrTail ? `: ${processState.stderrTail}` : ''}`); + const ready: unknown = JSON.parse(readFileSync(readyPath, 'utf8')); + if (!isRecord(ready) || typeof ready.port !== 'number' || !Number.isInteger(ready.port) + || ready.port < 1 || ready.port > 65_535) throw new Error('credential broker returned an invalid port'); + if (ready.host !== listenHost) throw new Error('credential broker returned an invalid host'); + const host = docker || networkMode === 'host' ? '127.0.0.1' : 'host.docker.internal'; + const heartbeat = docker ? setInterval(() => { + try { const time = new Date(); utimesSync(heartbeatPath, time, time); } catch { /* the broker stops itself */ } + }, 5_000).unref() : undefined; + return { child, root, ledgerPath, model, maxBudgetUsd: maxBudgetUsd ?? null, ...(heartbeat ? { heartbeat } : {}), + sessionToken, baseUrl: `http://${host}:${ready.port}`, listenHost, + endpointKind: docker ? 'container-credential-broker' : 'local-credential-broker', processState, + ...(container ? { container } : {}), + diagnosticSecrets, finalDiagnostics: null, finalLedger: null, certificateAuthority: certificate?.authority ?? null, + replaceCredential: next => { + diagnosticSecrets.push(next); + writeFileSync(`${credentialPath}.next`, next, { mode: 0o600 }); + renameSync(`${credentialPath}.next`, credentialPath); + } }; + } catch (error) { + // Keep private authority when Docker cleanup fails. Recovery uses the saved exact ID. + if (container) brokerDocker(['rm', '-f', container.id]); + if (child?.pid) killTree(child.pid); + rmSync(root, { recursive: true, force: true }); + throw error; + } +} + +function processAlive(pid: number | undefined): boolean { + if (typeof pid !== 'number' || !Number.isInteger(pid) || pid < 1) return false; + try { process.kill(pid, 0); return true; } + catch (error) { return !isRecord(error) || error.code !== 'ESRCH'; } +} + +function brokerExited(broker: CredentialBrokerHandle | null | undefined, alive: Alive): boolean { + if (broker?.container) { + try { + return brokerDocker(['inspect', '--format', '{{.State.Running}}', broker.container.id]) === 'false'; + } catch { + // Docker unavailability is not proof that a provider-capable process stopped. + return false; + } + } + const state = broker?.processState; + if (!state) return !alive(broker?.child?.pid); + if (state.exitedAt || state.exitCode !== null && state.exitCode !== undefined + || state.signal !== null && state.signal !== undefined + || broker?.child?.exitCode !== null && broker?.child?.exitCode !== undefined + || broker?.child?.signalCode !== null && broker?.child?.signalCode !== undefined) return true; + return !alive(broker?.child?.pid); +} + +async function waitForBrokerExit(broker: CredentialBrokerHandle, timeoutMs: number, + { sleep, now, alive }: { sleep: (ms: number) => Promise; now: () => number; alive: Alive }): Promise { + const deadline = now() + timeoutMs; + while (!brokerExited(broker, alive) && now() < deadline) await sleep(BROKER_DRAIN_POLL_MS); + return brokerExited(broker, alive); +} + +export function credentialBrokerDiagnostics(broker: CredentialBrokerHandle | null): BrokerDiagnostics | null { + if (!broker) return null; + if (broker.finalDiagnostics) return structuredClone(broker.finalDiagnostics); + const state = broker.processState ?? {}; + return { + schemaVersion: 1, + endpointKind: broker.endpointKind ?? 'local-credential-broker', + child: { pid: broker.child?.pid ?? null, + exitCode: state.exitCode ?? broker.child?.exitCode ?? null, + signal: state.signal ?? broker.child?.signalCode ?? null, + exitedAt: state.exitedAt ?? null, + stderrTail: state.stderrTail ? redactDiagnosticText(state.stderrTail, broker) : null, + stderrTruncated: state.stderrTruncated === true }, + drain: null, + termination: null, + ledger: null, + errors: [], + }; +} + +export async function stopCredentialBroker(broker: CredentialBrokerHandle | null, { + drainTimeoutMs = BROKER_DRAIN_TIMEOUT_MS, + pollMs = BROKER_DRAIN_POLL_MS, + gracefulTimeoutMs = BROKER_STOP_GRACE_MS, + forceTimeoutMs = BROKER_STOP_FORCE_MS, + readLedger = readCredentialBrokerLedger, + terminate = (pid: Parameters[0]) => { + if (broker?.container) signalBrokerContainer(broker.container, 'KILL'); + else killTree(pid); + }, + requestStop = (child: CredentialBrokerChild) => { + if (broker?.container) signalBrokerContainer(broker.container, 'TERM'); + else child.kill?.('SIGTERM'); + }, + alive = processAlive, + sleep = wait, + now = Date.now, +}: { drainTimeoutMs?: number; pollMs?: number; gracefulTimeoutMs?: number; forceTimeoutMs?: number; + readLedger?: typeof readCredentialBrokerLedger; terminate?: typeof killTree; + requestStop?: (child: CredentialBrokerChild) => boolean | void; alive?: Alive; + sleep?: (ms: number) => Promise; now?: () => number } = {}): Promise { + if (!broker) return null; + if (broker.finalDiagnostics) return structuredClone(broker.finalLedger ?? null); + let ledger: BrokerLedger | null = null; + const startedAt = now(); + let drainTimedOut = false; + let drainReason: string | null = null; + const errors: BrokerError[] = []; + const errorKeys = new Set(); + const recordError = (type: string, phase: string, error: unknown): void => { + const message = redactDiagnosticText(error instanceof Error ? error.message : error, broker) || 'unknown error'; + const key = `${type}:${phase}:${message}`; + if (errorKeys.has(key)) return; + errorKeys.add(key); + errors.push({ type, phase, message }); + }; + const read = (phase: string, expected: { model: string; maxBudgetUsd: number | null }): BrokerLedger | null => { + try { return readLedger(broker.ledgerPath, expected); } + catch (error) { recordError('ledger-read-error', phase, error); return null; } + }; + let gracefulRequested = false; + let forceRequested = false; + let exited = brokerExited(broker, alive); + const expected = { model: broker.model, maxBudgetUsd: broker.maxBudgetUsd }; + try { + const deadline = now() + drainTimeoutMs; + while (drainReason === null) { + ledger = read('drain', expected) ?? ledger; + if (ledger?.complete === true) { drainReason = 'ledger-complete'; break; } + exited = brokerExited(broker, alive); + if (exited) { drainReason = 'child-exited'; break; } + if (now() >= deadline) { drainTimedOut = true; drainReason = 'timeout'; break; } + await sleep(pollMs); + } + exited = brokerExited(broker, alive); + if (!exited) { + gracefulRequested = true; + try { requestStop(broker.child); } + catch (error) { recordError('termination-error', 'graceful-request', error); } + exited = await waitForBrokerExit(broker, gracefulTimeoutMs, { sleep, now, alive }); + } + if (!exited) { + forceRequested = true; + try { terminate(broker.child.pid); } + catch (error) { recordError('termination-error', 'force-request', error); } + exited = await waitForBrokerExit(broker, forceTimeoutMs, { sleep, now, alive }); + } + if (!exited) recordError('termination-error', 'exit-verification', + new Error('credential broker remained alive after forced termination')); + ledger = read('final', expected) ?? ledger; + } catch (error) { recordError('broker-stop-error', 'shutdown', error); } + finally { + clearInterval(broker.heartbeat); + const state = broker.processState ?? {}; + if (exited) { + try { + if (broker.container) brokerDocker(['rm', broker.container.id]); + // Grading can be interrupted before the caller saves its receipt. Keep the + // atomically written spend ledger; remove only the broker's credentials. + for (const name of ['config.json', 'ready.json', 'credential', 'lookup-key.pem']) { + rmSync(join(broker.root, name), { force: true }); + } + } + catch (error) { recordError('cleanup-error', 'private-root', error); } + } + broker.finalLedger = ledger; + broker.finalDiagnostics = { + ...(credentialBrokerDiagnostics(broker) as BrokerDiagnostics), + child: { pid: broker.child?.pid ?? null, + exitCode: state.exitCode ?? broker.child?.exitCode ?? null, + signal: state.signal ?? broker.child?.signalCode ?? null, + exitedAt: state.exitedAt ?? null, + stderrTail: state.stderrTail ? redactDiagnosticText(state.stderrTail, broker) : null, + stderrTruncated: state.stderrTruncated === true }, + drain: { timeoutMs: drainTimeoutMs, elapsedMs: Math.max(0, now() - startedAt), + timedOut: drainTimedOut, reason: drainReason, terminationRequested: gracefulRequested }, + termination: { gracefulRequested, forceRequested, exited, gracefulTimeoutMs, forceTimeoutMs }, + ledger: ledger ? structuredClone(ledger) : null, + errors, + }; + } + return ledger; +} diff --git a/tools/stack-bench/container/credential-broker.ts b/tools/stack-bench/container/credential-broker.ts new file mode 100644 index 00000000000..a135b5abeb9 --- /dev/null +++ b/tools/stack-bench/container/credential-broker.ts @@ -0,0 +1,541 @@ +#!/usr/bin/env node +import { createServer } from 'node:http'; +import type { ClientRequest, IncomingMessage, OutgoingHttpHeaders, ServerResponse } from 'node:http'; +import { request as httpsRequest } from 'node:https'; +import type { RequestOptions } from 'node:https'; +import { connect } from 'node:net'; +import type { Socket } from 'node:net'; +import { createSecureContext, TLSSocket } from 'node:tls'; +import { linkSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parseArgs as parseNodeArgs } from 'node:util'; +import { zstdDecompressSync } from 'node:zlib'; +import type { AddressInfo } from 'node:net'; +import { classifyProviderFailure } from '../src/agents/provider-failure.js'; +import type { ProviderFailure } from '../src/agents/provider-failure.js'; +import { brokerProtocol } from './broker-protocols.js'; + +import { normalizeClaudeUsage } from '../src/evidence/claude-usage-cost.js'; +import type { ClaudeUsage } from '../src/evidence/claude-usage-cost.js'; +import { BROKER_LEDGER_SCHEMA_VERSION, CLAUDE_USAGE_FIELDS, firstUnpricedReason, noEstimates, noUnpriced, + priceNormalizedClaudeUsage, validateBrokerConfig, + writeCredentialBrokerLedger } from './credential-broker-accounting.js'; +import type { BrokerConfig, EstimateReason, PricingRates, UnpricedReason } + from './credential-broker-accounting.js'; + +const MAX_REQUEST_BYTES = 32 * 1024 * 1024; +const BROKER_SERVER_CLOSE_GRACE_MS = 1_000; +const HOP_BY_HOP = ['connection', 'keep-alive', 'proxy-connection', 'te', 'trailer', 'transfer-encoding', 'upgrade']; + +// Grok Build zstd-compresses large request bodies. The broker reads them as JSON and +// forwards the plain body it re-serialises. +function requestBody(body: Buffer, encoding: string | undefined, limit: number): Buffer { + if (!encoding || encoding === 'identity') return body; + if (encoding === 'zstd') return zstdDecompressSync(body, { maxOutputLength: limit }); + throw new Error('unsupported request content-encoding'); +} +export type { ClaudeUsage } from '../src/evidence/claude-usage-cost.js'; +type JsonRecord = Record; +export interface BrokerStats { + acceptedRequests: number; + billableRequests: number; + completedBillableRequests: number; + estimatedBillableRequests: number; + spentUsd: number; + reservedUsd: number; +} + +export interface CreatedCredentialBroker { + server: ReturnType; + stats: () => BrokerStats; + abandonInFlight: () => void; +} + +type UpstreamRequest = (options: RequestOptions, + callback: (response: IncomingMessage) => void) => ClientRequest; + +const roundUsd = (value: number): number => Number(value.toFixed(6)); +const reserveUsd = (value: number): number => Math.ceil(value * 1e6) / 1e6; + +function fail(message: string): never { + throw new Error(`credential broker: ${message}`); +} + +function clientAuthorized(request: IncomingMessage, sessionToken: string): boolean { + return request.headers.authorization === `Bearer ${sessionToken}` + || request.headers['x-api-key'] === sessionToken || request.headers['x-goog-api-key'] === sessionToken; +} + +function requestPath(value: string | undefined): string | null { + try { return new URL(value ?? '', 'http://credential-broker.invalid').pathname; } + catch { return null; } +} + +function rejectRequest(request: IncomingMessage, response: ServerResponse, + status: number, message: string): void { + request.on('error', () => {}); + response.on('error', () => {}); + try { + response.shouldKeepAlive = false; + response.writeHead(status, { 'content-type': 'text/plain', connection: 'close' }); + response.end(message); + } catch { response.destroy(); } + request.resume(); +} + +function requestCostCeiling(bodyBytes: number, maxTokens: number, rates: PricingRates): number { + const inputRate = Math.max(rates.input, rates.cacheRead, rates.cacheWrite5m, rates.cacheWrite1h); + return bodyBytes * inputRate / 1e6 + maxTokens * rates.output / 1e6; +} + +export function createCredentialBroker(configInput: unknown, { + requestUpstream = httpsRequest as UpstreamRequest, + upstream, + maxRequestBytes = MAX_REQUEST_BYTES, +}: { requestUpstream?: UpstreamRequest; + upstream?: { protocol: string; hostname: string; port: number }; + maxRequestBytes?: number } = {}): CreatedCredentialBroker { + const config = validateBrokerConfig(configInput); + const protocol = brokerProtocol(config); + const destination = upstream ?? { protocol: 'https:', hostname: protocol.hostname, port: 443 }; + let acceptedRequests = 0; + // Settles each paid call still open when the broker stops. + const inFlight = new Set<() => void>(); + let lastResponseRequest = 0; + let providerFailure: ProviderFailure | null = null; + // A side call that finishes later must not hide the session's own refused call. + const sideCalls = new Set(); + const recordFailure = (request: number, failure: ProviderFailure | null): void => { + if (sideCalls.has(request)) return; + if (request >= lastResponseRequest) { lastResponseRequest = request; providerFailure = failure; } + }; + let billableRequests = 0; + let completedBillableRequests = 0; + let estimatedBillableRequests = 0; + const estimatedByReason = noEstimates(); + let unpricedBillableRequests = 0; + const unpricedByReason = noUnpriced(); + let webSearchCalls = 0; + let spentUsd = 0; + let providerReportedCostUsd = 0; + let providerIntegrityError: string | undefined; + const upstreamProviders = new Set(); + let reservedUsd = 0; + const usageTotals: ClaudeUsage = { input: 0, output: 0, cacheRead: 0, cacheWrite5m: 0, cacheWrite1h: 0 }; + const recordLedger = () => writeCredentialBrokerLedger(config.ledgerPath, { + schemaVersion: BROKER_LEDGER_SCHEMA_VERSION, + ...(config.provider === 'openrouter' ? { provider: config.provider, providerRoute: config.providerRoute, + providerReportedCostUsd: roundUsd(providerReportedCostUsd), upstreamProviders: [...upstreamProviders], + ...(providerIntegrityError ? { providerIntegrityError } : {}) } : {}), + providerFailure, + model: config.model, + maxBudgetUsd: config.maxBudgetUsd ?? null, + acceptedRequests, + billableRequests, + completedBillableRequests, + estimatedBillableRequests, + estimatedByReason, + unpricedBillableRequests, + unpricedByReason, + webSearchCalls, + spentUsd: Number(spentUsd.toFixed(6)), + reservedUsd: Number(reservedUsd.toFixed(6)), + usage: usageTotals, + complete: reservedUsd === 0 && completedBillableRequests === billableRequests, + updatedAt: new Date().toISOString(), + }); + recordLedger(); + // A renewed sign-in arrives as a new file; the next request uses it. + const reloadCredential = (): void => { + if (!config.credentialPath) return; + try { config.credential = readFileSync(config.credentialPath, 'utf8').trim() || config.credential; } + catch { /* keep the last credential */ } + }; + const server = createServer((request, response) => { + // A client can disappear while the broker is still draining an upstream + // response. Socket errors must not terminate the broker and strand a paid + // request reservation in the ledger. + request.on('error', () => {}); + request.on('aborted', () => {}); + response.on('error', () => {}); + const responseOpen = (): boolean => !response.destroyed && !response.writableEnded; + const writeHead = (status: number, headers: OutgoingHttpHeaders): void => { + if (!responseOpen() || response.headersSent) return; + try { response.writeHead(status, headers); } + catch { response.destroy(); } + }; + const endResponse = (body?: string | Buffer): void => { + if (!responseOpen()) return; + try { response.end(body); } + catch { response.destroy(); } + }; + if (!clientAuthorized(request, config.sessionToken)) { + rejectRequest(request, response, 401, 'unauthorized'); + return; + } + if (providerIntegrityError) { + rejectRequest(request, response, 502, 'provider accounting or routing validation failed'); + return; + } + reloadCredential(); + const path = requestPath(request.url); + const unbilled = path !== null && (request.method === 'GET' ? protocol.readPaths + : request.method === 'POST' ? protocol.controlPaths : undefined)?.has(path); + if (unbilled) { + // An account or configuration read: forwarded with the credential, outside the spend ledger. + const headers = protocol.headers(request); + for (const name of HOP_BY_HOP) delete headers[name]; + if (request.method === 'GET') delete headers['content-length']; + const read = requestUpstream({ protocol: destination.protocol, hostname: destination.hostname, port: destination.port, + method: request.method, path: protocol.upstreamPath(path + new URL(request.url ?? '', 'http://credential-broker.invalid').search), + headers }, upstreamResponse => { + writeHead(upstreamResponse.statusCode ?? 502, upstreamResponse.headers); + upstreamResponse.on('data', (chunk: Buffer) => { + if (!responseOpen()) return; + try { response.write(chunk); } catch { response.destroy(); } + }); + upstreamResponse.on('end', () => endResponse()); + upstreamResponse.once('error', () => { if (responseOpen()) response.destroy(); }); + }); + read.on('error', () => { writeHead(502, { 'content-type': 'text/plain' }); endResponse('upstream request failed'); }); + request.pipe(read); + return; + } + if (request.method !== 'POST' || path === null || !protocol.allowedPaths.has(path)) { + recordFailure(acceptedRequests + 1, { category: 'request', status: 404, code: 'broker-path' }); + recordLedger(); + rejectRequest(request, response, 404, 'not found'); + return; + } + acceptedRequests += 1; + const requestOrdinal = acceptedRequests; + recordLedger(); + + const chunks: Buffer[] = []; + let received = 0; + let tooLarge = false; + request.on('data', (chunk: Buffer) => { + if (tooLarge) return; + received += chunk.length; + if (received > maxRequestBytes) { + tooLarge = true; + recordFailure(requestOrdinal, { category: 'request', status: 413, code: 'broker-body-limit' }); + recordLedger(); + writeHead(413, { 'content-type': 'text/plain' }); + endResponse('request is too large'); + return; + } + chunks.push(chunk); + }); + request.on('end', () => { + if (tooLarge) return; + const body = Buffer.concat(chunks); + let payload: JsonRecord; + try { payload = protocol.parseRequest(requestBody(body, request.headers['content-encoding'], maxRequestBytes), path); } + catch { + recordFailure(requestOrdinal, { category: 'request', status: 400, code: 'broker-request-invalid' }); + recordLedger(); + writeHead(400, { 'content-type': 'text/plain' }); + endResponse('invalid provider request'); + return; + } + if (protocol.sideCall?.(path, payload)) sideCalls.add(requestOrdinal); + const billable = protocol.billable(path) && config.maxBudgetUsd != null; + // Preserve agent capabilities. This cap covers priced spend; receipts + // retain unpriced charges as unknown instead of refusing provider features. + const pricing = protocol.requestPricing(payload); + const costCeiling = billable + ? reserveUsd(requestCostCeiling(received + (protocol.inputTokenAdjustment?.(payload) ?? 0), protocol.outputLimit(payload), + config.pricingRates as PricingRates)) : 0; + const budget = config.maxBudgetUsd; + if (billable && budget !== null && budget !== undefined + && spentUsd + reservedUsd + costCeiling > budget) { + const measuredSpend = config.provider === 'openrouter' ? providerReportedCostUsd + : priceNormalizedClaudeUsage(usageTotals, config.pricingRates as PricingRates); + recordFailure(requestOrdinal, { category: 'broker-budget', status: 402, code: 'reservation-exceeds-budget', + budget: { maxBudgetUsd: budget, spentUsd, reservedUsd, requestCeilingUsd: costCeiling, + estimatedSpendUsd: roundUsd(Math.max(0, spentUsd - measuredSpend)) } }); + recordLedger(); + writeHead(402, { 'content-type': 'text/plain' }); + endResponse('session budget cannot cover the next request reservation'); + return; + } + if (billable) billableRequests += 1; + reservedUsd = roundUsd(reservedUsd + costCeiling); + recordLedger(); + let billableSettled = !billable; + let abandoned = false; + // Without response usage, a declared feature cannot be shown unused. + const unprovenPricing = firstUnpricedReason(pricing.unpriced, pricing.requiresUsage); + const settleBillable = ({ usage = null, estimated = null, reportedCost = null, unpriced = null }: + { usage?: ClaudeUsage | null; estimated?: EstimateReason | null; reportedCost?: number | null; + unpriced?: UnpricedReason | null } = {}): void => { + if (billableSettled) return; + billableSettled = true; + reservedUsd = roundUsd(reservedUsd - costCeiling); + completedBillableRequests += 1; + if (unpriced) { + unpricedBillableRequests += 1; + unpricedByReason[unpriced] += 1; + } + if (estimated) { + estimatedBillableRequests += 1; + estimatedByReason[estimated] += 1; + spentUsd = roundUsd(spentUsd + costCeiling); + } else if (usage) { + spentUsd = roundUsd(spentUsd + (reportedCost ?? priceNormalizedClaudeUsage(usage, config.pricingRates as PricingRates))); + if (reportedCost !== null) { + providerReportedCostUsd = roundUsd(providerReportedCostUsd + reportedCost); + if (pricing.bounded && reportedCost > costCeiling + 0.000001) { + providerIntegrityError = 'OpenRouter reported cost exceeds the request reservation'; + } + } + for (const field of CLAUDE_USAGE_FIELDS) usageTotals[field] += usage[field]; + } + recordLedger(); + }; + const headers = protocol.headers(request); + for (const name of [...HOP_BY_HOP, 'content-encoding']) delete headers[name]; + const forwardedBody = Buffer.from(JSON.stringify(payload)); + headers['content-length'] = String(forwardedBody.length); + const upstreamRequest = requestUpstream({ + protocol: destination.protocol, + hostname: destination.hostname, + port: destination.port, + method: request.method, + path: protocol.upstreamPath(path + new URL(request.url ?? '', 'http://credential-broker.invalid').search), + headers, + }, upstreamResponse => { + writeHead(upstreamResponse.statusCode ?? 502, upstreamResponse.headers); + const responseChunks: Buffer[] = []; + let responseBytes = 0; + upstreamResponse.on('data', (chunk: Buffer) => { + responseBytes += chunk.length; + if (responseBytes <= maxRequestBytes) responseChunks.push(chunk); + if (responseOpen()) { + try { response.write(chunk); } + catch { response.destroy(); } + } + }); + upstreamResponse.on('end', () => { + const status = upstreamResponse.statusCode ?? 502; + recordFailure(requestOrdinal, status >= 200 && status < 300 ? null + : classifyProviderFailure(status, Buffer.concat(responseChunks))); + recordLedger(); + endResponse(); + if (!billable) return; + if ((upstreamResponse.statusCode ?? 502) >= 200 + && (upstreamResponse.statusCode ?? 502) < 300) { + const usage = responseBytes <= maxRequestBytes + ? protocol.responseUsage(Buffer.concat(responseChunks), upstreamResponse.headers['content-encoding']) + : null; + const rejection = usage || responseBytes > maxRequestBytes ? null + : protocol.responseRejection?.(Buffer.concat(responseChunks), upstreamResponse.headers['content-encoding']) ?? null; + if (rejection) { + // The provider refused before producing output, so nothing is charged, as for an error status. + recordFailure(requestOrdinal, { ...rejection, status }); + settleBillable(); + } else if (!usage) { + if (config.provider === 'openrouter') providerIntegrityError = 'OpenRouter response lacks verified cost and routing metadata'; + recordFailure(requestOrdinal, { category: 'transport', status, code: 'incomplete-response' }); + settleBillable({ estimated: 'no-usage', unpriced: unprovenPricing }); + } + else try { + if (typeof usage.upstream_provider === 'string') upstreamProviders.add(usage.upstream_provider); + if (Number.isSafeInteger(usage.web_search_calls)) webSearchCalls += usage.web_search_calls as number; + settleBillable({ usage: normalizeClaudeUsage(usage), + reportedCost: typeof usage.provider_reported_cost_usd === 'number' ? usage.provider_reported_cost_usd : null, + unpriced: firstUnpricedReason(pricing.unpriced, usage.unpriced_reason as UnpricedReason | null) }); + } + catch { + recordFailure(requestOrdinal, { category: 'transport', status, code: 'invalid-usage' }); + settleBillable({ estimated: 'no-usage', unpriced: unprovenPricing }); + } + } else { + settleBillable(); + } + }); + const settleAbortedResponse = () => { + if (!abandoned) recordFailure(requestOrdinal, { category: 'transport', status: null, code: null }); + settleBillable({ estimated: 'response-aborted', unpriced: unprovenPricing }); + if (responseOpen()) response.destroy(); + }; + upstreamResponse.once('aborted', settleAbortedResponse); + upstreamResponse.once('error', settleAbortedResponse); + }); + // The broker stops only after the coding session has ended. A call still open + // then can no longer reach the agent; its spend is the reservation ceiling. + const abandon = () => { + abandoned = true; + settleBillable({ estimated: 'response-aborted', unpriced: unprovenPricing }); + upstreamRequest.destroy(); + }; + inFlight.add(abandon); + upstreamRequest.once('close', () => inFlight.delete(abandon)); + // A request whose connection never opened did not reach the provider. + let connected = false; + upstreamRequest.on('socket', socket => { + if (!socket.connecting) connected = true; + else socket.once(destination.protocol === 'https:' ? 'secureConnect' : 'connect', () => { connected = true; }); + }); + upstreamRequest.on('error', () => { + if (abandoned) return; + if (connected) { + recordFailure(requestOrdinal, { category: 'transport', status: null, code: null }); + settleBillable({ estimated: 'upstream-error', unpriced: unprovenPricing }); + } else { + recordFailure(requestOrdinal, { category: 'unreachable', status: null, code: null }); + settleBillable(); + } + writeHead(502, { 'content-type': 'text/plain' }); + endResponse('upstream request failed'); + }); + upstreamRequest.end(forwardedBody); + }); + }); + // The CLI sends its account check to another host through its HTTPS proxy, which is this + // broker. The broker terminates TLS for that host alone, with a per-session certificate the + // CLI trusts, and answers the check with the account's id and nothing that names the person. + // Every other host is tunnelled unchanged: the broker shares the coding container's network. + const lookup = protocol.accountLookup; + if (lookup && config.tlsKeyPath && config.tlsCertPath) { + const secureContext = createSecureContext({ key: readFileSync(config.tlsKeyPath), cert: readFileSync(config.tlsCertPath) }); + const lookupServer = createServer((request, response) => { + response.on('error', () => {}); + if (request.method !== 'GET' || requestPath(request.url) !== lookup.path + || !clientAuthorized(request, config.sessionToken)) { + rejectRequest(request, response, 404, 'not found'); + return; + } + reloadCredential(); + const target = upstream ?? { protocol: 'https:', hostname: lookup.hostname, port: 443 }; + requestUpstream({ ...target, method: 'GET', path: request.url, + headers: { authorization: `Bearer ${config.credential}`, accept: 'application/json' } }, upstreamResponse => { + const chunks: Buffer[] = []; + upstreamResponse.on('data', (chunk: Buffer) => chunks.push(chunk)); + upstreamResponse.on('end', () => { + const status = upstreamResponse.statusCode ?? 502; + let id: unknown = null; + try { id = (JSON.parse(Buffer.concat(chunks).toString('utf8')) as JsonRecord).id; } catch { /* no id */ } + const body = status === 200 && typeof id === 'string' + ? JSON.stringify({ id, email: 'agent@stack-bench.invalid', verified_email: true }) : '{}'; + response.writeHead(status, { 'content-type': 'application/json', 'content-length': Buffer.byteLength(body) }); + response.end(body); + }); + }).on('error', () => rejectRequest(request, response, 502, 'upstream request failed')).end(); + }); + server.on('connect', (request: IncomingMessage, client: Socket, head: Buffer) => { + client.on('error', () => {}); + const [host = '', port = '443'] = (request.url ?? '').split(':'); + if (host === lookup.hostname && port === '443') { + client.write('HTTP/1.1 200 Connection Established\r\n\r\n'); + lookupServer.emit('connection', new TLSSocket(client, { isServer: true, secureContext })); + return; + } + const tunnel = connect(Number(port), host, () => { + client.write('HTTP/1.1 200 Connection Established\r\n\r\n'); + tunnel.write(head); + tunnel.pipe(client); + client.pipe(tunnel); + }); + tunnel.on('error', () => client.destroy()); + client.on('close', () => tunnel.destroy()); + }); + } + // Node closes a connection idle for five seconds. A client that reuses it at that moment + // loses the request it was sending (seen from agy as a broken pipe). The broker serves + // one session and closes its connections when it stops, so idle ones stay open. + server.keepAliveTimeout = 0; + server.on('clientError', (_error: Error, socket: Socket) => { + socket.on('error', () => {}); + if (socket.writable) socket.end('HTTP/1.1 400 Bad Request\r\nConnection: close\r\n\r\n'); + else socket.destroy(); + }); + return { server, abandonInFlight: () => { for (const abandon of inFlight) abandon(); }, stats: () => ({ acceptedRequests, + billableRequests, completedBillableRequests, estimatedBillableRequests, + estimatedByReason: { ...estimatedByReason }, + spentUsd: Number(spentUsd.toFixed(6)), reservedUsd: Number(reservedUsd.toFixed(6)) }) }; +} + +function parseArgs(argv: string[]): string { + const { values } = parseNodeArgs({ args: argv, options: { config: { type: 'string' } } }); + const configPath = values.config; + if (!configPath || argv.length !== 2) fail('use --config '); + return resolve(configPath); +} + +async function main() { + const configPath = parseArgs(process.argv.slice(2)); + let config: BrokerConfig; + try { config = validateBrokerConfig(JSON.parse(readFileSync(configPath, 'utf8'))); } + finally { rmSync(configPath, { force: true }); } + if (!config.readyPath) fail('readyPath is invalid'); + const { server, abandonInFlight } = createCredentialBroker(config); + const sockets = new Set(); + server.on('connection', (socket: Socket) => { + sockets.add(socket); + socket.once('close', () => sockets.delete(socket)); + }); + server.on('error', (error: Error) => { + process.stderr.write(`credential broker: ${error.message}\n`); + process.exitCode = 1; + }); + const readyPath = config.readyPath; + server.listen(0, config.listenHost ?? '127.0.0.1', () => { + const address: string | AddressInfo | null = server.address(); + if (!address || typeof address === 'string') fail('listener address is unavailable'); + // Publish the file whole: the starter polls for it and reads it at once. + const pending = `${readyPath}.pending`; + writeFileSync(pending, `${JSON.stringify({ host: address.address, port: address.port })}\n`, + { flag: 'wx', mode: 0o600 }); + linkSync(pending, readyPath); + rmSync(pending); + }); + let stopping = false; + const stop = () => { + if (stopping) return; + stopping = true; + abandonInFlight(); + const force = setTimeout(() => { + for (const socket of sockets) socket.destroy(); + server.closeAllConnections?.(); + process.exit(0); + }, BROKER_SERVER_CLOSE_GRACE_MS); + force.unref(); + server.close(() => { + clearTimeout(force); + process.exit(0); + }); + server.closeIdleConnections?.(); + }; + const parentPid = config.parentPid; + if (parentPid) { + setInterval(() => { + try { process.kill(parentPid, 0); } + catch { stop(); } + }, 1_000).unref(); + } + // A Docker broker cannot see the controller process; the controller touches this + // file instead, and a stale file means it died. + const heartbeatPath = config.heartbeatPath; + if (heartbeatPath) { + setInterval(() => { if (heartbeatStale(heartbeatPath, Date.now())) stop(); }, 5_000).unref(); + } + const expiresAt = config.expiresAt; + if (expiresAt) setTimeout(stop, Math.max(1, expiresAt - Date.now())).unref(); + process.on('SIGINT', stop); + process.on('SIGTERM', stop); +} + +export function heartbeatStale(path: string, now: number): boolean { + try { return now - statSync(path).mtimeMs > 60_000; } + catch { return true; } +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + main().catch((error: unknown) => { + process.stderr.write(`${error instanceof Error ? error.stack ?? error.message : String(error)}\n`); + process.exit(1); + }); +} diff --git a/tools/stack-bench/container/reconcile-build-container.ts b/tools/stack-bench/container/reconcile-build-container.ts new file mode 100644 index 00000000000..a58fce33aaf --- /dev/null +++ b/tools/stack-bench/container/reconcile-build-container.ts @@ -0,0 +1,77 @@ +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import type { SpawnSyncOptionsWithStringEncoding, SpawnSyncReturns } from 'node:child_process'; + +export const BUILD_CONTAINER_CREATION_LABEL = 'com.clockworklabs.stack-bench.creation'; + +export function buildContainerName(lease: { runId: string; + resources: { buildContainer?: { name: string } | null } }): string { + return lease.resources.buildContainer?.name + ?? `sb-${createHash('sha256').update(lease.runId).digest('hex').slice(0, 16)}-build`; +} + +const CONTAINER_ID = /^[a-f0-9]{64}$/i; + +type DockerResult = SpawnSyncReturns; +type DockerExecute = (command: string, args: readonly string[], + options: SpawnSyncOptionsWithStringEncoding) => DockerResult; + +export interface RemoveFailedBuildContainerOptions { + containerName: string; + creationToken: string; + createdId?: string | null; + dockerEnv?: NodeJS.ProcessEnv; + timeoutMs?: number; + execute?: DockerExecute; +} + +export function containerIdFromDockerOutput(output: unknown): string | null { + return String(output ?? '').split(/\r?\n/).map(line => line.trim()) + .find(line => CONTAINER_ID.test(line)) ?? null; +} + +function detail(result: DockerResult): string { + return String(result.stderr || result.stdout || result.error?.message + || `exit ${result.status}`).trim(); +} + +export function removeFailedBuildContainer({ containerName, creationToken, createdId = null, + dockerEnv = process.env, timeoutMs = 120_000, + execute = spawnSync as DockerExecute }: RemoveFailedBuildContainerOptions): { + removed: boolean; absent: boolean; id?: string; +} { + if (typeof containerName !== 'string' || !containerName) { + throw new Error('failed build-container cleanup requires a container name'); + } + if (typeof creationToken !== 'string' || !creationToken) { + throw new Error('failed build-container cleanup requires a creation token'); + } + + let id = containerIdFromDockerOutput(createdId); + if (!id) { + const inspected = execute('docker', ['inspect', '--format', + `{{.Id}} {{index .Config.Labels "${BUILD_CONTAINER_CREATION_LABEL}"}}`, containerName], { + encoding: 'utf8', env: dockerEnv, timeout: timeoutMs, + }); + if (inspected.status !== 0) { + const reason = detail(inspected); + if (/no such (?:object|container)/i.test(reason)) return { removed: false, absent: true }; + throw new Error(`cannot prove cleanup of failed container ${containerName}: ${reason}`); + } + const [inspectedId, label, ...extra] = String(inspected.stdout ?? '').trim().split(/\s+/); + if (!CONTAINER_ID.test(inspectedId ?? '') || label !== creationToken || extra.length > 0) { + throw new Error(`refusing to remove ${containerName}: its creation identity does not match`); + } + id = inspectedId ?? null; + } + + if (!id) throw new Error(`cannot prove cleanup of failed container ${containerName}`); + + const removed = execute('docker', ['rm', '-f', id], { + encoding: 'utf8', env: dockerEnv, timeout: timeoutMs, + }); + if (removed.status !== 0) { + throw new Error(`could not remove failed build container ${id}: ${detail(removed)}`); + } + return { removed: true, absent: false, id }; +} diff --git a/tools/stack-bench/container/recover-build-container.ts b/tools/stack-bench/container/recover-build-container.ts new file mode 100644 index 00000000000..acefebcb228 --- /dev/null +++ b/tools/stack-bench/container/recover-build-container.ts @@ -0,0 +1,77 @@ +import { spawnSync } from 'node:child_process'; +import type { SpawnSyncOptionsWithStringEncoding } from 'node:child_process'; + +import { updateBackendLease } from '../src/runtime/backend-lease.js'; +import type { BackendLease } from '../src/runtime/backend-lease.js'; + +interface StoppedBuildContainer { + id: string; + running: false; +} + +interface LeaseContext { + path: string; + lease: BackendLease; +} + +export interface DockerExecuteResult { + status: number | null; + stdout?: string; + stderr?: string; + error?: Error; +} + +export type DockerExecute = (command: string, args: readonly string[], + options: SpawnSyncOptionsWithStringEncoding) => DockerExecuteResult; + +export interface RecoverStoppedBuildContainerOptions { + existing: StoppedBuildContainer; + containerName: string; + leaseContext: LeaseContext; + backend: string; + dockerEnv?: NodeJS.ProcessEnv; + timeoutMs?: number; + execute?: DockerExecute; +} + +function clearBuildContainerLease(leaseContext: LeaseContext, backend: string, + containerId: string, description: string): LeaseContext { + const lease = updateBackendLease(leaseContext.path, { + token: leaseContext.lease.ownershipToken, backend, runId: leaseContext.lease.runId, + }, next => { + if (next.resources.buildContainer?.id !== containerId) { + throw new Error(`${description} ownership changed before recovery`); + } + next.resources.buildContainer = null; + return next; + }); + return { path: leaseContext.path, lease }; +} + +export function recoverStoppedBuildContainer({ existing, containerName, leaseContext, backend, + dockerEnv = process.env, timeoutMs = 120_000, + execute = spawnSync }: RecoverStoppedBuildContainerOptions): LeaseContext { + const prior = leaseContext?.lease?.resources?.buildContainer ?? null; + if (!existing || existing.running) throw new Error('recovery requires a stopped container'); + if (!prior || prior.name !== containerName || prior.id !== existing.id) { + throw new Error('stopped container does not match the authenticated lease'); + } + const removed = execute('docker', ['rm', existing.id], { + encoding: 'utf8', env: dockerEnv, timeout: timeoutMs, + }); + if (removed.status !== 0) { + throw new Error(`could not remove exact stopped leased container ${existing.id}: ` + + String(removed.stderr || removed.stdout || removed.error?.message || `exit ${removed.status}`).trim()); + } + return clearBuildContainerLease(leaseContext, backend, existing.id, 'stopped container'); +} + +export function clearMissingBuildContainerLease({ containerName, leaseContext, backend }: { + containerName: string; leaseContext: LeaseContext; backend: string; +}): LeaseContext { + const prior = leaseContext?.lease?.resources?.buildContainer ?? null; + if (!prior || prior.name !== containerName) { + throw new Error('missing container does not match the authenticated lease'); + } + return clearBuildContainerLease(leaseContext, backend, prior.id, 'missing container'); +} diff --git a/tools/stack-bench/container/run-build.ts b/tools/stack-bench/container/run-build.ts new file mode 100644 index 00000000000..5d8c4081ad3 --- /dev/null +++ b/tools/stack-bench/container/run-build.ts @@ -0,0 +1,758 @@ +#!/usr/bin/env node +// The build container must not expose Stack Bench source or grading material. +import { spawnSync } from 'node:child_process'; +import { randomBytes } from 'node:crypto'; +import { chmodSync, existsSync, readFileSync, mkdirSync, writeFileSync, unlinkSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { resolve, dirname } from 'node:path'; +import { parseArgs } from 'node:util'; +import { leaseFromEnv, updateBackendLease } from '../src/runtime/backend-lease.js'; +import { resolveContainerImage } from '../src/runtime/container-image.js'; +import { leasedDatabaseEnvironment, STACK_ADAPTER_REGISTRY } from '../src/stacks/stack-adapters.js'; +import { BUILD_CONTAINER_RESOURCE_LIMITS, DEFAULT_BUILD_IMAGE } + from '../src/composition/product-config.js'; +import { dockerMountArguments } from '../src/runtime/container-mount.js'; +import type { ContainerMount } from '../src/runtime/container-mount.js'; +import { dockerHostGatewayArguments, requireAttemptNetwork, attemptControllerImage, + recordAttemptCreation, ATTEMPT_CREATION_LABEL } from '../src/runtime/docker-network.js'; +import { packageRegistry, packageRegistryEnvironment } from '../src/runtime/package-registry.js'; +import { resolveContainerAuth } from './container-auth.js'; +import { hasRequiredBuildContainerIsolation, inspectBuildContainer, parseCgroupResources, + parsePublishedPorts, waitForBuildContainerReady } + from './build-container-inspection.js'; +import { reconcileCredentialBrokerReceipt } from './credential-broker-accounting.js'; +import { credentialBrokerDiagnostics, startCredentialBroker, stopCredentialBroker } + from './credential-broker-process.js'; +import { clearMissingBuildContainerLease, + recoverStoppedBuildContainer } from './recover-build-container.js'; +import { BUILD_CONTAINER_CREATION_LABEL, buildContainerName, containerIdFromDockerOutput, + removeFailedBuildContainer } from './reconcile-build-container.js'; +import { CODING_SESSION_TIMEOUT_MS } from '../src/agents/coding-session-timeouts.js'; +import { CODING_CONTAINER_AGENT, CODING_CONTAINER_APP_ROOT, CODING_CONTAINER_CONTROL_DIR, + CODING_CONTAINER_PROCESS_IDENTITY, + codingContainerAgentEnvironment, codingContainerTranscriptHandoffCommands, + codingContainerWorkspaceHandoffCommands } + from '../src/runtime/coding-container-policy.js'; +import { PRICING_UNIT, validatePricingAuthority } + from '../src/evidence/pricing-authority.js'; +import { CODING_PROVIDERS, parseCodingProvider } from './coding-providers.js'; +import { ensureFreshGrokLogin } from '../src/agents/grok-login.js'; +import { ensureFreshAgyLogin } from '../src/agents/agy-login.js'; +import { agyRenewal } from './agy-renewal.js'; +import { validateProviderRoute, validateProviderOutputLimit } from '../src/agents/agent-adapter-contract.js'; +import { REPOSITORY_ROOT } from '../src/package-root.js'; +import { DEFAULT_TOOL_POLICY, parseToolPolicySwitch, unavailableToolsReason } from '../src/agents/tool-policy.js'; +import type { ToolPolicy } from '../src/agents/tool-policy.js'; + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +async function runBuild(): Promise { +const { values } = parseArgs({ options: { + app: { type: 'string' }, backend: { type: 'string' }, 'prepare-only': { type: 'boolean' }, + provider: { type: 'string' }, image: { type: 'string' }, effort: { type: 'string' }, model: { type: 'string' }, + 'provider-route': { type: 'string' }, + 'max-output-tokens': { type: 'string' }, + 'max-budget-usd': { type: 'string' }, 'pricing-json': { type: 'string' }, + 'resume-session': { type: 'string' }, 'prior-session-usage': { type: 'string' }, + 'recover-stopped-container': { type: 'boolean' }, + 'completion-marker': { type: 'string' }, ports: { type: 'string' }, + 'web-search': { type: 'string' }, subagents: { type: 'string' }, +} }); +const prepareOnly = values['prepare-only'] ?? false; + +const appDir = values.app; +if (!appDir) { console.error('run-build.js: --app is required'); process.exit(2); } +const backend = values.backend; +if (!backend) { console.error('run-build.js: --backend is required'); process.exit(2); } +let adapter; +try { adapter = STACK_ADAPTER_REGISTRY.get(backend); } +catch (error) { console.error(`run-build.js: ${errorMessage(error)}`); process.exit(2); } +const provider = parseCodingProvider(values.provider ?? 'anthropic'); +const providerRoute = validateProviderRoute(provider, values['provider-route']); +const maxOutputTokens = validateProviderOutputLimit(provider, + values['max-output-tokens'] === undefined ? undefined : Number(values['max-output-tokens'])); +const codingProvider = CODING_PROVIDERS[provider]; +const DOCKER_TIMEOUT_MS = 120_000; +const DOCKER_PROBE_TIMEOUT_MS = 10_000; +// A renewed Google token has an hour; checks every ten minutes keep at least half an hour. +const AGY_MIN_REMAINING_MS = 40 * 60_000; +const AGY_RENEWAL_INTERVAL_MS = 10 * 60_000; +const { uid: AGENT_UID, gid: AGENT_GID, home: AGENT_HOME } = CODING_CONTAINER_AGENT; +const CONTROLLER_GID = process.getgid?.() ?? 0; +const AGENT_ENVIRONMENT = codingContainerAgentEnvironment(); +const CONTROL_DIR = CODING_CONTAINER_CONTROL_DIR; +const REQUIRED_CAPABILITIES = Object.freeze([ + 'CHOWN', 'DAC_OVERRIDE', 'FOWNER', 'KILL', 'SETGID', 'SETUID', +]); +const REQUIRED_TMPFS = Object.freeze({ + '/tmp': 'rw,nosuid,nodev,mode=1777', + [AGENT_HOME]: `rw,nosuid,nodev,uid=${AGENT_UID},gid=${AGENT_GID},mode=0700`, + [`${AGENT_HOME}/.claude`]: `rw,nosuid,nodev,uid=${AGENT_UID},gid=${AGENT_GID},mode=0700`, + '/deps': 'rw,exec,nosuid,nodev,mode=0755', + [CONTROL_DIR]: 'rw,nosuid,nodev,mode=0700', +}); + +const REPO = REPOSITORY_ROOT; +const imageReference = values.image ?? DEFAULT_BUILD_IMAGE; +let imageIdentity; +try { imageIdentity = resolveContainerImage(imageReference); } +catch (error) { + console.error(`run-build.js: cannot resolve image ${imageReference}: ${errorMessage(error)}`); + process.exit(2); +} +const image = imageIdentity.id; +const effort = values.effort ?? ''; +const model = values.model ?? ''; +if (!prepareOnly && (!effort || !model)) { + console.error('run-build.js: --effort and --model are required'); + process.exit(2); +} +let tools: ToolPolicy; +try { + tools = { webSearch: parseToolPolicySwitch(values['web-search'], DEFAULT_TOOL_POLICY.webSearch, '--web-search'), + subagents: parseToolPolicySwitch(values.subagents, DEFAULT_TOOL_POLICY.subagents, '--subagents') }; + const unavailable = unavailableToolsReason(provider, tools); + if (unavailable) throw new Error(unavailable); +} catch (error) { + console.error(`run-build.js: ${errorMessage(error)}`); + process.exit(2); +} +const maxBudgetUsd = values['max-budget-usd'] ?? null; +if (!prepareOnly && codingProvider.requiresBudget && maxBudgetUsd === null) { + throw new Error('this coding provider requires --max-budget-usd and explicit pricing'); +} +if (maxBudgetUsd !== null && (!Number.isFinite(Number(maxBudgetUsd)) || Number(maxBudgetUsd) <= 0)) { + console.error('run-build.js: --max-budget-usd must be a positive number'); + process.exit(2); +} +let pricing = null; +try { + const supplied = values['pricing-json'] ?? null; + if (supplied !== null) { + pricing = validatePricingAuthority(JSON.parse(supplied), { at: '--pricing-json' }); + } else if (maxBudgetUsd !== null) { + const rates = codingProvider.rates(model); + if (!rates) throw new Error(`no default pricing is recorded for model ${model}`); + pricing = validatePricingAuthority({ unit: PRICING_UNIT, rates }, + { at: 'default pricing' }); + } +} catch (error) { + console.error(`run-build.js: ${errorMessage(error)}`); + process.exit(2); +} +const resumeSession = values['resume-session'] ?? null; +if (resumeSession !== null + && !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(resumeSession)) { + console.error('run-build.js: --resume-session must be a UUID'); + process.exit(2); +} +const priorSessionUsage = values['prior-session-usage'] ? JSON.parse(values['prior-session-usage']) : null; +if (priorSessionUsage !== null && (resumeSession === null || !['input_tokens', 'output_tokens', + 'cache_read_input_tokens', 'cache_creation_input_tokens'].every(key => + Number.isSafeInteger(priorSessionUsage[key]) && priorSessionUsage[key] >= 0))) { + console.error('run-build.js: --prior-session-usage needs --resume-session and four token counts'); + process.exit(2); +} +const recoverStoppedContainer = values['recover-stopped-container'] ?? false; +const completionMarker = values['completion-marker'] ?? null; +if (!prepareOnly && !/^[A-Z][A-Z0-9_]*$/.test(completionMarker ?? '')) { + console.error('run-build.js: --completion-marker must be an uppercase marker'); + process.exit(2); +} +let ports: string[] = []; +try { ports = parsePublishedPorts(values.ports); } +catch (error) { console.error(`run-build.js: ${errorMessage(error)}`); process.exit(2); } + +const containerPlan = adapter!.buildContainer.plan({ + repo: REPO, appDir, env: process.env, +}); + +// Auth is resolved in the controller. A short-lived broker forwards model API +// requests later. The coding container never receives the long-lived provider +// credential or a credential file. +const apiKey = process.env.STACK_BENCH_AGENT_API_KEY + ?? process.env[codingProvider.apiKeyEnvironment] ?? ''; +let auth = null; +if (!prepareOnly) { + // A SuperGrok token lasts hours and rotates on refresh: renew the shared sign-in + // so this session's snapshot outlives the session. + if (provider === 'xai' && !apiKey && process.env.GROK_AUTH_FILE) { + try { await ensureFreshGrokLogin(process.env.GROK_AUTH_FILE, CODING_SESSION_TIMEOUT_MS + 15 * 60_000); } + catch (error) { console.error(`run-build.js: ${errorMessage(error)}`); process.exit(2); } + } + try { auth = resolveContainerAuth({ provider, apiKey, env: process.env, credentialsPath: codingProvider.credentialPath }); } + catch (error) { console.error(`run-build.js: ${errorMessage(error)}`); process.exit(2); } +} + +// Persist this run's transcript without exposing other local sessions. +const projects = prepareOnly ? null : codingProvider.projects(appDir); +const containerTranscripts = codingProvider.containerTranscripts; +function ensureAgentDirectory(directory: string): void { + mkdirSync(directory, { recursive: true, + mode: process.env.STACK_BENCH_APPLIANCE === '1' ? 0o700 : 0o777 }); + if (process.env.STACK_BENCH_APPLIANCE !== '1') chmodSync(directory, 0o777); +} + +ensureAgentDirectory(appDir); +if (projects) ensureAgentDirectory(projects); +for (const directory of containerPlan.ensureDirectories) ensureAgentDirectory(directory); + +const dockerEnv: NodeJS.ProcessEnv = { ...process.env, MSYS_NO_PATHCONV: '1' }; + +function resolveNetworkMode(): string { + if (process.env.STACK_BENCH_APPLIANCE !== '1') return 'bridge'; + return requireAttemptNetwork(leaseFromEnv(process.env, { backend, active: true }).lease); +} + +let expectedNetworkMode: string; +try { expectedNetworkMode = resolveNetworkMode(); } +catch (error) { + console.error(`run-build.js: ${errorMessage(error)}`); + process.exit(2); +} + +const inspectContainer = (name: string) => inspectBuildContainer(name, + { env: dockerEnv, timeoutMs: DOCKER_TIMEOUT_MS }); + +const hasRequiredIsolation = (container: NonNullable>, + expectedMounts: ContainerMount[]): boolean => hasRequiredBuildContainerIsolation(container, { + expectedMounts, + requiredTmpfs: REQUIRED_TMPFS, + requiredCapabilities: REQUIRED_CAPABILITIES, + pidsLimit: BUILD_CONTAINER_RESOURCE_LIMITS.pids, + cpuCount: BUILD_CONTAINER_RESOURCE_LIMITS.cpuCount, + memoryBytes: BUILD_CONTAINER_RESOURCE_LIMITS.memoryBytes, + memorySwapBytes: BUILD_CONTAINER_RESOURCE_LIMITS.memorySwapBytes, + image, +}); + +const expectedMounts: ContainerMount[] = [ + { kind: 'bind' as const, source: resolve(appDir), target: CODING_CONTAINER_APP_ROOT, readOnly: false }, + ...(projects ? [{ kind: 'bind' as const, source: projects, + target: containerTranscripts, readOnly: false }] : []), + ...containerPlan.mounts, +]; + +// Only the lease's immutable container id grants reuse or deletion authority. +let leaseContext; +try { leaseContext = leaseFromEnv(process.env, { backend, active: true }); } +catch (error) { + console.error(`run-build.js: an authenticated active backend lease is required: ${errorMessage(error)}`); + process.exit(3); +} + +// App directories can share a parent. Container identity belongs to the lease. +const containerName = buildContainerName(leaseContext.lease); +let existing = inspectContainer(containerName); +const priorContainer = leaseContext.lease.resources.buildContainer ?? null; +if (existing) { + if (!priorContainer) { + console.error(`run-build.js: refusing to adopt existing unleased container ${containerName}`); + process.exit(3); + } + if (priorContainer.name !== containerName || priorContainer.id !== existing.id) { + console.error(`run-build.js: existing container ${containerName}/${existing.id} does not match lease ` + + `${priorContainer.name}/${priorContainer.id}`); + process.exit(3); + } + if (!existing.running) { + if (!recoverStoppedContainer) { + console.error(`run-build.js: leased container ${containerName} stopped unexpectedly; refusing to replace it`); + process.exit(3); + } + try { + leaseContext = recoverStoppedBuildContainer({ existing: { ...existing, running: false }, containerName, leaseContext, backend, + dockerEnv, timeoutMs: DOCKER_TIMEOUT_MS }); + existing = null; + } catch (error) { + console.error(`run-build.js: could not recover stopped container: ${errorMessage(error)}`); + process.exit(3); + } + } + if (existing && existing.networkMode !== expectedNetworkMode) { + console.error(`run-build.js: leased container ${containerName} uses network ${existing.networkMode}, ` + + `expected ${expectedNetworkMode}`); + process.exit(3); + } + if (existing?.unsafeCredentialExposure) { + console.error(`run-build.js: leased container ${containerName} was created with a provider credential; ` + + 'reconcile the run and start it with the isolated credential broker'); + process.exit(3); + } + if (existing && !hasRequiredIsolation(existing, expectedMounts)) { + console.error(`run-build.js: leased container ${containerName} does not have the required isolation`); + process.exit(3); + } +} else if (priorContainer) { + const leasedById = inspectContainer(priorContainer.id); + if (leasedById) { + console.error(`run-build.js: leased container ${priorContainer.id} still exists under an unexpected name`); + process.exit(3); + } + if (!recoverStoppedContainer) { + console.error(`run-build.js: leased container ${priorContainer.name}/${priorContainer.id} is missing`); + process.exit(3); + } + try { + leaseContext = clearMissingBuildContainerLease({ containerName, leaseContext, backend }); + } catch (error) { + console.error(`run-build.js: could not recover missing container lease: ${errorMessage(error)}`); + process.exit(3); + } +} + +// Create it if this is the first round of the run; reuse it for every round +// after, so a repair finds the app, its node_modules and its servers exactly +// where the build round left them. +let containerInspection = existing; +if (!existing) { + const creationToken = randomBytes(16).toString('hex'); + if (leaseContext.lease.resources.network) updateBackendLease(leaseContext.path, + { token: leaseContext.lease.ownershipToken }, next => { + (next.resources.creationIntents ??= {}).build = { name: containerName, creationToken }; + return next; + }); + const create = [ + 'create', '--init', '--name', containerName, + '--label', `${BUILD_CONTAINER_CREATION_LABEL}=${creationToken}`, + '--label', `${ATTEMPT_CREATION_LABEL}=${creationToken}`, + '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges:true', + '--pids-limit', String(BUILD_CONTAINER_RESOURCE_LIMITS.pids), + '--cpus', String(BUILD_CONTAINER_RESOURCE_LIMITS.cpuCount), + '--memory', String(BUILD_CONTAINER_RESOURCE_LIMITS.memoryBytes), + '--memory-swap', String(BUILD_CONTAINER_RESOURCE_LIMITS.memorySwapBytes), + // The agent may write the app, its own home directory, and temporary files. + // It must not replace system binaries or libraries used by later grading. + '--read-only', + '-v', `${resolve(appDir)}:${CODING_CONTAINER_APP_ROOT}`, + ]; + for (const capability of REQUIRED_CAPABILITIES) create.push('--cap-add', capability); + for (const [path, options] of Object.entries(REQUIRED_TMPFS)) { + create.push('--tmpfs', `${path}:${options}`); + } + create.push('--network', expectedNetworkMode); + create.push(...dockerHostGatewayArguments(expectedNetworkMode)); + if (projects) create.push('-v', `${projects}:${containerTranscripts}`); + // The selected adapter owns every stack-specific mount. Giving a treatment + // another stack's artifacts would violate the "only artifacts under test" + // boundary. + for (const requiredPath of containerPlan.requiredPaths) { + if (!existsSync(requiredPath)) { + console.error(`run-build.js: ${backend} container artifact is missing: ${requiredPath}`); + process.exit(2); + } + } + for (const mount of containerPlan.mounts) { + try { create.push(...dockerMountArguments(mount)); } + catch (error) { + console.error(`run-build.js: ${backend} adapter returned an invalid container mount: ${errorMessage(error)}`); + process.exit(2); + } + } + + // Publish the track's ports for the host grader. + if (expectedNetworkMode === 'bridge') for (const p of ports) create.push('-p', `127.0.0.1:${p}:${p}`); + // Container-level, so the agent's installs and every later exec share it. + for (const [key, value] of Object.entries( + packageRegistryEnvironment(packageRegistry(), expectedNetworkMode, leaseContext.lease.resources.network))) { + create.push('-e', `${key}=${value}`); + } + + // `--init` gives the container a real PID 1. Without it the dev servers the + // build leaves behind are reparented to `sleep`, which never reaps them. + const init = 'export HOME=/tmp npm_config_cache=/tmp/npm-cache; ' + + containerPlan.init; + create.push('-w', CODING_CONTAINER_APP_ROOT, image, 'sh', '-c', init); + + const made = spawnSync('docker', create, { + encoding: 'utf8', env: dockerEnv, timeout: DOCKER_TIMEOUT_MS, + }); + if (made.status !== 0) { + console.error(`run-build.js: could not create ${containerName}`); + console.error(made.stderr || made.stdout || made.error?.message || ''); + try { + removeFailedBuildContainer({ containerName, creationToken, + createdId: containerIdFromDockerOutput(made.stdout), dockerEnv, + timeoutMs: DOCKER_TIMEOUT_MS }); + } catch (cleanupError) { + console.error(`run-build.js: ${errorMessage(cleanupError)}`); + process.exit(3); + } + process.exit(2); + } + + const createdId = containerIdFromDockerOutput(made.stdout); + try { containerInspection = inspectContainer(containerName); } + catch (error) { + console.error(`run-build.js: cannot inspect ${containerName}: ${errorMessage(error)}`); + } + if (!containerInspection) { + try { + removeFailedBuildContainer({ containerName, creationToken, createdId, dockerEnv, + timeoutMs: DOCKER_TIMEOUT_MS }); + } catch (cleanupError) { + console.error(`run-build.js: ${errorMessage(cleanupError)}`); + process.exit(3); + } + console.error(`run-build.js: cannot inspect ${containerName}`); + process.exit(2); + } + if (containerInspection.unsafeCredentialExposure + || !hasRequiredIsolation(containerInspection, expectedMounts)) { + try { + removeFailedBuildContainer({ containerName, creationToken, createdId, dockerEnv, + timeoutMs: DOCKER_TIMEOUT_MS }); + } catch (cleanupError) { + console.error(`run-build.js: ${errorMessage(cleanupError)}`); + process.exit(3); + } + console.error(`run-build.js: created container ${containerName} does not have the required isolation`); + process.exit(2); + } +} + +if (!containerInspection) { + console.error(`run-build.js: cannot inspect ${containerName}`); + process.exit(2); +} +const { id: containerId, image: containerImage } = containerInspection; +try { + const { path, lease } = leaseContext; + const prior = lease.resources.buildContainer; + if (prior && (prior.name !== containerName || prior.id !== containerId)) { + throw new Error(`running container ${containerName}/${containerId} does not match lease ` + + `${prior.name}/${prior.id}`); + } + updateBackendLease(path, { token: lease.ownershipToken, backend, runId: lease.runId }, next => { + next.resources.buildContainer = { + name: containerName, id: containerId, image: containerImage, owned: true, running: existing !== null, + networkMode: expectedNetworkMode, + resourceLimits: structuredClone(BUILD_CONTAINER_RESOURCE_LIMITS), + }; + return next; + }); +} catch (error) { + // Creation succeeded but ownership recording did not. Remove only the exact + // id created by this invocation; leaving an unleased container is not safe. + if (!existing) { + spawnSync('docker', ['rm', '-f', containerId], { + stdio: 'ignore', env: dockerEnv, timeout: DOCKER_TIMEOUT_MS, + }); + } + console.error(`run-build.js: ${errorMessage(error)}`); + process.exit(3); +} + +if (!existing) { + const started = spawnSync('docker', ['start', containerId], { + encoding: 'utf8', env: dockerEnv, timeout: DOCKER_TIMEOUT_MS, + }); + if (started.status !== 0) { + console.error(`run-build.js: could not start leased container ${containerName}/${containerId}`); + console.error(started.stderr || started.stdout || started.error?.message || ''); + process.exit(2); + } + try { + const { path, lease } = leaseContext; + updateBackendLease(path, { token: lease.ownershipToken, backend, runId: lease.runId }, next => { + if (next.resources.buildContainer?.id !== containerId) { + throw new Error(`leased container changed before start: expected ${containerId}`); + } + next.resources.buildContainer.running = true; + return next; + }); + } catch (error) { + console.error(`run-build.js: started container ownership could not be recorded: ${errorMessage(error)}`); + process.exit(3); + } +} + +if (containerPlan.readyFile) { + // Wait until SDK staging finishes before starting the paid session. + try { + waitForBuildContainerReady(containerId, containerPlan.readyFile, + containerPlan.readyDescription ?? `${backend} setup`, { env: dockerEnv }); + } catch (error) { + console.error(`run-build.js: ${errorMessage(error)}`); + process.exit(2); + } +} + +if (process.env.STACK_BENCH_APPLIANCE === '1') { + const writableTargets = [AGENT_HOME, + ...expectedMounts.filter(mount => !mount.readOnly).map(mount => mount.target)]; + for (const [command, commandArgs] of [ + ['chown', ['-R', `${AGENT_UID}:${CONTROLLER_GID}`, '--', ...writableTargets]], + ['chmod', ['-R', 'u+rwX,g+rwX,o-rwx', '--', ...writableTargets]], + ] as const) { + const permissions = spawnSync('docker', ['exec', containerName, command, ...commandArgs], { + encoding: 'utf8', env: dockerEnv, timeout: DOCKER_PROBE_TIMEOUT_MS, + }); + if (permissions.status !== 0) { + console.error(`run-build.js: could not secure writable paths in ${containerName}`); + console.error(permissions.stderr || permissions.stdout || permissions.error?.message || ''); + process.exit(2); + } + } +} + +// A nested transcript mount makes Docker create its parent directories as +// root. Confirm that the coding runner can create its private session state before a +// provider request can spend money. +const homeProbe = spawnSync('docker', [ + 'exec', '--user', `${AGENT_UID}:${AGENT_GID}`, '-e', `HOME=${AGENT_HOME}`, + containerName, 'sh', '-c', + 'umask 077; mkdir -p "$1" && test -w "$1"', 'home-probe', dirname(containerTranscripts), +], { encoding: 'utf8', env: dockerEnv, timeout: DOCKER_PROBE_TIMEOUT_MS }); +if (homeProbe.status !== 0) { + console.error(`run-build.js: agent home is not writable in ${containerName}`); + console.error(homeProbe.stderr || homeProbe.stdout || homeProbe.error?.message || ''); + process.exit(2); +} + +// Docker bind sources must be the same filesystem the controller audits. +if (projects) { + const probe = `.mount-probe-${randomBytes(12).toString('hex')}`; + const expected = randomBytes(24).toString('hex'); + const path = resolve(projects, probe); + writeFileSync(path, expected, { mode: 0o644 }); + try { + const result = spawnSync('docker', ['exec', '--user', `${AGENT_UID}:${AGENT_GID}`, + containerName, 'cat', `${containerTranscripts}/${probe}`], + { encoding: 'utf8', env: dockerEnv, timeout: DOCKER_PROBE_TIMEOUT_MS }); + if (result.status !== 0 || result.stdout !== expected) { + throw new Error('Transcript mount is not shared with the controller. Use the shared controller HOME before starting a session.'); + } + } finally { unlinkSync(path); } +} + +if (prepareOnly) { + process.stdout.write(`${JSON.stringify({ containerName, + identity: `${containerId} ${containerImage}`, + networkMode: expectedNetworkMode })}\n`); + return; +} + +const args = ['exec', '-i', '--user', `${AGENT_UID}:${AGENT_GID}`, '-w', CODING_CONTAINER_APP_ROOT]; + +args.push('-e', `HOME=${AGENT_ENVIRONMENT.HOME}`, '-e', `USER=${AGENT_ENVIRONMENT.USER}`); +const leasedEnvironment = leasedDatabaseEnvironment(adapter!, { + database: leaseContext.lease.resources.database, networkMode: expectedNetworkMode, lease: leaseContext.lease, +}); +for (const key of Object.keys(leasedEnvironment)) args.push('-e', key); +const dockerExecEnv: NodeJS.ProcessEnv = { ...process.env, ...leasedEnvironment, MSYS_NO_PATHCONV: '1' }; +if (!projects) throw new Error('transcript directory is unavailable'); +// Forward only benchmark-owned environment settings. +if (provider === 'anthropic' && process.env.MAX_THINKING_TOKENS) { + args.push('-e', `MAX_THINKING_TOKENS=${process.env.MAX_THINKING_TOKENS}`); +} + +// Record the exact remote PID. Killing the local `docker exec` client does not +// guarantee that the coding runner stops inside the long-lived build container. +const invocationToken = randomBytes(16).toString('hex'); +const processRecord = `${CODING_CONTAINER_PROCESS_IDENTITY.recordPrefix}${invocationToken}.pid`; +const sessionWrapper = 'umask 022; record="$1"; shift; ' + + 'start="$(awk \'{print $22}\' /proc/$$/stat)" || exit 1; ' + + 'printf \'%s %s\\n\' "$$" "$start" > "$record"; exec "$@"'; + +if (!auth) throw new Error('container authentication is unavailable'); +// A Google account token lasts an hour, shorter than a session. The official CLI renews the +// shared sign-in under a lock; the broker takes each renewed token for its next request. If +// renewal fails the token runs out and Google refuses the session: it fails closed. +const agyAuthFile = provider === 'google' && auth.mode === 'subscription-token' ? process.env.AGY_AUTH_FILE : undefined; +const renewAgyLogin = agyAuthFile ? () => ensureFreshAgyLogin(agyAuthFile, AGY_MIN_REMAINING_MS, + agyRenewal(image, leaseContext.lease.resources.network ? dirname(leaseContext.path) : tmpdir())) : null; +if (renewAgyLogin) { + try { auth = { ...auth, credential: (await renewAgyLogin()).token }; } + catch (error) { console.error(`run-build.js: ${errorMessage(error)}`); process.exit(2); } +} +let credentialBroker: Awaited> | null = null; +try { + const docker = leaseContext.lease.resources.network ? (() => { + const intent = recordAttemptCreation(leaseContext.path, leaseContext.lease, 'broker'); + return { imageId: attemptControllerImage(), + networkContainerId: leaseContext.lease.resources.network!.namespaceContainerId!, + ...intent, privateDirectory: dirname(leaseContext.path), + onCreated: (container: import('./credential-broker-process.js').CredentialBrokerContainer) => { + updateBackendLease(leaseContext.path, { token: leaseContext.lease.ownershipToken }, next => { + next.resources.brokerContainer = container; return next; + }); + } }; + })() : undefined; + credentialBroker = await startCredentialBroker(auth, + { networkMode: expectedNetworkMode, deadlineMs: CODING_SESSION_TIMEOUT_MS, model, + providerRoute, maxOutputTokens, renewable: renewAgyLogin !== null, tools, + docker, + maxBudgetUsd: maxBudgetUsd === null ? null : Number(maxBudgetUsd), + pricingRates: maxBudgetUsd === null ? null : pricing!.rates }); +} catch (error) { + console.error(`run-build.js: ${errorMessage(error)}`); + process.exit(2); +} +if (!credentialBroker) throw new Error('credential broker is unavailable'); +const tokenEnvironment = codingProvider.tokenEnvironment; +dockerExecEnv[tokenEnvironment] = credentialBroker.sessionToken; +// Only the authority's certificate leaves the controller; its key is already gone. +if (credentialBroker.certificateAuthority) { + dockerExecEnv.BROKER_CA_CERT = credentialBroker.certificateAuthority; + args.push('-e', 'BROKER_CA_CERT'); +} +args.push('-e', tokenEnvironment, ...codingProvider.environment(credentialBroker.baseUrl, auth.mode).flatMap(value => ['-e', value]), + containerName, 'sh', '-c', sessionWrapper, CODING_CONTAINER_PROCESS_IDENTITY.sessionLabel, + processRecord, + codingProvider.executable, ...codingProvider.args({ model, effort, baseUrl: credentialBroker.baseUrl, + resumeSession, maxBudgetUsd, tools })); + +// MSYS_NO_PATHCONV: Git Bash rewrites container-side paths like /app into +// Windows paths (C:/Program Files/Git/app) and every mount silently lands +// somewhere wrong. +const promptInput = process.stdin.isTTY ? '' : readFileSync(0, 'utf8'); +function signalSession(signal: 'TERM' | 'KILL') { + const script = 'record="$1"; signal="$2"; test -r "$record" || exit 4; ' + + 'read -r pid expected < "$record"; ' + + 'current="$(awk \'{print $22}\' "/proc/$pid/stat" 2>/dev/null)" || exit 5; ' + + 'test "$current" = "$expected" || exit 3; kill "-$signal" "$pid"'; + // The agent can rewrite its own record, so signal with the agent's authority, not root's. + return spawnSync('docker', ['exec', '--user', `${AGENT_UID}:${AGENT_GID}`, containerName, 'sh', '-c', script, + CODING_CONTAINER_PROCESS_IDENTITY.stopLabel, processRecord, signal], { + encoding: 'utf8', env: dockerExecEnv, timeout: DOCKER_PROBE_TIMEOUT_MS, + }); +} +function terminateSession(child: { kill(signal?: NodeJS.Signals): boolean }): void { + const term = signalSession('TERM'); + if (term.status !== 0) child.kill('SIGTERM'); + const force = setTimeout(() => { + signalSession('KILL'); + child.kill('SIGKILL'); + }, 5_000); + force.unref(); +} + +let res: Awaited> | undefined; +let sessionError: unknown = null; +let brokerLedger = null; +let brokerDiagnostics = null; +const cleanupErrors: string[] = []; +const runCleanupCommand = (description: string, command: readonly string[]): void => { + const result = spawnSync('docker', ['exec', containerName, ...command], { + encoding: 'utf8', env: dockerExecEnv, timeout: DOCKER_PROBE_TIMEOUT_MS, + }); + if (result.status !== 0) cleanupErrors.push(`${description}: ${String(result.stderr || result.stdout + || result.error?.message || `exit ${result.status}`).trim()}`); +}; +const broker = credentialBroker; +const renewal = renewAgyLogin ? setInterval(() => { + renewAgyLogin().then(login => broker.replaceCredential(login.token)) + .catch(error => console.error(`run-build.js: ${errorMessage(error)}`)); +}, AGY_RENEWAL_INTERVAL_MS) : undefined; +try { + res = await codingProvider.run({ command: 'docker', args, input: promptInput, + env: dockerExecEnv, timeoutMs: CODING_SESSION_TIMEOUT_MS, terminate: terminateSession, + projects, containerId, marker: completionMarker as string, model, + pricingRates: pricing?.rates ?? null, resumeSession }); +} catch (error) { + sessionError = error; +} finally { + clearInterval(renewal); + brokerLedger = await stopCredentialBroker(credentialBroker); + brokerDiagnostics = credentialBrokerDiagnostics(credentialBroker); + if (credentialBroker.container && brokerDiagnostics?.termination?.exited + && !brokerDiagnostics.errors.some(error => error.type === 'cleanup-error')) { + updateBackendLease(leaseContext.path, { token: leaseContext.lease.ownershipToken }, next => { + delete next.resources.brokerContainer; + delete next.resources.creationIntents?.broker; + return next; + }); + } + for (const command of codingContainerTranscriptHandoffCommands(CONTROLLER_GID, containerTranscripts)) { + runCleanupCommand('transcript handoff', command); + } + const handoff = process.env.STACK_BENCH_APPLIANCE === '1' + ? codingContainerWorkspaceHandoffCommands(CONTROLLER_GID) + : [['chmod', '-R', 'a+rwX', CODING_CONTAINER_APP_ROOT]]; + for (const command of handoff) runCleanupCommand('workspace handoff', command); + runCleanupCommand('process-record cleanup', ['rm', '-f', processRecord]); +} + +if (sessionError) { + if (cleanupErrors.length) { + throw new AggregateError([sessionError, ...cleanupErrors.map(message => new Error(message))], + 'coding session and container cleanup failed'); + } + throw sessionError; +} +if (!res) throw new Error('coding session returned no process result'); +if (cleanupErrors.length) { + res.status = res.status === 0 ? 3 : res.status ?? 3; + res.stderr = `${res.stderr ?? ''}${res.stderr ? '\n' : ''}` + + `run-build.js: container cleanup failed: ${cleanupErrors.join('; ')}\n`; +} + +const cliResult = codingProvider.result(String(res.stdout ?? '').trim(), appDir, invocationToken, priorSessionUsage); +if (cliResult) cliResult.stack_bench_auth_mode = auth.mode; +const memory = spawnSync('docker', ['exec', containerName, 'sh', '-c', + 'for f in memory.events memory.current memory.peak memory.max pids.current pids.peak pids.max pids.events; do ' + + 'p="/sys/fs/cgroup/$f"; if test -r "$p"; then echo "[$f]"; cat "$p"; fi; done'], { + encoding: 'utf8', env: dockerExecEnv, timeout: DOCKER_PROBE_TIMEOUT_MS, +}); +const resources = { + ...(memory.status === 0 ? parseCgroupResources(memory.stdout) + : { buildContainerMemory: null, buildContainerPids: null }), + memoryProbeError: memory.status === 0 ? null + : memory.stderr?.trim() || (memory.error instanceof Error ? memory.error.message : null) + || `exit ${memory.status}`, +}; +if (maxBudgetUsd !== null) { + const reconciled = reconcileCredentialBrokerReceipt({ + ledger: brokerLedger, + provider, + cliResult, + model, + maxBudgetUsd: Number(maxBudgetUsd), + pricingRates: pricing!.rates, + brokerDiagnostics, + }); + reconciled.result.stack_bench_resources = resources; + res.stdout = `${JSON.stringify(reconciled.result)}\n`; + if (!reconciled.ok) { + res.status = res.status === 0 ? 3 : res.status ?? 3; + res.stderr = `${res.stderr ?? ''}${res.stderr ? '\n' : ''}` + + `run-build.js: ${reconciled.receipt.error}\n`; + } +} else if (cliResult && typeof cliResult === 'object' && !Array.isArray(cliResult)) { + cliResult.stack_bench_credential_broker = brokerDiagnostics; + cliResult.stack_bench_resources = resources; + res.stdout = `${JSON.stringify(cliResult)}\n`; +} + +if ((res.status ?? 1) !== 0) { + const state = spawnSync('docker', ['inspect', '--format', '{{json .State}}', containerName], { + encoding: 'utf8', env: dockerExecEnv, timeout: DOCKER_PROBE_TIMEOUT_MS, + }); + let containerState = null; + try { containerState = JSON.parse(state.stdout?.trim() || 'null'); } catch { /* retain raw text below */ } + const diagnostic = { + schemaVersion: 1, + kind: 'coding-process-exit', + status: res.status ?? null, + signal: res.signal ?? null, + error: res.error instanceof Error ? res.error.message : null, + container: containerState ?? { inspectError: state.stderr?.trim() + || (state.error instanceof Error ? state.error.message : null) }, + cgroupMemory: memory.stdout?.trim() || null, + cgroupProbeError: memory.status === 0 ? null + : memory.stderr?.trim() || (memory.error instanceof Error ? memory.error.message : null) + || `exit ${memory.status}`, + }; + process.stderr.write(`STACK_BENCH_CODING_PROCESS_DIAGNOSTIC ${JSON.stringify(diagnostic)}\n`); +} + +if (res.stdout) process.stdout.write(res.stdout); +if (res.stderr) process.stderr.write(res.stderr); +if (res.error) process.stderr.write(`run-build.js: coding session failed: ${errorMessage(res.error)}\n`); +process.exit(res.status ?? 1); +} + +await runBuild(); diff --git a/tools/stack-bench/container/spacetime-dev.ts b/tools/stack-bench/container/spacetime-dev.ts new file mode 100644 index 00000000000..161dee0b0a2 --- /dev/null +++ b/tools/stack-bench/container/spacetime-dev.ts @@ -0,0 +1,94 @@ +// Standalone development process command; Node built-ins only. +import { spawn } from 'node:child_process'; +import { existsSync, readFileSync, writeFileSync, openSync, closeSync, rmSync, realpathSync } from 'node:fs'; +import { resolve, relative, isAbsolute, dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { setTimeout as delay } from 'node:timers/promises'; + +function marker(pid: number): string | null { + try { + const stat = readFileSync(`/proc/${pid}/stat`, 'utf8'); + const fields = stat.slice(stat.lastIndexOf(')') + 2).split(' '); + return fields[0] === 'Z' ? null : fields[19] ?? null; + } catch { return null; } +} + +export function validateProject(root: string, server: string, database: string): void { + const read = (name: string) => JSON.parse(readFileSync(join(root, name), 'utf8')); + const config = { ...read('spacetime.json'), + ...(existsSync(join(root, 'spacetime.local.json')) ? read('spacetime.local.json') : {}) }; + if (config.server !== server || config.database !== database || config.publish !== undefined) { + throw new Error('Configure one database using the supplied server URL and database name.'); + } + const contained = (path: unknown): string => { + if (typeof path !== 'string' || !path.trim()) throw new Error('Configure module-path and generate out-dir.'); + const full = resolve(root, path); + let existing = full; + while (!existsSync(existing)) existing = dirname(existing); + for (const candidate of [full, realpathSync(existing)]) { + const rel = relative(realpathSync(root), candidate); + if (isAbsolute(rel) || rel === '..' || rel.startsWith('../')) throw new Error('Project paths must stay inside the application directory.'); + } + return full; + }; + if (!existsSync(contained(config['module-path']))) throw new Error('Create the module directory before starting development.'); + if (!Array.isArray(config.generate) || !config.generate.length) throw new Error('Configure at least one generate target.'); + for (const target of config.generate) { + if (!target || target.language !== 'typescript') throw new Error('Configure TypeScript generate targets.'); + contained(target['out-dir']); + if (target['module-path'] !== undefined) contained(target['module-path']); + if ((target.server !== undefined && target.server !== server) + || (target.database !== undefined && target.database !== database)) throw new Error('Generate targets must use the supplied database.'); + } +} + +// Calls are serialized by the installed flock wrapper. The detached watcher is +// still owned by the agent UID and the existing container teardown stops it. +export async function main(args: string[]): Promise { + const [root, state, cli, server, database, command = 'status', ...extra] = args; + if (!root || !state || !cli || !server || !database || extra.length + || !['start', 'status', 'stop'].includes(command)) throw new Error('Usage: spacetime-dev start|status|stop'); + const record = join(state, 'process.json'), ready = join(state, 'ready'), log = join(state, 'watcher.log'); + const previous = existsSync(record) ? JSON.parse(readFileSync(record, 'utf8')) : null; + const alive = () => previous && Number.isSafeInteger(previous.pid) && previous.pid > 1 + && typeof previous.marker === 'string' && marker(previous.pid) === previous.marker; + if (command === 'stop') { + if (alive()) { + // Stop the whole build group together, including a compiler child. + process.kill(-previous.pid, 'SIGKILL'); + for (let i = 0; i < 50 && alive(); i++) await delay(100); + if (alive()) throw new Error(`Watcher did not stop. Read ${log}`); + } + rmSync(record, { force: true }); rmSync(ready, { force: true }); + console.log(`Stopped. Log: ${log}`); return; + } + if (alive()) { + console.log(`${existsSync(ready) ? 'Running; initial publish and bindings completed' : 'Starting; initial publish not yet confirmed'}. Log: ${log}`); + return; + } + if (command === 'status') { console.log(`Not running. Log: ${log}`); return; } + validateProject(root, server, database); + rmSync(ready, { force: true }); + const output = openSync(log, 'w', 0o600); + const child = spawn(cli, ['dev', '--yes', '--delete-data=never', '--server-only', '--ready-file', ready], + { cwd: root, detached: true, stdio: ['ignore', output, output] }); + closeSync(output); + await new Promise((done, reject) => { child.once('spawn', done); child.once('error', reject); }); + const pid = child.pid!; + const identity = marker(pid); + if (!identity) throw new Error(`Watcher exited at startup. Read ${log}`); + writeFileSync(record, JSON.stringify({ pid, marker: identity })); + child.unref(); + // Bounded startup observation; a slow compile keeps running and status can + // inspect it later. Never mistake a live process for a completed publish. + for (let i = 0; i < 50; i++) { + if (marker(pid) !== identity) throw new Error(`Watcher exited. Read ${log}`); + if (existsSync(ready)) { console.log(`Running; initial publish and bindings completed. Log: ${log}`); return; } + await delay(100); + } + console.log(`Starting; initial publish not yet confirmed. Run spacetime-dev status. Log: ${log}`); +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main(process.argv.slice(2)).catch(error => { console.error(error.message); process.exitCode = 1; }); +} diff --git a/tools/stack-bench/container/spacetimedb-binaries.json b/tools/stack-bench/container/spacetimedb-binaries.json new file mode 100644 index 00000000000..51e4e8a5fd4 --- /dev/null +++ b/tools/stack-bench/container/spacetimedb-binaries.json @@ -0,0 +1,21 @@ +{ + "schemaVersion": 2, + "platform": "linux/amd64", + "builderImage": "rust:1.93-slim-bookworm@sha256:8f8609d448e821fbc0e44241bc5ca4ce49663cc6306ff1a17f655a0e2a7cd084", + "source": { + "identityScheme": "git-object-content-v1", + "revision": "81ab0550c6e07cd24b264a9c32d9240ff4cf2d75", + "sha256": "bacff6fdae08ea390eabbfd1fe908b3cf1338a744272c0684d7857c77f630f64", + "files": 2599 + }, + "binaries": { + "spacetimedb-cli": { + "sha256": "2f671b85f51beac7ab6d32ccf707f08e3ca936d876fb2bdc874e02e497e5287f", + "size": 47642320 + }, + "spacetimedb-standalone": { + "sha256": "53df19d77e81ef61426e0bfd154d78e94b4e9a04754647d79777ff4e056163c9", + "size": 132331544 + } + } +} diff --git a/tools/stack-bench/dashboard/README.md b/tools/stack-bench/dashboard/README.md new file mode 100644 index 00000000000..f6632aa8c92 --- /dev/null +++ b/tools/stack-bench/dashboard/README.md @@ -0,0 +1,81 @@ +# StackBench dashboard + +The dashboard is a local web view of campaign results and a way to start runs. +It reads the plans, campaign state, and evidence that the CLI writes, and starts +runs through the same job and campaign commands. It does not grade anything +itself. + +## Pages + +- **Campaigns** (`/`): a lane per running attempt, then one row per campaign + with its status and per-stack score. +- **Campaign** (`/c/:key`): plan facts, completion, scores, repairs, time, + spend, and attempts. `?chart=completion|cost|distribution` picks the chart and + `&unit=features|checks` the counting unit. Dependency campaigns add the + feature graph (`?questlines=grid|graph|replay`). +- **Attempt** (`/c/:key/a/:attemptId`): the attempt's figures and + `?tab=checks|transcript|screenshots|files|log`. Checks shows each check's + recorded status, expected value, and observation. Transcript shows the + agent's sessions, including tool calls. Log shows the controller's output. +- **New run** (`/new`): choose workload, level, stacks, models, guidance, + repetitions, repairs, and limits; review the attempt count and cost cap; start. + Defaults come from the installed presets under `results/run-presets/`. +- **Saved plans** (`/plans`): the exact configuration behind each run. +- **Checks** (`/checks`): search the installed check definitions and inspect + their steps, waits, and pass rules. + +"Before repairs" shows each level's first-build evidence. Later levels still +carry earlier fixes, so it is not a repair-free run. + +## Start and stop + +From `tools/stack-bench`, after [appliance setup](../appliance/README.md): + +```sh +docker compose --env-file operator.env -f appliance/docker-compose.yaml --profile dashboard up -d dashboard +docker compose --env-file operator.env -f appliance/docker-compose.yaml --profile dashboard stop dashboard +``` + +Open `localhost:7331`. Docker publishes the port on the loopback interface only, +and writes need the browser's origin and a per-server token, so there is no +password. Do not expose it on a shared network. Outside the appliance, +`npm run dashboard` serves a read-only view of `tools/stack-bench/results`; +`--port` and `--results` change the defaults. + +Start, Resume, and Stop run the CLI in a controller container. Stop interrupts +the attempt; it does not pause it. Resume continues scheduled dependency work. +Starting a run submits a job; starting again with the same reviewed settings +returns the same job. + +Cost shown as `~$` is a live estimate from completed responses at the plan's +rates. The final receipt replaces it, and estimates never enter scores or budgets. + +## Routes + +| route | returns | +| --- | --- | +| `GET /api/health` | `read-only` or `controller` | +| `GET /api/overview?page=N&filter=F` | campaign summaries, 20 per page | +| `GET /api/campaigns/:key` | the campaign sheet | +| `GET /api/campaigns/:key/live` | live spend, activity, and phase | +| `GET /api/campaigns/:key/progression` | the dependency graph and its replay | +| `GET /api/campaigns/:key/attempts/:id/checks` | per-check outcome and history | +| `GET /api/campaigns/:key/attempts/:id/package` | the evidence listing | +| `GET /api/campaigns/:key/attempts/:id/log?from=N` | log bytes after `N` | +| `GET /api/campaigns/:key/attempts/:id/transcript` | paged transcript messages | +| `GET /api/campaigns/:key/attempts/:id/time` | time allowance and grants | +| `POST /api/campaigns/:key/attempts/:id/time` | request more time | +| `GET /api/campaigns/:key/artifacts/:id` | one allowlisted artifact | +| `GET /api/events` | server-sent change events | +| `GET /api/plans` | the discovered plans | +| `GET /api/session` | whether controls are available, and the browser token | +| `GET /api/run-setup` | workload choices and defaults | +| `POST /api/runs/prepare` | the review of a run setup | +| `POST /api/runs` | start a reviewed run | +| `GET /api/reference-runs` | reference qualification runs | +| `POST /api/jobs`, `GET /api/jobs`, `GET /api/jobs/:id`, `POST /api/jobs/:id/start`, `POST /api/jobs/:id/cancel` | the job queue | +| `POST /api/campaigns/:key/resume` | run scheduled dependency work | +| `POST /api/campaigns/:key/stop` | stop the controller shown by the page | + +The dashboard records the controller operations it starts in +`/dashboard/operations.jsonl`. It never edits grades or source. diff --git a/tools/stack-bench/dashboard/check-guide-steps.ts b/tools/stack-bench/dashboard/check-guide-steps.ts new file mode 100644 index 00000000000..b58584148c3 --- /dev/null +++ b/tools/stack-bench/dashboard/check-guide-steps.ts @@ -0,0 +1,130 @@ +import type { CompiledStep } from '../src/composition/definition-compiler.js'; + +// Read the authored form so loops stay visible instead of expanding to thousands of rows. +export interface GuideStep extends CompiledStep { + repeat?: number; + forEach?: unknown[]; + steps?: GuideStep[]; + branches?: GuideStep[][]; + alongside?: GuideStep[]; + storage?: { cart?: boolean; warehouses?: boolean }; + swap?: { find: unknown; with: unknown }; + senders?: { actor: string; count: number; prefix: string }[]; +} +const q = (value: unknown) => JSON.stringify(value); +const words = (value: unknown) => String(value ?? '').replaceAll('-', ' '); +const actor = (s: GuideStep) => s.actor ? `In the ${s.actor} browser, ` : ''; +const target = (s: GuideStep) => `${words(s.testid)}${s.contains ? ` matching ${q(s.contains)}` : ''}${s.in ? ` inside ${words(s.in.testid)}${s.in.contains ? ` for ${q(s.in.contains)}` : ''}` : ''}`; +const seconds = (ms: unknown) => `${Number((Number(ms) / 1000).toFixed(3))} seconds`; +const wanted = (s: GuideStep) => s.relativeTo ? `the saved ${q(s.relativeTo)} value ${Number(s.plus ?? 0) < 0 ? 'minus' : 'plus'} ${Math.abs(s.plus ?? 0)}` + : s.equals !== undefined ? q(s.equals) : s.value !== undefined ? q(s.value) : s.containsText ? `text containing ${q(s.containsText)}` + : s.atLeast !== undefined ? `at least ${s.atLeast}` : s.atMost !== undefined ? `at most ${s.atMost}` : 'the required value'; +const meanings: Record string> = { + signUp: s => `${actor(s)}register ${q(s.name)}${s.password ? ' with the specified test password' : ''}.`, + signIn: s => `${actor(s)}sign in as ${q(s.name)}${s.exact ? ' using the exact seeded account name' : ''}.`, + ensureSignedIn: s => `${actor(s)}keep the current session if it is ready; otherwise sign in as ${q(s.name)}.`, + click: s => `${actor(s)}click ${target(s)}${s.ifAvailable ? ' if available' : ''}${s.unlessVisible ? `, unless ${Array.isArray(s.unlessVisible) ? s.unlessVisible.map(words).join(' or ') : words(s.unlessVisible)} is already visible` : ''}.`, + openItem: s => `${actor(s)}open the product ${q(s.item)}${s.unlessVisible ? ` unless ${words(s.unlessVisible)} is already visible` : ''}.`, + fill: s => `${actor(s)}set ${target(s)} to ${q(s.text)}${s.enter ? ' and press Enter' : ''}.`, + pressKey: s => `${actor(s)}press ${q(s.key)}.`, + reload: s => `${actor(s)}reload the page.`, + freshClient: s => `Open a separate browser for ${s.actor}${s.preserveStorage ? ', copying its cookies, local storage, IndexedDB and session storage to test retained access' : ' with clean storage'}; later steps refer to ${s.actor}-fresh.`, + openClient: s => `Reopen the ${s.actor} browser.`, closeClient: s => `Close the ${s.actor} browser.`, + setOffline: s => `${s.offline === false ? 'Restore' : 'Disconnect'} the ${s.actor} browser's network connection.`, + expect: s => `${actor(s)}${s.absent ? 'watch for and reject any visible' : 'check for visible'} ${target(s)}${s.count !== undefined ? `; require ${s.count} matching elements` : ''}${s.value !== undefined ? `; require value ${q(s.value)}` : ''}${s.nonEmpty ? '; require nonempty content' : ''}${s.notContains ? `; reject text containing ${q(s.notContains)}` : ''}${s.ignoreCase ? ' (ignore letter case)' : ''}.`, + expectNumber: s => `${actor(s)}check that ${target(s)} is ${s.comparison === 'atMost' ? 'no greater than ' : s.comparison === 'atLeast' ? 'at least ' : ''}${wanted(s)}.`, + expectWarehouseInventory: s => `${actor(s)}read all visible warehouse inventory rows in the open admin panel; require every declared item and warehouse holding, with correct stock in every copy.`, + expectElementCount: s => `${actor(s)}count ${target(s)}; require ${wanted(s)}.`, + expectSequence: s => `${actor(s)}read every ${target(s)} in displayed order; require exactly ${q(s.equals)}.`, + expectAgreement: s => `Compare ${target(s)} in browsers ${(s.actors ?? []).join(', ')}; require equal ${s.numeric ? 'numbers' : 'values'}.`, + expectActorsWith: s => `Require exactly ${s.equals} of browsers ${(s.actors ?? []).join(', ')} to show ${target(s)}${s.maxEach !== undefined ? `, with at most ${s.maxEach} per browser` : ''}.`, + expectUnavailable: s => `${actor(s)}check that ${target(s)} is absent, hidden or disabled.`, + waitUntilAbsent: s => `${actor(s)}wait until ${target(s)} is absent.`, + recordNumber: s => `${actor(s)}save the number from ${target(s)} as ${q(s.as)}.`, + recordTime: s => `Record the current time as ${q(s.as)}.`, + expectElapsed: s => `Require this observation to start within ${seconds(s.atMost)} of ${q(s.since)}; otherwise the measurement is inconclusive.`, + wait: s => s.since ? `Wait until ${seconds(s.ms)} after ${q(s.since)}.` : `Wait ${seconds(s.ms)}.`, + dbSetStock: s => `Set stored stock for ${q(s.item)} in ${q(s.warehouse)} to ${s.quantity}, independently of the app's handlers.`, + dbRecordStock: s => `Read authoritative stock for ${q(s.item)}${s.warehouse ? ` in ${q(s.warehouse)}` : ' across warehouses'}; save it as ${q(s.as)}.`, + dbExpectStock: s => `Independently read stored stock for ${q(s.item)}${s.warehouse ? ` in ${q(s.warehouse)}` : ''}; require ${wanted(s)}.`, + dbRecordCheckout: s => `Read stored account, product and order state for ${q(s.account)} and ${q(s.item)}; save it as ${q(s.as)}${s.storage ? ` (cart: ${!!s.storage.cart}; warehouse detail: ${!!s.storage.warehouses})` : ''}.`, + dbExpectCatalogItem: s => `Read the database after product creation. Relative to ${q(s.before)}, require exactly one new product named ${q(s.name)} with price ${s.priceMinor} minor units. This is also the committed-write barrier before the next creation.`, + dbExpectCheckout: s => `Compare stored state with ${q(s.before)} and the prepared cart ${q(s.prepared)}. Require complete checkout effects for quantity ${q(s.quantity)}${s.alongsideAdd ? ` while preserving a concurrent add of ${q(s.alongsideAdd)}` : ''}${s.actor ? `, reconciled with ${s.actor}'s request outcome` : ''}.`, + dbExpectCancellation: s => `Compare stored state with ${q(s.before)}. Require cancellation to restore the original stock and booked amount exactly once.`, + dbExpectNoPurchase: s => `Compare stored state with ${q(s.before)}. Require no purchase effects.`, + dbExpectPurchase: s => `Compare stored orders with ${q(s.before)}${s.stockBefore ? ` and stock with ${q(s.stockBefore)}` : ''}; require the purchase and its accounting effects.`, + dbExpectPurchases: s => `Reconcile stored orders and warehouse effects against the saved buyer snapshots ${q(s.before)}; require ${s.purchases} complete purchases.`, + dbExpectPurchaseCount: s => `Read all stored orders against snapshots ${q(s.before)}. Require ${s.purchasesEach} additional purchases for each account, their stored prices, and warehouse effects where those snapshots include warehouses.`, + dbExpectOperation: s => `Check the complete stored-state transition for ${s.actor}'s ${s.operation}, against ${q(s.before)}${s.otherBefore ? ` and ${q(s.otherBefore)}` : ''}.`, + callAction: s => `${actor(s)}send the real application ${q(s.action)} request${s.authentication === 'none' ? ' without credentials' : s.authentication === 'tampered-session' ? ' with a tampered session credential' : s.authentication === 'optional' ? ' with whatever credentials this browser actually has; allow it to be signed out' : " with this actor's credentials"}${s.input ? `, taking ${s.input.attribute} from ${words(s.input.testid)}${s.input.contains ? ` matching ${q(s.input.contains)}` : ''} in ${s.from ?? s.actor}'s browser` : ''}.`, + expectActionOutcome: s => `Require ${s.actor}'s application request to be ${s.outcome}.`, + callConcurrently: s => `Send ${s.requests ?? (s.actors ?? []).length} ${q(s.action)} requests across browsers ${(s.actors ?? []).join(', ')} concurrently; retain every response or unknown outcome${s.delayMs ? ` (delay this request group by ${seconds(s.delayMs)})` : ''}.`, + clickConcurrently: s => `In browsers ${(s.actors ?? []).join(', ')}, click ${target(s)} concurrently.`, + expectCallOutcomes: s => `Check all recorded concurrent request outcomes${s.accepted !== undefined ? `; require exactly ${s.accepted} accepted calls` : ''}; unknown outcomes remain inconclusive rather than assumed refusals.`, + replayAs: s => `Replay the observed ${s.namedAction?.id ?? s.action ?? s.match ?? 'write'} from ${s.from ?? 'the source actor'} with ${s.actor}'s authority${s.namedTarget ? ', selecting the declared target entity' : ''}.`, + repeatFormWrite: s => `Compare ${s.actor}'s confirmed form writes for ${q(s.match)} and ${q(s.control)}. Repeat only if they differ solely in that value, replacing it with ${q(s.replacement)}. Otherwise fill and submit the form. Choose the path before sending; never retry an uncertain or rejected write. The next database check must confirm the stored product.`, + replayConcurrently: s => `Replay recorded writes concurrently from ${(s.actors ?? []).join(', ')}.`, + expectReplayRejected: s => `Require the replay by ${s.actor} to be rejected. Later state assertions, where present, check its effects.`, + expectReplayCompleted: s => `Require ${s.actor}'s replay to complete${s.requireAccepted ? ' and be accepted' : ''}.`, + forgeWrite: s => `Send a write as ${s.actor} while claiming ${s.fromActor}'s identity.`, + expectForgeryRejected: s => `Require the forged write by ${s.actor} to be rejected.`, + prepareResponseLoss: s => `Prepare to intercept ${s.actor}'s checkout reply on its actual transport.`, + loseCheckoutResponse: s => `Submit ${s.actor}'s checkout, lose its reply, and independently inspect stored effects against ${q(s.before)} and ${q(s.prepared)}.`, + confirmCheckout: s => `Confirm a working authenticated checkout path for ${s.actor}.`, + crashCheckout: s => `Start ${s.requests} checkout requests from ${s.actor}; crash the ${s.target} process at offset ${s.offsetMs} ms. Recover it, inspect stored state against ${q(s.before)} and ${q(s.prepared)}, and save ${q(s.as)}.`, + expectCrashCheckout: s => `Read crash evidence ${q(s.from)} and require ${s.verdict === 'atomicity' ? 'either the prepared cart or one complete order, never partial effects' : 'acknowledged work and earlier orders to survive'}.`, + restartBackend: () => 'Restart the owned database backend with retained storage and wait for readiness.', + stopAppServer: () => 'Stop the application server.', startAppServer: () => 'Start the application server and wait for readiness.', + expectReceived: s => `Require ${s.actor}'s observed incoming data to contain ${q(s.contains)}.`, + expectNotReceived: s => `Watch ${s.actor}'s observed incoming data; reject ${q(s.contains)}. This is a bounded observation, not a proof against every possible leak.`, + markDelivery: s => s.phase === 'start' + ? `Watch ${s.actor}'s incoming data while idle, then start recording it as ${q(s.as)}.` + : `Stop recording ${s.actor}'s incoming data as ${q(s.as)}; records first seen during it belong to that write.`, + expectNotDelivered: s => `Watch ${s.actor}'s observed incoming data; reject any record recorded as ${q(s.as)} for ${s.from}. This is a bounded observation, not a proof against every possible leak.`, + armScriptCanary: s => `Install a harmless execution marker in ${s.actor}'s browser.`, + expectNoScriptExecution: s => `Require ${s.actor}'s execution marker to remain unchanged after the supplied content is displayed.`, + createRoom: s => `${actor(s)}create chat room ${q(s.room)}.`, enterRoom: s => `${actor(s)}enter chat room ${q(s.room)}.`, + send: s => `${actor(s)}send message ${q(s.text)}.`, typeInto: s => `${actor(s)}type ${q(s.text)} without sending.`, + clearInput: s => `${actor(s)}clear the message input.`, + sendMany: s => `${actor(s)}send ${s.count} numbered messages beginning ${q(s.prefix)}, ${seconds(s.delayMs ?? 0)} apart.`, + sendConcurrently: s => `Send messages concurrently: ${(s.senders ?? []).map(x=>`${x.actor}: ${x.count} beginning ${q(x.prefix)}`).join('; ')}; spacing ${seconds(s.delayMs ?? 0)}.`, + expectAllPresent: s => `${actor(s)}require all ${s.count} messages beginning ${q(s.prefix)}.`, + expectOrderMatches: s => `Require browsers ${(s.actors ?? []).join(', ')} to show the same nonempty message order matching ${q(s.prefix)}.`, + expectStable: s => `${actor(s)}sample ${target(s)} ${s.samples ?? 4} times, waiting ${seconds(s.intervalMs ?? 700)} after each sample; require a stable value.`, + runScript: s => `Run the application's ${q(s.script)} with arguments ${q(s.args ?? [])}.`, +}; +export function stepsText(steps: GuideStep[], depth = 0): string[] { + const lines = []; + for (const s of steps) { + const pad = ' '.repeat(depth); + if (s.repeat !== undefined || s.forEach) { + const list = s.forEach; + lines.push(`${pad}- Repeat ${list ? list.length : s.repeat} times${list ? `, substituting each listed value (first ${q(list[0])}, last ${q(list.at(-1))}; exact list in source)` : ''}:`); + lines.push(...stepsText(s.steps ?? [], depth + 1)); continue; + } + if (s.do === 'race') { + lines.push(`${pad}- Run these branches concurrently and wait for all of them:`); + (s.branches ?? []).forEach((branch, i) => { lines.push(`${pad} - Branch ${i + 1}:`); lines.push(...stepsText(branch, depth + 2)); }); + } else if (s.do === 'probeSignupClaims') { + lines.push(`${pad}- Create an ordinary account and verify that it can sign in. Record observed registration writes, using the same minimum two-second observation window for ordinary and modified signups. For the ordinary account and then a fresh account per write, run the observations below. In each fresh attempt, add ${q(s.fields)} to one write only. Missing targets or incomplete capture stay inconclusive.`); + lines.push(...stepsText(s.branches?.[0] ?? [], depth + 1)); + } else { + const describe = meanings[s.do]; + lines.push(`${pad}- ${describe ? describe(s) : `Run ${q(s.do)}. See the exact input below for its fields.`}`); + } + if (s.input?.overrides) lines.push(`${pad} Replace submitted fields with ${q(s.input.overrides)}; this is deliberately untrusted client input.`); + if (s.browserOrigin) lines.push(`${pad} Send from a ${s.browserOrigin === 'cross-site' ? 'different browser origin' : 'same-site browser origin'}.`); + if (s.swap) lines.push(`${pad} In the replayed request, replace ${q(s.swap.find)} with ${q(s.swap.with)}.`); + if (s.containsText) lines.push(`${pad} Require displayed text containing ${q(s.containsText)}.`); + if (s.attribute) lines.push(`${pad} Inspect the ${q(s.attribute)} attribute.`); + if (s.provenBy) lines.push(`${pad} This check uses proof recorded by ${q(s.provenBy)}.`); + if (s.reuseCombinedFrom) lines.push(`${pad} If application and database are the same process, reuse ${q(s.reuseCombinedFrom)} rather than counting another crash.`); + for (const group of s.alongside ?? []) lines.push(`${pad} At the same time: ${meanings.callConcurrently!(group)}`); + if (s.within !== undefined) lines.push(s.do === 'dbExpectStock' || s.do === 'dbExpectCheckout' + ? `${pad} Retry window: ${seconds(s.within)}; a started read may finish later.` + : `${pad} Observation deadline: ${seconds(s.within)}${s.absent || s.do === 'expectNotReceived' ? '; this includes an absence observation window' : '; normally completes sooner if the condition is met'}.`); + if (s.settleMs) lines.push(`${pad} Always wait another ${seconds(s.settleMs)} after this action.`); + + } + return lines; +} diff --git a/tools/stack-bench/dashboard/check-guide.ts b/tools/stack-bench/dashboard/check-guide.ts new file mode 100644 index 00000000000..2df8279fede --- /dev/null +++ b/tools/stack-bench/dashboard/check-guide.ts @@ -0,0 +1,135 @@ +import { readFileSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { createHash } from 'node:crypto'; +import { loadTrack, listTracks } from '../src/composition/tracks.js'; +import { resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { resolveFeatureCatalog } from '../src/progression/feature-catalog-selection.js'; +import { progressionLevels, selectFeatureCatalogLevels } from '../src/progression/progression-definition.js'; +import { resolveProgressionRecipeLevelSelection } from '../src/progression/progression-recipe-selection.js'; +import type { CompiledCriterion, CompiledFeature } from '../src/composition/definition-compiler.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { stepsText, type GuideStep } from './check-guide-steps.js'; +import { esc } from './public/format.js'; +import { topbar } from './public/views/plans.js'; + +interface Check { + key: string; + active: boolean; + source: string; + sourceSha256: string; + feature: CompiledFeature; + criterion: CompiledCriterion; + controls: string[]; +} + +function inventory() { + const track = loadTrack('ecommerce'); + const binding = resolveRecipeRelease(track, 3, 'ecommerce.progression-catalog'); + if (!binding) throw new Error('The ecommerce dependency recipe is unavailable.'); + const full = resolveFeatureCatalog('progression/ecommerce.json', track); + const catalog = selectFeatureCatalogLevels(full, progressionLevels(full).filter(level => level <= 3)); + const selected = new Set(resolveProgressionRecipeLevelSelection(binding, catalog, 3, + { cumulative: true }).grader.checkKeys); + const mutationRoot = join(STACK_BENCH_ROOT, 'grader', 'mutations'); + const mutations = readdirSync(mutationRoot).filter(file => file.endsWith('-ecommerce.json')).flatMap(file => { + const manifest = JSON.parse(readFileSync(join(mutationRoot, file), 'utf8')) as { + mutations: { id: string; desc: string; targets?: string[] }[]; + }; + return manifest.mutations.map(mutation => ({ ...mutation, backend: file.replace('-ecommerce.json', '') })); + }); + const sources = new Map(); + const read = (source: string) => { + let value = sources.get(source); + if (!value) { + const text = readFileSync(join(STACK_BENCH_ROOT, source), 'utf8'); + value = { ...JSON.parse(text), sha256: createHash('sha256').update(text).digest('hex') }; + sources.set(source, value!); + } + return value!; + }; + const checks: Check[] = binding.release.checkCatalog.map(check => { + const source = `tracks/ecommerce/${check.source}`; + const scenario = read(source); + const feature = scenario.features.find(feature => feature.id === check.featureId); + const criterion = feature?.criteria.find(criterion => criterion.id === check.criterionId); + if (!feature || !criterion) throw new Error(`Missing source for ${check.stableKey}`); + return { key: check.stableKey, active: selected.has(check.stableKey), source, + sourceSha256: scenario.sha256, feature, criterion, + controls: mutations.filter(mutation => mutation.targets?.includes(check.stableKey)) + .map(mutation => `${mutation.backend}: ${mutation.desc} (${mutation.id})`) }; + }); + const included = new Set(checks.map(c => `${c.source}:${c.feature.id}:${c.criterion.id}`)); + for (const name of listTracks()) { + const other = loadTrack(name); + for (const file of readdirSync(other.scenarios).filter(file => file.endsWith('.json')).sort()) { + const source = `tracks/${name}/scenarios/${file}`; + const scenario = read(source); + for (const feature of scenario.features) for (const criterion of feature.criteria) { + const key = `${source}:${feature.id}:${criterion.id}`; + if (!included.has(key)) checks.push({ key, active: false, source, + sourceSha256: scenario.sha256, feature, criterion, controls: [] }); + } + } + } + return { checks, selected: selected.size, recipe: binding.release.id, + sha256: binding.release.contentSha256 }; +} + +function procedure(steps: GuideStep[]): string { + return stepsText(steps).map(line => { + const text = line.trimStart(); + const kind = /Repeat \d|branches concurrently|Branch \d/.test(text) ? 'repeat' + : /^(?:- )?(?:Wait |Always wait|Observation deadline)/.test(text) ? 'wait' + : /require|reject|check that|check for|check all|compare stored|compare the|inspect stored/i.test(text) ? 'assert' + : 'action'; + const indent = Math.min(4, (line.length - text.length) / 2); + return `
${ + { repeat: 'Repeat / parallel', wait: 'Wait / deadline', assert: 'Verify', action: 'Action' }[kind] + }${esc(text.replace(/^- /, ''))}
`; + }).join(''); +} + +function entry(check: Check): string { + const { feature, criterion } = check; + const search = `${check.key} ${criterion.desc} ${feature.name} ${check.source}`.toLowerCase(); + const setup = procedure((feature.setup ?? []) as GuideStep[]); + return `
` + + `${esc(criterion.id)}${esc(criterion.desc)}` + + `${check.active ? 'L1–L3' : 'Other'}
` + + (criterion.statedBy ? `

Required behavior

${esc(criterion.statedBy)}

` : '') + + (criterion.provenBy ? `

Uses earlier proof: ${esc(criterion.provenBy)}

` : '') + + (setup ? '

Setup

' + setup : '') + + '

Steps

' + procedure(criterion.steps as GuideStep[]) + + '
Technical details' + + `

${esc(check.key)}

` + + `

${criterion.points} ${criterion.points === 1 ? 'point' : 'points'} · Browsers: ${esc((feature.actors ?? []).join(', ') || 'none')}

` + + (criterion.note ? `

${esc(criterion.note)}

` : '') + + '

Declared defect controls (targets, not proof of a passing qualification run):

' + + (check.controls.length ? '
    ' + check.controls.map(control => `
  • ${esc(control)}
    • `).join('') + '
' + : '

None linked to this exact check.

') + + `

${esc(check.source)} · SHA-256 ${check.sourceSha256}

` + + `
${esc(JSON.stringify({ actors: feature.actors, setup: feature.setup ?? [], criterion }, null, 2))}
` + + '
'; +} + +/** Current local definitions only. Historical campaign evidence retains its frozen version. */ +export function checkGuidePage(): string { + const data = inventory(); + return '' + + '' + + 'Checks · Stack Bench' + + '' + + topbar({ page: 'check-guide', key: '', canStart: false, resumable: false, error: '' }) + + '

Checks

' + + `

What each check does, step by step. ${data.selected} checks in the current ecommerce dependency L1–L3 selection.

` + + '
How checks are graded' + + '

A check passes only when every step runs and every Verify step holds. A failed prerequisite blocks the checks that depend on it. Unknown results and harness errors are not passes. Setup can be shared between checks; the runner keeps prerequisite order.

' + + '

This page shows the current local definitions, not a historical run\'s frozen version, and does not certify qualification. Waits and deadlines are authored values; omitted deadlines use grader defaults, so the steps are not an elapsed-time estimate.

' + + `

Recipe: ${esc(data.recipe)} · SHA-256 ${data.sha256}

` + + '
' + + '
' + + '' + + '
' + + '' + + data.checks.map(entry).join('') + '
'; +} diff --git a/tools/stack-bench/dashboard/dashboard-events.ts b/tools/stack-bench/dashboard/dashboard-events.ts new file mode 100644 index 00000000000..09bb3279b16 --- /dev/null +++ b/tools/stack-bench/dashboard/dashboard-events.ts @@ -0,0 +1,164 @@ +import { existsSync, readdirSync, statSync, watch } from 'node:fs'; +import type { FSWatcher } from 'node:fs'; +import { join } from 'node:path'; + +import { ARTIFACT_FILE } from '../src/evidence/artifacts.js'; +import { CAMPAIGN_FILE } from '../src/campaigns/campaign-path.js'; + +const DEBOUNCE_MS = 500; +const POLL_MS = 5000; +const LOG_FILE = 'process.stdout.log'; +const CAMPAIGN_FILES = [CAMPAIGN_FILE.plan, CAMPAIGN_FILE.state, 'depth-release.json'] as const; +const EXECUTION_FILES = [ARTIFACT_FILE.run, ARTIFACT_FILE.progressionState, 'depth-pause.json'] as const; + +export interface CampaignChange { + type: 'campaign' | 'log'; + key: string; + attemptId?: string; +} + +export type WatchMode = 'watch' | 'poll'; + +export interface CampaignWatcher { + close(): void; +} + +interface CampaignFingerprint { + campaign: string; + logs: Map; +} + +function stamp(path: string): string | null { + if (!existsSync(path)) return null; + const stat = statSync(path); + return `${stat.size}:${stat.mtimeMs}`; +} + +function directories(root: string): string[] { + if (!existsSync(root)) return []; + return readdirSync(root, { withFileTypes: true }) + .filter(entry => entry.isDirectory()).map(entry => entry.name); +} + +// The evidence a view reads, in one pass: the campaign files and each +// execution's result, and the log sizes that tell a follower there are new +// bytes to fetch. +function fingerprintCampaign(directory: string): CampaignFingerprint { + const parts = CAMPAIGN_FILES.map(file => `${file}:${stamp(join(directory, file)) ?? 'missing'}`); + const logs = new Map(); + const attemptsRoot = join(directory, 'attempts'); + for (const attempt of directories(attemptsRoot)) { + const attemptDirectory = join(attemptsRoot, attempt); + let bytes = 0; + for (const execution of directories(attemptDirectory)) { + const executionDirectory = join(attemptDirectory, execution); + for (const file of EXECUTION_FILES) { + const value = stamp(join(executionDirectory, file)); + if (value) parts.push(`${attempt}/${execution}/${file}:${value}`); + } + const log = join(executionDirectory, LOG_FILE); + if (existsSync(log)) bytes += statSync(log).size; + } + logs.set(attempt, bytes); + } + return { campaign: parts.sort().join('|'), logs }; +} + +// One watcher for the whole server: a recursive watch per campaign directory +// where the platform supports it (Windows, macOS, and Linux on Node 20 and +// later), and a poll of the same fingerprints where it does not. +export function watchCampaigns(campaignsRoot: string, + emit: (change: CampaignChange) => void, + onMode: (mode: WatchMode) => void = () => {}): CampaignWatcher { + const fingerprints = new Map(); + const watchers = new Map(); + const timers = new Map(); + let rootWatcher: FSWatcher | null = null; + let poll: NodeJS.Timeout | null = null; + let closed = false; + + const check = (key: string): void => { + timers.delete(key); + const directory = join(campaignsRoot, key); + if (!existsSync(directory)) { + fingerprints.delete(key); + watchers.get(key)?.close(); + watchers.delete(key); + return; + } + const next = fingerprintCampaign(directory); + const previous = fingerprints.get(key); + fingerprints.set(key, next); + if (!previous) return; + if (previous.campaign !== next.campaign) emit({ type: 'campaign', key }); + for (const [attemptId, bytes] of next.logs) { + if ((previous.logs.get(attemptId) ?? 0) !== bytes) emit({ type: 'log', key, attemptId }); + } + }; + + const schedule = (key: string): void => { + if (closed || timers.has(key)) return; + timers.set(key, setTimeout(() => check(key), DEBOUNCE_MS).unref()); + }; + + const startPoll = (): void => { + if (closed || poll) return; + onMode('poll'); + poll = setInterval(() => { + attach(); + for (const key of directories(campaignsRoot)) check(key); + }, POLL_MS).unref(); + }; + + const attach = (): void => { + if (closed) return; + if (!rootWatcher && existsSync(campaignsRoot)) { + try { + rootWatcher = watch(campaignsRoot, { persistent: false }, (_event, name) => { + const key = String(name ?? '').split(/[/\\]/)[0]; + if (key) schedule(key); + attach(); + }); + rootWatcher.once('error', () => { rootWatcher = null; startPoll(); }); + } catch { startPoll(); } + } + for (const key of directories(campaignsRoot)) { + // A campaign that appeared since the last pass has everything to report. + if (!fingerprints.has(key)) { + fingerprints.set(key, { campaign: '', logs: new Map() }); + schedule(key); + } + if (watchers.has(key) || poll) continue; + try { + const watcher = watch(join(campaignsRoot, key), { persistent: false, recursive: true }, + () => schedule(key)); + watcher.once('error', () => { watchers.delete(key); startPoll(); }); + watchers.set(key, watcher); + } catch { + // No recursive watch on this platform: the poll reads the same files. + startPoll(); + return; + } + } + }; + + for (const key of directories(campaignsRoot)) { + fingerprints.set(key, fingerprintCampaign(join(campaignsRoot, key))); + } + attach(); + if (!rootWatcher) startPoll(); + else if (!poll) onMode('watch'); + return { + close() { + closed = true; + for (const timer of timers.values()) clearTimeout(timer); + timers.clear(); + if (poll) clearInterval(poll); + poll = null; + rootWatcher?.close(); + rootWatcher = null; + for (const watcher of watchers.values()) watcher.close(); + watchers.clear(); + }, + }; +} diff --git a/tools/stack-bench/dashboard/dashboard-live-cost.ts b/tools/stack-bench/dashboard/dashboard-live-cost.ts new file mode 100644 index 00000000000..b8e501e6c1c --- /dev/null +++ b/tools/stack-bench/dashboard/dashboard-live-cost.ts @@ -0,0 +1,122 @@ +import { attemptTranscriptFiles } from './dashboard-transcript.js'; +import { normalizeClaudeUsage, priceClaudeUsage } from '../src/evidence/claude-usage-cost.js'; +import type { PricingRates } from '../src/evidence/pricing-authority.js'; + +interface UsagePoint { id: string; completedAt: string; costUsd: number; signature: string } +export function liveCostTotal(status: string, observed: number | undefined, saved: number | null): number | undefined { + return status === 'running' && observed !== undefined && observed >= (saved ?? 0) ? observed : undefined; +} +const object = (value: unknown): value is Record => + !!value && typeof value === 'object' && !Array.isArray(value); + +// Display-only usage. Never write these observations into benchmark evidence. +export function responseCosts(text: string, rates: PricingRates, model: string, startedAt: string): UsagePoint[] { + const points: UsagePoint[] = []; + for (const line of text.split('\n')) { + if (!line.trim()) continue; + let event: unknown; + try { event = JSON.parse(line); } catch { throw new Error('Incomplete usage transcript'); } + if (!object(event) || event.type !== 'assistant' || !object(event.message) || !event.message.usage) continue; + const message = event.message; + if (typeof message.stop_reason !== 'string' || !message.stop_reason) continue; + const timestamp = typeof event.timestamp === 'string' ? Date.parse(event.timestamp) : NaN; + if (!Number.isFinite(timestamp)) throw new Error('Usage timestamp unavailable'); + if (timestamp < Date.parse(startedAt)) continue; + if (typeof message.model !== 'string' || !(message.model === model || message.model.startsWith(`${model}-`))) { + throw new Error('No pinned price for transcript model'); + } + const id = event.requestId ?? message.id ?? event.uuid; + if (typeof id !== 'string' || !id) throw new Error('Usage request identity unavailable'); + points.push({ id, completedAt: new Date(timestamp).toISOString(), + costUsd: priceClaudeUsage(message.usage, rates), + signature: JSON.stringify([message.model, normalizeClaudeUsage(message.usage)]) }); + } + return points; +} + +export function cumulativeResponseCosts(points: readonly UsagePoint[]): Array<{ completedAt: string; costUsd: number }> { + const unique = new Map(); + for (const point of points) { + const prior = unique.get(point.id); + if (prior && prior.signature !== point.signature) throw new Error('Conflicting usage for request'); + if (!prior) unique.set(point.id, point); + } + let total = 0; + return [...unique.values()].sort((a, b) => a.completedAt.localeCompare(b.completedAt)).map(point => ({ + completedAt: point.completedAt, costUsd: Number((total += point.costUsd).toFixed(6)), + })); +} + +interface CodexUsageState { session?: string; model?: string; totals?: [number, number, number] } + +/** `state` carries session, model and cumulative totals across incremental reads; it is updated in place. */ +export function codexResponseCosts(text: string, rates: PricingRates, model: string, startedAt: string, + state: CodexUsageState = {}): UsagePoint[] { + const points: UsagePoint[] = []; + for (const line of text.split('\n')) { + if (!line.trim()) continue; + const event: unknown = JSON.parse(line); + if (!object(event) || !object(event.payload)) continue; + const payload = event.payload; + if (event.type === 'session_meta') { + if (typeof payload.id !== 'string' || !payload.id) throw new Error('Usage session identity unavailable'); + if (state.session && state.session !== payload.id) throw new Error('Usage session changed'); + state.session = payload.id; + } + if (event.type === 'turn_context') state.model = typeof payload.model === 'string' ? payload.model : undefined; + if (event.type !== 'event_msg' || payload.type !== 'token_count' || payload.info === null) continue; + const usage = object(payload.info) && object(payload.info.total_token_usage) + ? payload.info.total_token_usage : {}; + const totals = [usage.input_tokens, usage.cached_input_tokens, usage.output_tokens]; + if (!totals.every(value => typeof value === 'number' && Number.isSafeInteger(value) && value >= 0) + || Number(totals[1]) > Number(totals[0])) throw new Error('Invalid Codex token usage'); + const counts = totals as [number, number, number]; + const previous = state.totals ?? [0, 0, 0]; + const delta = [counts[0] - previous[0], counts[1] - previous[1], counts[2] - previous[2]] as const; + if (delta.some(value => value < 0) || delta[1] > delta[0]) throw new Error('Codex usage totals decreased'); + state.totals = counts; + const timestamp = typeof event.timestamp === 'string' ? Date.parse(event.timestamp) : NaN; + if (!Number.isFinite(timestamp)) throw new Error('Usage timestamp unavailable'); + if (timestamp < Date.parse(startedAt) || delta.every(value => value === 0)) continue; + if (!state.session) throw new Error('Usage session identity unavailable'); + if (state.model !== model) throw new Error('No pinned price for transcript model'); + points.push({ id: `${state.session}:${counts.join(':')}`, completedAt: new Date(timestamp).toISOString(), + costUsd: ((delta[0] - delta[1]) * rates.input + delta[1] * rates.cacheRead + delta[2] * rates.output) / 1e6, + signature: JSON.stringify([state.model, delta]) }); + } + return points; +} + +const cache = new Map(); + +export async function liveTranscriptCost(directory: string, adapter: string, rates: PricingRates, + model: string, startedAt: string) { + const files = await attemptTranscriptFiles([{ directory, label: 'Execution' }], adapter); + const activityUpdatedAt = files.length ? new Date(Math.max(...files.map(file => file.modified))).toISOString() : null; + const points: UsagePoint[] = []; + for (const file of files) { + const key = `${directory}/${file.id}/${startedAt}/${JSON.stringify(rates)}/${model}`; + const prior = cache.get(key); + if (prior?.size === file.size && prior.modified === file.modified) { points.push(...prior.points); continue; } + // Codex counters are cumulative; the cached state continues them from the last complete line. + const offsetStart = prior && file.size > prior.size ? prior.offset : 0; + const codex: CodexUsageState = offsetStart ? structuredClone(prior!.codex) : {}; + // Bound catch-up work; do not label a partial file as a complete live total. + if (file.size - offsetStart > 16 * 1024 * 1024) return { activityUpdatedAt, costs: [] }; + const chunks: Buffer[] = []; + for (let offset = offsetStart; offset < file.size; offset += 256 * 1024) { + chunks.push(await file.read(offset, Math.min(256 * 1024, file.size - offset))); + } + const bytes = Buffer.concat(chunks); + const end = bytes.lastIndexOf(10); + const text = end < 0 ? '' : bytes.subarray(0, end + 1).toString(); + const parsed = [...(offsetStart ? prior!.points : []), ...(adapter === 'codex' + ? codexResponseCosts(text, rates, model, startedAt, codex) + : responseCosts(text, rates, model, startedAt))]; + if (cache.size >= 128) cache.delete(cache.keys().next().value!); + cache.set(key, { size: file.size, modified: file.modified, offset: offsetStart + end + 1, points: parsed, codex }); + points.push(...parsed); + } + return { activityUpdatedAt, costs: cumulativeResponseCosts(points) }; +} diff --git a/tools/stack-bench/dashboard/dashboard-model.ts b/tools/stack-bench/dashboard/dashboard-model.ts new file mode 100644 index 00000000000..9f46d32e8a0 --- /dev/null +++ b/tools/stack-bench/dashboard/dashboard-model.ts @@ -0,0 +1,543 @@ +import { closeSync, existsSync, fstatSync, openSync, readFileSync, readSync, readdirSync, + lstatSync, realpathSync, statSync, +} from 'node:fs'; +import { basename, dirname, extname, join, relative, resolve, sep } from 'node:path'; + +import type { CompiledCampaignPlan } + from '../src/campaigns/campaign-compiler.js'; +import type { CampaignAttemptState } from '../src/campaigns/campaign-scheduler.js'; +import { ARTIFACT_FILE } from '../src/evidence/artifacts.js'; +import { compileCampaignFile } from '../src/campaigns/campaign-compiler.js'; +import { campaignLockIsActive, readCampaignLock } from '../src/campaigns/campaign-lock.js'; +import { readDepthPause } from '../src/campaigns/campaign-depth-pause.js'; +import { readCampaignState } from '../src/campaigns/campaign-scheduler.js'; +import { campaignFacts, inspectCampaignAttempt } from '../src/campaigns/campaign-inspection.js'; +import { resolveCalibrationForRelease } from '../src/composition/calibration-compiler.js'; +import type { CalibrationPlan } from '../src/composition/calibration-compiler.js'; +import { redactCredentials } from '../src/evidence/diagnostic-sanitizer.js'; +import { CAMPAIGN_FILE } from '../src/campaigns/campaign-path.js'; +import { repairBudgetLimit } from '../src/progression/repair-plan.js'; + +export const MAX_LOG_BYTES = 96 * 1024; +const MAX_PUBLIC_TEXT_BYTES = 8 * 1024 * 1024; +const MAX_ARTIFACTS_PER_EXECUTION = 512; +const IMAGE_TYPES = new Map([ + ['.png', 'image/png'], ['.jpg', 'image/jpeg'], ['.jpeg', 'image/jpeg'], ['.webp', 'image/webp'], +]); +const CAMPAIGN_ARTIFACT = /^(?:plan\.json|state\.json|report\/(?:report\.(?:html|json)|export-manifest\.json))$/; +const EXECUTION_ARTIFACT = /^(?:run\.json|preflight\.json|recovery\.json|progression-state\.json|process\.json|process\.(?:stdout|stderr)\.log|backend\.log|level-l\d+-checkpoint\.json|progression\/attempt-\d+\/(?:bundle\.json|contract-lint\.json|actions\.json|grading-[^/]+\.json|failure-media\/[^/]+\.(?:png|jpe?g|webp))|(?:first-build-l\d+-grading|l\d+-fix\d+-grading|grading)\/(?:bundle\.json|contract-lint\.json|actions\.json|grading-[^/]+\.json|failure-media\/[^/]+\.(?:png|jpe?g|webp)))$/i; + +type ControllerActive = (directory: string, campaign: CompiledCampaignPlan) => boolean; + +// The dashboard serves one unchanging build, so each calibration is compiled +// and its qualification evidence verified once, not on every campaign read. +const compiledCalibrations = new Map(); +export const calibrationResolver: typeof resolveCalibrationForRelease = (release, options) => + resolveCalibrationForRelease(release, { ...options, compiled: compiledCalibrations }); + +export interface DashboardArtifact { + id: string; + path: string; + name: string; + kind: 'visual' | 'report' | 'log' | 'data'; + contentType: string; + size: number; +} + +export interface ResolvedDashboardArtifact extends DashboardArtifact { + absolute: string; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +export function contained(root: string, path: string, label: string): string { + const absoluteRoot = resolve(root); + const absolute = resolve(absoluteRoot, path); + const rel = relative(absoluteRoot, absolute); + if (rel === '..' || rel.startsWith(`..${sep}`) || rel === '') { + throw new Error(`${label} is outside the configured dashboard root`); + } + return absolute; +} + +export function readTextTail(path: string, limit = MAX_LOG_BYTES): string { + if (!existsSync(path)) return ''; + const descriptor = openSync(path, 'r'); + try { + const size = fstatSync(descriptor).size; + const length = Math.min(size, limit); + const buffer = Buffer.alloc(length); + readSync(descriptor, buffer, 0, length, size - length); + return redactCredentials(buffer.toString('utf8')); + } finally { + closeSync(descriptor); + } +} + +function artifactId(relativePath: string): string { + return Buffer.from(relativePath, 'utf8').toString('base64url'); +} + +function artifactLabel(path: string): string { + const name = basename(path); + if (path === CAMPAIGN_FILE.plan) return 'Frozen plan'; + if (path === CAMPAIGN_FILE.state) return 'Campaign state'; + if (path === `report/${CAMPAIGN_FILE.reportHtml}`) return 'Campaign report'; + if (path === `report/${CAMPAIGN_FILE.reportJson}`) return 'Report data'; + if (name === ARTIFACT_FILE.run) return 'Run result'; + if (name === ARTIFACT_FILE.preflight) return 'Preflight result'; + if (name === ARTIFACT_FILE.recovery) return 'Recovery record'; + if (name === ARTIFACT_FILE.progressionState) return 'Dependency progress'; + if (name === 'process.stdout.log') return 'Run output'; + if (name === 'process.stderr.log') return 'Run errors'; + if (name === 'backend.log') return 'Backend output'; + if (name === ARTIFACT_FILE.gradeBundle) return `${basename(dirname(path))} bundle`; + if (name === ARTIFACT_FILE.actions) return `${basename(dirname(path))} actions`; + if (name === ARTIFACT_FILE.contractLint) return `${basename(dirname(path))} contract check`; + return name.replace(/[-_]/g, ' '); +} + +function artifactMetadata(campaignDirectory: string, path: string): DashboardArtifact { + const absolute = contained(campaignDirectory, path, 'campaign artifact'); + const size = statSync(absolute).size; + const extension = extname(path).toLowerCase(); + const kind = IMAGE_TYPES.has(extension) ? 'visual' + : path.endsWith('/report.html') ? 'report' + : path.endsWith('.log') ? 'log' : 'data'; + return { id: artifactId(path), path: path.replaceAll('\\', '/'), name: artifactLabel(path), + kind, contentType: IMAGE_TYPES.get(extension) ?? (kind === 'report' ? 'text/html' : 'text/plain'), + size }; +} + +function rejectSymlinkPath(root: string, path: string): void { + const rel = relative(resolve(root), resolve(path)); + let current = resolve(root); + for (const segment of rel.split(sep)) { + current = join(current, segment); + if (lstatSync(current).isSymbolicLink()) { + throw new Error('campaign artifact path contains a symbolic link'); + } + } +} + +export function walkPublicExecutionArtifacts(campaignDirectory: string, executionDirectory: string): { + artifacts: DashboardArtifact[]; + truncated: boolean; +} { + const found: DashboardArtifact[] = []; + let truncated = false; + const visit = (directory: string): void => { + const directoryRelative = relative(executionDirectory, directory).replaceAll('\\', '/'); + for (const entry of readdirSync(directory, { withFileTypes: true })) { + if (found.length >= MAX_ARTIFACTS_PER_EXECUTION) { + truncated = true; + return; + } + if (entry.isSymbolicLink()) continue; + const absolute = join(directory, entry.name); + if (entry.isDirectory()) { + const allowed = directoryRelative === '' + ? /^(?:first-build-l\d+-grading|l\d+-fix\d+-grading|grading|progression)$/i.test(entry.name) + : directoryRelative === 'progression' + ? /^attempt-\d+$/i.test(entry.name) + : (/^(?:progression\/attempt-\d+|(?:.*\/)?(?:first-build-l\d+-grading|l\d+-fix\d+-grading|grading))$/i + .test(directoryRelative) && entry.name === 'failure-media'); + if (allowed) visit(absolute); + if (truncated) return; + } + else if (entry.isFile()) { + const executionRelative = relative(executionDirectory, absolute).replaceAll('\\', '/'); + if (EXECUTION_ARTIFACT.test(executionRelative)) { + const campaignRelative = relative(campaignDirectory, absolute).replaceAll('\\', '/'); + found.push(artifactMetadata(campaignDirectory, campaignRelative)); + } + } + } + }; + if (existsSync(executionDirectory)) visit(executionDirectory); + return { artifacts: found.sort((left, right) => left.path.localeCompare(right.path)), truncated }; +} + +function campaignPackage(campaignDirectory: string, attempts: CampaignAttemptState[]) { + const campaign = [CAMPAIGN_FILE.plan, CAMPAIGN_FILE.state, + `report/${CAMPAIGN_FILE.reportHtml}`, `report/${CAMPAIGN_FILE.reportJson}`] + .filter(path => existsSync(join(campaignDirectory, path))) + .map(path => artifactMetadata(campaignDirectory, path)); + const executions: Array<{ + attemptId: string; + stack: string; + executionId: string; + ordinal: number; + status: string; + artifacts: DashboardArtifact[]; + visuals: DashboardArtifact[]; + truncated: boolean; + }> = []; + for (const attempt of attempts) { + for (const execution of attempt.executions) { + const directory = contained(campaignDirectory, execution.output, 'campaign execution'); + const scanned = walkPublicExecutionArtifacts(campaignDirectory, directory); + const artifacts = scanned.artifacts; + executions.push({ attemptId: attempt.plan.id, stack: attempt.plan.stack, + executionId: execution.id, ordinal: execution.ordinal, status: execution.status, + artifacts, visuals: artifacts.filter(artifact => artifact.kind === 'visual'), + truncated: scanned.truncated }); + } + } + return { campaign, executions }; +} + +export function resolveCampaignArtifact(resultsRoot: string, key: string, + id: string): ResolvedDashboardArtifact { + if (!/^[a-z0-9][a-z0-9.-]*$/.test(key)) throw new Error('campaign key is invalid'); + let path; + try { path = Buffer.from(id, 'base64url').toString('utf8'); } + catch { throw new Error('campaign artifact id is invalid'); } + if (!path || artifactId(path) !== id || path.includes('\\') || path.startsWith('/')) { + throw new Error('campaign artifact id is invalid'); + } + const executionMatch = path.match(/^attempts\/([^/]+)\/(execution-\d+)\/(.+)$/); + const allowed = CAMPAIGN_ARTIFACT.test(path) + || (executionMatch !== null && EXECUTION_ARTIFACT.test(executionMatch[3] ?? '')); + if (!allowed) { + throw new Error('campaign artifact is not available in the dashboard'); + } + const campaignsRoot = join(resolve(resultsRoot), 'campaigns'); + const campaignDirectory = contained(campaignsRoot, key, 'campaign'); + const absolute = contained(campaignDirectory, path, 'campaign artifact'); + if (!existsSync(absolute) || !statSync(absolute).isFile()) throw new Error('campaign artifact does not exist'); + rejectSymlinkPath(campaignsRoot, absolute); + const realCampaign = realpathSync(campaignDirectory); + const realArtifact = realpathSync(absolute); + contained(realCampaign, relative(realCampaign, realArtifact), 'campaign artifact'); + return { ...artifactMetadata(campaignDirectory, path), absolute }; +} + +export function readCampaignArtifactBody(artifact: ResolvedDashboardArtifact): Buffer { + if (artifact.kind === 'visual') return readFileSync(artifact.absolute); + if (artifact.size > MAX_PUBLIC_TEXT_BYTES) throw new Error('campaign artifact is too large to view'); + return Buffer.from(redactCredentials(readFileSync(artifact.absolute, 'utf8'))); +} + +function matches(text: string, pattern: RegExp): Array { + return [...text.matchAll(pattern)].map(match => Object.assign(match, { index: match.index ?? 0 })); +} + +export function parseRunProgress(log: string, { repairs = 0, running = true, status = null, + dependency = false }: { + repairs?: number; + running?: boolean; + status?: string | null; + dependency?: boolean; +} = {}) { + const totals = matches(log, /^\s*TOTAL\b.*?(\d+)\/(\d+)\s*$/gm) + .map(match => ({ index: match.index, score: Number(match[1]), max: Number(match[2]) })); + // A run-wide repair prints "repair N/M"; a feature repair prints + // "feature repair N: title" because its limit belongs to the feature. + const roundMarkers = matches(log, + /^--- (?:feature )?repair (\d+)(?:\/(\d+))?(?:: (.+))? ---$/gm) + .map(match => ({ index: match.index, round: Number(match[1]), + budget: match[2] === undefined ? null : Number(match[2]), + target: match[3] ?? null })); + const grading = matches(log, /^===\s+[^\n]*?-l(\d+)(?:-(?:first|fix(\d+)))?\s+\([^\n]+\)\s*===$/gm) + .map(match => ({ index: match.index, level: Number(match[1]), + round: match[2] ? Number(match[2]) : 0 })); + const latestTotal = totals.at(-1) ?? null; + const latestRound = roundMarkers.at(-1) ?? null; + const latestGrading = grading.at(-1) ?? null; + const latestIndex = Math.max(latestTotal?.index ?? -1, latestRound?.index ?? -1, + latestGrading?.index ?? -1); + let phase = status === 'pending' ? 'Waiting to start' + : running ? 'Building the generated app' + : status === 'invalid' ? 'Stopped without a valid result' : 'Finished'; + const level = latestGrading?.level ?? null; + const round = latestRound?.round ?? latestGrading?.round ?? 0; + const budget = latestRound ? latestRound.budget : repairs; + const target = latestRound?.target ? ` for ${latestRound.target}` : ''; + const of = (limit: number | null): string => limit === null ? '' : ` of ${limit}`; + const stage = (value: number): string => dependency ? `depth ${value}` : `L${value}`; + if (running && latestIndex === latestGrading?.index) { + phase = latestGrading.round + ? `Grading ${stage(latestGrading.level)} after repair ${round}${of(budget)}${target}` + : `Grading the first ${stage(latestGrading.level)} build`; + } else if (running && latestIndex === latestRound?.index) { + phase = latestRound.target + ? `Repairing ${latestRound.target} · ${latestRound.round}${of(latestRound.budget)}` + : `Repairing ${stage(latestGrading?.level ?? 1)} · round ${latestRound.round}${of(latestRound.budget)}`; + } else if (latestIndex === latestTotal?.index && running) { + phase = 'Preparing the next step'; + } + return { + phase, + level, + repair: { round, budget }, + firstScore: totals[0] ? { score: totals[0].score, max: totals[0].max } : null, + latestScore: latestTotal ? { score: latestTotal.score, max: latestTotal.max } : null, + completedGrades: totals.length, + // Every completed grade in order — the attempt's trajectory — carrying the + // level it graded and whether it was the unaided build of that level. A + // view can draw the climb with its bands, and a flat tail is the stall an + // operator otherwise discovers by diffing round logs. + series: totals.map(total => { + const mark = grading.findLast(entry => entry.index < total.index) ?? null; + return { score: total.score, max: total.max, level: mark?.level ?? null, + unaided: mark ? mark.round === 0 : false }; + }), + }; +} + +export function attemptPause(plan: CompiledCampaignPlan, attempt: CampaignAttemptState, + directory: string) { + const depth = plan.definition.mode.pauseAfterDepth; + const execution = attempt.executions.at(-1); + const lock = depth === undefined ? null : readCampaignLock(directory); + if (depth === undefined || !execution || !lock) return null; + return readDepthPause(contained(directory, execution.output, 'campaign execution'), { + directory, depth, campaignSha256: plan.contentSha256, + ownershipMarkerSha256: lock.ownershipMarkerSha256, + attemptId: attempt.plan.id, executionId: execution.id, + }); +} + +function summarizeAttempt(plan: CompiledCampaignPlan, attempt: CampaignAttemptState, + campaignDirectory: string, repairs: number, { includeLog = false }: { + includeLog?: boolean; + } = {}) { + const inspected = inspectCampaignAttempt(plan, attempt, campaignDirectory); + const execution = inspected.execution; + let executionDirectory = null; + let log = ''; + let logUpdatedAt = null; + if (execution) { + executionDirectory = contained(campaignDirectory, execution.output, 'campaign execution'); + const logPath = join(executionDirectory, 'process.stdout.log'); + log = readTextTail(logPath); + // When the run last wrote anything. A running attempt whose output has + // been silent for a long time is wedged in a way no score can show. + if (existsSync(logPath)) logUpdatedAt = new Date(statSync(logPath).mtimeMs).toISOString(); + } + const progress = parseRunProgress(log, { repairs, running: attempt.status === 'running', + status: attempt.status, dependency: plan.definition.mode.id === 'dependency' }); + const pause = attemptPause(plan, attempt, campaignDirectory); + const paused = attempt.status === 'running' && pause?.resumedAt === null; + if (paused) progress.phase = `Paused at L${pause.depth}`; + if (inspected.result?.score) progress.latestScore = inspected.result.score; + return { + ...inspected, + progress, + paused, + logUpdatedAt, + ...(includeLog ? { log: log.split(/\r?\n/).slice(-160).join('\n') } : {}), + }; +} + +export function summarizeCampaign(directory: string, { + includeLogs = false, + includePackage = false, + includeAttempts = true, + controllerActive = null, +}: { + includeLogs?: boolean; + includePackage?: boolean; + includeAttempts?: boolean; + controllerActive?: ControllerActive | null; +} = {}) { + const { plan, state } = readCampaignState(directory, { requireCurrentInputs: false }); + let attempts = includeAttempts + ? state.attempts.map(attempt => summarizeAttempt(plan, attempt, directory, + repairBudgetLimit(plan.definition.repair, { + features: plan.featureCatalog?.definition.nodes.length ?? 1, + depths: plan.definition.levels.length, + }), { includeLog: includeLogs })) + : []; + const interrupted = state.status === 'running' && controllerActive !== null + && !controllerActive(directory, plan); + if (interrupted) { + attempts = attempts.map(attempt => attempt.status !== 'running' ? attempt : ({ + ...attempt, + status: 'interrupted', + execution: attempt.execution ? { ...attempt.execution, status: 'interrupted' } : null, + progress: { ...attempt.progress, phase: 'Controller stopped before completion' }, + })); + } + return { + key: basename(resolve(directory)), + id: plan.id, + version: plan.version, + sha256: plan.contentSha256, + title: plan.title, + state: plan.state, + mode: plan.definition.mode?.id ?? 'sequential', + status: interrupted ? 'attention-required' : state.status, + track: plan.definition.track, + levels: plan.definition.levels, + stacks: plan.stacks.map(stack => stack.id), + repetitions: plan.definition.repetitions, + maxParallel: state.maxParallel, + createdAt: state.createdAt, + updatedAt: state.updatedAt, + summary: interrupted ? { ...state.summary, interrupted: state.summary.running, running: 0 } + : state.summary, + interrupted, + ...(interrupted ? { statusReason: 'The campaign controller is no longer running.' } : {}), + budgets: plan.definition.budgets, + facts: campaignFacts(plan, { calibrationResolver }), + attempts, + ...(includePackage ? { package: campaignPackage(directory, state.attempts) } : {}), + }; +} + +export interface UnreadableDashboardCampaign { + key: string; + id: string; + title: string; + status: 'unreadable'; + error: string; + attempts: []; +} + +export type DashboardCampaign = ReturnType; +export type DashboardCampaignSummary = DashboardCampaign | UnreadableDashboardCampaign; + +const overviewCampaignCache = new Map(); + +function summarizeOverviewCampaign(directory: string, includeAttempts: boolean, + controllerActive: ControllerActive): DashboardCampaign { + const fingerprint = [CAMPAIGN_FILE.plan, CAMPAIGN_FILE.state] + .map(file => { + const stat = statSync(join(directory, file)); + return `${stat.size}:${stat.mtimeMs}`; + }).join('|'); + const key = `${includeAttempts ? 'attempts' : 'summary'}:${directory}`; + const cached = overviewCampaignCache.get(key); + if (cached?.fingerprint === fingerprint) return cached.campaign; + const campaign = summarizeCampaign(directory, { includeAttempts, controllerActive }); + if (campaign.summary.running === 0) { + overviewCampaignCache.set(key, { fingerprint, campaign }); + } + return campaign; +} + +export function discoverCampaigns(campaignsRoot: string, { + includeLogs = false, + controllerActive = campaignLockIsActive, +}: { includeLogs?: boolean; controllerActive?: ControllerActive } = {}) { + if (!existsSync(campaignsRoot)) return []; + const campaigns: DashboardCampaignSummary[] = []; + for (const entry of readdirSync(campaignsRoot, { withFileTypes: true })) { + if (!entry.isDirectory()) continue; + const directory = join(campaignsRoot, entry.name); + if (!existsSync(join(directory, CAMPAIGN_FILE.state)) + || !existsSync(join(directory, CAMPAIGN_FILE.plan))) continue; + try { + campaigns.push(includeLogs + ? summarizeCampaign(directory, { includeLogs, controllerActive }) + : summarizeOverviewCampaign(directory, false, controllerActive)); + } catch (error) { + campaigns.push({ key: entry.name, id: entry.name, title: entry.name, + status: 'unreadable', error: errorMessage(error), attempts: [] }); + } + } + campaigns.sort((left, right) => String('updatedAt' in right ? right.updatedAt ?? '' : '') + .localeCompare(String('updatedAt' in left ? left.updatedAt ?? '' : ''))); + if (includeLogs) return campaigns; + + const verdict = campaigns.find(campaign => campaign.status === 'completed' + && 'facts' in campaign && campaign.facts.grading.status === 'qualified'); + return campaigns.map(campaign => { + if (campaign.status !== 'running' && campaign !== verdict) return campaign; + try { + return summarizeOverviewCampaign(join(campaignsRoot, campaign.key), true, controllerActive); + } catch (error) { + const unreadable: UnreadableDashboardCampaign = { + key: campaign.key, + id: campaign.id, + title: campaign.title, + status: 'unreadable', + error: errorMessage(error), + attempts: [], + }; + return unreadable; + } + }); +} + +export interface DashboardPlan { + id: string; + version?: string; + title: string; + state: string; + mode?: string; + track?: string; + levels?: number[]; + stacks?: string[]; + attempts?: number; + parallelism?: number; + budgets?: CompiledCampaignPlan['definition']['budgets']; + repairBudget?: number; + sha256?: string; + file: string; + error?: string; +} + +// Compiling a plan resolves its whole composition (tens of seconds each). The +// inputs outside the file ship with this release, so reuse a result until the +// file's size or modification time changes. +const planCache = new Map(); + +export function discoverPlans(plansRoot: string): DashboardPlan[] { + if (!existsSync(plansRoot)) return []; + const plans: DashboardPlan[] = []; + const seen = new Set(); + for (const entry of readdirSync(plansRoot, { withFileTypes: true })) { + if (!entry.isFile() || !entry.name.endsWith('.json')) continue; + const path = join(plansRoot, entry.name); + seen.add(path); + let fingerprint: string | null = null; + try { + const stat = statSync(path); + fingerprint = `${stat.size}:${stat.mtimeMs}`; + const cached = planCache.get(path); + if (cached?.fingerprint === fingerprint) { plans.push(cached.plan); continue; } + const plan = compileCampaignFile(path); + plans.push({ id: plan.id, version: plan.version, title: plan.title, state: plan.state, + mode: plan.definition.mode?.id ?? 'sequential', + track: plan.definition.track, levels: plan.definition.levels, + stacks: plan.stacks.map(stack => stack.id), attempts: plan.summary.attempts, + parallelism: plan.summary.parallelism, budgets: plan.definition.budgets, + repairBudget: repairBudgetLimit(plan.definition.repair, { + features: plan.featureCatalog?.definition.nodes.length ?? 1, + depths: plan.definition.levels.length, + }), + sha256: plan.contentSha256, file: entry.name }); + } catch (error) { + plans.push({ id: entry.name.slice(0, -5), title: entry.name, state: 'invalid', + error: errorMessage(error), file: entry.name }); + } + if (fingerprint) planCache.set(path, { fingerprint, plan: plans.at(-1)! }); + } + for (const path of planCache.keys()) if (dirname(path) === plansRoot && !seen.has(path)) planCache.delete(path); + return plans.sort((left, right) => left.title.localeCompare(right.title)); +} + +export function readJsonLines(path: string): unknown[] { + if (!existsSync(path)) return []; + const lines = readFileSync(path, 'utf8').split(/\r?\n/); + const last = lines.findLastIndex(line => line.trim() !== ''); + const events: unknown[] = []; + for (let index = 0; index <= last; index += 1) { + const line = lines[index]; + if (!line?.trim()) continue; + try { events.push(JSON.parse(line)); } + catch { + if (index === last) break; + throw new Error(`dashboard operation feed line ${index + 1} is invalid JSON`); + } + } + return events; +} diff --git a/tools/stack-bench/dashboard/dashboard-reader.ts b/tools/stack-bench/dashboard/dashboard-reader.ts new file mode 100644 index 00000000000..c524e73806c --- /dev/null +++ b/tools/stack-bench/dashboard/dashboard-reader.ts @@ -0,0 +1,58 @@ +import { isMainThread, parentPort, Worker } from 'node:worker_threads'; +import { overviewPage, campaignLiveSheet, campaignLiveUpdate, campaignLiveProgression, + attemptChecks, attemptPackage, attemptTranscript, attemptLogSlice } from './dashboard-views.js'; +import { discoverPlans } from './dashboard-model.js'; + +const reads = { overviewPage, campaignLiveSheet, campaignLiveUpdate, campaignLiveProgression, + attemptChecks, attemptPackage, attemptTranscript, attemptLogSlice, discoverPlans }; +type ReadName = keyof typeof reads; + +if (!isMainThread) { + parentPort!.on('message', async ({ id, name, args }: { id: number; name: ReadName; args: never[] }) => { + try { parentPort!.postMessage({ id, value: await (reads[name] as (...input: never[]) => unknown)(...args) }); } + catch (error) { parentPort!.postMessage({ id, error: error instanceof Error ? error.message : String(error) }); } + }); +} + +// One reader retains the existing view caches. CPU-heavy evidence validation +// must not block health, events, or run controls on the HTTP thread. +export function createDashboardReader() { + let worker: Worker | null = null; + let sequence = 0; + let closed = false; + const pending = new Map void; reject: (error: Error) => void }>(); + const inflight = new Map>(); + function read(name: Name, ...args: Parameters): Promise>> { + if (closed) return Promise.reject(new Error('Dashboard reader closed')); + const key = JSON.stringify([name, args]); + if (!worker) { + const active = worker = new Worker(new URL('./dashboard-reader.js', import.meta.url)); + const fail = (error: Error): void => { + if (worker !== active) return; + for (const request of pending.values()) request.reject(error); + pending.clear(); + inflight.clear(); + worker = null; + }; + active.on('message', ({ id, value, error }) => { + const request = pending.get(id); + pending.delete(id); + if (error !== undefined) request?.reject(new Error(error)); + else request?.resolve(value); + }); + active.once('error', fail); + active.once('exit', code => fail(new Error(`Dashboard reader exited (${code})`))); + } + let result = inflight.get(key); + if (!result) { + const id = ++sequence; + result = new Promise((resolve, reject) => { + pending.set(id, { resolve, reject }); + worker!.postMessage({ id, name, args }); + }).finally(() => inflight.delete(key)); + inflight.set(key, result); + } + return result as Promise>>; + } + return { read, close: () => { closed = true; void worker?.terminate(); } }; +} diff --git a/tools/stack-bench/dashboard/dashboard-reference-runs.ts b/tools/stack-bench/dashboard/dashboard-reference-runs.ts new file mode 100644 index 00000000000..90e7e8c64a9 --- /dev/null +++ b/tools/stack-bench/dashboard/dashboard-reference-runs.ts @@ -0,0 +1,81 @@ +import { execFile } from 'node:child_process'; +import { existsSync, readdirSync, readFileSync } from 'node:fs'; +import { basename, join, resolve } from 'node:path'; +import { promisify } from 'node:util'; +import { redactCredentials } from '../src/evidence/diagnostic-sanitizer.js'; + +const exec = promisify(execFile); +const docker = async (args: string[]) => { + const result = await exec('docker', args, + { timeout: 5000, maxBuffer: 4 * 1024 * 1024, windowsHide: true }); + return result.stdout + (args[0] === 'logs' ? result.stderr : ''); +}; + +export interface ReferenceRun { + id: string; + title: string; + status: 'running' | 'passed' | 'failed' | 'incomplete'; + updatedAt: string; + points: { passed: number; measured: number; planned: number | null } | null; + log: string; +} + +export function referenceLogPoints(log: string): ReferenceRun['points'] { + // A repetition starts a new measurement. Do not add repeated scores together. + const latest = log.split(/qualifying [^\n]+: clean run \d+\/\d+/).at(-1) ?? ''; + const rows = [...latest.matchAll(/^\s+selected-source-\d+ \.\.\. (\d+)\/(\d+)\s*$/gm)]; + if (!rows.length) return null; + const scope = latest.match(/scope: \d+ check\(s\), (\d+) point\(s\)/); + return { passed: rows.reduce((n, row) => n + Number(row[1]), 0), + measured: rows.reduce((n, row) => n + Number(row[2]), 0), planned: scope ? Number(scope[1]) : null }; +} + +export async function referenceRuns(resultsRoot: string, readDocker = docker): Promise<{ runs: ReferenceRun[]; error: string | null }> { + const root = resolve(resultsRoot, 'reference-live'); + const runs = new Map(); + // Finished artifacts remain visible after their controller containers are removed. + if (existsSync(root)) for (const file of readdirSync(root, { withFileTypes: true })) { + if (!file.isFile() || !file.name.endsWith('.json') || file.name.endsWith('.inputs.json')) continue; + try { + const value = JSON.parse(readFileSync(join(root, file.name), 'utf8')); + const artifact = value.payload; + if (value.kind !== 'reference_qualification') continue; + const score = String(artifact.runs?.at(-1)?.score ?? '').match(/^(\d+)\/(\d+)$/); + runs.set(file.name, { id: file.name, title: String(artifact.fixture ?? file.name), + status: artifact.ok === true ? 'passed' : 'failed', + updatedAt: String(value.timestamps?.completedAt ?? value.timestamps?.startedAt ?? ''), + points: score ? { passed: Number(score[1]), measured: Number(score[2]), planned: Number(score[2]) } : null, + log: redactCredentials((artifact.runs ?? []).flatMap((run: { failures?: string[] }) => run.failures ?? []).join('\n')) }); + } catch { /* A qualification artifact may be in the middle of an atomic replacement. */ } + } + try { + const listing = await readDocker(['ps', '-a', '--no-trunc', '--format', '{{json .}}']); + const ids = listing.trim().split('\n').filter(Boolean).map(line => JSON.parse(line)) + .filter(row => String(row.Command).includes('/references/reference-live.js')) + .map(row => String(row.ID)).filter(id => /^[a-f0-9]{64}$/.test(id)); + if (ids.length) { + const containers = JSON.parse(await readDocker(['inspect', ...ids])); + await Promise.all(containers.map(async (container: { Id: string; Args: string[]; + State: { Running: boolean; StartedAt: string; FinishedAt: string } }) => { + const args = container.Args; + const output = args[args.indexOf('--out') + 1]; + // Only this dashboard's reference output tree belongs here. + if (!args.includes('--out') || !output || resolve(output) !== join(root, basename(output))) return; + const id = basename(output), final = runs.get(id); + const level = args[args.indexOf('--level') + 1]; + const stack = args[args.indexOf('--backend') + 1]; + let log: string; + try { log = await readDocker(['logs', '--tail', '4000', container.Id]); } + catch { log = 'Controller log is unavailable.'; } + runs.set(id, { id, title: `${stack} L${level} reference validation`, + status: container.State.Running ? 'running' : final?.status ?? 'incomplete', + updatedAt: container.State.Running ? container.State.StartedAt : container.State.FinishedAt, + points: final?.points ?? referenceLogPoints(log), + log: redactCredentials(log).slice(-96 * 1024) + (final?.log ? `\n${final.log}` : '') }); + })); + } + return { runs: [...runs.values()].sort((a, b) => b.updatedAt.localeCompare(a.updatedAt)), error: null }; + } catch { + return { runs: [...runs.values()], error: 'Live reference status is unavailable. Docker could not be read.' }; + } +} diff --git a/tools/stack-bench/dashboard/dashboard-server.ts b/tools/stack-bench/dashboard/dashboard-server.ts new file mode 100644 index 00000000000..94c637fe946 --- /dev/null +++ b/tools/stack-bench/dashboard/dashboard-server.ts @@ -0,0 +1,664 @@ +#!/usr/bin/env node +import { prepareRun, runSetupCatalog, submitPreparedRun } from '../src/campaigns/run-setup.js'; +import { discoverModels } from '../src/agents/model-discovery.js'; + +import { randomBytes, randomUUID, timingSafeEqual } from 'node:crypto'; +import { appendFileSync, closeSync, createReadStream, existsSync, mkdirSync, openSync, statSync } from 'node:fs'; +import { createServer } from 'node:http'; +import type { IncomingMessage, ServerResponse } from 'node:http'; +import { basename, dirname, join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { spawn } from 'node:child_process'; +import { parseArgs as parseNodeArgs } from 'node:util'; + +import { contained, readCampaignArtifactBody, + readJsonLines, resolveCampaignArtifact, summarizeCampaign, +} from './dashboard-model.js'; +import type { DashboardPlan } from './dashboard-model.js'; +import { createDashboardReader } from './dashboard-reader.js'; +import type { CampaignFilter } from './dashboard-views.js'; +import { watchCampaigns } from './dashboard-events.js'; +import type { CampaignChange, CampaignWatcher } from './dashboard-events.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { referenceRuns } from './dashboard-reference-runs.js'; +import { checkGuidePage } from './check-guide.js'; +import { SAFE_RESULT_NAME, stackBenchResultsRoot } from '../src/runtime/operational-paths.js'; +import { controllerRuntimeCommand, controllerChildEnvironment } from '../appliance/controller.js'; +import { requestCampaignCancellation } from '../src/campaigns/campaign-lock.js'; +import { readCampaignTimeBudget, requestCampaignTimeGrant } from '../src/campaigns/campaign-time-grant.js'; +import { redactCredentials } from '../src/evidence/diagnostic-sanitizer.js'; +import { submitExecutionJob, listExecutionJobs, readExecutionJob, cancelExecutionJob } + from '../src/campaigns/execution-jobs.js'; + +const DASHBOARD_ROOT = dirname(fileURLToPath(import.meta.url)); +const PUBLIC_ROOT = join(DASHBOARD_ROOT, 'public'); +const SPA_PATH = /^\/(?:new|plans|c\/[^/]+(?:\/a\/[^/]+)?)$/; +const HEARTBEAT_MS = 25_000; +const LOOPBACK = new Set(['127.0.0.1', '::1', 'localhost']); +const STATIC = new Map([ + ['/', ['index.html', 'text/html; charset=utf-8']], + ['/app.js', ['app.js', 'text/javascript; charset=utf-8']], + ['/check-guide.js', ['check-guide.js', 'text/javascript; charset=utf-8']], + ['/climb.js', ['climb.js', 'text/javascript; charset=utf-8']], + ['/format.js', ['format.js', 'text/javascript; charset=utf-8']], + ['/progress-chart.js', ['progress-chart.js', 'text/javascript; charset=utf-8']], + ['/graph.js', ['graph.js', 'text/javascript; charset=utf-8']], + ['/metrics.js', ['metrics.js', 'text/javascript; charset=utf-8']], + // Shared with the CLI so a state has one name on every surface. + ['/src/evidence/status-words.js', ['../../src/evidence/status-words.js', 'text/javascript; charset=utf-8']], + ['/views/attempt.js', ['views/attempt.js', 'text/javascript; charset=utf-8']], + ['/views/campaign.js', ['views/campaign.js', 'text/javascript; charset=utf-8']], + ['/views/campaigns.js', ['views/campaigns.js', 'text/javascript; charset=utf-8']], + ['/views/plans.js', ['views/plans.js', 'text/javascript; charset=utf-8']], + ['/views/run-setup.js', ['views/run-setup.js', 'text/javascript; charset=utf-8']], + ['/styles.css', ['styles.css', 'text/css; charset=utf-8']], + ['/spacetimedb-mark.svg', ['spacetimedb-mark.svg', 'image/svg+xml']], + // The brand faces are served from here rather than a CDN: the dashboard's own + // content-security-policy allows 'self' only, and the appliance has no + // outbound access to fetch them at view time. + ['/fonts/inter-latin-variable.woff2', ['fonts/inter-latin-variable.woff2', 'font/woff2']], + ['/fonts/source-code-pro-latin-variable.woff2', ['fonts/source-code-pro-latin-variable.woff2', 'font/woff2']], +]); + +interface DashboardArgs { + host: string; + port: number; + resultsRoot: string; + plansRoot: string; + allowContainerBind: boolean; +} + +export interface DashboardOperation { + id: string; + updatedAt: string; + [key: string]: unknown; +} + +function dashboardOperation(value: unknown): DashboardOperation { + if (!value || typeof value !== 'object') throw new Error('dashboard operation must be an object'); + const id = 'id' in value ? value.id : undefined; + const updatedAt = 'updatedAt' in value ? value.updatedAt : undefined; + if (typeof id !== 'string' || !id) throw new Error('dashboard operation id is required'); + if (typeof updatedAt !== 'string' || !updatedAt) { + throw new Error('dashboard operation updatedAt is required'); + } + return { ...value, id, updatedAt }; +} + +export interface OperationFeed { + readonly path?: string; + append(event: DashboardOperation): void; + list(): DashboardOperation[]; +} + +export interface LaunchChild { + pid?: number; + once(event: 'error', listener: (error: Error) => void): unknown; + once(event: 'exit', listener: (code: number | null, signal: NodeJS.Signals | null) => void): unknown; +} + +export interface LaunchInput { + command: 'resume' | 'work'; + jobId?: string; + plan: DashboardPlan & { path: string }; + output: string; + operationId: string; + resultsRoot: string; + feed: OperationFeed; + env?: NodeJS.ProcessEnv; +} + +export interface DashboardServerOptions { + resultsRoot: string; + plansRoot: string; + allowLaunch?: boolean; + token?: string; + feed?: OperationFeed; + launch?: (input: LaunchInput) => LaunchChild; + plans?: () => DashboardPlan[] | Promise; +} + +function errorMessage(error: unknown): string { + return redactCredentials(error instanceof Error ? error.message : String(error)); +} + +function loopbackHost(value: unknown): boolean { + return /^(?:localhost|127\.0\.0\.1|\[::1\])(?::\d+)?$/i.test(String(value ?? '')); +} + +function sameSecret(actual: unknown, expected: unknown): boolean { + if (typeof actual !== 'string' || typeof expected !== 'string') return false; + const left = Buffer.from(actual); + const right = Buffer.from(expected); + return left.length === right.length && timingSafeEqual(left, right); +} + +function controlAuthorized(request: IncomingMessage, host: string | undefined, + csrfToken: string): boolean { + return request.headers.origin === `http://${host}` + && sameSecret(request.headers['x-stack-bench-token'], csrfToken); +} + +export function parseDashboardArgs(argv: string[], env: NodeJS.ProcessEnv = process.env): DashboardArgs { + const { values } = parseNodeArgs({ args: argv.slice(2), options: { + host: { type: 'string' }, port: { type: 'string' }, results: { type: 'string' }, + 'allow-container-bind': { type: 'boolean' }, + } }); + const args: DashboardArgs = { host: values.host ?? '127.0.0.1', + port: values.port === undefined ? 7331 : Number(values.port), + resultsRoot: stackBenchResultsRoot(STACK_BENCH_ROOT, env), + plansRoot: '', allowContainerBind: values['allow-container-bind'] ?? false }; + if (values.results) args.resultsRoot = resolve(values.results); + args.plansRoot = join(args.resultsRoot, 'plans'); + const applianceContainerBind = args.allowContainerBind + && env.STACK_BENCH_APPLIANCE === '1' && args.host === '0.0.0.0'; + if (!LOOPBACK.has(args.host) && !applianceContainerBind) { + throw new Error('dashboard must bind to localhost or a loopback address'); + } + if (!Number.isInteger(args.port) || args.port < 1 || args.port > 65535) { + throw new Error('dashboard port must be an integer from 1 through 65535'); + } + return args; +} + +function json(response: ServerResponse, status: number, value: unknown): void { + const body = Buffer.from(`${JSON.stringify(value)}\n`); + response.writeHead(status, { 'content-type': 'application/json; charset=utf-8', + 'content-length': body.length, 'cache-control': 'no-store' }); + response.end(body); +} + +function securityHeaders(response: ServerResponse): void { + response.setHeader('content-security-policy', "default-src 'self'; connect-src 'self'; img-src 'self' data:; script-src 'self'; style-src 'self'; frame-ancestors 'none'; base-uri 'none'; form-action 'self'"); + response.setHeader('x-content-type-options', 'nosniff'); + response.setHeader('x-frame-options', 'DENY'); + response.setHeader('referrer-policy', 'no-referrer'); +} + +class RequestBodyError extends Error {} + +async function body(request: IncomingMessage): Promise { + const chunks: Buffer[] = []; + let bytes = 0; + for await (const chunk of request) { + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + bytes += buffer.length; + if (bytes > 16 * 1024) throw new RequestBodyError('request body is too large'); + chunks.push(buffer); + } + try { return JSON.parse(Buffer.concat(chunks).toString('utf8')); + } catch { throw new RequestBodyError('request body must be valid JSON'); } +} + +function createOperationFeed(resultsRoot: string): OperationFeed { + const root = join(resolve(resultsRoot), 'dashboard'); + const path = join(root, 'operations.jsonl'); + mkdirSync(root, { recursive: true }); + return { + path, + append(event: DashboardOperation) { + appendFileSync(path, `${JSON.stringify(event)}\n`, { encoding: 'utf8', mode: 0o600 }); + }, + list() { + const latest = new Map(); + for (const value of readJsonLines(path)) { + const event = dashboardOperation(value); + latest.set(event.id, { ...(latest.get(event.id) ?? {}), ...event }); + } + return [...latest.values()].sort((left, right) => right.updatedAt.localeCompare(left.updatedAt)); + }, + }; +} + +function launchCampaign({ command, jobId, plan, output, operationId, resultsRoot, feed, + env = process.env }: LaunchInput): LaunchChild { + const runtime = controllerRuntimeCommand(command === 'work' + ? ['job', 'work', jobId!, '--results', resultsRoot, '--host', env.STACK_BENCH_HOST_ID ?? 'local'] + : jobId ? ['job', 'resume', jobId, '--results', resultsRoot] + : ['campaign', command, plan.path, '--out', output], env); + feed.append({ id: operationId, updatedAt: new Date().toISOString(), + containerName: runtime.containerName, ownershipLabel: runtime.ownershipLabel }); + const operationsRoot = join(resolve(resultsRoot), 'dashboard', 'operations'); + mkdirSync(operationsRoot, { recursive: true }); + const stdoutPath = join(operationsRoot, `${operationId}.stdout.log`); + const stderrPath = join(operationsRoot, `${operationId}.stderr.log`); + const stdout = openSync(stdoutPath, 'a', 0o600); + const stderr = openSync(stderrPath, 'a', 0o600); + let child; + try { + child = spawn(runtime.executable, runtime.args, { + cwd: STACK_BENCH_ROOT, env: runtime.env, stdio: ['ignore', stdout, stderr], windowsHide: true, + }); + } finally { + closeSync(stdout); + closeSync(stderr); + } + child.once('error', error => feed.append({ schemaVersion: 1, id: operationId, + status: 'failed', updatedAt: new Date().toISOString(), error: errorMessage(error) })); + child.once('exit', (code, signal) => feed.append({ schemaVersion: 1, id: operationId, + status: code === 0 ? 'completed' : 'failed', updatedAt: new Date().toISOString(), + exitCode: code, signal })); + return child; +} + +export function createDashboardServer(options: DashboardServerOptions) { + const resultsRoot = resolve(options.resultsRoot); + const plansRoot = resolve(options.plansRoot); + const allowLaunch = options.allowLaunch ?? process.env.STACK_BENCH_APPLIANCE === '1'; + const token = options.token ?? randomBytes(24).toString('base64url'); + const feed = options.feed ?? createOperationFeed(resultsRoot); + const launch = options.launch ?? launchCampaign; + // Plan compilation runs in the reader worker, which keeps its cache. + const plans = options.plans ?? (() => reader.read('discoverPlans', plansRoot)); + const launchReservations = new Set(); + const dispatchJob = (job: ReturnType) => { + const status = readExecutionJob(resultsRoot, job.id); + const key = `job-${job.id}`; + if (status.status === 'queued' && !launchReservations.has(key)) { + launchReservations.add(key); + const now = new Date().toISOString(); + const operation = { schemaVersion: 1, id: randomUUID(), type: 'campaign.run', status: 'running', + createdAt: now, updatedAt: now, actor: 'local-operator', campaignId: job.key, + campaignSha256: job.planSha256, outputName: key }; + feed.append(operation); + try { + const child = launch({ command: 'work', jobId: job.id, + plan: { id: job.key, title: job.key, state: 'frozen', file: 'plan.json', + path: join(resultsRoot, 'jobs', job.id, 'plan.json') }, + output: status.campaignDirectory, operationId: operation.id, resultsRoot, feed }); + child.once('error', () => launchReservations.delete(key)); + child.once('exit', () => launchReservations.delete(key)); + feed.append({ ...operation, pid: child.pid ?? null }); + } catch (error) { + launchReservations.delete(key); + feed.append({ ...operation, status: 'failed', error: errorMessage(error) }); + throw new Error(`Job ${job.id} is saved but dispatch failed. Retry Start with the same setup. ${errorMessage(error)}`); + } + } + return { ...readExecutionJob(resultsRoot, job.id), campaignKey: key }; + }; + const campaignsRoot = join(resultsRoot, 'campaigns'); + const listeners = new Set(); + const reader = createDashboardReader(); + let watcher: CampaignWatcher | null = null; + let heartbeat: NodeJS.Timeout | null = null; + const broadcast = (change: CampaignChange): void => { + const frame = `event: ${change.type}\ndata: ${JSON.stringify({ key: change.key, + ...(change.attemptId === undefined ? {} : { attemptId: change.attemptId }) })}\n\n`; + for (const listener of listeners) listener.write(frame); + }; + const stopEvents = (): void => { + watcher?.close(); + watcher = null; + if (heartbeat) clearInterval(heartbeat); + heartbeat = null; + }; + const server = createServer(async (request, response) => { + securityHeaders(response); + try { + if (!loopbackHost(request.headers.host)) { + return json(response, 421, { error: 'Dashboard requests must use a loopback host.' }); + } + const url = new URL(request.url ?? '/', `http://${request.headers.host ?? 'localhost'}`); + if (request.method === 'GET' && url.pathname === '/checks') { + const html = checkGuidePage(); + response.writeHead(200, { 'content-type': 'text/html; charset=utf-8', 'cache-control': 'no-store' }); + response.end(html); + return; + } + // The client routes are pages, not fragments: each serves the shell. + const staticFile = STATIC.get(url.pathname) + ?? (SPA_PATH.test(url.pathname) ? STATIC.get('/') : undefined); + if (request.method === 'GET' && staticFile) { + const [file, type] = staticFile; + const path = join(PUBLIC_ROOT, file); + const size = existsSync(path) ? statSync(path).size : 0; + if (!size) return json(response, 404, { error: 'Not found' }); + response.writeHead(200, { 'content-type': type, 'content-length': size, + 'cache-control': file === 'index.html' ? 'no-store' : 'no-cache' }); + createReadStream(path).pipe(response); + return; + } + if (request.method === 'GET' && url.pathname === '/api/run-setup') { + return json(response, 200, runSetupCatalog(resultsRoot)); + } + if (request.method === 'GET' && url.pathname === '/api/run-setup/models') { + if (!allowLaunch) return json(response, 503, { error: 'Model discovery requires the appliance.' }); + if (!sameSecret(request.headers['x-stack-bench-token'], token)) { + return json(response, 403, { error: 'The model request is not authorized.' }); + } + try { + const adapter = url.searchParams.get('adapter') ?? ''; + const profile = url.searchParams.get('profile') ?? undefined; + return json(response, 200, await discoverModels(adapter, profile)); + } catch (error) { return json(response, 400, { error: errorMessage(error) }); } + } + if (request.method === 'POST' && ['/api/runs/prepare', '/api/runs'].includes(url.pathname)) { + if (!allowLaunch) return json(response, 503, { error: 'Run controls require the appliance.' }); + if (!controlAuthorized(request, request.headers.host, token)) { + return json(response, 403, { error: 'The run request is not authorized.' }); + } + if (!String(request.headers['content-type'] ?? '').toLowerCase().startsWith('application/json')) { + return json(response, 415, { error: 'Run requests must use JSON.' }); + } + try { + const input = await body(request); + if (url.pathname.endsWith('/prepare')) return json(response, 200, prepareRun(resultsRoot, input, options.launch ? process.env : controllerChildEnvironment(process.env))); + // Validate dispatch configuration before publishing a job for workers. + if (!options.launch) controllerRuntimeCommand(['job', 'work'], process.env); + const job = submitPreparedRun(resultsRoot, input, options.launch ? process.env : controllerChildEnvironment(process.env)); + return json(response, 202, dispatchJob(job)); + } catch (error) { return json(response, 400, { error: errorMessage(error) }); } + } + if (request.method === 'GET' && url.pathname === '/api/health') { + return json(response, 200, { ok: true, mode: allowLaunch ? 'controller' : 'read-only' }); + } + if (request.method === 'GET' && url.pathname === '/api/overview') { + const page = Number(url.searchParams.get('page') ?? 1); + const filter = url.searchParams.get('filter') ?? 'all'; + if (!Number.isSafeInteger(page) || page < 1 || !['all', 'attention', 'completed', 'ready'].includes(filter)) { + return json(response, 400, { error: 'Use a positive page number and a valid campaign filter.' }); + } + return json(response, 200, { ...await reader.read('overviewPage', campaignsRoot, page, filter as CampaignFilter), + canStart: allowLaunch, csrfToken: token }); + } + if (request.method === 'GET' && url.pathname === '/api/reference-runs') { + return json(response, 200, await referenceRuns(resultsRoot)); + } + if (request.method === 'GET' && url.pathname === '/api/session') { + return json(response, 200, { canStart: allowLaunch, csrfToken: token }); + } + if (request.method === 'GET' && url.pathname === '/api/plans') { + return json(response, 200, await plans()); + } + const jobRoute = url.pathname.match(/^\/api\/jobs(?:\/([a-f0-9]{64})(?:\/(cancel|start))?)?$/); + if (jobRoute) { + const id = jobRoute[1], cancel = jobRoute[2] !== undefined; + if (request.method === 'GET' && !cancel) { + if (!id) { + const after = url.searchParams.get('after') ?? ''; + const limit = Number(url.searchParams.get('limit') ?? 50); + if ((after && !/^[a-f0-9]{64}$/.test(after)) || !Number.isSafeInteger(limit) || limit < 1 || limit > 200) { + return json(response, 400, { error: 'Use a valid job cursor and a page size from 1 through 200.' }); + } + return json(response, 200, listExecutionJobs(resultsRoot, { after, limit })); + } + if (!existsSync(join(resultsRoot, 'jobs', id, 'job.json'))) return json(response, 404, { error: 'Job not found.' }); + return json(response, 200, readExecutionJob(resultsRoot, id)); + } + if (request.method === 'POST' && (!id || cancel)) { + if (!allowLaunch) return json(response, 503, { error: 'Run controls are available inside the Stack Bench appliance.' }); + if (!controlAuthorized(request, request.headers.host, token)) { + return json(response, 403, { error: 'The job request is not authorized.' }); + } + if (id) { + if (!existsSync(join(resultsRoot, 'jobs', id, 'job.json'))) return json(response, 404, { error: 'Job not found.' }); + if (jobRoute[2] === 'start') { + if (!options.launch) controllerRuntimeCommand(['job', 'work'], process.env); + return json(response, 202, dispatchJob(readExecutionJob(resultsRoot, id).job)); + } + cancelExecutionJob(resultsRoot, id); + return json(response, 202, readExecutionJob(resultsRoot, id)); + } + if (!String(request.headers['content-type'] ?? '').toLowerCase().startsWith('application/json')) { + return json(response, 415, { error: 'Job submissions must use JSON.' }); + } + try { + const job = submitExecutionJob(resultsRoot, await body(request)); + return json(response, 202, readExecutionJob(resultsRoot, job.id)); + } catch (error) { return json(response, 400, { error: errorMessage(error) }); } + } + } + if (request.method === 'GET' && url.pathname === '/api/events') { + response.writeHead(200, { 'content-type': 'text/event-stream; charset=utf-8', + 'cache-control': 'no-store', connection: 'keep-alive' }); + response.write(': open\n\n'); + listeners.add(response); + watcher ??= watchCampaigns(campaignsRoot, broadcast, + mode => console.log(`Stack Bench dashboard: campaign watcher ${mode}`)); + // A silent connection is dropped by proxies long before a campaign + // writes anything. + heartbeat ??= setInterval(() => { + for (const listener of listeners) listener.write('event: reference\ndata: {}\n\n'); + }, HEARTBEAT_MS).unref(); + request.once('close', () => { + listeners.delete(response); + if (!listeners.size) stopEvents(); + }); + return; + } + const timeRoute = url.pathname.match(/^\/api\/campaigns\/([^/]+)\/attempts\/([^/]+)\/time$/); + if (timeRoute && (request.method === 'GET' || request.method === 'POST')) { + const key = decodeURIComponent(timeRoute[1] ?? ''); + if (!SAFE_RESULT_NAME.test(key)) return json(response, 400, { error: 'The campaign name is invalid.' }); + const directory = contained(campaignsRoot, key, 'campaign'); + const attemptId = decodeURIComponent(timeRoute[2] ?? ''); + if (request.method === 'GET') { + if (!existsSync(directory)) return json(response, 404, { error: 'Not found' }); + try { return json(response, 200, readCampaignTimeBudget(directory, attemptId)); } + catch (error) { + if (error instanceof Error && error.message === `unknown attempt ${attemptId}`) { + return json(response, 404, { error: 'Not found' }); + } + throw error; + } + } + if (!allowLaunch) return json(response, 503, { error: 'Run controls are available inside the Stack Bench appliance.' }); + if (!controlAuthorized(request, request.headers.host, token)) { + return json(response, 403, { error: 'The time request is not authorized.' }); + } + const input = await body(request) as { minutes?: unknown; grantId?: unknown } | null; + if (!input || typeof input.minutes !== 'number' || !Number.isSafeInteger(input.minutes * 60_000) + || !Number.isInteger(input.minutes) || input.minutes <= 0 || typeof input.grantId !== 'string') { + return json(response, 400, { error: 'Positive whole minutes and a grant ID are required.' }); + } + if (!existsSync(directory)) return json(response, 404, { error: 'Not found' }); + try { + const receipt = requestCampaignTimeGrant(directory, { + attemptId, grantId: input.grantId, minutes: input.minutes, + }); + return json(response, receipt.disposition === 'rejected' ? 409 : 202, + receipt.disposition === 'rejected' ? { ...receipt, error: receipt.reason } : receipt); + } catch (error) { + return json(response, 409, { error: errorMessage(error) }); + } + } + const stopRoute = url.pathname.match(/^\/api\/campaigns\/([^/]+)\/stop$/); + if (request.method === 'POST' && stopRoute) { + if (!allowLaunch) return json(response, 503, { error: 'Run controls are available inside the Stack Bench appliance.' }); + if (!controlAuthorized(request, request.headers.host, token)) { + return json(response, 403, { error: 'The stop request is not authorized.' }); + } + const key = decodeURIComponent(stopRoute[1] ?? ''); + if (!SAFE_RESULT_NAME.test(key)) return json(response, 400, { error: 'The campaign name is invalid.' }); + const input = await body(request); + const owner = input && typeof input === 'object' && 'owner' in input ? input.owner : null; + if (typeof owner !== 'string' || !/^[a-f0-9]{64}$/.test(owner)) { + return json(response, 400, { error: 'The current controller identity is required.' }); + } + const directory = contained(campaignsRoot, key, 'campaign'); + if (!existsSync(directory)) return json(response, 404, { error: 'Not found' }); + const campaign = summarizeCampaign(directory, { includeAttempts: false }); + if (!requestCampaignCancellation(directory, + { id: campaign.id, contentSha256: campaign.sha256 }, owner)) { + return json(response, 409, { error: 'The controller changed or stopped. Refresh the campaign.' }); + } + const now = new Date().toISOString(); + const operation = { id: randomUUID(), type: 'campaign.stop', status: 'requested', + updatedAt: now, campaignId: campaign.id, outputName: key, owner }; + feed.append(operation); + return json(response, 202, operation); + } + const resumeRoute = url.pathname.match(/^\/api\/campaigns\/([^/]+)\/resume$/); + if (request.method === 'POST' && resumeRoute) { + if (!allowLaunch) return json(response, 503, { error: 'Run controls are available inside the Stack Bench appliance.' }); + if (!controlAuthorized(request, request.headers.host, token)) { + return json(response, 403, { error: 'The run request is not authorized.' }); + } + const key = decodeURIComponent(resumeRoute[1] ?? ''); + if (!SAFE_RESULT_NAME.test(key)) return json(response, 400, { error: 'The campaign name is invalid.' }); + const directory = contained(campaignsRoot, key, 'campaign'); + if (!existsSync(directory)) return json(response, 404, { error: 'Not found' }); + const campaign = summarizeCampaign(directory, { includeAttempts: false }); + const priorExecutions = campaign.summary?.executions ?? 0; + if (campaign.mode !== 'dependency' || campaign.status !== 'prepared' || priorExecutions < 1) { + return json(response, 409, { error: 'Only an interrupted campaign that is ready can resume.' }); + } + const plan = (await plans()).find(item => item.id === campaign.id && item.sha256 === campaign.sha256); + // A draft campaign can only exist as a model-free trial, which `campaign resume` continues. + if (!plan || !['frozen', 'draft'].includes(plan.state)) { + return json(response, 409, { error: 'The test plan used by this campaign is unavailable.' }); + } + // A job's campaign resumes with the accounts and capacity policy the job saved. + const jobId = /^job-([a-f0-9]{64})$/.exec(key)?.[1]; + const savedJob = jobId && existsSync(join(resultsRoot, 'jobs', jobId, 'job.json')) ? jobId : undefined; + const reservation = `${campaign.id}:${campaign.sha256}:${key}`; + if (launchReservations.has(reservation)) { + return json(response, 409, { error: 'This campaign already has an active controller.' }); + } + launchReservations.add(reservation); + const now = new Date().toISOString(); + const operation = { schemaVersion: 1, id: randomUUID(), type: 'campaign.resume', + status: 'running', createdAt: now, updatedAt: now, actor: 'local-operator', + campaignId: campaign.id, campaignSha256: campaign.sha256, outputName: key }; + feed.append(operation); + const output = join(resultsRoot, 'campaigns', key); + try { + const child = launch({ command: 'resume', ...(savedJob ? { jobId: savedJob } : {}), + plan: { ...plan, path: join(plansRoot, plan.file) }, output, + operationId: operation.id, resultsRoot, feed, env: process.env }); + if (typeof child?.once === 'function') { + child.once('error', () => launchReservations.delete(reservation)); + child.once('exit', () => launchReservations.delete(reservation)); + } else { + launchReservations.delete(reservation); + } + feed.append({ ...operation, pid: child?.pid ?? null }); + } catch (error) { + launchReservations.delete(reservation); + feed.append({ schemaVersion: 1, id: operation.id, status: 'failed', + updatedAt: new Date().toISOString(), error: errorMessage(error) }); + throw error; + } + return json(response, 202, operation); + } + const artifactRoute = url.pathname.match(/^\/api\/campaigns\/([^/]+)\/artifacts\/([^/]+)$/); + if (request.method === 'GET' && artifactRoute) { + let artifact; + try { + artifact = resolveCampaignArtifact(resultsRoot, decodeURIComponent(artifactRoute[1] ?? ''), + decodeURIComponent(artifactRoute[2] ?? '')); + } catch { + return json(response, 404, { error: 'Campaign artifact not found.' }); + } + const body = readCampaignArtifactBody(artifact); + const download = url.searchParams.get('download') === '1'; + const type = artifact.kind === 'visual' ? artifact.contentType + : artifact.kind === 'report' && !download ? 'text/html; charset=utf-8' + : 'text/plain; charset=utf-8'; + if (artifact.kind === 'report' && !download) { + response.setHeader('content-security-policy', "sandbox; default-src 'none'; style-src 'unsafe-inline'; img-src data:"); + } + response.writeHead(200, { 'content-type': type, 'content-length': body.length, + 'cache-control': 'no-store', 'content-disposition': `${download ? 'attachment' : 'inline'}; filename*=UTF-8''${encodeURIComponent(basename(artifact.path))}` }); + response.end(body); + return; + } + const campaignRoute = url.pathname.match(/^\/api\/campaigns\/([^/]+)(?:\/(.*))?$/); + if (request.method === 'GET' && campaignRoute) { + const key = decodeURIComponent(campaignRoute[1] ?? ''); + const rest = campaignRoute[2] ?? ''; + if (!SAFE_RESULT_NAME.test(key)) { + return json(response, 400, { error: 'The campaign name is invalid.' }); + } + if (!rest && /^job-[a-f0-9]{64}$/.test(key) + && !existsSync(join(campaignsRoot, key, 'state.json')) + && existsSync(join(resultsRoot, 'jobs', key.slice(4), 'job.json'))) { + const pendingJob = readExecutionJob(resultsRoot, key.slice(4)); + const operation = feed.list().find(op => op.outputName === key); + return json(response, 200, { pendingJob, dispatchError: operation?.status === 'failed' ? operation.error ?? 'Worker exited before campaign startup.' : null }); + } + if (!existsSync(contained(campaignsRoot, key, 'campaign'))) { + return json(response, 404, { error: 'Not found' }); + } + const attemptRoute = rest.match(/^attempts\/([^/]+)\/(checks|package|log|transcript)$/); + const attemptId = attemptRoute ? decodeURIComponent(attemptRoute[1] ?? '') : ''; + if (attemptRoute && !SAFE_RESULT_NAME.test(attemptId)) { + return json(response, 400, { error: 'The attempt name is invalid.' }); + } + const from = url.searchParams.get('from') ?? '0'; + if (attemptRoute?.[2] === 'log' && (!/^\d+$/.test(from) || !Number.isSafeInteger(Number(from)))) { + return json(response, 400, { error: 'The log offset must be a whole number of bytes.' }); + } + try { + if (!rest) return json(response, 200, await reader.read('campaignLiveSheet', resultsRoot, key)); + if (rest === 'live') return json(response, 200, await reader.read('campaignLiveUpdate', resultsRoot, key)); + if (rest === 'progression') { + const progression = await reader.read('campaignLiveProgression', resultsRoot, key); + return progression + ? json(response, 200, progression) + : json(response, 404, { error: 'Progression is recorded for dependency campaigns only.' }); + } + if (attemptRoute?.[2] === 'transcript') { + const before = url.searchParams.get('before'); + if (before !== null && (!/^\d+$/.test(before) || !Number.isSafeInteger(Number(before)))) { + return json(response, 400, { error: 'Invalid transcript offset' }); + } + return json(response, 200, await reader.read('attemptTranscript', resultsRoot, key, attemptId, + url.searchParams.get('session') ?? '', before === null ? undefined : Number(before))); + } + if (attemptRoute?.[2] === 'checks') { + return json(response, 200, await reader.read('attemptChecks', resultsRoot, key, attemptId)); + } + if (attemptRoute?.[2] === 'package') { + return json(response, 200, await reader.read('attemptPackage', resultsRoot, key, attemptId)); + } + if (attemptRoute) { + const slice = await reader.read('attemptLogSlice', resultsRoot, key, attemptId, Number(from)); + const text = Buffer.from(slice.text); + response.writeHead(200, { 'content-type': 'text/plain; charset=utf-8', + 'content-length': text.length, 'cache-control': 'no-store', + 'x-stack-bench-log-offset': String(slice.offset) }); + response.end(text); + return; + } + } catch (error) { + if (error instanceof Error && error.message === 'campaign attempt does not exist') { + return json(response, 404, { error: 'Not found' }); + } + return json(response, 422, { error: `Cannot read campaign evidence: ${errorMessage(error)}` }); + } + return json(response, 404, { error: 'Not found' }); + } + return json(response, 404, { error: 'Not found' }); + } catch (error) { + const clientError = error instanceof RequestBodyError || error instanceof URIError; + return json(response, clientError ? 400 : 500, { error: errorMessage(error) }); + } + }); + // An open event stream is not an idle connection: the watchers stop and the + // streams end as the server closes, not once it has. + const closeServer = server.close.bind(server); + server.close = ((callback?: (error?: Error) => void) => { + reader.close(); + stopEvents(); + for (const listener of listeners) listener.end(); + listeners.clear(); + return closeServer(callback); + }) as typeof server.close; + return { server, token, allowLaunch }; +} + +async function main() { + const args = parseDashboardArgs(process.argv); + const { server, allowLaunch } = createDashboardServer(args); + await new Promise((resolveListen, reject) => { + server.once('error', reject); + server.listen(args.port, args.host, resolveListen); + }); + console.log(`Stack Bench dashboard: http://${args.host}:${args.port} (${allowLaunch ? 'controller' : 'read-only'})`); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + main().catch(error => { console.error(`stack-bench-dashboard: ${errorMessage(error)}`); process.exitCode = 2; }); +} diff --git a/tools/stack-bench/dashboard/dashboard-transcript.ts b/tools/stack-bench/dashboard/dashboard-transcript.ts new file mode 100644 index 00000000000..51a8459a332 --- /dev/null +++ b/tools/stack-bench/dashboard/dashboard-transcript.ts @@ -0,0 +1,180 @@ +import { sha256 } from '../src/evidence/provenance.js'; +import { loadTrack, workDirFor } from '../src/composition/tracks.js'; +import { execFile } from 'node:child_process'; +import { open, readdir, realpath, stat } from 'node:fs/promises'; +import { existsSync, realpathSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, relative, resolve, sep } from 'node:path'; +import { CODING_PROVIDERS } from '../container/coding-providers.js'; +import { CONTAINER_CLAUDE_TRANSCRIPT_READ } from '../container/claude-transcript-reader.js'; +import { AGENT_ADAPTER_REGISTRY } from '../src/agents/agent-adapters.js'; +import type { PublicBackendLease } from '../src/runtime/backend-lease.js'; +import { publicBackendLease, readBackendLease } from '../src/runtime/backend-lease.js'; +import { readArtifactPayload } from '../src/evidence/artifacts.js'; +import { codingContainerAgentExecOptions } from '../src/runtime/coding-container-policy.js'; +import { inspectBuildContainer } from '../src/stacks/hosted-lifecycle.js'; +import { redactCredentials } from '../src/evidence/diagnostic-sanitizer.js'; + +export interface TranscriptMessage { id: string; role: string; text: string; tool: boolean } +export interface TranscriptPage { + sessions: Array<{ id: string; label: string }>; + session: string; + before: number | null; + messages: TranscriptMessage[]; + skipped: number; +} +const record = (value: unknown): value is Record => + !!value && typeof value === 'object' && !Array.isArray(value); + +export function transcriptMessages(text: string): { messages: TranscriptMessage[]; skipped: number } { + const messages: TranscriptMessage[] = []; + let skipped = 0; + let eventId = ''; + let blockIndex = 0; + const add = (role: string, value: unknown, tool = false) => { + if (typeof value === 'string' && value.trim()) messages.push({ id: `${eventId}-${blockIndex++}`, role, + text: redactCredentials(value), tool }); + }; + for (const line of text.split('\n').filter(line => line.trim())) { + eventId = sha256(line); + blockIndex = 0; + let event: unknown; + try { event = JSON.parse(line); } catch { skipped++; continue; } + if (!record(event)) continue; + const message = record(event.message) ? event.message + : event.type === 'response_item' && record(event.payload) ? event.payload : null; + if (message) { + const role = String(message.role ?? event.type ?? 'Agent'); + if (typeof message.content === 'string') add(role, message.content); + if (Array.isArray(message.content)) for (const block of message.content) { + if (!record(block)) continue; + if (['text', 'input_text', 'output_text'].includes(String(block.type))) add(role, block.text); + if (block.type === 'tool_use') add(String(block.name ?? 'Tool'), JSON.stringify(block.input, null, 2), true); + if (block.type === 'tool_result') add('Tool result', typeof block.content === 'string' + ? block.content : JSON.stringify(block.content, null, 2), true); + } + if (message.type === 'function_call') add(String(message.name ?? 'Tool'), message.arguments, true); + if (message.type === 'function_call_output') add('Tool result', message.output, true); + } + if (event.type === 'item.completed' && record(event.item)) { + const item = event.item; + if (item.type === 'agent_message') add('assistant', item.text); + if (item.type === 'command_execution') add('Command', `${item.command ?? ''}\n${item.aggregated_output ?? ''}`, true); + if (item.type === 'file_change') add('File changes', JSON.stringify(item.changes, null, 2), true); + } + } + return { messages, skipped }; +} + +export interface TranscriptFile { + id: string; + label: string; + size: number; + modified: number; + read(start: number, count: number): Promise; +} +const pendingReads = new Map>(); +function dockerRead(args: string[]): Promise { + const key = JSON.stringify(args); + const pending = pendingReads.get(key); + if (pending) return pending; + const result = new Promise((resolve, reject) => { + execFile('docker', args, { timeout: 5_000, maxBuffer: 2 * 1024 * 1024, encoding: 'buffer' }, + (error, stdout) => error ? reject(error) : resolve(stdout)); + }).finally(() => pendingReads.delete(key)); + pendingReads.set(key, result); + return result; +} + +export function transcriptLease(directory: string, runtimeRoot = process.env.STACK_BENCH_RUNTIME_DIR + ?? join(tmpdir(), 'stack-bench-runtime')): PublicBackendLease | null { + const evidence = join(directory, 'backend-lease.json'); + if (existsSync(evidence)) return readArtifactPayload(evidence, + { expectedKind: 'backend_lease_evidence' }); + const runPath = join(directory, 'run.json'); + if (!existsSync(runPath)) return null; + const initial = readArtifactPayload<{ backendLease?: PublicBackendLease }>(runPath, + { expectedKind: 'benchmark_run' }).backendLease; + if (!initial) return null; + if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/.test(initial.runId)) throw new Error('Invalid transcript run identity'); + const runtime = resolve(runtimeRoot, initial.runId); + const path = join(runtime, 'backend-lease.json'); + if (!existsSync(path)) return null; + if (dirname(realpathSync(runtime)) !== realpathSync(resolve(runtimeRoot)) + || dirname(realpathSync(path)) !== realpathSync(runtime)) throw new Error('Transcript lease is outside runtime directory'); + const current = publicBackendLease(readBackendLease(path, { runId: initial.runId, backend: initial.backend, active: true })); + if (current.ownership.markerSha256 !== initial.ownership?.markerSha256 + || current.track !== initial.track || current.runIndex !== initial.runIndex) { + throw new Error('Transcript lease ownership changed'); + } + return current; +} + +// Reads only this attempt's transcript mounts. Never scans another account's sessions. +export async function attemptTranscriptFiles(executions: Array<{ directory: string; label: string }>, + adapterId: string): Promise { + const provider = AGENT_ADAPTER_REGISTRY.get(adapterId).provider; + if (!provider || !(provider in CODING_PROVIDERS)) return []; + const config = CODING_PROVIDERS[provider as keyof typeof CODING_PROVIDERS]; + const files: TranscriptFile[] = []; + for (const execution of executions) { + const lease = transcriptLease(execution.directory); + if (!lease) continue; + const root = config.projects(join(workDirFor(loadTrack(lease.track), lease.backend, lease.runIndex, lease.runId), 'app')); + let remote: ((name: string, start: number, count: number) => Promise) | null = null; + if (lease.state === 'active' && lease.resources.buildContainer?.owned) { + const actual = await dockerRead(['inspect', '--format', '{{.Id}}', lease.resources.buildContainer.name]); + const container = inspectBuildContainer(lease, () => actual.toString()); + remote = (name, start, count) => dockerRead(['exec', ...codingContainerAgentExecOptions(), + container.id, 'node', '-e', CONTAINER_CLAUDE_TRANSCRIPT_READ, + config.containerTranscripts, name, String(start), String(count)]); + } + let entries: Array<[string, number, number]>; + if (remote) entries = JSON.parse((await remote('', 0, 0)).toString()) as Array<[string, number, number]>; + else { + if (!existsSync(root)) continue; + const resolvedRoot = await realpath(root); + entries = []; + for (const entry of await readdir(root, { recursive: true, withFileTypes: true })) { + if (!entry.isFile() || !entry.name.endsWith('.jsonl')) continue; + const path = join(entry.parentPath, entry.name); + if (!(await realpath(path)).startsWith(resolvedRoot + sep)) continue; + const info = await stat(path); + entries.push([relative(root, path), info.size, info.mtimeMs]); + } + } + for (const [name, size, modified] of entries) { + const reader = remote; + files.push({ id: Buffer.from(`${execution.label}/${name}`).toString('base64url'), + label: `${execution.label} / ${new Date(modified).toISOString().replace('T', ' ').slice(0, 16)} UTC`, size, modified, + read: async (start, count) => { + if (!Number.isSafeInteger(start) || start < 0 || !Number.isSafeInteger(count) + || count < 0 || count > 256 * 1024) throw new Error('Invalid transcript range'); + if (reader) return reader(name, start, count); + const path = join(root, name); + if (!(await realpath(path)).startsWith(await realpath(root) + sep)) { + throw new Error('transcript is outside the attempt directory'); + } + const fd = await open(path, 'r'), buffer = Buffer.alloc(count); + try { return buffer.subarray(0, (await fd.read(buffer, 0, count, start)).bytesRead); } + finally { await fd.close(); } + } }); + } + } + return files.sort((a, b) => a.modified - b.modified); +} +export async function readAttemptTranscript(executions: Array<{ directory: string; label: string }>, + adapterId: string, session = '', before?: number): Promise { + const files = await attemptTranscriptFiles(executions, adapterId); + const file = (session ? files.find(file => file.id === session) : files.at(-1)); + if (session && !file) throw new Error('Transcript session not found'); + if (!file) return { sessions: [], session: '', before: null, messages: [], skipped: 0 }; + const end = Math.min(before ?? file.size, file.size); + const start = Math.max(0, end - 256 * 1024); + const bytes = await file.read(start, end - start); + const first = start ? bytes.indexOf(10) + 1 : 0; + const last = bytes.lastIndexOf(10); + const content = last >= first ? bytes.subarray(first, last + 1).toString('utf8') : ''; + return { sessions: files.map(({ id, label }) => ({ id, label })), session: file.id, + before: start > 0 ? start + first : null, ...transcriptMessages(content) }; +} diff --git a/tools/stack-bench/dashboard/dashboard-views.ts b/tools/stack-bench/dashboard/dashboard-views.ts new file mode 100644 index 00000000000..e02a9d3ee48 --- /dev/null +++ b/tools/stack-bench/dashboard/dashboard-views.ts @@ -0,0 +1,1094 @@ +import { readAttemptTranscript } from './dashboard-transcript.js'; +import { liveCostTotal, liveTranscriptCost } from './dashboard-live-cost.js'; +import { canonicalDefinitionJson } from '../src/composition/definition-plan.js'; +import { sha256 } from '../src/evidence/provenance.js'; +import { closeSync, existsSync, fstatSync, openSync, readFileSync, readSync, readdirSync, statSync } + from 'node:fs'; +import { basename, join, resolve } from 'node:path'; + +import type { CampaignAttemptState } from '../src/campaigns/campaign-scheduler.js'; +import type { DependencyPromptSelection, DependencyState } + from '../src/progression/dependency-mode.js'; +import type { ProgressionState } from '../src/progression/progression-state.js'; +import type { CompiledCampaignPlan } from '../src/campaigns/campaign-compiler.js'; +import type { DependencyProgress } from '../src/campaigns/campaign-inspection.js'; +import type { GradeBundlePayload } from '../src/evidence/benchmark-run.js'; +import { executionSpend } from '../src/campaigns/campaign-report.js'; +import type { CostEvidence } from '../src/evidence/cost-proof.js'; +import type { RunCheckpoint } from '../src/evidence/run-checkpoints.js'; +import { scoreDependencyState, dependencyCompletionBreakdown, type DependencyCompletionBreakdown } from '../src/progression/dependency-score.js'; +import type { CheckCompletion } from '../src/evidence/check-completion.js'; +import { ARTIFACT_FILE, readArtifact, readArtifactPayload } from '../src/evidence/artifacts.js'; +import { CAMPAIGN_FILE } from '../src/campaigns/campaign-path.js'; +import { campaignFacts, inspectCampaignAttempt } from '../src/campaigns/campaign-inspection.js'; +import { campaignLockIsActive, readCampaignLock } from '../src/campaigns/campaign-lock.js'; +import { campaignProgressionOwner } from '../src/campaigns/campaign-compiler.js'; +import { compileProgressionInput, dependencyRuntimeDefinition } + from '../src/progression/progression-definition.js'; +import { progressionEngine } from '../src/progression/progression-engine.js'; +import { readCampaignState } from '../src/campaigns/campaign-scheduler.js'; +import { readProgressionState } from '../src/progression/progression-state.js'; +import { redactCredentials } from '../src/evidence/diagnostic-sanitizer.js'; +import { CHECK_EVIDENCE_STATUSES, evidenceDisposition, type CheckEvidence } from '../src/evidence/check-evidence.js'; +import { renderFinding } from '../src/actions/action-findings.js'; +import { repairBudgetLimit, type RepairBudget } from '../src/progression/repair-plan.js'; +import { MAX_LOG_BYTES, contained, parseRunProgress, readTextTail, attemptPause, + walkPublicExecutionArtifacts, calibrationResolver } from './dashboard-model.js'; +import type { DashboardArtifact } from './dashboard-model.js'; +import { attemptExcluded, attemptMetrics, attemptStalling, compareCampaign, median } + from './public/metrics.js'; + +const CAMPAIGN_KEY = /^[a-z0-9][a-z0-9.-]*$/; +const GRADE_DIRECTORY = /^(?:first-build-l(\d+)-grading|l(\d+)-fix(\d+)-grading|grading)$/i; +const PROGRESSION_ATTEMPT = /^attempt-(\d+)$/i; +const LOG_FILE = 'process.stdout.log'; + +type ControllerActive = (directory: string, campaign: CompiledCampaignPlan) => boolean; +type InspectedAttempt = ReturnType; + +interface ViewOptions { + controllerActive?: ControllerActive; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function percentage(value: number | null | undefined): number | null { + return value == null ? null : Math.round(value * 1000) / 10; +} + +function campaignDirectory(resultsRoot: string, key: string): string { + if (!CAMPAIGN_KEY.test(key)) throw new Error('campaign key is invalid'); + return contained(join(resolve(resultsRoot), 'campaigns'), key, 'campaign'); +} + +function fileFingerprint(path: string): string | null { + if (!existsSync(path)) return null; + const stat = statSync(path); + return `${stat.size}:${stat.mtimeMs}`; +} + +const campaignStateCache = new Map; +}>(); + +// Share frozen plan/state validation across dashboard resources. Execution +// evidence and controller liveness keep their own freshness checks. +function dashboardCampaignState(directory: string): ReturnType { + const fingerprint = [CAMPAIGN_FILE.plan, CAMPAIGN_FILE.state] + .map(file => fileFingerprint(join(directory, file))).join('|'); + const cached = campaignStateCache.get(directory); + if (cached?.fingerprint === fingerprint) return cached.value; + const value = readCampaignState(directory, { requireCurrentInputs: false }); + campaignStateCache.set(directory, { fingerprint, value }); + return value; +} + +// Every file whose change can move a number in the view, and nothing else: a +// running campaign that has written nothing since the last read is unchanged. +function executionFingerprints(directory: string, files: readonly string[]): string[] { + const attemptsRoot = join(directory, 'attempts'); + if (!existsSync(attemptsRoot)) return []; + const parts: string[] = []; + for (const attempt of readdirSync(attemptsRoot, { withFileTypes: true })) { + if (!attempt.isDirectory()) continue; + const attemptDirectory = join(attemptsRoot, attempt.name); + for (const execution of readdirSync(attemptDirectory, { withFileTypes: true })) { + if (!execution.isDirectory()) continue; + for (const file of files) { + const stamp = fileFingerprint(join(attemptDirectory, execution.name, file)); + if (stamp) parts.push(`${attempt.name}/${execution.name}/${file}:${stamp}`); + } + } + } + return parts.sort(); +} + +function campaignFingerprint(directory: string, files: readonly string[], + executionFiles: readonly string[]): string { + return [...files.map(file => `${file}:${fileFingerprint(join(directory, file)) ?? 'missing'}`), + ...executionFingerprints(directory, executionFiles)].join('|'); +} + +// Overview + +export interface OverviewCampaign { + key: string; + id: string; + title: string; + status: string; + mode: string; + levels: number[]; + repetitions: number; + updatedAt: string | null; + // The mode's official score per stack, as a percentage; null until a stack + // has a comparable result. + scores: Record; + attempts: { total: number; running: number; completed: number }; +} + +export interface UnreadableOverviewCampaign { + key: string; + id: string; + title: string; + status: 'unreadable'; + error: string; +} + +export type OverviewEntry = OverviewCampaign | UnreadableOverviewCampaign; + +const overviewCache = new Map(); + +function overviewCampaign(directory: string): { + plan: CompiledCampaignPlan; + campaign: OverviewCampaign; +} { + const { plan, state } = dashboardCampaignState(directory); + // Only completed attempts can contribute to these scores. Live details and + // excluded evidence are read when their campaign is opened. + const attempts = state.attempts.filter(attempt => attempt.status === 'completed').map(attempt => + inspectCampaignAttempt(plan, attempt, directory)); + const comparison = compareCampaign({ attempts }); + const scores = Object.fromEntries(plan.stacks.map(stack => + [stack.id, percentage(comparison.rows.find(row => row.stack === stack.id)?.final ?? null)])); + return { + plan, + campaign: { + key: basename(resolve(directory)), + id: plan.id, + title: plan.title, + status: state.status, + mode: plan.definition.mode?.id ?? 'sequential', + levels: plan.definition.levels, + repetitions: plan.definition.repetitions, + // Current qualification is checked on the campaign sheet. Listing old + // runs must not recompile today's qualification for every history row. + updatedAt: state.updatedAt, + scores, + attempts: { total: state.summary.total, running: state.summary.running, + completed: state.summary.completed }, + }, + }; +} + +// Liveness is one more fact about a running campaign, not a condition of +// reading it: the read-only host view has no Docker socket to ask. +function controllerInterrupted(probe: ControllerActive, directory: string, + plan: CompiledCampaignPlan, status: string): boolean { + if (status !== 'running') return false; + try { return !probe(directory, plan); } catch { return false; } +} + +function withInterruption(campaign: OverviewCampaign, interrupted: boolean): OverviewCampaign { + if (!interrupted) return campaign; + return { ...campaign, status: 'attention-required', + attempts: { ...campaign.attempts, running: 0 } }; +} + +// Summaries only: no attempt list, no log, no plan. The fingerprint covers a +// running campaign too, so a poll that finds nothing changed costs one stat +// per evidence file instead of a full replay. +export function overviewSummary(campaignsRoot: string, + { controllerActive = campaignLockIsActive, keys }: ViewOptions & { keys?: readonly string[] } = {}): OverviewEntry[] { + if (!existsSync(campaignsRoot)) return []; + const campaigns: OverviewEntry[] = []; + for (const entry of readdirSync(campaignsRoot, { withFileTypes: true })) { + if (!entry.isDirectory()) continue; + if (keys && !keys.includes(entry.name)) continue; + const directory = join(campaignsRoot, entry.name); + if (!existsSync(join(directory, CAMPAIGN_FILE.state)) + || !existsSync(join(directory, CAMPAIGN_FILE.plan))) continue; + try { + const fingerprint = campaignFingerprint(directory, + [CAMPAIGN_FILE.plan, CAMPAIGN_FILE.state], [ARTIFACT_FILE.run]); + const cached = overviewCache.get(directory); + const fresh = cached?.fingerprint === fingerprint + ? { plan: cached.plan, campaign: cached.campaign } : overviewCampaign(directory); + overviewCache.set(directory, { fingerprint, ...fresh }); + campaigns.push(withInterruption(fresh.campaign, controllerInterrupted(controllerActive, + directory, fresh.plan, fresh.campaign.status))); + } catch (error) { + campaigns.push({ key: entry.name, id: entry.name, title: entry.name, + status: 'unreadable', error: errorMessage(error) }); + } + } + return campaigns.sort((left, right) => + String('updatedAt' in right ? right.updatedAt ?? '' : '') + .localeCompare(String('updatedAt' in left ? left.updatedAt ?? '' : ''))); +} + +export type CampaignFilter = 'all' | 'attention' | 'completed' | 'ready'; + +export function overviewPage(campaignsRoot: string, requestedPage = 1, + filter: CampaignFilter = 'all', options: ViewOptions = {}) { + const index: Array<{ key: string; status: string; updatedAt: string }> = []; + for (const entry of existsSync(campaignsRoot) ? readdirSync(campaignsRoot, { withFileTypes: true }) : []) { + if (!entry.isDirectory()) continue; + const directory = join(campaignsRoot, entry.name); + if (!existsSync(join(directory, CAMPAIGN_FILE.plan)) || !existsSync(join(directory, CAMPAIGN_FILE.state))) continue; + try { + const { plan, state } = dashboardCampaignState(directory); + const interrupted = controllerInterrupted(options.controllerActive ?? campaignLockIsActive, directory, plan, state.status); + index.push({ key: entry.name, status: interrupted ? 'attention-required' : state.status, updatedAt: state.updatedAt }); + } catch { index.push({ key: entry.name, status: 'unreadable', updatedAt: '' }); } + } + index.sort((a, b) => b.updatedAt.localeCompare(a.updatedAt) || a.key.localeCompare(b.key)); + const matches = (status: string, selection: CampaignFilter): boolean => selection === 'all' + || (selection === 'attention' ? ['attention-required', 'unreadable'].includes(status) + : status === (selection === 'ready' ? 'prepared' : 'completed')); + const counts = Object.fromEntries((['all', 'attention', 'completed', 'ready'] as const) + .map(selection => [selection, index.filter(item => matches(item.status, selection)).length])) as Record; + const selected = index.filter(item => matches(item.status, filter)); + const pageSize = 20; + const pages = Math.max(1, Math.ceil(selected.length / pageSize)); + const page = Math.min(Math.max(1, requestedPage), pages); + const keys = selected.slice((page - 1) * pageSize, page * pageSize).map(item => item.key); + const summaries = new Map(overviewSummary(campaignsRoot, { ...options, keys }).map(item => [item.key, item])); + return { campaigns: keys.map(key => summaries.get(key)!).filter(Boolean), page, pages, + total: selected.length, pageSize, counts }; +} + +export type OverviewPage = ReturnType; + +// Campaign sheet + +export interface SheetFacts { + mode: string; + workSelection: string | null; + repairSelection: string | null; + repairLimits: RepairBudget; + agent: string | null; + model: string | null; + guidance: string | null; + productionQuality: boolean | null; + recipes: Array<{ level: number; id: string | null; contentSha256: string | null }>; + timeLimitMinutes: number; + spendLimitUsd: number | null; + controllerImage: string | null; + buildImage: string | null; + planSha256: string; + grading: string; + gradingReasons: string[]; +} + +export interface ClimbPoint { + score: number; + max: number; + level: number | null; + unaided: boolean; +} + +export interface SheetAttempt { + model?: string; + effort?: string; + liveSpend?: number; + id: string; + repetition: number; + status: string; + phase: string; + stalling: boolean; + excluded: string | null; + continued: boolean; + logUpdatedAt: string | null; + activityUpdatedAt?: string | null; + paused?: boolean; + score: number | null; + unaided: number | null; + repairs: { used: number; budget: number }; + timeSec: number | null; + executionStartedAt: string | null; + executionCompletedAt: string | null; + executionCost?: CostEvidence; + spend: CostEvidence; + spendPending: boolean; + completion: CheckCompletion | null; + featureCompletion?: DependencyCompletionBreakdown['featureCompletion'] | null; + checkCategories?: DependencyCompletionBreakdown['checkCategories'] | null; + variant: string; + climb: ClimbPoint[]; +} + +export interface SheetLevel { + level: number; + unaided: { score: number; max: number } | null; + score: { score: number; max: number } | null; + repairs: number; +} + +export interface SheetQuestline { + id: string; + title: string; + score: number | null; + nodes: Array<{ id: string; status: string }>; +} + +export interface SheetStack { + liveSpend?: number; + stack: string; + costPerValidRun: number | null; + selectedAttemptId: string | null; + score: number | null; + points: { score: number; max: number } | null; + unaided: number | null; + continued: boolean; + regressions: number | null; + timeSec: number | null; + spend: CostEvidence; + spendPending: boolean; + completionRate: number | null; + n: number; + attempts: SheetAttempt[]; + levels: SheetLevel[] | null; + questlines: SheetQuestline[] | null; +} + +export interface CampaignSheet { + key: string; + id: string; + title: string; + status: string; + mode: string; + levels: number[]; + repetitions: number; + provisional: boolean; + mixedScope: boolean; + executions: number; + // A dependency campaign that stopped between executions is the one thing an + // operator can restart; the server checks the same three facts again. + resumable: boolean; + controllerOwner?: string | null; + reportFiles?: string[]; + createdAt: string; + updatedAt: string; + facts: SheetFacts; + stacks: SheetStack[]; +} + +interface SheetAttemptView { + inspected: InspectedAttempt; + attempt: SheetAttempt; +} + +function sheetFacts(plan: CompiledCampaignPlan): SheetFacts { + const mode = plan.definition.mode; + const policy = plan.dependencyPolicy?.definition ?? null; + const agent = plan.agents[0] ?? null; + const facts = campaignFacts(plan, { calibrationResolver }); + return { + mode: mode.id, + workSelection: policy?.workSelection ?? mode.workSelection ?? null, + repairSelection: policy?.repair.selection ?? plan.definition.repair.selection, + repairLimits: plan.definition.repair.budget, + agent: agent?.adapter ?? null, + model: agent?.model ?? null, + guidance: plan.attempts[0]?.guidance ?? null, + productionQuality: plan.attempts.every(attempt => attempt.condition.productionQuality === true) + ? true : plan.attempts.some(attempt => attempt.condition.productionQuality === true) ? null : false, + recipes: facts.recipes, + timeLimitMinutes: plan.definition.budgets.attemptTimeoutMinutes, + spendLimitUsd: plan.definition.budgets.maxCostUsdPerAttempt, + controllerImage: facts.runtime.controllerImage, + buildImage: facts.runtime.buildImage, + planSha256: plan.contentSha256, + grading: gradingStatus(facts.grading), + gradingReasons: [...new Set(facts.grading.levels.flatMap(level => level.reasons ?? []))], + }; +} + +// A campaign whose levels disagree is partly publishable and says so. +function gradingStatus(grading: ReturnType['grading']): string { + const levels = new Set(grading.levels.map(level => level.status)); + return levels.size > 1 ? 'partial' : grading.status; +} + +function dependencyRepairs(plan: CompiledCampaignPlan, + dependency: DependencyProgress): { used: number; budget: number } { + return { + used: dependency.history?.repairAttempts ?? 0, + budget: repairBudgetLimit(plan.definition.repair, { + features: dependency.nodes.length, + depths: plan.definition.levels.length, + }), + }; +} + +function attemptRegressions(attempt: InspectedAttempt): number { + if (attempt.dependency) return attempt.dependency.regressions ?? 0; + return attempt.result?.regressions ?? 0; +} + +// Continued: the attempt resumed on a repair grant, so its first grade is a +// checkpoint baseline rather than an unaided build. +function attemptContinued(attempt: InspectedAttempt): boolean { + if (attempt.dependency) { + return attempt.dependency.attempts.features.some(feature => + typeof feature.granted === 'number' && feature.granted > 0); + } + return (attempt.result?.levels ?? []).some(level => level.continued); +} + +function sheetLevels(attempt: InspectedAttempt | null): SheetLevel[] { + return (attempt?.result?.levels ?? []).map(level => ({ + level: level.level, + unaided: level.firstAbort ? null : level.firstScore, + score: level.finalScore, + repairs: level.used, + })); +} + +function sheetQuestlines(dependency: DependencyProgress): SheetQuestline[] { + const status = new Map(dependency.nodes.map(node => [node.id, node.status])); + const scored = new Map((dependency.score?.questlines ?? []) + .map(questline => [questline.id, questline.percentage ?? null])); + return (dependency.questlines ?? []).map(questline => ({ + id: questline.id, + title: questline.title, + score: scored.get(questline.id) ?? null, + nodes: questline.nodes.map(id => ({ id, status: status.get(id) ?? 'locked' })), + })); +} + +function sheetAttemptView(plan: CompiledCampaignPlan, state: CampaignAttemptState, + directory: string, interrupted: boolean): SheetAttemptView { + const inspected = inspectCampaignAttempt(plan, state, directory); + const execution = inspected.execution; + const logPath = execution + ? join(contained(directory, execution.output, 'campaign execution'), LOG_FILE) : null; + const log = logPath ? readTextTail(logPath) : ''; + const logUpdatedAt = logPath && existsSync(logPath) + ? new Date(statSync(logPath).mtimeMs).toISOString() : null; + const running = inspected.status === 'running' && !interrupted; + const pause = attemptPause(plan, state, directory); + const paused = running && pause?.resumedAt === null; + const repairLimit = repairBudgetLimit(plan.definition.repair, { + features: plan.featureCatalog?.definition.nodes.length ?? 1, + depths: plan.definition.levels.length, + }); + const progress = parseRunProgress(log, { repairs: repairLimit, + running, status: inspected.status, + dependency: plan.definition.mode?.id === 'dependency' }); + const metrics = attemptMetrics({ ...inspected, logUpdatedAt }); + const repairs = inspected.dependency ? dependencyRepairs(plan, inspected.dependency) : null; + return { + inspected, + attempt: { + id: inspected.id, + repetition: inspected.repetition, + status: interrupted && inspected.status === 'running' ? 'interrupted' : inspected.status, + phase: interrupted && inspected.status === 'running' + ? 'Controller stopped before completion' : paused ? `Paused at L${pause.depth}` : progress.phase, + stalling: attemptStalling({ ...inspected, paused }), + paused, + excluded: attemptExcluded(inspected), + continued: attemptContinued(inspected), + logUpdatedAt, + score: percentage(metrics?.final ?? null), + unaided: percentage(metrics?.first ?? null), + repairs: repairs ?? { used: metrics?.repairs ?? 0, budget: repairLimit }, + timeSec: metrics?.duration ?? null, + executionStartedAt: execution?.startedAt ?? null, + executionCompletedAt: execution?.completedAt ?? null, + executionCost: inspected.cost, + spend: inspected.spend, + spendPending: inspected.status === 'running' || inspected.status === 'pending', + completion: inspected.completion, + featureCompletion: inspected.dependency?.featureCompletion ?? null, + checkCategories: inspected.dependency?.checkCategories ?? null, + model: inspected.model, + effort: plan.attempts.find(entry => entry.id === inspected.id)?.effort, + variant: inspected.variantLabel, + climb: progress.series, + }, + }; +} + +const sheetCache = new Map(); + +// Facts and per-stack figures. No log text and no package walk: the climb and +// the phase come from the run output the controller already writes. +export function campaignSheet(resultsRoot: string, key: string, + { controllerActive = campaignLockIsActive }: ViewOptions = {}): CampaignSheet { + const directory = campaignDirectory(resultsRoot, key); + const reportPaths = ['report/report.html', 'report/export-manifest.json']; + const fingerprint = campaignFingerprint(directory, + [CAMPAIGN_FILE.plan, CAMPAIGN_FILE.state, `report/${CAMPAIGN_FILE.reportJson}`, ...reportPaths], + [ARTIFACT_FILE.run, ARTIFACT_FILE.progressionState, 'depth-pause.json']); + const { plan, state } = dashboardCampaignState(directory); + const interrupted = controllerInterrupted(controllerActive, directory, plan, state.status); + const controllerOwner = readCampaignLock(directory)?.ownershipMarkerSha256 ?? null; + const cacheKey = `${directory}:${interrupted ? 'interrupted' : 'live'}:${controllerOwner ?? ''}`; + const cached = sheetCache.get(cacheKey); + if (cached?.fingerprint === fingerprint) return cached.sheet; + const views = state.attempts.map(attempt => + sheetAttemptView(plan, attempt, directory, interrupted)); + const comparison = compareCampaign({ + attempts: views.map(view => view.inspected) }); + const dependency = plan.definition.mode?.id === 'dependency'; + const stacks = plan.stacks.map(stack => { + const owned = views.filter(view => view.inspected.stack === stack.id); + const row = comparison.rows.find(entry => entry.stack === stack.id); + const eligible = row?.scopes.length === 1 ? row.runs.map(run => run.attempt) : []; + // Figures a repetition cannot average — the climb, the questline board, the + // per-level rows — come from the newest attempt that actually ran. + const latest = owned.findLast(view => view.inspected.execution !== null) ?? null; + const lead = latest?.inspected ?? null; + const metrics = lead ? attemptMetrics(lead) : null; + return { + stack: stack.id, + costPerValidRun: row?.costPerValidRun ?? null, + selectedAttemptId: latest?.attempt.id ?? null, + score: percentage(row?.final ?? null), + points: dependency ? uniquePoints(lead?.dependency ?? null) : metrics?.raw.final ?? null, + unaided: percentage(row?.first ?? null), + continued: owned.some(view => view.attempt.continued), + regressions: median(eligible.map(attemptRegressions)), + timeSec: row?.duration ?? null, + spend: executionSpend(owned.map(view => ({ cost: view.inspected.spend, knownCostUsd: view.inspected.spend.knownCostUsd }))), + spendPending: owned.some(view => view.attempt.spendPending), + completionRate: eligible.every(attempt => attempt.completion?.rate != null) + ? median(eligible.flatMap(attempt => attempt.completion?.rate == null + ? [] : [attempt.completion.rate])) : null, + n: row?.n ?? 0, + attempts: owned.map(view => view.attempt), + levels: dependency ? null : sheetLevels(lead), + questlines: lead?.dependency ? sheetQuestlines(lead.dependency) : null, + }; + }); + const sheet: CampaignSheet = { + key: basename(resolve(directory)), + id: plan.id, + title: plan.title, + status: interrupted ? 'attention-required' : state.status, + mode: plan.definition.mode?.id ?? 'sequential', + levels: plan.definition.levels, + repetitions: plan.definition.repetitions, + provisional: campaignFacts(plan, { calibrationResolver }).grading.status !== 'qualified', + mixedScope: comparison.mixedScope, + executions: state.summary.executions, + resumable: dependency && state.status === 'prepared' && state.summary.executions > 0, + controllerOwner: interrupted ? null : controllerOwner, + reportFiles: reportCurrent(directory, plan, state.updatedAt) + ? reportPaths.filter(path => existsSync(join(directory, path)) && statSync(join(directory, path)).isFile()) : [], + createdAt: state.createdAt, + updatedAt: state.updatedAt, + facts: sheetFacts(plan), + stacks, + }; + sheetCache.set(cacheKey, { fingerprint, sheet }); + return sheet; +} + +// The report records the plan and state it was built from; a later state makes it an earlier report. +function reportCurrent(directory: string, plan: CompiledCampaignPlan, updatedAt: string): boolean { + try { + const report = readArtifact(join(directory, 'report', CAMPAIGN_FILE.reportJson), { expectedKind: 'campaign_report' }); + return report.identities.experiment?.sha256 === plan.contentSha256 && report.timestamps.completedAt === updatedAt; + } catch { return false; } +} + +function uniquePoints(dependency: DependencyProgress | null): { score: number; max: number } | null { + const unique = dependency?.score?.uniqueChecks; + if (!unique || unique.passedPoints == null || unique.availablePoints == null) return null; + return { score: unique.passedPoints, max: unique.availablePoints }; +} + +// Attempt sub-resources + +function attemptState(directory: string, attemptId: string): CampaignAttemptState { + const { state } = dashboardCampaignState(directory); + const attempt = state.attempts.find(item => item.plan.id === attemptId); + if (!attempt) throw new Error('campaign attempt does not exist'); + return attempt; +} + +export interface AttemptCheckGrade { + id: string; + level: number | null; + round: number | null; + score: { score: number; max: number } | null; + error?: string; +} + +export interface AttemptCheck { + category?: 'feature' | 'production' | 'interface' | null; + key: string; + id: string; + description: string; + points: number; + feature: string; + outcome: string; + regressed: boolean; + history: string[]; + observations: Array<{ status: string; summary: string | null; expected: string | null; actual: string | null; + context?: string } | null>; +} + +export interface AttemptChecks { + attemptId: string; + stack: string; + grades: AttemptCheckGrade[]; + checks: AttemptCheck[]; +} + +function checkOutcome(evidence: unknown): string { + const status = evidence !== null && typeof evidence === 'object' && 'status' in evidence + ? (evidence as { status?: unknown }).status : null; + if (status === 'passed') return 'pass'; + if (status === 'failed') return 'fail'; + return 'not-run'; +} + +function checkObservation(value: unknown): AttemptCheck['observations'][number] { + if (!value || typeof value !== 'object') return null; + const evidence = value as Partial; + if (!evidence.status || !CHECK_EVIDENCE_STATUSES.includes(evidence.status)) return null; + const status = evidenceDisposition(evidence.status).label; + if (evidence.sensitivity?.length) return { status, summary: 'Sensitive evidence omitted.', expected: null, actual: null }; + const text = (item: unknown): string | null => { + if (item == null) return null; + const result = redactCredentials(typeof item === 'string' ? item : JSON.stringify(item, null, 2)); + return result.length <= 12_000 ? result : `${result.slice(0, 12_000)}\n[Truncated. Full evidence is in Files.]`; + }; + const last = evidence.actions?.at(-1)?.evidence as { + status?: string; action?: { id?: string }; sensitivity?: string[]; + } | undefined; + const failedStep = last?.status && last.status !== 'passed' && !last.sensitivity?.length ? last.action?.id : null; + const interfaceFailure = evidence.status === 'failed' && evidence.finding + && ['interface-missing', 'interface-invalid', 'stock-interface-missing'].includes(evidence.finding.kind); + const context = [ + interfaceFailure ? 'Interface failure; remaining steps were not reached. This does not prove the intended guarantee failed.' + : evidence.status === 'blocked' ? 'A prerequisite failed; the target check was not reached.' : null, + failedStep ? `Stopped at ${failedStep}.` : null, + evidence.finding ? renderFinding(evidence.finding) : null, + ].filter(Boolean).join(' '); + return { status, summary: text(evidence.summary), expected: text(evidence.expected), actual: text(evidence.observation), + ...(context ? { context: text(context)! } : {}) }; +} + +function gradeDirectories(executionDirectory: string): AttemptCheckGrade[] { + if (!existsSync(executionDirectory)) return []; + const progression = join(executionDirectory, 'progression'); + if (existsSync(progression)) { + return readdirSync(progression, { withFileTypes: true }) + .filter(entry => entry.isDirectory() && PROGRESSION_ATTEMPT.test(entry.name)) + .map(entry => ({ id: `progression/${entry.name}`, + level: null, round: Number(PROGRESSION_ATTEMPT.exec(entry.name)?.[1] ?? 0), score: null })) + .sort((left, right) => (left.round ?? 0) - (right.round ?? 0)); + } + return readdirSync(executionDirectory, { withFileTypes: true }) + .filter(entry => entry.isDirectory() && GRADE_DIRECTORY.test(entry.name)) + .map(entry => { + const match = GRADE_DIRECTORY.exec(entry.name); + const level = match?.[1] ?? match?.[2] ?? null; + return { id: entry.name, level: level === null ? null : Number(level), + round: match?.[3] === undefined ? 0 : Number(match[3]), score: null }; + }) + // The final `grading` directory has no level and comes after every level's grades. + .sort((left, right) => (left.level ?? Infinity) - (right.level ?? Infinity) + || (left.round ?? 0) - (right.round ?? 0)); +} + +// Per-check outcome and the history of every grade that reported it: the +// question "did this ever pass" has no other answer in the evidence. +export function attemptChecks(resultsRoot: string, key: string, attemptId: string): AttemptChecks { + const directory = campaignDirectory(resultsRoot, key); + const attempt = attemptState(directory, attemptId); + const execution = attempt.executions.at(-1) ?? null; + if (!execution) return { attemptId, stack: attempt.plan.stack, grades: [], checks: [] }; + const executionDirectory = contained(directory, execution.output, 'campaign execution'); + const grades = gradeDirectories(executionDirectory); + const { plan } = dashboardCampaignState(directory); + const metadata = new Map(plan.featureCatalog?.definition.nodes.flatMap(node => + node.gradingChecks.map(check => [check.id, check] as const)) ?? []); + const checks = new Map(); + grades.forEach((grade, index) => { + const path = join(executionDirectory, grade.id, ARTIFACT_FILE.gradeBundle); + if (!existsSync(path)) { + grade.error = 'grade bundle is missing'; + return; + } + let payload; + try { + payload = readArtifactPayload(path, { expectedKind: 'grade_bundle' }); + } catch (error) { + grade.error = redactCredentials(errorMessage(error)); + return; + } + grade.score = payload.totals?.score == null || payload.totals.max == null + ? null : { score: payload.totals.score, max: payload.totals.max }; + for (const suite of Object.values(payload.suites ?? {})) { + for (const feature of suite.features ?? []) { + for (const criterion of feature.criteria ?? []) { + const stableKey = criterion.stableKey ?? `${feature.name ?? ''}.${criterion.id ?? ''}`; + const entry = checks.get(stableKey) ?? { key: stableKey, id: criterion.id ?? stableKey, + description: criterionDescription(criterion), points: criterion.points ?? 0, + category: metadata.get(stableKey)?.category ?? null, + feature: feature.name ?? '', outcome: 'not-run', regressed: false, + history: grades.map(() => 'not-run'), observations: grades.map(() => null) }; + entry.history[index] = checkOutcome(criterion.evidence); + entry.observations[index] = checkObservation(criterion.evidence); + checks.set(stableKey, entry); + } + } + } + }); + for (const check of checks.values()) { + const conclusive = check.history.filter(outcome => outcome !== 'not-run'); + check.outcome = conclusive.at(-1) ?? 'not-run'; + check.regressed = conclusive.some((outcome, index) => + outcome === 'fail' && conclusive.slice(0, index).includes('pass')); + } + return { attemptId, stack: attempt.plan.stack, grades, checks: [...checks.values()] }; +} + +// The grade bundle names the criterion text `desc`. +function criterionDescription(criterion: object): string { + const record = criterion as { desc?: unknown; description?: unknown }; + if (typeof record.desc === 'string') return record.desc; + return typeof record.description === 'string' ? record.description : ''; +} + +export interface AttemptPackage { + attemptId: string; + stack: string; + executions: Array<{ + executionId: string; + ordinal: number; + status: string; + artifacts: DashboardArtifact[]; + visuals: DashboardArtifact[]; + truncated: boolean; + }>; +} + +export function attemptPackage(resultsRoot: string, key: string, + attemptId: string): AttemptPackage { + const directory = campaignDirectory(resultsRoot, key); + const attempt = attemptState(directory, attemptId); + return { + attemptId, + stack: attempt.plan.stack, + executions: attempt.executions.map(execution => { + const scanned = walkPublicExecutionArtifacts(directory, + contained(directory, execution.output, 'campaign execution')); + return { executionId: execution.id, ordinal: execution.ordinal, status: execution.status, + artifacts: scanned.artifacts, + visuals: scanned.artifacts.filter(artifact => artifact.kind === 'visual'), + truncated: scanned.truncated }; + }), + }; +} + +export interface AttemptLogSlice { + attemptId: string; + from: number; + offset: number; + size: number; + text: string; +} + +// Bytes after an offset, so a following view pays for growth rather than for +// the whole log on every poll. +export function attemptLogSlice(resultsRoot: string, key: string, attemptId: string, + fromOffset = 0): AttemptLogSlice { + const directory = campaignDirectory(resultsRoot, key); + const attempt = attemptState(directory, attemptId); + const execution = attempt.executions.at(-1) ?? null; + const path = execution + ? join(contained(directory, execution.output, 'campaign execution'), LOG_FILE) : null; + if (!path || !existsSync(path)) { + return { attemptId, from: fromOffset, offset: 0, size: 0, text: '' }; + } + const descriptor = openSync(path, 'r'); + try { + const size = fstatSync(descriptor).size; + // A rotated or truncated log invalidates the caller's offset. + const start = Math.min(Math.max(0, fromOffset), size); + const length = Math.min(size - start, MAX_LOG_BYTES); + const buffer = Buffer.alloc(length); + if (length) readSync(descriptor, buffer, 0, length, start); + return { attemptId, from: fromOffset, offset: start + length, size, + text: redactCredentials(buffer.toString('utf8')) }; + } finally { + closeSync(descriptor); + } +} + +// Campaign progression + +export interface ProgressionCatalogNode { + id: string; + title: string; + questline: string; + depth: number; + dependencies: string[]; +} + +export interface ProgressionStep { + sequence: number; + completedAt?: string | null; + completion?: number | null; + featureCompletion?: number | null; + action: 'build' | 'repair' | 'grant'; + targets: string[]; + // Node status after the event, index-aligned with `nodes`. + statuses: string[]; + score: number | null; + repairs: number; +} + +export interface ProgressionTrack { + liveCosts?: Array<{ completedAt: string; costUsd: number }>; + stack: string; + attemptId: string; + updatedAt: string; + steps: ProgressionStep[]; + costs?: Array<{ completedAt: string; cost: CostEvidence }>; +} + +export interface CampaignProgression { + key: string; + depths: number[]; + questlines: Array<{ id: string; title: string; nodes: string[] }>; + nodes: ProgressionCatalogNode[]; + stacks: ProgressionTrack[]; +} + +function progressionSnapshot(state: ProgressionState, nodeIds: readonly string[]): { + statuses: string[]; + score: number | null; + repairs: number; +} { + // Passed points over selected points: the same measure as the weighted score. + const score = progressionEngine.score(state).uniqueChecks.percentage; + return { + statuses: nodeIds.map(id => state.nodes[id]?.status ?? 'locked'), + score: score == null ? null : Math.round(score * 10) / 10, + repairs: state.attempts.filter(attempt => attempt.repair !== undefined).length, + }; +} + +function progressionSteps(state: DependencyState, nodeIds: readonly string[], times: Map): ProgressionStep[] { + let replay = progressionEngine.initialize(state.definition); + return state.events.map(event => { + const action = progressionEngine.nextAction(replay); + if (event.type === 'repairs-granted') { + replay = progressionEngine.grantRepairs(replay, event.grant); + return { sequence: event.sequence, action: 'grant' as const, + targets: [...event.grant.nodeIds], ...progressionSnapshot(replay, nodeIds) }; + } + const targets = action.type === 'terminal' + ? [] : [...(action.prompt as DependencyPromptSelection).nodeIds]; + const repair = action.type === 'repair'; + replay = progressionEngine.recordResult(replay, event.result); + return { sequence: event.sequence, action: repair ? 'repair' as const : 'build' as const, + targets, ...progressionSnapshot(replay, nodeIds), + completedAt: event.result.evidence ? times.get(`${event.result.evidence.id}:${event.result.evidence.sha256}`) ?? null : null, + completion: scoreDependencyState(replay as DependencyState).completion.rate, + featureCompletion: dependencyCompletionBreakdown(replay as DependencyState).featureCompletion.rate }; + }); +} + +const progressionCache = new Map(); + +// The catalog subgraph the campaign runs, plus one node-status snapshot per +// progression event: the graph and its replay come from the same read. +export function campaignProgression(resultsRoot: string, key: string): CampaignProgression | null { + const directory = campaignDirectory(resultsRoot, key); + const { plan, state } = dashboardCampaignState(directory); + if (plan.definition.mode?.id !== 'dependency' || !plan.featureCatalog + || !plan.dependencyPolicy) return null; + const fingerprint = campaignFingerprint(directory, [CAMPAIGN_FILE.plan, CAMPAIGN_FILE.state], + [ARTIFACT_FILE.progressionState, ARTIFACT_FILE.run]); + const cached = progressionCache.get(directory); + if (cached?.fingerprint === fingerprint) return cached.view; + const progression = compileProgressionInput(dependencyRuntimeDefinition( + plan.featureCatalog, plan.dependencyPolicy)); + const definition = progression.definition; + const nodeIds = definition.nodes.map(node => node.id); + const owned = new Set(nodeIds); + const stacks: ProgressionTrack[] = []; + for (const attempt of state.attempts) { + const execution = attempt.executions.at(-1) ?? null; + if (!execution) continue; + const path = join(contained(directory, execution.output, 'campaign execution'), + ARTIFACT_FILE.progressionState); + if (!existsSync(path)) continue; + const stored = readProgressionState(path, { + progression, + featureCatalogIdentity: plan.featureCatalog.identity, + dependencyPolicyIdentity: plan.dependencyPolicy.identity, + owner: campaignProgressionOwner(plan, attempt.plan, { workspace: true }), + }); + const executionDirectory = contained(directory, execution.output, 'campaign execution'); + const times = new Map(); + for (const grade of gradeDirectories(executionDirectory)) { + const bundlePath = join(executionDirectory, grade.id, ARTIFACT_FILE.gradeBundle); + if (!existsSync(bundlePath)) continue; + try { + const bundle = readArtifact(bundlePath, { expectedKind: 'grade_bundle' }); + times.set(`${bundle.id}:${sha256(canonicalDefinitionJson(bundle))}`, bundle.timestamps.completedAt); + } catch { /* Missing or invalid evidence must not invent a chart timestamp. */ } + } + const costs: NonNullable = []; + try { + const run = readArtifact(join(executionDirectory, ARTIFACT_FILE.run), { expectedKind: 'benchmark_run' }); + if (run.attempt.parentId !== attempt.plan.id) throw new Error('Run belongs to another attempt'); + const checkpoints = (run.payload as { checkpoints?: RunCheckpoint[] }).checkpoints ?? []; + for (const checkpoint of checkpoints) { + try { + const evidencePath = contained(executionDirectory, checkpoint.evidence.path, 'cost checkpoint'); + if (sha256(readFileSync(evidencePath)) !== checkpoint.evidence.sha256) continue; + const grade = readArtifact(evidencePath, { expectedKind: 'grade_bundle' }); + if (grade.timestamps.completedAt && checkpoint.executionCost.status !== 'unknown') { + costs.push({ completedAt: grade.timestamps.completedAt, cost: checkpoint.executionCost }); + } + } catch { /* Missing evidence does not establish a timed cost. */ } + } + } catch { /* A run may not have saved its first checkpoint yet. */ } + stacks.push({ stack: attempt.plan.stack, attemptId: attempt.plan.id, + updatedAt: new Date(statSync(path).mtimeMs).toISOString(), costs, + steps: progressionSteps(stored.state as DependencyState, nodeIds, times) }); + } + const view: CampaignProgression = { + key: basename(resolve(directory)), + depths: [...new Set(definition.nodes.map(node => node.level))].sort((a, b) => a - b), + questlines: definition.questlines.map(questline => ({ id: questline.id, + title: questline.title, nodes: [...questline.nodes] })), + nodes: definition.nodes.map(node => ({ id: node.id, title: node.title, + questline: node.questline, depth: node.level, + dependencies: node.dependencies.filter(id => owned.has(id)) })), + stacks, + }; + progressionCache.set(directory, { fingerprint, view }); + return view; +} + +type LiveCosts = Map>>; +const liveCampaignCache = new Map(); + +function campaignLiveCosts(resultsRoot: string, key: string) { + const directory = campaignDirectory(resultsRoot, key); + const cached = liveCampaignCache.get(directory); + if (cached && (cached.pending || Date.now() - cached.at < 5000)) return cached.value; + const { state } = dashboardCampaignState(directory); + const entry = { at: Date.now(), pending: true, value: cached?.value ?? new Map() as LiveCosts }; + const refresh = async () => { + const costs = new Map>>(); + await Promise.all(state.attempts.map(async attempt => { + const execution = attempt.executions.at(-1); + // Restored executions can carry older transcript history. Keep their saved + // receipts until response identities are available across that boundary. + if (attempt.status !== 'running' || attempt.executions.length !== 1 || !execution?.startedAt + || !['claude-code', 'codex'].includes(attempt.plan.agentAdapter)) return; + try { + costs.set(attempt.plan.id, await liveTranscriptCost(contained(directory, execution.output, 'campaign execution'), + attempt.plan.agentAdapter, attempt.plan.pricing.rates, attempt.plan.model, execution.startedAt)); + } catch { /* Missing, conflicting, or unpriced usage leaves saved receipts visible. */ } + })); + entry.value = costs; + }; + if (liveCampaignCache.size >= 32) liveCampaignCache.delete(liveCampaignCache.keys().next().value!); + liveCampaignCache.set(directory, entry); + void refresh().catch(() => { entry.value = new Map(); }).finally(() => { + entry.pending = false; + entry.at = Date.now(); + }); + return entry.value; +} + +export function campaignLiveSheet(resultsRoot: string, key: string, options: ViewOptions = {}): CampaignSheet { + const sheet = structuredClone(campaignSheet(resultsRoot, key, options)); + const live = campaignLiveCosts(resultsRoot, key); + const directory = campaignDirectory(resultsRoot, key); + const { state } = dashboardCampaignState(directory); + for (const stack of sheet.stacks) { + for (const attempt of stack.attempts) { + const execution = state.attempts.find(entry => entry.plan.id === attempt.id)?.executions.at(-1); + if (execution) { + const path = join(contained(directory, execution.output, 'campaign execution'), LOG_FILE); + if (existsSync(path)) { + const updatedAt = new Date(statSync(path).mtimeMs).toISOString(); + if (attempt.logUpdatedAt !== updatedAt) { + attempt.logUpdatedAt = updatedAt; + const progress = parseRunProgress(readTextTail(path), { repairs: attempt.repairs.budget, + running: attempt.status === 'running', status: attempt.status, dependency: sheet.mode === 'dependency' }); + if (attempt.status === 'running' && !attempt.paused) attempt.phase = progress.phase; + attempt.climb = progress.series; + } + } + } + if (attempt.status !== 'running') continue; + const snapshot = live.get(attempt.id); + if (snapshot?.activityUpdatedAt) { + attempt.activityUpdatedAt = snapshot.activityUpdatedAt; + attempt.stalling = attemptStalling(attempt); + } + const total = snapshot?.costs.at(-1)?.costUsd; + attempt.liveSpend = liveCostTotal(attempt.status, total, attempt.spend.costUsd); + } + if (stack.attempts.some(attempt => attempt.liveSpend !== undefined) + && stack.attempts.every(attempt => attempt.liveSpend !== undefined || attempt.spend.status === 'exact')) { + stack.liveSpend = stack.attempts.reduce((sum, attempt) => sum + (attempt.liveSpend ?? attempt.spend.costUsd!), 0); + } + } + return sheet; +} + +// Transcript and log updates do not need another graph replay or evidence payload. +export function campaignLiveUpdate(resultsRoot: string, key: string, options: ViewOptions = {}) { + const sheet = campaignLiveSheet(resultsRoot, key, options); + const live = campaignLiveCosts(resultsRoot, key); + return { updatedAt: sheet.updatedAt, status: sheet.status, stacks: sheet.stacks.map(stack => ({ + stack: stack.stack, liveSpend: stack.liveSpend ?? null, + attempts: stack.attempts.filter(attempt => attempt.executionStartedAt !== null).map(attempt => ({ + id: attempt.id, phase: attempt.phase, logUpdatedAt: attempt.logUpdatedAt, + activityUpdatedAt: attempt.activityUpdatedAt ?? null, stalling: attempt.stalling, + climb: attempt.climb, liveSpend: attempt.liveSpend ?? null, + liveCosts: attempt.liveSpend === undefined ? [] : live.get(attempt.id)?.costs ?? [], + })), + })) }; +} +export type CampaignLiveUpdate = ReturnType; + +export function campaignLiveProgression(resultsRoot: string, key: string): CampaignProgression | null { + const progression = structuredClone(campaignProgression(resultsRoot, key)); + if (!progression) return null; + const live = campaignLiveCosts(resultsRoot, key); + const sheet = campaignSheet(resultsRoot, key); + const running = new Set(sheet.stacks.flatMap(stack => + stack.attempts.filter(attempt => attempt.status === 'running').map(attempt => attempt.id))); + for (const stack of sheet.stacks) for (const attempt of stack.attempts) { + const snapshot = live.get(attempt.id); + if (running.has(attempt.id) && snapshot?.costs.length + && !progression.stacks.some(track => track.attemptId === attempt.id)) { + progression.stacks.push({ stack: stack.stack, attemptId: attempt.id, + updatedAt: snapshot.activityUpdatedAt ?? sheet.updatedAt, steps: [], costs: [] }); + } + } + for (const track of progression.stacks) { + if (!running.has(track.attemptId)) continue; + const snapshot = live.get(track.attemptId); + if (snapshot?.costs.length && liveCostTotal('running', snapshot.costs.at(-1)!.costUsd, + track.costs?.at(-1)?.cost.costUsd ?? null) !== undefined) { + track.liveCosts = snapshot.costs; + } + } + return progression; +} + +export function attemptTranscript(resultsRoot: string, key: string, attemptId: string, + session: string, before?: number) { + const directory = campaignDirectory(resultsRoot, key); + const attempt = attemptState(directory, attemptId); + return readAttemptTranscript(attempt.executions.map((execution, index) => ({ + directory: contained(directory, execution.output, 'campaign execution'), + label: `Execution ${index + 1}`, + })), attempt.plan.agentAdapter, session, before); +} diff --git a/tools/stack-bench/dashboard/public/app.ts b/tools/stack-bench/dashboard/public/app.ts new file mode 100644 index 00000000000..00311baf867 --- /dev/null +++ b/tools/stack-bench/dashboard/public/app.ts @@ -0,0 +1,781 @@ +/// +/// +import type { readExecutionJob } from '../../src/campaigns/execution-jobs.js'; +import type { RunSetupCatalog, RunSetupRequest, RunSetupReview } from '../../src/campaigns/run-setup.js'; +import { initialRun, readRunForm, runSetupPage } from './views/run-setup.js'; +import type { TranscriptPage } from '../dashboard-transcript.js'; +import type { referenceRuns } from '../dashboard-reference-runs.js'; + + +// The client: real paths, one event stream, and keyed reconciliation so a +// refresh does not move what the pointer is on. Every view is a pure function +// of data; the only DOM work in the dashboard happens here. + +import type { AttemptChecks, AttemptPackage, CampaignLiveUpdate, CampaignProgression, CampaignSheet, OverviewEntry, OverviewPage } + from '../dashboard-views.js'; +import type { DashboardPlan } from '../dashboard-model.js'; +import type { readCampaignTimeBudget } from '../../src/campaigns/campaign-time-grant.js'; +import { type QuestlineView, campaignPage, replayTimeline, selectedProgression } from './views/campaign.js'; +import { type AttemptTab, attemptPage } from './views/attempt.js'; +import { type CampaignFilter, campaignsPage } from './views/campaigns.js'; +import { type Page, type RunForm, plansPage, topbar } + from './views/plans.js'; +import { duration, elapsed, esc } from './format.js'; + +const FALLBACK_MS = 15_000; +const TABS: readonly AttemptTab[] = ['checks', 'transcript', 'screenshots', 'files', 'log']; +const VIEWS: readonly QuestlineView[] = ['grid', 'graph', 'replay']; +const FILTERS: readonly CampaignFilter[] = ['all', 'attention', 'completed', 'ready']; + +interface Route { + key: string; + attempt: string; + plans: boolean; + newRun: boolean; + filter: CampaignFilter; + page: number; + view: QuestlineView; + chart: 'completion' | 'cost' | 'distribution'; + unit: 'checks' | 'features'; + step: number; + tab: AttemptTab; +} + +const state = { + overview: [] as OverviewEntry[], + overviewPage: null as OverviewPage | null, + overviewQuery: '', + references: { runs: [], error: null } as Awaited>, + plans: [] as DashboardPlan[], + overviewLoaded: false, + plansLoaded: false, + pendingJobs: new Map; dispatchError: string | null }>(), + setup: null as RunSetupCatalog | null, + setupRequest: null as RunSetupRequest | null, + setupReview: null as RunSetupReview | null, + canStart: false, + csrfToken: '', + readError: '', + form: { error: '' } as RunForm, + sheets: new Map(), + progression: new Map(), + hiddenChartRuns: new Map>(), + checks: new Map(), + evidence: new Map(), + timeBudgets: new Map>(), + timeGrantIds: new Map(), + timeGrantMinutes: '120', + transcript: { attempt: '', session: '', before: undefined as number | undefined, page: null as TranscriptPage | null }, + log: { attempt: '', text: '', offset: 0 }, +}; +let fallback = 0; +let events: EventSource | null = null; +let playing = 0; +let submitting = false; +let loading = false; +let loadVersion = 0; +let loadTask: Promise | null = null; +let loadController = new AbortController(); +let refreshPending = false; +let pendingNavigation = false; +let pendingOverview = false; + +function route(): Route { + const url = new URL(location.href); + const parts = url.pathname.split('/').filter(Boolean); + const pick = (values: readonly Value[], name: string, fall: Value): Value => + values.find(value => value === url.searchParams.get(name)) ?? fall; + return { + key: parts[0] === 'c' ? parts[1] ?? '' : '', + attempt: parts[2] === 'a' ? parts[3] ?? '' : '', + plans: parts[0] === 'plans' || parts[0] === 'new', + newRun: parts[0] === 'new', + filter: pick(FILTERS, 'filter', 'all'), + page: /^\d+$/.test(url.searchParams.get('page') ?? '1') && Number.isSafeInteger(Number(url.searchParams.get('page') ?? 1)) + ? Math.max(1, Number(url.searchParams.get('page') ?? 1)) : 1, + view: pick(VIEWS, 'questlines', 'grid'), + chart: pick(['completion', 'cost', 'distribution'] as const, 'chart', 'completion'), + unit: pick(['checks', 'features'] as const, 'unit', 'features'), + step: Math.max(0, Number(url.searchParams.get('step') ?? 0)), + tab: pick(TABS, 'tab', 'checks'), + }; +} + +async function read(url: string): Promise { + const version = loadVersion; + try { + const response = await fetch(url, { headers: { accept: 'application/json' }, + signal: AbortSignal.any([loadController.signal, AbortSignal.timeout(30_000)]) }); + if (!response.ok) { + const failure = await response.json().catch(() => ({})) as { error?: string }; + if (version === loadVersion) state.readError = failure.error ?? `Request failed (${response.status}).`; + return null; + } + const payload = await response.json() as Payload; + return version === loadVersion ? payload : null; + } catch { + if (version === loadVersion) state.readError = 'The dashboard did not respond. Check its connection and try again.'; + return null; + } +} + +function attemptUrl(current: Route, suffix: string): string { + return `/api/campaigns/${encodeURIComponent(current.key)}` + + `/attempts/${encodeURIComponent(current.attempt)}/${suffix}`; +} + +async function readLog(current: Route): Promise { + const version = loadVersion; + if (state.log.attempt !== current.attempt) state.log = { attempt: current.attempt, text: '', offset: 0 }; + try { + const response = await fetch(attemptUrl(current, `log?from=${state.log.offset}`), + { signal: AbortSignal.any([loadController.signal, AbortSignal.timeout(30_000)]) }); + if (!response.ok) throw new Error('Log request failed'); + const text = await response.text(); + if (version !== loadVersion || state.log.attempt !== current.attempt) return; + state.log.text += text; + state.log.offset = Number(response.headers.get('x-stack-bench-log-offset') ?? state.log.offset); + } catch { + if (version === loadVersion) state.readError = 'Could not load the run log. Try again.'; + } +} + +function chrome(current: Route): string { + const sheet = state.sheets.get(current.key) ?? null; + const page: Page = current.plans ? 'plans' + : current.key && !current.attempt ? 'campaign' : 'campaigns'; + return topbar({ page, key: current.key, canStart: state.canStart, error: state.form.error, + reportFiles: sheet?.reportFiles, + controllerOwner: page === 'campaign' ? sheet?.controllerOwner : null, + resumable: state.canStart && page === 'campaign' && (sheet?.resumable ?? false) }); +} + +function page(current: Route): string { + const sheet = state.sheets.get(current.key) ?? null; + const pending = state.pendingJobs.get(current.key); + if (pending) return `

${esc(pending.pendingJob.job.key)}

${esc(pending.pendingJob.status)}

` + + `

${esc(pending.dispatchError ?? pending.pendingJob.error ?? pending.pendingJob.capacityWait?.reason ?? 'Waiting for the campaign to start. This page updates automatically.')}

` + + (state.canStart && ['queued', 'running'].includes(pending.pendingJob.status) + ? '
' : '') + + (state.canStart && pending.pendingJob.status === 'queued' && pending.dispatchError + ? '
' : '') + + (state.form.error ? `

${esc(state.form.error)}

` : '') + '
'; + if (current.newRun) return runSetupPage(state.setup, state.setupRequest, state.setupReview, state.form.error, state.canStart); + if (current.plans) { + return plansPage({ plans: state.plans, + loading: loading && !state.plansLoaded }); + } + if (!current.key) { + return campaignsPage({ campaigns: state.overview, filter: current.filter, + pagination: state.overviewPage ?? undefined, + references: state.references, + loading: loading && (!state.overviewLoaded || state.overviewQuery !== `${current.filter}:${current.page}`) }); + } + if (!sheet) return `
Campaigns / ` + + `${esc(current.key)}
`; + if (current.attempt) { + return attemptPage({ sheet, progression: state.progression.get(current.key) ?? null, + attemptId: current.attempt, tab: current.tab, + timeBudget: state.timeBudgets.get(current.attempt), canControl: state.canStart, + controlError: state.form.error, + transcript: state.transcript.attempt === current.attempt ? state.transcript.page : null, + checks: state.checks.get(current.attempt) ?? null, + evidence: state.evidence.get(current.attempt) ?? null, + log: state.log.attempt === current.attempt ? state.log.text : '' }); + } + return campaignPage({ sheet, progression: state.progression.get(current.key) ?? null, + view: current.view, step: current.step, chart: current.chart, unit: current.unit, + hiddenChartRuns: state.hiddenChartRuns.get(current.key) }); +} + +function sync(current: Element, next: Element): void { + for (const name of [...current.getAttributeNames()]) { + if (current.tagName === 'DETAILS' && name === 'open') continue; + if (!next.hasAttribute(name)) current.removeAttribute(name); + } + for (const name of next.getAttributeNames()) { + if (current.getAttribute(name) !== next.getAttribute(name)) { + current.setAttribute(name, next.getAttribute(name) ?? ''); + } + } +} + +// Replace only what changed, matching children by position and data-key, so a +// row under the pointer keeps its hover across a refetch. +function patch(current: Element, next: Element): void { + const mine = [...current.children]; + const theirs = [...next.children]; + if (mine.length !== theirs.length || current.childNodes.length !== mine.length + || next.childNodes.length !== theirs.length) { + current.replaceChildren(...next.childNodes); + return; + } + mine.forEach((child, index) => { + const other = theirs[index]!; + if (child.tagName !== other.tagName + || child.getAttribute('data-key') !== other.getAttribute('data-key')) { + child.replaceWith(other); + return; + } + if (child.outerHTML === other.outerHTML) return; + if (!child.children.length || !other.children.length) { + child.replaceWith(other); + return; + } + // Moving option nodes can change a native select's value during reconciliation. + const selected = other instanceof HTMLSelectElement ? other.value : null; + sync(child, other); + patch(child, other); + if (child instanceof HTMLSelectElement && selected !== null) child.value = selected; + }); +} + +function updateTimeTotal(field: HTMLInputElement): void { + const total = field.form?.querySelector('[data-time-base]'); + if (total) total.textContent = field.validity.valid + ? `Limit after request: ${duration((Number(total.dataset.timeBase) + field.valueAsNumber) * 60)}` + : 'Enter positive whole minutes.'; +} + +function render(): void { + const current = route(); + const root = document.body; + const next = document.createElement('body'); + const ready = current.plans ? state.plansLoaded : current.key + ? state.sheets.has(current.key) : state.overviewLoaded; + next.innerHTML = `${chrome(current)}
` + + (state.readError ? `` : '') + + (loading && !ready && current.key ? `

${current.attempt ? 'Run details' : 'Campaign'}

` + + '
Loading…
' : page(current)) + + '
'; + const transcript = root.querySelector('.transcript'); + const scroll = transcript?.scrollTop ?? 0; + const follow = !transcript || transcript.scrollHeight - scroll - transcript.clientHeight < 40; + const openTools = [...root.querySelectorAll('.transcript details[open]')].map(el => el.dataset.key); + patch(root, next); + const updated = root.querySelector('.transcript'); + if (updated) { + for (const tool of updated.querySelectorAll('details')) tool.open = openTools.includes(tool.dataset.key); + updated.scrollTop = follow ? updated.scrollHeight : scroll; + } + // Keep the time limit input across background refreshes. + for (const field of document.querySelectorAll('form[data-run] input')) { + if (field.name === 'minutes') { + field.value = state.timeGrantMinutes; + updateTimeTotal(field); + } + } + for (const form of document.querySelectorAll('form[data-run]')) { + form.setAttribute('aria-busy', String(submitting)); + if (submitting && form.dataset.run?.startsWith('setup-')) { + const submit = form.querySelector('button[type=submit]'); + if (submit) submit.textContent = form.dataset.run === 'setup-review' ? 'Preparing review…' : 'Starting…'; + } + for (const button of form.querySelectorAll('button[type=submit]')) { + button.disabled = submitting || (!state.canStart && form.dataset.run?.startsWith('setup-')) || (form.dataset.run === 'grant-time' + && (state.timeBudgets.get(current.attempt)?.grants.some(grant => grant.disposition === 'pending') ?? false)); + } + } +} + +function load(navigation = false, liveOnly = false): Promise { + refreshPending = true; + pendingNavigation ||= navigation; + pendingOverview ||= !liveOnly; + if (navigation) { + state.form = { error: '' }; + ++loadVersion; + loadController.abort(); + } + if (loadTask) return loadTask; + if (document.hidden && !navigation) return Promise.resolve(); + loadTask = (async () => { + while (refreshPending && (!document.hidden || pendingNavigation)) { + const showLoading = pendingNavigation; + const refreshOverview = pendingOverview; + refreshPending = pendingNavigation = false; + pendingOverview = false; + const version = ++loadVersion; + loadController = new AbortController(); + loading = true; + state.readError = ''; + if (showLoading) render(); + try { + await loadData(version, refreshOverview); + } catch { + if (version === loadVersion) state.readError = 'Could not load this page. Try again.'; + } finally { + if (version === loadVersion) { + loading = false; + render(); + } + } + } + })().finally(() => { loadTask = null; }); + return loadTask; +} + +async function loadData(version: number, refreshOverview: boolean): Promise { + const current = route(); + if (refreshOverview && !current.key && !current.plans) { + const references = await read>>('/api/reference-runs'); + if (references && version === loadVersion) { state.references = references; render(); } + } + if (!refreshOverview) { + const keys = current.key ? [current.key] : []; + await Promise.all(keys.map(async key => { + const sheet = state.sheets.get(key); + if (!sheet) return; + const update = await read(`/api/campaigns/${encodeURIComponent(key)}/live`); + if (!update || version !== loadVersion) return; + if (update.updatedAt !== sheet.updatedAt || update.status !== sheet.status) { + void load(); // Evidence changed while a log refresh was in flight. + return; + } + const progression = state.progression.get(key); + for (const stack of sheet.stacks) { + const fresh = update.stacks.find(entry => entry.stack === stack.stack); + stack.liveSpend = fresh?.liveSpend ?? undefined; + for (const attempt of stack.attempts) { + const live = fresh?.attempts.find(entry => entry.id === attempt.id); + if (!live) continue; + const { liveCosts, liveSpend, ...fields } = live; + Object.assign(attempt, fields, { liveSpend: liveSpend ?? undefined }); + let track = progression?.stacks.find(entry => entry.attemptId === attempt.id); + if (!track && progression && liveCosts.length) { + track = { stack: stack.stack, attemptId: attempt.id, updatedAt: update.updatedAt, steps: [], costs: [] }; + progression.stacks.push(track); + } + if (track) track.liveCosts = liveCosts.length ? liveCosts : undefined; + } + } + })); + if (version !== loadVersion) return; + if (current.tab === 'transcript') await readTranscript(); + else if (current.attempt && current.tab === 'log') await readLog(current); + return; + } + const setupRequest = current.newRun && !state.setupRequest ? read('/api/run-setup').then(catalog => { + if (catalog && version === loadVersion) { + state.setup = catalog; state.setupRequest ??= initialRun(catalog); state.plansLoaded = true; render(); + } + }) : null; + const plansRequest = current.plans && !current.newRun ? read('/api/plans').then(plans => { + if (plans && version === loadVersion) { + state.plans = plans; + state.plansLoaded = true; + render(); + } + }) : null; + if (!state.csrfToken && (current.key || current.plans)) { + const session = await read<{ canStart: boolean; csrfToken: string }>('/api/session'); + if (version !== loadVersion) return; + if (session) Object.assign(state, session); + } + if (!current.key && !current.plans && (refreshOverview || !state.csrfToken)) { + const overview = await read(`/api/overview?filter=${current.filter}&page=${current.page}`); + if (version !== loadVersion) return; + if (overview) Object.assign(state, { overview: overview.campaigns, overviewLoaded: true, + overviewPage: overview, overviewQuery: `${current.filter}:${current.page}`, + canStart: overview.canStart, csrfToken: overview.csrfToken }); + render(); + } + if (current.plans) { + await Promise.all([plansRequest, setupRequest]); + if (version !== loadVersion) return; + render(); + return; + } + if (!current.key) return; + const result = await read; dispatchError: string | null }>(`/api/campaigns/${encodeURIComponent(current.key)}`); + if (version !== loadVersion) return; + if (result && 'pendingJob' in result) { state.pendingJobs.set(current.key, result); render(); return; } + state.pendingJobs.delete(current.key); + const sheet = result; + if (sheet) state.sheets.set(current.key, sheet); + render(); + if (sheet?.mode === 'dependency') { + const progression = await read( + `/api/campaigns/${encodeURIComponent(current.key)}/progression`); + if (version !== loadVersion) return; + if (progression) state.progression.set(current.key, progression); + render(); + } + if (!current.attempt) return; + const timeBudget = await read>(attemptUrl(current, 'time')); + if (version !== loadVersion) return; + if (timeBudget) { + state.timeBudgets.set(current.attempt, timeBudget); + if (timeBudget.grants.some(grant => grant.request.grantId === state.timeGrantIds.get(current.attempt) + && grant.disposition !== 'pending')) state.timeGrantIds.delete(current.attempt); + } + if (current.tab === 'checks') { + const checks = await read(attemptUrl(current, 'checks')); + if (checks) state.checks.set(current.attempt, checks); + } else if (current.tab === 'screenshots' || current.tab === 'files') { + const evidence = await read(attemptUrl(current, 'package')); + if (evidence) state.evidence.set(current.attempt, evidence); + } else if (current.tab === 'transcript') { + await readTranscript(); + } else if (current.tab === 'log') { + await readLog(current); + } + render(); +} + +function go(href: string): void { + history.pushState(null, '', href); + void load(true); +} + +function stepTo(offset: number): void { + const current = route(); + const progression = state.progression.get(current.key) ?? null; + const sheet = state.sheets.get(current.key); + if (!progression || !sheet) return; + const total = replayTimeline(selectedProgression(progression, sheet)).length; + const next = Math.min(Math.max(0, current.step + offset), Math.max(0, total - 1)); + const url = new URL(location.href); + url.searchParams.set('step', String(next)); + history.replaceState(null, '', `${url.pathname}${url.search}`); + render(); +} + +function subscribe(): void { + if (document.hidden || events) return; + const source = events = new EventSource('/api/events'); + const changed = (event: MessageEvent): void => { + const current = route(); + const message = JSON.parse(event.data) as { key?: string; attemptId?: string }; + const ids = message.attemptId ? [message.attemptId] + : state.sheets.get(message.key ?? '')?.stacks.flatMap(stack => stack.attempts.map(attempt => attempt.id)) ?? []; + for (const id of ids) { + // Keep the visible tab stable until its replacement data arrives. + if (message.key === current.key && id === current.attempt) continue; + state.checks.delete(id); + state.evidence.delete(id); + } + if (current.plans || (current.key && message.key !== current.key)) return; + if (current.attempt && message.attemptId && message.attemptId !== current.attempt) return; + void load(); + }; + source.addEventListener('campaign', changed); + source.addEventListener('reference', async () => { + const current = route(); + if (!current.key && !current.plans && !document.hidden) { + const references = await read>>('/api/reference-runs'); + if (references) { state.references = references; render(); } + } + }); + source.addEventListener('log', event => { + const current = route(); + const message = JSON.parse((event as MessageEvent).data) as { key: string }; + if (!current.plans && (!current.key || message.key === current.key)) void load(false, true); + }); + source.addEventListener('open', () => { + if (fallback) clearInterval(fallback); + fallback = 0; + void load(); + }); + // Recover missed campaign changes while the stream is down. + source.addEventListener('error', () => { + fallback ||= window.setInterval(() => void load(), FALLBACK_MS); + }); +} + +let helpClose = 0; +for (const type of ['pointerover', 'pointerout', 'focusin', 'focusout']) document.addEventListener(type, event => { + if (!(event.target instanceof Element)) return; + const series = event.target.closest('[data-chart-series]'); + if (!series) return; + const lines = [...series.closest('.page')?.querySelectorAll('[data-chart-series]') ?? []]; + const active = (type === 'pointerover' || type === 'focusin') + && lines.some(line => line.dataset.chartSeries === series.dataset.chartSeries); + for (const line of lines) { + line.classList.toggle('is-muted', active && line.dataset.chartSeries !== series.dataset.chartSeries); + line.classList.toggle('is-highlighted', active && line.dataset.chartSeries === series.dataset.chartSeries); + } +}); +for (const type of ['pointerover', 'focusin']) document.addEventListener(type, event => { + if (!(event.target instanceof Element)) return; + if (!event.target.closest('.metric-help, .metric-tooltip')) return; + clearTimeout(helpClose); + const trigger = event.target.closest('.metric-help'); + trigger?.click(); +}); +for (const type of ['pointerout', 'focusout']) document.addEventListener(type, event => { + if (!(event.target instanceof Element) + || !event.target.closest('.metric-help, .metric-tooltip')) return; + clearTimeout(helpClose); + helpClose = window.setTimeout(() => { + if (document.querySelector('.metric-help:hover, .metric-help:focus, .metric-tooltip:hover')) return; + document.querySelector('.metric-tooltip:popover-open')?.hidePopover(); + }, 150); +}); + +document.addEventListener('click', event => { + if (event.defaultPrevented || event.button !== 0 || event.metaKey || event.ctrlKey + || event.shiftKey || event.altKey) return; + const chartToggle = (event.target as Element | null)?.closest('[data-chart-run], [data-chart-stack]'); + if (chartToggle) { + const key = route().key; + const hidden = state.hiddenChartRuns.get(key) ?? new Set(); + const ids = chartToggle.dataset.chartRun !== undefined ? [chartToggle.dataset.chartRun] + : state.sheets.get(key)?.stacks.find(stack => stack.stack === chartToggle.dataset.chartStack) + ?.attempts.map(attempt => attempt.id) ?? []; + const hide = ids.some(id => !hidden.has(id)); + for (const id of ids) { if (hide) hidden.add(id); else hidden.delete(id); } + state.hiddenChartRuns.set(key, hidden); + render(); + return; + } + if ((event.target as Element | null)?.closest('[data-retry]')) { + void load(true); + return; + } + const shot = (event.target as Element | null)?.closest('[data-shot]'); + if (shot) { + const dialog = document.querySelector('.lightbox'); + const image = dialog?.querySelector('img'); + if (!dialog || !image) return; + image.src = shot.dataset.shot ?? ''; + image.alt = shot.dataset.shotName ?? ''; + dialog.showModal(); + return; + } + if (event.target instanceof HTMLDialogElement) event.target.close(); + const link = (event.target as Element | null)?.closest('a'); + const href = link?.getAttribute('href') ?? ''; + if (!href || href === '/checks' || href.startsWith('/api/') || !/^[/?]/.test(href)) return; + event.preventDefault(); + go(href.startsWith('?') ? `${location.pathname}${href}` : href); +}); + +// Controls require the browser token and same origin; the server re-reads the plan. +async function post(form: HTMLFormElement): Promise { + if (submitting) return; + const current = route(); + const data = new FormData(form); + const action = form.dataset.run; + // A response belongs to this form, even if the user leaves before it arrives. + const submittedForm = state.form = { error: '' }; + if (action === 'setup-review' || action === 'setup-start') { + if (action === 'setup-review') state.setupRequest = readRunForm(form, state.setup!); + submitting = true; render(); + try { + const response = await fetch(action === 'setup-review' ? '/api/runs/prepare' : '/api/runs', { + method: 'POST', headers: { 'content-type': 'application/json', 'x-stack-bench-token': state.csrfToken }, + body: JSON.stringify(action === 'setup-review' ? state.setupRequest + : { request: state.setupReview!.request, reviewId: state.setupReview!.reviewId }), + }); + const result = await response.json(); + if (state.form !== submittedForm) return; + if (!response.ok) { + submittedForm.error = result.error ?? `Request failed (${response.status})`; + if (response.status === 403) { state.csrfToken = ''; void load(); } + return; + } + if (action === 'setup-review') state.setupReview = result as RunSetupReview; + else { + state.setupReview = null; state.setupRequest = null; + go(`/c/${encodeURIComponent(result.campaignKey)}`); + } + } catch { submittedForm.error = 'Could not confirm the request. Retry with the same setup; it cannot create a second job.'; } + finally { submitting = false; render(); } + return; + } + + const resumeWithTime = action === 'grant-time' && form.dataset.resume === 'true'; + const existing = action === 'job-start' || action === 'job-cancel' || action === 'resume' || action === 'stop' || action === 'grant-time'; + if (!existing) return; + // Retain the ID after an uncertain response, so retry cannot add time twice. + if (action === 'grant-time' && !state.timeGrantIds.has(current.attempt)) { + state.timeGrantIds.set(current.attempt, crypto.randomUUID()); + } + const grantId = state.timeGrantIds.get(current.attempt); + submitting = true; + render(); + let timeAccepted = false; + try { + let response = await fetch(action === 'job-start' ? `/api/jobs/${current.key.slice(4)}/start` : action === 'job-cancel' ? `/api/jobs/${current.key.slice(4)}/cancel` : action === 'grant-time' ? attemptUrl(current, 'time') : `/api/campaigns/${encodeURIComponent(current.key)}/${action}`, { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-stack-bench-token': state.csrfToken }, + body: JSON.stringify(action === 'grant-time' ? { grantId, minutes: Number(data.get('minutes')) } + : action === 'stop' ? { owner: data.get('owner') } + : {}), + }); + if (response.ok && resumeWithTime) { + timeAccepted = true; + response = await fetch(`/api/campaigns/${encodeURIComponent(current.key)}/resume`, { + method: 'POST', headers: { 'content-type': 'application/json', 'x-stack-bench-token': state.csrfToken }, body: '{}', + }); + } + if (response.ok) { + return void load(); + } + const failure = await response.json().catch(() => ({})) as { error?: string }; + if (response.status === 403) { state.csrfToken = ''; void load(); } + submittedForm.error = + (timeAccepted ? 'Time was added, but resume failed. ' : '') + + (failure.error || `Request failed (HTTP ${response.status}). Check campaign status before retrying.`); + } catch { + submittedForm.error = (timeAccepted ? 'Time was added. Could not confirm resume. ' : 'Could not confirm the request. ') + + 'Check campaign status before retrying.'; + } finally { + submitting = false; + render(); + } +} + +document.addEventListener('submit', event => { + const form = event.target; + if (!(form instanceof HTMLFormElement) || !form.dataset.run) return; + event.preventDefault(); + void post(form); +}); + +// Keep typed settings across background refreshes. +document.addEventListener('input', event => { + const field = event.target as HTMLInputElement; + if (field.form?.dataset.run === 'setup-review') { + state.setupRequest = field.name === 'workload' ? initialRun(state.setup!, field.value) : readRunForm(field.form, state.setup!); + if (field.name === 'workload' || field.name === 'level') render(); + return; + } + + if (field.name === 'minutes') { + state.timeGrantMinutes = field.value; + updateTimeTotal(field); + } + +}); + +document.addEventListener('keydown', event => { + if (route().view !== 'replay') return; + if (event.target instanceof Element + && event.target.closest('input, textarea, select, button, summary, [contenteditable]')) return; + if (event.key === 'ArrowRight') stepTo(1); + else if (event.key === 'ArrowLeft') stepTo(-1); + else if (event.key === ' ') { + event.preventDefault(); + if (playing) { + clearInterval(playing); + playing = 0; + } else playing = window.setInterval(() => stepTo(1), 600); + return; + } else return; + if (playing) { + clearInterval(playing); + playing = 0; + } +}); + +window.setInterval(() => { + for (const clock of document.querySelectorAll('[data-started-at]')) { + clock.textContent = elapsed(clock.dataset.startedAt ?? null, null); + } +}, 1000); +window.addEventListener('popstate', () => void load(true)); +document.addEventListener('visibilitychange', () => { + if (document.hidden) { + // Hidden tabs must not consume the browser's limited HTTP connections. + events?.close(); + events = null; + clearInterval(fallback); + fallback = 0; + } else { + subscribe(); + void load(); + } +}); +subscribe(); +void load(true); + +let transcriptLoading = false; +let transcriptReload = false; +async function readTranscript(force = false): Promise { + const current = route(); + if (document.hidden || current.tab !== 'transcript' || !current.attempt) return; + if (transcriptLoading) { transcriptReload ||= force; return; } + if (state.transcript.attempt !== current.attempt) state.transcript = { + attempt: current.attempt, session: '', before: undefined, page: null }; + const pane = document.querySelector('.transcript'); + if (!force && state.transcript.page && pane + && pane.scrollHeight - pane.scrollTop - pane.clientHeight >= 40) return; + const selected = state.transcript; + transcriptLoading = true; + try { + const query = new URLSearchParams({ session: selected.session }); + if (selected.before !== undefined) query.set('before', String(selected.before)); + const page = await read(attemptUrl(current, `transcript?${query}`)); + if (page && state.transcript === selected) selected.page = page; + } finally { + transcriptLoading = false; + if (transcriptReload) { + transcriptReload = false; + await readTranscript(true); + } + } +} +setInterval(() => { + if (document.hidden) return; + const current = route(); + if (current.plans) return; + if (current.key && state.sheets.get(current.key)?.status === 'running') { + void load(false, true); + } else if (current.tab === 'transcript' && state.transcript.before === undefined) { + void readTranscript().then(render); + } +}, 5000); +document.addEventListener('change', event => { + const target = event.target; + if (target instanceof HTMLSelectElement && target.matches('[data-transcript-session]')) { + state.transcript = { ...state.transcript, session: target.value, before: undefined }; + void readTranscript(true).then(render); + } +}); +document.addEventListener('click', event => { + const target = event.target instanceof Element ? event.target.closest('[data-transcript-before], [data-transcript-latest]') : null; + if (!target) return; + state.transcript = { ...state.transcript, + session: state.transcript.page?.session ?? '', + before: target.hasAttribute('data-transcript-before') ? Number(target.dataset.transcriptBefore) : undefined }; + void readTranscript(true).then(() => { + render(); + const pane = document.querySelector('.transcript'); + if (pane && target.hasAttribute('data-transcript-latest')) pane.scrollTop = pane.scrollHeight; + }); +}); + +document.addEventListener('click', event => { + if (event.target instanceof Element && event.target.closest('[data-setup-edit]')) { + state.setupReview = null; state.form.error = ''; render(); + } +}); + +document.addEventListener('click', event => { + const button = event.target instanceof Element + ? event.target.closest('[data-model-discovery]') : null; + const form = button?.closest('form[data-run="setup-review"]'); + if (!button || !form) return; + const adapter = button.dataset.modelDiscovery!; + const profile = form.querySelector(`select[name="credential-${adapter}"]`)?.value; + const status = form.querySelector(`[data-model-discovery-status="${adapter}"]`); + const list = form.querySelector(`#models-${adapter}`); + if (!status || !list) return; + button.disabled = true; + status.textContent = 'Loading…'; + const query = new URLSearchParams({ adapter, ...(profile ? { profile } : {}) }); + void fetch(`/api/run-setup/models?${query}`, { headers: { 'x-stack-bench-token': state.csrfToken }, + signal: AbortSignal.timeout(30_000) }) + .then(async response => { + const result = await response.json() as { models?: Array<{ id: string }>; error?: string }; + if (!response.ok || !result.models) throw new Error(result.error ?? 'Model list is unavailable'); + list.replaceChildren(...result.models.map(model => { + const option = document.createElement('option'); option.value = model.id; return option; + })); + status.textContent = `${result.models.length} models listed. Select one, then record its pricing.`; + }).catch(error => { status.textContent = error instanceof Error ? error.message : 'Model list is unavailable'; }) + .finally(() => { button.disabled = false; }); +}); diff --git a/tools/stack-bench/dashboard/public/check-guide.ts b/tools/stack-bench/dashboard/public/check-guide.ts new file mode 100644 index 00000000000..3f721b9e5e7 --- /dev/null +++ b/tools/stack-bench/dashboard/public/check-guide.ts @@ -0,0 +1,31 @@ +/// +/// +export {}; +const search = document.querySelector('[data-guide-search]')!; +const others = document.querySelector('[data-guide-others]')!; +const checks = [...document.querySelectorAll('.guide-check')]; +const url = new URL(location.href); +search.value = url.searchParams.get('q') ?? ''; +others.checked = url.searchParams.get('scope') === 'all'; +function filter(): void { + let shown = 0; + for (const check of checks) { + check.hidden = (!others.checked && check.dataset.active !== 'true') + || !(check.dataset.search ?? '').includes(search.value.trim().toLowerCase()); + if (!check.hidden) shown++; + } + document.querySelector('[data-guide-count]')!.textContent = `${shown} ${shown === 1 ? 'check' : 'checks'} shown`; + document.querySelector('[data-guide-empty]')!.hidden = shown > 0; + if (search.value) url.searchParams.set('q', search.value); else url.searchParams.delete('q'); + if (others.checked) url.searchParams.set('scope', 'all'); else url.searchParams.delete('scope'); + history.replaceState(null, '', url); +} +search.addEventListener('input', filter); +others.addEventListener('change', filter); +const toggle = document.querySelector('[data-guide-toggle]')!; +toggle.addEventListener('click', () => { + const expand = toggle.textContent === 'Expand visible'; + for (const check of checks) check.open = expand && !check.hidden; + toggle.textContent = expand ? 'Collapse all' : 'Expand visible'; +}); +filter(); diff --git a/tools/stack-bench/dashboard/public/climb.ts b/tools/stack-bench/dashboard/public/climb.ts new file mode 100644 index 00000000000..4c1d7d72741 --- /dev/null +++ b/tools/stack-bench/dashboard/public/climb.ts @@ -0,0 +1,77 @@ +// The climb: one point per completed grade, unaided grades ringed, the current +// grade filled. Small in a lane or a sheet cell, large on the attempt page. + +import type { ClimbPoint } from '../dashboard-views.js'; +import { esc } from './format.js'; + +interface Plot { + x: number; + y: number; + point: ClimbPoint; +} + +function pointTitle(point: ClimbPoint): string { + return `${point.score} / ${point.max} points${point.unaided ? ' · First build at this level; earlier fixes and feedback retained' : ''}`; +} + +function plot(series: readonly ClimbPoint[], left: number, right: number, + top: number, bottom: number): Plot[] { + const span = Math.max(1, series.length - 1); + return series.map((point, index) => ({ + x: series.length === 1 ? (left + right) / 2 : left + (right - left) * index / span, + y: bottom - (bottom - top) * (point.max ? point.score / point.max : 0), + point, + })); +} + +function stepPath(plots: readonly Plot[]): string { + const head = plots[0]; + if (!head) return ''; + return plots.slice(1).reduce((path, item, index) => + `${path} L${item.x} ${plots[index]!.y} L${item.x} ${item.y}`, `M${head.x} ${head.y}`); +} + +// Full size: the same points with a band per depth or level, and a number at +// the first, the best and the current grade. +export function bigClimb(series: readonly ClimbPoint[], stage: (level: number) => string): string { + if (!series.length) return '

Awaiting first grade. The score history will appear here.

'; + const top = 10; + const bottom = 130; + const plots = plot(series, 100, 1010, top, bottom); + const bands: string[] = []; + let start = 0; + plots.forEach((item, index) => { + const next = plots[index + 1]; + if (next && next.point.level === item.point.level) return; + const level = item.point.level; + if (level !== null) { + const from = Math.max(60, plots[start]!.x - 40); + const width = Math.min(1050, item.x + 40) - from; + bands.push(`` + + `${esc(stage(level))}`); + } + start = index + 1; + }); + const line = stepPath(plots); + const first = plots[0]!; + const last = plots.at(-1)!; + const best = plots.reduce((top1, item) => item.y < top1.y ? item : top1, first); + const label = (item: Plot, tone: string): string => + `` + + `${Math.round(item.point.max ? 100 * item.point.score / item.point.max : 0)}`; + return `
Weighted score by completed grade. Each grade can cover a different scope.${bands.join('')}` + + [0, 50, 100].map(value => { + const y = bottom - (bottom - top) * value / 100; + return `` + + `${value}%`; + }).join('') + + `` + + `` + + plots.map(item => `${pointTitle(item.point)}`).join('') + + label(first, '#b6c0cf') + (best === first || best === last ? '' : label(best, '#b6c0cf')) + + (last === first ? '' : label(last, '#e6e9f0')) + '
'; +} diff --git a/tools/stack-bench/dashboard/public/fonts/inter-LICENSE.txt b/tools/stack-bench/dashboard/public/fonts/inter-LICENSE.txt new file mode 100644 index 00000000000..40589daa9de --- /dev/null +++ b/tools/stack-bench/dashboard/public/fonts/inter-LICENSE.txt @@ -0,0 +1,93 @@ +Copyright 2016 The Inter Project Authors (https://github.com/rsms/inter) Inter-Italic[opsz,wght].ttf: Copyright 2016 The Inter Project Authors (https://github.com/rsms/inter) + +This Font Software is licensed under the SIL Open Font License, Version 1.1. +This license is copied below, and is also available with a FAQ at: +http://scripts.sil.org/OFL + + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting -- in part or in whole -- any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/tools/stack-bench/dashboard/public/fonts/inter-latin-variable.woff2 b/tools/stack-bench/dashboard/public/fonts/inter-latin-variable.woff2 new file mode 100644 index 00000000000..d15208de03c Binary files /dev/null and b/tools/stack-bench/dashboard/public/fonts/inter-latin-variable.woff2 differ diff --git a/tools/stack-bench/dashboard/public/fonts/source-code-pro-LICENSE.txt b/tools/stack-bench/dashboard/public/fonts/source-code-pro-LICENSE.txt new file mode 100644 index 00000000000..046fc664900 --- /dev/null +++ b/tools/stack-bench/dashboard/public/fonts/source-code-pro-LICENSE.txt @@ -0,0 +1,93 @@ +Google Inc. + +This Font Software is licensed under the SIL Open Font License, Version 1.1. +This license is copied below, and is also available with a FAQ at: +http://scripts.sil.org/OFL + + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting -- in part or in whole -- any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/tools/stack-bench/dashboard/public/fonts/source-code-pro-latin-variable.woff2 b/tools/stack-bench/dashboard/public/fonts/source-code-pro-latin-variable.woff2 new file mode 100644 index 00000000000..bc303f50c5d Binary files /dev/null and b/tools/stack-bench/dashboard/public/fonts/source-code-pro-latin-variable.woff2 differ diff --git a/tools/stack-bench/dashboard/public/format.ts b/tools/stack-bench/dashboard/public/format.ts new file mode 100644 index 00000000000..6b9e27a33a7 --- /dev/null +++ b/tools/stack-bench/dashboard/public/format.ts @@ -0,0 +1,123 @@ +// One spelling per value. Every figure the dashboard prints goes through here, +// so a percentage, a duration and a dash look the same on every page. + +import type { SheetAttempt } from '../dashboard-views.js'; +import type { CostEvidence } from '../../src/evidence/cost-proof.js'; +import { statusWord } from '../../src/evidence/status-words.js'; +import { outputSilentMinutes } from './metrics.js'; + +export { statusWord }; + +const SILENCE_MINUTES = 10; + +export const STACK_LABEL: Record = { spacetime: 'SpacetimeDB', + postgres: 'PostgreSQL', mongodb: 'MongoDB', convex: 'Convex', supabase: 'Supabase' }; +export const DASH = '—'; + +export function esc(value: unknown): string { + return String(value ?? '').replace(/[&<>"']/g, character => + ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' })[character] ?? character); +} + +export function stackLabel(stack: string): string { + return STACK_LABEL[stack] ?? stack; +} + +export function metricLabel(label: string, description: string | undefined): string { + if (!description) return `${esc(label)}`; + const id = `help-${label.toLowerCase().replaceAll(' ', '-')}`; + return `` + + ``; +} + +export function pct(value: number | null | undefined): string { + return value == null ? DASH : `${Math.round(value)}%`; +} + +export function num(value: number | null | undefined): string { + return value == null ? DASH : String(Math.round(value)); +} + +// One value: the count and the total it is out of. +export function ratio(used: number | null | undefined, budget: number | null | undefined): string { + if (used == null) return DASH; + return budget == null ? String(used) : `${used} / ${budget}`; +} + +export function money(value: number | null | undefined): string { + if (value == null) return DASH; + return `$${value.toFixed(2)}`; +} + +export function spend(value: CostEvidence & { knownCostUsd?: number }, pending = false, liveSpend?: number): string { + return (liveSpend !== undefined + ? `~${money(liveSpend)}` + : value.status === 'unknown' ? value.lowerBoundUsd !== undefined ? `≥${money(value.lowerBoundUsd)}` : value.knownCostUsd + ? `${money(value.knownCostUsd)} recorded` : 'Unknown' + : `${value.status === 'upper-bound' ? '≤' : ''}${money(value.costUsd)}`) + + (pending ? ' ' : ''); +} + +export function duration(seconds: number | null | undefined): string { + if (seconds == null) return DASH; + const minutes = Math.round(seconds / 60); + return minutes < 60 ? `${minutes}m` : `${Math.floor(minutes / 60)}h ${minutes % 60}m`; +} + +// Wall time for the current execution, separate from the measured run duration. +export function elapsed(startedAt: string | null, completedAt: string | null, + now = Date.now()): string { + if (startedAt === null) return DASH; + const start = Date.parse(startedAt); + const end = completedAt === null ? now : Date.parse(completedAt); + if (!Number.isFinite(start) || !Number.isFinite(end)) return DASH; + const seconds = Math.floor(Math.max(0, end - start) / 1000); + const minutes = Math.floor(seconds / 60); + return `${minutes >= 60 ? `${Math.floor(minutes / 60)}h ` : ''}${minutes % 60}m ${seconds % 60}s`; +} + +export function executionClock(startedAt: string | null, completedAt: string | null): string { + return `${elapsed(startedAt, completedAt)}`; +} + +export function since(value: string | null | undefined, now = Date.now()): string { + if (!value) return DASH; + const minutes = Math.max(0, Math.floor((now - Date.parse(value)) / 60000)); + if (minutes < 60) return `${minutes}m`; + if (minutes < 60 * 48) return `${Math.floor(minutes / 60)}h`; + return `${Math.floor(minutes / 1440)}d`; +} + +export function phrase(attempt: SheetAttempt, now = Date.now()): string { + const parts = [attempt.phase]; + const silent = outputSilentMinutes(attempt, now); + if (silent >= SILENCE_MINUTES) parts.push(`no agent activity observed for ${silent}m`); + return parts.join(' · '); +} + +// depth 3 · 1× / L1–L3 · 3× +export function shape(mode: string, levels: readonly number[], repetitions: number): string { + const depth = levels.length ? Math.max(...levels) : 0; + const span = mode === 'dependency' ? `depth ${depth}` + : levels.length > 1 ? `L${Math.min(...levels)}–L${depth}` : `L${depth}`; + return `${span} · ${repetitions}×`; +} + +export function modelLabel(model?: string): string { + return ({ 'claude-sonnet-5': 'Sonnet 5', 'claude-fable-5-1': 'Fable 5.1', 'claude-opus-5': 'Opus 5', 'gpt-5.6-sol': 'Sol', 'gpt-6-astra': 'Astra' } as Record)[model ?? ''] ?? model ?? ''; +} + +// A finished attempt that does not count shows why instead of its completion. +export function excludedLabel(attempt: Pick): string | null { + if (!attempt.excluded || attempt.status === 'running' || attempt.status === 'pending') return null; + return attempt.status === 'completed' ? 'Excluded' : 'Incomplete'; +} + +export function completionLabel(attempt: Pick, + completion: Pick, 'passed' | 'selected'> | null = attempt.completion): string { + return excludedLabel(attempt) ?? (completion ? ratio(completion.passed, completion.selected) : DASH); +} + +export function runLabel(attempt: Pick, showRepetition = true): string { + return `${modelLabel(attempt.model)}${attempt.effort ? ` (${attempt.effort})` : ''}${showRepetition ? ` · Rep ${attempt.repetition}` : ''}`; +} diff --git a/tools/stack-bench/dashboard/public/graph.ts b/tools/stack-bench/dashboard/public/graph.ts new file mode 100644 index 00000000000..1c130c4a51a --- /dev/null +++ b/tools/stack-bench/dashboard/public/graph.ts @@ -0,0 +1,95 @@ +// One graph for the campaign: columns are depth, bands are questlines, edges +// are the catalog's own dependencies. Every stack builds the same catalog, so a +// node carries one dot per stack in fixed order. The renderer takes one +// node-status snapshot per stack, which is what the replay feeds it per step. + +import type { CampaignProgression } from '../dashboard-views.js'; +import { esc, stackLabel, statusWord } from './format.js'; + +export interface GraphStack { + stack: string; + statuses: readonly string[]; +} + +const DOT: Record = { passed: 'p', active: 'a', working: 'a', failed: 'f', + blocked: 'b', locked: 'o' }; +const DOT_START = 180; +const DOT_SPACING = 14; +const ROW = 30; + +interface Placed { + x: number; + y: number; + index: number; +} + +export function graph(view: CampaignProgression, stacks: readonly GraphStack[]): string { + if (!view.nodes.length) return '

No feature graph is available.

'; + const depths = view.depths; + const nodeWidth = DOT_START + Math.max(1, stacks.length) * DOT_SPACING; + const columnWidth = nodeWidth + 60; + const width = 150 + Math.max(1, depths.length) * columnWidth - 40; + const placed = new Map(); + const bands: string[] = []; + let top = 20; + for (const questline of view.questlines) { + const nodes = view.nodes.filter(node => node.questline === questline.id); + if (!nodes.length) continue; + const used = new Map(); + let rows = 0; + for (const node of nodes) { + const row = used.get(node.depth) ?? 0; + used.set(node.depth, row + 1); + rows = Math.max(rows, row + 1); + placed.set(node.id, { x: 150 + Math.max(0, depths.indexOf(node.depth)) * columnWidth, + y: top + 8 + row * ROW, index: view.nodes.indexOf(node) }); + } + const height = rows * ROW + 16; + bands.push(`${esc(questline.title.length > 19 ? `${questline.title.slice(0, 18)}…` : questline.title)}${esc(questline.title)}`); + top += height; + bands.push(``); + } + const failed = (index: number): boolean => + stacks.some(entry => entry.statuses[index] === 'failed'); + const blocked = (index: number): boolean => + stacks.some(entry => entry.statuses[index] === 'blocked'); + const edges = view.nodes.flatMap(node => { + const target = placed.get(node.id); + if (!target) return []; + return node.dependencies.flatMap(id => { + const source = placed.get(id); + if (!source) return []; + const cut = failed(source.index) || blocked(target.index); + return [``]; + }); + }); + const nodes = view.nodes.map(node => { + const at = placed.get(node.id); + if (!at) return ''; + const dots = stacks.map((entry, column) => + `` + + (entry.statuses[at.index] === 'passed' + ? `` : '')).join(''); + const hover = stacks.map(entry => + `${stackLabel(entry.stack)} ${statusWord(entry.statuses[at.index] ?? 'locked')}`).join(' · '); + return `${esc(`${node.title} · ${hover}`)}` + + `` + + `${esc(node.title.length > 22 + ? `${node.title.slice(0, 21)}…` : node.title)}${dots}`; + }); + const columns = depths.map((depth, index) => + `depth ${depth}`).join(''); + const order = stacks.map(entry => stackLabel(entry.stack)).join(', '); + const description = view.nodes.map((node, index) => `${node.title}: ${stacks.map(entry => + `${stackLabel(entry.stack)} ${statusWord(entry.statuses[index] ?? 'locked')}`).join(', ')}`).join('. '); + const key = [['p', 'Passed'], ['a', 'Active'], ['f', 'Failed'], ['b', 'Blocked'], ['o', 'Locked']] + .map(([tone, label]) => `${label}`).join(''); + return `
${stacks.length > 1 ? `

Feature dots, left to right: ${esc(order)}.

` : ''}${key}
` + + `
` + + `` + + `Feature dependencies and stack status${esc(description)}` + + `${bands.join('')}${columns}${edges.join('')}${nodes.join('')}
`; +} diff --git a/tools/stack-bench/dashboard/public/index.html b/tools/stack-bench/dashboard/public/index.html new file mode 100644 index 00000000000..e94a45b9a35 --- /dev/null +++ b/tools/stack-bench/dashboard/public/index.html @@ -0,0 +1,15 @@ + + + + + + + + Stack Bench + + + + + +
Loading Stack Bench…
+ diff --git a/tools/stack-bench/dashboard/public/metrics.ts b/tools/stack-bench/dashboard/public/metrics.ts new file mode 100644 index 00000000000..8804322b895 --- /dev/null +++ b/tools/stack-bench/dashboard/public/metrics.ts @@ -0,0 +1,203 @@ +import type { CampaignRunLevelResult, CampaignRunResult, DependencyProgress } + from '../../src/campaigns/campaign-inspection.js'; +import type { CostEvidence } from '../../src/evidence/cost-proof.js'; +import type { CheckCompletion } from '../../src/evidence/check-completion.js'; + +// The dashboard's vocabulary in one place: First builds, Score, Repairs, Regressions, +// Stalling and Excluded are defined here and nowhere else, so the server-rendered +// sheet and the browser read the same numbers from the same evidence. + +const EXCLUDED_OUTCOMES = new Set(['harness_failure', 'inconclusive', 'ungraded', 'contaminated']); +const SILENCE_MINUTES = 10; + +export interface MetricExecution { + outcome: string | null; + reason: string | null; +} + +export interface MetricAttempt { + id: string; + stack: string; + status: string; + repetition?: number; + logUpdatedAt?: string | null; + activityUpdatedAt?: string | null; + paused?: boolean; + execution: MetricExecution | null; + result: CampaignRunResult | null; + dependency: DependencyProgress | null; + spend?: CostEvidence; + measuredCost?: CostEvidence; + measuredDurationSec?: number | null; + completion?: CheckCompletion | null; + comparisonKey?: string; +} + +export interface AttemptMetrics { + first: number | null; + final: number; + repairs: number; + spend: number | null; + duration: number | null; + scope: string; + abortedFirst: number; + raw: { + first: { score: number; max: number } | null; + final: { score: number; max: number } | null; + }; +} + +export interface ComparisonEntry { + stack: string; + runs: Array<{ attempt: Attempt; metrics: AttemptMetrics }>; + excluded: Array<{ attempt: Attempt; reason: string }>; + pending: number; + spendSoFar: number | null; + abortedFirst: number; +} + +export type ComparisonRow = ComparisonEntry & { + n: number; scopes: string[]; first: number | null; final: number | null; + repairs: number | null; spend: number | null; duration: number | null; + costPerValidRun: number | null; + firstRange: { min: number; max: number } | null; + spendRange: { min: number; max: number } | null; + durationRange: { min: number; max: number } | null; +}; + +export function median(values: readonly number[]): number | null { + if (!values.length) return null; + const sorted = [...values].sort((left, right) => left - right); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 ? sorted[middle]! : (sorted[middle - 1]! + sorted[middle]!) / 2; +} + +export function attemptSpend(attempt: MetricAttempt): number | null { + return attempt.spend?.status === 'exact' ? attempt.spend.costUsd : null; +} + +// An ungraded first build has no score; it is not a zero. +export function attemptMetrics(attempt: MetricAttempt): AttemptMetrics | null { + const run = attempt.result; + if (!run || run.unreadable) return null; + const dependency = attempt.dependency; + if (dependency) { + const score = dependency.score; + const unique = score?.uniqueChecks; + if (score?.status !== 'final' || unique?.percentage == null) return null; + const available = unique.availablePoints ?? 0; + return { + first: run.firstBuildRate ?? null, + // Final completion counts each selected graph point once. First builds + // instead sum the cumulative scored scope at each depth. + final: unique.percentage / 100, + repairs: dependency.history?.repairAttempts ?? 0, + spend: attempt.measuredCost?.status === 'exact' ? attempt.measuredCost.costUsd : null, + duration: attempt.measuredDurationSec ?? null, + scope: `${attempt.comparisonKey ?? ''}:dependency:${dependency.nodes.length}:${available}`, + abortedFirst: 0, + raw: { first: null, final: unique.passedPoints == null + ? null : { score: unique.passedPoints, max: available } }, + }; + } + type FinalLevel = CampaignRunLevelResult & { finalScore: { score: number; max: number } }; + type ScoredLevel = FinalLevel & { firstScore: { score: number; max: number } }; + const levels = (run.levels ?? []) + .filter((level): level is FinalLevel => level.finalScore !== null); + if (!levels.length) return null; + const sum = (list: readonly Level[], + pick: (level: Level) => number): number => list.reduce((total, item) => total + pick(item), 0); + const scored = levels.filter((level): level is ScoredLevel => + level.firstScore !== null && level.firstAbort === null); + const abortedFirst = levels.filter(level => level.firstAbort).length; + const firstMax = sum(scored, level => level.firstScore.max); + const finalMax = sum(levels, level => level.finalScore.max) + (run.unreachedPoints ?? 0); + return { + first: run.firstBuildRate ?? null, + final: sum(levels, level => level.finalScore.score) / finalMax, + repairs: sum(levels, level => level.used ?? 0), + spend: attempt.measuredCost?.status === 'exact' ? attempt.measuredCost.costUsd : null, + duration: attempt.measuredDurationSec ?? null, + scope: `${attempt.comparisonKey ?? ''}:sequential`, + abortedFirst, + // Do not show a partial first-build sum when the complete rate is unknown. + raw: { first: run.firstBuildRate != null && firstMax ? { score: sum(scored, l => l.firstScore.score), max: firstMax + (run.unreachedPoints ?? 0) } : null, + final: { score: sum(levels, l => l.finalScore.score), max: finalMax } }, + }; +} + +export function attemptExcluded(attempt: MetricAttempt): string | null { + const outcome = attempt.execution?.outcome ?? attempt.result?.outcome; + if (attempt.status === 'invalid') return attempt.execution?.reason ?? outcome ?? 'excluded'; + if (attempt.result?.unreadable && attempt.status !== 'running') return `Result validation failed: ${attempt.result.unreadable}`; + // 'ungraded' on an attempt still running means "not yet", not "thrown out". + if (outcome && EXCLUDED_OUTCOMES.has(outcome) && attempt.status === 'completed') return outcome; + return null; +} + +// Compare results only when they share the same recorded test plan. +export function compareCampaign(campaign: { + attempts?: readonly Attempt[]; +}): { rows: Array>; usable: Array>; + priced: Array>; burn: Map; + mixedScope: boolean; comparable: boolean } { + const byStack = new Map>(); + const unknownSpend = new Set(); + for (const attempt of campaign.attempts ?? []) { + const entry = byStack.get(attempt.stack) + ?? { stack: attempt.stack, runs: [], excluded: [], pending: 0, spendSoFar: null, abortedFirst: 0 }; + byStack.set(attempt.stack, entry); + // Excluded attempts still contribute to actual spend. + const incurred = attemptSpend(attempt); + if (incurred === null) unknownSpend.add(attempt.stack); + else entry.spendSoFar = (entry.spendSoFar ?? 0) + incurred; + const reason = attemptExcluded(attempt); + if (reason) { entry.excluded.push({ attempt, reason }); continue; } + const metrics = attempt.status === 'completed' ? attemptMetrics(attempt) : null; + if (metrics) { + entry.runs.push({ attempt, metrics }); + entry.abortedFirst += metrics.abortedFirst; + } else entry.pending += 1; + } + const rows = [...byStack.values()] + .map(entry => { + const pick = (key: 'first' | 'final' | 'repairs' | 'spend' | 'duration'): number[] => + entry.runs.map(run => run.metrics[key]).filter((value): value is number => value !== null); + const range = (values: readonly number[]): { min: number; max: number } | null => + values.length ? { min: Math.min(...values), max: Math.max(...values) } : null; + const spend = pick('spend'); + const duration = pick('duration'); + const first = pick('first'); + const scopes = [...new Set(entry.runs.map(run => run.metrics.scope))].sort(); + return { ...entry, spendSoFar: unknownSpend.has(entry.stack) ? null : entry.spendSoFar, + n: entry.runs.length, scopes, + first: scopes.length === 1 ? median(first) : null, firstRange: range(first), + final: scopes.length === 1 ? median(pick('final')) : null, + repairs: scopes.length === 1 ? median(pick('repairs')) : null, + spend: scopes.length === 1 ? median(spend) : null, spendRange: scopes.length === 1 ? range(spend) : null, + costPerValidRun: scopes.length === 1 && spend.length > 0 && spend.length === entry.runs.length + ? spend.reduce((total, cost) => total + cost, 0) / spend.length : null, + duration: scopes.length === 1 ? median(duration) : null, + durationRange: scopes.length === 1 ? range(duration) : null }; + }); + const usable = rows.filter(row => row.n > 0); + const scopes = new Set(usable.flatMap(row => row.scopes)); + const priced = usable.filter(row => row.spend != null); + return { rows, usable, priced, + burn: new Map(rows.map(entry => [entry.stack, entry.spendSoFar])), + mixedScope: scopes.size > 1, + comparable: priced.length > 1 && scopes.size === 1 }; +} + +export function outputSilentMinutes(attempt: Pick, now = Date.now()): number { + if (attempt.status !== 'running' || attempt.paused || !attempt.activityUpdatedAt) return 0; + const updated = Date.parse(attempt.activityUpdatedAt); + return Number.isFinite(updated) ? Math.max(0, Math.floor((now - updated) / 60000)) : 0; +} + +// Flag observed agent inactivity, never infer it from controller output or scores. +export function attemptStalling(attempt: Pick, + now = Date.now()): boolean { + return outputSilentMinutes(attempt, now) >= SILENCE_MINUTES; +} diff --git a/tools/stack-bench/dashboard/public/progress-chart.ts b/tools/stack-bench/dashboard/public/progress-chart.ts new file mode 100644 index 00000000000..161879f29c9 --- /dev/null +++ b/tools/stack-bench/dashboard/public/progress-chart.ts @@ -0,0 +1,136 @@ +import type { CampaignProgression, CampaignSheet } from '../dashboard-views.js'; +import { duration, esc, runLabel, stackLabel } from './format.js'; + +export function progressChart(sheet: CampaignSheet, progression: CampaignProgression | null, + metric: 'completion' | 'cost' | 'distribution' = 'completion', view = 'grid', hidden: ReadonlySet = new Set(), unit: 'checks' | 'features' = 'features'): string { + const tracks = metric === 'distribution' ? sheet.stacks.flatMap(stack => stack.attempts.flatMap(attempt => { + const rate = unit === 'features' ? attempt.featureCompletion?.rate : attempt.completion?.rate; + return attempt.status === 'completed' && !attempt.excluded && rate != null && Number.isFinite(rate) + ? [{ stack: stack.stack, attempt, points: [{ elapsed: 0, value: rate * 100, upper: false }] }] : []; + })) : sheet.stacks.flatMap(stack => stack.attempts.map(attempt => + progression?.stacks.find(track => track.attemptId === attempt.id) + ?? { stack: stack.stack, attemptId: attempt.id, steps: [], costs: [], liveCosts: undefined })).flatMap(track => { + const attempt = sheet.stacks.find(stack => stack.stack === track.stack)?.attempts + .find(candidate => candidate.id === track.attemptId); + const start = Date.parse(attempt?.executionStartedAt ?? ''); + if (!attempt || !Number.isFinite(start)) return []; + const observations = (metric === 'cost' ? (track.liveCosts?.map(point => ({ + completedAt: point.completedAt, value: point.costUsd, upper: false, + })) ?? (track.costs ?? []).map(point => ({ + completedAt: point.completedAt, value: point.cost.costUsd, upper: point.cost.status === 'upper-bound', + }))) : track.steps.map(step => ({ completedAt: step.completedAt, + value: (unit === 'features' ? step.featureCompletion : step.completion) == null ? null + : (unit === 'features' ? step.featureCompletion! : step.completion!) * 100, upper: false }))).flatMap(step => { + const elapsed = (Date.parse(step.completedAt ?? '') - start) / 1000; + return Number.isFinite(elapsed) && elapsed >= 0 && step.value != null + && Number.isFinite(step.value) && step.value >= 0 + ? [{ elapsed, value: step.value, upper: step.upper }] : []; + }).sort((a, b) => a.elapsed - b.elapsed); + if (metric === 'cost' && attempt.executionCost) { + const liveTotal = track.liveCosts?.at(-1)?.costUsd; + const total = liveTotal ?? attempt.executionCost.costUsd; + const end = Date.parse(attempt.executionCompletedAt ?? attempt.activityUpdatedAt ?? attempt.logUpdatedAt ?? ''); + const elapsed = (end - start) / 1000; + if (total != null && Number.isFinite(total) && Number.isFinite(elapsed) && elapsed >= 0) { + // The sheet owns the total; histories can arrive in a different refresh. + while (observations.length && observations.at(-1)!.elapsed >= elapsed) observations.pop(); + observations.push({ elapsed, value: total, upper: liveTotal === undefined && attempt.executionCost.status === 'upper-bound' }); + } + } + const points = [{ elapsed: 0, value: 0, upper: false }, ...observations]; + return observations.length ? [{ stack: track.stack, attempt, points }] : []; + }); + const unitDescription = unit === 'features' + ? 'Features fully passed out of all selected features. A feature passes only when all its selected checks pass, including production guarantees.' + : 'Accepted checks passed out of all selected checks.'; + const label = metric === 'distribution' ? 'Completion distribution' : metric === 'cost' ? 'Cost' : 'Completion'; + const description = metric === 'distribution' + ? `One point per eligible completed run, grouped by stack. ${unitDescription} Running and excluded runs are not plotted.` + : metric === 'cost' + ? 'Current-execution cost, including repairs and excluded runs. Earlier executions remain in Total spend; Cost per valid run also includes explicit resume history. Live estimates use reported response usage; final receipts replace estimates. Other runs show saved grade checkpoints. Subscription costs use the pinned API-equivalent price snapshot, not invoice charges. Unknown costs are not plotted; upper bounds are labelled. Time starts at the current execution. Lines connect observations; intermediate values are not measured.' + : `${unitDescription} Each point is a saved grade. Zero marks run start. Each line is one repetition; elapsed time starts at that run. Excluded runs are labelled. Lines can fall after regressions. Intermediate values are not measured.`; + const heading = `

${label}${metric === 'distribution' ? '' : ' over time'}

' + + '' + + '
'; + const valueLabel = (value: number, upper = false, decimals = 1) => metric === 'cost' + ? `${upper ? '≤' : ''}$${value.toFixed(2)}` : `${value.toFixed(decimals)}%`; + const brandColors: Record = { spacetime: '#4cf490', mongodb: '#b45af2', postgres: '#336791', + supabase: '#1f9d6a' }; + const color = (stack: string) => brandColors[stack] + ?? `hsl(${Array.from(stack).reduce((hash, char) => (hash * 31 + char.charCodeAt(0)) % 360, 0)},65%,65%)`; + const marker = (repetition: number, x: number, y: number, title = '') => { + const shape = (repetition - 1) % 3; + return shape === 1 ? `${title}` + : shape === 2 ? `${title}` + : `${title}`; + }; + const controls = `
` + sheet.stacks.map(stack => { + const shown = stack.attempts.filter(attempt => !hidden.has(attempt.id)).length; + return `
` + + `` + + '
' + stack.attempts.map(attempt => { + const point = tracks.find(track => track.attempt.id === attempt.id)?.points.at(-1); + const label = runLabel(attempt, sheet.repetitions > 1) + + (point ? ` · ${metric === 'cost' && attempt.liveSpend !== undefined ? '~' : ''}${valueLabel(point.value, point.upper, 0)}` : ''); + const status = attempt.excluded || attempt.status === 'invalid' ? ' · Excluded' : ''; + return ``; + }).join('') + '
'; + }).join('') + '
'; + const visible = tracks.filter(track => !hidden.has(track.attempt.id)); + if (metric === 'distribution') { + const axisLeft = 150; + const position = (value: number) => axisLeft + (910 - axisLeft) * value / 100; + const rowHeight = Math.max(70, ...sheet.stacks.map(stack => stack.attempts.length * 20 + 24)); + const bottom = sheet.stacks.length * rowHeight + 24; + const ticks = [0, 25, 50, 75, 100].map(value => + `` + + `${value}%`).join(''); + const rows = sheet.stacks.map((stack, index) => { + const center = 24 + rowHeight * (index + 0.5); + const points = visible.filter(track => track.stack === stack.stack); + return `${esc(stackLabel(stack.stack))}` + + points.map(track => { + const value = track.points[0]!.value; + const at = center + (stack.attempts.findIndex(a => a.id === track.attempt.id) - (stack.attempts.length - 1) / 2) * 20; + const label = `${stackLabel(stack.stack)} · ${runLabel(track.attempt)}: ${valueLabel(value)}${track.attempt.excluded ? ' · Excluded' : ''}`; + return `` + + `${esc(label)}` + + marker(track.attempt.repetition, position(value), at) + + `${esc(runLabel(track.attempt))}` + + `${valueLabel(value, false, 0)}`; + }).join(''); + }).join(''); + return `
${heading}${controls}
` + + `${description}${ticks}${rows}
`; + } + if (!visible.length) return `
${heading}${controls}

${tracks.length > 0 && tracks.every(track => hidden.has(track.attempt.id)) ? 'Select a run to show its progress.' : metric === 'cost' ? 'Awaiting first timed cost receipt.' : 'Awaiting first timed grade.'}

`; + const ceiling = metric === 'cost' ? Math.max(0.01, ...tracks.flatMap(track => track.points.map(point => point.value))) : 100; + const maximum = Math.max(60, ...tracks.flatMap(track => track.points.map(point => point.elapsed))); + const left = metric === 'cost' ? 80 : 48; + const x = (seconds: number) => left + (948 - left) * seconds / maximum; + const y = (value: number) => 190 - 160 * value / ceiling; + const grid = [0, 0.25, 0.5, 0.75, 1].map(part => part * ceiling).map(value => + `${valueLabel(value, false, 0)}`).join(''); + const ticks = [0, 0.25, 0.5, 0.75, 1].map(part => + `${esc(part ? duration(part * maximum) : '0')}`).join(''); + const lines = visible.map(({ stack, attempt, points }) => { + // Saved observations are not continuous measurements. Stop at the last receipt. + let path = ''; + const marks = points.map((point, index) => { + path += index ? ` L${x(point.elapsed)} ${y(point.value)}` : `M${x(point.elapsed)} ${y(point.value)}`; + return marker(attempt.repetition, x(point.elapsed), y(point.value), `${esc(stackLabel(stack))} · ${esc(runLabel(attempt))}: ${valueLabel(point.value, point.upper)} at ${esc(duration(point.elapsed))}${index === 0 ? (metric === 'cost' ? ' · Run start; no recorded cost' : ' · Run start; no checks graded') : ''}${attempt.excluded ? ' · Excluded' : ''}`); + }).join(''); + return `${marks}`; + }).join(''); + return `
${heading}${controls}
` + + `${description}${grid}${ticks}${lines}Elapsed run time
` + + '
'; +} diff --git a/tools/stack-bench/dashboard/public/spacetimedb-mark.svg b/tools/stack-bench/dashboard/public/spacetimedb-mark.svg new file mode 100644 index 00000000000..f7957efa1ed --- /dev/null +++ b/tools/stack-bench/dashboard/public/spacetimedb-mark.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/tools/stack-bench/dashboard/public/styles.css b/tools/stack-bench/dashboard/public/styles.css new file mode 100644 index 00000000000..1689b3ac2f9 --- /dev/null +++ b/tools/stack-bench/dashboard/public/styles.css @@ -0,0 +1,402 @@ +/* Stack Bench dashboard: compact tables, sentence-case labels, and consistent spacing. + * Brand colors identify stacks in charts. Status: pulsing green active, static green checks passed, + * red failed, yellow warning, gray pending. Completion alone is not success. */ + +@font-face { + font-family: 'Inter Variable'; + font-style: normal; + font-display: swap; + font-weight: 100 900; + src: url(/fonts/inter-latin-variable.woff2) format('woff2-variations'); +} +@font-face { + font-family: 'Source Code Pro Variable'; + font-style: normal; + font-display: swap; + font-weight: 200 900; + src: url(/fonts/source-code-pro-latin-variable.woff2) format('woff2-variations'); +} + +:root { + --green: #4cf490; --green-25: #4cf49040; --green-10: #4cf4901a; + --active: var(--green); --active-ring: var(--green-25); + --yellow: #fbdc8e; --yellow-25: #fbdc8e40; --yellow-10: #fbdc8e1a; + --red: #ff4c4c; + --n1: #e6e9f0; --n2: #ced3e0; --n3: #b6c0cf; --n4: #8d98a5; --n5: #363840; --n7: #050505; + --shade1: #162d38; --shade4: #121e24; --shade5: #0f191f; --shade6: #0e161a; + --shade7: #0b1114; --shade8: #0b0e12; + --sans: 'Inter Variable', Inter, ui-sans-serif, system-ui, -apple-system, 'Segoe UI', sans-serif; + --mono: 'Source Code Pro Variable', 'Source Code Pro', ui-monospace, SFMono-Regular, Consolas, monospace; + color-scheme: dark; +} + +* { box-sizing: border-box; } +body { margin: 0; background: var(--shade7); color: var(--n3); font: 14px/1.5 var(--sans); } +a { color: var(--n1); } +:focus-visible { outline: 2px solid var(--green); outline-offset: 2px; } +::selection { background: var(--green-25); } + +.topbar { display: flex; align-items: center; gap: 22px; height: 48px; padding: 0 24px; border-bottom: 1px solid var(--shade4); } +.brand { display: flex; align-items: center; gap: 10px; color: var(--n1); text-decoration: none; } +.brand b { font: 600 12px/1 var(--mono); letter-spacing: .1em; } +.btn { display: inline-flex; align-items: center; min-height: 36px; padding: 0 14px; border-radius: 4px; border: 1px solid var(--shade1); color: var(--n1); background: transparent; font: 500 13px/1 var(--sans); cursor: pointer; list-style: none; } +.nav { display: flex; gap: 2px; } +.nav a { padding: 8px 10px; border-radius: 4px; color: var(--n4); text-decoration: none; font: 500 12.5px var(--sans); } +.nav a.on { color: var(--n1); background: var(--shade5); } +.btn.primary { background: var(--green); border-color: var(--green); color: var(--n7); text-decoration: none; } +.tools { display: flex; align-items: center; gap: 10px; margin-left: auto; } +.files { position: relative; z-index: 10; } +.files summary::-webkit-details-marker { display: none; } +.files div { position: absolute; right: 0; top: 36px; display: grid; gap: 2px; padding: 8px 10px; background: var(--shade6); border: 1px solid var(--shade1); border-radius: 4px; } +.files div a { padding: 6px 2px; white-space: nowrap; color: var(--n2); font: 12px var(--mono); text-decoration: none; } +.page { max-width: 1600px; margin-inline: auto; padding: 22px 24px 40px; } +.crumbs { color: var(--n4); font: 12px var(--mono); margin-bottom: 8px; } +.crumbs a { color: var(--n4); text-decoration: none; } +.crumbs b { color: var(--n2); font-weight: 500; } +.title { flex-wrap: wrap; display: flex; align-items: center; gap: 12px; margin-bottom: 16px; } +.title h2 { margin: 0; color: var(--n1); font: 600 25px/28px var(--sans); letter-spacing: -.01em; } +.title h2 span { color: var(--n4); font-weight: 400; } +.label { color: var(--n4); font: 500 12px/1.4 var(--sans); } +.state { color: var(--n4); font: 500 13px var(--sans); white-space: nowrap; } +.state.run { color: var(--active); } +.state.done { color: var(--n2); } +.state.warn { color: var(--yellow); } +.state.idle { color: var(--n4); } + +/* live lanes: stack, score, climb, phase */ + +.reference-run { margin-bottom: 16px; border: 1px solid var(--shade4); border-radius: 4px; background: var(--shade6); } +.reference-run > summary { display: grid; grid-template-columns: minmax(0, 1fr) auto auto; align-items: center; gap: 24px; padding: 14px 16px; cursor: pointer; list-style: none; } +.reference-run > summary::-webkit-details-marker { display: none; } +.reference-run > summary:hover { background: var(--shade5); } +.reference-identity { display: flex; flex-wrap: wrap; align-items: center; gap: 4px 12px; min-width: 0; } +.reference-identity b { color: var(--n1); font-weight: 600; overflow-wrap: anywhere; } +.reference-identity b::first-letter, .reference-identity .state { text-transform: capitalize; } +.reference-run small { display: block; color: var(--n4); font-size: 12px; font-weight: 400; } +.reference-identity small { flex-basis: 100%; } +.reference-score { white-space: nowrap; font-variant-numeric: tabular-nums; } +.reference-score strong { color: var(--n1); font-weight: 600; } +.reference-toggle { display: flex; align-items: center; gap: 10px; color: var(--n4); font-size: 12px; } +.reference-toggle span { font-size: 20px; line-height: 1; } +.reference-run[open] .reference-toggle span { transform: rotate(90deg); } +.reference-run > .log { margin: 0; border-top: 1px solid var(--shade4); border-radius: 0; } +@media (max-width: 600px) { + .reference-run > summary { grid-template-columns: minmax(0, 1fr) auto; gap: 12px; } + .reference-score { grid-column: 1; grid-row: 2; white-space: normal; } + .reference-toggle { grid-column: 2; grid-row: 1 / 3; } +} + +/* campaigns table */ +.tablewrap { border: 1px solid var(--shade4); border-radius: 4px; background: var(--shade6); overflow: hidden; } +.toolbar { display: flex; flex-wrap: wrap; gap: 6px; align-items: center; min-height: 44px; padding: 6px 12px; border-bottom: 1px solid var(--shade4); } +.chip { height: 24px; padding: 0 10px; border-radius: 4px; border: 1px solid var(--shade1); color: var(--n3); font: 500 12px/22px var(--sans); text-decoration: none; } +.chip.on { background: var(--shade1); color: var(--n1); } +.chip.sm { height: 24px; line-height: 22px; font-size: 12px; padding: 0 8px; } +.wrap { overflow-x: auto; } +table.runs { width: 100%; border-collapse: collapse; font-size: 13px; } +table.runs th, table.runs td { padding: 0 14px; height: 40px; text-align: left; vertical-align: middle; border-bottom: 1px solid var(--shade4); white-space: nowrap; } +table.runs thead th { color: var(--n4); font: 600 12px/1.4 var(--sans); height: 36px; } +table.runs tbody tr:last-child td { border-bottom: 0; } +table.runs tbody tr:hover td { background: var(--shade5); } +table.runs td.name a { color: var(--n1); font-weight: 600; text-decoration: none; } +table.runs td.shape { color: var(--n4); font: 12px var(--mono); } +table.runs td.stack { text-align: right; font: 13px var(--mono); font-variant-numeric: tabular-nums; color: var(--n1); width: 128px; } +table.runs th.stack:first-of-type, table.runs td.stack:first-of-type { border-left: 1px solid var(--shade4); } +table.runs td.stack.na { color: var(--n4); } +table.runs td.stack u { text-decoration-color: var(--green); text-underline-offset: 5px; text-decoration-thickness: 2px; } +table.runs th.when, table.runs td.when { color: var(--n4); font: 12px var(--mono); text-align: right; } +table.runs thead th.stack { text-align: right; } +table.runs thead th.when { color: var(--n4); font: 600 12px/1.4 var(--sans); } + +/* saved plans and shared controls */ +table.plans th.stack, table.plans td.stack { width: auto; } +table.plans td.name { color: var(--n1); font-weight: 600; min-width: 240px; max-width: 360px; white-space: normal; padding-block: 10px; overflow-wrap: anywhere; } +.secret input, .transcript-controls select, .transcript-controls button { height: 30px; padding: 0 10px; border: 1px solid var(--shade1); border-radius: 4px; background: var(--shade7); color: var(--n1); font: 13px var(--mono); } +.secret { display: flex; align-items: center; gap: 8px; } +.secret input { width: 168px; } +.err { color: var(--red); font: 12.5px var(--sans); align-self: center; } + +/* campaign sheet: stacks across, facts down */ +.page > h3 { margin: 28px 0 12px; font-size: 15px; font-weight: 600; color: var(--n1); } +.facts { display: flex; flex-wrap: wrap; gap: 4px 18px; margin: 24px 0 0; } +.facts div { display: flex; gap: 6px; align-items: baseline; min-width: 0; } +.facts b { color: var(--n3); font: 500 11.5px/1.5 var(--mono); overflow-wrap: anywhere; } +.sheet { width: 100%; border-collapse: collapse; background: var(--shade6); } +.sheet th, .sheet td { min-width: 185px; padding: 8px 16px; text-align: left; vertical-align: middle; border: 1px solid var(--shade4); } +.sheet th:first-child { min-width: 190px; width: 190px; } +.sheet .k { color: var(--n4); font: 500 13px/1.4 var(--sans); } +.sheet .h { height: 48px; color: var(--n1); font-size: 15px; font-weight: 600; } +.sheet .h a { text-decoration: none; } +.sheet .v { color: var(--n1); font: 13px var(--mono); font-variant-numeric: tabular-nums; } +.sheet .v i, .checks .group i { color: var(--n4); font-style: normal; margin-left: 6px; } +.sheet .big { color: var(--n1); font: 600 30px/1 var(--sans); letter-spacing: -.02em; } +.sheet .q { display: flex; align-items: center; gap: 6px; flex-wrap: wrap; min-height: 36px; padding-block: 6px; } +.sheet .q.k { display: table-cell; text-transform: none; letter-spacing: 0; font: 12.5px var(--sans); color: var(--n3); } +.sheet .q .pct { margin-left: auto; color: var(--n4); font: 11.5px var(--mono); font-variant-numeric: tabular-nums; } +.sheet .q .pct.full { color: var(--green); } + +.evhead { display: flex; flex-wrap: wrap; gap: 0; padding: 0; } +.evhead .ev { display: grid; gap: 5px; padding: 8px 16px; border-left: 1px solid var(--shade4); min-height: 48px; min-width: 130px; } +.evhead .ev:first-child { border-left: 0; } +.dot { width: 9px; height: 9px; border-radius: 50%; background: var(--shade1); flex: 0 0 auto; } +.dot.p { position: relative; background: var(--green); } +.dot.p::after { content: ""; position: absolute; left: 3px; top: 1px; width: 2px; height: 4px; border: solid var(--shade7); border-width: 0 1.5px 1.5px 0; transform: rotate(45deg); } +.dot.a { background: var(--active); box-shadow: 0 0 0 2px var(--active-ring); } +.dot.f { background: var(--red); } +.dot.b { background: transparent; border: 1.5px solid var(--red); } +.dot.o { background: transparent; border: 1.5px solid var(--n4); } + +/* graph */ +.dag { display: block; height: auto; } +.dag .band { font: 11px var(--mono); fill: var(--n4); } +.dag .col { font: 500 10.5px var(--mono); fill: var(--n4); letter-spacing: .08em; text-transform: uppercase; } +.dag .sep { stroke: var(--shade4); } +.dag .e { fill: none; stroke: var(--shade1); stroke-width: 1.1; opacity: .8; } +.dag .e.cut { stroke: var(--red); stroke-dasharray: 3 4; opacity: .7; } +.dag .n rect { fill: var(--shade5); stroke: var(--shade1); } +.dag .n text { font: 12px var(--mono); fill: var(--n2); } +.dag .d { fill: var(--shade1); } +.dag .d.p { fill: var(--green); } +.dag .d.a { fill: var(--active); } +.dag .d.f { fill: var(--red); } +.dag .d.b { fill: none; stroke: var(--red); stroke-width: 1.4; } +.dag .d.o { fill: none; stroke: var(--n4); stroke-width: 1.4; } + +/* replay */ +.replay { min-height: 40px; padding: 6px 16px; } +.strip { display: block; width: 100%; height: 28px; } +.strip .st { fill: var(--n4); } +.strip .st.b, .strip .st.r { fill: var(--active); } +.strip .st.g { fill: var(--n5); } +.strip .st.f { fill: var(--red); } +.strip .st.on { stroke: var(--yellow-25); stroke-width: 3; } +.strip .st.dim { opacity: .3; } +.strip .cur { stroke: var(--n1); stroke-width: 1; } + +/* attempt */ +.figs { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr)); gap: 1px; margin: 4px 0 20px; border: 1px solid var(--shade4); border-radius: 4px; overflow: hidden; background: var(--shade4); } +.figs > div { display: grid; align-content: start; gap: 6px; min-width: 0; padding: 12px 14px; background: var(--shade6); } +.metric-label { display: flex; align-items: center; flex-wrap: wrap; gap: 8px; min-height: 24px; } +.figs b { color: var(--n1); font: 600 20px/1.4 var(--sans); font-variant-numeric: tabular-nums; overflow-wrap: anywhere; } +.figs b.now { font: 500 14px/1.5 var(--sans); } +.figs b.now.warn { color: var(--yellow); } +.issue { margin: 0 0 20px; padding: 12px 14px; border-left: 2px solid var(--yellow); background: var(--yellow-10); } +.issue p { margin: 6px 0 0; color: var(--n2); } +.bigclimb { display: block; width: 100%; min-width: 1060px; height: auto; aspect-ratio: 1060 / 170; margin-bottom: 12px; } +.bigclimb text { font: 12px var(--mono); fill: var(--n4); } +.bigclimb .l { stroke: var(--green); stroke-width: 2; fill: none; stroke-linejoin: round; } +.bigclimb .a { fill: var(--green-10); } +.bigclimb .g { stroke: var(--shade4); } +.bigclimb .band { fill: var(--shade6); } +.bigclimb .ev { fill: var(--n1); } +.bigclimb .ev.first { fill: var(--shade7); stroke: var(--n3); stroke-width: 1.5; } +.bigclimb .ev.now { fill: var(--n1); } +.tabs { display: flex; gap: 2px; border-bottom: 1px solid var(--shade4); } +.tabs a { padding: 10px 12px; color: var(--n4); font: 500 13px var(--sans); border-bottom: 2px solid transparent; margin-bottom: -1px; text-decoration: none; } +.tabs a.on { color: var(--n1); border-bottom-color: var(--green); } +.tabs a i { color: var(--n4); font: 11.5px var(--mono); font-style: normal; margin-left: 6px; } +.checks { width: 100%; border-collapse: collapse; font-size: 13px; } +.checks th, .checks td { height: 34px; padding: 0 14px; text-align: left; border-bottom: 1px solid var(--shade4); white-space: nowrap; } +.checks thead th { color: var(--n4); font: 600 12px/1.4 var(--sans); height: 36px; } +.checks td.k { font-family: var(--mono); color: var(--n2); } +.checks td.d { color: var(--n3); white-space: normal; } +.checks td.h { color: var(--n4); font: 12px var(--mono); letter-spacing: .14em; } +.checks .h .p { color: var(--green); } +.checks .h .f { color: var(--red); } +.checks .h .x { color: var(--n4); } +.checks tr.group td { color: var(--n1); font-weight: 600; background: var(--shade6); } +.check-evidence { padding: 8px 0; min-width: 240px; } +.checks td:not(.d) { vertical-align: top; padding-top: 10px; } +.checks td.k { max-width: 180px; white-space: normal; overflow-wrap: anywhere; } +.check-evidence summary { cursor: pointer; } +.check-evidence summary:focus-visible { outline: 2px solid var(--n2); outline-offset: 3px; } +.check-evidence section { margin: 12px 0; padding-top: 12px; border-top: 1px solid var(--shade4); } +.check-evidence strong { font-size: 12px; } +.check-evidence p { margin: 8px 0; overflow-wrap: anywhere; } +.check-evidence pre { white-space: pre-wrap; overflow-wrap: anywhere; max-height: 240px; overflow: auto; font: 12px/1.5 var(--mono); } +.grade-key { flex-wrap: wrap; display: flex; gap: 18px; padding: 10px 14px; color: var(--n4); font: 12px/1.6 var(--sans); } +.grade-key .p { color: var(--green); } +.grade-key .f { color: var(--red); } +.grade-key .x { color: var(--n4); } +.shots { display: grid; grid-template-columns: repeat(auto-fill, minmax(240px, 1fr)); gap: 12px; padding: 14px 0; } +.shots button { padding: 0; border: 0; background: none; cursor: zoom-in; } +.shots img { width: 100%; border: 1px solid var(--shade4); border-radius: 4px; } +.lightbox { width: min(94vw, 1500px); max-height: 94vh; padding: 42px 12px 12px; border: 1px solid var(--shade1); border-radius: 4px; background: var(--shade8); } +.lightbox::backdrop { background: #000c; } +.lightbox form { position: absolute; top: 8px; right: 10px; } +.lightbox button { border: 0; background: none; color: var(--n2); cursor: pointer; } +.lightbox img { display: block; max-width: 100%; max-height: calc(94vh - 54px); margin: auto; } +.files-list { display: grid; gap: 6px; padding: 14px 0; } +.files-list a { color: var(--n2); font: 12px var(--mono); text-decoration: none; } +.log { margin: 14px 0 0; padding: 14px 16px; max-height: 520px; overflow: auto; background: var(--shade8); color: var(--n3); font: 12px/1.7 var(--mono); border-radius: 4px; } + +/* Local scrolling keeps labels readable without widening the page. */ +.crumbs, .title h2, .files-list a { overflow-wrap: anywhere; } +.sheet-scroll, .chart-scroll, .graph-scroll { max-width: 100%; overflow-x: auto; } +.chart-empty, .summary-note, .chart-caption { color: var(--n3); font: 12px/1.5 var(--sans); } +.chart-caption { margin: 0 0 16px; } +.graph-key { display: flex; flex-wrap: wrap; gap: 8px 18px; padding: 8px; font-size: 12px; } +.graph-key span { display: inline-flex; align-items: center; gap: 6px; } +.graph-key p { flex-basis: 100%; margin: 0; } +/* Secondary help stays out of the layout and above scrolling containers. */ +.metric-help { padding: 4px 0; border: 0; background: transparent; text-align: left; cursor: help; } +.metric-help:hover, .metric-help:focus-visible { color: var(--n1); } +.metric-tooltip { position: fixed; inset: auto; position-area: top span-right; position-try-fallbacks: flip-block, flip-inline; margin: 6px; width: min(260px, calc(100vw - 24px)); padding: 10px 12px; border: 1px solid var(--shade1); border-radius: 4px; background: var(--shade5); color: var(--n2); font: 12px/1.5 var(--sans); letter-spacing: normal; text-transform: none; overflow-wrap: anywhere; } +.btn:hover, .chip:hover, .nav a:hover, .tabs a:hover { color: var(--n1); background: var(--shade1); } +.btn.primary:hover { color: var(--n7); background: var(--green); filter: brightness(1.08); } +.btn:disabled { opacity: .55; cursor: not-allowed; } +.btn:disabled:hover { filter: none; } +@media (max-width: 900px) { + .topbar { height: auto; min-height: 48px; flex-wrap: wrap; padding: 10px 16px; gap: 12px; } + .tools { flex-wrap: wrap; margin-left: 0; } + .page { padding: 18px 16px 32px; } + .lane { grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 16px; } + .lane-identity, .lane .phase { grid-column: 1 / -1; } + .figs { grid-template-columns: repeat(2, minmax(0, 1fr)); } + .title h2 { font-size: 22px; } + .tabs { flex-wrap: wrap; } + .sheet-scroll::before { content: 'Scroll horizontally to compare stacks'; display: block; padding: 6px 0; color: var(--n4); font-size: 12px; } +} +@media (max-width: 420px) { + .page { padding-inline: 12px; } + .secret { flex-wrap: wrap; } + .secret input { max-width: 100%; } +} + +.feature-progress { margin-block: 28px; } +.section-heading { display: flex; align-items: center; justify-content: space-between; flex-wrap: wrap; gap: 12px; } +.section-heading h3 { margin: 0; color: var(--n1); font-size: 15px; font-weight: 600; } +.section-heading nav { display: flex; gap: 6px; } + +.explore { position: relative; } +.explore summary { cursor: pointer; color: var(--n3); font-size: 12px; padding: 6px 0; } +.explore nav { position: absolute; right: 0; z-index: 2; padding: 10px; border: 1px solid var(--shade1); border-radius: 4px; background: var(--shade5); } +table.attempt-list { table-layout: fixed; min-width: 960px; } +table.attempt-list th:first-child { width: 16%; } +table.attempt-list th:nth-child(2) { width: 18%; } +table.attempt-list th:nth-child(3), table.attempt-list th:nth-child(4) { width: 10%; } +table.attempt-list th:nth-last-child(2) { width: 10%; } +table.attempt-list td { white-space: normal; overflow-wrap: anywhere; } +table.attempt-list .run-name { display: block; padding-block: 8px; color: var(--n1); text-decoration: none; } +table.attempt-list .run-effort { font-weight: 400; color: var(--n4); } +table.attempt-list .run-name:hover { text-decoration: underline; } +table.attempt-list .run-status { padding-block: 10px; } +.run-status summary { cursor: pointer; color: var(--yellow); } +.loading { display: flex; align-items: center; gap: 12px; min-height: 96px; color: var(--n2); font-size: 14px; } +.loading::before { content: ''; width: 16px; height: 16px; flex-shrink: 0; border: 2px solid var(--shade1); border-top-color: var(--n2); border-radius: 50%; animation: loading-spin .8s linear infinite; } +@keyframes loading-spin { to { transform: rotate(360deg); } } +main[aria-busy="true"]:not(:has(.loading))::before { content: ''; position: fixed; top: 0; left: 0; width: 30%; height: 2px; z-index: 10; background: var(--n2); animation: loading-progress 1.4s ease-in-out infinite; } +@keyframes loading-progress { from { transform: translateX(-100%); } to { transform: translateX(334%); } } +@media (prefers-reduced-motion: reduce) { main[aria-busy="true"]:not(:has(.loading))::before, .loading::before { animation: none; } } +[data-started-at] { font-variant-numeric: tabular-nums; white-space: nowrap; } +.time-grant { display: flex; flex-wrap: wrap; align-items: center; gap: .75rem; margin: 1rem 0; } +.time-grant label { display: flex; align-items: center; gap: .5rem; } +.time-grant input[type="number"] { width: 6rem; } + +.progress-heading { margin: 24px 0 8px; } +.progress-chart { display: block; width: 100%; min-width: 520px; max-height: 300px; } +.progress-chart text { fill: var(--n3); font: 12px var(--sans); } +.progress-grid { stroke: var(--n3); opacity: .15; } +.progress-panel { margin-bottom: 24px; } +.progress-controls { display: grid; grid-template-columns: max-content minmax(0, 1fr); gap: 6px 8px; margin: 12px 0; } +.progress-stack { display: grid; grid-template-columns: subgrid; grid-column: 1 / -1; align-items: start; } +.chart-runs { display: flex; flex-wrap: wrap; gap: 4px; min-width: 0; } +.progress-controls .chart-stack-toggle { max-width: 12rem; overflow-wrap: anywhere; } +.progress-controls .chart-run-toggle { min-width: 7.5rem; font-variant-numeric: tabular-nums; } +.progress-controls button { display: inline-flex; align-items: center; gap: 6px; min-height: 32px; padding: 4px 8px; border: 1px solid transparent; border-radius: 4px; background: transparent; color: var(--n2); font: 12px var(--sans); cursor: pointer; } +.progress-controls button:hover { background: var(--shade1); } +.progress-controls button:focus-visible { outline: 2px solid var(--n2); outline-offset: 2px; } +.progress-controls button[aria-pressed="false"] { color: var(--n4); } +.progress-controls button[aria-pressed="false"] svg, .progress-controls button[aria-pressed="false"] .chart-swatch { opacity: .25; } +.progress-controls .chart-run-toggle[aria-pressed="true"] { border-color: var(--shade1); background: var(--shade5); } +.chart-swatch { flex-shrink: 0; } +.chart-run-toggle svg { flex-shrink: 0; } +.progress-series.is-muted { opacity: .15; } +.progress-series.is-highlighted .progress-line { stroke-width: 3; } +.distribution-run-label { display: none; pointer-events: none; paint-order: stroke; stroke: var(--shade8); stroke-width: 4px; } +.progress-series.is-highlighted .distribution-run-label { display: block; } + +.transcript-controls { margin-top: 16px; display: flex; flex-wrap: wrap; align-items: center; gap: 12px; margin-bottom: 12px; } +.transcript-controls label { min-width: 0; flex: 1; } +.transcript-controls select { max-width: 100%; width: min(100%, 560px); } +.transcript { max-height: 65vh; overflow: auto; overflow-anchor: none; border: 1px solid var(--shade4); border-radius: 4px; background: var(--shade8); padding: 16px; } +.transcript article, .transcript details { margin-bottom: 16px; } +.transcript pre { white-space: pre-wrap; overflow-wrap: anywhere; margin: 8px 0; font-size: 12px; line-height: 1.6; } +.transcript strong, .transcript summary { font-size: 12px; color: var(--n3); } + +.transcript-controls button { cursor: pointer; } +.transcript-controls button:hover { border-color: var(--green); } + +@media (max-width: 600px) { + .progress-chart { min-width: 640px; } + .progress-chart text { font-size: 16px; } +} + +.spend-pending { display: inline-block; width: 6px; height: 6px; vertical-align: middle; margin-left: 3px; } +.dot.a, .dag .d.a { animation: activity-pulse 2.8s ease-in-out infinite; } +@keyframes activity-pulse { 50% { opacity: .4; } } +@media (prefers-reduced-motion: reduce) { .dot.a, .dag .d.a { animation: none; } } + +.dag .passed-check { fill: none; stroke: var(--shade7); stroke-width: 1.2; } + +.chart-options { display: flex; flex-wrap: wrap; gap: 16px; align-items: center; } + +.chip[aria-disabled="true"] { opacity: .4; pointer-events: none; } + +.attempt-list tr.is-highlighted td, .chart-run-toggle.is-highlighted { background: var(--shade1); } + +/* Run setup uses native controls and one review step. */ +.setup{max-width:900px}.setup-form{display:grid;gap:24px}.setup-fields{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:16px}.setup label{display:flex;gap:8px}.setup-fields>label,.setup-actions>label{flex-direction:column}.setup label>span,.setup legend{color:var(--muted,#9eabb9);font-size:13px}.setup input:not([type=checkbox]),.setup select{min-width:0;background:#101c22;color:inherit;border:1px solid #29404b;border-radius:4px;padding:10px;font:inherit}.setup fieldset{border:1px solid #24383f;border-radius:5px;padding:16px}.setup-choices{display:flex;flex-wrap:wrap;gap:20px}.setup-model{display:flex;align-items:center;justify-content:space-between;gap:16px;padding:7px 0}.setup-model select{width:130px}.setup-actions{display:flex;align-items:end;justify-content:flex-end;gap:12px}.setup details>div{margin-top:16px}.setup-review{display:grid;grid-template-columns:160px 1fr;gap:14px}.setup-review dt{color:#9eabb9}.setup-review dd{margin:0}.setup .warning{border-left:3px solid #eabc65;padding:12px;background:#211e15}.setup pre{overflow:auto;max-height:260px}.setup .summary-note{margin:20px 0}@media(max-width:600px){.setup-fields{grid-template-columns:1fr}.setup-actions{align-items:stretch;flex-direction:column}.setup-review{grid-template-columns:1fr;gap:6px}.setup-review dd{margin-bottom:12px}} + +/* Definition guide: neutral panels; color labels distinguish work from assertions. */ +.guide { max-width: 1160px; } +.guide h1 { margin: 0; color: var(--n1); font-size: 28px; letter-spacing: -.02em; } +.guide-intro { margin: 8px 0 20px; font-size: 16px; color: var(--n2); } +.guide-help { margin-bottom: 20px; } +.guide summary { cursor: pointer; } +.guide-help > summary, .guide-evidence > summary { padding: 10px 0; color: var(--n2); font-weight: 500; } +.guide-key { overflow-wrap: anywhere; font: 12px/1.7 var(--mono); color: var(--n4); } +.guide-toolbar { position: sticky; top: 0; z-index: 2; background: var(--shade7); padding: 12px 0; border-bottom: 1px solid var(--shade1); } +.guide-search { display: grid; gap: 6px; color: var(--n2); font-weight: 500; } +.guide-search input { width: 100%; padding: 12px 14px; border: 1px solid var(--shade1); border-radius: 5px; background: var(--shade6); color: var(--n1); font: inherit; } +.guide-search input::placeholder { color: var(--n4); } +.guide-controls { display: flex; flex-wrap: wrap; align-items: center; gap: 12px; margin-top: 12px; } +.guide-controls label { display: flex; align-items: center; gap: 6px; margin-right: auto; } +.guide-controls input { accent-color: var(--green); } +.guide-controls [role=status] { color: var(--n4); font-size: 12px; } +.guide-kind { font: 600 11px/1.6 var(--sans); letter-spacing: .015em; } +.guide-kind.action { color: #9ecbff; } +.guide-kind.assert { color: #87deb0; } +.guide-kind.wait { color: var(--yellow); } +.guide-kind.repeat { color: #c3b4fa; } +.guide-check { margin: 10px 0; border: 1px solid var(--shade1); border-radius: 6px; background: var(--shade6); overflow: hidden; } +.guide-check[hidden] { display: none; } +.guide-check > summary { display: flex; align-items: baseline; gap: 14px; padding: 17px 18px; color: var(--n1); font-weight: 500; list-style: none; } +.guide-check > summary::-webkit-details-marker { display: none; } +.guide-check > summary::before { content: '\203A'; color: var(--n4); font-size: 20px; line-height: 1; } +.guide-check[open] > summary::before { transform: rotate(90deg); } +.guide-check > summary:hover { background: var(--shade5); } +.guide-id { color: var(--n4); font: 12px var(--mono); min-width: 40px; } +.guide-scope { margin-left: auto; white-space: nowrap; color: var(--n4); font-size: 11px; } +.guide-body { padding: 0 22px 20px; border-top: 1px solid var(--shade1); } +.guide-body h2 { margin: 24px 0 10px; font-size: 15px; color: var(--n1); } +.guide-step { display: grid; grid-template-columns: 110px minmax(0, 1fr); gap: 16px; padding: 10px 0; border-bottom: 1px solid var(--shade4); line-height: 1.7; overflow-wrap: anywhere; } +.guide-step .guide-kind { padding-top: 3px; } +.guide-indent-1 { padding-left: 18px; } +.guide-indent-2 { padding-left: 36px; } +.guide-indent-3 { padding-left: 54px; } +.guide-indent-4 { padding-left: 72px; } +.guide-note { color: var(--n4); } +.guide-evidence { border-top: 1px solid var(--shade1); } +.guide-evidence li { margin: 8px 0; overflow-wrap: anywhere; } +.guide pre { padding: 16px; background: var(--shade8); border-radius: 4px; white-space: pre-wrap; overflow-wrap: anywhere; color: var(--n2); font: 12px/1.7 var(--mono); } +@media (max-width: 640px) { + body:has(.guide) .topbar { height: auto; min-height: 48px; padding: 12px 16px; flex-wrap: wrap; gap: 12px; } + .guide { padding-inline: 16px; } + .guide-toolbar { position: static; } + .guide-check > summary { gap: 10px; padding: 14px 12px; } + .guide-scope { display: none; } + .guide-body { padding-inline: 14px; } + .guide-step { grid-template-columns: minmax(0, 1fr); gap: 2px; } + .guide pre { padding: 10px; } +} diff --git a/tools/stack-bench/dashboard/public/views/attempt.ts b/tools/stack-bench/dashboard/public/views/attempt.ts new file mode 100644 index 00000000000..eff46ff5e04 --- /dev/null +++ b/tools/stack-bench/dashboard/public/views/attempt.ts @@ -0,0 +1,208 @@ +import type { TranscriptPage } from '../../dashboard-transcript.js'; +// One attempt: figures, feature dependencies, and the evidence behind tabs. +// Each tab is a link, so what is open survives a reload and a back button. + +import type { AttemptCheck, AttemptChecks, AttemptPackage, CampaignProgression, CampaignSheet, SheetAttempt, SheetStack } + from '../../dashboard-views.js'; +import type { readCampaignTimeBudget } from '../../../src/campaigns/campaign-time-grant.js'; +import { graph } from '../graph.js'; +import { bigClimb } from '../climb.js'; +import { DASH, completionLabel, duration, executionClock, esc, metricLabel, spend, pct, phrase, ratio, stackLabel } from '../format.js'; + +export type AttemptTab = 'checks' | 'screenshots' | 'files' | 'log' | 'transcript'; + +export interface AttemptPageInput { + sheet: CampaignSheet; + progression?: CampaignProgression | null; + attemptId: string; + tab: AttemptTab; + checks: AttemptChecks | null; + evidence: AttemptPackage | null; + log: string; + transcript?: TranscriptPage | null; + timeBudget?: ReturnType | null; + canControl?: boolean; + controlError?: string; +} + +const GLYPH: Record = { pass: '✓', + fail: '✕', 'not-run': '·' }; + +function locate(sheet: CampaignSheet, attemptId: string): { + stack: SheetStack; + attempt: SheetAttempt; +} | null { + for (const stack of sheet.stacks) { + const attempt = stack.attempts.find(item => item.id === attemptId); + if (attempt) return { stack, attempt }; + } + return null; +} + +const CATEGORY = { feature: 'Feature', production: 'Production', interface: 'Interface', unknown: 'Unclassified' }; + +function checksTable(checks: AttemptChecks | null): string { + const errors = checks?.grades.filter(grade => grade.error).map(grade => + `

${esc(grade.id)}: ${esc(grade.error)}

`).join('') ?? ''; + if (!checks?.checks.length) return errors + '

No check results are recorded yet. Check the log for current work or an execution error.

'; + const features = new Map(); + for (const check of checks.checks) { + features.set(check.feature, [...features.get(check.feature) ?? [], check]); + } + const groups = [...features.entries()].map(([feature, items]) => { + return `${esc(feature)}` + + items.map(check => `${esc(check.id)}` + + `
` + + `${esc(check.description || check.id)}` + + check.observations.map((observation, index) => { + const grade = checks.grades[index]; + const label = `Grade ${index + 1}${grade?.level == null ? '' : ` · L${grade.level}`}`; + return `
${esc(label)} · ${esc(observation?.status ?? 'NO RESULT')}` + + (observation?.context ? `

${esc(observation.context)}

` : '') + + (observation?.summary ? `

${esc(observation.summary)}

` : '') + + (observation?.expected != null ? `
Expected
${esc(observation.expected)}
` : '') + + (observation?.actual != null ? `
Observed
${esc(observation.actual)}
` : '') + + (!observation ? `

${esc(grade?.error ?? 'This check has no recorded result in this grade.')}

` + : observation.expected == null && observation.actual == null && !observation.summary + ? '

No observation details were recorded.

' : '') + + '
'; + }).join('') + '
' + + `${CATEGORY[check.category ?? 'unknown']}` + + `${check.history.map((outcome, index) => `${GLYPH[outcome] ?? GLYPH['not-run']}`).join('')}` + + '').join(''); + }).join(''); + return errors + '
Recorded grades, oldest first. Expand a check for evidence. ' + + '✓ Pass✕ Fail' + + '· No pass/fail result Categories name the intended property. Failed checks can share one cause.
' + + '
' + + `${groups}
CheckRequirementCategoryGrades
`; +} + +function artifacts(evidence: AttemptPackage | null, key: string, visual: boolean): string { + const items = (evidence?.executions ?? []).flatMap(execution => + visual ? execution.visuals : execution.artifacts.filter(item => item.kind !== 'visual')); + const link = (id: string): string => + `/api/campaigns/${encodeURIComponent(key)}/artifacts/${encodeURIComponent(id)}`; + if (!items.length) return `

No ${visual ? 'screenshots' : 'files'} are available for this attempt.

`; + if (visual) { + return `
${items.map(item => { + const source = link(item.id); + return ``; + }).join('')}
` + + '
'; + } + return `
${items.map(item => + `${esc(item.path)}`).join('')}
`; +} + +export function attemptPage({ sheet, attemptId, tab, checks, evidence, log, transcript, + progression, timeBudget, canControl = false, controlError = '' }: AttemptPageInput): string { + const found = locate(sheet, attemptId); + const crumbs = (tail: string): string => `
Campaigns / ` + + `${esc(sheet.title)} / ` + + `${esc(tail)}
`; + if (!found) { + return `
${crumbs(attemptId)}` + + '

Attempt not found

'; + } + const { stack, attempt } = found; + const clock = timeBudget?.observedAt + ? executionClock(new Date(Date.parse(timeBudget.observedAt) - timeBudget.consumedMs).toISOString(), + attempt.status === 'running' ? null : timeBudget.observedAt) + : executionClock(attempt.executionStartedAt, attempt.executionCompletedAt); + const latestGrant = timeBudget?.grants.slice().sort((a, b) => + a.request.requestedAt.localeCompare(b.request.requestedAt)).at(-1); + const pending = timeBudget?.grants.some(grant => grant.disposition === 'pending') ?? false; + const resumeWithTime = attempt.status !== 'running' && timeBudget?.continuation?.eligible === true; + const timeControls = canControl && timeBudget && (resumeWithTime || (attempt.status === 'running' && timeBudget.liveGrantSupported)) + ? `
` + + '' + + `` + + `Limit after request: ${duration((timeBudget.effectiveMinutes + 120) * 60)}` + + `${resumeWithTime ? 'Continues from the verified checkpoint.' : 'Keeps the agent running.'} Cost and repair limits stay fixed.
` + : canControl && attempt.status === 'running' && timeBudget?.liveGrantSupported === false + ? '

This controller does not support live time extensions.

' + : canControl && timeBudget?.continuation?.reason + ? `

Cannot resume: ${esc(timeBudget.continuation.reason)}

` : ''; + const grantStatus = latestGrant ? `

${esc( + latestGrant.disposition === 'pending' ? 'Time request pending. The limit has not changed yet.' + : latestGrant.disposition === 'accepted' ? `Time added. Limit: ${duration(timeBudget!.effectiveMinutes * 60)}.` + : `Time request rejected: ${latestGrant.reason ?? 'See the grant evidence.'}`)}

` : ''; + const name = `${stackLabel(stack.stack)} rep ${attempt.repetition}`; + const counts: Record = { + checks: checks ? String(checks.checks.length) : '', + screenshots: evidence + ? String(evidence.executions.reduce((total, item) => total + item.visuals.length, 0)) : '', + files: evidence ? String(evidence.executions.reduce((total, item) => + total + item.artifacts.filter(entry => entry.kind !== 'visual').length, 0)) : '', + transcript: attempt.status === 'running' ? 'live' : '', + log: attempt.status === 'running' ? 'live' : '', + }; + const tabs = (['checks', 'transcript', 'screenshots', 'files', 'log'] as const).map(entry => + `` + + `${entry[0]!.toUpperCase()}${entry.slice(1)}` + + `${counts[entry] ? `${esc(counts[entry])}` : ''}`).join(''); + const help: Record = { + 'Checks passed': 'Accepted checks passed / selected, including checks not reached.', + 'Weighted score': 'Earned points / available points.', + 'First build': 'First build at each level. Earlier fixes and feedback are retained.', + Repairs: 'Completed repairs / allowance. Per-feature limits apply.', + Elapsed: 'Consumed time across executions / effective time limit. Includes coding, grading, repairs, and host sleep. Time between executions is excluded.' + + (timeBudget ? ` Original limit: ${duration(timeBudget.originalMinutes * 60)}. Accepted extensions: ${timeBudget.extensionCount}.` : ''), + Time: 'Recorded attempt duration.', + Spend: 'Saved receipts or live usage at pinned prices. ~ estimate; ≤ upper bound.', + }; + const figure = (label: string, text: string, tone = ''): string => + `
${metricLabel(label, help[label])}
${text}
`; + const panel = tab === 'transcript' ? transcriptPanel(transcript) + : tab === 'checks' ? checksTable(checks) + : tab === 'log' ? (log ? `
${esc(log)}
` : '

No log output is recorded yet.

') + : artifacts(evidence, sheet.key, tab === 'screenshots'); + const issue = attempt.excluded + ? `
Why this run was excluded` + + `

${esc(attempt.excluded)}

` : ''; + const categories = Object.entries(attempt.checkCategories ?? {}).filter(([, value]) => value.selected > 0); + const categorySummary = categories.some(([category]) => category !== 'unknown') ? '
' + categories.map(([category, value]) => + figure(CATEGORY[category as keyof typeof CATEGORY], ratio(value.passed, value.selected))).join('') + '
' : ''; + const track = progression?.stacks.find(entry => entry.attemptId === attemptId); + const history = sheet.mode === 'dependency' + ? '

Feature dependencies

' + (progression && track + ? graph(progression, [{ stack: track.stack, + statuses: track.steps.at(-1)?.statuses ?? progression.nodes.map(() => 'locked') }]) + : '

Feature graph unavailable.

') + : `

Grade history

${bigClimb(attempt.climb, level => `L${level}`)}`; + return `
${crumbs(name)}` + + `

${esc(stackLabel(stack.stack))} ` + + `rep ${attempt.repetition}

` + + + `
${figure('Checks passed', completionLabel(attempt))}` + + figure('Spend', spend(attempt.spend, attempt.spendPending, attempt.liveSpend)) + + figure('Status', esc(phrase(attempt)), attempt.stalling ? 'now warn' : 'now') + + figure('Weighted score', pct(attempt.score)) + + figure('First build', pct(attempt.unaided)) + + figure('Repairs', ratio(attempt.repairs.used, attempt.repairs.budget)) + + figure('Elapsed', attempt.status === 'running' || attempt.executionCompletedAt + ? clock + + ` / ${duration((timeBudget?.effectiveMinutes ?? sheet.facts.timeLimitMinutes) * 60)}` : DASH) + + figure('Time', duration(attempt.timeSec)) + + `
${timeControls}${grantStatus}${controlError ? `` : ''}${issue}${history}` + + `
${tabs}
${tab === 'checks' ? categorySummary : ''}${panel}
`; +} + +function transcriptPanel(page?: TranscriptPage | null): string { + if (!page) return '
Loading transcript�
'; + if (!page.sessions.length) return '

No transcript is available for this run yet.

'; + return '
' + + (page.before === null ? '' : ``) + + '
' + + (page.skipped ? '

Some malformed transcript records could not be displayed.

' : '') + + '
' + + page.messages.map(message => message.tool + ? `
${esc(message.role)}
${esc(message.text)}
` + : `
${esc(message.role)}
${esc(message.text)}
`).join('') + + '
'; +} diff --git a/tools/stack-bench/dashboard/public/views/campaign.ts b/tools/stack-bench/dashboard/public/views/campaign.ts new file mode 100644 index 00000000000..d6e3e8a967b --- /dev/null +++ b/tools/stack-bench/dashboard/public/views/campaign.ts @@ -0,0 +1,258 @@ +// Results compare stacks. Runs expose individual evidence. Feature views use +// the same selected attempt per stack, separate from aggregate results. + +import type { CampaignProgression, CampaignSheet, ProgressionStep, SheetAttempt, SheetStack } + from '../../dashboard-views.js'; +import { DASH, completionLabel, modelLabel, duration, executionClock, esc, metricLabel, spend, num, pct, phrase, ratio, stackLabel, statusWord } from '../format.js'; +import { progressChart } from '../progress-chart.js'; +import { graph } from '../graph.js'; + +export type QuestlineView = 'grid' | 'graph' | 'replay'; + +export interface CampaignPageInput { + sheet: CampaignSheet; + progression: CampaignProgression | null; + view: QuestlineView; + chart?: 'completion' | 'cost' | 'distribution'; + unit?: 'checks' | 'features'; + hiddenChartRuns?: ReadonlySet; + step: number; +} + +export interface ReplayEvent { + stack: string; + ordinal: number; + step: ProgressionStep; +} + +const DOT: Record = { passed: 'p', active: 'a', working: 'a', failed: 'f', + blocked: 'b', locked: 'o' }; + +function short(value: string | null): string { + return value ? value.slice(0, 12) : DASH; +} + +function latest(stack: SheetStack): SheetAttempt | null { + return stack.attempts.find(attempt => attempt.id === stack.selectedAttemptId) ?? null; +} + +function facts(sheet: CampaignSheet): string { + const fact = sheet.facts; + const dependency = sheet.mode === 'dependency'; + const depth = sheet.levels.length ? Math.max(...sheet.levels) : 0; + const cells: Array<[string, string, string]> = [['Mode', fact.mode, '']]; + const limits = fact.repairLimits; + const repairBudget = [ + limits.perFeature === undefined ? '' : `${limits.perFeature} per feature`, + limits.perDepth === undefined ? '' : `${limits.perDepth.count} per depth${limits.perDepth.carry ? " (carry forward)" : ""}`, + limits.total === undefined ? '' : `${limits.total} total`, + ].filter(Boolean).join(' · '); + cells.push(dependency ? ['Depth', String(depth), ''] + : ['Levels', sheet.levels.map(level => `L${level}`).join('–'), '']); + if (dependency) { + cells.push(['Work', fact.workSelection ?? DASH, ''], + ['Repair', fact.repairSelection ?? DASH, ''], + ['Repair budget', repairBudget || 'No count limit', 'Limits apply to each attempt. When limits overlap, the tightest remaining limit applies.']); + } else { + cells.push(['Repair budget', repairBudget || 'No count limit', 'Limits apply to each attempt.']); + } + cells.push(['Repetitions', String(sheet.repetitions), ''], + ['Agent', fact.agent ?? DASH, ''], ['Model', fact.model ?? DASH, ''], + ['Guidance', fact.guidance ?? DASH, ''], + ['Production quality', fact.productionQuality === null ? 'Mixed' + : fact.productionQuality ? 'Requested' : 'Not requested', 'Whether the prompt explicitly requests a production-quality app.'], + ['Recipe', [...new Set(fact.recipes.map(recipe => + [recipe.id, short(recipe.contentSha256)].filter(Boolean).join(' ')))].join(' · ') || DASH, ''], + ['Time limit', `${fact.timeLimitMinutes} min`, ''], + ['Spend limit', fact.spendLimitUsd === null ? DASH + : `$${fact.spendLimitUsd} per attempt`, ''], + ['Controller', short(fact.controllerImage), ''], ['Plan', short(fact.planSha256), '']); + if (sheet.mixedScope) cells.push(['Scope', 'mixed', 'attempts do not share one test plan']); + const continued = sheet.stacks.filter(stack => stack.continued).length; + if (continued) cells.push(['Continued', String(continued), '']); + + return `
${cells.map(([label, value, hover]) => + `
${esc(label)}` + + `${esc(value)}
`).join('')}
`; +} + +function questlineRows(sheet: CampaignSheet, stacks: readonly SheetStack[]): string { + const lead = stacks.find(stack => stack.questlines?.length)?.questlines ?? []; + const rows = lead.map(questline => { + const cells = stacks.map(stack => { + const owned = stack.questlines?.find(entry => entry.id === questline.id) ?? null; + const dots = (owned?.nodes ?? []).map(node => + ``).join(''); + const score = owned?.score ?? null; + return `
${dots}` + + `${pct(score)}
`; + }).join(''); + return `${esc(questline.title)}${cells}`; + }).join(''); + if (sheet.mode !== 'dependency') return ''; + return rows || `Feature progress appears after the first recorded grade.`; +} + +function levelRows(stacks: readonly SheetStack[]): string { + const levels = stacks.find(stack => stack.levels?.length)?.levels ?? []; + return levels.map(level => ['unaided', 'score'].map(kind => { + const cells = stacks.map(stack => { + const owned = stack.levels?.find(entry => entry.level === level.level) ?? null; + const points = kind === 'unaided' ? owned?.unaided ?? null : owned?.score ?? null; + return `
${points + ? ratio(points.score, points.max) : DASH}
`; + }).join(''); + return `L${level.level} ${kind === 'unaided' ? 'first build' : 'score'}${cells}`; + }).join('')).join(''); +} + +export function selectedProgression(progression: CampaignProgression, sheet: CampaignSheet): CampaignProgression { + return { ...progression, stacks: sheet.stacks.flatMap(stack => progression.stacks.filter(track => + stack.stack === track.stack && stack.selectedAttemptId === track.attemptId)) }; +} + +export function replayTimeline(progression: CampaignProgression): ReplayEvent[] { + const tracks = progression.stacks; + const depth = Math.max(0, ...tracks.map(track => track.steps.length)); + const events: ReplayEvent[] = []; + for (let ordinal = 0; ordinal < depth; ordinal += 1) { + for (const track of tracks) { + const step = track.steps[ordinal]; + if (step) events.push({ stack: track.stack, ordinal, step }); + } + } + return events; +} + +function marker(step: ProgressionStep, failed: boolean): string { + if (failed) return 'f'; + if (step.action === 'repair') return 'r'; + return step.action === 'grant' ? 'g' : 'b'; +} + +function replay(progression: CampaignProgression, cursor: number): string { + const events = replayTimeline(progression); + cursor = Math.min(Math.max(0, cursor), Math.max(0, events.length - 1)); + const span = Math.max(1, events.length - 1); + const selected = events[cursor] ?? events.at(-1) ?? null; + const failedAt = (step: ProgressionStep): boolean => step.targets.some(target => + step.statuses[progression.nodes.findIndex(node => node.id === target)] === 'failed'); + const title = (id: string): string => + progression.nodes.find(node => node.id === id)?.title ?? id; + const head = selected ? [['Step', ratio(cursor + 1, events.length)], + ['Stack', esc(stackLabel(selected.stack))], ['Action', esc(selected.step.action)], + ['Feature', selected.step.targets.length === 1 + ? esc(title(selected.step.targets[0] ?? '')) : `${selected.step.targets.length} features`], + ['Score', pct(selected.step.score)], ['Repairs', num(selected.step.repairs)]] + .map(([label, value]) => `
${label}` + + `${value}
`).join('') : ''; + // Drawn as one SVG per stack: the dashboard's policy allows no inline style, + // and a marker's position is geometry, not decoration. + const at = (index: number): number => 20 + 960 * index / span; + const rows = progression.stacks.map(track => { + const marks = events.map((event, index) => event.stack !== track.stack ? '' : + ``).join(''); + return `${esc(stackLabel(track.stack))}` + + '' + + `${marks}`; + }).join(''); + const snapshot = progression.stacks.map(track => { + const step = events.filter((event, index) => + event.stack === track.stack && index <= cursor).at(-1)?.step ?? null; + return { stack: track.stack, + statuses: step?.statuses ?? progression.nodes.map(() => 'locked') }; + }); + return `
${head}
` + + graph(progression, snapshot) + + `
${rows}
`; +} + +function board({ sheet, progression, view, step }: CampaignPageInput, + stacks: readonly SheetStack[]): string { + const chips = (['grid', 'graph', 'replay'] as const).map(entry => + `` + + `${entry[0]!.toUpperCase()}${entry.slice(1)}`).join(''); + const heading = stacks.map(stack => `${esc(stackLabel(stack.stack))}`).join(''); + const grid = (rows: string): string => `
${heading}${rows}
Feature
`; + let content: string; + if (sheet.mode !== 'dependency') content = grid(levelRows(stacks)); + else if (view === 'grid' || !progression) content = grid(questlineRows(sheet, stacks)); + else if (view === 'graph') { + const selected = selectedProgression(progression, sheet); + const snapshot = selected.stacks.map(track => ({ stack: track.stack, + statuses: track.steps.at(-1)?.statuses ?? selected.nodes.map(() => 'locked') })); + content = graph(selected, snapshot); + } else content = replay(selectedProgression(progression, sheet), step); + return '
' + + '

Feature progress

' + + (sheet.mode === 'dependency' ? `
Explore · ${esc(view)}
` : '') + + '
' + + content + '
'; +} + +export function campaignPage(input: CampaignPageInput): string { + const sheet = input.sheet; + const stacks = sheet.stacks; + const showRepairs = stacks.some(stack => stack.attempts.some(attempt => + attempt.repairs.budget > 0 || attempt.repairs.used > 0)); + const cell = (render: (stack: SheetStack) => string): string => + stacks.map(stack => `${render(stack)}`).join(''); + const help: Record = { + 'Checks passed': 'Median percentage of selected checks passed, across valid completed runs.', + 'Weighted score': 'Median score weighted by check points, across valid completed runs.', + 'First builds': 'Summed first-build points across levels. Earlier fixes and feedback are retained; this is not an unaided run.', + Regressions: 'Median count of previously passing checks that later failed, across valid completed runs.', + 'Valid runs': 'Completed runs with usable evidence; not necessarily all checks passed.', + Excluded: 'Invalid or incomplete evidence. Spend is retained in Total spend.', + 'Active time': 'Median measured-run time, excluding recorded provider waits and operator pauses. Run Elapsed shows wall time.', + 'Cost per valid run': 'Mean exact measured-run cost, including explicit resume history. Independent failed retries remain in Total spend.', + 'Total spend': 'All runs, including excluded. ~ estimate; ≤ upper bound. Pinned prices.', + }; + const row = (label: string, render: (stack: SheetStack) => string): string => + `${metricLabel(label, help[label])}${cell(render)}`; + const value = (text: string): string => `
${text}
`; + const heads = stacks.map(stack => { + const attempt = latest(stack); + const label = esc(stackLabel(stack.stack)); + return `${attempt + ? `` + + `${label}` : label}`; + }).join(''); + const results = '

Results

' + + `
${heads}` + + row('Checks passed', stack => `
${pct(stack.completionRate === null ? null : 100 * stack.completionRate)}
`) + + row('Cost per valid run', stack => value(stack.costPerValidRun === null ? (stack.n ? 'Unknown' : 'Awaiting valid runs') : `$${stack.costPerValidRun.toFixed(2)}`)) + + row('Weighted score', stack => value(pct(stack.score))) + + (showRepairs ? row('First builds', stack => value(pct(stack.unaided))) : '') + + row('Regressions', stack => value(num(stack.regressions))) + + row('Active time', stack => value(duration(stack.timeSec))) + + row('Valid runs', stack => value(ratio(stack.n, stack.attempts.length))) + + row('Excluded', stack => value(num(stack.attempts.filter(attempt => attempt.excluded).length))) + + row('Total spend', stack => value(spend(stack.spend, stack.spendPending, stack.liveSpend))) + + '
Metric
'; + return `
Campaigns / ` + + `${esc(sheet.key)}
` + + `

${esc(sheet.title)}

` + + `${sheet.provisional ? 'Provisional' : esc(statusWord(sheet.status))}
` + + (sheet.mode === 'dependency' ? progressChart(sheet, input.progression, input.chart, input.view, input.hiddenChartRuns, input.unit) : '') + + board(input, stacks) + + '

Runs

' + + `
${showRepairs ? '' : ''}` + + stacks.flatMap(stack => stack.attempts.map(attempt => { + const href = `/c/${encodeURIComponent(sheet.key)}/a/${encodeURIComponent(attempt.id)}`; + const effort = attempt.effort ? ` (${attempt.effort})` : ''; + return `` + + `` + + `` + + `` + + (showRepairs ? `` : '') + + `` + + ``; + })).join('') + + '
RunModelFeatures passedChecks passedSpendRepairsElapsedStatus
${esc(stackLabel(stack.stack))} ${attempt.repetition}${esc(modelLabel(attempt.model))}${esc(effort)}${completionLabel(attempt, attempt.featureCompletion ?? null)}${completionLabel(attempt)}${spend(attempt.spend, attempt.spendPending, attempt.liveSpend)}${ratio(attempt.repairs.used, attempt.repairs.budget)}${attempt.status === 'running' || attempt.executionCompletedAt ? executionClock(attempt.executionStartedAt, attempt.executionCompletedAt) : DASH}${attempt.excluded + ? `
Excluded · show reason

${esc(attempt.excluded)}

` + : esc(phrase(attempt))}
' + results + facts(sheet) + '
'; +} diff --git a/tools/stack-bench/dashboard/public/views/campaigns.ts b/tools/stack-bench/dashboard/public/views/campaigns.ts new file mode 100644 index 00000000000..135ed1e36b9 --- /dev/null +++ b/tools/stack-bench/dashboard/public/views/campaigns.ts @@ -0,0 +1,87 @@ +// The campaign history, with any reference run in progress above it. + +import type { OverviewCampaign, OverviewEntry, OverviewPage } from '../../dashboard-views.js'; +import { DASH, esc, pct, shape, since, stackLabel, statusWord } from '../format.js'; +import type { ReferenceRun } from '../../dashboard-reference-runs.js'; + +import type { CampaignFilter } from '../../dashboard-views.js'; +export type { CampaignFilter } from '../../dashboard-views.js'; + +const FILTERS: Array<{ id: CampaignFilter; label: string }> = [{ id: 'all', label: 'All' }, + { id: 'attention', label: 'Needs attention' }, { id: 'completed', label: 'Completed' }, + { id: 'ready', label: 'Ready' }]; + +function readable(campaign: OverviewEntry): campaign is OverviewCampaign { + return 'scores' in campaign; +} + +function matches(campaign: OverviewEntry, filter: CampaignFilter): boolean { + if (filter === 'all') return true; + if (filter === 'attention') { + return campaign.status === 'attention-required' || campaign.status === 'unreadable'; + } + if (filter === 'completed') return campaign.status === 'completed'; + return campaign.status === 'prepared'; +} + +function stackCell(campaign: OverviewEntry, stack: string): string { + const score = readable(campaign) ? campaign.scores[stack] ?? null : null; + if (score === null) return `${DASH}`; + return `` + + `${pct(score)}`; +} + +function tone(status: string): string { + if (status === 'running') return 'run'; + if (status === 'completed') return 'done'; + if (status === 'attention-required' || status === 'unreadable') return 'warn'; + return 'idle'; +} + +function row(campaign: OverviewEntry, stacks: readonly string[]): string { + const summary = readable(campaign) ? campaign : null; + return `` + + `${esc(campaign.title)}` + + `${summary + ? esc(shape(summary.mode, summary.levels, summary.repetitions)) : DASH}` + + `${esc(statusWord(campaign.status))}` + + stacks.map(stack => stackCell(campaign, stack)).join('') + + `${summary ? esc(since(summary.updatedAt)) : DASH}`; +} + +export function campaignsPage({ campaigns, filter, loading = false, references, pagination }: { + campaigns: readonly OverviewEntry[]; + filter: CampaignFilter; + loading?: boolean; + references?: { runs: ReferenceRun[]; error: string | null }; + pagination?: Pick; +}): string { + const stacks = [...new Set(campaigns.flatMap(campaign => readable(campaign) ? Object.keys(campaign.scores) : []))]; + const shown = campaigns.filter(campaign => matches(campaign, filter)); + const chips = FILTERS.map(entry => + `` + + `${entry.label}${loading ? '' : ` ${pagination?.counts[entry.id] ?? campaigns.filter(campaign => matches(campaign, entry.id)).length}`}`).join(''); + const body = loading ? `
Loading campaigns…
` + : shown.length ? shown.map(campaign => row(campaign, stacks)).join('') + : `No campaigns match this filter.`; + const pager = pagination && !loading ? `' : ''; + const validations = references?.runs.filter(run => run.status === 'running') + .map(run => `
` + + `${esc(run.title)}` + + `${esc(run.status)}${run.points + ? `${run.points.passed}${run.points.planned === null ? '' : `/${run.points.planned}`} points passed` + : 'Awaiting results'}` + + `Log ` + + `
${esc(run.log || 'No log recorded.')}
`).join('') ?? ''; + return `

Campaigns

` + + validations + + (references?.error ? `

${esc(references.error)}

` : '') + + `
${chips}
` + + '' + + stacks.map(stack => ``).join('') + + `${body}
CampaignScopeStatus${esc(stackLabel(stack))}Updated
${pager}
`; +} diff --git a/tools/stack-bench/dashboard/public/views/plans.ts b/tools/stack-bench/dashboard/public/views/plans.ts new file mode 100644 index 00000000000..1511ed19633 --- /dev/null +++ b/tools/stack-bench/dashboard/public/views/plans.ts @@ -0,0 +1,93 @@ +// Saved plan summaries and shared navigation. New runs use the setup page. + +import type { DashboardPlan } from '../../dashboard-model.js'; +import { DASH, duration, esc, money, num } from '../format.js'; + +export type Page = 'campaigns' | 'plans' | 'campaign' | 'check-guide'; + +export interface RunForm { + error: string; +} + +const HEADS: Array<[string, string]> = [['Plan', 'name'], ['Mode', 'shape'], ['Shape', 'shape'], + ['Stacks', 'stack'], ['Attempts', 'stack'], ['Parallel', 'stack'], ['Repairs', 'stack'], + ['Time limit', 'stack'], ['Attempt cap', 'stack'], ['Campaign cap', 'stack'], ['State', 'state']]; + +export function runName(planId: string, now: Date): string { + const pad = (value: number): string => String(value).padStart(2, '0'); + const stamp = `${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}` + + `-${pad(now.getHours())}${pad(now.getMinutes())}`; + return `${planId}-${stamp}`.toLowerCase().replace(/[^a-z0-9.-]+/g, '-') + .replace(/^[^a-z0-9]+/, '').slice(0, 120); +} + +export function topbar({ page, key, canStart, resumable, controllerOwner, error, reportFiles = [] }: { + page: Page; key: string; canStart: boolean; resumable: boolean; controllerOwner?: string | null; error: string; + reportFiles?: string[]; +}): string { + const artifact = (path: string): string => `/api/campaigns/${encodeURIComponent(key)}/artifacts/` + + btoa(path).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, ''); + const files = page === 'campaign' + ? '
Files
' + + `planstate` + + (reportFiles.includes('report/report.html') ? `report` : '') + + (reportFiles.includes('report/export-manifest.json') ? `export manifest` : '') + + '
' : ''; + const resume = resumable + ? '
' + + '' + + (error ? `${esc(error)}` : '') + '
' : ''; + const stop = canStart && controllerOwner + ? `
` + + '' + + (error ? `${esc(error)}` : '') + '
' : ''; + const nav = (on: boolean, label: string, href: string): string => + `${label}`; + return '
' + + 'STACK BENCH' + + `
${stop}${resume}${files}` + + 'New run
'; +} + +function shapeOf(plan: DashboardPlan): string { + const levels = plan.levels ?? []; + if (!levels.length) return DASH; + const depth = Math.max(...levels); + if (plan.mode === 'dependency') return `depth ${depth}`; + return levels.length > 1 ? `L${Math.min(...levels)}–L${depth}` : `L${depth}`; +} + +function planRow(plan: DashboardPlan): string { + const budgets = plan.budgets ?? null; + const stacks = plan.stacks ?? []; + const cell = (value: string, hover = ''): string => + `${value}`; + return `` + + `${esc(plan.title)}` + + `${esc(plan.mode ?? DASH)}` + + `${esc(shapeOf(plan))}` + + cell(stacks.length ? num(stacks.length) : DASH, stacks.join(' · ')) + + cell(num(plan.attempts)) + cell(num(plan.parallelism)) + + cell(plan.repairBudget === undefined ? DASH : num(plan.repairBudget)) + + cell(budgets ? duration(budgets.attemptTimeoutMinutes * 60) : DASH) + + cell(budgets ? money(budgets.maxCostUsdPerAttempt) : DASH) + + cell(budgets?.maxCostUsdPerAttempt != null && plan.attempts != null + ? money(budgets.maxCostUsdPerAttempt * plan.attempts) : DASH, + 'Maximum across planned attempts; each attempt cap includes its retries') + + `${esc(plan.state)}`; +} + +export function plansPage({ plans, loading = false }: { + plans: readonly DashboardPlan[]; loading?: boolean; +}): string { + return `

Saved plans

` + + '

Plans record the exact configuration behind a run. Use New run to select its settings.

' + + '
' + + HEADS.map(([label, kind]) => ``).join('') + + `${loading + ? `` + : plans.length ? plans.map(planRow).join('') + : ``}
${label}
Loading plans…
No plans
`; +} diff --git a/tools/stack-bench/dashboard/public/views/run-setup.ts b/tools/stack-bench/dashboard/public/views/run-setup.ts new file mode 100644 index 00000000000..f533bc08fdf --- /dev/null +++ b/tools/stack-bench/dashboard/public/views/run-setup.ts @@ -0,0 +1,168 @@ +import type { RunSetupCatalog, RunSetupRequest, RunSetupReview } from '../../../src/campaigns/run-setup.js'; +import { esc, money, modelLabel, stackLabel } from '../format.js'; +import { runName } from './plans.js'; + +const guidanceLabel = (id: string) => ({ neutral: 'Standard skills', + 'neutral-no-sdk': 'No SDK skills or dev workflow', + 'neutral-dev': 'Standard skills + dev workflow', + 'neutral-dev-no-sdk': 'Dev workflow without SDK skills' } as Record)[id] ?? id; + +export function initialRun(catalog: RunSetupCatalog, id?: string): RunSetupRequest | null { + const w = catalog.workloads.find(w => w.id === id) + ?? catalog.workloads.find(w => w.mode === 'dependency' && w.workSelection === 'progressive') + ?? catalog.workloads[0]; + if (!w) return null; + return { key: runName(w.track, new Date()) + '-' + crypto.randomUUID().slice(0, 8), + workload: w.id, workloadSha256: w.sha256, level: Math.max(...w.levels), stacks: [...w.stacks], + agents: [{ index: 0, effort: w.agents[0]!.effort ?? 'medium' }], + conditions: [(w.conditions.find(c => c.guidance === 'neutral-dev') + ?? w.conditions.find(c => c.guidance === 'neutral') ?? w.conditions[0])!.id], ...w.defaults, + productionQuality: true, maxCostUsd: w.defaults.maxCostUsd ?? 0, credentials: {} }; +} + +export function selectGuidance(conditions: RunSetupCatalog['workloads'][number]['conditions'], sdk: string, dev: string): string[] { + return conditions.filter(c => c.sdkSkills === (sdk === 'on') + && c.devWorkflow === (dev === 'on')).map(c => c.id); +} + +export function readRunForm(form: HTMLFormElement, catalog: RunSetupCatalog): RunSetupRequest { + const data = new FormData(form); + const workload = catalog.workloads.find(w => w.id === data.get('workload'))!; + const custom = [...new Set(workload.agents.map(a => a.adapter))].flatMap(adapter => { + const index = workload.agents.findIndex(a => a.adapter === adapter); + const model = String(data.get(`custom-model-${index}`) ?? '').trim(); + if (!model) return []; + const rate = (field: string) => { + const value = String(data.get(`custom-${index}-${field}`) ?? '').trim(); + return value ? Number(value) : Number.NaN; + }; + return [{ index, effort: String(data.get(`custom-effort-${index}`)) as RunSetupRequest['agents'][number]['effort'], + model, pricing: { input: rate('input'), output: rate('output'), + cacheWrite5m: rate('cacheWrite5m'), cacheWrite1h: rate('cacheWrite1h'), cacheRead: rate('cacheRead') }, + pricingSource: String(data.get(`custom-${index}-pricingSource`) ?? '').trim(), + pricingCapturedAt: `${String(data.get(`custom-${index}-pricingCapturedAt`))}T00:00:00.000Z`, + ...(data.get(`custom-${index}-maxOutputTokens`) ? { maxOutputTokens: rate('maxOutputTokens'), + outputLimitSource: String(data.get(`custom-${index}-outputLimitSource`) ?? '').trim() } : {}), + ...(data.get(`custom-${index}-providerRoute`) ? { providerRoute: String(data.get(`custom-${index}-providerRoute`)).trim() } : {}) }]; + }); + return { key: String(data.get('key')), workload: String(data.get('workload')), workloadSha256: String(data.get('workloadSha256')), + level: Number(data.get('level')), stacks: data.getAll('stack').map(String), + agents: [...data.getAll('agent').map(index => ({ index: Number(index), + effort: String(data.get(`effort-${index}`)) as RunSetupRequest['agents'][number]['effort'] })), ...custom], + conditions: data.has('sdkSkills') ? selectGuidance(catalog.workloads.find(w => w.id === data.get('workload'))!.conditions, + String(data.get('sdkSkills')), String(data.get('devWorkflow'))) : data.getAll('condition').map(String), repetitions: Number(data.get('repetitions')), + productionQuality: data.has('productionQuality'), + parallelism: Number(data.get('parallelism')), repairs: Number(data.get('repairs')), + timeoutMinutes: Number(data.get('timeoutMinutes')), maxCostUsd: Number(data.get('maxCostUsd')), + pauseAfterDepth: data.get('pauseAfterDepth') ? Number(data.get('pauseAfterDepth')) : null, + credentials: { adapters: Object.fromEntries([...data].filter(([key, value]) => + key.startsWith('credential-') && value).map(([key, value]) => [key.slice(11), String(value)])) } }; +} + +export function runSetupPage(catalog: RunSetupCatalog | null, request: RunSetupRequest | null, + review: RunSetupReview | null, error: string, canStart: boolean): string { + const field = (label: string, input: string) => ``; + const option = (value: string | number, label: string, selected: boolean) => + ``; + const integer = (name: string, value: number, min = 1) => ``; + const alert = error ? `` : ''; + const head = '

New run

'; + if (!catalog) return head + '

Loading setup…

' + alert + '
'; + if (!request || !catalog.workloads.length) return head + '

No runnable workloads are configured. Run appliance setup to install the workload presets.

' + + catalog.errors.map(error => `

${esc(error)}

`).join('') + ''; + const w = catalog.workloads.find(w => w.id === request.workload)!; + const delivery = w.mode === 'dependency' + ? ({ progressive: 'Progressive dependency graph', feature: 'One ready feature at a time', + 'all-at-once': 'Full graph in one build' }[w.workSelection as string] ?? w.workSelection) + : 'Sequential levels'; + const splitGuidance = w.conditions.length === 4 + && new Set(w.conditions.map(c => `${c.sdkSkills}:${c.devWorkflow}`)).size === 4; + const guidanceChoice = (key: 'sdkSkills' | 'devWorkflow', label: string) => { + const value = w.conditions.find(c => request.conditions.includes(c.id))?.[key] ? 'on' : 'off'; + return field(label, ``); + }; + const model = (index: number) => w.agents[index]!; + const customModel = (adapter: string) => request.agents.find(a => a.model + && w.agents[a.index]?.adapter === adapter); + const customRow = (adapter: string) => { + const index = w.agents.findIndex(a => a.adapter === adapter); + const value = customModel(adapter); + const fields = { pricingSource: value?.pricingSource, maxOutputTokens: value?.maxOutputTokens, + outputLimitSource: value?.outputLimitSource, providerRoute: value?.providerRoute }; + const input = (key: keyof typeof fields, label: string, type = 'text') => field(label, + ``); + const rate = (key: keyof NonNullable['pricing']>, label: string) => field(label, + ``); + return `Add ${esc(adapter)} model` + + `

` + + `

` + + field('Model ID', `` + + ``) + + field('Reasoning', ``) + + rate('input', 'Input / million tokens (USD)') + rate('output', 'Output / million tokens (USD)') + + rate('cacheWrite5m', '5-minute cache write') + rate('cacheWrite1h', '1-hour cache write') + + rate('cacheRead', 'Cache read') + input('pricingSource', 'Pricing source (URL)', 'url') + + field('Pricing checked on', ``) + + (adapter === 'codex' || adapter === 'openrouter' || adapter === 'grok-build' || adapter === 'antigravity' + ? input('maxOutputTokens', adapter === 'openrouter' ? 'Maximum output tokens' : 'Documented model maximum output tokens', 'number') + + input('outputLimitSource', 'Output limit source (URL)', 'url') : '') + + (adapter === 'openrouter' ? input('providerRoute', 'Provider route') : '') + + '
'; + }; + if (review) { + const rows = [ + ['Workload', `${w.title} · L${request.level}`], + ['Work delivery', delivery], + ['Stacks', request.stacks.map(stackLabel).join(', ')], + ['Models', request.agents.map(a => `${modelLabel(a.model ?? model(a.index).model)} (${a.effort})`).join(', ')], + ['Guidance', request.conditions.map(id => guidanceLabel(w.conditions.find(c => c.id === id)!.guidance)).join(', ')], + ['Production-quality app', request.productionQuality ? 'Requested' : 'Not requested'], + ['Runs', `${review.attempts} attempts · ${request.repetitions} per combination · ${review.parallelism} concurrent`], + ['Repairs', `${request.repairs} per attempt`], + ['Limits', `${request.timeoutMinutes} minutes and ${money(request.maxCostUsd)} per attempt`], + ['Total cost cap', money(review.maxCostUsd)], + ['Pause', request.pauseAfterDepth ? `After L${request.pauseAfterDepth}` : 'None'], + ['Account', review.authentication.map(a => `${a.adapter}: ${a.profile ? `${a.profile.id} (${a.profile.mode})` : `appliance default (${a.source})`}`).join(', ')], + ]; + return head + '

Review run

' + rows.map(([key, value]) => + `
${esc(key!)}
${esc(value!)}
`).join('') + '
' + + (review.qualification === 'pending' ? '

Grading qualification is pending. Results will be provisional.

' : '') + + '

Cost caps use recorded token pricing. Subscription usage is not an invoice charge.

' + + `
Recorded pricing and runtime
${esc(JSON.stringify({ pricing: review.pricing, runtime: review.runtime }, null, 2))}
` + + `
` + + '
' + alert + ''; + } + return head + (canStart ? '' : '

This dashboard is read-only. Start the appliance to run a study.

') + + `
` + + '
' + + field('Workload', ``) + + field('Target', ``) + + `

${esc(String(delivery))}

` + + '
Stacks
' + + w.stacks.map(id => ``).join('') + + '
Models and reasoning' + + w.agents.map((agent, index) => `
` + + `
`).join('') + + [...new Set(w.agents.filter(a => a.provider).map(a => a.adapter))].map(customRow).join('') + + '
App requirement' + + `` + + '

Build a production-quality application suitable for real users, not a prototype or demo.

' + + '
SpacetimeDB guidance' + + (splitGuidance ? '
' + guidanceChoice('sdkSkills', 'SDK skills') + + guidanceChoice('devWorkflow', 'Dev workflow') : '
' + w.conditions.map(c => ``).join('')) + + '
' + + field('Repetitions per combination', integer('repetitions', request.repetitions)) + + field('Concurrent attempts', integer('parallelism', request.parallelism)) + + field('Repairs per attempt', integer('repairs', request.repairs, 0)) + + field('Minutes per attempt', integer('timeoutMinutes', request.timeoutMinutes)) + + field('Cost cap per attempt (USD)', ``) + + '
Pause, run name and accounts
' + + field('Pause', `') + + field('Run name', ``) + + [...new Set(w.agents.map(a => a.adapter))].map(adapter => field(esc(adapter), `')).join('') + + '
' + + '
' + alert + '
'; +} diff --git a/tools/stack-bench/docker-compose.yaml b/tools/stack-bench/docker-compose.yaml new file mode 100644 index 00000000000..7c3675d51ed --- /dev/null +++ b/tools/stack-bench/docker-compose.yaml @@ -0,0 +1,49 @@ +# Databases for the Postgres and MongoDB backends. +# +# Development-only ports, container names and volumes keep this stack separate +# from the appliance. The SpacetimeDB +# backend needs no service here; run `spacetime start` for it. +# +# docker compose -f tools/stack-bench/docker-compose.yaml up -d +# +name: stack-bench + +services: + postgres: + image: postgres:16@sha256:219341e4cedb06c8634f80af40851da3425b41b76603fd890272f58e37e139f7 + container_name: stack-bench-dev-postgres + ports: + - "127.0.0.1:6532:5432" + environment: + POSTGRES_USER: appuser + POSTGRES_PASSWORD: local-app-password + POSTGRES_DB: app + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U appuser -d app"] + interval: 5s + timeout: 5s + retries: 12 + + # One local replica-set member exposes native transactions and change streams. + # This provides neither failover nor a multi-node availability test. + mongodb: + image: mongo:7@sha256:554a9bb1ec6e00c40ba078a41974a834d1a9a8ab1772645b69142afecc87f082 + command: ["mongod", "--replSet", "rs0", "--bind_ip_all"] + container_name: stack-bench-dev-mongodb + ports: + - "127.0.0.1:6537:27017" + volumes: + - mongodata:/data/db + healthcheck: + test: ["CMD", "mongosh", "--quiet", "--eval", "try { rs.status(); } catch (e) { if (e.code !== 94) throw e; rs.initiate({_id:'rs0',members:[{_id:0,host:'localhost:27017'}]}); } if (!db.hello().isWritablePrimary) quit(1)"] + interval: 5s + timeout: 5s + retries: 12 + +volumes: + pgdata: + name: stack-bench-dev-pgdata + mongodata: + name: stack-bench-dev-mongodata diff --git a/tools/stack-bench/docs/README.md b/tools/stack-bench/docs/README.md new file mode 100644 index 00000000000..e2ce93ea163 --- /dev/null +++ b/tools/stack-bench/docs/README.md @@ -0,0 +1,38 @@ +# StackBench documentation + +Start with the [README](../README.md), then [Getting started](../GETTING-STARTED.md) +for a first run. + +## Run it + +- [Appliance guide](../appliance/README.md): setup, credentials, plans, campaigns, + jobs, upload, results +- [Dashboard](../dashboard/README.md): pages and routes +- Skills, step-by-step procedures a person or a coding agent can follow: + [run a campaign](../skills/run-campaign/SKILL.md), + [recover a run](../skills/recover-run/SKILL.md), + [archive evidence](../skills/archive-evidence/SKILL.md), + [qualify the checks](../skills/qualify-checks/SKILL.md), + [build a release](../skills/release/SKILL.md) + +## How it works + +- [Prompting](prompting.md): what the agent receives and what is withheld +- [Grading](grading.md): outcomes, scoring, what the checks observe, and what + they do not prove +- [Study method](study-method.md): running a comparison that holds up +- [System design](system-design.md): owners, terms, data flow, and isolation +- [Reference apps](../reference-apps/README.md): the fixtures that prove the + checks +- [Grader](../grader/README.md): running and authoring scenarios + +## Change it + +- [Development](development.md): install, build, and which check to run +- [Authoring](authoring.md): add features, checks, weights, and study conditions +- [Ecommerce track](../tracks/ecommerce/LEVELS.md) and its + [composition](../tracks/ecommerce/composition/README.md) +- [Dependency graph](dependency-graph.html), generated by `npm run graph` + +Markdown under `backends/`, `conditions/`, `tracks/*/prompts`, and +`tracks/*/contracts` is benchmark input the agent receives. diff --git a/tools/stack-bench/docs/authoring.md b/tools/stack-bench/docs/authoring.md new file mode 100644 index 00000000000..8d819f834d9 --- /dev/null +++ b/tools/stack-bench/docs/authoring.md @@ -0,0 +1,107 @@ +# Author a benchmark change + +Product text lives in `tracks/`, shared actions in `src/actions/`, and stack +operations in `src/stacks/`. A feature or rule change does not need runtime, +report, or dashboard edits. Each command below builds `dist` first. + +## Add a feature + +The customer profile is the worked example. Its complete path: + +- `tracks/ecommerce/prompts/modular/customer-profile.md`: the product request +- `tracks/ecommerce/contracts/customer-profile.md`: the application interface +- `tracks/ecommerce/scenarios/progression-customer-profile.json`: the checks +- `tracks/ecommerce/composition/packs/progression-customer-profile.json`: the + feature pack, its dependencies, and its selected checks +- `tracks/ecommerce/composition/recipes/progression-catalog.json`: the recipe + that makes the pack available +- `tracks/ecommerce/progression/ecommerce.json`: the graph node and its parents + +For a new feature, follow that path with new IDs. Write the request as a +customer would ("A customer can save and view a delivery note"). Put the +sample data in the scenario. Use existing actions +(`signUp`, `click`, `fill`, `expect`). Keep one criterion per behavior; do not +award several points for several selectors that prove the same thing. Add the +behavior to every reference app. + +Validate, then review the rendered prompt for every stack and depth: + +```sh +npm run pack -- validate tracks/ecommerce/composition/packs/.json --track ecommerce +npm run recipe -- validate tracks/ecommerce/composition/recipes/progression-catalog.json --track ecommerce +npm run recipe -- show tracks/ecommerce/composition/recipes/progression-catalog.json --track ecommerce +npm run check:scenarios +npm run check:composition +npm run check:prompts +``` + +## Add a production check + +Put it in a `moduleType: "specification"` pack with a one-line product reason +(private delivery instructions belong to their owner), and select it through +the graph node's grading group. The feature request stays as it is. Do not add +probe strings or negative-test instructions to any stack's guidance. + +Then add a seeded bug that breaks exactly that behavior while everything else +keeps working, declare the scenario and check ID it must fail, and run +`npm run test:mutation-definitions`. Static tests prove nothing about +detection; see [Qualification](#qualification). + +## Match the observation to the claim + +- **Session persistence**: reload, then observe the signed-in user with no + sign-in step in between. +- **Server persistence**: observe from an independent client with no copied + browser storage. A reload alone can pass from browser storage. +- **Restart survival**: prove the behavior works first, restart the owned + runtime without reseeding, observe from a fresh client, and record which + process restarted. A runtime-control failure is not an app defect. +- **Live updates**: set the observer's initial state, change data through + another actor, then observe without a reload. The writer's own optimistic + update is not evidence. +- **Autonomous execution**: closing clients is not enough if the next request + runs the overdue work. Use an observation that cannot trigger it. +- **Absence**: establish readiness first. `waitUntilAbsent` tests eventual + disappearance; `expect` with `absent: true` tests continued absence for its + window. Neither proves permanent absence. +- **Contention**: prove a serial operation first, use independent actors, + classify every request, and reconcile stored state after the burst. A seeded + race bug must keep the serial operation working. +- **Timing**: prefer completion signals. Keep a fixed wait only when time is + the behavior under test, and say why in the scenario. + +Scenario steps can repeat with `{ "repeat": 10, "steps": [...] }` or +`{ "forEach": ["Beta", "Alpha"], "steps": [...] }`, where `"{value}"` takes +each listed string. Both compile to ordinary actions, at most 1,000 repetitions +and 10,000 steps. + +When a valid store design reveals an assumption in a shared action, fix the shared action or +the scenario's entry sequence, and add the smallest executor test that shows +the failure. + +## Change a requirement or a weight + +Edit the request, the interface if needed, and the scenario assertions +together, keeping test input in the scenario. Keep a stable check ID only while +the criterion means the same thing; a materially different behavior gets a new +ID. Weights are the criterion's `points` in its scenario; nowhere else. + +Save `recipe show` before and after, and compare selected checks, points, task +text, and hashes (`npm run recipe -- diff --track ecommerce`). Rerun +the scenario, composition, prompt, and seeded-bug checks, and `npm run graph` +if the graph changed. + +## Add a study condition + +Copy `conditions/guidance/neutral.json`, give it a new ID, register it in +`conditions/catalog.json`, and change only the treatment you want to compare. +Select it in a campaign definition. Do not add an agent-adapter branch for a +guidance change. + +## Qualification + +Any change to a prompt, scenario, reference app, weight, or the grading path +makes the existing evidence stale; produce new evidence with the +[qualify-checks skill](../skills/qualify-checks/SKILL.md). StackBench reads +qualification from evidence bound to the recipe hash. Do not add status fields +to recipes or references. diff --git a/tools/stack-bench/docs/dependency-graph.html b/tools/stack-bench/docs/dependency-graph.html new file mode 100644 index 00000000000..d1905959aba --- /dev/null +++ b/tools/stack-bench/docs/dependency-graph.html @@ -0,0 +1,1377 @@ + + + + + +Stack Bench | Dependency Graph + + + +
+
+
+

Ecommerce dependency graph

+

Each row is a questline. Select a feature to see what it needs and what it unlocks.

+
+
+ +
+
+ Qualified feature pack + Draft feature pack +
+
+ +
+
+ +
+
+
+
+ + + + diff --git a/tools/stack-bench/docs/development.md b/tools/stack-bench/docs/development.md new file mode 100644 index 00000000000..a1a59536878 --- /dev/null +++ b/tools/stack-bench/docs/development.md @@ -0,0 +1,87 @@ +# Development + +This guide covers working on the StackBench source. Running the benchmark is +in the [appliance guide](../appliance/README.md). + +## Set up + +You need Node.js 22 or newer, the repository's pinned pnpm through Corepack, +Docker Engine with Compose v2, and Chromium through the pinned Playwright +dependency. Campaign and resource-lock tests use Linux `flock`, so on Windows +or macOS run them in the source container below. + +```bash +pnpm --filter spacetimedb install --frozen-lockfile --ignore-scripts # crates/bindings-typescript, the SDK +cd tools/stack-bench +npm ci +npm run bootstrap:browsers +docker build --platform linux/amd64 -t stack-bench-build:local container +``` + +The last command builds the build image used by local tests. Real stack +execution needs the appliance images from the +[appliance guide](../appliance/README.md#set-up-for-model-runs). + +## Which check to run + +Run the smallest check that covers the change. Every test and check script builds `dist` +first and a build clears it, so do not run two at once; to run several checks +against one build, run `npm run build` and then call `node --test dist/tests/...` +or `node dist/commands/...` directly. + +| Change | Check | Runs | +|---|---|---| +| TypeScript | `npm run typecheck`, then the focused test | | +| Engine code | `npm test` | `tests/*.test.ts` | +| Dashboard read model, routes, pages | `npm run test:dashboard` | `tests/dashboard/*.test.ts`; writes thirty campaigns of fixture evidence | +| Tracks, prompts, references, repository policy, campaign definitions | `npm run test:contracts` | `tests/*.contract.ts` | +| Reference source, grading checks, seeded-bug manifests | `npm run test:mutation-definitions` | `tests/*.mutation.ts`; model-free | +| Browser, process, or Docker boundary | `npm run test:integration` | `tests/*.integration.ts`; serial, owns browsers, ports, and Docker | +| Prompt composition | `npm run check:prompts` | | +| Scenarios | `npm run check:scenarios` | | +| Packs and recipes | `npm run check:composition` | | +| Calibrations | `npm run check:calibration` | | +| Reference registry | `npm run check:references` | `reference-apps/registry.json` and each fixture's hash | +| Scenario, contract, reference, or SDK source behind a pinned hash | `npm run repin` | | +| Feature graph | `npm run graph` | | +| SpacetimeDB transport | `npm run build`, then `node --test dist/tests/spacetime-captured-replay.integration.js` | | +| Container preparation and reset publish | `npm run test:container` | needs Docker, the build image, and the local SpacetimeDB CLI | +| Teardown during a stack restart | `npm run test:faults` | same requirements | +| Whole bench loop with the deterministic agent | `npm run test:loop` | stub backend, `loop` track | + +Once a shared runtime, composition, grading, campaign, or release change is +stable, run the source gate once: + +```bash +npm run lint +npm run typecheck +npm run test:all +``` + +`test:all` runs the unit, dashboard, and contract tiers; integration and +mutation-definition tests stay separate. On a non-Linux host, run it in the +controller image's `source` target, which holds the source, development +dependencies, and compiled tests without the Rust binaries: + +```sh +docker build --platform linux/amd64 --target source -f tools/stack-bench/appliance/Controller.Dockerfile -t stack-bench-source-tests:local . +docker run --rm --init --network none stack-bench-source-tests:local npm run test:all +``` + +Browser tests under `tests/dashboard/*.browser.ts` run by hand with +`STACK_BENCH_BROWSER_TEST_URL` pointing at an isolated appliance. + +A passing check stays valid until one of its inputs changes. Add a test only +for an invariant no existing test covers. Documentation changes need link and +formatting checks only. + +## Pack budgets + +A pack's runtime budget is the time its checks may take on a clean reference. +For a pack whose budget is `unmeasured`, collect timings with +`qualify-reference --timing-only`. Timing artifacts cannot qualify a release. +Then run `pack-budget recommend`. It sets the budget to twice the slowest +passing timing, rounded up to the next second, and never below the waits the +pack's checks declare. A pack that exceeds its budget fails, so set a growing +pack's budget back to `unmeasured` before timing it again. Then qualify the frozen candidate as usual with the +[qualify-checks skill](../skills/qualify-checks/SKILL.md). diff --git a/tools/stack-bench/docs/grading.md b/tools/stack-bench/docs/grading.md new file mode 100644 index 00000000000..579f548adcc --- /dev/null +++ b/tools/stack-bench/docs/grading.md @@ -0,0 +1,166 @@ +# Grading + +This is how StackBench scores a store: what each check outcome means, how +outcomes become a score, what the checks observe, and what they do not prove. +Check counts and points come from the compiled recipe, and qualification status +from `qualification status`; this document holds neither. + +## Outcomes + +Every check ends in one outcome: + +- `passed`: every assertion held. +- `failed`: an assertion did not hold. This is a conclusive application failure. +- `blocked`: a prerequisite of the check failed, so the check never ran. This + is not evidence about the check's own assertion. +- `inconclusive`: StackBench could not reach a verdict, for example a response + that never finished or a fault that missed its window. +- `harness_failure`: StackBench itself failed, such as an invalid selector or a + browser protocol error. + +A result is **conclusive** when it is `passed` or `failed`. Only `passed` adds +the check's points. The other outcomes add nothing and never change the +denominator. Harness and provider failures never become application failures. + +Rules that decide the outcome: + +- A refusal is the stack's defined error result: HTTP 400, 401, 403, 409, or + 422; HTTP 404 only after the same operation succeeded for an authorized + caller; a SpacetimeDB reducer failure (HTTP 530, which also covers a panic); + or a thrown `ConvexError`. HTTP 500 is never a refusal. A missing route or an + incomplete response cannot prove authorization, so a refusal check without a + proven route is inconclusive. +- A page navigation timeout is inconclusive, because external resources can + delay readiness. A refused connection is a measured failure. +- If the store stops answering mid-grading, checks already measured keep their + outcomes, the rest of the current work fails, and unreached checks are + recorded as not run. +- Concurrent calls keep every request's outcome. A lost response or a timeout + is unknown; it is not proof of a rejection or a commit, and makes the response + assertion inconclusive. HTTP success alone does not prove the stored effect. +- A check file whose inconclusive results are all retryable runs once more on a + fresh database. Failures, cleanup failures, and harness failures do not retry. + +## Categories and counting + +Each check has a category, which only splits reports: + +- **Feature**: a requested product operation or result. +- **Production**: an access boundary, ownership rule, durability, concurrency, + deduplication, or synchronization property. +- **Interface**: presentation or a test hook with no separate assertion. +- **Unknown**: the frozen definition predates categories. + +Three measures share one denominator, the frozen selection, including blocked +and unmeasured work: + +- **Check completion**: passed checks over selected point-scoring checks. +- **Feature completion**: fully passed features over selected features. A + feature with a failed production check is not complete. +- **Weighted score**: passed points over selected points. + +Score on the leaderboard is feature completion averaged over valid runs. Checks +that score zero points, such as setup controls, are evidence only. + +## What the checks observe + +- **Pages.** Scenarios work with separate pages and single-page views. After a + reload they reopen the declared entry control and require the destination to + be visible before reading it. An absence assertion never passes because the + whole view is closed. +- **Readiness markers.** A destination marker identifies the opened view. An + `aria-busy="false"` marker means the signed-in account's list loaded, so a + failed read cannot earn empty-list credit. Markers are app-reported; the + business effect is still read fresh. +- **Stored data.** StackBench reads only the stock, order, and cart tables the + prompt requires, through the leased database. Missing or ambiguous data is + inconclusive, never a fabricated zero. +- **Business effects.** Price, transfer, cancellation, and return checks prove + the original effect before testing its preservation or reversal. An authorized + call proves the route before a refusal check uses it. +- **Live updates.** Live-update checks keep the observer on the page it already + had open. Every other check reads a fresh view. +- **Concurrent calls.** Timing is recorded on the client. Overlap on the way to + the server is not proof of overlap inside it. +- **Privacy.** StackBench records every HTTP response and WebSocket message a + browser receives, with the cache off, and searches for the owner's marker. + The owner's own recording must contain it. Unreadable or unfinished traffic + makes the check inconclusive; absence is proven only from complete evidence. +- **Scheduled work.** Checks anchor on the time the work was scheduled and + observe early and late. A missed window is inconclusive. Clocks are never moved. +- **Source.** Snapshots exclude logs, pid files, installed packages, and build + output. The store must start from the saved source alone. + +## Production check families + +| Family | What it requires | +| --- | --- | +| `state-durability` | Saved data survives a restart and a fresh sign-in; sessions survive a reload. | +| `access-control` | Customer data and staff operations are refused to the wrong caller, through direct server calls as well as the page. | +| `live-state` | Shared views reflect other users' changes without a reload. Polling that meets the window counts. | +| `concurrency-safety` | Several customers can act at once without overselling, lost orders, or double counting. | +| `external-data-sync` | A direct database write to stock shows in the store. | +| `transactional-integrity` | Stock and money are never created or lost by a partial operation. | +| `cancellation-queue`, `cancellation-accounting` | A cancelled order leaves the queue and reverses exactly its own effects, once. | +| `price-accounting`, `price-history` | Price edits do not rewrite earned revenue; receipts keep the agreed price. | +| `inventory-conservation` | A transfer moves stock without changing the total; an overdraw has no partial effect. | +| `operations-access`, `review-access`, `deferred-access` | Staff, review, and scheduling operations follow the product's roles on the server. | +| `deferred-durability`, `deferred-integrity`, `server-time` | Scheduled work survives a restart, runs once, and runs on time with the browser closed. | + +These checks cover the declared product behaviors. A score is not "production +readiness"; it says nothing about accessibility, operations, performance, or +security beyond the listed boundaries. + +Notes on individual checks: + +- **Shipping** waits for the declared submission state, then reads fresh staff + and customer views; it does not require a live queue update. +- **Stock alerts** require a confirmed alert request before the first restock. + The duplicate-alert check samples a fresh client ten seconds after the second + restock; it does not watch continuously. +- **Staff roles (621b, 621d)** request a role change through HTTP and reducer + replay and reload the administrator view. They do not test token revocation. +- **Purchase session (101a)** proves a direct purchase works signed in, then + requires refusal with no session, after rejected sign-ins, with a tampered + session token, and from an untrusted origin, with stock and orders unchanged. + A probe that cannot be built, such as mixed cookie and bearer credentials, is + inconclusive. +- **Checkout crash** kills the application and database processes separately + on PostgreSQL and MongoDB, and the single combined process on SpacetimeDB, + Convex, and Supabase. A missed fault window is inconclusive. + +## What the checks do not prove + +| Observation | Does not establish | +| --- | --- | +| Restart with retained data | Database crash recovery, power loss, or corruption recovery | +| Checkout interrupted by a process kill, then reconciled | Power loss, disk corruption, every crash timing, or external payment durability | +| Private marker absent from captured responses | Every endpoint, encoding, or object-reference attack | +| Exact final stock, orders, and totals | Every intermediate state, general serializability, or an external ledger | +| Bounded concurrent requests | Sustained throughput or overlap inside the server | +| Serial promotion redemption limit | A race for the last redemption | +| Hidden return and activity controls | Server-side return authorization or audit-log confidentiality | + +## Review a failed check + +A failed check records an observation that did not meet an assertion. It does +not identify a bug or its cause, and several checks can fail from one missing +update path or one failed setup step. + +For each failure you investigate, keep a short note beside the evidence: the +campaign, execution, source hash, and check IDs; the observed result, separate +from the proposed cause; whether the evidence shows an application defect, a +harness defect, or neither; and what evidence would settle it. Group checks +only when evidence shows a shared cause, and never report the group count as a +bug count. + +Stay inside the measured boundary. A button shown to a guest proves a +visibility failure only if the contract forbids it; it does not prove the +server accepts a guest purchase. A missing stock number does not prove +overselling. Use direct-call results and stored quantities for those claims. + +If the review finds a grader defect, keep the original artifacts, fix the +grader, and regrade the saved store into separate evidence with the +[regrade command](../appliance/README.md#regrade-a-saved-attempt). A regrade +is not a new build, and it cannot reconstruct the different path a dependency +run would have taken under the corrected gate. diff --git a/tools/stack-bench/docs/prompting.md b/tools/stack-bench/docs/prompting.md new file mode 100644 index 00000000000..6213bfaf3aa --- /dev/null +++ b/tools/stack-bench/docs/prompting.md @@ -0,0 +1,154 @@ +# Prompting + +StackBench gives the coding agent a normal software request. It does not say +that the agent is in a benchmark. What the prompt asks for and what StackBench +measures are separate choices, and this document covers the first. + +## What the agent receives + +Each prompt is assembled from versioned files and bound to the campaign by +hash: + +| Part | Owner | +| --- | --- | +| Framing: build a new app, or add to the existing one | Recipe | +| The features to build now | Feature packs | +| Production requirements the campaign chose to disclose | Specification packs | +| Stack setup: connection details, start command, selected SDK material | Guidance profile and backend document | +| The starting catalog | Fixture | +| Application interface: the element names and server operations the checks use | Feature contracts | +| A repair report, in repair runs only | Repair policy | + +A new-build prompt looks like this, abridged: + +```text +Build the application described below and leave it running. + +Build the app in /app. +The web application must listen on 0.0.0.0. + +## Stack + + +## New application + + +## + + +## Starting catalog + + +## Application interface + +``` + +The prompt never contains grader source, scenario files, check names, points, +scores, comparison results, the exact values a scenario uses, features that are +not ready yet, or production requirements under the `expected` and `observed` +treatments. + +By default the prompt also contains one line asking for production quality: +"Build a production-quality application suitable for real users, not a +prototype or demo." It is on for each condition with `productionQuality: true` +and recorded with the run. It changes the request, not the checks. + +## Production requirements and their treatment + +A feature is product work. A production requirement says how features must +behave under conditions such as reload, reconnect, concurrent writes, access +boundaries, or direct data changes. Each selected requirement has one +treatment: + +| Treatment | In the prompt | In the score | Repair feedback | +| --- | --- | --- | --- | +| `requested` | Yes | Yes | Yes | +| `expected` | No | Yes | After a conclusive failure | +| `observed` | No | No | No | + +For example, a campaign selects accounts as current work and session +durability as `expected`. The agent sees only the account feature. StackBench +still checks that a signed-in session survives a reload, and a failure counts. +Switching the same requirement to `requested` adds one line to the prompt and +changes nothing else, so the two conditions are comparable. + +`expected` is what lets a no-repair study measure production behavior that was +not asked for. The features, interfaces, and SDK material can still hint at a +requirement, so audit their text before claiming a behavior was unprompted. + +A product request states what the product does, clearly enough that it is not +ambiguous: reviews, cancellation before shipping, reservations, restock +scheduling. It does not give implementation instructions for restart survival, +exactly-once execution, or cross-account isolation. Those are specification +packs and enter the prompt only as `requested`. + +## Stack setup + +Every stack's setup names the stack, its connection details, and the start +contract. The agent chooses libraries, architecture, and project structure. +Stacks that are newer and less represented in model training also get SDK +skills; today that is SpacetimeDB. A condition's `guidanceProfile` picks one +of these, from [`conditions/catalog.json`](../conditions/catalog.json): + +| Profile | SpacetimeDB skills | +| --- | --- | +| `neutral` (default) | TypeScript server, TypeScript client, CLI | +| `neutral-dev` | The same, plus the `spacetime dev` watch workflow | +| `neutral-managed-dev` | The same, plus a managed start/status/stop helper | +| `neutral-no-sdk` | None | +| `neutral-dev-no-sdk` | Dev workflow only | +| `prescribed` | SDK skills plus stack documents with design advice | + +Only the SpacetimeDB material differs between the neutral profiles. The skills +are study inputs: keep their text intact, and compare profiles in separate runs. + +## Agent tools + +Every agent has files and a shell. Two tool groups follow the campaign's +`tools` setting, the same for every agent: `webSearch` (the agent's own search +and fetch tools) and `subagents`. Both are off by default, and `subagents` is +refused for every agent, because the contamination audit cannot see what a +sub-agent does. The container can reach public websites whatever the setting. + +The audit treats a fetch of StackBench's own source as contamination: anything +under `tools/stack-bench` in `clockworklabs/SpacetimeDB` on any host, GitHub or +a mirror, and its pull request. The rest of SpacetimeDB's source is open. + +## Application interfaces + +Feature contracts name the element IDs and server operations the checks use. +They do not prescribe layout, data model, framework, or visual design. The one +data-shaped exception is the stock interface, which other systems write to +directly. A contracted action must be the real UI action, not a separate +endpoint that can pass while the product is broken. + +## Dependency progression + +Each level's prompt contains the features that are ready now: a new app gets +the root features, an upgrade gets the newly unlocked ones. Upgrades keep the +original catalog and the interface contracts from earlier prompts +(`mode.retainPriorContracts`, default `true`), and never reset live data. +Blocked features are left out until their prerequisites pass. + +## Repair requests + +In repair runs, a repair starts only after grading records a conclusive +application failure. The agent gets a plain bug report: + +```text +Fix the reported application bugs. + +Expected: The signed-in account remains active after a reload. +Actual: The page returned to the signed-out state after reload. + +Change only what is needed. Do not alter behavior that is already correct. +``` + +The report gives the expected behavior, the grader's finding in one sentence, +the measured values, and the store's own console errors. It never includes +scenario scripts, fixture data, or implementation advice. Provider failures, +harness failures, and interrupted work never become bug reports. + +A repair can disclose an `expected` requirement, under the same rule for every +stack. Later levels carry earlier repairs, so a later level's first build is a +pre-repair checkpoint, not a fresh measure of unprompted behavior. diff --git a/tools/stack-bench/docs/study-method.md b/tools/stack-bench/docs/study-method.md new file mode 100644 index 00000000000..0e5a4dc2b4b --- /dev/null +++ b/tools/stack-bench/docs/study-method.md @@ -0,0 +1,115 @@ +# Study method + +How to run a comparison that holds up: what to decide before the first run, +what to freeze, what to measure, and what the results can and cannot claim. +The campaign plan and its evidence record what a study ran; this covers the +decisions around them. + +## What a study measures + +A study measures how much model usage each stack needs to build the same store, +and how much of the selected behavior passes. It compares stack packages as an +agent receives them, including the selected SDK skills. It is not a +database-only experiment, and no result can say how much of a difference came +from the database, the API, the documentation, or the model's familiarity. + +The baseline design is dependency mode, progressive work selection, no repairs, +no retries. A repair-enabled study is a separate experiment with repair cost +included, and a claim that repairs help needs matching repair and no-repair +cohorts. + +## Before the first run + +Set the sample count from the question, and start every repetition from a +clean store and a fresh agent session. A pilot count is not a power calculation. + +| Stage | Collection | Exit condition | +| --- | --- | --- | +| Pilot | One block: one attempt per stack | The harness produces valid measurements | +| Comparison | A fixed number of blocks under one frozen protocol | Every result and its variation are shown | +| Confirmation | A separate batch, sized from pilot variance and a decision threshold | A stated claim is tested with uncertainty | +| Wider scope | Deeper work, another product, or another model | Findings extend, or do not, beyond the first condition | + +Freeze before the comparison batch, and start a new dataset if any of it +changes: the feature graph and target depth, the selected checks and their +treatments, the guidance profile, the images, the model and adapter versions, +the time and cost caps, the repetition count and concurrency, the exclusion +rule, and the analysis. The plan records most of this; keep the research +question and analysis decisions in a short note beside it. + +Set one per-attempt cost cap, well above observed usage. It is a safety stop, +not a condition. An attempt that hits it is recorded as capped and rerun under +the plan's rule; it stays in the assigned population and leaves the +completed-only distributions. Never add runs until a preferred stack wins, and +never choose representative apps after seeing scores. + +Run balanced waves, with every stack in each wave, in the plan's rotated order. +Host capacity delays a launch; it never changes the requested parallelism. +Report contention separately from application defects, and keep batch identity +if capacity changes between batches. + +## Measures + +Cost and completion are the two primary outcomes. Never fold them into one +composite. + +| Measure | Interpretation | +| --- | --- | +| Check completion | Passed checks over selected checks, with both counts | +| Feature completion | Fully passed features over selected features | +| Full target delivery | Complete targets over all assigned attempts; stops and exclusions stay in the denominator | +| Reach | Features started, passed, failed, and blocked at each depth | +| Cost | From receipts at the plan's rates; exact, upper bound, and unknown stay distinct, and unknown is never zero | +| Tokens | Input, output, cache reads, and cache writes separately | +| Time | Wall time, planned pauses, and working time separately | +| Reliability | Harness and provider failures, cleanup failures, and cap stops | +| Regression | Checks that passed and later failed, with the checkpoint that lost them | + +Eligibility is per measure: a completed run keeps its completion when its cost +is unknown, but a run whose missing receipts prevent cap verification is not +eligible for comparison. Blocked work is not a measured failure; label it +blocked. Do not sum repeated checks across depths, and do not report completion +only among runs that reached the target depth. + +Show every attempt with the median, interquartile range, and mean, and state +the sample size beside every comparison. Analyze whole attempts or blocks; a +check is not an independent sample, and levels, repairs, and regrades of one +store are not new builds. Use a binomial interval for full-target success +rates. For a confirmation, choose the smallest decision-relevant difference +first and size the batch from observed between-build variation. Three runs per +stack support an initial comparison, not a claim of superiority. + +## What the results can claim + +- A package-level difference: this agent, with this stack setup, delivered + more or less. Not why. +- Delivery under the gating policy. A failed feature blocks the features built + on it, so two attempts can receive different later work. Show the originating + failure and its blocked work. +- Coverage of the frozen checklist, on this ecommerce workload. For a broader + claim, preselect materially different workloads. +- Completion that depends on what the checks can observe. Report measurement + failures separately and inspect their distribution by stack. +- Cost conditional on the cap. Report any raised-cap rerun. + +Qualification of the exact selection, which proves every check on every +stack's reference app, seeded bugs, and the empty app, is required before +results are published as verified; see the +[qualify-checks skill](../skills/qualify-checks/SKILL.md). Collection can start +before that, labelled provisional. + +## Failure review and external review + +Review failures with the [grading procedure](grading.md#review-a-failed-check), +without stack labels where feasible. Do not repair generated stores by hand in +the primary dataset. A new prompt or interface requirement needs a new cohort. + +Before a public comparative claim, get independent external review of the +frozen protocol, exclusions, scoring, and analysis; record unresolved +objections and reviewer affiliations. Review by parallel agents is an internal +check. Do not claim replication until another team reproduces the method. + +Keep the complete campaign evidence privately, archived with the +[archive-evidence skill](../skills/archive-evidence/SKILL.md). The research +pack from `campaign export` omits source, transcripts, and media; review its +free text before sharing it. diff --git a/tools/stack-bench/docs/system-design.md b/tools/stack-bench/docs/system-design.md new file mode 100644 index 00000000000..342ceb37835 --- /dev/null +++ b/tools/stack-bench/docs/system-design.md @@ -0,0 +1,153 @@ +# System design + +StackBench turns one versioned plan into comparison evidence that can be traced +without chat history or operator memory. Every decision, action, result, and +cost has one owner and one durable record. + +## One owner for each fact + +| Layer | Owns | Durable output | +| --- | --- | --- | +| Definitions | Product work, prompt modules, checks, stacks, models, budgets | Versioned source files | +| Compiler | The exact work matrix and every bound identity | `plan.json` | +| Job store and worker | Submission, host placement, credential references, the execution claim | Job and claim records | +| Admission | Whether this plan can run on this appliance | Admission artifact | +| Scheduler | Attempt order, concurrency, continuations, terminal state | `state.json` | +| Run engine | Build, grade, repair, resource ownership, cleanup | Attempt directory | +| Grader | Typed check results and evidence | Grade artifacts | +| Progression engine | Open, passed, failed, and blocked features | `progression-state.json` | +| Report | A reproducible view of the evidence | `report.json`, `report.html` | + +No layer replaces a decision from the layer above it. A view can summarize +durable data; it cannot create run state. + +## Where code lives + +The source is TypeScript, built to ESM JavaScript in `dist/`. + +| Path | Owns | +| --- | --- | +| `src/campaigns/` | Plan compilation, scheduling, jobs, budgets, reporting | +| `src/progression/` | Feature dependencies, work selection, repair state | +| `src/composition/` | Pack and recipe compilation, prompt composition, identities | +| `tracks/` | Product work, interfaces, scenarios, feature graphs | +| `src/agents/`, `container/` | Coding adapters, provider protocols, the broker, transcripts | +| `src/stacks/`, `src/runtime/` | Stack operations, isolation, source checkpoints, cleanup | +| `grader/`, `src/evidence/` | Executable checks, seeded bugs, evidence validation | +| `commands/`, `dashboard/` | CLI and browser interfaces to the same engine | +| `skills/` | Operator procedures | + +## Terms + +One word per thing. The CLI, dashboard, and artifacts use these. + +- **campaign**: one comparison job, defined by a + [plan](../appliance/README.md#plans); a result directory holds everything it + produced. The dashboard calls a campaign a run. +- **attempt**: one stack building the product once inside a campaign. +- **execution**: one process run of an attempt. A retried attempt has two. +- **session**: one conversation with the coding agent: a build session or a + repair session. +- **stack**: the technology under test. Flags still spell it `--backend`. +- **level**: one rung of a sequential campaign. **depth**: how far down the + feature graph a dependency campaign has reached. Same field, different meaning. +- **feature**: one node of the graph, the unit the agent builds and the grader + scores. It opens when its parents pass. +- **questline**: a named path of features through the graph, such as + `identity-profiles` or `orders-fulfillment`. +- **check**: one scored criterion with a stable ID such as `601b`. Feature + checks measure product behavior; production checks measure required + properties of it. +- **treatment**: how a specification reaches the agent; see + [prompting](prompting.md). +- **first build**: the score before any repair. **repair**: one paid session + that reacts to a failure report, plus the regrade after it. +- **passed**, **failed**, **blocked**, **inconclusive**, **harness failure**: + the check outcomes, defined in [grading](grading.md). A **provider failure**, + a harness failure, and a **contaminated** attempt are excluded from comparison. +- **qualification**: matching reference, seeded-bug, and empty-app evidence for + an exact check selection. Without it, results are provisional. +- **preflight**: the checks before an attempt starts. **admission**: the record + that preflight and policy allowed it. +- **coding container**: where the agent works, created from the build image. +- **clean source**: the accepted source with nothing the agent's process left + behind. Every grade starts the store from clean source. +- **broker**: the proxy that holds the provider key so the coding container + never sees it. Its **cost receipt** proves what a session spent. +- **lease**: the record of the containers, ports, database, and locks an + attempt owns, so cleanup and recovery touch only those. + +## Data flow + +```text +versioned definitions + | + v +compiled plan -> admission -> scheduler -> run engine -> grader + | | | + v v v + state.json run.json evidence + \ | / + \ v / + -> inspection -> report +``` + +What the agent receives is in [prompting](prompting.md). + +## Appliance isolation + +The controller owns definitions, grading, results, provider secrets, and the +Docker socket. It runs on Docker's host network so it can reach the ports each +attempt publishes on loopback. The dashboard runs on a bridge and publishes only +`127.0.0.1:7331`; its controls start a fresh controller through Compose. + +Each attempt owns one database container and one bridge network. The database +container anchors the network namespace: the coding container, browser, smoke +container, and broker join that namespace and share nothing with other +attempts. Before creating anything, the controller writes the attempt's lease and +claims its port and capacity locks. It records creation authority before +`docker create` and exact IDs before `docker start`. A short helper with +`NET_ADMIN` then loads the nftables rules, and only then does the database +start. Generated code and the browser cannot change the firewall. + +The namespace may reach its own loopback, the package cache at its address on +port 4873, and public IPv4 HTTP and HTTPS. It cannot reach the host, private +ranges, other attempts, or IPv6. The package cache joins each attempt's bridge +for the attempt's life; publishing and sign-up are disabled on it. + +The browser has no Docker socket, provider secret, app mount, or control port; +Playwright drives it through `docker exec`. The coding container gets its app +workspace, its transcript directory, and the selected stack's dependency +mounts, with a read-only root filesystem and `no-new-privileges`. It never +gets the repository, grader, scenarios, results, Docker socket, or credential +file. The broker gets the credential file; the agent gets a session token. + +All state lives in the `stack-bench-state` volume, mounted in the controller +at Docker's own mountpoint so child bind mounts name the same bytes on Docker +Desktop and Linux. PostgreSQL and MongoDB get private per-attempt credentials +and unpublished ports. MongoDB runs as a single-node replica set, which allows +transactions and change streams but does not test failover. + +Teardown removes the exact owned containers, volumes, and network, releases the +locks, and leaves anything it cannot prove it owns. A restart of the database +container invalidates the recorded namespace and requires recovery of the whole +attempt; see the [recover-run skill](../skills/recover-run/SKILL.md). + +## Operator loop + +1. **Define**: select one versioned plan. Do not rebuild a plan from flags. +2. **Validate**: compile it and inspect the attempts, model, prompt policy, + checks, points, budgets, images, and parallelism. +3. **Admit**: prove credentials, images, ports, capacity, and stack access + before model work starts. +4. **Run**: start the stored plan or an eligible continuation. A paid action is + always explicit. +5. **Observe**: read durable campaign state first; open logs only to diagnose. +6. **Decide**: continue only through a legal state transition. Never hide an + invalid attempt or retry it outside the frozen policy. +7. **Report**: generate the result from retained evidence. +8. **Clean**: remove temporary owned resources and keep the campaign package. + +The CLI and the dashboard share the compiler, scheduler, state reader, and run +commands. Machine-facing commands return stable JSON that names the plan, +campaign state, active work, cost, failures, and legal next actions. diff --git a/tools/stack-bench/grader/README.md b/tools/stack-bench/grader/README.md new file mode 100644 index 00000000000..a3166335185 --- /dev/null +++ b/tools/stack-bench/grader/README.md @@ -0,0 +1,77 @@ +# Grader + +The grader runs versioned scenarios against a generated store and records +browser, transport, lifecycle, and database evidence for each check. What the +outcomes mean and what the checks observe is in [grading](../docs/grading.md). + +Each scenario actor gets its own browser context. A live-update check passes +only when the page that was already open changes. The grader never reloads a +failed assertion to try again. + +## Database reset + +Between check files, every stack runs the store's normal startup after the +database is emptied, including SpacetimeDB stores that initialize outside the +module's `init`. The prompt requires startup to create the starting data on an +empty database and to preserve existing data otherwise. PostgreSQL resets +recreate only the leased database. Durability probes restart services without +resetting data. + +Convex runs a pinned self-hosted backend in each attempt's private network. A +Convex backend crash also stops its application functions, so the grader +measures that boundary once. + +## Failure reports + +An action never fails with a sentence. It fails with a finding from the closed +catalog in `src/actions/action-findings.ts`: a kind and its fields, where a +field is a control name, an action ID, an actor, a number, or an HTTP status. +Every reader renders the finding from its one template. Raw diagnostics travel +in a `detail` field that is never rendered. + +## Scenarios + +Scenario JSON holds actors, setup steps, actions, and scored checks. The +actions are compiled and registered in `src/actions/`; scenario prose is not +executable. Each stack adapter declares the capabilities it provides, and the +campaign compiler refuses a campaign that a selected stack could not measure. + +When writing assertions: + +- scope repeated elements to their owning row, order, or user; +- assert visible values, never the presence of an empty container; +- keep the original open page for live-update behavior; +- use separate actors for identity boundaries; +- say in the criterion's `note` why its points differ from its neighbors'. + +```json +{ "do": "expect", "actor": "watcher", "testid": "item-stock", + "in": { "testid": "item-card", "contains": "Air Purifier" }, "within": 10000 } +``` + +## Run it + +`dist/commands/run-suite.js` is the normal entry: it owns database reset, +provenance checks, contract linting, scenario execution, logs, and the grade +bundle. Direct `dist/grader/grade.js` runs are for authoring one scenario: + +```bash +node dist/grader/grade.js --url http://localhost:6173 --spec tracks/ecommerce/scenarios/01-account-create.json --label spacetime-l1 --out report.json +``` + +If the grader exits before writing JSON, read the retained +`grader-.stdout.log` and `grader-.stderr.log`. + +`--media ` records videos and failure screenshots; `--trace` adds a +Playwright trace, viewed with `npx playwright show-trace `. Inspect +the failing actor's evidence before attributing a failure. Media is not tracked +in the repository. + +Preflight binds the stack adapter, database or module name, ports, container +identity, and run lease, and the suite runner verifies that target before +grading. A mismatch is a harness failure. + +Proving the checks against the reference apps, seeded bugs, and the empty app +is the [qualify-checks skill](../skills/qualify-checks/SKILL.md). +`npm run check:mutations -- --app --mutations ` +checks seeded-bug definitions and source anchors without starting a store. diff --git a/tools/stack-bench/grader/grade.ts b/tools/stack-bench/grader/grade.ts new file mode 100644 index 00000000000..28f4d1cbabc --- /dev/null +++ b/tools/stack-bench/grader/grade.ts @@ -0,0 +1,1408 @@ +#!/usr/bin/env node +/// +// Score declared criteria from one observed run in isolated actor contexts. +// +import { chromium } from 'playwright'; +import { attemptBrowserLaunchOptions } from '../container/browser-pipe.js'; +import type { Browser, BrowserContext, Page, Request, Response } from 'playwright'; +import { sanitiseConsoleError } from '../src/evidence/diagnostic-sanitizer.js'; +import { inspectSavedDiagnostic } from '../src/runtime/saved-diagnostic.js'; +import { randomUUID } from 'node:crypto'; +import { readFileSync, mkdirSync } from 'node:fs'; +import { basename, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseArgs as parseNodeArgs } from 'node:util'; +import { harnessBrowserFailure, harnessProcessFailure, + runBrowserInfrastructureOperation } from '../src/evidence/harness-errors.js'; +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; +import { materializeScenarioCredentials } from '../src/composition/credential-aliases.js'; +import { loadTrack } from '../src/composition/tracks.js'; +import { isFinding } from '../src/actions/action-findings.js'; +import type { Finding } from '../src/actions/action-findings.js'; +import { recipeArtifactIdentities, writeArtifact } from '../src/evidence/artifacts.js'; +import { resolveCalibrationForRelease } from '../src/composition/calibration-compiler.js'; +import { resolveGradeRecipeArtifactBinding } from '../src/composition/recipe-release.js'; +import { resolveBoundRecipeTaskRequest, selectScenarioChecks } from '../src/composition/recipe-selection.js'; +import { ACTION_REGISTRY } from '../src/actions/action-catalog.js'; +import { ActionApplicationFailure, ActionInconclusive, executeAction } from '../src/actions/action-contract.js'; +import { fail, inconclusive } from '../src/actions/actor-action-runtime.js'; +import { runApplicationNavigation } from '../src/actions/browser-navigation.js'; +import { createCheckEvidence, evidenceIsMeasured, evidencePassed } from '../src/evidence/check-evidence.js'; +import { evidenceNowMs } from '../src/evidence/evidence-timing.js'; +import { renderEvidenceConsoleLine } from '../src/evidence/evidence-presentation.js'; +import { measureGradePackRuntime } from '../src/composition/pack-runtime.js'; +import { STACK_ADAPTER_REGISTRY } from '../src/stacks/stack-adapters.js'; +import { stackDatabaseRuntime } from '../src/stacks/process-crash.js'; +import { stableElementSelector } from '../src/actions/element-selector.js'; +import { + createNamedActionsCapability, +} from '../src/actions/actor-transport-action-executors.js'; +import type { ConcurrentCallResult } + from '../src/actions/actor-transport-action-executors.js'; +import { + createDatabaseWriteCapability, + createDatabaseReadCapability, + createLifecycleCapability, +} from '../src/actions/runtime-action-executors.js'; +import type { LeasedDatabase } from '../src/stacks/backend-reset-guard.js'; +import { createMongoDbOrderDataReader } from '../src/stacks/backends/mongodb-operations.js'; +import { createConvexOrderDataReader } from '../src/stacks/backends/convex-operations.js'; +import { controlAppServer, controlBackendRuntime, parseRuntimeControlSpec, prepareRuntimeCrash } + from '../src/runtime/backend-control.js'; +import type { RuntimeControlSpec } from '../src/runtime/backend-control.js'; +import { leaseFromEnv } from '../src/runtime/backend-lease.js'; +import type { LeasedSpacetimeTarget } from '../src/runtime/spacetime-target.js'; + +import { STACK_BENCH_ROOT as ROOT } from '../src/package-root.js'; +import { captureResponses, ReceivedTransport, requestDiagnostic, urlDiagnostic } from './transport-frames.js'; +import { installResponseLoss } from './response-loss.js'; +import { installAuthWebSocketCapture, type PassiveSocketFrame } from '../src/actions/auth-request-patch.js'; +import type { PlatformAuthPatch } from '../src/actions/auth-request-patch.js'; +import type { DeliveryMark } from '../src/actions/private-delivery.js'; +import { applicationWrites } from '../src/actions/named-action-runtime.js'; +import { installSpacetimeWriteCapture } from '../src/stacks/backends/spacetime-browser-session.js'; +import { startNetworkInterruption, type NetworkInterruption } from '../src/actions/network-interruption.js'; +import { recordConvexSession, recordConvexMutationResult } from '../src/stacks/backends/convex-browser-session.js'; +import type { ActionEvidence } from '../src/actions/action-contract.js'; +import type { CheckEvidence, CheckEvidenceAttachment, CheckEvidencePhase, + CheckEvidenceStatus } from '../src/evidence/check-evidence.js'; +import type { CompletedGradeFeatureResult, CompletedGradeReport, GradeCleanupFailure } + from '../src/evidence/grade-report.js'; +import type { CompiledFeature, CompiledScenarioDefinition, + CompiledStep } from '../src/composition/definition-compiler.js'; +import type { RecipeCheck, RecipeGradeRelease, RecipeRelease } from '../src/composition/recipe-release.js'; +import type { TrackAction } from '../src/composition/tracks.js'; + +type JsonRecord = Record; +type ActorWrite = { + accepted?: boolean; + confirmed?: boolean; + url: string; + method: string; + headers: Record; + body: JsonRecord | null; +}; +type ActorWebSocketWrite = { event: unknown; body: JsonRecord }; +type ActorContextEntry = { context: BrowserContext; name: string; page: Page | null; traceStarted?: boolean }; +type CleanupBrowserContext = { + tracing: { stop(options: { path: string }): Promise }; + close(): Promise; +}; +type CleanupVideo = { saveAs(path: string): Promise; delete(): Promise }; +type CleanupPage = { video(): CleanupVideo | null }; +type CleanupActorContextEntry = { + traceStarted?: boolean; + context: CleanupBrowserContext; + name: string; + page: CleanupPage | null; +}; +type FeatureResult = Omit & { + setupEvidence?: CheckEvidence; +}; +type GradeArgs = { + url?: string; + level: number; + headed: boolean; + selectedCheckKeys: string[]; + out?: string; + label?: string; + feature?: number; + spec?: string; + restartSpec?: RuntimeControlSpec; + backend?: string; + track?: string; + recipe?: string; + recipeTask?: Parameters[1]; + expectedRecipeSha256?: string; + credentialAliases?: unknown; + selectionSha256?: string; + parentAttemptId?: string; + dbName?: string; + app?: string; + media?: string; + failureMedia?: string; + trace?: boolean; + nullControl: boolean; + diagnostic?: boolean; + savedDiagnostic?: ReturnType; + browserWsEndpoint?: string; + timeLimitMs?: number; +}; +type GradeRunContext = { + contractIds?: readonly string[]; + savedReader?: { path: string; sha256: string }; + checkoutActivity?: { unsettled: boolean }; + checkoutSnapshots?: ReturnType['checkoutSnapshots']; + mongoOrderReader?: ReturnType; + convexOrderReader?: ReturnType; + authRequestPatch?: PlatformAuthPatch | null; + authReadEndpoints?: readonly string[]; + passiveSocketFrame?: PassiveSocketFrame; + applicationWriteEndpoints?: readonly string[]; + actionCancellation?: { reason: string | null }; + timeLimit?: { limitMs: number; deadlineAtMs: number }; + runId: string; + roomName: (base: string) => string; + restartSpec?: RuntimeControlSpec; + url: string; + backend?: string; + actions: TrackAction[]; + spacetime: LeasedSpacetimeTarget | null; + dbName?: string; + databaseLease?: LeasedDatabase | null; + appDir?: string; + scope?: string; + extraContexts?: ActorContextEntry[]; + interruptions?: NetworkInterruption[]; + privacyActors?: ReadonlySet; + recorded?: Record; + unverified?: string[]; + verified?: string[]; + actionEvidence?: Array<{ actor: string | null; evidence: ActionEvidence }>; + serverCheck?: string | null; + lastCalls?: ConcurrentCallResult | null; + defaultWithin?: number; + nullControl: boolean; + // True while a scenario step has stopped the application server and no + // later step or restore has started it again. + applicationStopped?: boolean; +}; +type ActionFailure = Error & { actionEvidence?: ActionEvidence; actionActor?: string | null }; +const APPLICATION_RESTORE_SETTLE_MS = 8000; +const APPLICATION_RESTORE_TIMEOUT_MS = 60_000; +// A check still running at the grading time limit has not passed. Recording it keeps +// the rest of the level measured instead of losing the grader process. +function enforceTimeLimit(ctx: GradeRunContext): void { + if (ctx.timeLimit && Date.now() >= ctx.timeLimit.deadlineAtMs) { + fail('grading-time-limit', { limitMs: ctx.timeLimit.limitMs }); + } +} +class ApplicationNotRestored extends Error { + constructor(reason: string) { + super(`the application server stopped by the harness was not restored: ${reason}`); + } +} +type CheckFailure = { + status: CheckEvidenceStatus; + code: string; + actor: string | null; + summary: string | null; + finding: Finding | null; + observation: unknown; + expected: unknown; + retryable: boolean; +}; + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function actionFailure(error: unknown): ActionFailure | null { + return error instanceof Error ? error as ActionFailure : null; +} +// The sentence the coding agent was given for this behaviour travels with the +// grade, so a repair report can repeat it instead of describing the check. +const authored = (criterion: { statedBy?: string }): { statedBy?: string } => + criterion.statedBy ? { statedBy: criterion.statedBy } : {}; +const DEFAULT_WITHIN = 5000; +const SETUP_WITHIN = 20000; +// Keep the cause when Playwright prefixes it with locator retry details. +function keepReason(detail: unknown, limit = 600): string { + const s = String(detail ?? ''); + if (s.length <= limit) return s; + const [head, ...rest] = s.split('\n'); + const reasons = rest + .map(l => l.trim()) + .filter(l => /^-\s/.test(l)) + .map(l => l.replace(/^-\s*/, '')) + .filter(l => !/^(waiting for|retrying|attempting|scrolling|done scrolling|locator resolved to|\d+ ×)/i.test(l)); + const kept = [...new Set(reasons)].slice(0, 4); + const out = kept.length ? `${head}\n - ${kept.join('\n - ')}` : s.slice(0, limit); + return out.length > limit ? out.slice(0, limit) : out; +} + +export function parseGradeArgs(argv: readonly string[]): GradeArgs { + const { values } = parseNodeArgs({ args: [...argv.slice(2)], options: { + url: { type: 'string' }, level: { type: 'string' }, out: { type: 'string' }, + label: { type: 'string' }, feature: { type: 'string' }, spec: { type: 'string' }, + 'restart-spec': { type: 'string' }, backend: { type: 'string' }, track: { type: 'string' }, + recipe: { type: 'string' }, 'expected-recipe-sha256': { type: 'string' }, + 'recipe-task-json': { type: 'string' }, + 'selected-check': { type: 'string', multiple: true }, + 'credential-aliases-json': { type: 'string' }, 'selection-sha256': { type: 'string' }, + 'parent-attempt-id': { type: 'string' }, 'db-name': { type: 'string' }, + app: { type: 'string' }, media: { type: 'string' }, 'failure-media': { type: 'string' }, + trace: { type: 'boolean' }, headed: { type: 'boolean' }, + 'null-control': { type: 'boolean' }, + diagnostic: { type: 'boolean' }, + 'saved-diagnostic': { type: 'string' }, + 'browser-ws-endpoint': { type: 'string' }, + 'time-limit-ms': { type: 'string' }, + } }); + const args: GradeArgs = { url: values.url, level: values.level === undefined ? 1 : Number(values.level), + out: values.out, label: values.label, + feature: values.feature === undefined ? undefined : Number(values.feature), spec: values.spec, + restartSpec: values['restart-spec'] === undefined ? undefined + : parseRuntimeControlSpec(JSON.parse(values['restart-spec'])), + backend: values.backend, track: values.track, recipe: values.recipe, + recipeTask: values['recipe-task-json'] === undefined ? undefined : JSON.parse(values['recipe-task-json']), + expectedRecipeSha256: values['expected-recipe-sha256'], + selectedCheckKeys: values['selected-check'] ?? [], + credentialAliases: values['credential-aliases-json'] === undefined + ? undefined : JSON.parse(values['credential-aliases-json']), + selectionSha256: values['selection-sha256'], parentAttemptId: values['parent-attempt-id'], + dbName: values['db-name'], app: values.app, media: values.media, + failureMedia: values['failure-media'], trace: values.trace, headed: values.headed ?? false, + nullControl: values['null-control'] ?? false, + diagnostic: values.diagnostic ?? false, + browserWsEndpoint: values['browser-ws-endpoint'], + timeLimitMs: values['time-limit-ms'] === undefined ? undefined : Number(values['time-limit-ms']) }; + if (args.timeLimitMs !== undefined && !(Number.isSafeInteger(args.timeLimitMs) && args.timeLimitMs > 0)) { + throw new Error('--time-limit-ms must be a positive integer'); + } + if (!args.url || !args.spec) { + throw new Error('Usage: node dist/grader/grade.js --url --spec ' + + '--level [--out ] [--label ] [--feature ]'); + } + if (args.diagnostic && (args.recipe || args.recipeTask !== undefined || args.expectedRecipeSha256 || args.selectedCheckKeys.length)) { + throw new Error('diagnostic grades cannot select a scored recipe or check catalog'); + } + if (values['saved-diagnostic']) { + if (!args.diagnostic) throw new Error('saved readers require zero-point diagnostics'); + args.savedDiagnostic = inspectSavedDiagnostic(JSON.parse(values['saved-diagnostic']), process.cwd()); + if (args.backend !== args.savedDiagnostic.backend) throw new Error('saved diagnostic backend mismatch'); + } + let url: URL; + try { url = new URL(args.url); } + catch { throw new Error('--url must be a valid HTTP or HTTPS URL'); } + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + throw new Error('--url must use HTTP or HTTPS'); + } + if (!Number.isInteger(args.level) || args.level < 1) { + throw new Error('--level must be a positive integer'); + } + if (args.feature !== undefined && (!Number.isInteger(args.feature) || args.feature < 1)) { + throw new Error('--feature must be a positive integer'); + } + if (args.selectionSha256 && !/^[a-f0-9]{64}$/.test(args.selectionSha256)) { + throw new Error('--selection-sha256 must be 64 lowercase hexadecimal characters'); + } + if (args.browserWsEndpoint) { + let endpoint: URL; + try { endpoint = new URL(args.browserWsEndpoint); } + catch { throw new Error('--browser-ws-endpoint must be a valid WebSocket URL'); } + if (!['ws:', 'wss:'].includes(endpoint.protocol)) { + throw new Error('--browser-ws-endpoint must use ws or wss'); + } + } + return args; +} + +const tid = stableElementSelector; +const uniq = () => randomUUID().slice(0, 16); +const MAX_CONSOLE_ERRORS = 200; + +// Isolated browser actor + +// Which requests count as writes worth capturing for replay and forgery. The +// default covers chat's routes; a scenario spec can widen it for an application +// whose endpoints are named differently (`writeUrlPattern`). +const DEFAULT_WRITE_URL = '\\/api\\/|\\/rooms|\\/messages'; +let WRITE_URL_RE = new RegExp(DEFAULT_WRITE_URL); + + +export class Actor { + readonly name: string; + readonly context: BrowserContext; + page!: Page; + readonly consoleErrors: string[]; + private readonly transport = new ReceivedTransport(); + private readonly completedTransport = new ReceivedTransport(); + private readonly httpReceipts = new Map(); + private serviceWorkerSeen = false; + private pendingHttp = 0; + private readonly pendingRequests = new Map, { request: Request; startedAt: number }>(); + readonly ready: Promise; + get received(): readonly string[] { return this.networkInterruption?.observesHttp + ? this.completedTransport.chunks : this.transport.chunks; } + get pendingReceived(): number { return this.transport.pending + this.pendingHttp + this.unobservedRequests().length; } + get deliveries(): readonly { atMs: number; payload: string | Buffer }[] { return this.transport.deliveries; } + get deliveriesDropped(): boolean { return this.transport.deliveriesDropped; } + readonly deliveryMarks = new Map(); + lastWrite: ActorWrite | null = null; + lastWrites: Record = {}; + writes: ActorWrite[] = []; + lastWsWrite: ActorWebSocketWrite | null = null; + annotate = false; + responseLoss?: Awaited>; + networkInterruption?: NetworkInterruption; + + constructor(name: string, page: Page, context: BrowserContext, readonly patchAuthentication = false, + readonly replaySpacetime = false, readonly spacetimeBackend = false, + // A stack's platform endpoints that receive the application's writes, whatever they are named. + readonly writeEndpoints: readonly string[] = [], readonly freshResponses = false, + networkInterruption?: NetworkInterruption, readonly passiveSocketFrame?: PassiveSocketFrame) { + this.name = name; + this.context = context; + this.consoleErrors = []; + this.networkInterruption = networkInterruption; + this.serviceWorkerSeen = context.serviceWorkers().length > 0; + context.on('serviceworker', () => { this.serviceWorkerSeen = true; }); + // Test privacy against delivered payloads, not rendered content. + this.ready = this.attach(page); + } + async attach(page: Page): Promise { + this.page = page; + if (this.patchAuthentication) await installAuthWebSocketCapture(page, this.passiveSocketFrame); + if (this.replaySpacetime || this.patchAuthentication && this.spacetimeBackend) await installSpacetimeWriteCapture(page); + // Capture writes so checks can replay them with changed fields or actors. + this.lastWrite = null; + this.lastWrites = {}; + this.writes = []; + this.lastWsWrite = null; + page.on('dialog', dialog => { + // Each step performs the action it names, so a confirmation or leave-page warning is accepted. + void dialog.accept().catch(error => { + if (page.isClosed()) return; + this.consoleErrors.push(`dialog handling failed: ${errorMessage(error)}`); + if (this.consoleErrors.length > MAX_CONSOLE_ERRORS) this.consoleErrors.shift(); + }); + }); + // Capture wire data separately from what the application renders. + page.on('websocket', ws => { + ws.on('framesent', f => { + recordConvexSession(page, ws.url(), f.payload, ws); + const p = typeof f.payload === 'string' ? f.payload : ''; + const m = p.match(/^\d+(\[.*\])$/s); + if (!m) return; + try { + const [event, arg] = JSON.parse(m[1] as string) as unknown[]; + if (arg && typeof arg === 'object' && !Array.isArray(arg)) { + this.lastWsWrite = { event, body: arg as JsonRecord }; + } + } catch { /* not a socket.io event frame */ } + }); + // Binary frames are decoded as UTF-8 too, after any SpacetimeDB frame + // compression: a binary wire format still carries message text as + // inline UTF-8 bytes, so a substring search finds it without the + // harness knowing the encoding. + ws.on('framereceived', f => { + recordConvexMutationResult(page, ws, f.payload); + this.record(f.payload); + }); + }); + page.on('request', req => { + const isWrite = req.method() !== 'GET' && req.method() !== 'OPTIONS'; + // Native EventSource messages have their own continuous capture. Other + // privacy reads must finish before reload, even before headers arrive. + if (!isWrite && (!this.freshResponses || req.resourceType() === 'eventsource')) return; + const completed = req.response().then(async response => + response !== null && await response.finished() === null && response.ok()).catch(() => false); + if (this.freshResponses) { + this.pendingRequests.set(completed, { request: req, startedAt: Date.now() }); + void completed.finally(() => this.pendingRequests.delete(completed)); + } + if (!isWrite) return; + const url = req.url(); + if (!WRITE_URL_RE.test(url) && !this.writeEndpoints.some(endpoint => url.startsWith(endpoint))) return; + let body: JsonRecord | null = null; + try { + const candidate: unknown = JSON.parse(req.postData() ?? ''); + if (candidate && typeof candidate === 'object' && !Array.isArray(candidate)) { + body = candidate as JsonRecord; + } + } catch { /* bodyless, e.g. a DELETE */ } + // Forging needs a body to tamper with; replaying does not — a privileged + // action is often a bare DELETE whose meaning is entirely in the URL. + const write: ActorWrite = { url, method: req.method(), headers: req.headers(), body }; + // Only a fully received successful response can supply bulk setup replay. + void completed.then(confirmed => { write.confirmed = confirmed; }); + // A success status shows the server accepted the request. A response whose body the + // page does not read finishes late, and never once the page has been replaced. + void req.response().then(response => { write.accepted = response?.ok() === true; }).catch(() => {}); + this.writes.push(write); + if (this.writes.length > 200) this.writes.shift(); + if (body && typeof body === 'object') { + this.lastWrite = write; + this.lastWrites[req.method()] = write; + } + }); + page.on('console', m => { + if (m.type() !== 'error') return; + const text = m.text(); + // Expected 4xx responses are not application console failures. + if (/Failed to load resource.*status of 4\d\d/.test(text)) return; + this.consoleErrors.push(text.slice(0, 200)); + if (this.consoleErrors.length > MAX_CONSOLE_ERRORS) this.consoleErrors.shift(); + }); + page.on('pageerror', e => { + this.consoleErrors.push(`pageerror: ${e.message.slice(0, 200)}`); + if (this.consoleErrors.length > MAX_CONSOLE_ERRORS) this.consoleErrors.shift(); + }); + await captureResponses(page, this.transport, this.freshResponses, this.networkInterruption?.observesHttp + ? { covered: response => this.coveredHttpResponse(response), completed: this.completedTransport } : undefined); + } + record(payload: string | Buffer): void { + this.transport.record(payload); + if (this.networkInterruption?.observesHttp) this.completedTransport.record(payload); + } + private unobservedRequests() { + return [...this.pendingRequests.values()].filter(({ request }) => + !this.networkInterruption?.observesHttp || this.serviceWorkerSeen || this.patchAuthentication + || this.responseLoss || !request.url().startsWith('http:')); + } + private async coveredHttpResponse(response: Response): Promise { + if (!response.url().startsWith('http:') || response.fromServiceWorker()) return false; + const id = response.headers()['x-stack-bench-capture']; + if (!id || this.httpReceipts.has(id)) return false; + const address = await response.serverAddr(); + const proxy = new URL(this.networkInterruption!.proxy.server); + // Synthetic/cache responses cannot borrow an old proxy receipt. The exact + // request record is checked again at the snapshot boundary. + if (!address || address.port !== Number(proxy.port) + || !['127.0.0.1', '::ffff:127.0.0.1'].includes(address.ipAddress)) return false; + this.httpReceipts.set(id, { url: response.url(), method: response.request().method(), + status: response.status(), contentType: response.headers()['content-type'] ?? '' }); + return true; + } + async syncReceived(): Promise { + if (!this.networkInterruption?.observesHttp) return; + try { + const take = async () => { + const snapshot = await this.networkInterruption!.httpSnapshot(); + return { snapshot, records: new Map(snapshot.records.map(record => [record.id, record])) }; + }; + // The proxy records a response before the browser sees it, so a receipt missing from a + // snapshot arrived while that snapshot was in flight. A later snapshot holds it. + let { snapshot, records } = await take(); + for (let again = 0; again < 3 && [...this.httpReceipts.keys()].some(id => !records.has(id)); again++) { + ({ snapshot, records } = await take()); + } + this.pendingHttp = snapshot.pending; + for (const record of snapshot.records) this.transport.record(record.body, true); + const unmatched = [...this.httpReceipts].find(([id, expected]) => { + const actual = records.get(id); + return !actual || actual.url !== expected.url || actual.method !== expected.method + || actual.status !== expected.status || actual.contentType !== expected.contentType; + }); + const reason = snapshot.incomplete ? 'snapshot-incomplete' + : records.size !== snapshot.records.length ? 'duplicate-records' + : unmatched ? records.has(unmatched[0]) ? 'receipt-mismatch' : 'receipt-missing' : null; + if (reason) this.captureIncomplete(reason, unmatched?.[1]); + } catch { + this.captureIncomplete('snapshot-failed'); + } + } + // Every incomplete capture says why; absence verdicts depend on it. + private captureIncompleteReports = 0; + private captureIncomplete(reason: string, receipt?: { url: string; method: string; status: number }): void { + this.transport.markIncomplete('bodyReadFailures'); + if (this.captureIncompleteReports++ >= 8) return; + try { process.stderr.write(`transport proxy capture incomplete ${JSON.stringify({ reason, + ...(receipt ? { ...urlDiagnostic(receipt.url), method: receipt.method, status: receipt.status } : {}), + })}\n`); } catch { /* Diagnostics must not change the verdict. */ } + } + private reportPendingRequests(phase: 'navigation' | 'absence'): void { + for (const { request, startedAt } of [...this.pendingRequests.values()].slice(0, 8)) { + try { process.stderr.write(`transport request pending ${JSON.stringify({ + ...requestDiagnostic(request), phase, elapsedMs: Date.now() - startedAt, + })}\n`); } catch { /* Diagnostics must not change the verdict. */ } + } + } + async prepareNavigation(within: number, signal: AbortSignal): Promise { + if (!this.freshResponses) return; + // An ordinary reload must not abort the response this privacy observer is recording. + // Fault actions navigate directly and deliberately do not use this wait. + const deadline = Date.now() + within; + await this.syncReceived(); + while (this.pendingReceived) { + if (Date.now() >= deadline) { + // Preserve a captured disclosure; incomplete capture only prevents proving absence. + this.reportPendingRequests('navigation'); + this.transport.markIncomplete('navigationInterrupted'); + return; + } + await abortableSleep(Math.min(25, deadline - Date.now()), signal); + await this.syncReceived(); + } + } + // `settled`: every request of the observation window had finished; one that began later is outside it. + wasSent(needle: string, requireComplete = true, settled = false): boolean { + const found = this.transport.contains(needle, requireComplete); + if (!found && requireComplete && !settled && (this.pendingHttp || this.unobservedRequests().length)) { + this.reportPendingRequests('absence'); + inconclusive('transport-incomplete', {}); + } + return found; + } + loc(testid: string, { contains, scope, editable }: + { contains?: string; scope?: { testid: string; contains?: string }; editable?: boolean } = {}) { + // `scope` narrows the search to inside a specific container (e.g. the badge + // belonging to ONE room), so a stale element elsewhere can't satisfy it. + const root = scope + ? this.page.locator(tid(scope.testid), { hasText: scope.contains }).filter({ visible: true }).first() + : this.page; + const selector = editable ? `:is(${tid(testid)}):is(input, textarea, select, [contenteditable]:not([contenteditable="false"]))` + : tid(testid); + return (contains + ? root.locator(selector, { hasText: contains }) + : root.locator(selector)).filter({ visible: true }).first(); + } +} + +// Expand scenario aliases to the run-scoped values used by the app. +const expand = (s: unknown, ctx: GradeRunContext): unknown => + typeof s === 'string' + ? s.replace(/\{room:([^}]+)\}/g, (_, b) => ctx.roomName(b)) + // Keep generated usernames alphanumeric so ordinary validators accept them. + .replace(/\{user:([^}]+)\}/g, (_, n) => `${n}${ctx.scope}`) + : s; + + +// Put test context in recordings without exposing it to scoped app selectors. + +const OVERLAY_ID = '__stackbench_overlay'; + +async function annotate(actor: Actor | undefined, { feature, criterion, step, status }: + { feature?: string; criterion?: string; step?: string; status?: 'fail' | 'pass' } = {}): Promise { + if (!actor?.annotate) return; + await actor.page.evaluate(({ id, feature, criterion, step, status, who }) => { + let el = document.getElementById(id); + if (!el) { + el = document.createElement('div'); + el.id = id; + el.style.cssText = [ + 'position:fixed', 'inset:0 0 auto 0', 'z-index:2147483647', + 'font:12px/1.5 ui-monospace,SFMono-Regular,Menlo,monospace', + 'padding:6px 10px', 'pointer-events:none', 'white-space:pre', + 'background:rgba(12,12,16,.92)', 'color:#e8e8ef', + 'border-bottom:2px solid #4c8dff', + ].join(';'); + document.documentElement.appendChild(el); + } + const colour = status === 'fail' ? '#ff5c5c' : status === 'pass' ? '#3ddc84' : '#4c8dff'; + el.style.borderBottomColor = colour; + el.textContent = [ + `${who} ${feature ?? ''}`, + criterion ? ` ${status === 'fail' ? 'FAILED' : 'checking'}: ${criterion}` : '', + step ? ` > ${step}` : '', + ].filter(Boolean).join(String.fromCharCode(10)); + }, { id: OVERLAY_ID, feature, criterion, step, status, who: actor.name }).catch(() => {}); +} + +// Step execution + +function abortableSleep(ms: number, signal: AbortSignal | null = null): Promise { + if (signal?.aborted) return Promise.reject(signal.reason ?? new Error('action cancelled')); + return new Promise((resolve, reject) => { + const timer = setTimeout(done, ms); + function done() { + signal?.removeEventListener('abort', cancelled); + resolve(); + } + function cancelled() { + clearTimeout(timer); + signal?.removeEventListener('abort', cancelled); + reject(signal?.reason ?? new Error('action cancelled')); + } + signal?.addEventListener('abort', cancelled, { once: true }); + }); +} + +// One accessor per run: per-run action state (such as proven routes) is keyed by it. +const actorAccessors = new WeakMap, Readonly<{ get(name: string): Actor | undefined }>>(); +function actorAccessFor(actors: Map): Readonly<{ get(name: string): Actor | undefined }> { + let access = actorAccessors.get(actors); + if (!access) actorAccessors.set(actors, access = Object.freeze({ get: (name: string) => actors.get(name) })); + return access; +} + +function browserActionCapabilities(actors: Map, ctx: GradeRunContext): Readonly> { + const defaultWithin = ctx.defaultWithin ?? DEFAULT_WITHIN; + const actorAccess = actorAccessFor(actors); + const runtimeValues = Object.freeze({ + sequenceScopeFallback: ctx.contractIds?.includes('ecommerce.progression.faceted-search-hooks') + ? { testid: 'item-name', from: 'item-list', to: 'search-results' } : undefined, + applicationUrl: ctx.url, + authRequestPatch: ctx.authRequestPatch ?? null, + authReadEndpoints: ctx.authReadEndpoints ?? [], + defaultWithin, + expand: (value: unknown) => expand(value, ctx), + hyphenatedScopedUser: (name: string) => `${name}-${ctx.scope}`, + roomName: (base: string) => ctx.roomName(base), + scopedUser: (name: string) => `${name}${ctx.scope}`, + recorded: Object.freeze({ + get: (key: string) => ctx.recorded?.[key], + set: (key: string, value: unknown) => { (ctx.recorded ??= {})[key] = value; }, + }), + sleep: abortableSleep, + testId: tid, + clients: Object.freeze({ + async open(actor: Actor, settleMs: number, signal: AbortSignal) { + const fresh = await actor.context.newPage(); + if (actor.networkInterruption) await actor.networkInterruption.attach(actor.context, fresh); + fresh.setDefaultTimeout(defaultWithin); + await actor.attach(fresh); + await runApplicationNavigation(() => fresh.goto(ctx.url, { waitUntil: 'domcontentloaded', timeout: 20000 }), fresh); + await abortableSleep(settleMs, signal); + }, + async fresh(actor: Actor, sourceName: string, preserveStorage: boolean) { + const browser = actor.page.context().browser(); + if (!browser) throw new Error('actor browser is unavailable'); + const name = `${sourceName}-fresh`; + const interruption = ctx.privacyActors?.has(name) ? await startNetworkInterruption(true) : undefined; + if (interruption) ctx.interruptions?.push(interruption); + const context = await browser.newContext({ + ...(preserveStorage ? { storageState: await actor.context.storageState({ indexedDB: true }) } : {}), + ...(interruption ? { proxy: interruption.proxy } : {}), + }); + // Own cleanup before page creation or navigation can fail. + const entry: ActorContextEntry = { context, name, page: null }; + ctx.extraContexts?.push(entry); + const fresh = await context.newPage(); + entry.page = fresh; + if (interruption) await interruption.attach(context, fresh); + fresh.setDefaultTimeout(defaultWithin); + const observer = new Actor(`${actor.name}-fresh`, fresh, context, actor.patchAuthentication, actor.replaySpacetime, + actor.spacetimeBackend, actor.writeEndpoints, ctx.privacyActors?.has(name), interruption, actor.passiveSocketFrame); + await observer.ready; + // storageState omits sessionStorage. Seed the first document only; + // later reloads must retain the application's own storage changes. + const session = preserveStorage ? await context.newCDPSession(fresh) : null; + let seed: string | undefined; + try { + if (session) { + await session.send('Page.enable'); + const state = await actor.page.evaluate(() => ({ + origin: location.origin, entries: Object.entries(sessionStorage), + })); + const script = await session.send('Page.addScriptToEvaluateOnNewDocument', { + source: `if (window === window.top && location.origin === ${JSON.stringify(state.origin)}) { + for (const [key, value] of ${JSON.stringify(state.entries)}) sessionStorage.setItem(key, value); + }`, + }); + seed = script.identifier; + } + await runApplicationNavigation(() => fresh.goto(ctx.url, { waitUntil: 'domcontentloaded', timeout: 20000 }), fresh); + } finally { + if (session) { + try { if (seed) await session.send('Page.removeScriptToEvaluateOnNewDocument', { identifier: seed }); } + finally { await session.detach(); } + } + } + observer.annotate = actor.annotate; + actors.set(name, observer); + return name; + }, + }), + }); + const transportObservation = Object.freeze({ + defaultWithin, + expand: (value: unknown) => expand(value, ctx), + sleep: abortableSleep, + verification: Object.freeze({ + structural(message: string) { + ctx.verified?.push(message); + ctx.serverCheck = ctx.serverCheck ?? 'structural'; + }, + unverified(message: string) { + (ctx.unverified ??= []).push(message); + ctx.serverCheck = 'unverified'; + }, + verified(message: string) { + ctx.verified?.push(message); + ctx.serverCheck = 'verified'; + }, + }), + }); + const namedActions = createNamedActionsCapability({ + actions: ctx.actions, + backend: ctx.backend!, + url: ctx.url, + spacetime: ctx.spacetime, + lastCalls: Object.freeze({ + get: () => ctx.lastCalls ?? null, + set: value => { ctx.lastCalls = value; }, + }), + sleep: abortableSleep, + applicationWriteEndpoints: ctx.applicationWriteEndpoints, + }); + const concurrency = Object.freeze({ + defaultWithin, + dispatch: (step: CompiledStep, signal: AbortSignal) => runRegisteredAction(step, actors, ctx, signal), + expand: (value: unknown) => expand(value, ctx), + sleep: abortableSleep, + testId: tid, + }); + return Object.freeze({ + actors: actorAccess, + 'response-loss': Object.freeze({ + async prepare(name: string) { + const actor = actors.get(name); + if (!actor || actor.responseLoss) throw new Error('response-loss actor is missing or already prepared'); + actor.responseLoss = await installResponseLoss(actor.context); + // Existing sockets predate interception. Replace them before setup writes. + await runApplicationNavigation(() => actor.page.reload({ waitUntil: 'domcontentloaded', timeout: 20000 }), actor.page); + }, + get(name: string) { + const gate = actors.get(name)?.responseLoss; + if (!gate) throw new Error('response loss was not prepared'); + return gate; + }, + }), + 'application-files': Object.freeze({ root: ctx.appDir ?? null, expand: (value: unknown) => expand(value, ctx) }), + 'application-lifecycle': applicationLifecycle(ctx), + 'backend-lifecycle': createLifecycleCapability({ + restartSpec: ctx.restartSpec, + target: 'backend-runtime', + control: async (...args) => { + await closeOrderReaders(ctx); + return controlBackendRuntime(...args); + }, + sleep: abortableSleep, + }), + 'browser-interaction': runtimeValues, + 'browser-observation': runtimeValues, + clock: Object.freeze({ sleep: abortableSleep }), + concurrency, + 'database-read': createDatabaseReadCapability({ + contractIds: ctx.contractIds, + savedReader: ctx.savedReader, + checkoutSnapshots: ctx.checkoutSnapshots ??= new Map(), + checkoutActivity: ctx.checkoutActivity ??= { unsettled: false }, + app: ctx.appDir, + backend: ctx.backend, + spacetime: ctx.spacetime, + databaseLease: ctx.databaseLease, + skip: ctx.nullControl, + expand: (value: string) => String(expand(value, ctx)), + readMongoDbOrders: async input => { + if (!ctx.databaseLease) throw new Error('MongoDB observation requires an authenticated lease'); + const reader = ctx.mongoOrderReader ??= createMongoDbOrderDataReader({ lease: ctx.databaseLease }); + try { return await reader.read(input); } + catch (error) { + await closeOrderReaders(ctx); + throw error; + } + }, + readConvexOrders: input => { + const reader = ctx.convexOrderReader ??= createConvexOrderDataReader( + leaseFromEnv(process.env, { backend: 'convex', active: true })); + return reader.read(input); + }, + }), + 'database-write': createDatabaseWriteCapability({ + backend: ctx.backend, + spacetime: ctx.spacetime, + databaseLease: ctx.databaseLease, + skip: ctx.nullControl, + expand: (value: string) => String(expand(value, ctx)), + }), + 'named-actions': namedActions, + 'process-crash': Object.freeze({ combinedBoundary: !ctx.nullControl + && Boolean(stackDatabaseRuntime(ctx.restartSpec?.backend ?? '')?.combinedBoundary), + prepare: async (target: 'application' | 'database') => { + if (!ctx.restartSpec || ctx.nullControl) throw new Error('process crash requires an owned grading runtime'); + await closeOrderReaders(ctx); + return prepareRuntimeCrash(ctx.restartSpec, target); + } }), + subprocess: Object.freeze({ sleep: abortableSleep }), + 'transport-observation': transportObservation, + }); +} + +async function closeOrderReaders(ctx: GradeRunContext): Promise { + ctx.convexOrderReader?.close(); + delete ctx.convexOrderReader; + const reader = ctx.mongoOrderReader; + await reader?.close(); + if (ctx.mongoOrderReader === reader) delete ctx.mongoOrderReader; +} + +function applicationLifecycle(ctx: GradeRunContext) { + return createLifecycleCapability({ + restartSpec: ctx.restartSpec, + target: 'app-server', + control: controlAppServer, + sleep: abortableSleep, + onOperated: mode => { ctx.applicationStopped = mode === 'stop'; }, + }); +} + +async function runRegisteredAction(step: CompiledStep, actors: Map, ctx: GradeRunContext, + signal: AbortSignal | null = null): Promise { + if (ctx.actionCancellation?.reason) throw new Error(ctx.actionCancellation.reason); + const actionEvidence = await executeAction(ACTION_REGISTRY, step.do, step, + { + capabilities: browserActionCapabilities(actors, ctx), + signal, + // Closing this grader's connection cancels pending Playwright calls before + // the action executor drains them. It does not stop the app or its database. + onAbort: ACTION_REGISTRY.get(step.do).capabilities.includes('actors') + ? async () => { + if (ctx.actionCancellation) ctx.actionCancellation.reason = 'browser session cancelled; no further actions are permitted'; + const browsers = new Set([...actors.values()].map(actor => actor.page.context().browser())); + await Promise.all([...browsers].map(browser => browser?.close())); + } : undefined, + }); + ctx.actionEvidence?.push({ actor: step.actor ?? null, evidence: actionEvidence }); + if (actionEvidence.status === 'passed') return actionEvidence.observation; + const error = new Error(actionEvidence.summary ?? `${step.do} did not complete`); + Object.defineProperty(error, 'actionEvidence', { value: actionEvidence }); + Object.defineProperty(error, 'actionActor', { value: step.actor ?? null }); + throw error; +} + +function classifyCheckFailure(error: unknown, fallbackActor: string | null = null): CheckFailure { + const actionError = actionFailure(error); + const actionEvidence = actionError?.actionEvidence; + if (actionEvidence) { + return { + status: actionEvidence.status, + code: actionEvidence.code, + actor: actionError?.actionActor ?? fallbackActor, + summary: actionEvidence.summary ?? `${actionEvidence.action.id} did not complete`, + finding: actionEvidence.finding, + observation: actionEvidence.observation, + expected: actionEvidence.expected, + retryable: actionEvidence.retryable, + }; + } + if (error instanceof ApplicationNotRestored) { + return { status: 'harness_failure', code: 'application_not_restored', actor: fallbackActor, + summary: error.message, finding: null, observation: null, expected: null, retryable: false }; + } + const processFailure = harnessProcessFailure(error); + if (processFailure) return { status: 'harness_failure', code: 'process_failure', actor: fallbackActor, + summary: processFailure, finding: null, observation: null, expected: null, retryable: false }; + const browserFailure = harnessBrowserFailure(error); + if (browserFailure) return { status: 'harness_failure', code: 'browser_failure', actor: fallbackActor, + summary: browserFailure, finding: null, observation: null, expected: null, retryable: false }; + if (error instanceof ActionApplicationFailure || error instanceof ActionInconclusive) { + return { status: error instanceof ActionInconclusive ? 'inconclusive' : 'failed', + code: error.classification, actor: fallbackActor, + summary: error.message, finding: isFinding(error.details.finding) ? error.details.finding : null, + observation: error.details.observation ?? null, + expected: error.details.expected ?? null, retryable: error.details.retryable === true }; + } + return { status: 'harness_failure', code: 'unclassified_exception', actor: fallbackActor, + summary: errorMessage(error ?? 'unknown grader failure'), + finding: null, observation: null, expected: null, retryable: false }; +} + +function buildCheckEvidence({ ctx, phase, startedAtMs, failure = null, actor = null, summary = null, + attachments = [], actions = ctx.actionEvidence ?? [], sensitivity = null }: { + ctx: GradeRunContext; phase: CheckEvidencePhase; startedAtMs: number; failure?: unknown; + actor?: string | null; summary?: string | null; attachments?: Array; + actions?: Array<{ actor: string | null; evidence: ActionEvidence }>; + sensitivity?: readonly string[] | null; + }): CheckEvidence { + const classified: CheckFailure = failure ? classifyCheckFailure(failure, actor) : { + status: 'passed', code: 'completed', actor: null, summary: null, finding: null, + observation: null, expected: null, retryable: false, + }; + const completedAtMs = Math.max(startedAtMs, evidenceNowMs()); + const evidenceSummary = summary ?? classified.summary; + return createCheckEvidence({ + ...classified, + phase, + summary: evidenceSummary == null ? null : keepReason(evidenceSummary), + startedAtMs, + completedAtMs, + actions, + attachments: attachments.map(attachment => typeof attachment === 'string' + ? { kind: 'screenshot', ref: basename(attachment) } : attachment), + sensitivity: sensitivity ?? actions.flatMap(entry => entry.evidence?.sensitivity ?? []), + }); +} + +async function runStep(step: CompiledStep, actors: Map, ctx: GradeRunContext): Promise { + return runRegisteredAction(step, actors, ctx); +} + +export async function closeActorContexts(entries: readonly CleanupActorContextEntry[], { + trace = false, media = null, slug = 'grade', +}: { trace?: boolean; media?: string | null; slug?: string } = {}): Promise { + const failures: GradeCleanupFailure[] = []; + const record = (name: string, stage: string, error: unknown): void => { failures.push({ + actor: name, + stage, + reason: keepReason(errorMessage(error)), + }); }; + for (const { context, name, page, traceStarted } of entries) { + if (trace && traceStarted) { + try { + await context.tracing.stop({ path: join(media ?? '.', `${slug}-${name}.trace.zip`) }); + } catch (error) { record(name, 'trace', error); } + } + let video = null; + if (media && page) { + try { video = page.video(); } + catch (error) { record(name, 'video-handle', error); } + } + try { await context.close(); } + catch (error) { record(name, 'context-close', error); } + if (video) { + try { await video.saveAs(join(media!, `${slug}-${name}.webm`)); } + catch (error) { record(name, 'video-save', error); } + try { await video.delete(); } + catch (error) { record(name, 'video-delete', error); } + } + } + return failures; +} + +// Feature grading + +function completedFeatureResult(result: FeatureResult): CompletedGradeFeatureResult { + if (!result.setupEvidence) { + throw new Error(`feature ${result.id} completed without setup evidence`); + } + return { ...result, setupEvidence: result.setupEvidence }; +} + +export async function gradeFeature(browser: Browser, feature: CompiledFeature, args: GradeArgs, + runCtx: GradeRunContext): Promise { + // Features share the app's DATABASE even though each gets fresh browser + // contexts, so user and room names are scoped per feature — otherwise a + // defect in one feature (e.g. a hijacked account) corrupts later setups. + const scope = `${runCtx.runId}f${feature.id}`; + runCtx.checkoutActivity ??= { unsettled: false }; + const extraContexts: ActorContextEntry[] = []; + const ctx: GradeRunContext = { ...runCtx, scope, roomName: (base: string) => `${base}-${scope}`, extraContexts, recorded: {}, checkoutSnapshots: new Map(), + unverified: [], verified: [], actionEvidence: [] }; + const actors = new Map(); + const contexts: ActorContextEntry[] = []; + const interruptions: NetworkInterruption[] = []; + ctx.interruptions = interruptions; + const slug = `${args.label ?? 'run'}-f${feature.id}`; + + // A feature is worth what its criteria are worth. An explicit `max` is only + // a consistency check enforced by check-scenarios, never a top-up. + const featureMax = feature.criteria.reduce((n, c) => n + (c.points ?? 1), 0); + const result: FeatureResult = { + id: feature.id, name: feature.name, score: 0, max: featureMax, + criteria: [], consoleErrors: [], + }; + const restoreFailures: GradeCleanupFailure[] = []; + const closeAll = async () => { + try { await closeOrderReaders(ctx); } + catch (error) { restoreFailures.push({ actor: null, stage: 'database-reader-close', reason: keepReason(errorMessage(error)) }); } + // Interruption proxies run where the browser runs; close them even after cancellation closed the browser. + await Promise.all(interruptions.splice(0).map(async interruption => { + try { await interruption.dispose(); } + catch (error) { restoreFailures.push({ actor: null, stage: 'network-interruption-close', + reason: keepReason(errorMessage(error)) }); } + })); + for (const actor of actors.values()) { + for (const message of actor.consoleErrors) { + result.consoleErrors.push(`[${actor.name}] ${sanitiseConsoleError(message)}`); + } + } + // The abort hook already closed this connection, or reported that closure + // could not be confirmed. Do not hang again while collecting browser media. + if (ctx.actionCancellation?.reason) { + const failures = [...restoreFailures, { actor: null, stage: 'browser-cancel', reason: ctx.actionCancellation.reason }]; + result.cleanupEvidence = { status: 'harness_failure', failures }; + return failures; + } + const failures = [...restoreFailures, ...await closeActorContexts([...contexts, ...extraContexts], { + trace: args.trace, media: args.media, slug, + })]; + if (failures.length) result.cleanupEvidence = { status: 'harness_failure', failures }; + return failures; + }; + // A criterion that stops the application server owns it only for its own + // steps. Whatever the outcome, the server is running again before the next + // criterion; a restore the harness cannot complete is the harness's failure + // and every later criterion in the feature is unmeasured, not failed. + const restoreApplicationServer = async () => { + if (!ctx.applicationStopped || restoreFailures.length) return; + try { + await applicationLifecycle(ctx) + .operate('start', APPLICATION_RESTORE_SETTLE_MS, AbortSignal.timeout(APPLICATION_RESTORE_TIMEOUT_MS)); + } catch (error) { + restoreFailures.push({ actor: null, stage: 'application-restore', + reason: keepReason(errorMessage(error)) }); + } + }; + const initializationStartedAtMs = evidenceNowMs(); + try { + if (ctx.actionCancellation?.reason) throw new Error(ctx.actionCancellation.reason); + // Actors this feature takes offline send their traffic through a harness proxy. + const steps = [...feature.setup ?? [], ...feature.criteria.flatMap(criterion => criterion.steps)]; + const offlineActors = new Set(steps + .filter(step => step.do === 'setOffline').map(step => step.actor)); + // Routing replaces native WebSocket delivery. Credential probes and explicit write completion need it; + // passive transport observation must not pay for forwarding every live update. + const needsAuthPatch = (step: CompiledStep): boolean => step.requestPatch !== undefined || step.do === 'probeSignupClaims' || step.awaitWrites === true + || Boolean(step.branches?.some(branch => branch.some(needsAuthPatch))); + const patchAuthentication = steps.some(needsAuthPatch); + const privacyActors = new Set(); + const collectPrivacyActors = (step: CompiledStep): void => { + if ((step.do === 'expectNotReceived' || step.do === 'expectNotDelivered') && step.actor) privacyActors.add(step.actor); + step.branches?.forEach(branch => branch.forEach(collectPrivacyActors)); + }; + steps.forEach(collectPrivacyActors); + ctx.privacyActors = privacyActors; + const replayActors = new Set(); + const collectReplayActors = (step: CompiledStep): void => { + if (step.do === 'repeatFormWrite' && step.actor) replayActors.add(step.actor); + step.branches?.forEach(branch => branch.forEach(collectReplayActors)); + }; + if (args.backend === 'spacetime') steps.forEach(collectReplayActors); + for (const name of feature.actors!) { + // Isolated storage per actor. Video is per-context, so each actor gets its + // own recording — you can watch what every participant saw, side by side. + const networkInterruption = offlineActors.has(name) || privacyActors.has(name) + ? await runBrowserInfrastructureOperation('browser proxy start', () => startNetworkInterruption(privacyActors.has(name))) + : undefined; + if (networkInterruption) interruptions.push(networkInterruption); + const context = await runBrowserInfrastructureOperation('context creation', () => + browser.newContext({ + ...(args.media ? { recordVideo: { dir: args.media, size: { width: 1280, height: 800 } } } : {}), + ...(networkInterruption ? { proxy: networkInterruption.proxy } : {}), + })); + contexts.push({ context, name, page: null }); + if (args.trace) { + await runBrowserInfrastructureOperation('trace start', () => + context.tracing.start({ screenshots: true, snapshots: true })); + contexts[contexts.length - 1]!.traceStarted = true; + } + const page = await runBrowserInfrastructureOperation('page creation', () => context.newPage()); + contexts[contexts.length - 1]!.page = page; + if (networkInterruption) await runBrowserInfrastructureOperation('network interruption attach', () => + networkInterruption.attach(context, page)); + page.setDefaultTimeout(SETUP_WITHIN); + const actor = new Actor(name, page, context, patchAuthentication, replayActors.has(name), args.backend === 'spacetime', + ctx.applicationWriteEndpoints, privacyActors.has(name), networkInterruption, ctx.passiveSocketFrame); + await actor.ready; + actor.annotate = Boolean(args.media); + actors.set(name, actor); + await runApplicationNavigation(() => page.goto(args.url!, { waitUntil: 'domcontentloaded', timeout: 20000 }), page); + } + } catch (error) { + const classified = classifyCheckFailure(error); + const reason = keepReason((classified.summary ?? '').trim()); + result.setupEvidence = buildCheckEvidence({ ctx, phase: 'setup', startedAtMs: initializationStartedAtMs, + failure: error, summary: reason }); + for (const criterion of feature.criteria) { + const points = criterion.points ?? 1; + const evidence = buildCheckEvidence({ ctx, phase: 'setup', startedAtMs: initializationStartedAtMs, + failure: error, summary: `browser setup failed: ${reason}`, actions: [] }); + if (evidence.status === 'failed') evidence.status = 'blocked'; + result.criteria.push({ id: criterion.id, desc: criterion.desc, points, evidence, + ...authored(criterion) }); + if (!evidenceIsMeasured(evidence)) result.inconclusive = [...(result.inconclusive ?? []), + { id: criterion.id, points, status: evidence.status, code: evidence.code, + phase: evidence.phase, summary: evidence.summary }]; + } + await closeAll(); + return completedFeatureResult(result); + } + + const captureFailureScreenshots = async (label: string): Promise => { + if (ctx.actionCancellation?.reason || !args.failureMedia) return []; + mkdirSync(args.failureMedia, { recursive: true }); + const captured: string[] = []; + for (const { name, page } of [...contexts, ...extraContexts]) { + if (!page) continue; + const path = join(args.failureMedia, `${slug}-${label}-${name}.png`); + const ok = await page.screenshot({ path, fullPage: true, timeout: 5000 }) + .then(() => true, () => false); + if (ok) captured.push(path); + } + return captured; + }; + + const setupStartedAtMs = evidenceNowMs(); + ctx.defaultWithin = SETUP_WITHIN; + ctx.actionEvidence = []; + try { + // Setup is not scored, but a failure makes the feature untestable (0). + for (const step of feature.setup) { + if (ctx.actionCancellation?.reason) throw new Error(ctx.actionCancellation.reason); + enforceTimeLimit(ctx); + await annotate(actors.get(step.actor), { feature: feature.name, criterion: 'setup', step: step.do }); + await runStep(step, actors, ctx); + } + } catch (err) { + // Preserve the typed setup failure on every affected criterion. + const classified = classifyCheckFailure(err); + const why = keepReason((classified.summary ?? '').trim()); + const screenshots = await captureFailureScreenshots('setup'); + result.setupEvidence = buildCheckEvidence({ ctx, phase: 'setup', startedAtMs: setupStartedAtMs, + failure: err, summary: why, attachments: screenshots }); + for (const c of feature.criteria) { + const base = why ? `Blocked by a failed prerequisite: ${why}` : 'Blocked by a failed prerequisite'; + const points = c.points ?? 1; + const evidence = buildCheckEvidence({ ctx, phase: 'setup', startedAtMs: setupStartedAtMs, + failure: err, summary: base, actions: [], sensitivity: result.setupEvidence.sensitivity, + attachments: [{ kind: 'check-evidence', ref: 'feature.setupEvidence' }, ...screenshots] }); + if (evidence.status === 'failed') evidence.status = 'blocked'; + const recorded = { id: c.id, desc: c.desc, points, evidence, ...authored(c) }; + result.criteria.push(recorded); + if (!evidenceIsMeasured(evidence)) { + result.inconclusive = [...(result.inconclusive ?? []), + { id: c.id, points, status: evidence.status, code: evidence.code, + phase: evidence.phase, summary: evidence.summary }]; + } + } + if (screenshots.length) result.screenshots = screenshots; + await closeAll(); + return completedFeatureResult(result); + } + result.setupEvidence = buildCheckEvidence({ ctx, phase: 'setup', startedAtMs: setupStartedAtMs }); + ctx.defaultWithin = DEFAULT_WITHIN; + for (const actor of actors.values()) actor.page.setDefaultTimeout(DEFAULT_WITHIN); + + for (const criterion of feature.criteria) { + let failure: unknown = null, detail: string | null = null, activeActor: string | null = null; + let criterionScreenshots: string[] = []; + const criterionStartedAtMs = evidenceNowMs(); + ctx.actionEvidence = []; + ctx.serverCheck = null; + try { + if (restoreFailures.length) throw new ApplicationNotRestored(restoreFailures[0]!.reason); + for (const step of criterion.steps) { + if (ctx.actionCancellation?.reason) throw new Error(ctx.actionCancellation.reason); + enforceTimeLimit(ctx); + activeActor = step.actor ?? activeActor; + await annotate(actors.get(step.actor) ?? actors.values().next().value, + { feature: feature.name, criterion: criterion.id, step: step.do }); + await runStep(step, actors, ctx); + } + for (const a of actors.values()) { + if (ctx.actionCancellation?.reason) throw new Error(ctx.actionCancellation.reason); + await annotate(a, { feature: feature.name, criterion: criterion.id, step: 'passed', status: 'pass' }); + } + } catch (err) { + failure = err; + const classified = classifyCheckFailure(err, activeActor); + detail = classified.summary; + // captureFailureScreenshots also refuses a cancelled session. + if (!ctx.actionCancellation?.reason && args.media) { + for (const a of actors.values()) { + await annotate(a, { feature: feature.name, criterion: criterion.id, + step: errorMessage(err).slice(0, 120), status: 'fail' }); + } + const shotActor = actors.get(criterion.steps[criterion.steps.length - 1]?.actor) ?? actors.values().next().value; + const shot = join(args.media, `${slug}-${criterion.id}.png`); + const captured = await shotActor.page.screenshot({ path: shot, fullPage: true }) + .then(() => true, () => false); + if (captured) criterionScreenshots.push(shot); + } else { + criterionScreenshots = await captureFailureScreenshots(criterion.id); + } + if (criterionScreenshots.length) { + result.screenshots = [...(result.screenshots ?? []), ...criterionScreenshots]; + } + } + await restoreApplicationServer(); + const evidence = buildCheckEvidence({ ctx, phase: 'assertion', startedAtMs: criterionStartedAtMs, + failure, actor: activeActor, summary: detail, attachments: criterionScreenshots }); + result.criteria.push({ id: criterion.id, desc: criterion.desc, points: criterion.points, + evidence, ...authored(criterion), + ...(ctx.serverCheck ? { serverCheck: ctx.serverCheck } : {}) }); + if (evidencePassed(evidence)) result.score += criterion.points; + else if (!evidenceIsMeasured(evidence)) { + result.inconclusive = [...(result.inconclusive ?? []), + { id: criterion.id, points: criterion.points, status: evidence.status, code: evidence.code, + phase: evidence.phase, summary: evidence.summary }]; + } + } + + // Retain diagnostics for server-side checks that could not execute. The + // action executor marks those criteria inconclusive, so they cannot score. + if (ctx.unverified?.length) result.unverified = ctx.unverified; + if (ctx.verified?.length) result.verified = ctx.verified; + + await closeAll(); + if (args.media) result.videos = contexts.map(c => join(args.media!, `${slug}-${c.name}.webm`)); + return completedFeatureResult(result); +} + +// Main + +export function gradeDatabaseLease(backend?: string, env: NodeJS.ProcessEnv = process.env): LeasedDatabase | null { + if (!backend || !(env.STACK_BENCH_LEASE || env.STACK_BENCH_LEASE_TOKEN)) return null; + // A stack whose observers read and write a leased database container declares how. + const { databaseLease } = STACK_ADAPTER_REGISTRY.get(backend).grading; + return databaseLease ? databaseLease(leaseFromEnv(env, { backend, active: true }).lease) : null; +} + +// A platform that serves accounts itself supplies how its own password requests are changed. +function platformAuthPatch(backend: string): PlatformAuthPatch | null { + const { authRequestPatch } = STACK_ADAPTER_REGISTRY.get(backend).grading; + return authRequestPatch ? authRequestPatch(leaseFromEnv(process.env, { backend, active: true }).lease) : null; +} + +async function main(): Promise { + const startedAt = new Date().toISOString(); + const args = parseGradeArgs(process.argv); + const specPath = args.spec!; + let spec: CompiledScenarioDefinition; + try { + const compiled = compileScenarioDefinition(JSON.parse(readFileSync(specPath, 'utf8')), + { source: specPath }); + spec = materializeScenarioCredentials(compiled, args.credentialAliases); + } catch (error) { + throw new Error(`cannot compile scenario ${specPath}: ${errorMessage(error)}`, { cause: error }); + } + + if (typeof spec.writeUrlPattern === 'string' && spec.writeUrlPattern) { + WRITE_URL_RE = new RegExp(spec.writeUrlPattern); + } + + const candidateFeatures = args.feature ? spec.features.filter(f => f.id === args.feature) : spec.features; + if (args.diagnostic && candidateFeatures.some(feature => feature.criteria.some(check => check.points !== 0))) { + throw new Error('diagnostic grades require zero-point checks'); + } + if (args.feature && candidateFeatures.length === 0) { + throw new Error(`scenario ${specPath} has no feature ${args.feature}`); + } + const runId = uniq(); + // Where the named actions live. The track declares their names; the + // authenticated backend lease—not generated application config—selects the + // SpacetimeDB host, module and exact build container used for direct SQL. + let actions: TrackAction[] = [], spacetime: LeasedSpacetimeTarget | null = null, + selectedTask: ReturnType | null = null, + recipeRelease: RecipeGradeRelease | null = null, + recipeIdentityRelease: RecipeRelease | null = null, + calibration: ReturnType | null = null; + if (args.track) { + const track = loadTrack(args.track); + actions = track.actions; + const binding = args.diagnostic ? null : resolveGradeRecipeArtifactBinding(track, args.level, specPath, + args.feature ?? null, args.recipe); + recipeRelease = binding?.release ?? null; + recipeIdentityRelease = binding?.sourceRelease ?? null; + if (args.recipeTask !== undefined) { + if (!binding) throw new Error('recipe task requires a bound grade recipe'); + selectedTask = resolveBoundRecipeTaskRequest(binding.binding, args.recipeTask); + } + } + if (args.recipeTask !== undefined && !selectedTask) throw new Error('recipe task requires a bound grade recipe'); + if (args.expectedRecipeSha256 + && recipeRelease?.contentSha256 !== args.expectedRecipeSha256) { + throw new Error(`recipe changed before grading: expected ${args.expectedRecipeSha256}, ` + + `resolved ${recipeRelease?.contentSha256 ?? 'no recipe'}`); + } + const selectedScenario = selectScenarioChecks( + { ...spec, features: candidateFeatures }, recipeRelease, args.selectedCheckKeys); + const features = selectedScenario.features; + const selectedChecks = selectedScenario.checks; + if (!features.length) throw new Error(`scenario ${specPath} has no selected checks`); + if (args.track) { + const track = loadTrack(args.track); + calibration = resolveCalibrationForRelease(recipeIdentityRelease, { + trackRoot: track.dir, + stackBenchRoot: ROOT, + alias: `L${args.level}`, + }); + } + spacetime = args.backend + ? STACK_ADAPTER_REGISTRY.get(args.backend).grading.context({ requireBuildContainer: true }) + : null; + const databaseLease = gradeDatabaseLease(args.backend); + + const ctx: GradeRunContext = { actionCancellation: { reason: null }, + ...(args.timeLimitMs === undefined ? {} : { timeLimit: { limitMs: args.timeLimitMs, deadlineAtMs: Date.now() + args.timeLimitMs } }), + runId, roomName: (base: string) => `${base}-${runId}`, + restartSpec: args.restartSpec, url: args.url!, + backend: args.backend, actions, spacetime, dbName: args.dbName, + databaseLease, authRequestPatch: args.backend ? platformAuthPatch(args.backend) : null, + authReadEndpoints: args.backend && STACK_ADAPTER_REGISTRY.get(args.backend).grading.authReadEndpoints + ? STACK_ADAPTER_REGISTRY.get(args.backend).grading.authReadEndpoints!(leaseFromEnv(process.env, { backend: args.backend, active: true }).lease) : [], + passiveSocketFrame: args.backend ? STACK_ADAPTER_REGISTRY.get(args.backend).grading.passiveSocketFrame + ?.(leaseFromEnv(process.env, { backend: args.backend, active: true }).lease) : undefined, + applicationWriteEndpoints: args.backend ? applicationWrites(args.backend) : [], + nullControl: args.nullControl, + contractIds: selectedTask?.task.contractIds, + appDir: args.app, savedReader: args.savedDiagnostic?.reader }; + + const browser = args.browserWsEndpoint + ? await chromium.connect(args.browserWsEndpoint) + : await chromium.launch({ headless: !args.headed, ...attemptBrowserLaunchOptions() }); + const report: JsonRecord & CompletedGradeReport & { + inconclusive?: Array; + cleanupEvidence?: { status: 'harness_failure'; failures: GradeCleanupFailure[] }; + } = { + definitionSchemaVersion: spec.schemaVersion, + recipeRelease, + ...(selectedTask ? { recipeTask: selectedTask.request } : {}), + label: args.label ?? null, url: args.url, level: args.level, runId, + total: 0, max: features.reduce((n, f) => n + f.criteria.reduce((m, c) => m + (c.points ?? 1), 0), 0), features: [], + selection: recipeRelease ? { + ...(args.selectionSha256 ? { sha256: args.selectionSha256 } : {}), + checks: selectedChecks.map(({ stableKey, packId, checkGroupId, featureId, criterionId, + description, points }) => { + if (!packId) throw new Error(`selected check ${stableKey} has no pack id`); + return { stableKey, packId, checkGroupId, featureId, criterionId, description, points }; + }), + } : null, + }; + const checkByCriterion = new Map(selectedChecks.map(check => [ + `${String(check.featureId)}\0${String(check.criterionId)}`, check, + ])); + + try { + for (const feature of features) { + process.stdout.write(`Feature ${feature.id}: ${feature.name} ... `); + const r = await gradeFeature(browser, feature, args, ctx); + if (recipeRelease) { + for (const criterion of r.criteria) { + const check = checkByCriterion.get(`${String(feature.id)}\0${String(criterion.id)}`); + if (!check) throw new Error(`graded criterion ${feature.id}/${criterion.id} has no recipe check`); + criterion.stableKey = check.stableKey; + } + } + report.features.push(r); + report.total += r.score; + // The recipe owns the denominator. An unmeasured criterion earns zero and + // remains explicitly inconclusive; it must never change the contract. + if (r.inconclusive?.length) { + report.inconclusive = [...(report.inconclusive ?? []), + ...r.inconclusive.map(c => ({ feature: r.id, ...c }))]; + } + console.log(`${r.score}/${r.max}`); + for (const c of r.criteria.filter(c => !evidencePassed(c.evidence))) { + console.log(` ${renderEvidenceConsoleLine(c.evidence, c.id)}`); + } + } + } finally { + try { if (!ctx.actionCancellation?.reason) await browser.close(); } + catch (error) { + report.cleanupEvidence = { status: 'harness_failure', failures: [{ + actor: null, stage: 'browser-close', reason: keepReason(errorMessage(error)), + }] }; + } + } + + if (recipeRelease) report.packRuntime = measureGradePackRuntime(report); + + console.log(`\nTOTAL ${report.total}/${report.max}`); + if (args.out) { + const artifactId = `grade-${runId}`; + writeArtifact(args.out, { + kind: 'grade', + id: artifactId, + attempt: { id: artifactId, parentId: args.parentAttemptId ?? null }, + timestamps: { startedAt, completedAt: new Date().toISOString() }, + identities: recipeArtifactIdentities(recipeIdentityRelease, { + calibration: calibration ? { id: calibration.id, + sha256: calibration.contentSha256 } : null, + stackAdapter: args.backend ? { id: args.backend } : null, + }), + payload: report, + }); + console.log(`Report written to ${args.out}`); + } +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + main().catch(error => { + console.error(error instanceof Error ? (error.stack ?? error.message) : String(error)); + process.exitCode = 2; + }); +} diff --git a/tools/stack-bench/grader/mutation-test.ts b/tools/stack-bench/grader/mutation-test.ts new file mode 100644 index 00000000000..00d3f6c4119 --- /dev/null +++ b/tools/stack-bench/grader/mutation-test.ts @@ -0,0 +1,865 @@ +#!/usr/bin/env node +// A valid mutation fails only its declared criterion against a passing baseline. +import { + copyFileSync, + closeSync, + existsSync, + mkdirSync, + mkdtempSync, + openSync, + readFileSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { createHash, randomUUID } from "node:crypto"; +import { dirname, join, relative, resolve, sep } from "node:path"; +import { fileURLToPath } from "node:url"; +import { execFileSync } from "node:child_process"; +import { parseArgs as parseNodeArgs } from "node:util"; +import { currentEngineIdentity, emptyArtifactIdentities, readArtifactPayload, + writeRunJson } from "../src/evidence/artifacts.js"; +import { controlAppServer, parseRuntimeControlSpec } from "../src/runtime/backend-control.js"; +import type { RuntimeControlSpec } from "../src/runtime/backend-control.js"; +import { leaseFromEnv } from '../src/runtime/backend-lease.js'; +import { inspectBuildContainer } from '../src/stacks/hosted-lifecycle.js'; +import { CODING_CONTAINER_APP_ROOT, codingContainerAgentCommand, codingContainerAgentExecOptions } + from '../src/runtime/coding-container-policy.js'; +import { + classifyMutationResult, + groupMutationsByScenario, + isRetryableMutationBaseline, + isRetryableMutationResult, + mutationFileEdits, + mutationTargetKeys, + readMutationManifest, + releaseScenarioCheckKeys, + resolveMutationScenarioPath, + reusableMutationBaseline, + resolveMutationFile, + validateMutationBaseline, + validateMutationDefinitions, +} from "../src/evidence/mutation-analysis.js"; +import { dbName, loadTrack, TRACK_MANIFEST_FILE } from "../src/composition/tracks.js"; +import { resolveRecipeRelease } from "../src/composition/recipe-release.js"; +import { createBoundRecipeTaskRequest } from '../src/composition/recipe-selection.js'; +import { resetBackend } from "../src/stacks/backend-reset.js"; +import { STACK_ADAPTER_REGISTRY } from "../src/stacks/stack-adapters.js"; +import { mutationShard } from "../src/evidence/mutation-shards.js"; +import { reusableMutationEvidence } from "../src/evidence/mutation-checkpoint.js"; +import { mutationGradeTimeoutMs } + from "../src/evidence/mutation-control.js"; +import { GRADER_SOURCE_TIMEOUT_MS, gradingSourceTimeoutMs } from '../src/runtime/grading-timeout.js'; +import { assertAppSourceIdentity } from "../src/runtime/source-snapshot.js"; +import type { TextCommandExecutor } from '../src/runtime/command-executor.js'; +import type { LoadedMutationManifest, MutationDefinition } from '../src/evidence/mutation-analysis.js'; +import { redactCredentials } from '../src/evidence/diagnostic-sanitizer.js'; +import type { MutationCheckpointBaseline, MutationCheckpointIdentity, + MutationCheckpointResult } from '../src/evidence/mutation-checkpoint.js'; + +type JsonRecord = Record; +type MutationSpec = LoadedMutationManifest; +type MutationArgs = { + app?: string; url?: string; mutations?: string; level?: string; spec?: string; backend?: string; + track?: string; recipe?: string; selectedCheckKeys?: string[]; dbName?: string; runIndex?: string; + restartSpec?: RuntimeControlSpec; out?: string; parentAttemptId?: string; + mutationShardIndex?: number; mutationShardCount?: number; resumeFrom?: string; checkpointOut?: string; + baselineBundle?: string; expectedCalibrationIdentity?: JsonRecord; maxRuntimeMinutes?: number; + imageId?: string; mutationAttemptId?: string; expectedRecipeSha256?: string; + reseedOnReset?: boolean; + gradeTimeoutMs?: number; + recipeTask?: ReturnType['request']; +}; +type ParsedMutationArgs = MutationArgs & { + app: string; + url: string; + mutations: string; + level: string; + recipe: string; + maxRuntimeMinutes: number; + mutationAttemptId: string; +}; +type GradeReport = { total?: unknown; max?: unknown; + features?: Array<{ id?: unknown; score?: unknown; + criteria?: Array<{ id?: string; stableKey?: unknown; evidence?: unknown }> }>; + [key: string]: unknown }; +type MutationResult = ReturnType & { id: string; scenario: string; targets: string[] }; +type BaselineEntry = MutationCheckpointBaseline & { total: unknown; max: unknown }; +type MutationFile = { target: string; backup: string; original: string; edits: ReturnType }; + +export function mutationFailureMessage(error: unknown): string { + return redactCredentials(error instanceof AggregateError + ? `${error.message}: ${error.errors.map(mutationFailureMessage).join('; ')}` + : errorMessage(error)); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +export function restoreMutationSource(file: { target: string; backup: string; original: string }): void { + // Restart hands files to the app UID. copyFileSync can fail chmod and unlink + // its destination even when the controller can write through group access. + writeFileSync(file.target, file.original); + if (readFileSync(file.target, 'utf8') !== file.original) { + throw new Error(`restore verification failed for ${file.target}`); + } + unlinkSync(file.backup); +} + +function jsonObject(value: unknown, label: string): JsonRecord { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${label} must be an object`); + } + return value as JsonRecord; +} + +const HERE = dirname(fileURLToPath(import.meta.url)); +const GRADER = join(HERE, "grade.js"); + +export function parseMutationArgs(argv: readonly string[]): ParsedMutationArgs { + const { values } = parseNodeArgs({ args: [...argv.slice(2)], options: { + app: { type: 'string' }, url: { type: 'string' }, mutations: { type: 'string' }, + level: { type: 'string' }, backend: { type: 'string' }, + track: { type: 'string' }, recipe: { type: 'string' }, + 'selected-check': { type: 'string', multiple: true }, + 'run-index': { type: 'string' }, + 'restart-spec': { type: 'string' }, out: { type: 'string' }, 'parent-attempt-id': { type: 'string' }, + 'mutation-shard-index': { type: 'string' }, 'mutation-shard-count': { type: 'string' }, + 'resume-from': { type: 'string' }, 'checkpoint-out': { type: 'string' }, + 'baseline-bundle': { type: 'string' }, 'expected-calibration-json': { type: 'string' }, + 'max-runtime-minutes': { type: 'string' }, 'image-id': { type: 'string' }, + } }); + const a: MutationArgs = { app: values.app, url: values.url, mutations: values.mutations, + level: values.level, backend: values.backend, track: values.track, + recipe: values.recipe, selectedCheckKeys: values['selected-check'], + runIndex: values['run-index'] ?? '0', + restartSpec: values['restart-spec'] === undefined ? undefined + : parseRuntimeControlSpec(JSON.parse(values['restart-spec'])), + out: values.out, parentAttemptId: values['parent-attempt-id'], + mutationShardIndex: values['mutation-shard-index'] === undefined + ? undefined : Number(values['mutation-shard-index']), + mutationShardCount: values['mutation-shard-count'] === undefined + ? undefined : Number(values['mutation-shard-count']), + resumeFrom: values['resume-from'] && resolve(values['resume-from']), + checkpointOut: values['checkpoint-out'] && resolve(values['checkpoint-out']), + baselineBundle: values['baseline-bundle'] && resolve(values['baseline-bundle']), + expectedCalibrationIdentity: values['expected-calibration-json'] === undefined + ? undefined : JSON.parse(values['expected-calibration-json']) as JsonRecord, + maxRuntimeMinutes: values['max-runtime-minutes'] === undefined + ? 60 : Number(values['max-runtime-minutes']), + imageId: values['image-id'] }; + if (!a.app || !a.url || !a.mutations || !a.level || !a.recipe) { + throw new Error( + "Usage: node dist/grader/mutation-test.js --app --url --mutations " + + "--level --recipe ", + ); + } + let url: URL; + try { url = new URL(a.url); } + catch { throw new Error('--url must be a valid HTTP or HTTPS URL'); } + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + throw new Error('--url must use HTTP or HTTPS'); + } + if (!Number.isInteger(Number(a.level)) || Number(a.level) < 1) { + throw new Error('--level must be a positive integer'); + } + const shardFields = [a.mutationShardIndex, a.mutationShardCount] + .filter(value => value !== undefined); + if (shardFields.length === 1) { + throw new Error('--mutation-shard-index and --mutation-shard-count must be supplied together'); + } + if (a.mutationShardCount !== undefined + && (!Number.isInteger(a.mutationShardIndex) || !Number.isInteger(a.mutationShardCount) + || a.mutationShardIndex! < 0 || a.mutationShardCount < 1 + || a.mutationShardIndex! >= a.mutationShardCount)) { + throw new Error('--mutation-shard-index must be within the positive shard count'); + } + a.maxRuntimeMinutes ??= 60; + if (!Number.isFinite(a.maxRuntimeMinutes) || a.maxRuntimeMinutes < 1 + || a.maxRuntimeMinutes > 120) { + throw new Error('--max-runtime-minutes must be from 1 through 120'); + } + if (a.resumeFrom && !a.checkpointOut) a.checkpointOut = a.resumeFrom; + return { ...a, app: a.app, url: a.url, mutations: a.mutations, level: a.level, recipe: a.recipe, + maxRuntimeMinutes: a.maxRuntimeMinutes, + mutationAttemptId: `mutation-${new Date().toISOString().replace(/[:.]/g, "-")}` }; +} + +class MutationBatchDeadlineError extends Error {} + +export function remainingMutationBatchMs(deadlineMs: number, nowMs: number = Date.now()): number { + const remaining = Math.floor(deadlineMs - nowMs); + if (remaining <= 0) throw new MutationBatchDeadlineError('mutation batch deadline reached'); + return remaining; +} + +// Startup owns application initialization, including migrations outside module init. +export async function resetMutationDatabase(a: MutationArgs, deadlineMs: number | null, + control = controlAppServer): Promise { + const exec: TextCommandExecutor = deadlineMs === null ? execFileSync : ((file, commandArgs, options) => + execFileSync(file, commandArgs, { ...options, + timeout: Math.min(options.timeout, remainingMutationBatchMs(deadlineMs)) })); + try { + const requiresReseed = STACK_ADAPTER_REGISTRY.get(a.backend!).reset.requiresReseed; + const restartSpec = a.reseedOnReset && requiresReseed ? a.restartSpec : undefined; + if (a.reseedOnReset && requiresReseed && !restartSpec) { + throw new Error(`track ${a.track} requires a lease-authenticated --restart-spec to reseed after reset`); + } + const signal = deadlineMs === null ? null + : AbortSignal.timeout(remainingMutationBatchMs(deadlineMs)); + if (restartSpec) await control(restartSpec, "stop", { signal, exec }); + await resetBackend({ backend: a.backend!, app: a.app!, exec }); + if (restartSpec) await control(restartSpec, "start", { signal, exec }); + } catch (error) { + if (deadlineMs !== null && Date.now() >= deadlineMs) { + throw new MutationBatchDeadlineError('mutation batch deadline reached', { cause: error }); + } + throw error; + } +} + +export function mutationClientCommand(backend: string, command: string, args: readonly string[], + timeout: number, exec: TextCommandExecutor = execFileSync): string { + const { lease } = leaseFromEnv(process.env, { backend, active: true }); + const container = inspectBuildContainer(lease, exec); + return exec('docker', ['exec', ...codingContainerAgentExecOptions(), + '-w', `${CODING_CONTAINER_APP_ROOT}/client`, container.id, + ...codingContainerAgentCommand(command, args)], + { encoding: 'utf8', stdio: 'pipe', timeout }); +} + +function rebuildClientAfterSourceChange(a: MutationArgs, deadlineMs: number): void { + const timeout = deadlineMs - Date.now(); + if (timeout <= 0) throw new MutationBatchDeadlineError(); + try { + mutationClientCommand(a.backend!, 'npm', ['run', 'build'], timeout); + } catch (cause) { + if (Date.now() >= deadlineMs) throw new MutationBatchDeadlineError(); + throw new Error(`client build failed after source change: ${mutationFailureMessage(cause)}`, { cause }); + } +} + +export function mutationGradeArguments(a: MutationArgs, reportPath: string): string[] { + const gradeArgs: string[] = [ + GRADER, + "--url", + a.url!, + "--level", + a.level!, + "--out", + reportPath, + "--spec", + a.spec!, + "--backend", + a.backend!, + "--track", + a.track!, + "--app", + a.app!, + ]; + if (a.dbName) gradeArgs.push("--db-name", a.dbName); + if (a.restartSpec) gradeArgs.push("--restart-spec", JSON.stringify(a.restartSpec)); + if (a.mutationAttemptId) gradeArgs.push("--parent-attempt-id", a.mutationAttemptId); + if (a.recipe) gradeArgs.push("--recipe", a.recipe); + if (a.recipeTask) gradeArgs.push('--recipe-task-json', JSON.stringify(a.recipeTask)); + if (a.expectedRecipeSha256) { + gradeArgs.push("--expected-recipe-sha256", a.expectedRecipeSha256); + } + for (const stableKey of a.selectedCheckKeys ?? []) { + gradeArgs.push("--selected-check", stableKey); + } + if (a.selectedCheckKeys?.length) { + gradeArgs.push("--selection-sha256", sha256(JSON.stringify([...a.selectedCheckKeys].sort()))); + } + return gradeArgs; +} + +async function grade(a: MutationArgs, reportPath: string, deadlineMs: number | null = null): Promise { + await resetMutationDatabase(a, deadlineMs); + if (existsSync(reportPath)) unlinkSync(reportPath); + const gradeArgs = mutationGradeArguments(a, reportPath); + const sourceTimeout = a.gradeTimeoutMs ?? GRADER_SOURCE_TIMEOUT_MS; + const timeout = deadlineMs === null + ? sourceTimeout + : mutationGradeTimeoutMs(deadlineMs, Date.now(), sourceTimeout); + if (timeout === 0) throw new MutationBatchDeadlineError('mutation batch deadline reached'); + const stderr = openSync(`${reportPath}.stderr.log`, 'wx'); + try { + execFileSync(process.execPath, gradeArgs, { + stdio: ['pipe', 'pipe', stderr], + encoding: "utf8", + timeout, + }); + } catch (error) { + if (jsonObject(error, 'grader process error').code === 'ETIMEDOUT' && timeout < sourceTimeout) { + throw new MutationBatchDeadlineError('mutation grade reached the remaining batch deadline'); + } + throw error; + } finally { + closeSync(stderr); + } + if (!existsSync(reportPath)) { + throw new Error("grader completed without producing its report"); + } + return readArtifactPayload(reportPath, { expectedKind: "grade" }); +} + +interface MutationGradeReceipt { + scenario: string; + mutationId: string | null; + status: 'running' | 'returned' | 'threw'; + report: { path: string; sha256: string | null }; +} + +// Reports stay private beside their control artifact, including failed and retried invocations. +export async function retainMutationGrade(outputPath: string, + context: Pick, + invoke: (path: string) => Promise, record: (receipt: MutationGradeReceipt) => void): Promise { + const directory = `${outputPath}.grades`; + mkdirSync(directory, { recursive: true }); + const path = join(mkdtempSync(join(directory, 'invocation-')), 'grade.json'); + const receipt: MutationGradeReceipt = { ...context, status: 'running', report: { + path: relative(dirname(outputPath), path).replaceAll('\\', '/'), sha256: null, + } }; + record(receipt); + try { + const result = await invoke(path); + receipt.status = 'returned'; + return result; + } catch (error) { + receipt.status = 'threw'; + throw error; + } finally { + if (existsSync(path)) receipt.report.sha256 = sha256(readFileSync(path)); + record(receipt); + } +} + +let args: ParsedMutationArgs; +let startedAt: number; +let startedIso: string; +const artifactPath = (id: string) => + resolve(args.out ?? join(HERE, "..", "results", `${id}.json`)); +let spec!: MutationSpec; +const gradeReports: MutationGradeReceipt[] = []; +let priorMutationControl: { path: string; sha256: string } | null = null; +let baselineBundleEvidence: { path: string; sha256: string } | null = null; +let currentControlArtifact: (() => Record) | null = null; + +function sha256(value: string | Buffer): string { + return createHash('sha256').update(value).digest('hex'); +} + +function scenarioKey(path: string): string { + return relative(resolve(HERE, '..'), path).replaceAll('\\', '/'); +} + +function checkpointGroup(path: string, mutations: MutationDefinition[], selectedCheckKeys: readonly string[]): + MutationCheckpointIdentity['groups'][number] { + const scenario = scenarioKey(path); + const scenarioSha256 = sha256(readFileSync(path)); + const mutationSha256 = sha256(JSON.stringify(mutations)); + const selectionSha256 = sha256(JSON.stringify([...selectedCheckKeys].sort())); + return { scenario, scenarioSha256, mutationSha256, selectionSha256, + identitySha256: sha256(JSON.stringify({ scenarioSha256, mutationSha256, selectionSha256 })), + mutationIds: mutations.map(mutation => mutation.id as string) }; +} + +function checkpointIdentity(groups: MutationCheckpointIdentity['groups'], shard: { index: number; count: number; + mutationIds: string[] }, track: ReturnType): MutationCheckpointIdentity { + return { + schemaVersion: 1, + engineSha256: currentEngineIdentity().sha256, + recipeSha256: args.expectedRecipeSha256, + fixtureSha256: spec.fixtureSha256, + calibrationSha256: args.expectedCalibrationIdentity?.sha256 ?? null, + imageId: args.imageId ?? null, + backend: args.backend, + track: args.track, + level: Number(args.level), + trackSha256: sha256(readFileSync(join(track.dir, TRACK_MANIFEST_FILE))), + shard: { index: shard.index, count: shard.count, mutationIds: shard.mutationIds }, + groups, + }; +} + +function resumableEvidence(path: string | undefined, identity: MutationCheckpointIdentity): + { results: MutationCheckpointResult[]; baselines: MutationCheckpointBaseline[] } { + if (!path || !existsSync(path)) return { results: [], baselines: [] }; + const prior = readArtifactPayload(path, { expectedKind: 'mutation_control' }); + const { results, baselines } = reusableMutationEvidence(prior, identity); + const shard = identity.shard as { mutationIds: string[] }; + console.log(`Resuming ${results.length}/${shard.mutationIds.length} completed mutations from ${path}`); + return { results, baselines }; +} + +export function mutationHarnessFailureArtifact(current: Record, reason: string, + completedAt: string): Record { + return { ...current, completedAt, ok: false, + outcome: { kind: 'harness_failure', phase: 'mutation-control', reason } }; +} + +function recordHarnessFailure(error: unknown): void { + const generatedAt = new Date().toISOString(); + const id = args.mutationAttemptId; + const artifact = { + id, + kind: "mutation_control", + startedAt: startedIso, + completedAt: generatedAt, + parentAttemptId: args.parentAttemptId ?? null, + identities: emptyArtifactIdentities({ + fixture: spec?.fixtureSha256 ? { id: "source-under-mutation", sha256: spec.fixtureSha256 } : null, + stackAdapter: (args.backend ?? spec?.backend) ? { id: args.backend ?? spec.backend } : null, + }), + durationMs: Date.now() - startedAt, + app: resolve(args.app), + mutations: resolve(args.mutations), + fixtureSha256: spec?.fixtureSha256 ?? null, + spec: args.spec ? resolve(args.spec) : null, + backend: args.backend ?? spec?.backend ?? null, + track: args.track ?? spec?.track ?? null, + ok: false, + gradeReports, + priorMutationControl, + baseline: { sourceBundle: baselineBundleEvidence }, + outcome: { + kind: "harness_failure", + phase: "mutation-control", + reason: mutationFailureMessage(error), + }, + }; + try { + const outputPath = artifactPath(id); + writeRunJson(outputPath, mutationHarnessFailureArtifact( + currentControlArtifact?.() ?? artifact, mutationFailureMessage(error), generatedAt)); + console.error( + `mutation harness failure: ${mutationFailureMessage(error)}\nartifact: ${outputPath}`, + ); + } catch (artifactError) { + console.error( + `mutation harness failure: ${mutationFailureMessage(error)}\nfailed to write failure artifact: ${errorMessage(artifactError)}`, + ); + } + process.exitCode = 2; +} + +async function main(): Promise { + spec = readMutationManifest(args.mutations!); + const fullMutations = spec.mutations; + const shard = args.mutationShardCount === undefined + ? { index: 0, count: 1, mutationIds: fullMutations.map(mutation => mutation.id as string), + mutations: fullMutations } + : mutationShard(fullMutations, + { index: args.mutationShardIndex!, count: args.mutationShardCount, + defaultScenario: spec.scenario }); + if (shard.mutations.length === 0) throw new Error('mutation shard has no assigned mutations'); + spec.mutations = shard.mutations; + if (args.backend && args.backend !== spec.backend) { + throw new Error( + `--backend conflicts with manifest backend ${spec.backend}`, + ); + } + if (args.track && args.track !== spec.track) { + throw new Error(`--track conflicts with manifest track ${spec.track}`); + } + args.backend = spec.backend; + args.track = spec.track; + const track = loadTrack(args.track); + const binding = resolveRecipeRelease(track, Number(args.level), args.recipe); + if (!binding) throw new Error(`${args.track} L${args.level} has no recipe release`); + args.recipe = binding.release.id; + args.expectedRecipeSha256 = binding.release.contentSha256; + args.recipeTask = createBoundRecipeTaskRequest(binding, { + taskMode: binding.plan.recipe.task.mode === 'action' ? 'fresh' : undefined, + }).request; + const recipeRelease = binding.release; + args.dbName ??= dbName(track, Number(args.runIndex)); + args.reseedOnReset = track.reseedOnReset; + const definitions = validateMutationDefinitions(spec.mutations, + { defaultScenario: spec.scenario, requireScenario: true }); + if (!definitions.ok) { + throw new Error( + `invalid mutation manifest: ${ + definitions.issues.map((issue) => + `${issue.mutation ?? ""}:${issue.kind}` + ).join(", ") + }`, + ); + } + const groups = new Map(); + for (const [scenario, mutations] of groupMutationsByScenario(spec)) { + const declaredSpec = resolveMutationScenarioPath(scenario); + groups.set(declaredSpec, mutations); + } + // Hosted apps serve this build; development servers compile client source on demand. + const clientDist = `${CODING_CONTAINER_APP_ROOT}/client/dist`; + const cleanClientDist = spec.mutations.some(mutation => + mutationFileEdits(mutation).some(edit => edit.file.replaceAll('\\', '/').startsWith('client/'))) + && existsSync(join(args.app, 'client', 'dist')) ? `/tmp/stack-bench-mutation-${randomUUID()}-client-dist` : null; + if (cleanClientDist) mutationClientCommand(args.backend!, 'cp', + ['-R', '--', clientDist, cleanClientDist], 120_000); + + // Reject backups left by an interrupted run before grading the baseline. + for (const m of spec.mutations) { + for (const file of new Set(mutationFileEdits(m).map(edit => edit.file))) { + const stale = resolveMutationFile(args.app, file) + ".mutation-backup"; + if (existsSync(stale)) { + throw new Error( + `${stale} exists; restore the interrupted mutation backup before running again`, + ); + } + } + } + + // Catch dirty source even when no backup file remains. + assertAppSourceIdentity(args.app, spec.fixtureSha256, 'mutation fixture'); + + // Reject missing or ambiguous edit anchors before baseline grading. + for (const m of spec.mutations) { + for (const edit of mutationFileEdits(m)) { + const source = readFileSync(resolveMutationFile(args.app, edit.file), "utf8"); + const matches = source.split(edit.find).length - 1; + if (matches !== 1) { + throw new Error( + `${m.id} anchor matched ${matches} times in ${edit.file}; expected exactly once`, + ); + } + } + } + + const plans = [...groups].map(([scenarioPath, mutations]) => { + const selectedCheckKeys = releaseScenarioCheckKeys(recipeRelease, track.dir, scenarioPath, + args.selectedCheckKeys ?? null); + return { scenarioPath, scenario: scenarioKey(scenarioPath), mutations, selectedCheckKeys, + checkpoint: checkpointGroup(scenarioPath, mutations, selectedCheckKeys) }; + }); + const cleanBaselineBundle = args.baselineBundle + ? readArtifactPayload(args.baselineBundle, { expectedKind: 'grade_bundle' }) + : null; + if (cleanBaselineBundle && !args.expectedCalibrationIdentity) { + throw new Error('a reusable clean baseline requires its expected calibration identity'); + } + const checkpoint = checkpointIdentity(plans.map(plan => plan.checkpoint), shard, track); + const resumed = resumableEvidence(args.resumeFrom, checkpoint); + const results: MutationResult[] = [...resumed.results] as MutationResult[]; + const baselines: BaselineEntry[] = [...resumed.baselines] as BaselineEntry[]; + const completedIds = new Set(results.map(result => result.id)); + if (completedIds.size !== results.length) { + throw new Error('mutation checkpoint contains duplicate results'); + } + const outputPath = artifactPath(args.mutationAttemptId); + if (args.baselineBundle) baselineBundleEvidence = { + path: relative(dirname(outputPath), resolve(args.baselineBundle)).replaceAll('\\', '/'), + sha256: sha256(readFileSync(args.baselineBundle)), + }; + if (args.resumeFrom) { + let priorPath = resolve(args.resumeFrom); + if (priorPath === outputPath || (args.checkpointOut && priorPath === resolve(args.checkpointOut))) { + const retainedPath = `${priorPath}.prior-${randomUUID()}.json`; + copyFileSync(priorPath, retainedPath); + priorPath = retainedPath; + } + priorMutationControl = { path: relative(dirname(outputPath), priorPath).replaceAll('\\', '/'), + sha256: sha256(readFileSync(priorPath)) }; + } + const deadline = startedAt + args.maxRuntimeMinutes * 60_000; + + const createControlArtifact = (status: 'running' | 'incomplete' | 'complete', reason: string | null = null) => { + const ordered = [...results].sort((left, right) => + shard.mutationIds.indexOf(left.id) - shard.mutationIds.indexOf(right.id)); + const clean = ordered.filter(result => result.status === 'CAUGHT'); + const orderedBaselines = plans.map(plan => baselines.find(entry => + entry.scenario === plan.scenario)).filter((entry): entry is BaselineEntry => Boolean(entry)); + const remaining = shard.mutationIds.filter(id => !completedIds.has(id)); + return { + id: args.mutationAttemptId, + kind: 'mutation_control', + startedAt: startedIso, + completedAt: status === 'running' ? null : new Date().toISOString(), + parentAttemptId: args.parentAttemptId ?? null, + identities: emptyArtifactIdentities({ + fixture: { id: 'source-under-mutation', sha256: spec.fixtureSha256 }, + recipe: { id: recipeRelease.id, sha256: recipeRelease.contentSha256 }, + stackAdapter: { id: args.backend }, + }), + durationMs: Date.now() - startedAt, + app: resolve(args.app), + mutations: resolve(args.mutations), + fixtureSha256: spec.fixtureSha256, + spec: plans.map(plan => plan.scenario), + backend: args.backend, + track: args.track, + shard: { index: shard.index, count: shard.count, mutationIds: shard.mutationIds }, + baseline: { + sourceBundle: baselineBundleEvidence, + total: orderedBaselines.reduce((sum, entry) => sum + Number(entry.total), 0), + max: orderedBaselines.reduce((sum, entry) => sum + Number(entry.max), 0), + scenarios: orderedBaselines, + }, + ok: status === 'complete' && clean.length === ordered.length + && ordered.length === shard.mutationIds.length, + ...(status === 'complete' ? {} : { outcome: { kind: 'incomplete', + phase: 'mutation-control', reason: reason ?? 'mutation batch is in progress' } }), + summary: { caught: clean.length, completed: ordered.length, + total: shard.mutationIds.length, remaining: remaining.length }, + results: ordered, + gradeReports, + priorMutationControl, + checkpoint: { ...checkpoint, status, maxRuntimeMinutes: args.maxRuntimeMinutes, + updatedAt: new Date().toISOString() }, + }; + }; + currentControlArtifact = () => createControlArtifact('incomplete'); + + const persist = (status: 'running' | 'incomplete' | 'complete', reason: string | null = null) => { + assertAppSourceIdentity(args.app, spec.fixtureSha256, + 'mutation fixture before checkpoint'); + const artifact = createControlArtifact(status, reason); + writeRunJson(outputPath, artifact); + if (args.checkpointOut && resolve(args.checkpointOut) !== outputPath) { + const rebase = (ref: { path: string; sha256: string | null }) => ({ ...ref, + path: relative(dirname(resolve(args.checkpointOut!)), resolve(dirname(outputPath), ref.path)).replaceAll('\\', '/') }); + writeRunJson(args.checkpointOut, { ...artifact, + baseline: { ...artifact.baseline, sourceBundle: baselineBundleEvidence ? rebase(baselineBundleEvidence) : null }, + gradeReports: gradeReports.map(receipt => ({ ...receipt, report: rebase(receipt.report) })), + priorMutationControl: priorMutationControl ? rebase(priorMutationControl) : null }); + } + return artifact; + }; + const invokeGrade = (mutationId: string | null) => { + const index = gradeReports.length; + return retainMutationGrade(outputPath, { scenario: scenarioKey(args.spec!), mutationId }, + path => grade(args, path, deadline), receipt => { + gradeReports[index] = receipt; + // During a mutant invocation source differs by design; this is a running receipt, not acceptance. + writeRunJson(outputPath, createControlArtifact('running')); + }); + }; + const stopAtBudget = () => { + const artifact = persist('incomplete', + `mutation batch reached its ${args.maxRuntimeMinutes} minute limit`); + console.log(`\n${artifact.summary.completed}/${artifact.summary.total} mutations completed; ` + + `${artifact.summary.remaining} remain`); + console.log(`checkpoint: ${args.checkpointOut ?? outputPath}`); + process.exitCode = 3; + }; + + for (const plan of plans) { + const { scenarioPath, scenario, mutations, selectedCheckKeys } = plan; + const pending = mutations.filter((mutation: MutationDefinition) => !completedIds.has(mutation.id as string)); + if (pending.length === 0) continue; + if (Date.now() >= deadline) return stopAtBudget(); + args.spec = scenarioPath; + args.selectedCheckKeys = selectedCheckKeys; + args.gradeTimeoutMs = gradingSourceTimeoutMs(binding.plan.packs, + recipeRelease.checkCatalog.filter(check => selectedCheckKeys.includes(check.stableKey))); + let baseline; + if (cleanBaselineBundle) { + const reused = reusableMutationBaseline(cleanBaselineBundle, { + backend: args.backend, + track: args.track, + level: Number(args.level), + fixtureSha256: spec.fixtureSha256, + recipe: { id: recipeRelease.id, sha256: recipeRelease.contentSha256 }, + identities: { + engine: currentEngineIdentity(), + calibration: args.expectedCalibrationIdentity, + stackAdapter: { id: args.backend }, + }, + selectedCheckKeys, + }); + if (!reused.ok) { + throw new Error(`cannot reuse clean baseline for ${scenarioPath}: ${reused.reason}`); + } + baseline = reused.report; + console.log(`Baseline (verified clean evidence, ${scenarioPath})...`); + } else { + console.log(`Baseline (unmutated app, ${scenarioPath})...`); + try { + baseline = await invokeGrade(null); + } catch (error) { + if (error instanceof MutationBatchDeadlineError) return stopAtBudget(); + throw error; + } + const validation = validateMutationBaseline(baseline, mutations); + if (!validation.ok && isRetryableMutationBaseline(validation.issues)) { + console.log(' transient baseline failure; retrying once'); + try { + baseline = await invokeGrade(null); + } catch (error) { + if (error instanceof MutationBatchDeadlineError) return stopAtBudget(); + throw error; + } + } + } + const baselineValidation = validateMutationBaseline(baseline, mutations); + if (!baselineValidation.ok) { + throw new Error( + `reference baseline is not known-good for ${scenarioPath}: ${ + JSON.stringify(baselineValidation.issues) + }`, + ); + } + console.log( + ` baseline: ${baseline.total}/${baseline.max} ${ + (baseline.features ?? []).map((f) => `F${f.id}:${(f as NonNullable[number]).score}`).join(" ") + }\n`, + ); + const baselineEntry = { scenario, identitySha256: plan.checkpoint.identitySha256, + total: baseline.total, max: baseline.max }; + const priorBaseline = baselines.findIndex(entry => entry.scenario === scenario); + if (priorBaseline === -1) baselines.push(baselineEntry); + else baselines[priorBaseline] = baselineEntry; + + for (const m of pending) { + if (Date.now() >= deadline) return stopAtBudget(); + const byFile = new Map(); + for (const edit of mutationFileEdits(m)) { + const target = resolveMutationFile(args.app, edit.file); + if (!byFile.has(target)) { + byFile.set(target, { + target, + backup: `${target}.mutation-backup`, + original: readFileSync(target, "utf8"), + edits: [], + }); + } + byFile.get(target)!.edits.push(edit); + } + const files = [...byFile.values()]; + const clientChanged = files.some(file => relative(args.app!, file.target) + .split(sep)[0] === 'client'); + const backedUp: MutationFile[] = []; + let r: GradeReport | undefined; + let classified: ReturnType | undefined; + let deadlineReached = false; + let mutationError: unknown = null; + try { + for (const file of files) { + copyFileSync(file.target, file.backup); + backedUp.push(file); + } + for (const file of files) { + writeFileSync(file.target, + // A function inserts the replacement as written; a string would expand $' and $&. + file.edits.reduce((src, edit) => src.replace(edit.find, () => edit.replace), + file.original)); + } + if (clientChanged) rebuildClientAfterSourceChange(args, deadline); + r = await invokeGrade(m.id as string); + classified = classifyMutationResult( + baseline as Parameters[0], + r as Parameters[1], + m, + ); + if (isRetryableMutationResult(classified.status)) { + console.log(` ${classified.status} result; retrying once`); + r = await invokeGrade(m.id as string); + classified = classifyMutationResult( + baseline as Parameters[0], + r as Parameters[1], + m, + ); + } + } catch (error) { + if (error instanceof MutationBatchDeadlineError) deadlineReached = true; + else mutationError = error; + } + + const cleanupErrors: Error[] = []; + for (const file of backedUp) { + try { + restoreMutationSource(file); + } catch (error) { + cleanupErrors.push(new Error(`cannot restore ${file.target}: ${errorMessage(error)}`, + { cause: error })); + } + } + for (const file of files) { + try { + if (existsSync(file.backup) || readFileSync(file.target, 'utf8') !== file.original) { + cleanupErrors.push(new Error(`restore verification failed for ${file.target}`)); + } + } catch (error) { + cleanupErrors.push(new Error(`cannot verify restored source ${file.target}: ${errorMessage(error)}`, + { cause: error })); + } + } + if (clientChanged) { + try { + mutationClientCommand(args.backend!, 'rm', ['-rf', '--', clientDist], 120_000); + if (cleanClientDist) mutationClientCommand(args.backend!, 'cp', + ['-R', '--', cleanClientDist, clientDist], 120_000); + } catch (error) { + cleanupErrors.push(new Error(`cannot restore the built client: ${errorMessage(error)}`, + { cause: error })); + } + } + // A normal error leaves enough budget to restore the clean runtime. A + // deadline stop leaves clean source and lets the lease owner stop it. + if (mutationError !== null && cleanupErrors.length === 0) { + try { + await resetMutationDatabase(args, deadline); + } catch (error) { + cleanupErrors.push(new Error(`cannot restore the clean runtime: ${errorMessage(error)}`, + { cause: error })); + } + } + if (cleanupErrors.length > 0) { + const errors = mutationError === null ? cleanupErrors : [mutationError, ...cleanupErrors]; + throw new AggregateError(errors, + 'mutation cleanup failed; do not reuse this app source'); + } + if (mutationError !== null) throw mutationError; + if (deadlineReached) return stopAtBudget(); + if (!r || !classified) throw new Error('mutation grade completed without a result'); + results.push({ id: m.id, scenario, + targets: mutationTargetKeys(m), ...classified }); + completedIds.add(m.id); + persist('running'); + console.log( + `${classified.status.padEnd(20)} ${m.id} — expected ${ + classified.targetKeys.join(", ") + }`, + ); + if (classified.regressions.length) { + console.log( + ` failed criteria: ${ + classified.regressions.map((item) => item.key).join(", ") + }`, + ); + } + if (classified.targetOffAssertion.length) { + console.log(` stopped before an observation: ${classified.targetOffAssertion + .map(item => `${item.key} at ${item.action ?? 'no recorded action'}`).join(', ')}`); + } + } + } + + // Detect any source change outside the files restored above. + assertAppSourceIdentity(args.app, spec.fixtureSha256, 'mutation fixture after worker completion'); + // Restore the clean runtime and database before releasing the worker lease. + await resetMutationDatabase(args, null); + + if (cleanClientDist) mutationClientCommand(args.backend!, 'rm', ['-rf', '--', cleanClientDist], 120_000); + const artifact = persist('complete'); + console.log(`\n${artifact.summary.caught}/${artifact.summary.total} mutations cleanly caught`); + console.log(`artifact: ${outputPath}`); + if (!artifact.ok) process.exitCode = 1; +} + +function run(): void { + try { + args = parseMutationArgs(process.argv); + } catch (error) { + console.error(errorMessage(error)); + process.exitCode = 2; + return; + } + startedAt = Date.now(); + startedIso = new Date(startedAt).toISOString(); + main().catch(recordHarnessFailure); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) run(); diff --git a/tools/stack-bench/grader/mutations/convex-ecommerce.json b/tools/stack-bench/grader/mutations/convex-ecommerce.json new file mode 100644 index 00000000000..7d335a1e2b3 --- /dev/null +++ b/tools/stack-bench/grader/mutations/convex-ecommerce.json @@ -0,0 +1,2366 @@ +{ + "schemaVersion": 3, + "fixtureSha256": "8c647a96fbab91b8f51c1d7b6831f926a99d8e681b145b66add7ae52ce24e6c6", + "backend": "convex", + "track": "ecommerce", + "note": "Native Convex selected-L3 defect controls. Authored coverage is not qualification; every baseline and mutant requires execution against this exact source hash. Recovery controls use real native reconnect callbacks and writes.", + "mutations": [ + { + "id": "signup-does-not-expose-created-account", + "scenario": "tracks/ecommerce/scenarios/01-account-create.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1a" + ], + "desc": "Authenticated account views omit the created username, including native subscription updates.", + "file": "convex/accounts.js", + "edits": [ + { + "find": "id: a._id, username: a.username, roles: a.roles", + "replace": "id: a._id, username: '', roles: a.roles" + } + ] + }, + { + "id": "session-token-not-persisted", + "scenario": "tracks/ecommerce/scenarios/01-account-reload.json", + "targets": [ + "ecommerce.spec.state-durability.session-reload.1e" + ], + "desc": "Fresh sign-in works, but each new document discards the stored access token before initializing native authentication, so reload cannot restore the session.", + "file": "client/src/request.ts", + "edits": [ + { + "find": "export const TOKEN_KEY = 'convex_shop_token';", + "replace": "export const TOKEN_KEY = 'convex_shop_token';\nlocalStorage.removeItem(TOKEN_KEY);" + } + ] + }, + { + "id": "reload-hydrates-an-empty-cart", + "scenario": "tracks/ecommerce/scenarios/01-cart.json", + "targets": [ + "ecommerce.spec.state-durability.cart-reload.4b" + ], + "desc": "Existing-token page loads discard cart hydration from the native subscription; the first signup page and its working cart are unchanged.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "const CATALOG_PAGE_SIZE = 10;", + "replace": "const CATALOG_PAGE_SIZE = 10;\nconst discardHydratedCart = Boolean(localStorage.getItem(TOKEN_KEY));" + }, + { + "find": " setCart(state.cart);", + "replace": " setCart(discardHydratedCart ? { items: [], total: 0 } : state.cart);" + } + ] + }, + { + "id": "warehouse-view-omits-one-location", + "scenario": "tracks/ecommerce/scenarios/01-warehouse-admin-staff.json", + "targets": [ + "ecommerce.feature.warehouse-admin.warehouse-view.7b" + ], + "desc": "The admin warehouse projection truncates the final item-location row.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " {overview.locations.map((loc) => (", + "replace": " {overview.locations.slice(0, -1).map((loc) => (" + } + ] + }, + { + "id": "sign-in-starts-an-empty-cart", + "scenario": "tracks/ecommerce/scenarios/progression-account-state-reload.json", + "targets": [ + "ecommerce.spec.state-durability.account-state-recovery.105a" + ], + "desc": "A successful sign-in deletes the account's stored cart lines, so a fresh sign-in after a restart finds an empty cart; sign-up, token-restored sessions and orders are unchanged.", + "file": "convex/accounts.js", + "edits": [ + { + "find": " if (await ctx.db.query('order_account').withIndex('auth_user', q => q.eq('auth_user_id', userId)).unique()) return;", + "replace": " const existing = await ctx.db.query('order_account').withIndex('auth_user', q => q.eq('auth_user_id', userId)).unique();\n if (existing) {\n // mutant: each sign-in starts a new cart instead of restoring the account cart\n for (const line of await ctx.db.query('order_cart').withIndex('account', q => q.eq('account_id', existing._id)).collect()) await ctx.db.delete(line._id);\n return;\n }" + } + ] + }, + { + "id": "purchased-review-ui-does-not-submit", + "scenario": "tracks/ecommerce/scenarios/01-review-eligibility.json", + "targets": [ + "ecommerce.spec.access-control.review-eligibility.108b" + ], + "desc": "The review form discards valid customer submissions. Direct authorized and unauthorized review calls retain their normal behavior.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const handleReviewSubmit = async (itemId: string, rating: number, comment: string) => {\n setReviewError(\"\");", + "replace": " const handleReviewSubmit = async (itemId: string, rating: number, comment: string) => {\n setReviewError(\"Review submission unavailable\");\n return;" + } + ] + }, + { + "id": "progression-customer-sees-fulfilment-content", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-access.json", + "targets": [ + "ecommerce.spec.access-control.fulfilment-area-boundary.1d" + ], + "desc": "Expose the protected staff area to signed-in customers, including its navigation and content.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " {(currentUser?.isStaff || currentUser?.isAdmin) && (\n ", + "replace": " setPreference(value => ({ ...value, order: !value.order }))} />" + }, + { + "find": "function SupportTicket({ ticket, user, act }: any) {", + "replace": "function OrderNotificationToggle({ on, onToggle }: { on: boolean; onToggle: () => void }) {\n const [shown] = useState(on); // mutant: the toggle keeps the state loaded when the card opened\n return ;\n}\n\nfunction SupportTicket({ ticket, user, act }: any) {" + } + ] + }, + { + "id": "role-editor-snaps-back-to-stored-role", + "scenario": "tracks/ecommerce/scenarios/progression-staff-roles.json", + "targets": [ + "ecommerce.progression.staff-roles.staff-roles.621c" + ], + "desc": "Pressing Save persists the role but resets the dropdown to the role that was stored before the save, so the assignment is not visible until a reload.", + "file": "client/src/ProgressionPanel.tsx", + "edits": [ + { + "find": " ", + "replace": " " + } + ] + }, + { + "id": "review-script-unsafe-render", + "scenario": "tracks/ecommerce/scenarios/progression-review-script.json", + "targets": [ + "ecommerce.progression.review-access-specifications.stored-review-script.9180a" + ], + "desc": "Render stored review text as HTML so it executes in another customer session.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "
{r.comment}
", + "replace": "
" + } + ] + }, + { + "id": "shipping-counts-sale-twice", + "scenario": "tracks/ecommerce/scenarios/progression-shipping-accounting.json", + "targets": [ + "ecommerce.inventory-operations.shipping-accounting.202e" + ], + "desc": "Shipping doubles the authoritative booked order total.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(orderId, { status: 'shipped' });", + "replace": "await ctx.db.patch(orderId, { status: 'shipped', total: order.total * 2 });" + } + ] + }, + { + "id": "checkout-retains-purchased-cart", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Checkout does not consume its cart; concurrent calls create duplicate effects.", + "file": "convex/shop.js", + "edits": [ + { + "find": "for (const line of lines) await ctx.db.delete(line._id);", + "replace": "// Defect: the completed cart remains available for another checkout." + } + ] + }, + { + "id": "checkout-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Reject every checkout, including the positive control; refusal alone is insufficient.", + "file": "convex/shop.js", + "edits": [ + { + "find": "if (!lines.length) throw new ConvexError('Cart is empty');", + "replace": "throw new ConvexError('Checkout rejected');" + } + ] + }, + { + "id": "purchases-do-not-decrement-stock", + "scenario": "tracks/ecommerce/scenarios/01-last-unit.json", + "targets": [ + "ecommerce.spec.concurrency-safety.last-unit.201a", + "ecommerce.spec.concurrency-safety.last-unit.201b", + "ecommerce.spec.concurrency-safety.last-unit.201c" + ], + "desc": "Record purchases without decrementing stock, causing unchanged inventory, excess orders and excess revenue in the same purchase burst.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(row._id, { quantity: row.quantity - take });", + "replace": "await ctx.db.patch(row._id, { quantity: row.quantity });" + } + ] + }, + { + "id": "purchase-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/01-last-unit.json", + "targets": [ + "ecommerce.spec.concurrency-safety.last-unit.201a", + "ecommerce.spec.concurrency-safety.last-unit.201b", + "ecommerce.spec.concurrency-safety.last-unit.201c" + ], + "desc": "Reject every purchase, leaving unchanged stock, zero successful orders and zero revenue in the required purchase burst.", + "file": "convex/shop.js", + "edits": [ + { + "find": "const orderId = await ctx.db.insert('order_header',", + "replace": "throw new ConvexError('Purchases rejected');\n const orderId = await ctx.db.insert('order_header'," + } + ] + }, + { + "id": "order-history-not-owner-scoped", + "scenario": "tracks/ecommerce/scenarios/01-order-ownership.json", + "targets": [ + "ecommerce.spec.access-control.order-ownership.106a" + ], + "desc": "Order history exposes every customer's orders instead of only the signed-in customer's orders.", + "file": "convex/shop.js", + "edits": [ + { + "find": "orders: orderViews.filter(o => ownOrderIds.has(o.id)),", + "replace": "orders: orderViews," + } + ] + }, + { + "id": "profile-read-not-owner-scoped", + "scenario": "tracks/ecommerce/scenarios/progression-customer-profile.json", + "targets": [ + "ecommerce.spec.access-control.customer-profile-privacy.620b" + ], + "desc": "Profile reads expose another account saved address.", + "file": "convex/progression.js", + "edits": [ + { + "find": "profile: user?.profile || null", + "replace": "profile: (await rows(ctx, 'order_account')).find(a => a.profile.address)?.profile || null" + } + ] + }, + { + "id": "support-read-not-owner-scoped", + "scenario": "tracks/ecommerce/scenarios/progression-support-history.json", + "targets": [ + "ecommerce.spec.access-control.support-history-privacy.612b", + "ecommerce.spec.access-control.support-history-logout.612d" + ], + "desc": "All customers and signed-out clients receive private support history.", + "file": "convex/progression.js", + "edits": [ + { + "find": ".filter(t => staff || (user && t.accountId === user._id))", + "replace": ".filter(() => true)" + } + ] + }, + { + "id": "support-reply-not-owner-scoped", + "scenario": "tracks/ecommerce/scenarios/progression-managed-support-privacy.json", + "targets": [ + "ecommerce.spec.access-control.managed-support-privacy.613b" + ], + "desc": "A customer can reply to another customer support case.", + "file": "convex/progression.js", + "edits": [ + { + "find": "if (!ticket || (!isStaff(user) && ticket.accountId !== user._id)) throw new ConvexError('Ticket not found');", + "replace": "if (!ticket) throw new ConvexError('Ticket not found');" + } + ] + }, + { + "id": "preference-read-not-owner-scoped", + "scenario": "tracks/ecommerce/scenarios/progression-notification-preferences.json", + "targets": [ + "ecommerce.spec.access-control.notification-preferences-privacy.630b" + ], + "desc": "Notification settings expose another customer saved preference.", + "file": "convex/progression.js", + "edits": [ + { + "find": "preference: user?.preference || { order: false, stock: false }", + "replace": "preference: (await rows(ctx, 'order_account')).find(a => a.preference.order || a.preference.stock)?.preference || { order: false, stock: false }" + } + ] + }, + { + "id": "staff-admin-access-survives-role-removal", + "scenario": "tracks/ecommerce/scenarios/progression-staff-roles.json", + "targets": [ + "ecommerce.spec.access-control.staff-role-revocation.621d" + ], + "desc": "Role reassignment retains the previous administrator privilege.", + "file": "convex/progression.js", + "edits": [ + { + "find": "await ctx.db.patch(accountId, { roles: [role] });", + "replace": "await ctx.db.patch(accountId, { roles: [role, ...(await ctx.db.get(accountId)).roles.filter(previous => previous === 'admin' && previous !== role)] });" + } + ] + }, + { + "id": "role-assignment-trusts-sign-in-admin-flag", + "scenario": "tracks/ecommerce/scenarios/progression-staff-roles.json", + "targets": [ + "ecommerce.spec.access-control.staff-role-revocation.621d" + ], + "desc": "Each sign-in records on the account whether it was an administrator at that moment, and role assignment authorizes from that recorded flag instead of the current roles. An administrator who signed in before being changed back to staff keeps assigning roles until signing in again.", + "file": "convex/progression.js", + "edits": [ + { + "file": "convex/schema.js", + "find": " preference: v.object({ order: v.boolean(), stock: v.boolean() }),\n }).index('auth_user', ['auth_user_id'])", + "replace": " preference: v.object({ order: v.boolean(), stock: v.boolean() }),\n sign_in_admin: v.optional(v.boolean()),\n }).index('auth_user', ['auth_user_id'])" + }, + { + "file": "convex/accounts.js", + "find": " if (await ctx.db.query('order_account').withIndex('auth_user', q => q.eq('auth_user_id', userId)).unique()) return;", + "replace": " const existing = await ctx.db.query('order_account').withIndex('auth_user', q => q.eq('auth_user_id', userId)).unique();\n if (existing) { await ctx.db.patch(existing._id, { sign_in_admin: isAdmin(existing) }); return; }" + }, + { + "find": "await operator(ctx, true);\n if (!['staff', 'inventory', 'admin'].includes(role))", + "replace": "if (!(await account(ctx)).sign_in_admin) throw new ConvexError('Access denied');\n if (!['staff', 'inventory', 'admin'].includes(role))" + } + ] + }, + { + "id": "review-eligibility-not-enforced", + "scenario": "tracks/ecommerce/scenarios/progression-review-access.json", + "targets": [ + "ecommerce.progression.review-access-specifications.review-eligibility-direct.618a" + ], + "desc": "Native review operation permits an account that never purchased the item.", + "file": "convex/shop.js", + "edits": [ + { + "find": "if (!purchased) throw new ConvexError('Purchase this item before reviewing');", + "replace": "// Defect: non-buyers may submit a review." + } + ] + }, + { + "id": "review-script-reject-all", + "scenario": "tracks/ecommerce/scenarios/progression-review-script.json", + "targets": [ + "ecommerce.progression.review-access-specifications.stored-review-script.9180a" + ], + "desc": "Reject every review, including benign text; rejecting all input is not safe rendering.", + "file": "convex/shop.js", + "edits": [ + { + "find": "if (!purchased) throw new ConvexError('Purchase this item before reviewing');", + "replace": "throw new ConvexError('Reviews rejected');" + } + ] + }, + { + "id": "duplicate-review-accepted", + "scenario": "tracks/ecommerce/scenarios/01-review-uniqueness.json", + "targets": [ + "ecommerce.spec.transactional-integrity.unique-review.6b" + ], + "desc": "Same purchaser can record multiple reviews for one item.", + "file": "convex/shop.js", + "edits": [ + { + "find": "if (await ctx.db.query('review').withIndex('author', q => q.eq('itemId', args.itemId).eq('accountId', user._id)).unique()) throw new ConvexError('Already reviewed');", + "replace": "// Defect: duplicate author reviews are accepted." + } + ] + }, + { + "id": "external-stock-client-snapshot-open", + "scenario": "tracks/ecommerce/scenarios/01-external-live-sync.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock.901a" + ], + "desc": "Keep the first subscribed catalog values instead of accepting later native stock changes.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setItems(state.items);", + "replace": " setItems(current => current.length ? current : state.items);" + } + ] + }, + { + "id": "external-stock-client-snapshot-reconnect", + "scenario": "tracks/ecommerce/scenarios/01-external-reconnect-sync.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock.901d" + ], + "desc": "Keep the first subscribed catalog values instead of accepting later native stock changes.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setItems(state.items);", + "replace": " setItems(current => current.length ? current : state.items);" + } + ] + }, + { + "id": "duplicate-signup-reports-success", + "scenario": "tracks/ecommerce/scenarios/01-account-duplicate.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1b" + ], + "desc": "Duplicate signup authenticates the existing account without its password.", + "file": "convex/auth.js", + "edits": [ + { + "find": "const { user } = await createAccount(ctx, {", + "replace": "const existing = await retrieveAccount(ctx, { provider: 'password', account: { id: params.username } }).catch(() => null);\n const { user } = existing ?? await createAccount(ctx, {" + }, + { + "find": "if (user.registrationNonce !== registrationNonce) throw new ConvexError('Username already exists');", + "replace": "// Defect: creating an existing account grants a session." + } + ] + }, + { + "id": "signin-skips-password-verification", + "scenario": "tracks/ecommerce/scenarios/01-account-password.json", + "targets": [ + "ecommerce.spec.access-control.account-password.1c" + ], + "desc": "Signin accepts an existing account without requiring its password to match.", + "file": "convex/auth.js", + "edits": [ + { + "find": "verifySecret: (password, hash) => new Scrypt().verify(hash, password),", + "replace": "verifySecret: async () => true," + } + ] + }, + { + "id": "signout-keeps-current-account", + "scenario": "tracks/ecommerce/scenarios/01-account-signout.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1d" + ], + "desc": "Signout disconnects the token state but leaves the current account and persisted credential in place.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " await signOut();\n clearSession();", + "replace": " // Defect: sign-out leaves the authenticated account intact." + } + ] + }, + { + "id": "cart-repeat-does-not-increment", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4a" + ], + "desc": "An existing cart line remains at its old quantity.", + "file": "convex/shop.js", + "edits": [ + { + "find": "const next = add ? (existing?.quantity || 0) + amount : amount;", + "replace": "const next = add ? amount : amount;" + } + ] + }, + { + "id": "checkout-leaves-cart-claimed", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "Checkout creates an order without emptying the cart.", + "file": "convex/shop.js", + "edits": [ + { + "find": "for (const line of lines) await ctx.db.delete(line._id);", + "replace": "// Defect: checked-out cart lines survive." + } + ] + }, + { + "id": "catalog-price-is-offset", + "scenario": "tracks/ecommerce/scenarios/01-catalog-values.json", + "targets": [ + "ecommerce.feature.catalog.catalog-values.2a" + ], + "desc": "Public catalog prices are reported one unit too high.", + "file": "convex/seed.js", + "edits": [ + { + "find": "name: entry.name, price: entry.price,", + "replace": "name: entry.name, price: entry.price + 1," + } + ] + }, + { + "id": "purchase-order-uses-zero-price", + "scenario": "tracks/ecommerce/scenarios/progression-purchasing.json", + "targets": [ + "ecommerce.feature.purchasing.purchase-order.3c" + ], + "desc": "A direct purchase records the item but stores a zero order total instead of the price paid.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(orderId, { total: total / 100 });", + "replace": "await ctx.db.patch(orderId, { total: 0 });" + } + ] + }, + { + "id": "review-comment-is-not-persisted", + "scenario": "tracks/ecommerce/scenarios/01-review-visibility.json", + "targets": [ + "ecommerce.feature.reviews.reviews.6a" + ], + "desc": "Review submission persists an empty comment rather than the customer's submitted text.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.insert('review', { ...args, accountId: user._id });", + "replace": "await ctx.db.insert('review', { ...args, comment: '', accountId: user._id });" + } + ] + }, + { + "id": "authorized-restock-does-not-change-stock", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.feature.warehouse-admin.admin-write.103a", + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Accept an administrator restock without changing warehouse stock.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(stock._id, { quantity: stock.quantity + args.quantity });", + "replace": "await ctx.db.patch(stock._id, { quantity: stock.quantity });" + } + ] + }, + { + "id": "purchases-do-not-affect-best-sellers", + "scenario": "tracks/ecommerce/scenarios/02-operational-best-sellers.json", + "targets": [ + "ecommerce.inventory-operations.operational-views.5d" + ], + "desc": "Rank signed-out recommendations without purchase counts.", + "file": "convex/shop.js", + "edits": [ + { + "find": "recommended: recommendations", + "replace": "recommended: [...catalog].sort((a, b) => a.name.localeCompare(b.name)).slice(0, 10)" + } + ] + }, + { + "id": "initial-dashboard-load-omits-low-stock", + "scenario": "tracks/ecommerce/scenarios/02-low-stock.json", + "targets": [ + "ecommerce.inventory-operations.operational-views.5e", + "ecommerce.spec.live-state.inventory-dashboard.5a" + ], + "desc": "The dashboard's initial load reports no low-stock items, so an item already at ten units or fewer is missing until stock moves; 5a's opening precondition re-asserts the same listing, so it necessarily regresses with 5e.", + "file": "convex/shop.js", + "edits": [ + { + "find": "lowStock: catalog.filter(i => i.stock <= 10).sort((a, b) => a.stock - b.stock)", + "replace": "lowStock: []" + } + ] + }, + { + "id": "category-totals-skip-the-newest-order", + "scenario": "tracks/ecommerce/scenarios/02-operational-category-totals.json", + "targets": [ + "ecommerce.inventory-operations.operational-views.5f" + ], + "desc": "Category totals leave out the most recent order, so the reloaded dashboard shows a purchase only once a later order pushes it into the aggregate.", + "file": "convex/shop.js", + "edits": [ + { + "find": "const sold = standing.filter(l => itemById.get(l.item_id)?.category === category);", + "replace": "const sold = standing.filter(l => itemById.get(l.item_id)?.category === category).slice(0, -1);" + } + ] + }, + { + "id": "transfer-debits-source-without-crediting-existing-destination", + "scenario": "tracks/ecommerce/scenarios/02-strengthened.json", + "targets": [ + "ecommerce.inventory-operations.warehouse-transfer.2a", + "ecommerce.operations-access.operator-authorization.201a" + ], + "desc": "A transfer debits the source but credits zero to the destination. This breaks both movement/conservation assertions and the authorized-transfer positive observation inside the authorization check; that last failure is not evidence of an unauthorized transfer.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(target._id, { quantity: target.quantity + args.quantity });", + "replace": "await ctx.db.patch(target._id, { quantity: target.quantity });" + } + ] + }, + { + "id": "cancel-does-not-restore-stock-fresh-client", + "scenario": "tracks/ecommerce/scenarios/02-self-contained.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202b" + ], + "desc": "Cancellation changes order state but skips restoration, so a fresh client reads the persisted shortfall.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(stock._id, { quantity: stock.quantity + allocation.quantity });", + "replace": "await ctx.db.patch(stock._id, { quantity: stock.quantity });" + } + ] + }, + { + "id": "transfer-creates-stock-during-race", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202d" + ], + "desc": "A transfer creates one extra unit while a purchase runs.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(target._id, { quantity: target.quantity + args.quantity });", + "replace": "await ctx.db.patch(target._id, { quantity: target.quantity + args.quantity + 1 });" + } + ] + }, + { + "id": "customer-can-cancel-scheduled-restock", + "scenario": "tracks/ecommerce/scenarios/03-deferred-access.json", + "targets": [ + "ecommerce.l3.deferred-access.scheduled-work-access.317a" + ], + "desc": "Scheduled-restock cancellation routes omit their staff checks.", + "file": "convex/progression.js", + "edits": [ + { + "find": "await operator(ctx, true);\n const restock = await ctx.db.get(restockId)", + "replace": "await account(ctx);\n const restock = await ctx.db.get(restockId)" + } + ] + }, + { + "id": "completed-restock-is-replayed", + "scenario": "tracks/ecommerce/scenarios/03-deferred-integrity.json", + "targets": [ + "ecommerce.l3.deferred-integrity.exactly-once.311a" + ], + "desc": "Completing the deferred work applies its stock effect twice.", + "file": "convex/progression.js", + "edits": [ + { + "find": "await ctx.db.patch(stock._id, { quantity: stock.quantity + restock.quantity });", + "replace": "await ctx.db.patch(stock._id, { quantity: stock.quantity + 2 * restock.quantity });" + } + ] + }, + { + "id": "due-restock-does-not-change-stock", + "scenario": "tracks/ecommerce/scenarios/03-scheduled-restock-apply.json", + "targets": [ + "ecommerce.l3.scheduled-restocks.scheduled-restocks.305a" + ], + "desc": "A due restock is recorded without adding stock.", + "file": "convex/progression.js", + "edits": [ + { + "find": "await ctx.db.patch(stock._id, { quantity: stock.quantity + restock.quantity });", + "replace": "await ctx.db.patch(stock._id, { quantity: stock.quantity });" + } + ] + }, + { + "id": "cancelled-restock-remains-pending", + "scenario": "tracks/ecommerce/scenarios/03-scheduled-restock-cancel.json", + "targets": [ + "ecommerce.l3.scheduled-restocks.scheduled-restocks.306a" + ], + "desc": "Cancelling a restock leaves it pending so it later applies.", + "file": "convex/progression.js", + "edits": [ + { + "find": "await ctx.scheduler.cancel(restock.scheduledId);\n await ctx.db.patch(restock._id, { status: 'cancelled' });", + "replace": "// Defect: cancellation does not cancel scheduled work." + } + ] + }, + { + "id": "server-time-restock-never-completes", + "scenario": "tracks/ecommerce/scenarios/03-server-time.json", + "targets": [ + "ecommerce.l3.server-time.server-time.312a" + ], + "desc": "Server time never selects a due restock after restart.", + "file": "convex/progression.js", + "edits": [ + { + "find": "ctx.scheduler.runAfter(args.delaySeconds * 1000,", + "replace": "ctx.scheduler.runAfter(86400000," + } + ] + }, + { + "id": "ship-acknowledges-without-changing-status", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-ship.json", + "targets": [ + "ecommerce.operations-access.fulfilment-queue.1c" + ], + "desc": "Shipping returns success but writes pending back to the order, leaving both live views unchanged.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(orderId, { status: 'shipped' });", + "replace": "// Defect: shipping acknowledges but leaves the order pending." + } + ] + }, + { + "id": "customer-can-ship-order-direct-1-1", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "The shipping route keeps authentication but drops its staff role gate.", + "file": "convex/shop.js", + "edits": [ + { + "find": "const user = await operator(ctx);", + "replace": "const user = await account(ctx);" + } + ] + }, + { + "id": "customer-can-cancel-foreign-order-1-1", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.order-owner.204a" + ], + "desc": "Cancellation retains authentication and pending-state validation but drops order ownership.", + "file": "convex/shop.js", + "edits": [ + { + "find": "if (!order || order.account_id !== user._id) throw new ConvexError('Order not found');", + "replace": "if (!order) throw new ConvexError('Order not found');" + } + ] + }, + { + "id": "promotion-save-drops-bounded-values", + "scenario": "tracks/ecommerce/scenarios/progression-promotion-rules.json", + "targets": [ + "ecommerce.progression.promotion-rules.promotion-rule-values.620a" + ], + "desc": "Promotion creation drops its bounded discount and limit.", + "file": "convex/progression.js", + "edits": [ + { + "find": "await ctx.db.insert('promotion', args);", + "replace": "await ctx.db.insert('promotion', { ...args, discountPercent: 0, usageLimit: 0 });" + } + ] + }, + { + "id": "stock-alert-delivery-is-suppressed", + "scenario": "tracks/ecommerce/scenarios/progression-stock-alert-delivery.json", + "targets": [ + "ecommerce.progression.stock-alerts.stock-alert-delivery.631c" + ], + "desc": "A request is accepted but restored stock does not deliver its alert.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.insert('notification', { accountId: alert.accountId, type: 'stock', message: `${item.name} is back in stock` });", + "replace": "// Defect: no notification is delivered." + } + ] + }, + { + "id": "support-intake-returns-no-reference", + "scenario": "tracks/ecommerce/scenarios/progression-support-intake.json", + "targets": [ + "ecommerce.progression.support-intake.support-intake.610a" + ], + "desc": "Support intake creates a ticket but omits its reference from the response.", + "file": "convex/progression.js", + "edits": [ + { + "find": "const reference = `SUP-${id.slice(-10)}`;", + "replace": "const reference = '';" + } + ] + }, + { + "id": "support-triage-discards-updates", + "scenario": "tracks/ecommerce/scenarios/progression-support-triage.json", + "targets": [ + "ecommerce.progression.support-triage.support-assignment.611a", + "ecommerce.progression.support-triage.support-priority.611b", + "ecommerce.progression.support-triage.support-status.611c" + ], + "desc": "Support triage accepts but discards assignment, priority, and status updates.", + "file": "convex/progression.js", + "edits": [ + { + "find": "await ctx.db.patch(ticketId, change);", + "replace": "// Defect: support edits are acknowledged but discarded." + } + ] + }, + { + "id": "cancel-does-not-restore-stock-feature", + "scenario": "tracks/ecommerce/scenarios/02-order-cancellation-core.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3a" + ], + "desc": "Cancellation changes order state but skips restoration of its recorded warehouse allocations.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(stock._id, { quantity: stock.quantity + allocation.quantity });", + "replace": "await ctx.db.patch(stock._id, { quantity: stock.quantity });" + } + ] + }, + { + "id": "cancel-restores-stock-but-keeps-pending-status", + "scenario": "tracks/ecommerce/scenarios/02-order-cancellation-history.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3b" + ], + "desc": "Cancellation restores allocations but writes pending back to order history.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(orderId, { status: 'cancelled', refunded: order.total });", + "replace": "await ctx.db.patch(orderId, { status: 'pending', refunded: order.total });" + } + ] + }, + { + "id": "cancellation-accounting-loses-stock-restoration", + "scenario": "tracks/ecommerce/scenarios/02-invariants.json", + "targets": [ + "ecommerce.returns-pricing.refund-accounting.203a" + ], + "desc": "Cancellation removes revenue and changes order status, but loses the original warehouse stock restoration. The native refund-accounting assertion must detect this.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(stock._id, { quantity: stock.quantity + allocation.quantity });", + "replace": "await ctx.db.patch(stock._id, { quantity: stock.quantity });" + } + ] + }, + { + "id": "negative-cart-quantity-is-accepted", + "scenario": "tracks/ecommerce/scenarios/01-cart-boundary.json", + "targets": [ + "ecommerce.spec.access-control.cart-boundary.109b" + ], + "desc": "Cart quantity updates accept values below one.", + "file": "convex/shop.js", + "edits": [ + { + "find": " quantity(amount, !add);", + "replace": " if (amount >= 0) quantity(amount, !add);" + } + ] + }, + { + "id": "customer-can-create-promotion", + "scenario": "tracks/ecommerce/scenarios/progression-promotion-rules.json", + "targets": [ + "ecommerce.spec.access-control.promotion-management-boundary.620b" + ], + "desc": "Promotion creation omits its staff authorization check.", + "file": "convex/progression.js", + "edits": [ + { + "find": "await operator(ctx);\n if (!args.code.trim()", + "replace": "await account(ctx);\n if (!args.code.trim()" + } + ] + }, + { + "id": "unpurchased-review-is-accepted", + "scenario": "tracks/ecommerce/scenarios/01-review-eligibility.json", + "targets": [ + "ecommerce.spec.access-control.review-eligibility.108a" + ], + "desc": "The review endpoint bypasses its completed-purchase eligibility check.", + "file": "convex/shop.js", + "edits": [ + { + "find": "if (!purchased) throw new ConvexError('Purchase this item before reviewing');", + "replace": "// Defect: review requires no purchase." + } + ] + }, + { + "id": "customer-signin-gains-staff-role", + "scenario": "tracks/ecommerce/scenarios/progression-staff-access.json", + "targets": [ + "ecommerce.spec.access-control.staff-area-boundary.601b" + ], + "desc": "Treat every authenticated account, including the seeded customer, as staff in authorization and public account state.", + "file": "convex/accounts.js", + "edits": [ + { + "find": "export const isStaff = a => Boolean(a?.roles.some(role => ['admin', 'staff', 'inventory'].includes(role)));", + "replace": "export const isStaff = a => Boolean(a);" + } + ] + }, + { + "id": "staff-can-assign-roles", + "scenario": "tracks/ecommerce/scenarios/progression-staff-roles.json", + "targets": [ + "ecommerce.spec.access-control.staff-role-boundary.621b", + "ecommerce.spec.access-control.staff-role-revocation.621d" + ], + "desc": "Staff can call the role assignment endpoint, including after administrator access is revoked.", + "file": "convex/progression.js", + "edits": [ + { + "find": "await operator(ctx, true);\n if (!['staff', 'inventory', 'admin'].includes(role))", + "replace": "await operator(ctx);\n if (!['staff', 'inventory', 'admin'].includes(role))" + } + ] + }, + { + "id": "stock-alerts-are-not-owner-scoped", + "scenario": "tracks/ecommerce/scenarios/progression-stock-alerts.json", + "targets": [ + "ecommerce.spec.access-control.stock-alert-privacy.631b" + ], + "desc": "Notification reads expose alerts from other accounts.", + "file": "convex/progression.js", + "edits": [ + { + "find": "(await ctx.db.query('notification').withIndex('account', q => q.eq('accountId', user._id)).collect()).map(publicRow)", + "replace": "(await rows(ctx, 'notification')).map(publicRow)" + } + ] + }, + { + "id": "staff-can-use-direct-restock", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "The direct restock route accepts staff instead of administrators only.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await operator(ctx, true); quantity(args.quantity);\n const stock", + "replace": "await operator(ctx); quantity(args.quantity);\n const stock" + } + ] + }, + { + "id": "concurrent-cart-add-does-not-increment", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203a" + ], + "desc": "Concurrent cart adds leave the line quantity and its reservation list unchanged, so the existing cart remains checkable at checkout.", + "file": "convex/shop.js", + "edits": [ + { + "find": "const next = add ? (existing?.quantity || 0) + amount : amount;", + "replace": "const next = add ? amount : amount;" + } + ] + }, + { + "id": "restock-race-records-wrong-order-total", + "scenario": "tracks/ecommerce/scenarios/01-restock-race.json", + "targets": [ + "ecommerce.spec.concurrency-safety.restock-race.202a" + ], + "desc": "Purchases preserve stock and visible order counts but record the wrong booked total. Native mixed-race reconciliation must reject them.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(orderId, { total: total / 100 });", + "replace": "await ctx.db.patch(orderId, { total: total / 100 + 1 });" + } + ] + }, + { + "id": "queue-ignores-live-fulfilment-updates", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-live.json", + "targets": [ + "ecommerce.spec.live-state.fulfilment-queue.1a" + ], + "desc": "The open staff queue ignores live fulfilment events, so a new order appears only after a reload.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setFulfilmentQueue(state.fulfilment || { orders: [], depth: 0 });", + "replace": " // Defect: the subscribed fulfilment view is ignored." + } + ] + }, + { + "id": "managed-support-live-refresh-keeps-stale-tickets", + "scenario": "tracks/ecommerce/scenarios/progression-managed-support-shared.json", + "targets": [ + "ecommerce.spec.live-state.managed-support.613a" + ], + "desc": "The native subscription keeps the first support ticket snapshot. Saved replies remain visible on initial load, but another open client does not receive updates.", + "file": "client/src/ProgressionPanel.tsx", + "edits": [ + { + "find": " useEffect(() => subscribeProgression(next => {\n setState({ ...next, loadedToken: token });", + "replace": " useEffect(() => subscribeProgression(next => {\n setState((current: any) => ({ ...next, tickets: current.loadedToken === token ? current.tickets : next.tickets, loadedToken: token }));" + } + ] + }, + { + "id": "open-review-list-ignores-live-update", + "scenario": "tracks/ecommerce/scenarios/progression-open-list-live.json", + "targets": [ + "ecommerce.spec.live-state.open-list.902a" + ], + "desc": "The native detail subscription retains its first review list while other item data stays current. New reviews appear on reload but not in the already-open view.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setItemDetails(state.details);", + "replace": " setItemDetails(previous => state.details.map((item: ItemDetailT) => ({ ...item, reviews: previous.find(old => old.id === item.id)?.reviews || item.reviews })));" + } + ] + }, + { + "id": "espresso-stock-row-ignores-live-updates", + "scenario": "tracks/ecommerce/scenarios/01-buying.json", + "targets": [ + "ecommerce.spec.live-state.purchase-stock.3b" + ], + "desc": "The live catalogue handler preserves a stale Espresso Machine stock projection while applying all other item updates.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setItems(state.items);", + "replace": " setItems(current => state.items.map((item: ItemT) => item.name === 'Espresso Machine' ? current.find(previous => previous.id === item.id) || item : item));" + } + ] + }, + { + "id": "purchase-counts-never-affect-ranking", + "scenario": "tracks/ecommerce/scenarios/01-core.json", + "targets": [ + "ecommerce.spec.live-state.ranking.2c" + ], + "desc": "The catalogue ranking ignores recorded purchases and therefore never promotes the bought item.", + "file": "convex/shop.js", + "edits": [ + { + "find": "purchaseCount: standing.filter(l => l.item_id === i._id).reduce((n, l) => n + l.quantity, 0)", + "replace": "purchaseCount: 0" + } + ] + }, + { + "id": "live-review-average-uses-an-extra-divisor", + "scenario": "tracks/ecommerce/scenarios/01-review-rating-live.json", + "targets": [ + "ecommerce.spec.live-state.rating.6c" + ], + "desc": "The live review event divides the rating sum by one more review than actually exists.", + "file": "convex/shop.js", + "edits": [ + { + "find": "/ itemReviews.length : 0", + "replace": "/ (itemReviews.length + 1) : 0" + } + ] + }, + { + "id": "live-admin-updates-keep-stale-category-totals", + "scenario": "tracks/ecommerce/scenarios/02-operational-category-totals.json", + "targets": [ + "ecommerce.spec.live-state.sales-dashboard.5b" + ], + "desc": "Live admin updates keep the category totals loaded at page load, so a purchase does not move units or revenue until a reload.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setAdminOverview(state.admin);", + "replace": " setAdminOverview(previous => state.admin && previous ? { ...state.admin, categories: previous.categories } : state.admin);" + } + ] + }, + { + "id": "shared-cart-live-events-ignored", + "scenario": "tracks/ecommerce/scenarios/01-cart.json", + "targets": [ + "ecommerce.spec.live-state.shared-cart.4c" + ], + "desc": "The cart accepts initial snapshots and updates after a local write, but an idle second session ignores remote cart changes.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const [cart, setCart] = useState({ items: [], total: 0 });", + "replace": " const [cart, setCart] = useState({ items: [], total: 0 });\n const cartSnapshotAccount = useRef(null);\n const cartWrittenHere = useRef(false);" + }, + { + "find": " setCart(state.cart);", + "replace": " const cartAccount = state.user?.id ?? \"anonymous\";\n if (cartSnapshotAccount.current !== cartAccount) cartWrittenHere.current = false;\n if (cartSnapshotAccount.current !== cartAccount || cartWrittenHere.current) setCart(state.cart);\n cartSnapshotAccount.current = cartAccount;" + }, + { + "find": " await mutate(\"api:add_to_cart\", { itemId, quantity: 1 });", + "replace": " cartWrittenHere.current = true;\n await mutate(\"api:add_to_cart\", { itemId, quantity: 1 });" + } + ] + }, + { + "id": "restock-does-not-increase-stock", + "scenario": "tracks/ecommerce/scenarios/01-warehouse-stock-live-staff.json", + "targets": [ + "ecommerce.spec.live-state.warehouse-stock.7c" + ], + "desc": "The restock endpoint accepts a positive quantity but increments authoritative stock by zero.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(stock._id, { quantity: stock.quantity + args.quantity });", + "replace": "await ctx.db.patch(stock._id, { quantity: stock.quantity });" + } + ] + }, + { + "id": "notification-preference-is-not-saved", + "scenario": "tracks/ecommerce/scenarios/progression-notification-preferences.json", + "targets": [ + "ecommerce.spec.state-durability.notification-preferences-reload.630a", + "ecommerce.spec.access-control.notification-preferences-privacy.630b" + ], + "desc": "Saved notification preferences remain disabled after reload. It also breaks the owner's saved-choice positive control in 630b; that coupled failure is not evidence of a privacy defect.", + "file": "convex/progression.js", + "edits": [ + { + "find": "await ctx.db.patch((await account(ctx))._id, { preference });", + "replace": "await ctx.db.patch((await account(ctx))._id, { preference: { order: false, stock: false } });" + } + ] + }, + { + "id": "role-assignment-drops-role", + "scenario": "tracks/ecommerce/scenarios/progression-staff-roles.json", + "targets": [ + "ecommerce.spec.state-durability.staff-role-reload.621a", + "ecommerce.spec.access-control.staff-role-boundary.621b", + "ecommerce.spec.access-control.staff-role-revocation.621d" + ], + "desc": "Inventory assignments are not saved, including the authorized baselines for access-control checks; staff and administrator role transitions still work.", + "file": "convex/progression.js", + "edits": [ + { + "find": "await ctx.db.patch(accountId, { roles: [role] });", + "replace": "await ctx.db.patch(accountId, { roles: [role === 'inventory' ? 'staff' : role] });" + } + ] + }, + { + "id": "revenue-aggregation-ignores-order-totals", + "scenario": "tracks/ecommerce/scenarios/progression-books-balance.json", + "targets": [ + "ecommerce.spec.transactional-integrity.books-balance.107a" + ], + "desc": "The admin revenue aggregation counts every order as zero regardless of its stored total.", + "file": "convex/shop.js", + "edits": [ + { + "find": "revenue: money(headers.reduce((n, o) => n + o.total - o.refunded, 0))", + "replace": "revenue: 0" + } + ] + }, + { + "id": "purchase-does-not-reduce-warehouse-stock", + "scenario": "tracks/ecommerce/scenarios/progression-books-balance.json", + "targets": [ + "ecommerce.spec.transactional-integrity.books-balance.107b" + ], + "desc": "Purchases create orders without reducing warehouse stock.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(row._id, { quantity: row.quantity - take });", + "replace": "await ctx.db.patch(row._id, { quantity: row.quantity });" + } + ] + }, + { + "id": "direct-purchase-total-ignores-store-price", + "scenario": "tracks/ecommerce/scenarios/01-server-price.json", + "targets": [ + "ecommerce.spec.transactional-integrity.server-price.104a" + ], + "desc": "The direct purchase creates one order but records a zero total rather than the store's current price.", + "file": "convex/shop.js", + "edits": [ + { + "find": "total += Math.round(item.price * 100) * line.quantity;", + "replace": "total += 100 * line.quantity;" + } + ] + }, + { + "id": "overdraw-transfer-is-accepted", + "scenario": "tracks/ecommerce/scenarios/02-transfer-overdraw.json", + "targets": [ + "ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c" + ], + "desc": "The atomic source debit no longer requires sufficient quantity, so an overdrawn transfer succeeds and moves both warehouse totals.", + "file": "convex/shop.js", + "edits": [ + { + "find": "if (source.quantity < args.quantity) throw new ConvexError('Not enough stock');", + "replace": "// Defect: transfer may debit below zero." + } + ] + }, + { + "id": "recommendations-ignore-pending-purchases", + "scenario": "tracks/ecommerce/scenarios/02-operational-recommendations.json", + "targets": [ + "ecommerce.inventory-operations.operational-views.5c" + ], + "desc": "Never use customer purchases to choose recommended categories.", + "file": "convex/shop.js", + "edits": [ + { + "find": "const purchasedCategories = new Set(standing.filter(l => ownOrderIds.has(l.order_id)).map(l => itemById.get(l.item_id)?.category));", + "replace": "const purchasedCategories = new Set();" + } + ] + }, + { + "id": "cart-add-line-lookup-not-owner-scoped", + "scenario": "tracks/ecommerce/scenarios/01-cart-boundary.json", + "targets": [ + "ecommerce.spec.access-control.cart-boundary.109a" + ], + "desc": "Cart line lookup ignores account ownership, so a second customer changes the first customer’s item quantity.", + "file": "convex/shop.js", + "edits": [ + { + "find": "const existing = await ctx.db.query('order_cart').withIndex('line', q => q.eq('account_id', user._id).eq('item_id', itemId)).unique();", + "replace": "const existing = (await ctx.db.query('order_cart').collect()).find(row => row.item_id === itemId);" + } + ] + }, + { + "id": "direct-purchase-is-attributed-to-another-account", + "scenario": "tracks/ecommerce/scenarios/01-purchase-attribution.json", + "targets": [ + "ecommerce.spec.access-control.purchase-attribution.102a" + ], + "desc": "Purchase order is attributed to another account despite the real caller session.", + "file": "convex/shop.js", + "edits": [ + { + "find": "account_id: user._id, total: 0, refunded: 0, status: 'pending'", + "replace": "account_id: (await rows(ctx, 'order_account')).find(a => a._id !== user._id)._id, total: 0, refunded: 0, status: 'pending'" + } + ] + }, + { + "id": "unauthenticated-purchase-defaults-to-admin", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Anonymous native buy defaults to the administrator account and commits a purchase.", + "file": "convex/shop.js", + "edits": [ + { + "find": "purchase(ctx, await account(ctx), [{ item_id: itemId, quantity: 1 }])", + "replace": "purchase(ctx, (await account(ctx, false)) || (await rows(ctx, 'order_account')).find(isAdmin), [{ item_id: itemId, quantity: 1 }])" + } + ] + }, + { + "id": "signed-out-visitor-purchase-is-accepted", + "scenario": "tracks/ecommerce/scenarios/progression-signed-out-purchase.json", + "targets": [ + "ecommerce.spec.access-control.signed-out-purchase.3a" + ], + "desc": "Expose the purchase control while signed out and accept its anonymous buy as an administrator-owned purchase.", + "file": "convex/shop.js", + "edits": [ + { + "find": "purchase(ctx, await account(ctx), [{ item_id: itemId, quantity: 1 }])", + "replace": "purchase(ctx, (await account(ctx, false)) || (await rows(ctx, 'order_account')).find(isAdmin), [{ item_id: itemId, quantity: 1 }])" + }, + { + "file": "client/src/App.tsx", + "find": "const isCustomer = !!currentUser && !currentUser.isAdmin && !currentUser.isStaff;", + "replace": "const isCustomer = !currentUser || (!currentUser.isAdmin && !currentUser.isStaff);" + } + ] + }, + { + "id": "staff-can-see-admin-navigation", + "scenario": "tracks/ecommerce/scenarios/01-warehouse-admin-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-area-boundary.7a" + ], + "desc": "Staff are incorrectly treated as administrators in reads, navigation, and writes.", + "file": "convex/accounts.js", + "edits": [ + { + "find": "export const isAdmin = a => Boolean(a?.roles.includes('admin'));", + "replace": "export const isAdmin = a => Boolean(a?.roles.some(role => ['admin', 'staff'].includes(role)));" + } + ] + }, + { + "id": "reconnect-hydration-loses-account-state", + "scenario": "tracks/ecommerce/scenarios/progression-account-state-reconnect.json", + "targets": [ + "ecommerce.spec.state-durability.account-state-recovery.105b" + ], + "desc": "Network loss permanently closes the live client, so the account misses peer cart changes after reconnect.", + "file": "client/src/request.ts", + "edits": [ + { + "find": "followAuth();\nexport async function signOut()", + "replace": "followAuth();\nwindow.addEventListener('offline', () => { live.close(); }, { once: true });\nexport async function signOut()" + } + ] + }, + { + "id": "profile-data-is-lost-on-server-restart", + "scenario": "tracks/ecommerce/scenarios/progression-customer-profile.json", + "targets": [ + "ecommerce.spec.state-durability.customer-profile-reload.620a" + ], + "desc": "Signing in from the fresh recovery browser resets a stored profile; the initial save and token reload remain valid.", + "file": "convex/accounts.js", + "edits": [ + { + "find": "if (await ctx.db.query('order_account').withIndex('auth_user', q => q.eq('auth_user_id', userId)).unique()) return;", + "replace": "const existing = await ctx.db.query('order_account').withIndex('auth_user', q => q.eq('auth_user_id', userId)).unique();\n if (existing) { await ctx.db.patch(existing._id, { profile: { name: '', address: '' } }); return; }" + } + ] + }, + { + "id": "stock-alert-repeats-while-in-stock", + "scenario": "tracks/ecommerce/scenarios/progression-stock-alerts.json", + "targets": [ + "ecommerce.spec.transactional-integrity.stock-alert-deduplication.631a" + ], + "desc": "One restock delivers two independent notifications for the same pending stock alert.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.insert('notification', { accountId: alert.accountId, type: 'stock', message: `${item.name} is back in stock` });", + "replace": "await ctx.db.insert('notification', { accountId: alert.accountId, type: 'stock', message: `${item.name} is back in stock` });\n await ctx.db.insert('notification', { accountId: alert.accountId, type: 'stock', message: `${item.name} is back in stock` });" + } + ] + }, + { + "id": "last-unit-allows-negative-stock", + "scenario": "tracks/ecommerce/scenarios/01-last-unit.json", + "targets": [ + "ecommerce.spec.concurrency-safety.last-unit.201a", + "ecommerce.spec.concurrency-safety.last-unit.201b", + "ecommerce.spec.concurrency-safety.last-unit.201c" + ], + "desc": "Allocation permits an oversell and debits the first warehouse below zero.", + "file": "convex/shop.js", + "edits": [ + { + "find": "if (stocks.reduce((sum, row) => sum + row.quantity, 0) < count) throw new ConvexError('Not enough stock');", + "replace": "// Defect: insufficient stock is accepted." + }, + { + "find": "const take = Math.min(row.quantity, count);", + "replace": "const take = count;" + } + ] + }, + { + "id": "scheduled-restock-never-becomes-due-after-restart", + "scenario": "tracks/ecommerce/scenarios/03-server-time.json", + "targets": [ + "ecommerce.l3.server-time.server-time.312a" + ], + "desc": "A reconnect handler mistakenly cancels pending deferred work; ordinary scheduling before a reconnect still completes.", + "file": "client/src/request.ts", + "edits": [ + { + "find": "followAuth();\nexport async function signOut()", + "replace": "followAuth();\nlet lastRecoveryConnection = 1;\nlive.subscribeToConnectionState(({ isWebSocketConnected, connectionCount }) => {\n if (!isWebSocketConnected || connectionCount <= lastRecoveryConnection) return;\n lastRecoveryConnection = connectionCount;\n void (async () => {\n const token = await accessToken();\n if (!token) return;\n http.setAuth(token);\n const current = await http.query(makeFunctionReference<'query'>('progression:state'), {});\n if (current.user?.isAdmin) for (const task of current.scheduledRestocks) await mutate('api:cancel_scheduled_restock', { restockId: task.id });\n })().catch(() => undefined);\n});\nexport async function signOut()" + } + ] + }, + { + "id": "checkout-crash-integrity", + "scenario": "tracks/ecommerce/scenarios/progression-checkout-crash.json", + "targets": [ + "ecommerce.spec.state-durability.checkout-crash-integrity.910a" + ], + "desc": "A disconnect-triggered recovery worker restores a wrong cart quantity through the native write path, producing neither the prepared cart nor a completed checkout. The worker retries native calls while the backend is down; it does not wait for the browser WebSocket reconnect backoff.", + "file": "client/src/request.ts", + "edits": [ + { + "find": "followAuth();\nexport async function signOut()", + "replace": "followAuth();\nlet recoveryStarted = false;\nlive.subscribeToConnectionState(({ isWebSocketConnected, connectionCount }) => {\n if (isWebSocketConnected || !connectionCount || recoveryStarted) return;\n recoveryStarted = true;\n void (async () => {\n const token = await accessToken();\n if (!token) return;\n http.setAuth(token);\n const deadline = Date.now() + 10000;\n while (true) {\n try {\n const current = await state();\n const line = current.cart.items[0] || current.orders[0]?.items[0];\n if (!line) throw new Error('Crash control has no cart or prior order');\n await mutate('api:update_cart_quantity', { itemId: line.itemId, quantity: 2 });\n console.error('Crash control committed', Date.now());\n return;\n } catch (error) {\n if (Date.now() >= deadline) throw error;\n await new Promise(resolve => setTimeout(resolve, 25));\n }\n }\n })().catch(error => console.error('Crash control failed', String(error)));\n});\nexport async function signOut()" + } + ] + }, + { + "id": "checkout-crash-sibling-stock", + "scenario": "tracks/ecommerce/scenarios/progression-checkout-crash.json", + "targets": [ + "ecommerce.spec.state-durability.checkout-crash-integrity.910a" + ], + "desc": "A disconnect-triggered recovery worker removes stock for Coffee Grinder while leaving cart and order data intact. The second-product stock discrepancy must fail crash atomicity. The worker retries native calls while the backend is down; it does not wait for the browser WebSocket reconnect backoff.", + "file": "client/src/request.ts", + "edits": [ + { + "find": "followAuth();\nexport async function signOut()", + "replace": "followAuth();\nlet recoveryStarted = false;\nlive.subscribeToConnectionState(({ isWebSocketConnected, connectionCount }) => {\n if (isWebSocketConnected || !connectionCount || recoveryStarted) return;\n recoveryStarted = true;\n void (async () => {\n const token = await accessToken();\n if (!token) return;\n http.setAuth(token);\n const deadline = Date.now() + 10000;\n while (true) {\n try {\n const current = await state();\n const priorOrder = current.orders.find((order: any) => order.status === 'pending');\n if (!priorOrder) throw new Error('Crash control has no pending order');\n await mutate('api:cancel_order', { orderId: priorOrder.id });\n console.error('Crash control committed', Date.now());\n return;\n } catch (error) {\n if (Date.now() >= deadline) throw error;\n await new Promise(resolve => setTimeout(resolve, 25));\n }\n }\n })().catch(error => console.error('Crash control failed', String(error)));\n});\nexport async function signOut()" + }, + { + "file": "convex/shop.js", + "find": " for (const line of await ctx.db.query('order_line').withIndex('order', q => q.eq('order_id', orderId)).collect()) await restoreLine(ctx, line);\n await ctx.db.patch(orderId, { status: 'cancelled', refunded: order.total });", + "replace": " for (const line of await ctx.db.query('order_line').withIndex('order', q => q.eq('order_id', orderId)).collect()) {\n if ((await ctx.db.get(line.item_id))?.name !== 'Coffee Grinder') continue;\n const stock = await ctx.db.query('stock').withIndex('item', q => q.eq('item_id', line.item_id)).first();\n if (stock) await ctx.db.patch(stock._id, { quantity: stock.quantity - 1 });\n }" + } + ] + }, + { + "id": "checkout-crash-durability", + "scenario": "tracks/ecommerce/scenarios/progression-checkout-crash.json", + "targets": [ + "ecommerce.spec.state-durability.checkout-crash-durability.910b" + ], + "desc": "A disconnect-triggered recovery worker corrupts the total of an earlier acknowledged order through its native write path, without changing stock, allocations or cart state. The worker retries native calls while the backend is down; it does not wait for the browser WebSocket reconnect backoff.", + "file": "client/src/request.ts", + "edits": [ + { + "find": "followAuth();\nexport async function signOut()", + "replace": "followAuth();\nlet recoveryStarted = false;\nlive.subscribeToConnectionState(({ isWebSocketConnected, connectionCount }) => {\n if (isWebSocketConnected || !connectionCount || recoveryStarted) return;\n recoveryStarted = true;\n void (async () => {\n const token = await accessToken();\n if (!token) return;\n http.setAuth(token);\n const deadline = Date.now() + 10000;\n while (true) {\n try {\n const current = await state();\n const priorOrder = current.orders.find((order: any) => order.status === 'pending');\n if (!priorOrder) throw new Error('Crash control has no pending order');\n await mutate('api:cancel_order', { orderId: priorOrder.id });\n console.error('Crash control committed', Date.now());\n return;\n } catch (error) {\n if (Date.now() >= deadline) throw error;\n await new Promise(resolve => setTimeout(resolve, 25));\n }\n }\n })().catch(error => console.error('Crash control failed', String(error)));\n});\nexport async function signOut()" + }, + { + "file": "convex/shop.js", + "find": " for (const line of await ctx.db.query('order_line').withIndex('order', q => q.eq('order_id', orderId)).collect()) await restoreLine(ctx, line);\n await ctx.db.patch(orderId, { status: 'cancelled', refunded: order.total });", + "replace": " await ctx.db.patch(orderId, { total: 0 });" + } + ] + }, + { + "id": "restock-trusts-submitted-admin-role", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Trust an administrator role supplied by staff in the restock request. Normal staff requests are still refused.", + "file": "convex/shop.js", + "edits": [ + { + "find": "export const restock = mutationGeneric({ args: { itemId: v.id('item'), warehouseId: v.id('warehouse'), quantity: v.number() },\n handler: async (ctx, args) => {\n await operator(ctx, true); quantity(args.quantity);", + "replace": "export const restock = mutationGeneric({ args: { itemId: v.id('item'), warehouseId: v.id('warehouse'), quantity: v.number(), role: v.optional(v.string()) },\n handler: async (ctx, args) => {\n await operator(ctx, args.role !== 'admin'); quantity(args.quantity);" + } + ] + }, + { + "id": "direct-purchase-trusts-submitted-price", + "scenario": "tracks/ecommerce/scenarios/01-server-price.json", + "targets": [ + "ecommerce.spec.transactional-integrity.server-price.104a" + ], + "desc": "Ordinary purchases use the catalog price; an extra price field overrides the booked order and line price.", + "file": "convex/shop.js", + "edits": [ + { + "find": "export const buy = mutationGeneric({ args: { itemId: v.id('item') }, handler: async (ctx, { itemId }) =>\n purchase(ctx, await account(ctx), [{ item_id: itemId, quantity: 1 }]) });", + "replace": "export const buy = mutationGeneric({ args: { itemId: v.id('item'), price: v.optional(v.number()) }, handler: async (ctx, { itemId, price }) =>\n purchase(ctx, await account(ctx), [{ item_id: itemId, quantity: 1, price }]) });" + }, + { + "find": " if (!item) throw new ConvexError('Unknown item');\n const lineId", + "replace": " if (!item) throw new ConvexError('Unknown item');\n if (line.price !== undefined) item.price = line.price;\n const lineId" + } + ] + }, + { + "id": "order-cache-survives-account-switch", + "scenario": "tracks/ecommerce/scenarios/01-order-ownership.json", + "targets": [ + "ecommerce.spec.access-control.order-ownership.106a" + ], + "desc": "Merge order updates into a browser cache that is never cleared when the account changes. Separate-browser ownership still works.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const [orders, setOrders] = useState([]);", + "replace": " const [orders, setStoredOrders] = useState([]);\n const setOrders: typeof setStoredOrders = update => setStoredOrders(previous => {\n const incoming = typeof update === 'function' ? update(previous) : update;\n return [...new Map([...previous, ...incoming].map(order => [order.id, order])).values()];\n });" + } + ] + }, + { + "id": "checkout-trusts-submitted-price", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "Accept an optional price field and overwrite the checkout total with it.", + "file": "convex/shop.js", + "edits": [ + { + "find": "export const checkout = mutationGeneric({ args: {}, handler: async ctx => {", + "replace": "export const checkout = mutationGeneric({ args: { price: v.optional(v.number()) }, handler: async (ctx, args) => {" + }, + { + "find": " const result = await purchase(ctx, user, lines);", + "replace": " const result = await purchase(ctx, user, lines);\n if (args.price !== undefined) await ctx.db.patch(result.orderId, { total: args.price });" + } + ] + }, + { + "id": "checkout-missing-sibling-line", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "Commit the cart checkout with one product line missing while retaining the other product and the full order total.", + "file": "convex/shop.js", + "edits": [ + { + "find": " const result = await purchase(ctx, user, lines);", + "replace": " const result = await purchase(ctx, user, lines);\n for (const line of await rows(ctx, 'order_line')) {\n if (line.order_id === result.orderId && (await ctx.db.get(line.item_id)).name === 'Coffee Grinder') await ctx.db.delete(line._id);\n }" + } + ] + }, + { + "id": "shipping-cancellation-cancel-shipped", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Cancellation accepts a shipped order and restores its stock after shipping.", + "file": "convex/shop.js", + "edits": [ + { + "find": "if (order.status !== 'pending')", + "replace": "if (false)" + } + ] + }, + { + "id": "shipping-cancellation-resurrect-cancelled", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Shipping accepts a cancelled order after its stock was restored.", + "file": "convex/shop.js", + "edits": [ + { + "find": "if (!order || order.status !== 'pending')", + "replace": "if (!order)" + } + ] + }, + { + "id": "shipping-cancellation-combined-race", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Cancellation restores stock but records a shipped status, combining mutually exclusive effects.", + "file": "convex/shop.js", + "edits": [ + { + "find": "if (order.status !== 'pending')", + "replace": "if (false)" + }, + { + "find": "await ctx.db.patch(orderId, { status: 'cancelled', refunded: order.total });", + "replace": "await ctx.db.patch(orderId, { status: 'shipped', refunded: order.total });" + } + ] + }, + { + "id": "shipping-cancellation-reject-both", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Both cancellation and shipping refuse pending orders; useful progress must remain required.", + "file": "convex/shop.js", + "edits": [ + { + "find": "if (order.status !== 'pending')", + "replace": "if (true)" + }, + { + "find": "if (!order || order.status !== 'pending')", + "replace": "if (true)" + } + ] + }, + { + "id": "shipping-cancellation-queue-retains-removed-order", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "The live staff queue accepts new orders but retains rows after cancellation; the database state remains correct.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setFulfilmentQueue(state.fulfilment || { orders: [], depth: 0 });", + "replace": " setFulfilmentQueue(previous => { const next = state.fulfilment || { orders: [], depth: 0 }; return next.orders.length < previous.orders.length ? previous : next; });" + } + ] + }, + { + "id": "opposing-transfer-ignored", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202d" + ], + "desc": "One valid opposing transfer is acknowledged without moving stock.", + "file": "convex/shop.js", + "edits": [ + { + "find": " if (args.fromWarehouseId === args.toWarehouseId) throw new ConvexError('Choose different warehouses');", + "replace": " if (args.fromWarehouseId === args.toWarehouseId) throw new ConvexError('Choose different warehouses');\n if (args.quantity === 7) return;" + } + ] + }, + { + "id": "opposing-transfer-rejected", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202d" + ], + "desc": "One valid opposing transfer is refused despite ample source stock.", + "file": "convex/shop.js", + "edits": [ + { + "find": " if (args.fromWarehouseId === args.toWarehouseId) throw new ConvexError('Choose different warehouses');", + "replace": " if (args.fromWarehouseId === args.toWarehouseId) throw new ConvexError('Choose different warehouses');\n if (args.quantity === 7) throw new ConvexError('Transfer disabled');" + } + ] + }, + { + "id": "overlapping-cart-add-ignored", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "The overlapping second-product cart add is ignored while ordinary cart setup still works.", + "file": "convex/shop.js", + "edits": [ + { + "find": " const next = add ? (existing?.quantity || 0) + amount : amount;", + "replace": " if (add && user.username.includes('checkout-overlap') && (await ctx.db.get(itemId)).name === 'Coffee Grinder') return;\n const next = add ? (existing?.quantity || 0) + amount : amount;" + } + ] + }, + { + "id": "overlapping-cart-add-rejected", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "The overlapping second-product cart add is rejected while ordinary cart setup still works.", + "file": "convex/shop.js", + "edits": [ + { + "find": " const next = add ? (existing?.quantity || 0) + amount : amount;", + "replace": " if (add && user.username.includes('checkout-overlap') && (await ctx.db.get(itemId)).name === 'Coffee Grinder') throw new ConvexError('Cart add disabled');\n const next = add ? (existing?.quantity || 0) + amount : amount;" + } + ] + }, + { + "id": "mixed-history-lost-transfer", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: lost-transfer. The stored-state comparison must catch the defect at its first effect.", + "file": "convex/shop.js", + "edits": [ + { + "find": "await ctx.db.patch(target._id, { quantity: target.quantity + args.quantity });", + "replace": "await ctx.db.patch(target._id, { quantity: target.quantity });" + } + ] + }, + { + "id": "mixed-history-wrong-price", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: wrong-price. The stored-state comparison must catch the defect at its first effect.", + "file": "convex/shop.js", + "edits": [ + { + "find": "total += Math.round(item.price * 100) * line.quantity;", + "replace": "total += 100 * line.quantity;" + } + ] + }, + { + "id": "mixed-history-ignored-write", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: ignored-write. The stored-state comparison must catch the defect at its first effect.", + "file": "convex/shop.js", + "edits": [ + { + "find": " const next = add ? (existing?.quantity || 0) + amount : amount;", + "replace": " if (add && user.username.includes('history-') && (await ctx.db.get(itemId)).name === 'Coffee Grinder') return;\n const next = add ? (existing?.quantity || 0) + amount : amount;" + } + ] + }, + { + "id": "mixed-history-reject-valid-write", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: reject-valid-write. The stored-state comparison must catch the defect at its first effect.", + "file": "convex/shop.js", + "edits": [ + { + "find": " const next = add ? (existing?.quantity || 0) + amount : amount;", + "replace": " if (add && user.username.includes('history-') && (await ctx.db.get(itemId)).name === 'Coffee Grinder') throw new ConvexError('Cart add disabled');\n const next = add ? (existing?.quantity || 0) + amount : amount;" + } + ] + }, + { + "id": "mixed-history-repeat-cancel", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: repeat-cancel. The stored-state comparison must catch the defect at its first effect.", + "file": "convex/shop.js", + "edits": [ + { + "find": " if (order.status !== 'pending') throw new ConvexError('Only pending orders can be cancelled');", + "replace": " if (!['pending', 'cancelled'].includes(order.status)) throw new ConvexError('Only pending orders can be cancelled');" + } + ] + }, + { + "id": "mixed-history-wrong-owner", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: wrong-owner. The stored-state comparison must catch the defect at its first effect.", + "file": "convex/shop.js", + "edits": [ + { + "find": "account_id: user._id, total: 0, refunded: 0, status: 'pending'", + "replace": "account_id: (await rows(ctx, 'order_account')).find(a => a._id !== user._id)._id, total: 0, refunded: 0, status: 'pending'" + } + ] + }, + { + "id": "checkout-empty-replay-creates-order", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Allow retrying the consumed empty cart to create another order after a lost reply.", + "file": "convex/shop.js", + "edits": [ + { + "find": " if (!lines.length) throw new ConvexError('Cart is empty');", + "replace": "// Defect: an empty consumed cart may create another order." + } + ] + }, + { + "id": "checkout-replay-loses-prior-order", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Retrying the consumed cart deletes an earlier acknowledged order.", + "file": "convex/shop.js", + "edits": [ + { + "find": " if (!lines.length) throw new ConvexError('Cart is empty');", + "replace": " if (!lines.length) { const prior = (await rows(ctx, 'order_header')).find(order => order.account_id === user._id); if (prior) await ctx.db.delete(prior._id); return { orderId: null }; }" + } + ] + }, + { + "id": "checkout-clears-cart-without-order", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Checkout returns success and empties the cart without recording an order.", + "file": "convex/shop.js", + "edits": [ + { + "find": " const result = await purchase(ctx, user, lines);", + "replace": " const result = { orderId: null };" + } + ] + }, + { + "id": "revenue-aggregation-truncates-order-population", + "scenario": "tracks/ecommerce/scenarios/progression-books-balance.json", + "targets": [ + "ecommerce.spec.transactional-integrity.books-balance.107a" + ], + "desc": "Calculate revenue from only the first 256 stored orders while preserving all acknowledged purchases.", + "file": "convex/shop.js", + "edits": [ + { + "find": "revenue: money(headers.reduce((n, o) => n + o.total - o.refunded, 0))", + "replace": "revenue: money(headers.slice(0, 256).reduce((n, o) => n + o.total - o.refunded, 0))" + } + ] + }, + { + "id": "catalog-truncates-created-population", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-volume.json", + "targets": [ + "ecommerce.spec.transactional-integrity.catalog-volume.622c" + ], + "desc": "Omit committed products after the first 256 search results.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": ".slice(searchPage * CATALOG_PAGE_SIZE,", + "replace": ".slice(0, 256).slice(searchPage * CATALOG_PAGE_SIZE," + } + ] + }, + { + "id": "catalog-discards-product-writes", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-volume.json", + "targets": [ + "ecommerce.spec.transactional-integrity.catalog-volume.622c" + ], + "desc": "Acknowledge catalog creation without committing a product.", + "file": "convex/progression.js", + "edits": [ + { + "find": " if (!args.name.trim() || !args.category.trim() || !Number.isFinite(args.price) || args.price < 0) throw new ConvexError('Invalid product');", + "replace": " if (args.name.length >= 0) return; // mutant: successful mutation without product\n if (!args.name.trim() || !args.category.trim() || !Number.isFinite(args.price) || args.price < 0) throw new ConvexError('Invalid product');" + } + ] + }, + { + "id": "catalog-rejects-all-product-writes", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-volume.json", + "targets": [ + "ecommerce.spec.transactional-integrity.catalog-volume.622c" + ], + "desc": "Reject every catalog creation while keeping sign-in and catalog reads available.", + "file": "convex/progression.js", + "edits": [ + { + "find": " if (!args.name.trim() || !args.category.trim() || !Number.isFinite(args.price) || args.price < 0) throw new ConvexError('Invalid product');", + "replace": " throw new ConvexError('Catalog writes refused');\n if (!args.name.trim() || !args.category.trim() || !Number.isFinite(args.price) || args.price < 0) throw new ConvexError('Invalid product');" + } + ] + }, + { + "id": "shipping-trusts-submitted-staff-role", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "Trust a staff role supplied by a customer; ordinary customers remain refused.", + "file": "convex/shop.js", + "edits": [ + { + "find": "export const ship = mutationGeneric({ args: { orderId: v.id('order_header') }, handler: async (ctx, { orderId }) => {\n const user = await operator(ctx);", + "replace": "export const ship = mutationGeneric({ args: { orderId: v.id('order_header'), role: v.optional(v.string()) }, handler: async (ctx, { orderId, role }) => {\n const user = role === 'staff' ? await account(ctx) : await operator(ctx);" + } + ] + }, + { + "id": "shipping-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "Reject legitimate staff shipping as well as customer requests.", + "file": "convex/shop.js", + "edits": [ + { + "find": "export const ship = mutationGeneric({ args: { orderId: v.id('order_header') }, handler: async (ctx, { orderId }) => {\n const user = await operator(ctx);", + "replace": "export const ship = mutationGeneric({ args: { orderId: v.id('order_header') }, handler: async (ctx, { orderId }) => {\n throw new ConvexError('Shipping refused');\n const user = await operator(ctx);" + } + ] + }, + { + "id": "cancel-trusts-submitted-owner", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.order-owner.204a" + ], + "desc": "Trust a username submitted by another signed-in customer as the order owner.", + "file": "convex/shop.js", + "edits": [ + { + "find": "export const cancel = mutationGeneric({ args: { orderId: v.id('order_header') }, handler: async (ctx, { orderId }) => {\n const order = await ownedOrder(ctx, orderId);", + "replace": "export const cancel = mutationGeneric({ args: { orderId: v.id('order_header'), username: v.optional(v.string()) }, handler: async (ctx, { orderId, username }) => {\n const user = await account(ctx);\n const claimed = username ? await ctx.db.query('order_account').withIndex('username', q => q.eq('username', username)).unique() : null;\n const order = await ctx.db.get(orderId);\n if (!order || order.account_id !== (claimed?._id ?? user._id)) throw new ConvexError('Order not found');" + } + ] + }, + { + "id": "cancel-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.order-owner.204a" + ], + "desc": "Reject legitimate cancellation as well as foreign requests.", + "file": "convex/shop.js", + "edits": [ + { + "find": "export const cancel = mutationGeneric({ args: { orderId: v.id('order_header') }, handler: async (ctx, { orderId }) => {\n const order = await ownedOrder(ctx, orderId);", + "replace": "export const cancel = mutationGeneric({ args: { orderId: v.id('order_header') }, handler: async (ctx, { orderId }) => {\n const order = await ownedOrder(ctx, orderId);\n throw new ConvexError('Cancellation refused');" + } + ] + }, + { + "id": "account-password-in-public-data", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Expose the actual submitted account password through data available to other clients.", + "edits": [ + { + "file": "convex/auth.js", + "find": "profile: { name: params.username, registrationNonce },", + "replace": "profile: { name: params.username, registrationNonce, image: params.password }," + }, + { + "file": "convex/accounts.js", + "find": " const user = await ctx.db.get(userId);", + "replace": " const user = await ctx.db.get(userId);\n const publicItem = await ctx.db.query('item').first();\n if (publicItem && user.image) await ctx.db.patch(publicItem._id, { name: publicItem.name + ' ' + user.image });" + } + ] + }, + { + "edits": [ + { + "file": "convex/auth.js", + "find": "verifySecret: (password, hash) => new Scrypt().verify(hash, password),", + "replace": "verifySecret: (password, hash) => password === \"' OR '1'='1\" || new Scrypt().verify(hash, password)," + } + ], + "id": "query-like-password-bypasses-signin", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Interpret the query-like password as a login bypass while retaining normal password verification." + }, + { + "edits": [ + { + "file": "convex/auth.js", + "find": "verifySecret: (password, hash) => new Scrypt().verify(hash, password),", + "replace": "verifySecret: (_password, _hash) => false," + } + ], + "id": "account-disclosure-rejects-login", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Reject every valid login; absent disclosures alone must not earn a pass." + }, + { + "id": "signout-keeps-session", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Show a signed-out interface while retaining the browser credential and its purchase authority.", + "edits": [ + { + "file": "client/src/App.tsx", + "find": " const clearSession = () => {\n localStorage.removeItem(TOKEN_KEY);", + "replace": " const clearSession = () => {\n // mutant: retain the current request token after signout" + }, + { + "file": "client/src/App.tsx", + "find": " await signOut();\n clearSession();", + "replace": " clearSession(); // mutant: hide the account without removing server authority" + }, + { + "file": "client/src/App.tsx", + "find": " {currentUser ? (", + "replace": " {currentUser && token ? (" + } + ] + }, + { + "edits": [ + { + "file": "convex/auth.js", + "find": "verifySecret: (password, hash) => new Scrypt().verify(hash, password),", + "replace": "verifySecret: (_password, _hash) => false," + } + ], + "id": "purchase-session-rejects-returning-login", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Refuse every returning login; blocking all access after logout must not pass the session check." + }, + { + "id": "account-password-in-script", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Ship the account password sentinel in the public JavaScript bundle while normal signup and login still work.", + "edits": [ + { + "file": "client/src/main.tsx", + "find": "import React from \"react\";", + "replace": "import React from \"react\";\nObject.assign(window, { debugPassword: 'sb-private-pw-76dc492ea153' });" + } + ] + }, + { + "id": "account-password-in-style", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Ship the account password sentinel in a public stylesheet while normal signup and login still work.", + "edits": [ + { + "file": "client/src/index.css", + "find": ":root {", + "replace": ":root {\n --debug-password: \"sb-private-pw-76dc492ea153\";" + } + ] + }, + { + "id": "signup-accepts-admin-claim", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Trust a submitted signup administrator claim and grant protected stock-write authority.", + "edits": [ + { + "file": "convex/schema.js", + "find": "registrationNonce: v.optional(v.string()),", + "replace": "registrationNonce: v.optional(v.string()), isAdmin: v.optional(v.boolean())," + }, + { + "file": "convex/auth.js", + "find": "profile: { name: params.username, registrationNonce },", + "replace": "profile: { name: params.username, registrationNonce, isAdmin: params.role === 'admin' }," + }, + { + "file": "convex/accounts.js", + "find": "username: user.name, roles: [],", + "replace": "username: user.name, roles: user.isAdmin ? ['admin'] : []," + } + ] + }, + { + "id": "admin-boundary-rejects-all-signups", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Keep fixture provisioning available, but refuse every new customer registration; blocking useful signup must not pass the authority check.", + "edits": [ + { + "file": "convex/auth.js", + "find": "if (params.flow === 'signUp') {", + "replace": "if (params.flow === 'signUp') {\n if (!['admin', 'staff', 'staff2', 'customer'].includes(params.username)) throw new ConvexError('Registration disabled');" + } + ] + }, + { + "id": "stock-alert-misses-direct-stock-writes", + "scenario": "tracks/ecommerce/scenarios/progression-stock-alert-direct-restore.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock-alert.631d" + ], + "desc": "Stock alerts are delivered only by the restock mutations, so stock written directly to the database never delivers its alert.", + "file": "convex/crons.js", + "edits": [ + { + "find": "crons.interval('deliver restored stock alerts', { seconds: 2 }, makeFunctionReference('progression:deliverRestoredAlerts'));\n", + "replace": "// Stock alerts are delivered by the restock mutations.\n" + } + ] + } + ] +} diff --git a/tools/stack-bench/grader/mutations/mongodb-ecommerce.json b/tools/stack-bench/grader/mutations/mongodb-ecommerce.json new file mode 100644 index 00000000000..bd5895df281 --- /dev/null +++ b/tools/stack-bench/grader/mutations/mongodb-ecommerce.json @@ -0,0 +1,3812 @@ +{ + "schemaVersion": 3, + "fixtureSha256": "8a167a23e422534dcf58d664edcd0506ec454c5880b54e170da391c600dfe6c2", + "backend": "mongodb", + "track": "ecommerce", + "note": "Mutation definitions for the MongoDB ecommerce reference.", + "mutations": [ + { + "id": "recommendation-dismissal-lost-on-restart", + "scenario": "tracks/ecommerce/scenarios/progression-recommendation-feedback.json", + "targets": [ + "ecommerce.spec.state-durability.recommendation-feedback-restart.504c" + ], + "desc": "Erase saved recommendation dismissals when the application starts again.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await mongoose.connect(DATABASE_URL);", + "replace": " await mongoose.connect(DATABASE_URL);\n await Dismissal.deleteMany({});" + } + ] + }, + { + "id": "pending-order-item-return-accepted", + "scenario": "tracks/ecommerce/scenarios/progression-order-return-boundary.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3f" + ], + "desc": "Accept a pending order return and restore its stock before shipment.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "if (!['shipped', 'delivered'].includes(value.status)) throw new Error('No shipped order found');", + "replace": "if (!['pending', 'shipped', 'delivered'].includes(value.status)) throw new Error('No shipped order found');" + } + ] + }, + { + "id": "staff-admin-access-survives-role-removal", + "scenario": "tracks/ecommerce/scenarios/progression-staff-roles.json", + "targets": [ + "ecommerce.spec.access-control.staff-role-revocation.621d" + ], + "desc": "Keep administrator access after changing the assigned role back to staff.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " target.isAdmin = role === \"admin\";", + "replace": " target.isAdmin = target.isAdmin || role === \"admin\";" + } + ] + }, + { + "id": "role-assignment-trusts-sign-in-admin-claim", + "scenario": "tracks/ecommerce/scenarios/progression-staff-roles.json", + "targets": [ + "ecommerce.spec.access-control.staff-role-revocation.621d" + ], + "desc": "Sign-in writes the account's administrator flag into the token as a claim, and role assignment authorizes from that claim instead of the stored account. A token issued while the account was an administrator keeps assigning roles after the account is changed back to staff; tokens issued after the change are refused.", + "file": "server/src/progression.ts", + "edits": [ + { + "file": "server/src/index.ts", + "find": "function signToken(userId: string) {\n return jwt.sign({ sub: userId }, JWT_SECRET, { expiresIn: \"30d\" });", + "replace": "function signToken(userId: string, admin = false) {\n return jwt.sign({ sub: userId, admin }, JWT_SECRET, { expiresIn: \"30d\" });" + }, + { + "file": "server/src/index.ts", + "find": " if (!valid) return res.status(401).json({ error: \"Invalid username or password\" });\n const token = signToken(user._id.toString());", + "replace": " if (!valid) return res.status(401).json({ error: \"Invalid username or password\" });\n const token = signToken(user._id.toString(), Boolean(user.isAdmin));" + }, + { + "find": "app.put(\"/api/staff/:id/role\", auth, admin, async (req: Request, res) => {", + "replace": "app.put(\"/api/staff/:id/role\", auth, async (req: Request, res) => {\n const claims = jwt.decode(String(req.headers.authorization).slice(7), { json: true });\n if (!claims?.admin) return res.status(403).json({ error: \"Admin access required\" });" + } + ] + }, + { + "id": "shipping-counts-sale-twice", + "scenario": "tracks/ecommerce/scenarios/progression-shipping-accounting.json", + "targets": [ + "ecommerce.inventory-operations.shipping-accounting.202e" + ], + "desc": "Shipping succeeds but doubles the completed sale value in authoritative revenue.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " { $set: { status: \"shipped\" } },", + "replace": " { $set: { status: \"shipped\" }, $mul: { total: 2 } }," + } + ] + }, + { + "id": "signup-does-not-expose-created-account", + "scenario": "tracks/ecommerce/scenarios/01-account-create.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1a" + ], + "desc": "Signup succeeds but the client discards the created account identity from its current session view.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " saveSession(data.token, data.user);\n };\n\n const handleSignIn", + "replace": " saveSession(data.token, { ...data.user, username: \"\" });\n };\n\n const handleSignIn" + } + ] + }, + { + "id": "duplicate-signup-reports-success", + "scenario": "tracks/ecommerce/scenarios/01-account-duplicate.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1b" + ], + "desc": "A duplicate username is reported as a successful empty signup response instead of a refusal.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (existing) return res.status(409).json({ error: \"Username is already taken\" });", + "replace": " if (existing) return res.json({}); // mutant: duplicate signup is falsely accepted" + } + ] + }, + { + "id": "signin-skips-password-verification", + "scenario": "tracks/ecommerce/scenarios/01-account-password.json", + "targets": [ + "ecommerce.spec.access-control.account-password.1c" + ], + "desc": "Signin accepts an existing account without requiring its password to match.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (!valid) return res.status(401).json({ error: \"Invalid username or password\" });", + "replace": " if (false && !valid) return res.status(401).json({ error: \"Invalid username or password\" });" + } + ] + }, + { + "id": "signout-keeps-current-account", + "scenario": "tracks/ecommerce/scenarios/01-account-signout.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1d" + ], + "desc": "Signout disconnects the token state but leaves the current account and persisted credential in place.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const handleSignOut = () => {\n clearSession();\n };", + "replace": " const handleSignOut = () => {\n setToken(null); // mutant: visible and persisted account state is not cleared\n };" + } + ] + }, + { + "id": "session-token-not-persisted", + "scenario": "tracks/ecommerce/scenarios/01-account-reload.json", + "targets": [ + "ecommerce.spec.state-durability.session-reload.1e" + ], + "desc": "The active session is kept only in React state and is unavailable after a page reload.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " localStorage.setItem(TOKEN_KEY, tok);\n setToken(tok);", + "replace": " void tok; // mutant: the session token is never persisted\n setToken(tok);" + } + ] + }, + { + "id": "purchase-counts-never-affect-ranking", + "scenario": "tracks/ecommerce/scenarios/01-core.json", + "targets": [ + "ecommerce.spec.live-state.ranking.2c" + ], + "desc": "The catalogue ranking ignores recorded purchases and therefore never promotes the bought item.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " purchaseCount: purchaseMap.get(id) || 0,", + "replace": " purchaseCount: 0, // mutant: ranking ignores durable purchase counts" + } + ] + }, + { + "id": "signed-out-visitor-purchase-is-accepted", + "scenario": "tracks/ecommerce/scenarios/progression-signed-out-purchase.json", + "targets": [ + "ecommerce.spec.access-control.signed-out-purchase.3a" + ], + "desc": "The UI exposes purchase controls to visitors and the buy route assigns unauthenticated requests an unverified identity, allowing an actual stock-debiting order.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const isCustomer = !!currentUser && !currentUser.isAdmin && !currentUser.isStaff;", + "replace": " const isCustomer = !currentUser?.isAdmin && !currentUser?.isStaff;" + }, + { + "file": "server/src/index.ts", + "find": "app.post(\"/api/items/:id/buy\", requireAuth, async (req, res) => {", + "replace": "app.post(\"/api/items/:id/buy\", async (req, _res, next) => {\n (req as any).user = await userFromToken(extractToken(req)) || { _id: new Types.ObjectId() };\n next();\n}, async (req, res) => {" + } + ] + }, + { + "id": "espresso-stock-row-ignores-live-updates", + "scenario": "tracks/ecommerce/scenarios/01-buying.json", + "targets": [ + "ecommerce.spec.live-state.purchase-stock.3b" + ], + "desc": "The live catalogue handler preserves a stale Espresso Machine stock projection while applying all other item updates.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " socket.on(\"items:update\", (data: ItemT[]) => setItems(data));", + "replace": " socket.on(\"items:update\", (data: ItemT[]) => setItems((previous) => data.map((item) => item.name === \"Espresso Machine\" ? { ...item, stock: previous.find((old) => old.id === item.id)?.stock ?? item.stock } : item)));" + } + ] + }, + { + "id": "restock-race-records-wrong-order-total", + "scenario": "tracks/ecommerce/scenarios/01-restock-race.json", + "targets": [ + "ecommerce.spec.concurrency-safety.restock-race.202a" + ], + "desc": "Purchases preserve stock and visible order counts but record the wrong booked total. Native mixed-race reconciliation must reject them.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " total: item.price,\n });", + "replace": " total: 0, // mutant: purchase receipt loses the authoritative price\n });" + } + ] + }, + { + "id": "purchase-order-uses-zero-price", + "scenario": "tracks/ecommerce/scenarios/progression-purchasing.json", + "targets": [ + "ecommerce.feature.purchasing.purchase-order.3c" + ], + "desc": "A direct purchase records the item but stores a zero order total instead of the price paid.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " total: item.price,\n });", + "replace": " total: 0, // mutant: purchase receipt loses the authoritative price\n });" + } + ] + }, + { + "id": "reload-hydrates-an-empty-cart", + "scenario": "tracks/ecommerce/scenarios/01-cart.json", + "targets": [ + "ecommerce.spec.state-durability.cart-reload.4b" + ], + "desc": "Cart hydration discards the persisted server response after reload.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const refreshCart = useCallback(async (tok: string) => {\n const data = await apiFetch(\"/api/cart\", tok);\n setCart(data);\n }, []);", + "replace": " const refreshCart = useCallback(async (tok: string) => {\n await apiFetch(\"/api/cart\", tok);\n setCart({ items: [], total: 0 }); // mutant: persisted cart response is discarded\n }, []);" + } + ] + }, + { + "id": "shared-cart-live-events-ignored", + "scenario": "tracks/ecommerce/scenarios/01-cart.json", + "targets": [ + "ecommerce.spec.live-state.shared-cart.4c" + ], + "desc": "An already-open second session ignores committed cart update events.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " socket.on(\"cart:update\", (data: CartT) => setCart(data));", + "replace": " socket.on(\"cart:update\", (data: CartT) => setCart(current => current.items.length === 0 ? current : data)); // mutant: an empty second-session cart ignores its first remote update" + } + ] + }, + { + "id": "review-comment-is-not-persisted", + "scenario": "tracks/ecommerce/scenarios/01-review-visibility.json", + "targets": [ + "ecommerce.feature.reviews.reviews.6a" + ], + "desc": "Review submission persists an empty comment rather than the customer's submitted text.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " { itemId, userId: user._id, username: user.username, rating: ratingNum,\n comment: typeof comment === \"string\" ? comment : \"\" },", + "replace": " { itemId, userId: user._id, username: user.username, rating: ratingNum,\n comment: \"\" }," + } + ] + }, + { + "id": "repeat-review-uses-a-new-owner-key", + "scenario": "tracks/ecommerce/scenarios/01-review-uniqueness.json", + "targets": [ + "ecommerce.spec.transactional-integrity.unique-review.6b" + ], + "desc": "Each review submission is stored under a fresh owner key, bypassing the one-review-per-customer constraint.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " { itemId, userId: user._id },\n { itemId, userId: user._id, username: user.username, rating: ratingNum,\n comment: typeof comment === \"string\" ? comment : \"\" },", + "replace": " { itemId, userId: new Types.ObjectId() },\n { itemId, userId: new Types.ObjectId(), username: user.username, rating: ratingNum,\n comment: typeof comment === \"string\" ? comment : \"\" }," + } + ] + }, + { + "id": "live-review-average-uses-an-extra-divisor", + "scenario": "tracks/ecommerce/scenarios/01-review-rating-live.json", + "targets": [ + "ecommerce.spec.live-state.rating.6c" + ], + "desc": "The live review event divides the rating sum by one more review than actually exists.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "async function broadcastReviews(itemId: string) {\n const reviews = await Review.find({ itemId }).sort({ createdAt: -1 });\n const average = reviews.length ? reviews.reduce((s, r) => s + r.rating, 0) / reviews.length : 0;", + "replace": "async function broadcastReviews(itemId: string) {\n const reviews = await Review.find({ itemId }).sort({ createdAt: -1 });\n const average = reviews.length ? reviews.reduce((s, r) => s + r.rating, 0) / (reviews.length + 1) : 0;" + } + ] + }, + { + "id": "warehouse-view-omits-one-location", + "scenario": "tracks/ecommerce/scenarios/01-warehouse-admin-staff.json", + "targets": [ + "ecommerce.feature.warehouse-admin.warehouse-view.7b" + ], + "desc": "The admin warehouse projection truncates the final item-location row.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " {overview.locations.map((loc) => (", + "replace": " {overview.locations.slice(0, -1).map((loc) => (" + } + ] + }, + { + "id": "unauthenticated-purchase-defaults-to-admin", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "The purchase endpoint drops authentication and assigns sessionless purchases to the seeded administrator.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\"/api/items/:id/buy\", requireAuth, async (req, res) => {", + "replace": "app.post(\"/api/items/:id/buy\", async (req, res) => {" + }, + { + "find": " const user = (req as any).user;\n const order = await Order.create({", + "replace": " const user = (req as any).user || await User.findOne({ username: \"admin\" });\n const order = await Order.create({" + } + ] + }, + { + "id": "direct-purchase-total-ignores-store-price", + "scenario": "tracks/ecommerce/scenarios/01-server-price.json", + "targets": [ + "ecommerce.spec.transactional-integrity.server-price.104a" + ], + "desc": "The direct purchase creates one order but records a zero total rather than the store's current price.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " total: item.price,\n });", + "replace": " total: 0, // mutant: direct purchase ignores the authoritative price\n });" + } + ] + }, + { + "id": "backend-restart-discards-account-carts", + "scenario": "tracks/ecommerce/scenarios/progression-account-state-reload.json", + "targets": [ + "ecommerce.spec.state-durability.account-state-recovery.105a" + ], + "desc": "Server startup deletes every cart as leftover session state, so an account cart does not survive a backend restart; accounts, sessions and orders persist, and carts work normally between restarts.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await initializeOrderData(mongoose.connection);\n server.listen(PORT, () => {", + "replace": " await initializeOrderData(mongoose.connection);\n // mutant: carts are treated as process-lifetime session state\n await Cart.deleteMany({});\n server.listen(PORT, () => {" + } + ] + }, + { + "id": "reconnect-hydration-loses-account-state", + "scenario": "tracks/ecommerce/scenarios/progression-account-state-reconnect.json", + "targets": [ + "ecommerce.spec.state-durability.account-state-recovery.105b" + ], + "desc": "The initial account cart loads correctly, but after network restoration the client ignores both refreshed and pushed cart state.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " useEffect(() => {\n const socket = io({ auth: token ? { token } : {} });", + "replace": " useEffect(() => {\n const clearAccountOffline = () => {\n setCurrentUser(null);\n setCart({ items: [], total: 0 });\n };\n window.addEventListener(\"offline\", clearAccountOffline, { once: true });\n const socket = io({ auth: token ? { token } : {} });" + } + ] + }, + { + "id": "order-history-is-not-owner-scoped", + "scenario": "tracks/ecommerce/scenarios/01-order-ownership.json", + "targets": [ + "ecommerce.spec.access-control.order-ownership.106a" + ], + "desc": "Order history returns every customer's orders instead of filtering by the authenticated owner.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const orders = await Order.find({ userId }).sort({ createdAt: -1 });", + "replace": " const orders = await Order.find({}).sort({ createdAt: -1 });" + } + ] + }, + { + "id": "revenue-aggregation-ignores-order-totals", + "scenario": "tracks/ecommerce/scenarios/progression-books-balance.json", + "targets": [ + "ecommerce.spec.transactional-integrity.books-balance.107a" + ], + "desc": "The admin revenue aggregation counts every order as zero regardless of its stored total.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " { $group: { _id: null, total: { $sum: { $subtract: [\"$total\", { $ifNull: [\"$refundTotal\", 0] }] } } } },", + "replace": " { $group: { _id: null, total: { $sum: 0 } } }," + } + ] + }, + { + "id": "unpurchased-review-is-accepted", + "scenario": "tracks/ecommerce/scenarios/01-review-eligibility.json", + "targets": [ + "ecommerce.spec.access-control.review-eligibility.108a" + ], + "desc": "The review endpoint bypasses its completed-purchase eligibility check.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (!hasPurchased) {\n return res.status(403).json({ error: \"You can only review items you have purchased\" });\n }", + "replace": " if (false && !hasPurchased) {\n return res.status(403).json({ error: \"You can only review items you have purchased\" });\n }" + } + ] + }, + { + "id": "purchased-review-ui-does-not-submit", + "scenario": "tracks/ecommerce/scenarios/01-review-eligibility.json", + "targets": [ + "ecommerce.spec.access-control.review-eligibility.108b" + ], + "desc": "The review form discards valid customer submissions. Direct authorized and unauthorized review calls retain their normal behavior.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const handleReviewSubmit = async (itemId: string, rating: number, comment: string) => {\n setReviewError(\"\");", + "replace": " const handleReviewSubmit = async (itemId: string, rating: number, comment: string) => {\n setReviewError(\"Review submission unavailable\");\n return;" + } + ] + }, + { + "id": "external-stock-polling-disabled", + "scenario": "tracks/ecommerce/scenarios/01-external-live-sync.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock.901a" + ], + "desc": "The server stops reconciling direct database stock writes into live catalogue events.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " broadcastItems().catch((err) => console.error(\"broadcastItems poll failed\", err));", + "replace": " // mutant: direct database stock changes are never reconciled" + } + ] + }, + { + "id": "reconnect-generation-ignores-current-catalog", + "scenario": "tracks/ecommerce/scenarios/01-external-reconnect-sync.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock.901d" + ], + "desc": "After the browser goes offline, the existing page ignores reconnect refreshes and subsequent catalogue events.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const socketRef = useRef(null);\n\n const saveSession", + "replace": " const socketRef = useRef(null);\n const acceptCatalogUpdates = useRef(true);\n useEffect(() => {\n const stopCatalogRecovery = () => { acceptCatalogUpdates.current = false; };\n window.addEventListener(\"offline\", stopCatalogRecovery);\n return () => window.removeEventListener(\"offline\", stopCatalogRecovery);\n }, []);\n\n const saveSession" + }, + { + "find": " setItems(data.items);", + "replace": " if (acceptCatalogUpdates.current) setItems(data.items);" + }, + { + "find": " socket.on(\"items:update\", (data: ItemT[]) => setItems(data));", + "replace": " socket.on(\"items:update\", (data: ItemT[]) => {\n if (acceptCatalogUpdates.current) setItems(data);\n });" + } + ] + }, + { + "id": "open-review-list-ignores-live-update", + "scenario": "tracks/ecommerce/scenarios/progression-open-list-live.json", + "targets": [ + "ecommerce.spec.live-state.open-list.902a" + ], + "desc": "The already-open review list ignores a committed review update from another client.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setItemDetail((prev) => (prev && prev.id === payload.itemId ? { ...prev, reviews: payload.reviews, average: payload.average } : prev));", + "replace": " void payload; // mutant: the already-open review list ignores committed updates" + } + ] + }, + { + "id": "cancel-does-not-restore-stock-feature", + "scenario": "tracks/ecommerce/scenarios/02-order-cancellation-core.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3a" + ], + "desc": "Cancellation changes order state but skips restoration of its recorded warehouse allocations.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " {item_id:line.itemId,warehouse_id:allocation.warehouseId}, {$inc:{quantity:allocation.quantity}}, {session});", + "replace": " {item_id:line.itemId,warehouse_id:allocation.warehouseId}, {$inc:{quantity:0}}, {session});" + } + ] + }, + { + "id": "cancellation-accounting-loses-stock-restoration", + "scenario": "tracks/ecommerce/scenarios/02-invariants.json", + "targets": [ + "ecommerce.returns-pricing.refund-accounting.203a" + ], + "desc": "Cancellation removes revenue and changes order status, but loses the original warehouse stock restoration. The native refund-accounting assertion must detect this.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " {item_id:line.itemId,warehouse_id:allocation.warehouseId}, {$inc:{quantity:allocation.quantity}}, {session});", + "replace": " {item_id:line.itemId,warehouse_id:allocation.warehouseId}, {$inc:{quantity:0}}, {session});" + } + ] + }, + { + "id": "cancel-does-not-restore-stock-fresh-client", + "scenario": "tracks/ecommerce/scenarios/02-self-contained.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202b" + ], + "desc": "Cancellation changes order state but skips restoration, so a fresh client reads the persisted shortfall.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " {item_id:line.itemId,warehouse_id:allocation.warehouseId}, {$inc:{quantity:allocation.quantity}}, {session});", + "replace": " {item_id:line.itemId,warehouse_id:allocation.warehouseId}, {$inc:{quantity:0}}, {session});" + } + ] + }, + { + "id": "cancel-restores-stock-but-keeps-pending-status", + "scenario": "tracks/ecommerce/scenarios/02-order-cancellation-history.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3b" + ], + "desc": "Cancellation restores allocations but writes pending back to order history.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " value.status = \"cancelled\";\n await value.save({session});", + "replace": " value.status = \"pending\";\n await value.save({session});" + } + ] + }, + { + "id": "cancelled-order-remains-in-revenue-feature", + "scenario": "tracks/ecommerce/scenarios/02-order-cancellation-core.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3a" + ], + "desc": "Admin revenue includes cancelled orders even though cancellation otherwise succeeds.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "async function getRevenue(): Promise {\n const rows = await Order.aggregate([\n { $match: { status: { $ne: \"cancelled\" } } },", + "replace": "async function getRevenue(): Promise {\n const rows = await Order.aggregate([\n { $match: {} }," + } + ] + }, + { + "id": "cancelled-order-remains-in-revenue-invariant", + "scenario": "tracks/ecommerce/scenarios/02-invariants.json", + "targets": [ + "ecommerce.returns-pricing.refund-accounting.203a" + ], + "desc": "Admin revenue includes cancelled orders even though cancellation otherwise succeeds.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "async function getRevenue(): Promise {\n const rows = await Order.aggregate([\n { $match: { status: { $ne: \"cancelled\" } } },", + "replace": "async function getRevenue(): Promise {\n const rows = await Order.aggregate([\n { $match: {} }," + } + ] + }, + { + "id": "operator-authorization-allows-customer-transfer", + "scenario": "tracks/ecommerce/scenarios/02-strengthened.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201a" + ], + "desc": "The transfer route keeps authentication but drops its administrator role gate.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\"/api/admin/transfer\", requireAuth, requireAdmin, async (req, res) => {", + "replace": "app.post(\"/api/admin/transfer\", requireAuth, async (req, res) => {" + } + ] + }, + { + "id": "customer-can-ship-order-direct-1-1", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "The shipping route keeps authentication but drops its staff role gate.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\"/api/fulfilment/ship\", requireAuth, requireStaff, async (req, res) => {", + "replace": "app.post(\"/api/fulfilment/ship\", requireAuth, async (req, res) => {" + } + ] + }, + { + "id": "customer-can-cancel-foreign-order-1-1", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.order-owner.204a" + ], + "desc": "Cancellation retains authentication and pending-state validation but drops order ownership.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const value = orderId ? await Order.findOne({ _id: orderId, userId: user._id }).session(session) : null;\n if (!value) return null;\n if (value.status !== \"pending\") throw new Error(\"Order cannot be cancelled\");", + "replace": " const value = orderId ? await Order.findOne({ _id: orderId }).session(session) : null;\n if (!value) return null;\n if (value.status !== \"pending\") throw new Error(\"Order cannot be cancelled\");" + } + ] + }, + { + "id": "queue-depth-lags-one-order", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-live.json", + "targets": [ + "ecommerce.operations-access.fulfilment-queue.1a" + ], + "desc": "The queue renders every order but its visible depth remains one behind.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "Orders waiting: {queue.depth}", + "replace": "Orders waiting: {Math.max(0, queue.depth - 1)}" + } + ] + }, + { + "id": "ship-acknowledges-without-changing-status", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-ship.json", + "targets": [ + "ecommerce.operations-access.fulfilment-queue.1c" + ], + "desc": "Shipping returns success but writes pending back to the order, leaving both live views unchanged.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " { $set: { status: \"shipped\" } },", + "replace": " { $set: { status: \"pending\" } }," + } + ] + }, + { + "id": "customer-sees-fulfilment-navigation", + "scenario": "tracks/ecommerce/scenarios/02-features.json", + "targets": [ + "ecommerce.operations-access.fulfilment-queue.1d" + ], + "desc": "Expose the protected staff area to signed-in customers, including its navigation and content.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " {(currentUser?.isStaff || currentUser?.isAdmin) && (\n ", + "replace": " setPreference(value => ({ ...value, order: !value.order }))} />" + }, + { + "find": "function SupportTicket({ ticket, user, orders, act }: any) {", + "replace": "function OrderNotificationToggle({ on, onToggle }: { on: boolean; onToggle: () => void }) {\n const [shown] = useState(on); // mutant: the toggle keeps the state loaded when the card opened\n return ;\n}\n\nfunction SupportTicket({ ticket, user, orders, act }: any) {" + } + ] + }, + { + "id": "role-editor-snaps-back-to-stored-role", + "scenario": "tracks/ecommerce/scenarios/progression-staff-roles.json", + "targets": [ + "ecommerce.progression.staff-roles.staff-roles.621c" + ], + "desc": "Pressing Save persists the role but resets the dropdown to the role that was stored before the save, so the assignment is not visible until a reload.", + "file": "client/src/ProgressionPanel.tsx", + "edits": [ + { + "find": " ", + "replace": " " + } + ] + }, + { + "id": "queue-ignores-live-fulfilment-updates", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-live.json", + "targets": [ + "ecommerce.spec.live-state.fulfilment-queue.1a" + ], + "desc": "The open staff queue ignores live fulfilment events, so a new order appears only after a reload.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " socket.on(\"fulfilment:update\", (data: FulfilmentQueueT) => setFulfilmentQueue(data));", + "replace": " socket.on(\"fulfilment:update\", (data: FulfilmentQueueT) => { void data; }); // mutant: the open staff queue ignores live fulfilment updates" + } + ] + }, + { + "id": "low-stock-boundary-excludes-ten-live", + "scenario": "tracks/ecommerce/scenarios/02-low-stock.json", + "targets": [ + "ecommerce.spec.live-state.inventory-dashboard.5a" + ], + "desc": "The low-stock view uses a strict boundary, so an item that falls to exactly ten units never joins the list.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " .filter((it) => it.stock <= 10)", + "replace": " .filter((it) => it.stock < 10)" + } + ] + }, + { + "id": "live-admin-updates-keep-stale-category-totals", + "scenario": "tracks/ecommerce/scenarios/02-operational-category-totals.json", + "targets": [ + "ecommerce.spec.live-state.sales-dashboard.5b" + ], + "desc": "Live admin updates keep the category totals loaded at page load, so a purchase does not move units or revenue until a reload.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " socket.on(\"admin:update\", (data: AdminOverviewT) => setAdminOverview(data));", + "replace": " socket.on(\"admin:update\", (data: AdminOverviewT) => setAdminOverview((previous) => previous ? { ...data, categories: previous.categories } : data)); // mutant: live admin updates keep the category totals loaded at page load" + } + ] + }, + { + "id": "overdraw-transfer-is-accepted", + "scenario": "tracks/ecommerce/scenarios/02-transfer-overdraw.json", + "targets": [ + "ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c" + ], + "desc": "The atomic source debit no longer requires sufficient quantity, so an overdrawn transfer succeeds and moves both warehouse totals.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "{ item_id: itemId, warehouse_id: fromWarehouseId, quantity: { $gte: qty } }", + "replace": "{ item_id: itemId, warehouse_id: fromWarehouseId }" + } + ] + }, + { + "id": "credit-checkout-ignores-wallet", + "desc": "A credit checkout pays entirely externally despite available credit.", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.feature.store-credit.store-credit-750.750a" + ], + "file": "server/src/credit.ts", + "edits": [ + { + "find": " const creditMinor = useCredit ? Math.min(user.creditMinor, totalMinor) : 0;", + "replace": " const creditMinor = 0;" + } + ] + }, + { + "id": "credit-grant-replay-increments-balance", + "desc": "Replaying a grant applies its credit to the wallet again.", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.spec.store-credit.production-752.752a" + ], + "file": "server/src/credit.ts", + "edits": [ + { + "find": " if (existing.amountMinor !== amountMinor) throw new Error('Reference already identifies another grant');\n return;", + "replace": " if (existing.amountMinor !== amountMinor) throw new Error('Reference already identifies another grant');\n await User.updateOne({ _id: accountId }, { $inc: { creditMinor: amountMinor } }, { session });\n return;" + } + ] + }, + { + "id": "customer-can-grant-credit", + "desc": "Customer authentication is accepted without staff authorization.", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.spec.store-credit.production-753.753a" + ], + "file": "server/src/credit.ts", + "edits": [ + { + "find": " app.post('/api/staff/credit', auth, staff, async (req, res) => {", + "replace": " app.post('/api/staff/credit', auth, async (req, res) => {" + } + ] + }, + { + "id": "credit-checkout-retains-purchased-cart", + "desc": "A second checkout can reuse the purchased cart and create another order.", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.spec.store-credit.production-754.754a" + ], + "file": "server/src/credit.ts", + "edits": [ + { + "find": " cart.items = cart.items.filter(line => line.reservationExpiresAt && line.reservationExpiresAt <= now) as any;", + "replace": " // mutant: purchased cart lines remain" + } + ] + }, + { + "id": "split-refund-does-not-restore-credit", + "desc": "The refund is recorded but its original wallet credit is not restored.", + "scenario": "tracks/ecommerce/scenarios/progression-split-tender-refunds.json", + "targets": [ + "ecommerce.feature.split-tender-refunds.split-tender-refunds-751.751a" + ], + "file": "server/src/progression.ts", + "edits": [ + { + "find": " await refundCredit(order, session);", + "replace": " // mutant: omit wallet restoration" + } + ] + }, + { + "id": "split-refund-duplicates-credit", + "desc": "A refund credits the wallet twice while recording one refund.", + "scenario": "tracks/ecommerce/scenarios/progression-split-tender-refunds.json", + "targets": [ + "ecommerce.spec.split-tender-refunds.production-756.756a" + ], + "file": "server/src/credit.ts", + "edits": [ + { + "find": "{ $inc: { creditMinor: delta } }, { session });", + "replace": "{ $inc: { creditMinor: delta * 2 } }, { session });" + } + ] + }, + { + "id": "subscription-skips-due-purchase", + "desc": "Due deliveries are recorded as skipped although stock is available.", + "scenario": "tracks/ecommerce/scenarios/progression-subscriptions.json", + "targets": [ + "ecommerce.feature.subscriptions.subscriptions-760.760a" + ], + "file": "server/src/subscriptions.ts", + "edits": [ + { + "find": " const allocation = await reserveStock(row.itemId, row.quantity, session);", + "replace": " const allocation = row.quantity < 0 ? await reserveStock(row.itemId, row.quantity, session) : null;" + } + ] + }, + { + "id": "subscription-allows-foreign-cancellation", + "desc": "A customer can cancel another customer subscription.", + "scenario": "tracks/ecommerce/scenarios/progression-subscriptions.json", + "targets": [ + "ecommerce.spec.subscriptions.production-762.762a" + ], + "file": "server/src/subscriptions.ts", + "edits": [ + { + "find": " if (!row || String(row.userId) !== String((req as any).user._id)) return false;", + "replace": " if (!row) return false;" + } + ] + }, + { + "id": "subscription-pause-is-not-recorded", + "desc": "Pause acknowledges the request but the subscription remains active.", + "scenario": "tracks/ecommerce/scenarios/progression-subscriptions.json", + "targets": [ + "ecommerce.spec.subscriptions.production-763.763a" + ], + "file": "server/src/subscriptions.ts", + "edits": [ + { + "find": " row.status = 'paused'; row.pausedAt = new Date();", + "replace": " row.status = 'active'; row.pausedAt = new Date();" + } + ] + }, + { + "id": "credit-balance-is-cleared-at-startup", + "desc": "Restart clears an issued wallet balance while leaving accounts present.", + "file": "server/src/index.ts", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.spec.store-credit.production-755.755a" + ], + "edits": [ + { + "find": " await seed();", + "replace": " await seed();\n await User.updateMany({}, { $set: { creditMinor: 0 } });" + } + ] + }, + { + "id": "pending-subscriptions-are-cleared-at-startup", + "desc": "Restart erases pending subscription work while preserving ordinary timer execution.", + "file": "server/src/index.ts", + "scenario": "tracks/ecommerce/scenarios/progression-subscriptions.json", + "targets": [ + "ecommerce.spec.subscriptions.production-761.761a" + ], + "edits": [ + { + "find": " await seed();", + "replace": " await seed();\n await mongoose.connection.collection(\"purchasesubscriptions\").deleteMany({ status: \"active\" });" + } + ] + }, + { + "id": "bundle-definition-loses-component-quantity", + "scenario": "tracks/ecommerce/scenarios/progression-product-bundles.json", + "targets": [ + "ecommerce.feature.product-bundles.product-bundles.740a" + ], + "desc": "definition loses component quantity", + "file": "server/src/bundles.ts", + "edits": [ + { + "find": "quantity: component.quantity });", + "replace": "quantity: 1 });" + } + ] + }, + { + "id": "bundle-catalog-write-allows-customers", + "scenario": "tracks/ecommerce/scenarios/progression-product-bundles.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-743.743a" + ], + "desc": "catalog write allows customers", + "file": "server/src/bundles.ts", + "edits": [ + { + "find": "if (!actor.isAdmin && !actor.roles?.includes('catalog'))", + "replace": "if (false)" + } + ] + }, + { + "id": "bundle-checkout-price-not-snapshot", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-checkout.json", + "targets": [ + "ecommerce.feature.bundle-checkout.bundle-checkout.741a" + ], + "desc": "checkout price not snapshot", + "file": "server/src/bundles.ts", + "edits": [ + { + "find": "bundlePrice: bundle.price,", + "replace": "bundlePrice: bundle.price + 1," + } + ] + }, + { + "id": "bundle-expiry-does-not-release-components", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-checkout.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-746.746a" + ], + "desc": "expiry does not release components", + "file": "server/src/bundles.ts", + "edits": [ + { + "find": "await releaseBundle(line.componentAllocations as Allocation[], session);\n line.componentAllocations = [] as any;", + "replace": "line.componentAllocations = [] as any; // mutant: component holds leak" + } + ] + }, + { + "id": "bundle-return-loses-original-components", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-returns.json", + "targets": [ + "ecommerce.feature.bundle-returns.bundle-returns.742a" + ], + "desc": "return loses original components", + "file": "server/src/bundles.ts", + "edits": [ + { + "find": "for (const line of bundles) { await releaseBundle(line.componentAllocations as Allocation[], session); line.returned = true; }", + "replace": "for (const line of bundles) { line.returned = true; }" + } + ] + }, + { + "id": "bundle-return-replay-restocks-again", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-returns.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-742.742b" + ], + "desc": "return replay restocks again", + "file": "server/src/bundles.ts", + "edits": [ + { + "find": "line.isBundle && !line.returned", + "replace": "line.isBundle" + } + ] + }, + { + "id": "bundle-return-crosses-account-boundary", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-returns.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-748.748a" + ], + "desc": "return crosses account boundary", + "file": "server/src/bundles.ts", + "edits": [ + { + "find": "{ _id: req.params.orderId, userId, status: { $in: ['shipped', 'delivered'] } }", + "replace": "{ _id: req.params.orderId, status: { $in: ['shipped', 'delivered'] } }" + } + ] + }, + { + "id": "bundle-components-can-overdraw", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-checkout.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-744.744a", + "ecommerce.spec.bundle-integrity.bundle-745.745a" + ], + "desc": "components can overdraw", + "file": "server/src/stock-reservations.ts", + "edits": [ + { + "find": "{ item_id: itemId, quantity: { $gte: 1 } }", + "replace": "{ item_id: itemId }" + } + ] + }, + { + "id": "bundle-checkout-reuses-reservation", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-checkout.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-747.747a" + ], + "desc": "checkout reuses reservation", + "file": "server/src/credit.ts", + "edits": [ + { + "find": "cart.items = cart.items.filter(line => line.reservationExpiresAt && line.reservationExpiresAt <= now) as any;", + "replace": "// mutant: active cart survives checkout" + } + ] + }, + { + "id": "return-after-support-refund-is-blocked", + "scenario": "tracks/ecommerce/scenarios/progression-support-return-interaction.json", + "targets": [ + "ecommerce.feature.split-tender-refunds.return-refund-interaction.757a" + ], + "desc": "Reject a valid physical return after a financial refund.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (!line || line.returned) throw new Error('No returnable item found');", + "replace": " if (!line || line.returned || value.refundTotal > 0) throw new Error('No returnable item found');" + } + ] + }, + { + "id": "support-refund-after-return-pays-twice", + "scenario": "tracks/ecommerce/scenarios/progression-support-return-interaction.json", + "targets": [ + "ecommerce.feature.split-tender-refunds.return-refund-interaction.757b" + ], + "desc": "Pay the full order again after a physical return.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " order.refundTotal = order.total;", + "replace": " order.refundTotal += order.total;" + } + ] + }, + { + "id": "support-history-leaks-to-signed-out-visitors", + "scenario": "tracks/ecommerce/scenarios/progression-support-history.json", + "targets": [ + "ecommerce.spec.access-control.support-history-logout.612d" + ], + "desc": "Disclose support tickets to signed-out visitors while leaving signed-in ownership filtering unchanged.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": "const ticketFilter = isStaff ? {} : user ? { userId: user._id } : { _id: null };", + "replace": "const ticketFilter = isStaff ? {} : user ? { userId: user._id } : {};" + } + ] + }, + { + "id": "checkout-crash-integrity", + "scenario": "tracks/ecommerce/scenarios/progression-checkout-crash.json", + "targets": [ + "ecommerce.spec.state-durability.checkout-crash-integrity.910a" + ], + "desc": "A lifecycle callback changes a prepared or cleared cart to quantity two, leaving neither a legal rollback nor a complete checkout.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await initializeOrderData(mongoose.connection);", + "replace": " await initializeOrderData(mongoose.connection);\n for (const cart of await Cart.find()) { const order = await Order.findOne({ userId: cart.userId }); if (order?.items[0]) await Cart.updateOne({ _id: cart._id }, { $set: { items: [{ itemId: order.items[0].itemId, quantity: 2 }] } }); }" + } + ] + }, + { + "id": "checkout-crash-sibling-stock", + "scenario": "tracks/ecommerce/scenarios/progression-checkout-crash.json", + "targets": [ + "ecommerce.spec.state-durability.checkout-crash-integrity.910a" + ], + "desc": "A recovery callback removes stock for Coffee Grinder while leaving cart and order data intact. The second-product stock discrepancy must fail crash atomicity.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await initializeOrderData(mongoose.connection);", + "replace": " await initializeOrderData(mongoose.connection);\n if (await Order.exists({})) { const item = await Item.findOne({ name: 'Coffee Grinder' }); if (item) await Stock.updateOne({ item_id: item._id }, { $inc: { quantity: -1 } }); }" + } + ] + }, + { + "id": "checkout-crash-durability", + "scenario": "tracks/ecommerce/scenarios/progression-checkout-crash.json", + "targets": [ + "ecommerce.spec.state-durability.checkout-crash-durability.910b" + ], + "desc": "A lifecycle callback corrupts earlier order totals. The unchanged prepared snapshots and acknowledged baseline must expose the damage.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await initializeOrderData(mongoose.connection);", + "replace": " await initializeOrderData(mongoose.connection);\n await Order.updateMany({}, { $set: { total: 0 } });" + } + ] + }, + { + "id": "review-script-unsafe-render", + "scenario": "tracks/ecommerce/scenarios/progression-review-script.json", + "targets": [ + "ecommerce.progression.review-access-specifications.stored-review-script.9180a" + ], + "desc": "Render stored review text as HTML so it executes in another customer session.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "
{r.comment}
", + "replace": "
" + } + ] + }, + { + "id": "review-script-reject-all", + "scenario": "tracks/ecommerce/scenarios/progression-review-script.json", + "targets": [ + "ecommerce.progression.review-access-specifications.stored-review-script.9180a" + ], + "desc": "Reject all nonempty reviews, including the required benign review; absence of script is insufficient.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "const { rating, comment } = req.body || {};", + "replace": "const { rating, comment } = req.body || {};\n if (String(comment).length > 0) { res.status(400).json({ error: \"Review rejected\" }); return; }" + } + ] + }, + { + "id": "review-owner-trust-username", + "scenario": "tracks/ecommerce/scenarios/progression-review-access.json", + "targets": [ + "ecommerce.progression.review-access-specifications.review-eligibility-direct.618a" + ], + "desc": "Trust the submitted buyer username instead of the authenticated caller.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "const user = (req as any).user;\n const hasPurchased = await Order.exists", + "replace": "const user = req.body?.username ? await User.findOne({ username: req.body.username }) : (req as any).user;\n const hasPurchased = await Order.exists" + } + ] + }, + { + "id": "review-owner-deny-after-write", + "scenario": "tracks/ecommerce/scenarios/progression-review-access.json", + "targets": [ + "ecommerce.progression.review-access-specifications.review-eligibility-direct.618a" + ], + "desc": "Replace the buyer review under a forged username, then return a refusal.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "const user = (req as any).user;\n const hasPurchased = await Order.exists", + "replace": "const user = req.body?.username ? await User.findOne({ username: req.body.username }) : (req as any).user;\n const hasPurchased = await Order.exists" + }, + { + "find": "await broadcastReviews(String(itemId));", + "replace": "if (req.body?.username) return res.status(403).json({ error: \"Refused after write\" });\n await broadcastReviews(String(itemId));" + } + ] + }, + { + "id": "review-owner-reject-all", + "scenario": "tracks/ecommerce/scenarios/progression-review-access.json", + "targets": [ + "ecommerce.progression.review-access-specifications.review-eligibility-direct.618a" + ], + "desc": "Reject the legitimate buyer review as well as unauthorized reviews.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "const { rating, comment } = req.body || {};", + "replace": "const { rating, comment } = req.body || {};\n if (String(comment).length > 0) { res.status(403).json({ error: \"Reject all reviews\" }); return; }" + } + ] + }, + { + "id": "password-verification-truncates-utf8", + "scenario": "tracks/ecommerce/scenarios/01-account-password.json", + "targets": [ + "ecommerce.spec.access-control.account-password.1c" + ], + "desc": "Hash and verify only the first 72 UTF-8 bytes, accepting a different password suffix.", + "file": "server/src/auth.ts", + "edits": [ + { + "find": "scrypt(password, salt, 64,", + "replace": "scrypt(Buffer.from(password).subarray(0, 72), salt, 64," + } + ] + }, + { + "id": "password-verification-rejects-every-login", + "scenario": "tracks/ecommerce/scenarios/01-account-password.json", + "targets": [ + "ecommerce.spec.access-control.account-password.1c" + ], + "desc": "Refuse even a correct password; refusal alone must not earn credit.", + "file": "server/src/auth.ts", + "edits": [ + { + "find": "return crypto.timingSafeEqual", + "replace": "return false && crypto.timingSafeEqual" + } + ] + }, + { + "id": "tampered-session-can-purchase", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Accept a modified session credential and allow its purchase.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "jwt.verify(token, JWT_SECRET) as { sub: string }", + "replace": "jwt.decode(token) as { sub: string }" + } + ] + }, + { + "id": "tampered-session-commits-then-refuses", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Commit the forged purchase but return an access refusal; stored-state evidence must catch it.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "jwt.verify(token, JWT_SECRET) as { sub: string }", + "replace": "jwt.decode(token) as { sub: string }" + }, + { + "find": "res.json({ order: order.toJSON() });\n});\n\n// ---------------------------------------------------------------------------\n// Cart routes", + "replace": "try { jwt.verify(extractToken(req)!, JWT_SECRET); } catch { res.status(401); }\n res.json({ order: order.toJSON() });\n});\n\n// ---------------------------------------------------------------------------\n// Cart routes" + } + ] + }, + { + "id": "restock-trusts-submitted-admin-role", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Trust an administrator role supplied by staff in the restock request. Normal staff requests are still refused.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\"/api/admin/restock\", requireAuth, requireAdmin, async (req, res) => {", + "replace": "app.post(\"/api/admin/restock\", requireAuth, (req, res, next) => req.body?.role === \"admin\" ? next() : requireAdmin(req, res, next), async (req, res) => {" + } + ] + }, + { + "id": "direct-purchase-trusts-submitted-price", + "scenario": "tracks/ecommerce/scenarios/01-server-price.json", + "targets": [ + "ecommerce.spec.transactional-integrity.server-price.104a" + ], + "desc": "Ordinary purchases use the catalog price; an extra price field overrides the booked order and line price.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const user = (req as any).user;\n const order = await Order.create({", + "replace": " const user = (req as any).user;\n if (req.body?.price !== undefined) item.price = req.body.price;\n const order = await Order.create({" + } + ] + }, + { + "id": "order-cache-survives-account-switch", + "scenario": "tracks/ecommerce/scenarios/01-order-ownership.json", + "targets": [ + "ecommerce.spec.access-control.order-ownership.106a" + ], + "desc": "Merge order updates into a browser cache that is never cleared when the account changes. Separate-browser ownership still works.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const [orders, setOrders] = useState([]);", + "replace": " const [orders, setStoredOrders] = useState([]);\n const setOrders: typeof setStoredOrders = update => setStoredOrders(previous => {\n const incoming = typeof update === 'function' ? update(previous) : update;\n return [...new Map([...previous, ...incoming].map(order => [order.id, order])).values()];\n });" + } + ] + }, + { + "id": "checkout-trusts-submitted-price", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "Book the submitted checkout price instead of the stored price.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const order = await checkoutAtomic(userId);", + "replace": " const order = await checkoutAtomic(userId);\n if (req.body?.price !== undefined) { order.total = req.body.price; await order.save(); }" + } + ] + }, + { + "id": "checkout-missing-sibling-line", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "Commit the cart checkout with one product line missing while retaining the other product and the full order total.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const order = await checkoutAtomic(userId);", + "replace": " const order = await checkoutAtomic(userId);\n order.items = order.items.filter(line => line.name !== 'Coffee Grinder') as any; await order.save();" + } + ] + }, + { + "id": "shipping-cancellation-cancel-shipped", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Cancellation accepts a shipped order and restores its stock after shipping.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "if (value.status !== \"pending\")", + "replace": "if (false)" + } + ] + }, + { + "id": "shipping-cancellation-resurrect-cancelled", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Shipping accepts a cancelled order after its stock was restored.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "if (order.status !== \"pending\")", + "replace": "if (false)" + }, + { + "find": " { _id: orderId, status: \"pending\" },", + "replace": " { _id: orderId }," + } + ] + }, + { + "id": "shipping-cancellation-combined-race", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Cancellation restores stock but records a shipped status, combining mutually exclusive effects.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "if (value.status !== \"pending\")", + "replace": "if (false)" + }, + { + "find": "value.status = \"cancelled\";", + "replace": "value.status = \"shipped\";" + } + ] + }, + { + "id": "shipping-cancellation-reject-both", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Both cancellation and shipping refuse pending orders; useful progress must remain required.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "if (value.status !== \"pending\")", + "replace": "if (true)" + }, + { + "find": "if (order.status !== \"pending\")", + "replace": "if (true)" + } + ] + }, + { + "id": "shipping-cancellation-queue-retains-removed-order", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "The live staff queue accepts new orders but retains rows after cancellation; the database state remains correct.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " socket.on(\"fulfilment:update\", (data: FulfilmentQueueT) => setFulfilmentQueue(data));", + "replace": " socket.on(\"fulfilment:update\", (data: FulfilmentQueueT) => setFulfilmentQueue(previous => data.orders.length < previous.orders.length ? previous : data));" + } + ] + }, + { + "id": "opposing-transfer-ignored", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202d" + ], + "desc": "One valid opposing transfer is acknowledged without moving stock.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const { itemId, fromWarehouseId, toWarehouseId, quantity } = req.body || {};", + "replace": " const { itemId, fromWarehouseId, toWarehouseId, quantity } = req.body || {};\n if (Number(quantity) === 7) return res.json({});" + } + ] + }, + { + "id": "opposing-transfer-rejected", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202d" + ], + "desc": "One valid opposing transfer is refused despite ample source stock.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const { itemId, fromWarehouseId, toWarehouseId, quantity } = req.body || {};", + "replace": " const { itemId, fromWarehouseId, toWarehouseId, quantity } = req.body || {};\n if (Number(quantity) === 7) return res.status(409).json({ error: \"Transfer disabled\" });" + } + ] + }, + { + "id": "overlapping-cart-add-ignored", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "The overlapping second-product cart add is ignored while ordinary cart setup still works.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const userId = (req as any).user._id.toString();\n const reservedWarehouseIds = await reserveStock(item._id, qty);", + "replace": " if ((req as any).user.username.includes(\"checkout-overlap\") && item.name === \"Coffee Grinder\") return res.json({});\n const userId = (req as any).user._id.toString();\n const reservedWarehouseIds = await reserveStock(item._id, qty);" + } + ] + }, + { + "id": "overlapping-cart-add-rejected", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "The overlapping second-product cart add is rejected while ordinary cart setup still works.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const userId = (req as any).user._id.toString();\n const reservedWarehouseIds = await reserveStock(item._id, qty);", + "replace": " if ((req as any).user.username.includes(\"checkout-overlap\") && item.name === \"Coffee Grinder\") return res.status(409).json({ error: \"Cart add disabled\" });\n const userId = (req as any).user._id.toString();\n const reservedWarehouseIds = await reserveStock(item._id, qty);" + } + ] + }, + { + "id": "mixed-history-lost-transfer", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: lost-transfer. The stored-state comparison must catch the defect at its first effect.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await Stock.findOneAndUpdate(\n { item_id: itemId, warehouse_id: toWarehouseId },\n { $inc: { quantity: qty } },\n { upsert: true }\n );", + "replace": " await Stock.findOneAndUpdate(\n { item_id: itemId, warehouse_id: toWarehouseId },\n { $inc: { quantity: 0 } },\n { upsert: true }\n );" + } + ] + }, + { + "id": "mixed-history-wrong-price", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: wrong-price. The stored-state comparison must catch the defect at its first effect.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " total: item.price,\n });", + "replace": " total: 0, // mutant: direct purchase ignores the authoritative price\n });" + } + ] + }, + { + "id": "mixed-history-ignored-write", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: ignored-write. The stored-state comparison must catch the defect at its first effect.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const userId = (req as any).user._id.toString();\n const reservedWarehouseIds = await reserveStock(item._id, qty);", + "replace": " if ((req as any).user.username.includes(\"history-\") && item.name === \"Coffee Grinder\") return res.json({});\n const userId = (req as any).user._id.toString();\n const reservedWarehouseIds = await reserveStock(item._id, qty);" + } + ] + }, + { + "id": "mixed-history-reject-valid-write", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: reject-valid-write. The stored-state comparison must catch the defect at its first effect.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const userId = (req as any).user._id.toString();\n const reservedWarehouseIds = await reserveStock(item._id, qty);", + "replace": " if ((req as any).user.username.includes(\"history-\") && item.name === \"Coffee Grinder\") return res.status(409).json({ error: \"Cart add disabled\" });\n const userId = (req as any).user._id.toString();\n const reservedWarehouseIds = await reserveStock(item._id, qty);" + } + ] + }, + { + "id": "mixed-history-repeat-cancel", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: repeat-cancel. The stored-state comparison must catch the defect at its first effect.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (value.status !== \"pending\") throw new Error(\"Order cannot be cancelled\");", + "replace": " if (![\"pending\", \"cancelled\"].includes(value.status)) throw new Error(\"Order cannot be cancelled\");" + } + ] + }, + { + "id": "mixed-history-wrong-owner", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: wrong-owner. The stored-state comparison must catch the defect at its first effect.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " userId: user._id,\n items: [", + "replace": " userId: (await User.findOne({ _id: { $ne: user._id } }))!._id,\n items: [" + } + ] + }, + { + "id": "checkout-empty-replay-creates-order", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Allow retrying the consumed empty cart to create another order after a lost reply.", + "file": "server/src/credit.ts", + "edits": [ + { + "find": " if (!cart?.items.length) throw new Error('Cart is empty');", + "replace": " if (!cart) throw new Error('Cart is missing');" + }, + { + "find": " if (!active.length) throw new Error('Reservation expired');", + "replace": "// Defect: an empty consumed cart may create another order." + } + ] + }, + { + "id": "checkout-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Reject every valid checkout; refusal cannot earn duplicate protection credit.", + "file": "server/src/credit.ts", + "edits": [ + { + "find": " if (!cart?.items.length) throw new Error('Cart is empty');", + "replace": " if (!cart?.items.length) throw new Error('Cart is empty');\n throw new Error('Checkout disabled');" + } + ] + }, + { + "id": "checkout-replay-loses-prior-order", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Retrying the consumed cart deletes an earlier acknowledged order.", + "file": "server/src/credit.ts", + "edits": [ + { + "find": " if (!cart?.items.length) throw new Error('Cart is empty');", + "replace": " if (!cart?.items.length) { const prior = await Order.findOne({ userId }).sort({ _id: 1 }).session(session); if (prior) await prior.deleteOne({ session }); return null; }" + } + ] + }, + { + "id": "checkout-clears-cart-without-order", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Checkout returns success and empties the cart without recording an order.", + "file": "server/src/credit.ts", + "edits": [ + { + "find": " if (!cart?.items.length) throw new Error('Cart is empty');", + "replace": " if (!cart?.items.length) throw new Error('Cart is empty');\n if (cart.items.length > 0) { cart.items = [] as any; await cart.save({ session }); return null; }" + } + ] + }, + { + "id": "revenue-aggregation-truncates-order-population", + "scenario": "tracks/ecommerce/scenarios/progression-books-balance.json", + "targets": [ + "ecommerce.spec.transactional-integrity.books-balance.107a" + ], + "desc": "Calculate revenue from only the first 256 stored orders while preserving all acknowledged purchases.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "async function getRevenue(): Promise {\n const rows = await Order.aggregate([", + "replace": "async function getRevenue(): Promise {\n const rows = await Order.aggregate([\n { $limit: 256 }," + } + ] + }, + { + "id": "catalog-truncates-created-population", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-volume.json", + "targets": [ + "ecommerce.spec.transactional-integrity.catalog-volume.622c" + ], + "desc": "Omit committed products after the first 256 search results.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": ".slice(searchPage * CATALOG_PAGE_SIZE,", + "replace": ".slice(0, 256).slice(searchPage * CATALOG_PAGE_SIZE," + } + ] + }, + { + "id": "catalog-discards-product-writes", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-volume.json", + "targets": [ + "ecommerce.spec.transactional-integrity.catalog-volume.622c" + ], + "desc": "Acknowledge catalog creation without committing a product.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " router.post(\"/catalog\", auth, staff, async (req: Request, res) => {", + "replace": " router.post(\"/catalog\", auth, staff, async (req: Request, res) => {\n if (req.body) { res.json({ok:true}); return; } // mutant: successful response without product" + } + ] + }, + { + "id": "catalog-rejects-all-product-writes", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-volume.json", + "targets": [ + "ecommerce.spec.transactional-integrity.catalog-volume.622c" + ], + "desc": "Reject every catalog creation while keeping sign-in and catalog reads available.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " router.post(\"/catalog\", auth, staff, async (req: Request, res) => {", + "replace": " router.post(\"/catalog\", auth, staff, async (req: Request, res) => {\n res.status(403).json({error:'Catalog writes refused'}); return;" + } + ] + }, + { + "id": "shipping-trusts-submitted-staff-role", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "Trust a staff role supplied by a customer; ordinary customers remain refused.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\"/api/fulfilment/ship\", requireAuth, requireStaff, async (req, res) => {", + "replace": "app.post(\"/api/fulfilment/ship\", requireAuth, (req, res, next) => req.body?.role === \"staff\" ? next() : requireStaff(req, res, next), async (req, res) => {" + } + ] + }, + { + "id": "shipping-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "Reject legitimate staff shipping as well as customer requests.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\"/api/fulfilment/ship\", requireAuth, requireStaff, async (req, res) => {", + "replace": "app.post(\"/api/fulfilment/ship\", requireAuth, requireStaff, async (req, res) => {\n return res.status(403).json({ error: \"Shipping refused\" });" + } + ] + }, + { + "id": "cancel-trusts-submitted-owner", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.order-owner.204a" + ], + "desc": "Trust a username submitted by another signed-in customer as the order owner.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\"/api/orders/:id/cancel\", requireAuth, async (req, res) => {\n const user = (req as any).user;", + "replace": "app.post(\"/api/orders/:id/cancel\", requireAuth, async (req, res) => {\n const user = (typeof req.body?.username === 'string' ? await User.findOne({username:req.body.username}) : null) ?? (req as any).user;" + } + ] + }, + { + "id": "cancel-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.order-owner.204a" + ], + "desc": "Reject legitimate cancellation as well as foreign requests.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\"/api/orders/:id/cancel\", requireAuth, async (req, res) => {\n const user = (req as any).user;", + "replace": "app.post(\"/api/orders/:id/cancel\", requireAuth, async (req, res) => {\n const user = (req as any).user;\n return res.status(403).json({error:\"Cancellation refused\"});" + } + ] + }, + { + "id": "account-password-in-public-data", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Expose the actual submitted account password through data available to other clients.", + "edits": [ + { + "file": "server/src/index.ts", + "find": "const app = express();", + "replace": "const app = express();\nconst leakedPasswords: string[] = [];\napp.use((_req, res, next) => { const json = res.json.bind(res); res.json = (body: any) => json({ ...body, debugCredentials: leakedPasswords }); next(); });" + }, + { + "file": "server/src/index.ts", + "find": "const passwordHash = await hashPassword(password);", + "replace": "const passwordHash = await hashPassword(password);\n leakedPasswords.push(password);" + } + ] + }, + { + "id": "account-disclosure-rejects-login", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Reject every valid login; absent disclosures alone must not earn a pass.", + "file": "server/src/auth.ts", + "edits": [ + { + "find": "return crypto.timingSafeEqual", + "replace": "return false && crypto.timingSafeEqual" + } + ] + }, + { + "id": "signout-keeps-session", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Show a signed-out interface while retaining the browser credential and its purchase authority.", + "edits": [ + { + "file": "client/src/App.tsx", + "find": " const clearSession = () => {\n localStorage.removeItem(TOKEN_KEY);", + "replace": " const clearSession = () => {\n // mutant: retain the current request token after signout" + } + ] + }, + { + "id": "query-like-password-bypasses-signin", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Interpret the query-like password as a login bypass while retaining normal password verification.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const valid = await verifyPassword(password, user.passwordHash);", + "replace": " const valid = password === \"' OR '1'='1\" || await verifyPassword(password, user.passwordHash);" + } + ] + }, + { + "id": "purchase-session-rejects-returning-login", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Refuse every returning login; blocking all access after logout must not pass the session check.", + "file": "server/src/auth.ts", + "edits": [ + { + "find": "return crypto.timingSafeEqual", + "replace": "return false && crypto.timingSafeEqual" + } + ] + }, + { + "id": "rejected-query-login-creates-order", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Create an order during the query-like login, leave stock unchanged and still refuse access; a refusal and stock-only observation must not earn a pass.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const valid = await verifyPassword(password, user.passwordHash);", + "replace": " if (password === \"' OR '1'='1\") {\n const item = await Item.findOne({ name: \"Bluetooth Speaker\" });\n if (!item) throw new Error(\"mutation requires the seeded item\");\n await Order.create({ userId: user._id, items: [{ itemId: item._id, name: item.name, price: item.price, quantity: 1, allocations: [] }], total: item.price });\n }\n const valid = await verifyPassword(password, user.passwordHash);" + } + ] + }, + { + "id": "account-password-in-script", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Ship the account password sentinel in the public JavaScript bundle while normal signup and login still work.", + "edits": [ + { + "file": "client/src/main.tsx", + "find": "import React from \"react\";", + "replace": "import React from \"react\";\nObject.assign(window, { debugPassword: 'sb-private-pw-76dc492ea153' });" + } + ] + }, + { + "id": "account-password-in-style", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Ship the account password sentinel in a public stylesheet while normal signup and login still work.", + "edits": [ + { + "file": "client/src/index.css", + "find": ":root {", + "replace": ":root {\n --debug-password: \"sb-private-pw-76dc492ea153\";" + } + ] + }, + { + "id": "signup-accepts-admin-claim", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Trust a submitted signup administrator claim and grant protected stock-write authority.", + "edits": [ + { + "file": "server/src/index.ts", + "find": "User.create({ username: username.trim(), passwordHash, isAdmin: false })", + "replace": "User.create({ username: username.trim(), passwordHash, isAdmin: req.body.role === 'admin' })" + } + ] + }, + { + "id": "admin-boundary-rejects-all-signups", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Refuse every new registration; blocking useful signup must not pass the authority check.", + "edits": [ + { + "file": "server/src/index.ts", + "find": "app.post(\"/api/auth/signup\", async (req, res) => {", + "replace": "app.post(\"/api/auth/signup\", async (req, res) => {\n return res.status(403).json({ error: \"Registration disabled\" });" + } + ] + }, + { + "id": "inactive-cart-expiry-reset-by-reservation-release", + "scenario": "tracks/ecommerce/scenarios/03-cart-expiration.json", + "targets": [ + "ecommerce.l3.cart-expiration.cart-expiration.304a" + ], + "desc": "Treat the server releasing an expired reservation as cart activity, so an abandoned cart is still full after five minutes.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " } } }, { $set: { \"items.$.reservedWarehouseIds\": [] } }, { timestamps: false });", + "replace": " } } }, { $set: { \"items.$.reservedWarehouseIds\": [], inactiveExpiresAt: new Date(Date.now() + 300_000) } }, { timestamps: false });" + } + ] + }, + { + "id": "shipped-order-delivery-waits-an-hour", + "scenario": "tracks/ecommerce/scenarios/03-order-delivery.json", + "targets": [ + "ecommerce.l3.order-delivery.order-delivery.303a" + ], + "desc": "Deliver a shipped order after an hour instead of after 60 seconds.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": "const shipped = await Order.find({ status: \"shipped\", updatedAt: { $lte: new Date(Date.now() - 60_000) } });", + "replace": "const shipped = await Order.find({ status: \"shipped\", updatedAt: { $lte: new Date(Date.now() - 3_600_000) } });" + } + ] + }, + { + "id": "delivery-timer-advances-cancelled-orders", + "scenario": "tracks/ecommerce/scenarios/03-order-delivery.json", + "targets": [ + "ecommerce.l3.order-delivery.order-delivery.305a" + ], + "desc": "The delivery timer advances cancelled orders to delivered as well as shipped ones.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": "const shipped = await Order.find({ status: \"shipped\", updatedAt: { $lte: new Date(Date.now() - 60_000) } });", + "replace": "const shipped = await Order.find({ status: { $in: [\"shipped\", \"cancelled\"] }, updatedAt: { $lte: new Date(Date.now() - 60_000) } });" + } + ] + }, + { + "id": "customer-can-save-reorder-rule", + "scenario": "tracks/ecommerce/scenarios/progression-automatic-reorder-access.json", + "targets": [ + "ecommerce.spec.access-control.automatic-reorder-access.502c" + ], + "desc": "The reorder rule endpoint omits its staff check, so a customer can overwrite a rule.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": "app.put(\"/api/reorders/:itemId\", auth, staff, async (req: Request, res) => {", + "replace": "app.put(\"/api/reorders/:itemId\", auth, async (req: Request, res) => {" + } + ] + }, + { + "id": "automatic-restock-dedupe-keyed-by-stock-level", + "scenario": "tracks/ecommerce/scenarios/progression-automatic-reorder-duplicate.json", + "targets": [ + "ecommerce.spec.transactional-integrity.automatic-reorder-deduplication.502b" + ], + "desc": "Deduplicate automatic restocks by the stock level that triggered them, so each further sale below the threshold adds another pending restock.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " const pending = await ScheduledRestock.exists({ itemId: rule.itemId, status: \"pending\" });", + "replace": " const pending = await ScheduledRestock.exists({ itemId: rule.itemId, status: \"pending\",\n source: `automatic:${total[0]?.total || 0}` });" + }, + { + "find": " dueAt: new Date(Date.now() + 60_000), source: \"automatic\" });", + "replace": " dueAt: new Date(Date.now() + 60_000), source: `automatic:${total[0]?.total || 0}` });" + } + ] + }, + { + "id": "automatic-restock-skips-threshold-boundary", + "scenario": "tracks/ecommerce/scenarios/progression-automatic-reorder.json", + "targets": [ + "ecommerce.progression.automatic-reorder.automatic-reorder.502a" + ], + "desc": "Create an automatic restock only when stock falls below the threshold, not when it reaches it.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " if ((total[0]?.total || 0) > rule.threshold) continue;", + "replace": " if ((total[0]?.total || 0) >= rule.threshold) continue;" + } + ] + }, + { + "id": "full-cart-restore-shows-unavailable-warning", + "scenario": "tracks/ecommerce/scenarios/progression-cart-recovery.json", + "targets": [ + "ecommerce.progression.cart-recovery.cart-recovery.503a" + ], + "desc": "Show the unavailable-items warning after every restore, even when every item was restored.", + "file": "client/src/ProgressionPanel.tsx", + "edits": [ + { + "find": " setRestoreWarning(result.unavailable.join(\", \"));", + "replace": " setRestoreWarning(`Unavailable: ${result.unavailable.join(\", \")}`);" + } + ] + }, + { + "id": "partial-restore-drops-unavailable-silently", + "scenario": "tracks/ecommerce/scenarios/progression-cart-recovery.json", + "targets": [ + "ecommerce.progression.cart-recovery.cart-recovery.503b" + ], + "desc": "Skip an unavailable item during a cart restore without naming it.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " if (!held) { unavailable.push(item.name); continue; }", + "replace": " if (!held) continue;" + } + ] + }, + { + "id": "delivery-notifications-are-not-owner-scoped", + "scenario": "tracks/ecommerce/scenarios/progression-delivery-notifications.json", + "targets": [ + "ecommerce.spec.access-control.delivery-notification-privacy.501b" + ], + "desc": "Notification reads return every account's notifications, so another customer sees the owner's delivery.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": "user ? Notification.find({ userId: user._id }).sort({ createdAt: -1 }) : [],", + "replace": "user ? Notification.find({}).sort({ createdAt: -1 }) : []," + } + ] + }, + { + "id": "pending-order-shows-return-button", + "scenario": "tracks/ecommerce/scenarios/progression-order-return-boundary.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3e" + ], + "desc": "Offer item return buttons on every order that is not cancelled, including pending ones.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " {[\"shipped\", \"delivered\"].includes(order.status) &&\n order.items", + "replace": " {order.status !== \"cancelled\" &&\n order.items" + } + ] + }, + { + "id": "support-order-link-skips-order-ownership", + "scenario": "tracks/ecommerce/scenarios/progression-order-support-boundary.json", + "targets": [ + "ecommerce.spec.access-control.order-support-ownership.614b" + ], + "desc": "Link any order to a support case without checking that the order belongs to the case owner.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " const order = await Order.findOne({ _id: orderId, userId: req.progressionUser._id });", + "replace": " const order = await Order.findById(orderId);" + } + ] + }, + { + "id": "support-order-link-is-not-saved", + "scenario": "tracks/ecommerce/scenarios/progression-order-support-owned.json", + "targets": [ + "ecommerce.progression.order-support.order-support-owned.614a" + ], + "desc": "Accept an order link on a support case without saving it.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " ticket.orderId = order._id;\n await ticket.save();", + "replace": " ticket.orderId = order._id;" + } + ] + }, + { + "id": "promotion-redemption-counted-when-applied", + "scenario": "tracks/ecommerce/scenarios/progression-promotion-reporting.json", + "targets": [ + "ecommerce.progression.promotion-reporting.promotion-report-redemptions.622a" + ], + "desc": "Count a promotion redemption when the code is applied to the cart and again at checkout.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " const cart = await Cart.findOneAndUpdate({ userId: req.progressionUser._id },\n { $set: { promotionCode: code, discount: promotion.discount } }, { new: true });", + "replace": " await Promotion.updateOne({ code }, { $inc: { redemptions: 1 } });\n const cart = await Cart.findOneAndUpdate({ userId: req.progressionUser._id },\n { $set: { promotionCode: code, discount: promotion.discount } }, { new: true });" + } + ] + }, + { + "id": "promotion-revenue-records-undiscounted-subtotal", + "scenario": "tracks/ecommerce/scenarios/progression-promotion-reporting.json", + "targets": [ + "ecommerce.progression.promotion-reporting.promotion-report-revenue.622b" + ], + "desc": "Record the subtotal before the discount as promotion revenue.", + "file": "server/src/credit.ts", + "edits": [ + { + "find": "{ $inc: { redemptions: 1, revenue: totalMinor / 100 } }", + "replace": "{ $inc: { redemptions: 1, revenue: subtotal } }" + } + ] + }, + { + "id": "recommendation-dismissals-are-not-owner-scoped", + "scenario": "tracks/ecommerce/scenarios/progression-recommendation-feedback.json", + "targets": [ + "ecommerce.spec.access-control.recommendation-feedback-privacy.504b" + ], + "desc": "Recommendations exclude items dismissed by any account, not only the viewer's own dismissals.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "const dismissedItemIds = new Set((await Dismissal.find({ userId })).map(entry => entry.itemId.toString()));", + "replace": "const dismissedItemIds = new Set((await Dismissal.find({})).map(entry => entry.itemId.toString()));" + } + ] + }, + { + "id": "catalog-creation-not-recorded-in-activity", + "scenario": "tracks/ecommerce/scenarios/progression-staff-activity.json", + "targets": [ + "ecommerce.progression.staff-activity.staff-activity.624a" + ], + "desc": "Create a catalog item without recording it in staff activity.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " await recordActivity(req.progressionUser, \"created catalog item\", name);\n changed();", + "replace": " changed();" + } + ] + }, + { + "id": "customer-sees-staff-activity-tools", + "scenario": "tracks/ecommerce/scenarios/progression-staff-activity.json", + "targets": [ + "ecommerce.spec.access-control.staff-activity-privacy.624b" + ], + "desc": "Render the staff tools, including the activity link, for every signed-in account.", + "edits": [ + { + "file": "client/src/App.tsx", + "find": "{!activeView && (currentUser?.isStaff || currentUser?.isAdmin) &&", + "replace": "{!activeView && currentUser &&" + }, + { + "file": "client/src/ProgressionPanel.tsx", + "find": " if (!(user?.isStaff || user?.isAdmin)) return null;", + "replace": " if (!user) return null;" + } + ] + }, + { + "id": "customer-can-issue-support-refund", + "scenario": "tracks/ecommerce/scenarios/progression-support-refunds-access.json", + "targets": [ + "ecommerce.spec.access-control.support-refund-access.615c" + ], + "desc": "The support refund endpoint omits its staff check, so the case owner can refund their own order.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " supportRouter.post(\"/cases/:caseId/refund\", auth, staff, async (req: Request, res) => {", + "replace": " supportRouter.post(\"/cases/:caseId/refund\", auth, async (req: Request, res) => {" + } + ] + }, + { + "id": "support-refund-replay-overwrites-case-total", + "scenario": "tracks/ecommerce/scenarios/progression-support-refunds-accounting.json", + "targets": [ + "ecommerce.spec.transactional-integrity.support-refund-accounting.615b" + ], + "desc": "Accept a repeated refund on an already refunded case, which rewrites the case's refunded amount.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " if (!ticket?.orderId || ticket.refundTotal > 0) throw new Error('No refundable order-linked case');", + "replace": " if (!ticket?.orderId) throw new Error('No refundable order-linked case');" + } + ] + }, + { + "id": "support-refund-leaves-case-open", + "scenario": "tracks/ecommerce/scenarios/progression-support-refunds-resolution.json", + "targets": [ + "ecommerce.progression.support-refunds.support-refunds-resolution.615a" + ], + "desc": "Refund the linked order but leave the support case open.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " ticket.status = 'resolved'; ticket.refundTotal = amount;", + "replace": " ticket.refundTotal = amount;" + } + ] + }, + { + "id": "delivery-due-time-kept-only-in-memory", + "scenario": "tracks/ecommerce/scenarios/03-deferred-durability.json", + "targets": [ + "ecommerce.l3.deferred-durability.restart-survival.315a" + ], + "desc": "Shipping records the delivery due time only in server memory, so an order shipped before a restart stays shipped forever.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": "export function installProgressionRoutes(app: express.Express, io: SocketIOServer,", + "replace": "export const deliveryDueAt = new Map();\n\nexport function installProgressionRoutes(app: express.Express, io: SocketIOServer," + }, + { + "find": "const shipped = await Order.find({ status: \"shipped\", updatedAt: { $lte: new Date(Date.now() - 60_000) } });", + "replace": "const shipped = await Order.find({ status: \"shipped\", _id: { $in: [...deliveryDueAt]\n .filter(([, dueAt]) => dueAt <= Date.now()).map(([id]) => new Types.ObjectId(id)) } });" + }, + { + "file": "server/src/index.ts", + "find": "import { installProgressionRoutes } from \"./progression.js\";", + "replace": "import { deliveryDueAt, installProgressionRoutes } from \"./progression.js\";" + }, + { + "file": "server/src/index.ts", + "find": " if (!shipped) return res.status(400).json({ error: \"Order is not waiting to be shipped\" });", + "replace": " if (!shipped) return res.status(400).json({ error: \"Order is not waiting to be shipped\" });\n deliveryDueAt.set(String(shipped._id), Date.now() + 60_000);" + } + ] + }, + { + "id": "startup-delivery-catch-up-duplicates-delivered-orders", + "scenario": "tracks/ecommerce/scenarios/03-deferred-integrity.json", + "targets": [ + "ecommerce.l3.deferred-integrity.exactly-once.312a" + ], + "desc": "A startup delivery catch-up writes a new delivered order record for every order already delivered, so each restart duplicates them.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await initializeOrderData(mongoose.connection);", + "replace": " await initializeOrderData(mongoose.connection);\n for (const { _id, ...order } of await Order.collection.find({ status: \"delivered\" }).toArray()) {\n await Order.collection.insertOne({ ...order, status: \"delivered\" });\n }" + } + ] + }, + { + "id": "startup-restarts-cart-inactivity-deadlines", + "scenario": "tracks/ecommerce/scenarios/03-deferred-durability.json", + "targets": [ + "ecommerce.l3.deferred-durability.restart-survival.316a" + ], + "desc": "Startup gives every open cart a fresh five-minute inactivity deadline, so a cart left idle before a restart expires minutes late.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await seed();", + "replace": " await seed();\n await Cart.updateMany({ items: { $ne: [] } }, { $set: { inactiveExpiresAt: new Date(Date.now() + 300_000) } });" + } + ] + }, + { + "id": "delivery-notification-addressed-to-order-id", + "scenario": "tracks/ecommerce/scenarios/progression-delivery-notifications.json", + "targets": [ + "ecommerce.progression.delivery-notifications.delivery-notification-delivery.501a", + "ecommerce.spec.access-control.delivery-notification-privacy.501b" + ], + "desc": "Delivery notifications are stored under the order's id instead of the owner's account, so the owner never receives one.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": "await Notification.updateOne({ userId: order.userId, key: `delivery:${order._id}` },\n { $setOnInsert: { userId: order.userId, key: `delivery:${order._id}`,", + "replace": "await Notification.updateOne({ userId: order._id, key: `delivery:${order._id}` },\n { $setOnInsert: { userId: order._id, key: `delivery:${order._id}`," + } + ] + }, + { + "id": "dismissal-does-not-refresh-open-recommendations", + "scenario": "tracks/ecommerce/scenarios/progression-recommendation-feedback.json", + "targets": [ + "ecommerce.progression.recommendation-feedback.recommendation-feedback.504a", + "ecommerce.spec.access-control.recommendation-feedback-privacy.504b" + ], + "desc": "A dismissal is saved, but nothing refreshes the open recommendation list, so the item stays on screen until the page reloads.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "{ method: \"POST\" }); setRecommended(value => value.filter(entry => entry.id !== item.id)); }}>", + "replace": "{ method: \"POST\" }); }}>" + }, + { + "find": " socket.on(\"progression:update\", () => {\n refreshItems().catch((err) => console.error(err));\n refreshRecommended(token).catch((err) => console.error(err));\n", + "replace": " socket.on(\"progression:update\", () => {\n refreshItems().catch((err) => console.error(err));\n" + }, + { + "file": "server/src/index.ts", + "find": " broadcastFulfilment().catch((err) => console.error(\"broadcastFulfilment poll failed\", err));\n broadcastRecommendedForAll().catch((err) => console.error(\"broadcastRecommended poll failed\", err));\n", + "replace": " broadcastFulfilment().catch((err) => console.error(\"broadcastFulfilment poll failed\", err));\n" + } + ] + }, + { + "id": "recommendations-capped-at-highest-purchase-price", + "scenario": "tracks/ecommerce/scenarios/progression-personalized-recommendations.json", + "targets": [ + "ecommerce.progression.personalized-recommendations.personalized-recommendations.403a" + ], + "desc": "Recommendations leave out items priced above the most expensive item the customer has bought.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const categories = new Set(purchasedItems.map((i) => i.category));", + "replace": " const categories = new Set(purchasedItems.map((i) => i.category));\n const priceCeiling = Math.max(...purchasedItems.map((i) => i.price));" + }, + { + "find": " return itemsLive.filter((it) => categories.has(it.category) && !purchasedItemIds.includes(it.id)", + "replace": " return itemsLive.filter((it) => categories.has(it.category) && it.price <= priceCeiling && !purchasedItemIds.includes(it.id)" + } + ] + }, + { + "id": "recommendations-exclude-only-first-order-items", + "scenario": "tracks/ecommerce/scenarios/progression-personalized-recommendations.json", + "targets": [ + "ecommerce.spec.access-control.recommendation-profile-isolation.403b" + ], + "desc": "Recommendations exclude only items from the customer's first order, so an item bought later stays recommended.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " return itemsLive.filter((it) => categories.has(it.category) && !purchasedItemIds.includes(it.id)", + "replace": " const firstOrder = await Order.findOne({ userId, status: { $ne: \"cancelled\" } }).sort({ createdAt: 1 });\n const firstOrderItemIds = (firstOrder?.items || []).map((line) => line.itemId.toString());\n return itemsLive.filter((it) => categories.has(it.category) && !firstOrderItemIds.includes(it.id)" + } + ] + }, + { + "id": "stock-alert-misses-direct-stock-writes", + "scenario": "tracks/ecommerce/scenarios/progression-stock-alert-direct-restore.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock-alert.631d" + ], + "desc": "The alert timer checks only items the admin restock route has just raised, so stock written directly to the database never delivers its alert.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": "export function installProgressionRoutes(app: express.Express, io: SocketIOServer,", + "replace": "export const appRestockedItems = new Set();\n\nexport function installProgressionRoutes(app: express.Express, io: SocketIOServer," + }, + { + "find": " const alerts = await StockAlert.find({ sent: false });", + "replace": " const restockedNow = [...appRestockedItems];\n appRestockedItems.clear();\n const alerts = await StockAlert.find({ sent: false, itemId: { $in: restockedNow } });" + }, + { + "file": "server/src/index.ts", + "find": "import { installProgressionRoutes } from \"./progression.js\";", + "replace": "import { appRestockedItems, installProgressionRoutes } from \"./progression.js\";" + }, + { + "file": "server/src/index.ts", + "find": " { $inc: { quantity: qty } },\n { upsert: true }\n );\n\n await Promise.all([broadcastItems(), broadcastAdmin()]);", + "replace": " { $inc: { quantity: qty } },\n { upsert: true }\n );\n appRestockedItems.add(String(item._id));\n\n await Promise.all([broadcastItems(), broadcastAdmin()]);" + } + ] + } + ] +} diff --git a/tools/stack-bench/grader/mutations/postgres-ecommerce.json b/tools/stack-bench/grader/mutations/postgres-ecommerce.json new file mode 100644 index 00000000000..f6c9d0f0300 --- /dev/null +++ b/tools/stack-bench/grader/mutations/postgres-ecommerce.json @@ -0,0 +1,3952 @@ +{ + "schemaVersion": 3, + "fixtureSha256": "b54d02f7b19b4f2e59349c776780fbec6560575a998f2c8592cf4cda6d24acac", + "backend": "postgres", + "track": "ecommerce", + "note": "Mutation definitions for the PostgreSQL ecommerce reference.", + "mutations": [ + { + "id": "recommendation-dismissal-lost-on-restart", + "scenario": "tracks/ecommerce/scenarios/progression-recommendation-feedback.json", + "targets": [ + "ecommerce.spec.state-durability.recommendation-feedback-restart.504c" + ], + "desc": "Erase saved recommendation dismissals when the application starts again.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await initializeProgressionSchema(pool);", + "replace": " await initializeProgressionSchema(pool);\n await pool.query('DELETE FROM recommendation_dismissal');" + } + ] + }, + { + "id": "pending-order-item-return-accepted", + "scenario": "tracks/ecommerce/scenarios/progression-order-return-boundary.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3f" + ], + "desc": "Accept a pending order return and restore its stock before shipment.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "if (!['shipped', 'delivered'].includes(orderRow.rows[0].status)) {", + "replace": "if (!['pending', 'shipped', 'delivered'].includes(orderRow.rows[0].status)) {" + } + ] + }, + { + "id": "staff-admin-access-survives-role-removal", + "scenario": "tracks/ecommerce/scenarios/progression-staff-roles.json", + "targets": [ + "ecommerce.spec.access-control.staff-role-revocation.621d" + ], + "desc": "Keep administrator access after changing the assigned role back to staff.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": "is_admin = ($1 = 'admin')", + "replace": "is_admin = (is_admin OR $1 = 'admin')" + } + ] + }, + { + "id": "role-assignment-trusts-sign-in-admin-flag", + "scenario": "tracks/ecommerce/scenarios/progression-staff-roles.json", + "targets": [ + "ecommerce.spec.access-control.staff-role-revocation.621d" + ], + "desc": "Sign-in copies the account's administrator flag into the session row, and role assignment authorizes from that copied flag instead of the stored account. A session opened while the account was an administrator keeps assigning roles after the account is changed back to staff; sessions opened after the change are refused.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " ALTER TABLE account ADD COLUMN IF NOT EXISTS notify_order boolean NOT NULL DEFAULT false;", + "replace": " ALTER TABLE account ADD COLUMN IF NOT EXISTS notify_order boolean NOT NULL DEFAULT false;\n ALTER TABLE session ADD COLUMN IF NOT EXISTS signin_is_admin boolean NOT NULL DEFAULT false;" + }, + { + "file": "server/src/index.ts", + "find": " await db.insert(session).values({ id: token, accountId: acc.id });", + "replace": " await db.insert(session).values({ id: token, accountId: acc.id });\n await pool.query(\"UPDATE session SET signin_is_admin = $1 WHERE id = $2\", [acc.isAdmin, token]);" + }, + { + "find": "app.put(\"/api/staff/:id/role\", dependencies.requireAdmin, asyncRoute(async (req, res) => {", + "replace": "app.put(\"/api/staff/:id/role\", dependencies.requireAuth, asyncRoute(async (req, res) => {\n const signIn = await deps.pool.query(\"SELECT signin_is_admin FROM session WHERE id = $1\", [req.cookies?.sid]);\n if (!signIn.rows[0]?.signin_is_admin) { res.status(403).json({ error: \"admin only\" }); return; }" + } + ] + }, + { + "id": "shipping-counts-sale-twice", + "scenario": "tracks/ecommerce/scenarios/progression-shipping-accounting.json", + "targets": [ + "ecommerce.inventory-operations.shipping-accounting.202e" + ], + "desc": "Shipping succeeds but doubles the completed sale value in authoritative revenue.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "`UPDATE orders SET status = 'shipped', shipped_at = now()\n WHERE id = $1 AND status = 'pending' RETURNING account_id`", + "replace": "`UPDATE orders SET status = 'shipped', shipped_at = now(), total = total * 2\n WHERE id = $1 AND status = 'pending' RETURNING account_id`" + } + ] + }, + { + "id": "signup-shows-a-different-account", + "scenario": "tracks/ecommerce/scenarios/01-account-create.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1a" + ], + "desc": "Create the requested account but display a different signed-in account name.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const r = await api<{ account: Account }>(\"/api/auth/signup\", {\n method: \"POST\",\n body: JSON.stringify({ username, password }),\n });\n setAccount(r.account);", + "replace": " const r = await api<{ account: Account }>(\"/api/auth/signup\", {\n method: \"POST\",\n body: JSON.stringify({ username, password }),\n });\n setAccount(r.account ? { ...r.account, username: \"different-account\" } : null);" + } + ] + }, + { + "id": "duplicate-signup-authenticates-existing-account", + "scenario": "tracks/ecommerce/scenarios/01-account-duplicate.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1b" + ], + "desc": "Treat a duplicate signup as a successful session for the pre-existing account.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (existing.length > 0) {\n res.status(409).json({ error: \"username already taken\" });\n return;\n }", + "replace": " if (existing.length > 0) {\n const token = newToken();\n await db.insert(session).values({ id: token, accountId: existing[0].id });\n res.cookie(\"sid\", token, { httpOnly: true, sameSite: \"lax\", path: \"/\" });\n res.json({ account: { id: existing[0].id, username, isAdmin: false, isStaff: false } });\n return;\n }" + } + ] + }, + { + "id": "signin-skips-password-verification", + "scenario": "tracks/ecommerce/scenarios/01-account-password.json", + "targets": [ + "ecommerce.spec.access-control.account-password.1c" + ], + "desc": "Accept an existing account regardless of its password; valid login remains possible.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (rows.length === 0 || !(await verifyPassword(password, rows[0].passwordHash))) {", + "replace": " if (rows.length === 0) {" + } + ] + }, + { + "id": "correct-signin-is-refused", + "scenario": "tracks/ecommerce/scenarios/01-account-signout.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1d" + ], + "desc": "Preserve wrong-password refusal but reject an otherwise valid sign-in, preventing a signed-out account from returning.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const acc = rows[0];\n const token = newToken();", + "replace": " const acc = rows[0];\n if (username === acc.username) {\n res.status(401).json({ error: \"sign in is unavailable\" });\n return;\n }\n const token = newToken();" + } + ] + }, + { + "id": "reload-discards-session-identity", + "scenario": "tracks/ecommerce/scenarios/01-account-reload.json", + "targets": [ + "ecommerce.spec.state-durability.session-reload.1e" + ], + "desc": "Ignore the authenticated identity returned during initial page hydration.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setAccount(me.account);", + "replace": " setAccount(null);" + } + ] + }, + { + "id": "purchase-does-not-broadcast-ranking", + "scenario": "tracks/ecommerce/scenarios/01-core.json", + "targets": [ + "ecommerce.spec.live-state.ranking.2c" + ], + "desc": "Commit the purchase but omit the catalog broadcast that updates already-open rankings.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " lastCatalogJson = json;\n io.emit(\"items:update\", { items: catalog });", + "replace": " lastCatalogJson = json;\n // mutant: changed catalog state is not broadcast" + } + ] + }, + { + "id": "signed-out-purchase-uses-default-account", + "scenario": "tracks/ecommerce/scenarios/progression-signed-out-purchase.json", + "targets": [ + "ecommerce.spec.access-control.signed-out-purchase.3a" + ], + "desc": "Expose purchase controls to guests and let the purchase route charge the first stored account when no caller is authenticated.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const canBuy = !!account && !account.isAdmin && !account.isStaff;", + "replace": " const canBuy = !account || (!account.isAdmin && !account.isStaff);" + }, + { + "file": "server/src/index.ts", + "find": " \"/api/items/:id/buy\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const itemId = Number(req.params.id);\n const accountId = req.account!.id;", + "replace": " \"/api/items/:id/buy\",\n asyncHandler(async (req, res) => {\n const itemId = Number(req.params.id);\n const accountId = req.account?.id ?? (await pool.query(`SELECT id FROM account ORDER BY id LIMIT 1`)).rows[0].id;" + } + ] + }, + { + "id": "purchase-stock-change-is-not-broadcast--01-buying", + "scenario": "tracks/ecommerce/scenarios/01-buying.json", + "targets": [ + "ecommerce.spec.live-state.purchase-stock.3b" + ], + "desc": "Commit purchases without broadcasting their stock changes, breaking live stock and sold-out visibility.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " lastCatalogJson = json;\n io.emit(\"items:update\", { items: catalog });", + "replace": " lastCatalogJson = json;\n // mutant: changed stock is not broadcast" + } + ] + }, + { + "id": "purchase-stock-change-is-not-broadcast--stock-limit", + "scenario": "tracks/ecommerce/scenarios/progression-stock-limit.json", + "targets": [ + "ecommerce.spec.concurrency-safety.stock-limit.3d" + ], + "desc": "Commit purchases without broadcasting their stock changes, breaking live stock and sold-out visibility.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " lastCatalogJson = json;\n io.emit(\"items:update\", { items: catalog });", + "replace": " lastCatalogJson = json;\n // mutant: changed stock is not broadcast" + } + ] + }, + { + "id": "restock-race-records-wrong-order-total", + "scenario": "tracks/ecommerce/scenarios/01-restock-race.json", + "targets": [ + "ecommerce.spec.concurrency-safety.restock-race.202a" + ], + "desc": "Purchases preserve stock and visible order counts but record the wrong booked total. Native mixed-race reconciliation must reject them.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " [accountId, price]\n );", + "replace": " [accountId, Number(price) + 1]\n );" + } + ] + }, + { + "id": "direct-purchase-order-total-is-offset", + "scenario": "tracks/ecommerce/scenarios/progression-purchasing.json", + "targets": [ + "ecommerce.feature.purchasing.purchase-order.3c" + ], + "desc": "Record a direct purchase one dollar above the stored price.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " [accountId, price]\n );", + "replace": " [accountId, Number(price) + 1]\n );" + } + ] + }, + { + "id": "reload-hydrates-an-empty-cart", + "scenario": "tracks/ecommerce/scenarios/01-cart.json", + "targets": [ + "ecommerce.spec.state-durability.cart-reload.4b" + ], + "desc": "Return an empty cart from both reload hydration paths while preserving later live cart broadcasts.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const state = await buildCartState(req.account!.id);\n res.json(state);", + "replace": " await buildCartState(req.account!.id);\n res.json({ items: [], total: 0 });" + }, + { + "find": " const cartState = await buildCartState(acc.id);\n socket.emit(\"cart:update\", cartState);", + "replace": " await buildCartState(acc.id);\n socket.emit(\"cart:update\", { items: [], total: 0 });" + } + ] + }, + { + "id": "signed-out-visitors-do-not-see-reviews", + "scenario": "tracks/ecommerce/scenarios/01-review-visibility.json", + "targets": [ + "ecommerce.feature.reviews.reviews.6a" + ], + "desc": "Hide an item's reviews whenever the viewer is signed out.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " {reviews.length === 0 ? (", + "replace": " {!account || reviews.length === 0 ? (" + } + ] + }, + { + "id": "review-average-update-is-not-broadcast", + "scenario": "tracks/ecommerce/scenarios/01-review-rating-live.json", + "targets": [ + "ecommerce.spec.live-state.rating.6c" + ], + "desc": "Return the new average to the submitter but omit the live review update to other viewers.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " io.emit(\"review:update\", { itemId, reviews, average });", + "replace": " // mutant: other open review views do not receive the new average" + } + ] + }, + { + "id": "admin-warehouse-view-drops-one-location", + "scenario": "tracks/ecommerce/scenarios/01-warehouse-admin-staff.json", + "targets": [ + "ecommerce.feature.warehouse-admin.warehouse-view.7b" + ], + "desc": "Render only 23 of the 24 item-by-warehouse stock locations.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " {admin.locations.map((loc) => {", + "replace": " {admin.locations.slice(0, 23).map((loc) => {" + } + ] + }, + { + "id": "unauthenticated-direct-purchase-uses-default-account", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Remove purchase authentication and attribute unauthenticated requests to a default account.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " \"/api/items/:id/buy\",\n requireAuth,", + "replace": " \"/api/items/:id/buy\"," + }, + { + "find": " const itemId = Number(req.params.id);\n const accountId = req.account!.id;\n\n const client = await pool.connect();", + "replace": " const itemId = Number(req.params.id);\n const accountId = req.account?.id ?? 1;\n\n const client = await pool.connect();" + } + ] + }, + { + "id": "direct-purchase-is-attributed-to-previous-account", + "scenario": "tracks/ecommerce/scenarios/01-purchase-attribution.json", + "targets": [ + "ecommerce.spec.access-control.purchase-attribution.102a" + ], + "desc": "Create the direct-purchase order for the preceding account id rather than the authenticated caller.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const itemId = Number(req.params.id);\n const accountId = req.account!.id;\n\n const client = await pool.connect();", + "replace": " const itemId = Number(req.params.id);\n const accountId = req.account!.id - 1;\n\n const client = await pool.connect();" + } + ] + }, + { + "id": "direct-purchase-uses-constant-price", + "scenario": "tracks/ecommerce/scenarios/01-server-price.json", + "targets": [ + "ecommerce.spec.transactional-integrity.server-price.104a" + ], + "desc": "Create a direct-purchase order at a hard-coded price instead of the current stored price.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " [accountId, price]\n );", + "replace": " [accountId, \"1.00\"]\n );" + } + ] + }, + { + "id": "backend-restart-discards-account-carts", + "scenario": "tracks/ecommerce/scenarios/progression-account-state-reload.json", + "targets": [ + "ecommerce.spec.state-durability.account-state-recovery.105a" + ], + "desc": "Server startup deletes every cart line as leftover session state, so an account cart does not survive a backend restart; accounts, sessions and orders persist, and carts work normally between restarts.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await seed();\n httpServer.listen(PORT, () => {", + "replace": " await seed();\n // mutant: carts are treated as process-lifetime session state\n await pool.query(\"DELETE FROM cart_item\");\n httpServer.listen(PORT, () => {" + } + ] + }, + { + "id": "offline-event-clears-account-state", + "scenario": "tracks/ecommerce/scenarios/progression-account-state-reconnect.json", + "targets": [ + "ecommerce.spec.state-durability.account-state-recovery.105b" + ], + "desc": "Treat a temporary offline event as a sign-out and clear the account and cart state.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " useEffect(() => {\n const socket = io({ path: \"/socket.io\" });", + "replace": " useEffect(() => {\n const clearAccountOffline = () => {\n setAccount(null);\n setCart({ items: [], total: 0 });\n };\n window.addEventListener(\"offline\", clearAccountOffline, { once: true });\n const socket = io({ path: \"/socket.io\" });" + } + ] + }, + { + "id": "purchase-does-not-decrement-warehouse-stock", + "scenario": "tracks/ecommerce/scenarios/progression-books-balance.json", + "targets": [ + "ecommerce.spec.transactional-integrity.books-balance.107b" + ], + "desc": "Create purchase orders without decrementing their selected warehouse stock row.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " UPDATE stock s SET quantity = quantity - 1\n FROM target t", + "replace": " UPDATE stock s SET quantity = quantity\n FROM target t" + } + ] + }, + { + "id": "review-route-skips-purchase-eligibility", + "scenario": "tracks/ecommerce/scenarios/01-review-eligibility.json", + "targets": [ + "ecommerce.spec.access-control.review-eligibility.108a" + ], + "desc": "Allow review creation even when the caller has never purchased the item.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (purchased.rowCount === 0) {", + "replace": " if (false && purchased.rowCount === 0) {" + } + ] + }, + { + "id": "only-shipped-orders-earn-review-eligibility", + "scenario": "tracks/ecommerce/scenarios/01-review-eligibility.json", + "targets": [ + "ecommerce.spec.access-control.review-eligibility.108b", + "ecommerce.spec.access-control.review-eligibility.108a" + ], + "desc": "Incorrectly require an order to be shipped before its buyer may review the item. The same restriction also rejects the required successful buyer control in 108a; it does not independently test non-buyer denial.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " WHERE o.account_id = $1 AND oi.item_id = $2 LIMIT 1`,", + "replace": " WHERE o.account_id = $1 AND oi.item_id = $2 AND o.status = 'shipped' LIMIT 1`," + } + ] + }, + { + "id": "cart-update-accepts-negative-quantity", + "scenario": "tracks/ecommerce/scenarios/01-cart-boundary.json", + "targets": [ + "ecommerce.spec.access-control.cart-boundary.109b" + ], + "desc": "Accept a negative cart quantity update and persist it instead of refusing the named action.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (!Number.isInteger(quantity) || quantity < 1) {", + "replace": " if (!Number.isInteger(quantity)) {" + } + ] + }, + { + "id": "oversell-no-row-lock", + "scenario": "tracks/ecommerce/scenarios/01-last-unit.json", + "targets": [ + "ecommerce.spec.concurrency-safety.last-unit.201a", + "ecommerce.spec.concurrency-safety.last-unit.201b", + "ecommerce.spec.concurrency-safety.last-unit.201c" + ], + "desc": "Drop the row lock so simultaneous buyers can select the same remaining units.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " FOR UPDATE\n LIMIT 1", + "replace": " LIMIT 1" + } + ] + }, + { + "id": "purchase-read-write-loses-concurrent-stock", + "scenario": "tracks/ecommerce/scenarios/01-restock-race.json", + "targets": [ + "ecommerce.spec.concurrency-safety.restock-race.202a" + ], + "desc": "Replace atomic stock reservation with an unlocked read and absolute write. A fixed pause widens scheduling overlap; serial purchases and restocks retain their stock effects. Concurrent reservations or restocking can lose updates.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const decrement = await client.query(\n `WITH target AS (\n SELECT item_id, warehouse_id FROM stock\n WHERE item_id = $1 AND quantity > 0\n ORDER BY warehouse_id\n FOR UPDATE\n LIMIT 1\n )\n UPDATE stock s SET quantity = quantity - 1\n FROM target t\n WHERE s.item_id = t.item_id AND s.warehouse_id = t.warehouse_id\n RETURNING s.item_id, s.warehouse_id`,\n [itemId]\n );\n", + "replace": " const snapshot = await client.query(\n `SELECT item_id, warehouse_id, quantity FROM stock\n WHERE item_id = $1 AND quantity > 0 ORDER BY warehouse_id LIMIT 1`, [itemId]\n );\n // Mutant: widen the unlocked read/write window without changing serial behavior.\n await new Promise(resolve => setTimeout(resolve, 500));\n const decrement = snapshot.rowCount === 0 ? snapshot : await client.query(\n `UPDATE stock SET quantity = $3 WHERE item_id = $1 AND warehouse_id = $2\n RETURNING item_id, warehouse_id`,\n [itemId, snapshot.rows[0].warehouse_id, snapshot.rows[0].quantity - 1]\n );\n" + } + ] + }, + { + "id": "external-stock-polling-disabled", + "scenario": "tracks/ecommerce/scenarios/01-external-live-sync.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock.901a" + ], + "desc": "Stop reconciling direct database changes while the server remains online.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " broadcastCatalog().catch((err) => console.error(\"poll broadcast failed\", err));", + "replace": " // mutant: direct database catalog changes are never reconciled" + } + ] + }, + { + "id": "reconnect-does-not-send-current-catalog", + "scenario": "tracks/ecommerce/scenarios/01-external-reconnect-sync.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock.901d" + ], + "desc": "Follow catalog changes until the browser goes offline, then ignore updates after restoration.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const cartObservationRef = useRef(0);\n\n function applyCartResponse", + "replace": " const cartObservationRef = useRef(0);\n const acceptCatalogUpdates = useRef(true);\n useEffect(() => {\n const stopCatalogRecovery = () => { acceptCatalogUpdates.current = false; };\n window.addEventListener(\"offline\", stopCatalogRecovery);\n return () => window.removeEventListener(\"offline\", stopCatalogRecovery);\n }, []);\n\n function applyCartResponse" + }, + { + "find": " socket.on(\"items:update\", (payload: { items: Item[] }) => setItems(payload.items));", + "replace": " socket.on(\"items:update\", (payload: { items: Item[] }) => {\n if (acceptCatalogUpdates.current) setItems(payload.items);\n });" + } + ] + }, + { + "id": "open-review-list-ignores-live-update", + "scenario": "tracks/ecommerce/scenarios/progression-open-list-live.json", + "targets": [ + "ecommerce.spec.live-state.open-list.902a" + ], + "desc": "Ignore committed review updates in a detail view that is already open.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setItemDetail({ reviews: payload.reviews, average: payload.average });", + "replace": " // mutant: the already-open review list ignores committed updates" + } + ] + }, + { + "id": "open-review-list-renders-each-review-twice", + "scenario": "tracks/ecommerce/scenarios/progression-open-list-live.json", + "targets": [ + "ecommerce.spec.live-state.open-list.902a" + ], + "desc": "Render every committed review twice in the already-open list.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " reviews.map((r) => (", + "replace": " [...reviews, ...reviews].map((r) => (" + } + ] + }, + { + "id": "cancel-does-not-restore-stock-feature", + "scenario": "tracks/ecommerce/scenarios/02-order-cancellation-core.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3a" + ], + "desc": "Cancellation commits but restores zero units to each recorded warehouse row.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " [l.item_id, l.warehouse_id, l.quantity]\n );\n }\n await refundCredit(client, orderRow.rows[0]);\n await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);", + "replace": " [l.item_id, l.warehouse_id, 0]\n );\n }\n await refundCredit(client, orderRow.rows[0]);\n await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);" + } + ] + }, + { + "id": "cancellation-accounting-loses-stock-restoration", + "scenario": "tracks/ecommerce/scenarios/02-invariants.json", + "targets": [ + "ecommerce.returns-pricing.refund-accounting.203a" + ], + "desc": "Cancellation removes revenue and changes order status, but loses the original warehouse stock restoration. The native refund-accounting assertion must detect this.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " [l.item_id, l.warehouse_id, l.quantity]\n );\n }\n await refundCredit(client, orderRow.rows[0]);\n await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);", + "replace": " [l.item_id, l.warehouse_id, 0]\n );\n }\n await refundCredit(client, orderRow.rows[0]);\n await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);" + } + ] + }, + { + "id": "cancel-does-not-restore-stock-fresh-client", + "scenario": "tracks/ecommerce/scenarios/02-self-contained.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202b" + ], + "desc": "Cancellation commits but restores zero units, so a fresh client reads the persisted shortfall.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " [l.item_id, l.warehouse_id, l.quantity]\n );\n }\n await refundCredit(client, orderRow.rows[0]);\n await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);", + "replace": " [l.item_id, l.warehouse_id, 0]\n );\n }\n await refundCredit(client, orderRow.rows[0]);\n await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);" + } + ] + }, + { + "id": "cancel-restores-stock-but-keeps-pending-status", + "scenario": "tracks/ecommerce/scenarios/02-order-cancellation-history.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3b" + ], + "desc": "Cancellation restores allocations but writes pending back to order history.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);", + "replace": "await client.query(`UPDATE orders SET status = 'pending' WHERE id = $1`, [orderId]);" + } + ] + }, + { + "id": "operator-authorization-allows-customer-transfer", + "scenario": "tracks/ecommerce/scenarios/02-strengthened.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201a" + ], + "desc": "The transfer route replaces its administrator gate with ordinary authentication.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\n \"/api/admin/transfer\",\n requireAdmin,", + "replace": "app.post(\n \"/api/admin/transfer\",\n requireAuth," + } + ] + }, + { + "id": "customer-can-ship-order-direct-1-1", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "The shipping route replaces its staff gate with ordinary authentication.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\n \"/api/fulfilment/ship\",\n requireStaff,", + "replace": "app.post(\n \"/api/fulfilment/ship\",\n requireAuth," + } + ] + }, + { + "id": "customer-can-cancel-foreign-order-1-1", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.order-owner.204a" + ], + "desc": "Cancellation retains authentication and pending-state validation but drops order ownership.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\n \"/api/orders/:id/cancel\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const orderId = Number(req.params.id);\n const accountId = req.account!.id;\n const client = await pool.connect();\n try {\n await client.query(\"BEGIN\");\n const orderRow = await client.query(`SELECT * FROM orders WHERE id = $1 FOR UPDATE`, [orderId]);\n if (orderRow.rowCount === 0 || orderRow.rows[0].account_id !== accountId) {", + "replace": "app.post(\n \"/api/orders/:id/cancel\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const orderId = Number(req.params.id);\n const accountId = req.account!.id;\n const client = await pool.connect();\n try {\n await client.query(\"BEGIN\");\n const orderRow = await client.query(`SELECT * FROM orders WHERE id = $1 FOR UPDATE`, [orderId]);\n if (orderRow.rowCount === 0) {" + } + ] + }, + { + "id": "queue-depth-lags-one-order", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-live.json", + "targets": [ + "ecommerce.operations-access.fulfilment-queue.1a" + ], + "desc": "The queue renders every order but its visible depth remains one behind.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "{queue.depth}", + "replace": "{Math.max(0, queue.depth - 1)}" + } + ] + }, + { + "id": "customer-sees-fulfilment-navigation", + "scenario": "tracks/ecommerce/scenarios/02-features.json", + "targets": [ + "ecommerce.operations-access.fulfilment-queue.1d" + ], + "desc": "Expose the protected staff area to signed-in customers, including its navigation and content.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " {account && (account.isStaff || account.isAdmin) && (\n
)}", + "replace": "
)}" + } + ] + }, + { + "id": "purchase-does-not-broadcast-fulfilment-queue", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-live.json", + "targets": [ + "ecommerce.spec.live-state.fulfilment-queue.1a" + ], + "desc": "A direct purchase commits its pending order but omits the fulfilment queue broadcast, so an open staff queue never learns about the new order.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await broadcastCatalog();\n await broadcastOrders(accountId);\n await broadcastFulfilment();\n await broadcastRecommended(accountId);\n res.json({ ok: true });\n })\n);\n\n// ---------- cart routes ----------", + "replace": " await broadcastCatalog();\n await broadcastOrders(accountId);\n // mutant: the new pending order is not pushed to open fulfilment queues\n await broadcastRecommended(accountId);\n res.json({ ok: true });\n })\n);\n\n// ---------- cart routes ----------" + } + ] + }, + { + "id": "admin-state-change-is-not-broadcast", + "scenario": "tracks/ecommerce/scenarios/02-low-stock.json", + "targets": [ + "ecommerce.spec.live-state.inventory-dashboard.5a" + ], + "desc": "Changed admin dashboard state is never broadcast to open admin views, so a customer purchase that drops an item to ten units does not re-enter the low-stock list live.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " lastAdminJson = adminJson;\n io.to(\"admin\").emit(\"admin:update\", adminState);", + "replace": " lastAdminJson = adminJson;\n // mutant: changed admin state is not broadcast" + } + ] + }, + { + "id": "admin-sockets-do-not-join-admin-room", + "scenario": "tracks/ecommerce/scenarios/02-operational-category-totals.json", + "targets": [ + "ecommerce.spec.live-state.sales-dashboard.5b" + ], + "desc": "Admin sockets receive their dashboard state on connection but never join the admin room, so a customer purchase does not update the open category totals live.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (acc.isAdmin) socket.join(\"admin\");", + "replace": " // mutant: admin sockets never join the admin room" + } + ] + }, + { + "id": "transfer-overdraft-guard-skips-bulk-transfers", + "scenario": "tracks/ecommerce/scenarios/02-transfer-overdraw.json", + "targets": [ + "ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c" + ], + "desc": "The insufficient-stock guard is only evaluated for transfers under 1000 units, so a bulk transfer that overdraws the source warehouse commits instead of being refused.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (available < qty) {", + "replace": " if (available < qty && qty < 1000) {" + } + ] + }, + { + "id": "transfer-does-not-publish-warehouse-totals", + "scenario": "tracks/ecommerce/scenarios/02-strengthened.json", + "targets": [ + "ecommerce.inventory-operations.warehouse-transfer.2a" + ], + "desc": "A transfer commits but answers with the pre-transfer admin snapshot and admin state is never broadcast, so the open warehouse totals do not move.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const client = await pool.connect();\n try {\n await client.query(\"BEGIN\");\n // Lock both warehouse rows", + "replace": " const state = await buildAdminState();\n const client = await pool.connect();\n try {\n await client.query(\"BEGIN\");\n // Lock both warehouse rows" + }, + { + "find": " await broadcastCatalog();\n const state = await buildAdminState();\n res.json(state);\n })\n);\n\napp.post(\n \"/api/admin/price\",", + "replace": " await broadcastCatalog();\n res.json(state);\n })\n);\n\napp.post(\n \"/api/admin/price\"," + }, + { + "find": " lastAdminJson = adminJson;\n io.to(\"admin\").emit(\"admin:update\", adminState);", + "replace": " lastAdminJson = adminJson;\n // mutant: changed admin state is not broadcast" + } + ] + }, + { + "id": "credit-checkout-ignores-wallet", + "desc": "A credit checkout pays entirely externally despite available credit.", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.feature.store-credit.store-credit-750.750a" + ], + "file": "server/src/credit.ts", + "edits": [ + { + "find": " const creditMinor = Math.min(Number(account.rows[0].credit_minor), totalMinor);", + "replace": " const creditMinor = 0;" + } + ] + }, + { + "id": "credit-grant-replay-increments-balance", + "desc": "Replaying a grant applies its credit to the wallet again.", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.spec.store-credit.production-752.752a" + ], + "file": "server/src/credit.ts", + "edits": [ + { + "find": " if (!existing.rows.length) {", + "replace": " if (existing.rows.length) await client.query('UPDATE account SET credit_minor=credit_minor+$1 WHERE id=$2', [amountMinor, accountId]);\n if (!existing.rows.length) {" + } + ] + }, + { + "id": "customer-can-grant-credit", + "desc": "Customer authentication is accepted without staff authorization.", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.spec.store-credit.production-753.753a" + ], + "file": "server/src/credit.ts", + "edits": [ + { + "find": " app.post('/api/staff/credit', auth, staff, async (req, res) => {", + "replace": " app.post('/api/staff/credit', auth, async (req, res) => {" + } + ] + }, + { + "id": "split-refund-does-not-restore-credit", + "desc": "The refund is recorded but its original wallet credit is not restored.", + "scenario": "tracks/ecommerce/scenarios/progression-split-tender-refunds.json", + "targets": [ + "ecommerce.feature.split-tender-refunds.split-tender-refunds-751.751a" + ], + "file": "server/src/progression.ts", + "edits": [ + { + "find": " await refundCredit(client, order.rows[0]);", + "replace": " // mutant: omit wallet restoration" + } + ] + }, + { + "id": "split-refund-duplicates-credit", + "desc": "A refund credits the wallet twice while recording one refund.", + "scenario": "tracks/ecommerce/scenarios/progression-split-tender-refunds.json", + "targets": [ + "ecommerce.spec.split-tender-refunds.production-756.756a" + ], + "file": "server/src/credit.ts", + "edits": [ + { + "find": "[delta, order.account_id]);", + "replace": "[delta * 2, order.account_id]);" + } + ] + }, + { + "id": "subscription-skips-due-purchase", + "desc": "Due deliveries are recorded as skipped although stock is available.", + "scenario": "tracks/ecommerce/scenarios/progression-subscriptions.json", + "targets": [ + "ecommerce.feature.subscriptions.subscriptions-760.760a" + ], + "file": "server/src/subscriptions.ts", + "edits": [ + { + "find": " if (stock.rows.reduce((sum, row) => sum + row.quantity, 0) >= subscription.quantity) {", + "replace": " if (false && stock.rows.reduce((sum, row) => sum + row.quantity, 0) >= subscription.quantity) {" + } + ] + }, + { + "id": "subscription-allows-foreign-cancellation", + "desc": "A customer can cancel another customer subscription.", + "scenario": "tracks/ecommerce/scenarios/progression-subscriptions.json", + "targets": [ + "ecommerce.spec.subscriptions.production-762.762a" + ], + "file": "server/src/subscriptions.ts", + "edits": [ + { + "find": " if (!row || row.account_id !== req.account!.id) {", + "replace": " if (!row) {" + } + ] + }, + { + "id": "subscription-pause-is-not-recorded", + "desc": "Pause acknowledges the request but the subscription remains active.", + "scenario": "tracks/ecommerce/scenarios/progression-subscriptions.json", + "targets": [ + "ecommerce.spec.subscriptions.production-763.763a" + ], + "file": "server/src/subscriptions.ts", + "edits": [ + { + "find": "UPDATE purchase_subscription SET status='paused',paused_at=now() WHERE id=$1", + "replace": "UPDATE purchase_subscription SET status='active',paused_at=now() WHERE id=$1" + } + ] + }, + { + "id": "credit-checkout-retains-purchased-cart", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.spec.store-credit.production-754.754a" + ], + "desc": "The purchased cart remains available instead of being consumed by checkout.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " await client.query(`DELETE FROM cart_item WHERE cart_id = $1`, [cart.rows[0].id]);", + "replace": " // Mutation: keep checked-out cart lines." + } + ] + }, + { + "id": "pending-subscriptions-are-cleared-at-startup", + "desc": "Restart erases pending subscription work while preserving ordinary timer execution.", + "file": "server/src/subscriptions.ts", + "scenario": "tracks/ecommerce/scenarios/progression-subscriptions.json", + "targets": [ + "ecommerce.spec.subscriptions.production-761.761a" + ], + "edits": [ + { + "find": " `);\n}\n\nexport function registerSubscriptions", + "replace": " `);\n await pool.query(\"UPDATE purchase_subscription SET status='cancelled' WHERE status='active'\");\n}\n\nexport function registerSubscriptions" + } + ] + }, + { + "id": "credit-balance-is-cleared-at-startup", + "desc": "Restart clears an issued wallet balance while leaving accounts present.", + "file": "server/src/credit.ts", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.spec.store-credit.production-755.755a" + ], + "edits": [ + { + "find": " `);\n}\n\nexport async function spendCredit", + "replace": " `);\n await pool.query(\"UPDATE account SET credit_minor=0\");\n}\n\nexport async function spendCredit" + } + ] + }, + { + "id": "bundle-definition-loses-component-quantity", + "scenario": "tracks/ecommerce/scenarios/progression-product-bundles.json", + "targets": [ + "ecommerce.feature.product-bundles.product-bundles.740a" + ], + "desc": "definition loses component quantity", + "file": "server/src/bundles.ts", + "edits": [ + { + "find": "values.push({ ...component, itemId: item.rows[0].id });", + "replace": "values.push({ ...component, quantity: 1, itemId: item.rows[0].id });" + } + ] + }, + { + "id": "bundle-catalog-write-allows-customers", + "scenario": "tracks/ecommerce/scenarios/progression-product-bundles.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-743.743a" + ], + "desc": "catalog write allows customers", + "file": "server/src/bundles.ts", + "edits": [ + { + "find": "if (!actor?.is_admin && actor?.staff_role !== 'catalog')", + "replace": "if (false)" + } + ] + }, + { + "id": "bundle-checkout-price-not-snapshot", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-checkout.json", + "targets": [ + "ecommerce.feature.bundle-checkout.bundle-checkout.741a" + ], + "desc": "checkout price not snapshot", + "file": "server/src/bundles.ts", + "edits": [ + { + "find": "[cart.rows[0].id, bundleId, bundle.price, JSON.stringify(allocations), JSON.stringify(bundle.bundle_components)]", + "replace": "[cart.rows[0].id, bundleId, bundle.price + 1, JSON.stringify(allocations), JSON.stringify(bundle.bundle_components)]" + } + ] + }, + { + "id": "bundle-expiry-does-not-release-components", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-checkout.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-746.746a" + ], + "desc": "expiry does not release components", + "file": "server/src/progression.ts", + "edits": [ + { + "find": "await releaseBundle(client, bundle.rows[0].component_allocations);", + "replace": "/* mutant: component holds leak after expiration */" + } + ] + }, + { + "id": "bundle-return-loses-original-components", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-returns.json", + "targets": [ + "ecommerce.feature.bundle-returns.bundle-returns.742a" + ], + "desc": "return loses original components", + "file": "server/src/bundles.ts", + "edits": [ + { + "find": "await releaseBundle(client, line.component_allocations);", + "replace": "/* mutant: purchased components are not restored */" + } + ] + }, + { + "id": "bundle-return-replay-restocks-again", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-returns.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-742.742b" + ], + "desc": "return replay restocks again", + "file": "server/src/bundles.ts", + "edits": [ + { + "find": "WHERE order_id=$1 AND is_bundle AND NOT returned FOR UPDATE", + "replace": "WHERE order_id=$1 AND is_bundle FOR UPDATE" + } + ] + }, + { + "id": "bundle-return-crosses-account-boundary", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-returns.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-748.748a" + ], + "desc": "return crosses account boundary", + "file": "server/src/bundles.ts", + "edits": [ + { + "find": "WHERE id=$1 AND account_id=$2 AND status IN ('shipped','delivered')", + "replace": "WHERE id=$1 AND $2::integer=$2::integer AND status IN ('shipped','delivered')" + } + ] + }, + { + "id": "bundle-components-can-overdraw", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-checkout.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-744.744a", + "ecommerce.spec.bundle-integrity.bundle-745.745a" + ], + "desc": "components can overdraw", + "file": "server/src/bundles.ts", + "edits": [ + { + "find": "if (rows.rows.reduce((sum, row) => sum + row.quantity, 0) < component.quantity) throw new Error('A component is unavailable');", + "replace": "// mutant: incomplete component reservation is accepted" + } + ] + }, + { + "id": "bundle-checkout-reuses-reservation", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-checkout.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-747.747a" + ], + "desc": "checkout reuses reservation", + "file": "server/src/progression.ts", + "edits": [ + { + "find": "await client.query(`DELETE FROM cart_item WHERE cart_id = $1`, [cart.rows[0].id]);", + "replace": "// mutant: cart survives checkout" + } + ] + }, + { + "id": "return-after-support-refund-is-blocked", + "scenario": "tracks/ecommerce/scenarios/progression-support-return-interaction.json", + "targets": [ + "ecommerce.feature.split-tender-refunds.return-refund-interaction.757a" + ], + "desc": "Reject a valid physical return after a financial refund.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (lineRow.rows[0].returned) {", + "replace": " if (lineRow.rows[0].returned || Number(orderRow.rows[0].refund_total) > 0) {" + } + ] + }, + { + "id": "support-refund-after-return-pays-twice", + "scenario": "tracks/ecommerce/scenarios/progression-support-return-interaction.json", + "targets": [ + "ecommerce.feature.split-tender-refunds.return-refund-interaction.757b" + ], + "desc": "Pay the full order again after a physical return.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " await client.query(`UPDATE orders SET refund_total = total, status = CASE", + "replace": " await client.query(`UPDATE orders SET refund_total = refund_total + total, status = CASE" + } + ] + }, + { + "id": "progression-support-history-anonymous-leak", + "scenario": "tracks/ecommerce/scenarios/progression-support-history.json", + "targets": [ + "ecommerce.spec.access-control.support-history-logout.612d" + ], + "desc": "Return private support subjects to signed-out visitors while preserving authenticated ownership filtering.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " if (!account) return [];\n const where = account.isAdmin || account.isStaff ? \"TRUE\" : \"sc.account_id = $1\";\n const values = account.isAdmin || account.isStaff ? [] : [account.id];", + "replace": " const where = !account || account.isAdmin || account.isStaff ? \"TRUE\" : \"sc.account_id = $1\";\n const values = !account || account.isAdmin || account.isStaff ? [] : [account.id];" + } + ] + }, + { + "id": "checkout-crash-integrity", + "scenario": "tracks/ecommerce/scenarios/progression-checkout-crash.json", + "targets": [ + "ecommerce.spec.state-durability.checkout-crash-integrity.910a" + ], + "desc": "A lifecycle callback changes a prepared or cleared cart to quantity two, leaving neither a legal rollback nor a complete checkout.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await initializeProgressionSchema(pool);", + "replace": " await initializeProgressionSchema(pool);\n await pool.query(`INSERT INTO cart_item (cart_id, item_id, quantity) SELECT c.id, MIN(oi.item_id), 2 FROM cart c JOIN orders o ON o.account_id=c.account_id JOIN order_item oi ON oi.order_id=o.id GROUP BY c.id ON CONFLICT (cart_id,item_id) DO UPDATE SET quantity=2`);" + } + ] + }, + { + "id": "checkout-crash-sibling-stock", + "scenario": "tracks/ecommerce/scenarios/progression-checkout-crash.json", + "targets": [ + "ecommerce.spec.state-durability.checkout-crash-integrity.910a" + ], + "desc": "A recovery callback removes stock for Coffee Grinder while leaving cart and order data intact. The second-product stock discrepancy must fail crash atomicity.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await initializeProgressionSchema(pool);", + "replace": " await initializeProgressionSchema(pool);\n await pool.query(`UPDATE stock SET quantity=quantity-1 WHERE item_id=(SELECT id FROM item WHERE name='Coffee Grinder') AND EXISTS (SELECT 1 FROM orders)`);" + } + ] + }, + { + "id": "checkout-crash-durability", + "scenario": "tracks/ecommerce/scenarios/progression-checkout-crash.json", + "targets": [ + "ecommerce.spec.state-durability.checkout-crash-durability.910b" + ], + "desc": "A lifecycle callback corrupts earlier order totals. The unchanged prepared snapshots and acknowledged baseline must expose the damage.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await initializeProgressionSchema(pool);", + "replace": " await initializeProgressionSchema(pool);\n await pool.query('UPDATE orders SET total=0');" + } + ] + }, + { + "id": "review-script-unsafe-render", + "scenario": "tracks/ecommerce/scenarios/progression-review-script.json", + "targets": [ + "ecommerce.progression.review-access-specifications.stored-review-script.9180a" + ], + "desc": "Render stored review text as HTML so it executes in another customer session.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "
{r.comment}
", + "replace": "
" + } + ] + }, + { + "id": "review-script-reject-all", + "scenario": "tracks/ecommerce/scenarios/progression-review-script.json", + "targets": [ + "ecommerce.progression.review-access-specifications.stored-review-script.9180a" + ], + "desc": "Reject all nonempty reviews, including the required benign review; absence of script is insufficient.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "const { rating, comment } = req.body ?? {};", + "replace": "const { rating, comment } = req.body ?? {};\n if (String(comment).length > 0) { res.status(400).json({ error: \"Review rejected\" }); return; }" + } + ] + }, + { + "id": "review-owner-trust-username", + "scenario": "tracks/ecommerce/scenarios/progression-review-access.json", + "targets": [ + "ecommerce.progression.review-access-specifications.review-eligibility-direct.618a" + ], + "desc": "Trust the submitted buyer username instead of the authenticated caller.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "\"/api/items/:id/reviews\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const itemId = Number(req.params.id);\n const accountId = req.account!.id;", + "replace": "\"/api/items/:id/reviews\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const itemId = Number(req.params.id);\n const accountId = req.body?.username ? (await pool.query(\"SELECT id FROM account WHERE username = $1\", [req.body.username])).rows[0]?.id : req.account!.id;" + } + ] + }, + { + "id": "review-owner-deny-after-write", + "scenario": "tracks/ecommerce/scenarios/progression-review-access.json", + "targets": [ + "ecommerce.progression.review-access-specifications.review-eligibility-direct.618a" + ], + "desc": "Replace the buyer review under a forged username, then return a refusal.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "\"/api/items/:id/reviews\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const itemId = Number(req.params.id);\n const accountId = req.account!.id;", + "replace": "\"/api/items/:id/reviews\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const itemId = Number(req.params.id);\n const accountId = req.body?.username ? (await pool.query(\"SELECT id FROM account WHERE username = $1\", [req.body.username])).rows[0]?.id : req.account!.id;" + }, + { + "find": "const { reviews, average } = await buildItemReviews(itemId);\n io.emit(\"review:update\", { itemId, reviews, average });", + "replace": "if (req.body?.username) { res.status(403).json({ error: \"Refused after write\" }); return; }\n const { reviews, average } = await buildItemReviews(itemId);\n io.emit(\"review:update\", { itemId, reviews, average });" + } + ] + }, + { + "id": "review-owner-reject-all", + "scenario": "tracks/ecommerce/scenarios/progression-review-access.json", + "targets": [ + "ecommerce.progression.review-access-specifications.review-eligibility-direct.618a" + ], + "desc": "Reject the legitimate buyer review as well as unauthorized reviews.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "const { rating, comment } = req.body ?? {};", + "replace": "const { rating, comment } = req.body ?? {};\n if (String(comment).length > 0) { res.status(403).json({ error: \"Reject all reviews\" }); return; }" + } + ] + }, + { + "id": "password-verification-truncates-utf8", + "scenario": "tracks/ecommerce/scenarios/01-account-password.json", + "targets": [ + "ecommerce.spec.access-control.account-password.1c" + ], + "desc": "Hash and verify only the first 72 UTF-8 bytes, accepting a different password suffix.", + "file": "server/src/auth.ts", + "edits": [ + { + "find": "scrypt(password, salt, 64,", + "replace": "scrypt(Buffer.from(password).subarray(0, 72), salt, 64," + } + ] + }, + { + "id": "password-verification-rejects-every-login", + "scenario": "tracks/ecommerce/scenarios/01-account-password.json", + "targets": [ + "ecommerce.spec.access-control.account-password.1c" + ], + "desc": "Refuse even a correct password; refusal alone must not earn credit.", + "file": "server/src/auth.ts", + "edits": [ + { + "find": "return crypto.timingSafeEqual", + "replace": "return false && crypto.timingSafeEqual" + } + ] + }, + { + "id": "tampered-session-can-purchase", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Accept a modified session credential and allow its purchase.", + "file": "server/src/index.ts", + "edits": [ + { + "find": ".where(eq(session.id, token))\n .limit(1)", + "replace": ".where(sql`true`)\n .limit(1)" + } + ] + }, + { + "id": "restock-trusts-submitted-admin-role", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Trust an administrator role supplied by staff in the restock request. Normal staff requests are still refused.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\n \"/api/admin/restock\",\n requireAdmin,", + "replace": "app.post(\n \"/api/admin/restock\",\n requireAuth,\n (req, res, next) => req.body?.role === \"admin\" ? next() : requireAdmin(req, res, next)," + } + ] + }, + { + "id": "restock-admin-claim-commits-then-refuses", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Trust the submitted administrator role, change stored stock, then return 403.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\n \"/api/admin/restock\",\n requireAdmin,", + "replace": "app.post(\n \"/api/admin/restock\",\n requireAuth,\n (req, res, next) => req.body?.role === \"admin\" ? next() : requireAdmin(req, res, next)," + }, + { + "find": " await processImmediateRestock(itemId);\n await broadcastCatalog();\n const state = await buildAdminState();\n res.json(state);", + "replace": " await processImmediateRestock(itemId);\n await broadcastCatalog();\n const state = await buildAdminState();\n if (req.body?.role === \"admin\") { res.status(403).json({ error: \"refused after write\" }); return; }\n res.json(state);" + } + ] + }, + { + "id": "restock-rejects-administrator-control", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.feature.warehouse-admin.admin-write.103a", + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Reject every direct restock, including the legitimate administrator control.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\n \"/api/admin/restock\",\n requireAdmin,", + "replace": "app.post(\n \"/api/admin/restock\",\n (_req, res) => { res.status(403).json({ error: \"restock disabled\" }); }," + } + ] + }, + { + "id": "direct-purchase-trusts-submitted-price", + "scenario": "tracks/ecommerce/scenarios/01-server-price.json", + "targets": [ + "ecommerce.spec.transactional-integrity.server-price.104a" + ], + "desc": "Ordinary purchases use the catalog price; an extra price field overrides the booked order and line price.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const { name, price } = itemRow.rows[0];", + "replace": " const { name } = itemRow.rows[0];\n const price = req.body?.price ?? itemRow.rows[0].price;" + } + ] + }, + { + "id": "direct-purchase-price-commits-then-refuses", + "scenario": "tracks/ecommerce/scenarios/01-server-price.json", + "targets": [ + "ecommerce.spec.transactional-integrity.server-price.104a" + ], + "desc": "A request with an extra price field commits an otherwise correct purchase, then returns 403.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await broadcastRecommended(accountId);\n res.json({ ok: true });\n })\n);\n\n// ---------- cart routes", + "replace": " await broadcastRecommended(accountId);\n if (req.body?.price !== undefined) { res.status(403).json({ error: \"Price denied\" }); return; }\n res.json({ ok: true });\n })\n);\n\n// ---------- cart routes" + } + ] + }, + { + "id": "direct-purchase-rejects-all", + "scenario": "tracks/ecommerce/scenarios/01-server-price.json", + "targets": [ + "ecommerce.spec.transactional-integrity.server-price.104a" + ], + "desc": "Reject every direct purchase, including the valid purchase controls.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " \"/api/items/:id/buy\",\n requireAuth,", + "replace": " \"/api/items/:id/buy\",\n (_req, res) => { res.status(403).json({ error: \"Purchases denied\" }); },\n requireAuth," + } + ] + }, + { + "id": "order-cache-survives-account-switch", + "scenario": "tracks/ecommerce/scenarios/01-order-ownership.json", + "targets": [ + "ecommerce.spec.access-control.order-ownership.106a" + ], + "desc": "Merge order updates into a browser cache that is never cleared when the account changes. Separate-browser ownership still works.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const [orders, setOrders] = useState([]);", + "replace": " const [orders, setStoredOrders] = useState([]);\n const setOrders: typeof setStoredOrders = update => setStoredOrders(previous => {\n const incoming = typeof update === 'function' ? update(previous) : update;\n return [...new Map([...previous, ...incoming].map(order => [order.id, order])).values()];\n });" + } + ] + }, + { + "id": "order-history-is-always-empty", + "scenario": "tracks/ecommerce/scenarios/01-order-ownership.json", + "targets": [ + "ecommerce.spec.access-control.order-ownership.106a" + ], + "desc": "Hide all orders, including the required own-account positive views.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "FROM orders WHERE account_id = $1 ORDER BY created_at DESC`,", + "replace": "FROM orders WHERE account_id = $1 AND FALSE ORDER BY created_at DESC`," + } + ] + }, + { + "id": "checkout-trusts-submitted-price", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "Book the submitted checkout price instead of the stored price.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await checkoutReservedCart(accountId);", + "replace": " await checkoutReservedCart(accountId);\n if (req.body?.price !== undefined) await pool.query('UPDATE orders SET total=$1 WHERE id=(SELECT MAX(id) FROM orders WHERE account_id=$2)', [req.body.price, accountId]);" + } + ] + }, + { + "id": "checkout-commits-then-refuses-price", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "Commit checkout before refusing the extra price field.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await checkoutReservedCart(accountId);", + "replace": " await checkoutReservedCart(accountId);\n if (req.body?.price !== undefined) { res.status(400).json({ error: \"Invalid price field\" }); return; }" + } + ] + }, + { + "id": "checkout-rejects-all-price-control", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "Refuse all checkouts, including the normal positive path.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " \"/api/checkout\",\n requireAuth,", + "replace": " \"/api/checkout\",\n (_req, res) => { res.status(400).json({ error: \"Checkout denied\" }); },\n requireAuth," + } + ] + }, + { + "id": "checkout-missing-sibling-line", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "Commit the cart checkout with one product line missing while retaining the other product and the full order total.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await checkoutReservedCart(accountId);", + "replace": " await checkoutReservedCart(accountId);\n await pool.query(\"DELETE FROM order_item WHERE order_id=(SELECT MAX(id) FROM orders WHERE account_id=$1) AND item_name='Coffee Grinder'\", [accountId]);" + } + ] + }, + { + "id": "shipping-cancellation-cancel-shipped", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Cancellation accepts a shipped order and restores its stock after shipping.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "if (orderRow.rows[0].status !== \"pending\")", + "replace": "if (false)" + } + ] + }, + { + "id": "shipping-cancellation-resurrect-cancelled", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Shipping accepts a cancelled order after its stock was restored.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "WHERE id = $1 AND status = 'pending' RETURNING account_id", + "replace": "WHERE id = $1 RETURNING account_id" + } + ] + }, + { + "id": "shipping-cancellation-combined-race", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Cancellation restores stock but records a shipped status, combining mutually exclusive effects.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "if (orderRow.rows[0].status !== \"pending\")", + "replace": "if (false)" + }, + { + "find": "UPDATE orders SET status = 'cancelled' WHERE id = $1", + "replace": "UPDATE orders SET status = 'shipped' WHERE id = $1" + } + ] + }, + { + "id": "shipping-cancellation-reject-both", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Both cancellation and shipping refuse pending orders; useful progress must remain required.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "if (orderRow.rows[0].status !== \"pending\")", + "replace": "if (true)" + }, + { + "find": "WHERE id = $1 AND status = 'pending' RETURNING account_id", + "replace": "WHERE id = $1 AND false RETURNING account_id" + } + ] + }, + { + "id": "shipping-cancellation-queue-retains-removed-order", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "The live staff queue accepts new orders but retains rows after cancellation; the database state remains correct.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " socket.on(\"queue:update\", (payload: QueueState) => setQueue(payload));", + "replace": " socket.on(\"queue:update\", (payload: QueueState) => setQueue(previous => payload.queue.length < previous.queue.length ? previous : payload));" + } + ] + }, + { + "id": "opposing-transfer-ignored", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202d" + ], + "desc": "One valid opposing transfer is acknowledged without moving stock.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const { itemId, fromWarehouseId, toWarehouseId, quantity } = req.body ?? {};", + "replace": " const { itemId, fromWarehouseId, toWarehouseId, quantity } = req.body ?? {};\n if (Number(quantity) === 7) return res.json({});" + } + ] + }, + { + "id": "opposing-transfer-rejected", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202d" + ], + "desc": "One valid opposing transfer is refused despite ample source stock.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const { itemId, fromWarehouseId, toWarehouseId, quantity } = req.body ?? {};", + "replace": " const { itemId, fromWarehouseId, toWarehouseId, quantity } = req.body ?? {};\n if (Number(quantity) === 7) return res.status(409).json({ error: \"Transfer disabled\" });" + } + ] + }, + { + "id": "overlapping-cart-add-ignored", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "The overlapping second-product cart add is ignored while ordinary cart setup still works.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " try {\n await reserveCartItem(accountId, itemId, qty);", + "replace": " if (req.account!.username.includes(\"checkout-overlap\") && (await db.select().from(item).where(eq(item.id, itemId)))[0]?.name === \"Coffee Grinder\") return res.json({});\n try {\n await reserveCartItem(accountId, itemId, qty);" + } + ] + }, + { + "id": "overlapping-cart-add-rejected", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "The overlapping second-product cart add is rejected while ordinary cart setup still works.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " try {\n await reserveCartItem(accountId, itemId, qty);", + "replace": " if (req.account!.username.includes(\"checkout-overlap\") && (await db.select().from(item).where(eq(item.id, itemId)))[0]?.name === \"Coffee Grinder\") return res.status(409).json({ error: \"Cart add disabled\" });\n try {\n await reserveCartItem(accountId, itemId, qty);" + } + ] + }, + { + "id": "mixed-history-lost-transfer", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: lost-transfer. The stored-state comparison must catch the defect at its first effect.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await client.query(\n `INSERT INTO stock (item_id, warehouse_id, quantity) VALUES ($1, $2, $3)\n ON CONFLICT (item_id, warehouse_id) DO UPDATE SET quantity = stock.quantity + excluded.quantity`,\n [itemId, toWarehouseId, qty]\n );", + "replace": " await client.query(\n `INSERT INTO stock (item_id, warehouse_id, quantity) VALUES ($1, $2, $3)\n ON CONFLICT (item_id, warehouse_id) DO UPDATE SET quantity = stock.quantity + excluded.quantity`,\n [itemId, toWarehouseId, 0]\n );" + } + ] + }, + { + "id": "mixed-history-wrong-price", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: wrong-price. The stored-state comparison must catch the defect at its first effect.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " [accountId, price]\n );", + "replace": " [accountId, \"1.00\"]\n );" + } + ] + }, + { + "id": "mixed-history-ignored-write", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: ignored-write. The stored-state comparison must catch the defect at its first effect.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " try {\n await reserveCartItem(accountId, itemId, qty);", + "replace": " if (req.account!.username.includes(\"history-\") && (await db.select().from(item).where(eq(item.id, itemId)))[0]?.name === \"Coffee Grinder\") return res.json({});\n try {\n await reserveCartItem(accountId, itemId, qty);" + } + ] + }, + { + "id": "mixed-history-reject-valid-write", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: reject-valid-write. The stored-state comparison must catch the defect at its first effect.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " try {\n await reserveCartItem(accountId, itemId, qty);", + "replace": " if (req.account!.username.includes(\"history-\") && (await db.select().from(item).where(eq(item.id, itemId)))[0]?.name === \"Coffee Grinder\") return res.status(409).json({ error: \"Cart add disabled\" });\n try {\n await reserveCartItem(accountId, itemId, qty);" + } + ] + }, + { + "id": "mixed-history-repeat-cancel", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: repeat-cancel. The stored-state comparison must catch the defect at its first effect.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (orderRow.rows[0].status !== \"pending\") {", + "replace": " if (![\"pending\", \"cancelled\"].includes(orderRow.rows[0].status)) {" + } + ] + }, + { + "id": "mixed-history-wrong-owner", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: wrong-owner. The stored-state comparison must catch the defect at its first effect.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const itemId = Number(req.params.id);\n const accountId = req.account!.id;\n\n const client = await pool.connect();", + "replace": " const itemId = Number(req.params.id);\n const accountId = req.account!.id - 1;\n\n const client = await pool.connect();" + } + ] + }, + { + "id": "checkout-empty-replay-creates-order", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Allow retrying the consumed empty cart to create another order after a lost reply.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " if (lines.rows.length === 0) throw new Error(\"cart is empty\");", + "replace": "// Defect: an empty consumed cart may create another order." + } + ] + }, + { + "id": "checkout-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Reject every valid checkout; refusal cannot earn duplicate protection credit.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " if (lines.rows.length === 0) throw new Error(\"cart is empty\");", + "replace": " if (lines.rows.length === 0) throw new Error(\"cart is empty\");\n throw new Error('Checkout disabled');" + } + ] + }, + { + "id": "checkout-replay-loses-prior-order", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Retrying the consumed cart deletes an earlier acknowledged order.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " if (lines.rows.length === 0) throw new Error(\"cart is empty\");", + "replace": " if (lines.rows.length === 0) { await client.query(\"DELETE FROM orders WHERE id = (SELECT min(id) FROM orders WHERE account_id = $1)\", [accountId]); await client.query(\"COMMIT\"); return 0; }" + } + ] + }, + { + "id": "checkout-clears-cart-without-order", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Checkout returns success and empties the cart without recording an order.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " if (lines.rows.length === 0) throw new Error(\"cart is empty\");", + "replace": " if (lines.rows.length === 0) throw new Error(\"cart is empty\");\n if (lines.rows.length > 0) { await client.query(\"DELETE FROM cart_item WHERE cart_id = $1\", [cart.rows[0].id]); await client.query(\"COMMIT\"); return 0; }" + } + ] + }, + { + "id": "revenue-aggregation-truncates-order-population", + "scenario": "tracks/ecommerce/scenarios/progression-books-balance.json", + "targets": [ + "ecommerce.spec.transactional-integrity.books-balance.107a" + ], + "desc": "Calculate revenue from only the first 256 stored orders while preserving all acknowledged purchases.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "COALESCE((SELECT SUM(total - refund_total) FROM orders WHERE status != 'cancelled'), 0)", + "replace": "COALESCE((SELECT SUM(total - refund_total) FROM (SELECT * FROM orders WHERE status != 'cancelled' ORDER BY id LIMIT 256) limited_orders), 0)" + } + ] + }, + { + "id": "catalog-truncates-created-population", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-volume.json", + "targets": [ + "ecommerce.spec.transactional-integrity.catalog-volume.622c" + ], + "desc": "Omit committed products after the first 256 search results.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": ".slice(searchPage * CATALOG_PAGE_SIZE,", + "replace": ".slice(0, 256).slice(searchPage * CATALOG_PAGE_SIZE," + } + ] + }, + { + "id": "catalog-discards-product-writes", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-volume.json", + "targets": [ + "ecommerce.spec.transactional-integrity.catalog-volume.622c" + ], + "desc": "Acknowledge catalog creation without committing a product.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " app.post(\"/api/catalog/products\", dependencies.requireStaff, asyncRoute(async (req, res) => {", + "replace": " app.post(\"/api/catalog/products\", dependencies.requireStaff, asyncRoute(async (req, res) => {\n if (req.body) { res.json({ok:true}); return; } // mutant: successful response without product" + } + ] + }, + { + "id": "catalog-rejects-all-product-writes", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-volume.json", + "targets": [ + "ecommerce.spec.transactional-integrity.catalog-volume.622c" + ], + "desc": "Reject every catalog creation while keeping sign-in and catalog reads available.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " app.post(\"/api/catalog/products\", dependencies.requireStaff, asyncRoute(async (req, res) => {", + "replace": " app.post(\"/api/catalog/products\", dependencies.requireStaff, asyncRoute(async (req, res) => {\n res.status(403).json({error:'Catalog writes refused'}); return;" + } + ] + }, + { + "id": "shipping-trusts-submitted-staff-role", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "Trust a staff role supplied by a customer; ordinary customers remain refused.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\n \"/api/fulfilment/ship\",\n requireStaff,", + "replace": "app.post(\n \"/api/fulfilment/ship\",\n (req, res, next) => req.body?.role === \"staff\" ? requireAuth(req, res, next) : requireStaff(req, res, next)," + } + ] + }, + { + "id": "shipping-claim-commits-then-refuses", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "Ship the order using the claimed role, then return 403. A refusal response cannot hide the stored effect.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\n \"/api/fulfilment/ship\",\n requireStaff,", + "replace": "app.post(\n \"/api/fulfilment/ship\",\n (req, res, next) => req.body?.role === \"staff\" ? requireAuth(req, res, next) : requireStaff(req, res, next)," + }, + { + "find": " const accountId = result.rows[0].account_id;", + "replace": " if (req.body?.role === \"staff\") { res.status(403).json({error:\"Shipping refused\"}); return; }\n const accountId = result.rows[0].account_id;" + } + ] + }, + { + "id": "shipping-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "Reject legitimate staff shipping as well as customer requests.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\n \"/api/fulfilment/ship\",\n requireStaff,", + "replace": "app.post(\n \"/api/fulfilment/ship\",\n (_req, res) => { res.status(403).json({ error: \"Shipping refused\" }); }," + } + ] + }, + { + "id": "cancel-trusts-submitted-owner", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.order-owner.204a" + ], + "desc": "Trust a username submitted by another signed-in customer as the order owner.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\n \"/api/orders/:id/cancel\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const orderId = Number(req.params.id);\n const accountId = req.account!.id;", + "replace": "app.post(\n \"/api/orders/:id/cancel\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const orderId = Number(req.params.id);\n const claimed = typeof req.body?.username === 'string' ? (await pool.query('SELECT id FROM account WHERE username = $1', [req.body.username])).rows[0] : null;\n const accountId = claimed?.id ?? req.account!.id;" + } + ] + }, + { + "id": "cancel-claim-commits-then-refuses", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.order-owner.204a" + ], + "desc": "Cancel using the submitted owner, then return 403; fresh state must expose the unauthorized effect.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\n \"/api/orders/:id/cancel\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const orderId = Number(req.params.id);\n const accountId = req.account!.id;", + "replace": "app.post(\n \"/api/orders/:id/cancel\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const orderId = Number(req.params.id);\n const claimed = typeof req.body?.username === 'string' ? (await pool.query('SELECT id FROM account WHERE username = $1', [req.body.username])).rows[0] : null;\n const accountId = claimed?.id ?? req.account!.id;" + }, + { + "find": " await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);\n await client.query(\"COMMIT\");", + "replace": " await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);\n await client.query(\"COMMIT\");\n if (req.body?.username) { res.status(403).json({error:\"Cancellation refused\"}); return; }" + } + ] + }, + { + "id": "cancel-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.order-owner.204a" + ], + "desc": "Reject legitimate cancellation as well as foreign requests.", + "file": "server/src/index.ts", + "edits": [ + { + "find": "app.post(\n \"/api/orders/:id/cancel\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const orderId = Number(req.params.id);\n const accountId = req.account!.id;", + "replace": "app.post(\n \"/api/orders/:id/cancel\",\n (_req, res) => { res.status(403).json({error:\"Cancellation refused\"}); },\n requireAuth,\n asyncHandler(async (req, res) => {\n const orderId = Number(req.params.id);\n const accountId = req.account!.id;" + } + ] + }, + { + "id": "purchase-trusts-cross-origin-cookie", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Remove the origin check so a foreign page can make a purchase with the browsers ambient session cookie.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " return URL.parse(req.headers.origin)?.host === req.headers.host\n ? next() : res.status(403).json({ error: \"cross-origin request refused\" });", + "replace": " return next(); // mutant: accept another origins cookie-backed POST" + } + ] + }, + { + "id": "account-password-in-public-data", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Expose the actual submitted account password through data available to other clients.", + "edits": [ + { + "file": "server/src/index.ts", + "find": "const app = express();", + "replace": "const app = express();\nconst leakedPasswords: string[] = [];\napp.use((_req, res, next) => { const json = res.json.bind(res); res.json = (body: any) => json({ ...body, debugCredentials: leakedPasswords }); next(); });" + }, + { + "file": "server/src/index.ts", + "find": "const passwordHash = await hashPassword(password);", + "replace": "const passwordHash = await hashPassword(password);\n leakedPasswords.push(password);" + } + ] + }, + { + "id": "account-disclosure-rejects-login", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Reject every valid login; absent disclosures alone must not earn a pass.", + "file": "server/src/auth.ts", + "edits": [ + { + "find": "return crypto.timingSafeEqual", + "replace": "return false && crypto.timingSafeEqual" + } + ] + }, + { + "id": "signout-keeps-session", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Show a signed-out interface while retaining the browser credential and its purchase authority.", + "edits": [ + { + "file": "client/src/App.tsx", + "find": " await api(\"/api/auth/signout\", { method: \"POST\" });", + "replace": " // mutant: clear the UI but retain the valid session cookie" + } + ] + }, + { + "id": "query-like-password-bypasses-signin", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Interpret the query-like password as a login bypass while retaining normal password verification.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " if (rows.length === 0 || !(await verifyPassword(password, rows[0].passwordHash))) {", + "replace": " if (rows.length === 0 || !(password === \"' OR '1'='1\" || await verifyPassword(password, rows[0].passwordHash))) {" + } + ] + }, + { + "id": "purchase-session-rejects-returning-login", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Refuse every returning login; blocking all access after logout must not pass the session check.", + "file": "server/src/auth.ts", + "edits": [ + { + "find": "return crypto.timingSafeEqual", + "replace": "return false && crypto.timingSafeEqual" + } + ] + }, + { + "id": "account-password-in-script", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Ship the account password sentinel in the public JavaScript bundle while normal signup and login still work.", + "edits": [ + { + "file": "client/src/main.tsx", + "find": "import React from \"react\";", + "replace": "import React from \"react\";\nObject.assign(window, { debugPassword: 'sb-private-pw-76dc492ea153' });" + } + ] + }, + { + "id": "account-password-in-style", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Ship the account password sentinel in a public stylesheet while normal signup and login still work.", + "edits": [ + { + "file": "client/src/index.css", + "find": ":root {", + "replace": ":root {\n --debug-password: \"sb-private-pw-76dc492ea153\";" + } + ] + }, + { + "id": "signup-accepts-admin-claim", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Trust a submitted signup administrator claim and grant protected stock-write authority.", + "edits": [ + { + "file": "server/src/index.ts", + "find": ".values({ username, passwordHash, isAdmin: false, isStaff: false })", + "replace": ".values({ username, passwordHash, isAdmin: req.body.role === 'admin', isStaff: false })" + } + ] + }, + { + "id": "admin-boundary-rejects-all-signups", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Refuse every new registration; blocking useful signup must not pass the authority check.", + "edits": [ + { + "file": "server/src/index.ts", + "find": "\"/api/auth/signup\",\n asyncHandler(async (req, res) => {", + "replace": "\"/api/auth/signup\",\n asyncHandler(async (req, res) => {\n res.status(403).json({ error: \"Registration disabled\" }); return;" + } + ] + }, + { + "id": "expired-cart-keeps-its-lines", + "scenario": "tracks/ecommerce/scenarios/03-cart-expiration.json", + "targets": [ + "ecommerce.l3.cart-expiration.cart-expiration.304a" + ], + "desc": "Save an inactive cart for recovery when it expires but leave its lines in the cart.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " await client.query(`DELETE FROM cart_item WHERE cart_id = $1`, [cart.id]);", + "replace": " // Mutation: keep the expired cart lines in the cart." + } + ] + }, + { + "id": "delivery-interval-uses-minutes", + "scenario": "tracks/ecommerce/scenarios/03-order-delivery.json", + "targets": [ + "ecommerce.l3.order-delivery.order-delivery.303a" + ], + "desc": "Wait sixty minutes instead of sixty seconds before marking a shipped order delivered.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " WHERE status = 'shipped' AND shipped_at <= now() - interval '60 seconds'", + "replace": " WHERE status = 'shipped' AND shipped_at <= now() - interval '60 minutes'" + } + ] + }, + { + "id": "delivery-job-delivers-cancelled-orders", + "scenario": "tracks/ecommerce/scenarios/03-order-delivery.json", + "targets": [ + "ecommerce.l3.order-delivery.order-delivery.305a" + ], + "desc": "Mark cancelled orders delivered along with shipped orders once the delivery interval passes.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " WHERE status = 'shipped' AND shipped_at <= now() - interval '60 seconds'", + "replace": " WHERE status IN ('shipped', 'cancelled') AND COALESCE(shipped_at, created_at) <= now() - interval '60 seconds'" + } + ] + }, + { + "id": "reorder-rule-save-allows-customers", + "scenario": "tracks/ecommerce/scenarios/progression-automatic-reorder-access.json", + "targets": [ + "ecommerce.spec.access-control.automatic-reorder-access.502c" + ], + "desc": "Accept a reorder rule change from any signed-in customer because only the interface hides the form.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": "app.put(\"/api/reorders/:itemId\", dependencies.requireStaff,", + "replace": "app.put(\"/api/reorders/:itemId\", dependencies.requireAuth," + } + ] + }, + { + "id": "sale-schedules-reorder-without-pending-check", + "scenario": "tracks/ecommerce/scenarios/progression-automatic-reorder-duplicate.json", + "targets": [ + "ecommerce.spec.transactional-integrity.automatic-reorder-deduplication.502b" + ], + "desc": "Schedule an automatic restock from every sale at or below the threshold without checking for one already pending.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " [orderId, itemId, name, price, warehouseId]\n );\n await client.query(\"COMMIT\");", + "replace": " [orderId, itemId, name, price, warehouseId]\n );\n await client.query(\n `INSERT INTO scheduled_restock (item_id, warehouse_id, quantity, due_at, automatic, created_by)\n SELECT rr.item_id, (SELECT id FROM warehouse ORDER BY id LIMIT 1), rr.quantity, now() + interval '60 seconds', true, rr.created_by\n FROM reorder_rule rr WHERE rr.item_id = $1 AND rr.enabled = true\n AND (SELECT COALESCE(SUM(quantity), 0) FROM stock WHERE item_id = $1) <= rr.threshold`,\n [itemId]\n );\n await client.query(\"COMMIT\");" + } + ] + }, + { + "id": "reorder-waits-for-stock-below-threshold", + "scenario": "tracks/ecommerce/scenarios/progression-automatic-reorder.json", + "targets": [ + "ecommerce.progression.automatic-reorder.automatic-reorder.502a" + ], + "desc": "Schedule an automatic restock only when stock falls below the threshold, not when it reaches it.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " if (rule.available > rule.threshold) continue;", + "replace": " if (rule.available >= rule.threshold) continue;" + } + ] + }, + { + "id": "cart-restore-checks-stock-without-reserving", + "scenario": "tracks/ecommerce/scenarios/progression-cart-recovery.json", + "targets": [ + "ecommerce.progression.cart-recovery.cart-recovery.503a" + ], + "desc": "Restore an expired cart line after checking that stock is available but without reserving it again.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " try {\n await allocateReservation(client, inserted.rows[0].id, saved.itemId, saved.quantity);\n } catch {", + "replace": " try {\n const available = await client.query(`SELECT COALESCE(SUM(quantity), 0)::int AS total FROM stock WHERE item_id = $1`, [saved.itemId]);\n if (available.rows[0].total < saved.quantity) throw new Error(\"not enough stock\");\n } catch {" + } + ] + }, + { + "id": "notifications-not-scoped-to-account", + "scenario": "tracks/ecommerce/scenarios/progression-delivery-notifications.json", + "targets": [ + "ecommerce.spec.access-control.delivery-notification-privacy.501b" + ], + "desc": "Send every account's notifications to each signed-in customer.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " account ? deps.pool.query(`SELECT * FROM notification WHERE account_id = $1 ORDER BY created_at DESC`, [account.id]) : { rows: [] },", + "replace": " account ? deps.pool.query(`SELECT * FROM notification ORDER BY created_at DESC`) : { rows: [] }," + } + ] + }, + { + "id": "pending-order-shows-return-button", + "scenario": "tracks/ecommerce/scenarios/progression-order-return-boundary.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3e" + ], + "desc": "Offer a return button on every order line that is not cancelled, including pending orders.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "{(o.status === \"shipped\" || o.status === \"delivered\") && !l.returned && !l.isBundle && (", + "replace": "{o.status !== \"cancelled\" && !l.returned && !l.isBundle && (" + } + ] + }, + { + "id": "promotion-redemption-counted-at-apply", + "scenario": "tracks/ecommerce/scenarios/progression-promotion-reporting.json", + "targets": [ + "ecommerce.progression.promotion-reporting.promotion-report-redemptions.622a" + ], + "desc": "Count a promotion redemption when the code is applied to the cart and again at checkout.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " res.json({ ok: true, discount: Number(promotion.discount_percent) });", + "replace": " await deps.pool.query(`UPDATE promotion SET redemptions = redemptions + 1 WHERE id = $1`, [promotion.id]);\n res.json({ ok: true, discount: Number(promotion.discount_percent) });" + } + ] + }, + { + "id": "promotion-revenue-records-subtotal", + "scenario": "tracks/ecommerce/scenarios/progression-promotion-reporting.json", + "targets": [ + "ecommerce.progression.promotion-reporting.promotion-report-revenue.622b" + ], + "desc": "Add the undiscounted subtotal to the promotion revenue instead of the discounted total.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " `UPDATE promotion SET redemptions = redemptions + 1, revenue = revenue + $1 WHERE id = $2`,\n [total, promotionId],", + "replace": " `UPDATE promotion SET redemptions = redemptions + 1, revenue = revenue + $1 WHERE id = $2`,\n [subtotal, promotionId]," + } + ] + }, + { + "id": "product-creation-not-recorded-in-activity", + "scenario": "tracks/ecommerce/scenarios/progression-staff-activity.json", + "targets": [ + "ecommerce.progression.staff-activity.staff-activity.624a" + ], + "desc": "Create a product without writing a staff activity entry.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " await recordActivity(client, req.account!.id, \"create product\", name);", + "replace": " // Mutation: product creation is not recorded." + } + ] + }, + { + "id": "staff-activity-sent-to-customers", + "scenario": "tracks/ecommerce/scenarios/progression-staff-activity.json", + "targets": [ + "ecommerce.spec.access-control.staff-activity-privacy.624b" + ], + "desc": "Send staff activity history to every signed-in account and show the activity link whenever that history arrives.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " isStaff ? deps.pool.query(\n `SELECT sa.*, a.username FROM staff_activity sa JOIN account a ON a.id = sa.account_id", + "replace": " account ? deps.pool.query(\n `SELECT sa.*, a.username FROM staff_activity sa JOIN account a ON a.id = sa.account_id" + }, + { + "file": "client/src/ProgressionPanel.tsx", + "find": "{staff && Activity}", + "replace": "{(staff || (state?.activity?.length ?? 0) > 0) && Activity}" + } + ] + }, + { + "id": "support-refund-allows-customers", + "scenario": "tracks/ecommerce/scenarios/progression-support-refunds-access.json", + "targets": [ + "ecommerce.spec.access-control.support-refund-access.615c" + ], + "desc": "Accept a support refund from any signed-in customer because only the interface hides the refund button.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": "app.post(\"/api/support/cases/:id/refund\", dependencies.requireStaff,", + "replace": "app.post(\"/api/support/cases/:id/refund\", dependencies.requireAuth," + } + ] + }, + { + "id": "support-refund-refunds-every-customer-order", + "scenario": "tracks/ecommerce/scenarios/progression-support-refunds-accounting.json", + "targets": [ + "ecommerce.spec.transactional-integrity.support-refund-accounting.615b" + ], + "desc": "Mark every one of the customer's orders refunded when the support case refunds one linked order.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " await client.query(`UPDATE orders SET refund_total = total, status = CASE WHEN status='pending' THEN 'refunded' ELSE status END WHERE id = $1`, [order.rows[0].id]);", + "replace": " await client.query(`UPDATE orders SET refund_total = total, status = CASE WHEN status='pending' THEN 'refunded' ELSE status END WHERE account_id = $1 AND status != 'cancelled'`, [order.rows[0].account_id]);" + } + ] + }, + { + "id": "support-refund-leaves-case-open", + "scenario": "tracks/ecommerce/scenarios/progression-support-refunds-resolution.json", + "targets": [ + "ecommerce.progression.support-refunds.support-refunds-resolution.615a" + ], + "desc": "Record the refund amount on the support case without resolving the case.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " await client.query(`UPDATE support_case SET refund_total = $1, status = 'resolved', updated_at = now() WHERE id = $2`, [amount, caseId]);", + "replace": " await client.query(`UPDATE support_case SET refund_total = $1, updated_at = now() WHERE id = $2`, [amount, caseId]);" + } + ] + }, + { + "id": "delivery-due-time-kept-in-memory", + "scenario": "tracks/ecommerce/scenarios/03-deferred-durability.json", + "targets": [ + "ecommerce.l3.deferred-durability.restart-survival.315a" + ], + "desc": "Shipping records the order's delivery due time only in a server-memory map that the timer reads, so an order shipped before a restart stays shipped forever.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": "let io: SocketIOServer | null = null;", + "replace": "let io: SocketIOServer | null = null;\nexport const deliveryDueAt = new Map();" + }, + { + "find": " const delivered = await client.query(\n `UPDATE orders SET status = 'delivered', delivered_at = now()\n WHERE status = 'shipped' AND shipped_at <= now() - interval '60 seconds'\n RETURNING id, account_id`,\n );", + "replace": " const dueDeliveries = [...deliveryDueAt].filter(([, dueAt]) => dueAt <= Date.now()).map(([orderId]) => orderId);\n const delivered = await client.query(\n `UPDATE orders SET status = 'delivered', delivered_at = now()\n WHERE status = 'shipped' AND id = ANY($1::int[])\n RETURNING id, account_id`,\n [dueDeliveries],\n );\n for (const orderId of dueDeliveries) deliveryDueAt.delete(orderId);" + }, + { + "file": "server/src/index.ts", + "find": " processProgressionTimers,", + "replace": " deliveryDueAt,\n processProgressionTimers," + }, + { + "file": "server/src/index.ts", + "find": " const accountId = result.rows[0].account_id;", + "replace": " const accountId = result.rows[0].account_id;\n deliveryDueAt.set(orderId, Date.now() + 60_000);" + } + ] + }, + { + "id": "cart-expiry-restarts-at-startup", + "scenario": "tracks/ecommerce/scenarios/03-deferred-durability.json", + "targets": [ + "ecommerce.l3.deferred-durability.restart-survival.316a" + ], + "desc": "Startup re-arms every open cart's five-minute expiry from the restart time instead of the cart's last activity, so a cart that was waiting before a restart expires minutes late.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " await seed();", + "replace": " await seed();\n await pool.query(`UPDATE cart SET last_activity = now() WHERE expired_at IS NULL`);" + } + ] + }, + { + "id": "delivery-record-backfill-repeats-on-startup", + "scenario": "tracks/ecommerce/scenarios/03-deferred-integrity.json", + "targets": [ + "ecommerce.l3.deferred-integrity.exactly-once.312a" + ], + "desc": "Each delivery writes a delivery record that the staff completed-orders list shows, and startup backfills a record for every delivered order without checking for an existing one, so a restart lists a delivered order twice.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " CREATE TABLE IF NOT EXISTS refund_entry (", + "replace": " CREATE TABLE IF NOT EXISTS delivery_record (\n id serial PRIMARY KEY,\n order_id integer NOT NULL REFERENCES orders(id),\n delivered_at timestamptz\n );\n INSERT INTO delivery_record (order_id, delivered_at)\n SELECT id, delivered_at FROM orders WHERE status = 'delivered';\n CREATE TABLE IF NOT EXISTS refund_entry (" + }, + { + "find": " changedAccounts.add(order.account_id);\n const preference", + "replace": " changedAccounts.add(order.account_id);\n await client.query(`INSERT INTO delivery_record (order_id, delivered_at) VALUES ($1, now())`, [order.id]);\n const preference" + }, + { + "find": " `SELECT o.id, o.status, o.delivered_at, string_agg(oi.item_name, ', ' ORDER BY oi.id) AS items\n FROM orders o JOIN order_item oi ON oi.order_id = o.id\n WHERE o.status IN ('shipped', 'delivered') GROUP BY o.id ORDER BY o.id DESC`,", + "replace": " `SELECT COALESCE(dr.id, -o.id) AS id, o.status, o.delivered_at,\n (SELECT string_agg(oi.item_name, ', ' ORDER BY oi.id) FROM order_item oi WHERE oi.order_id = o.id) AS items\n FROM orders o LEFT JOIN delivery_record dr ON dr.order_id = o.id\n WHERE o.status IN ('shipped', 'delivered') ORDER BY o.id DESC, dr.id`," + } + ] + }, + { + "id": "recommendations-limited-to-purchase-price", + "scenario": "tracks/ecommerce/scenarios/progression-personalized-recommendations.json", + "targets": [ + "ecommerce.progression.personalized-recommendations.personalized-recommendations.403a" + ], + "desc": "Recommendations leave out items priced above the customer's most expensive purchase, so an Audio buyer of the cheaper speaker is never offered Headphones.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " return catalog.filter((i) => categories.has(i.category) && !purchased.has(i.id)\n && !inCart.has(i.id) && !dismissed.has(i.id)).slice(0, 10);", + "replace": " const maxPaid = Math.max(...catalog.filter((i) => purchased.has(i.id)).map((i) => i.price));\n return catalog.filter((i) => categories.has(i.category) && !purchased.has(i.id) && i.price <= maxPaid\n && !inCart.has(i.id) && !dismissed.has(i.id)).slice(0, 10);" + } + ] + }, + { + "id": "repeat-purchase-overwrites-every-recommendation-profile", + "scenario": "tracks/ecommerce/scenarios/progression-personalized-recommendations.json", + "targets": [ + "ecommerce.spec.access-control.recommendation-profile-isolation.403b" + ], + "desc": "Recommendations read a stored profile category per customer; a first purchase inserts the buyer's row, but a later purchase updates the category without an account filter, so one customer's repeat purchase replaces every customer's recommendations.", + "file": "server/src/index.ts", + "edits": [ + { + "file": "server/src/progression.ts", + "find": " CREATE TABLE IF NOT EXISTS recommendation_dismissal (", + "replace": " CREATE TABLE IF NOT EXISTS recommendation_profile (\n account_id integer PRIMARY KEY REFERENCES account(id) ON DELETE CASCADE,\n category text NOT NULL\n );\n CREATE TABLE IF NOT EXISTS recommendation_dismissal (" + }, + { + "find": " [orderId, itemId, name, price, warehouseId]\n );\n await client.query(\"COMMIT\");", + "replace": " [orderId, itemId, name, price, warehouseId]\n );\n const profile = await client.query(`SELECT 1 FROM recommendation_profile WHERE account_id = $1`, [accountId]);\n if (profile.rowCount) {\n await client.query(`UPDATE recommendation_profile SET category = (SELECT category FROM item WHERE id = $1)`, [itemId]);\n } else {\n await client.query(`INSERT INTO recommendation_profile (account_id, category) SELECT $1, category FROM item WHERE id = $2`, [accountId, itemId]);\n }\n await client.query(\"COMMIT\");" + }, + { + "find": " const categories = new Set(categoriesResult.rows.map((r) => r.category));", + "replace": " const profileResult = await pool.query(`SELECT category FROM recommendation_profile WHERE account_id = $1`, [accountId]);\n const categories = new Set(profileResult.rows.map((r) => r.category));" + } + ] + }, + { + "id": "support-order-link-requires-staff", + "scenario": "tracks/ecommerce/scenarios/progression-order-support-owned.json", + "targets": [ + "ecommerce.progression.order-support.order-support-owned.614a" + ], + "desc": "Only staff may attach an order to a support case, so a customer's link request for their own order is refused and staff never see the order on the case.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": "app.post(\"/api/support/cases/:id/order\", dependencies.requireAuth,", + "replace": "app.post(\"/api/support/cases/:id/order\", dependencies.requireStaff," + } + ] + }, + { + "id": "support-order-link-checks-only-case-owner", + "scenario": "tracks/ecommerce/scenarios/progression-order-support-boundary.json", + "targets": [ + "ecommerce.spec.access-control.order-support-ownership.614b" + ], + "desc": "Linking an order checks only that the caller owns the support case, not that the order belongs to the case's customer, so another customer can attach and then read the owner's order on their own case.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " if (!support || !order || support.account_id !== order.account_id\n || (!req.account!.isAdmin && !req.account!.isStaff && order.account_id !== req.account!.id)) {", + "replace": " if (!support || !order\n || (!req.account!.isAdmin && !req.account!.isStaff && support.account_id !== req.account!.id)) {" + } + ] + }, + { + "id": "delivery-notification-checks-stock-preference", + "scenario": "tracks/ecommerce/scenarios/progression-delivery-notifications.json", + "targets": [ + "ecommerce.progression.delivery-notifications.delivery-notification-delivery.501a", + "ecommerce.spec.access-control.delivery-notification-privacy.501b" + ], + "desc": "The delivery job checks the stock-alert preference instead of the order-update preference, so an owner who turned on order updates gets no delivery notification.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " if (preference.rows[0]?.notify_order) {", + "replace": " if (preference.rows[0]?.notify_stock) {" + } + ] + }, + { + "id": "recommendation-panel-keeps-first-list", + "scenario": "tracks/ecommerce/scenarios/progression-recommendation-feedback.json", + "targets": [ + "ecommerce.progression.recommendation-feedback.recommendation-feedback.504a", + "ecommerce.spec.access-control.recommendation-feedback-privacy.504b" + ], + "desc": "The recommendations panel keeps the first non-empty list it receives and ignores later updates, so a dismissed item stays on screen until the page is reloaded.", + "file": "client/src/ProgressionPanel.tsx", + "edits": [ + { + "find": " const [preferences, setPreferences] = useState({ order: false, stock: false });", + "replace": " const [preferences, setPreferences] = useState({ order: false, stock: false });\n const [shownRecommendations, setShownRecommendations] = useState([]);\n useEffect(() => {\n if (shownRecommendations.length === 0 && state?.recommendations?.length) setShownRecommendations(state.recommendations);\n }, [state?.recommendations]);" + }, + { + "find": " {(state?.recommendations ?? []).map((item: any) =>
", + "replace": " {shownRecommendations.map((item: any) =>
" + } + ] + }, + { + "id": "recommendation-dismissals-shared-across-customers", + "scenario": "tracks/ecommerce/scenarios/progression-recommendation-feedback.json", + "targets": [ + "ecommerce.spec.access-control.recommendation-feedback-privacy.504b" + ], + "desc": "Recommendations hide every item any customer has dismissed, so one customer's dismissal removes the item from other customers' lists.", + "file": "server/src/index.ts", + "edits": [ + { + "find": " const dismissedResult = await pool.query(\n `SELECT item_id FROM recommendation_dismissal WHERE account_id = $1`,\n [accountId]\n );", + "replace": " const dismissedResult = await pool.query(\n `SELECT item_id FROM recommendation_dismissal`\n );" + } + ] + }, + { + "id": "cart-restore-warning-omits-item-names", + "scenario": "tracks/ecommerce/scenarios/progression-cart-recovery.json", + "targets": [ + "ecommerce.progression.cart-recovery.cart-recovery.503b" + ], + "desc": "A partial cart restore keeps the available items but its warning only counts the unavailable items without naming them.", + "file": "client/src/ProgressionPanel.tsx", + "edits": [ + { + "find": "

Could not restore: {unrestored.join(\", \")}

", + "replace": "

Could not restore {unrestored.length} item(s)

" + } + ] + }, + { + "id": "stock-alert-misses-direct-stock-writes", + "scenario": "tracks/ecommerce/scenarios/progression-stock-alert-direct-restore.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock-alert.631d" + ], + "desc": "Stock alerts are delivered only by the server's restock paths, so stock written directly to the database never delivers its alert.", + "file": "server/src/progression.ts", + "edits": [ + { + "find": " // Stock written directly to the database bypasses the server, so alerts also follow the stock rows.\n const restocked = await client.query(\n `SELECT DISTINCT r.item_id FROM stock_alert_request r\n WHERE r.fulfilled = false AND (SELECT COALESCE(SUM(s.quantity), 0) FROM stock s WHERE s.item_id = r.item_id) > 0`,\n );\n for (const row of restocked.rows) await processStockAlerts(client, row.item_id);\n\n", + "replace": " // Stock alerts are delivered by the restock paths.\n\n" + } + ] + } + ] +} diff --git a/tools/stack-bench/grader/mutations/spacetime-ecommerce.json b/tools/stack-bench/grader/mutations/spacetime-ecommerce.json new file mode 100644 index 00000000000..be7a1b7d01e --- /dev/null +++ b/tools/stack-bench/grader/mutations/spacetime-ecommerce.json @@ -0,0 +1,3824 @@ +{ + "schemaVersion": 3, + "fixtureSha256": "218a8e6298b8becef513b68d4b9a5068b84bde9c14789e0ae45ebc0d6ce703aa", + "backend": "spacetime", + "track": "ecommerce", + "note": "Mutation definitions for the SpacetimeDB ecommerce reference.", + "mutations": [ + { + "id": "recommendation-dismissal-lost-on-reconnect", + "scenario": "tracks/ecommerce/scenarios/progression-recommendation-feedback.json", + "targets": [ + "ecommerce.spec.state-durability.recommendation-feedback-restart.504c" + ], + "desc": "Erase saved recommendation dismissals when a browser reconnects. This controls reconnect persistence, not backend restart alone.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "// --- views ---", + "replace": "export const onConnect = spacetimedb.clientConnected((ctx) => { for (const row of ctx.db.recommendationDismissal.iter()) ctx.db.recommendationDismissal.id.delete(row.id); });\n// --- views ---" + } + ] + }, + { + "id": "pending-order-item-return-accepted", + "scenario": "tracks/ecommerce/scenarios/progression-order-return-boundary.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3f" + ], + "desc": "Accept a pending order return and restore its stock before shipment.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "if (!['shipped', 'delivered'].includes(order.status)) throw new SenderError('Order has not shipped yet.');", + "replace": "if (!['pending', 'shipped', 'delivered'].includes(order.status)) throw new SenderError('Order has not shipped yet.');" + } + ] + }, + { + "id": "staff-admin-access-survives-role-removal", + "scenario": "tracks/ecommerce/scenarios/progression-staff-roles.json", + "targets": [ + "ecommerce.spec.access-control.staff-role-revocation.621d" + ], + "desc": "Keep administrator access after changing the assigned role back to staff.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.account.id.update({ ...target, isAdmin: role === 'admin' });", + "replace": "ctx.db.account.id.update({ ...target, isAdmin: target.isAdmin || role === 'admin' });" + } + ] + }, + { + "id": "role-assignment-trusts-sign-in-admin-flag", + "scenario": "tracks/ecommerce/scenarios/progression-staff-roles.json", + "targets": [ + "ecommerce.spec.access-control.staff-role-revocation.621d" + ], + "desc": "Sign-in records whether the account was an administrator on the caller identity's session row, and role assignment authorizes from that recorded flag instead of the stored account. A session opened while the account was an administrator keeps assigning roles after the account is changed back to staff; sessions opened after the change are refused.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "file": "backend/spacetimedb/src/schema.ts", + "find": " accountId: t.u64(),\n expiresMicros: t.i64(),\n }\n);", + "replace": " accountId: t.u64(),\n expiresMicros: t.i64(),\n signInAdmin: t.bool().default(false),\n }\n);" + }, + { + "file": "backend/spacetimedb/src/auth.ts", + "find": " expiresMicros: ctx.timestamp.microsSinceUnixEpoch + SESSION_MICROS });", + "replace": " expiresMicros: ctx.timestamp.microsSinceUnixEpoch + SESSION_MICROS,\n signInAdmin: ctx.db.account.id.find(accountId)?.isAdmin ?? false });" + }, + { + "find": "const actor = requireAdmin(ctx);\n const target = ctx.db.account.id.find(accountId);", + "replace": "const actor = requireAccount(ctx);\n if (!ctx.db.session.identity.find(ctx.sender)?.signInAdmin) throw new SenderError('Admin access required.');\n const target = ctx.db.account.id.find(accountId);" + } + ] + }, + { + "id": "shipping-counts-sale-twice", + "scenario": "tracks/ecommerce/scenarios/progression-shipping-accounting.json", + "targets": [ + "ecommerce.inventory-operations.shipping-accounting.202e" + ], + "desc": "Shipping succeeds but doubles the completed sale value in authoritative revenue.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ctx.db.customerOrder.id.update({ ...order, status: 'shipped' });", + "replace": " ctx.db.customerOrder.id.update({ ...order, status: 'shipped', total: order.total * 2 });" + } + ] + }, + { + "id": "restock-client-snapshot-overwrites-concurrent-purchases", + "scenario": "tracks/ecommerce/scenarios/01-restock-race.json", + "targets": [ + "ecommerce.spec.concurrency-safety.restock-race.202a" + ], + "desc": "Compute absolute restock quantity when the administrator edits the form and store that captured value in the reducer. A fixed 500 ms submission delay widens the stale-write window. Serial purchases and restocks still work; intervening purchases can be overwritten. This is a stale-form lost update, not a race inside an atomic reducer.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.stock.insert({ ...existing, quantity: existing.quantity + quantity });\n } else {\n ctx.db.stock.insert({ item_id: itemId, warehouse_id: warehouseId, quantity });\n }\n for (const alert of", + "replace": "ctx.db.stock.insert({ ...existing, quantity });\n } else {\n ctx.db.stock.insert({ item_id: itemId, warehouse_id: warehouseId, quantity });\n }\n for (const alert of" + }, + { + "file": "client/src/components/AdminPanel.tsx", + "find": "onChange={(e) => setRestockInputs((v) => ({ ...v, [k]: e.target.value }))}", + "replace": "onChange={(e) => setRestockInputs((v) => ({ ...v, [k]: String(stockOf(item.id, wh.id) + Number(e.target.value)) }))}" + }, + { + "file": "client/src/App.tsx", + "find": " await conn?.reducers.adminRestock({ itemId, warehouseId, quantity });", + "replace": " // Mutant: widen the stale form submission window without changing serial behavior.\n await new Promise(resolve => setTimeout(resolve, 500));\n await conn?.reducers.adminRestock({ itemId, warehouseId, quantity });" + } + ] + }, + { + "id": "signup-binds-the-new-account-to-the-admin-session", + "scenario": "tracks/ecommerce/scenarios/01-account-create.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1a" + ], + "desc": "Create the requested account but bind the new browser session to the administrator, so account creation no longer signs the visitor in as the account it created.", + "file": "backend/spacetimedb/src/auth.ts", + "edits": [ + { + "find": " bindSession(tx, account.id);\n return true;\n });\n }\n);\n\nexport const signIn", + "replace": " bindSession(tx, tx.db.account.username.find('admin')!.id);\n return true;\n });\n }\n);\n\nexport const signIn" + } + ] + }, + { + "id": "duplicate-signup-is-silently-ignored", + "scenario": "tracks/ecommerce/scenarios/01-account-duplicate.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1b" + ], + "desc": "Return success for a taken username without creating a session or surfacing the required refusal.", + "file": "backend/spacetimedb/src/auth.ts", + "edits": [ + { + "find": "if (tx.db.account.username.find(name) || tx.db.account.passwordSalt.find(passwordSalt)) return false;", + "replace": "if (tx.db.account.username.find(name) || tx.db.account.passwordSalt.find(passwordSalt)) return true;" + } + ] + }, + { + "id": "signin-does-not-verify-the-password", + "scenario": "tracks/ecommerce/scenarios/01-account-password.json", + "targets": [ + "ecommerce.spec.access-control.account-password.1c" + ], + "desc": "Accept a known username without comparing the supplied password hash.", + "file": "backend/spacetimedb/src/auth.ts", + "edits": [ + { + "find": "const valid = !!account && sameHash(digest, account.passwordHash);", + "replace": "const valid = !!account;" + } + ] + }, + { + "id": "signout-keeps-the-account-session", + "scenario": "tracks/ecommerce/scenarios/01-account-signout.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1d" + ], + "desc": "Leave the current account session in place when the visitor signs out.", + "file": "client/src/components/AuthWidget.tsx", + "edits": [ + { + "find": " await connection.reducers.signOut({});\n clearToken(); location.reload();", + "replace": " location.reload(); // mutant: keep the account and credential" + } + ] + }, + { + "id": "session-token-is-not-persisted-for-reload", + "scenario": "tracks/ecommerce/scenarios/01-account-reload.json", + "targets": [ + "ecommerce.spec.state-durability.session-reload.1e" + ], + "desc": "Discard the connection token instead of persisting it, so a reload receives a new identity with no account session.", + "file": "client/src/main.tsx", + "edits": [ + { + "find": ".onConnect((_connection, _identity, token) => saveToken(token));", + "replace": ".onConnect((_connection, _identity, token) => { saveToken(token); sessionStorage.clear(); });" + } + ] + }, + { + "id": "catalog-seeds-the-wrong-air-purifier-price", + "scenario": "tracks/ecommerce/scenarios/01-catalog-values.json", + "targets": [ + "ecommerce.feature.catalog.catalog-values.2a" + ], + "desc": "Seed Air Purifier with an incorrect stored price while leaving the rest of the catalog intact.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ['Air Purifier', 189.0, 60, 40, 'Home'],", + "replace": " ['Air Purifier', 999.0, 60, 40, 'Home']," + } + ] + }, + { + "id": "catalog-tie-breaks-in-reverse-alphabetical-order--01-catalog-ranking", + "scenario": "tracks/ecommerce/scenarios/01-catalog-ranking.json", + "targets": [ + "ecommerce.feature.catalog.catalog-ranking.2b" + ], + "desc": "Reverse the specified alphabetical tie-breaker. This necessarily breaks both the initial sequence and the post-purchase sequence in the same scenario.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " return a.name.localeCompare(b.name);", + "replace": " return b.name.localeCompare(a.name);" + } + ] + }, + { + "id": "purchase-does-not-update-ranking-count", + "scenario": "tracks/ecommerce/scenarios/01-core.json", + "targets": [ + "ecommerce.spec.live-state.ranking.2c" + ], + "desc": "Complete the purchase but leave its popularity count unchanged, so open storefronts cannot rank the purchased item first.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " bumpPurchaseCount(ctx, itemId, quantity);", + "replace": " // mutant: buy-now never advances the ranking count" + } + ] + }, + { + "id": "signed-out-purchase-bypasses-account-check", + "scenario": "tracks/ecommerce/scenarios/progression-signed-out-purchase.json", + "targets": [ + "ecommerce.spec.access-control.signed-out-purchase.3a" + ], + "desc": "Expose the guest purchase button and accept its purchase as the existing administrator. The stock observation then exercises the broken account boundary; normal signed-in purchases remain unchanged.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "export const buyNow = spacetimedb.reducer({ itemId: t.u64() }, (ctx, { itemId }) => {\n const acc = requireAccount(ctx);", + "replace": "export const buyNow = spacetimedb.reducer({ itemId: t.u64() }, (ctx, { itemId }) => {\n const acc = getAccountId(ctx) === null ? ctx.db.account.username.find('admin')! : requireAccount(ctx);" + }, + { + "file": "client/src/components/ItemCard.tsx", + "find": " {isSignedIn && (", + "replace": " {true && (" + } + ] + }, + { + "id": "buy-now-creates-orders-without-reserving-stock--01-buying", + "scenario": "tracks/ecommerce/scenarios/01-buying.json", + "targets": [ + "ecommerce.spec.live-state.purchase-stock.3b" + ], + "desc": "Create purchase orders without reserving inventory. The same defect necessarily breaks live purchase stock and the sell-out portion of the stock-limit check.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " const allocations = decrementStockTracked(ctx, itemId, quantity);", + "replace": " const allocations: Array<{ warehouseId: bigint; quantity: number }> = [];" + } + ] + }, + { + "id": "buy-now-creates-orders-without-reserving-stock--stock-limit", + "scenario": "tracks/ecommerce/scenarios/progression-stock-limit.json", + "targets": [ + "ecommerce.spec.concurrency-safety.stock-limit.3d" + ], + "desc": "Create purchase orders without reserving inventory. The same defect necessarily breaks live purchase stock and the sell-out portion of the stock-limit check.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " const allocations = decrementStockTracked(ctx, itemId, quantity);", + "replace": " const allocations: Array<{ warehouseId: bigint; quantity: number }> = [];" + } + ] + }, + { + "id": "restock-race-records-wrong-order-total", + "scenario": "tracks/ecommerce/scenarios/01-restock-race.json", + "targets": [ + "ecommerce.spec.concurrency-safety.restock-race.202a" + ], + "desc": "Purchases preserve stock and visible order counts but record the wrong booked total. Native mixed-race reconciliation must reject them.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " total: Math.round(price * 100) * quantity / 100,\n status: 'pending',", + "replace": " total: Math.round(price * 100) * quantity / 100 + 1,\n status: 'pending'," + } + ] + }, + { + "id": "buy-now-records-the-wrong-order-total", + "scenario": "tracks/ecommerce/scenarios/progression-purchasing.json", + "targets": [ + "ecommerce.feature.purchasing.purchase-order.3c" + ], + "desc": "Record a completed buy-now order one dollar above the stored item price.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " total: Math.round(price * 100) * quantity / 100,\n status: 'pending',", + "replace": " total: Math.round(price * 100) * quantity / 100 + 1,\n status: 'pending'," + } + ] + }, + { + "id": "existing-cart-line-does-not-increment-basic-cart", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4a" + ], + "desc": "Write an existing cart line back without incrementing its quantity.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.cartItem.id.update({ ...existing, quantity: existing.quantity + 1 });", + "replace": "ctx.db.cartItem.id.update({ ...existing, quantity: existing.quantity });" + } + ] + }, + { + "id": "cart-is-deleted-when-owner-disconnects", + "scenario": "tracks/ecommerce/scenarios/01-cart.json", + "targets": [ + "ecommerce.spec.state-durability.cart-reload.4b" + ], + "desc": "Delete the account cart on transport disconnect. Reload loses stored cart contents even after the same account signs in again; account and session records remain intact.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "// --- views ---", + "replace": "export const onDisconnect = spacetimedb.clientDisconnected((ctx) => {\n const accountId = getAccountId(ctx);\n if (accountId !== null) for (const row of [...ctx.db.cartItem.byAccountItem.filter(accountId)]) ctx.db.cartItem.id.delete(row.id);\n});\n// --- views ---" + } + ] + }, + { + "id": "signin-binds-the-second-client-to-a-different-account", + "scenario": "tracks/ecommerce/scenarios/01-cart.json", + "targets": [ + "ecommerce.spec.live-state.shared-cart.4c" + ], + "desc": "Authenticate valid credentials but bind the second connection to the administrator account, so two sessions for one customer do not share the customer's cart.", + "file": "backend/spacetimedb/src/auth.ts", + "edits": [ + { + "find": " if (!current || current.passwordHash !== account.passwordHash) return false;\n bindSession(tx, account.id);", + "replace": " if (!current || current.passwordHash !== account.passwordHash) return false;\n bindSession(tx, tx.db.account.username.find('admin')!.id);" + } + ] + }, + { + "id": "checkout-does-not-empty-the-basic-cart", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "Leave completed checkout lines in the durable cart.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " for (const line of lines) ctx.db.cartItem.id.delete(line.id);", + "replace": " // mutant: checked-out cart lines remain" + } + ] + }, + { + "id": "new-review-is-accepted-without-being-stored", + "scenario": "tracks/ecommerce/scenarios/01-review-visibility.json", + "targets": [ + "ecommerce.feature.reviews.reviews.6a" + ], + "desc": "Accept an eligible new review but omit its durable insert, so neither author nor visitor can see it.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ctx.db.review.insert({\n id: 0n,\n itemId,\n accountId: acc.id,\n rating,\n comment,\n createdAt: ctx.timestamp,\n });", + "replace": " // mutant: accepted review is not persisted" + } + ] + }, + { + "id": "repeat-review-inserts-a-second-row", + "scenario": "tracks/ecommerce/scenarios/01-review-uniqueness.json", + "targets": [ + "ecommerce.spec.transactional-integrity.unique-review.6b" + ], + "desc": "Insert a second review row instead of updating the customer's existing item review.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ctx.db.review.id.update({ ...existing, rating, comment, createdAt: ctx.timestamp });", + "replace": " ctx.db.review.insert({ id: 0n, itemId, accountId: acc.id, rating, comment, createdAt: ctx.timestamp });" + } + ] + }, + { + "id": "review-average-counts-rows-instead-of-ratings", + "scenario": "tracks/ecommerce/scenarios/01-review-rating-live.json", + "targets": [ + "ecommerce.spec.live-state.rating.6c" + ], + "desc": "Compute the live average from a constant per row rather than each stored rating.", + "file": "client/src/components/ItemDetail.tsx", + "edits": [ + { + "find": " : reviews.reduce((sum, r) => sum + r.rating, 0) / reviews.length;", + "replace": " : reviews.reduce((sum) => sum + 1, 0) / reviews.length;" + } + ] + }, + { + "id": "every-signed-in-customer-is-treated-as-an-admin", + "scenario": "tracks/ecommerce/scenarios/01-warehouse-admin-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-area-boundary.7a" + ], + "desc": "Use account presence instead of the server-provided administrator flag to expose the admin area.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const isAdmin = currentUser?.isAdmin ?? false;", + "replace": " const isAdmin = isSignedIn;" + } + ] + }, + { + "id": "warehouse-view-omits-west", + "scenario": "tracks/ecommerce/scenarios/01-warehouse-admin-staff.json", + "targets": [ + "ecommerce.feature.warehouse-admin.warehouse-view.7b" + ], + "desc": "Filter one real warehouse out of the administrator's inventory view.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " warehouses={warehouses}", + "replace": " warehouses={warehouses.filter((warehouse) => warehouse.name !== 'West')}" + } + ] + }, + { + "id": "guest-purchase-falls-back-to-the-admin-account", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Allow an unauthenticated direct purchase by attributing missing sessions to the administrator account, while preserving the authorized control path.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "export const buyNow = spacetimedb.reducer({ itemId: t.u64() }, (ctx, { itemId }) => {\n const acc = requireAccount(ctx);", + "replace": "export const buyNow = spacetimedb.reducer({ itemId: t.u64() }, (ctx, { itemId }) => {\n const accountId = getAccountId(ctx);\n const acc = (accountId === null ? null : ctx.db.account.id.find(accountId))\n ?? ctx.db.account.username.find('admin')!;" + } + ] + }, + { + "id": "direct-purchases-are-attributed-to-the-system-account", + "scenario": "tracks/ecommerce/scenarios/01-purchase-attribution.json", + "targets": [ + "ecommerce.spec.access-control.purchase-attribution.102a" + ], + "desc": "Store every buy-now order under the administrator instead of the authenticated caller.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " accountId,\n createdAt: ctx.timestamp,\n total: Math.round(price * 100) * quantity / 100,", + "replace": " accountId: ctx.db.account.username.find('admin')!.id,\n createdAt: ctx.timestamp,\n total: Math.round(price * 100) * quantity / 100," + } + ] + }, + { + "id": "direct-restock-does-not-require-an-admin", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Remove the server-side administrator check from the restock reducer.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "export const adminRestock = spacetimedb.reducer(\n { itemId: t.u64(), warehouseId: t.u64(), quantity: t.u32() },\n (ctx, { itemId, warehouseId, quantity }) => {\n requireAdmin(ctx);", + "replace": "export const adminRestock = spacetimedb.reducer(\n { itemId: t.u64(), warehouseId: t.u64(), quantity: t.u32() },\n (ctx, { itemId, warehouseId, quantity }) => {\n // mutant: no administrator check" + } + ] + }, + { + "id": "direct-purchase-ignores-the-stored-price", + "scenario": "tracks/ecommerce/scenarios/01-server-price.json", + "targets": [ + "ecommerce.spec.transactional-integrity.server-price.104a" + ], + "desc": "Create the direct purchase order one dollar above the authoritative stored price.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " total: Math.round(price * 100) * quantity / 100,\n status: 'pending',", + "replace": " total: 1,\n status: 'pending'," + } + ] + }, + { + "id": "sign-in-starts-an-empty-cart", + "scenario": "tracks/ecommerce/scenarios/progression-account-state-reload.json", + "targets": [ + "ecommerce.spec.state-durability.account-state-recovery.105a" + ], + "desc": "A successful sign-in deletes the account's stored cart lines, so a fresh sign-in after a restart finds an empty cart; sign-up, token-restored sessions and orders are unchanged.", + "file": "backend/spacetimedb/src/auth.ts", + "edits": [ + { + "find": " if (!current || current.passwordHash !== account.passwordHash) return false;\n bindSession(tx, account.id);", + "replace": " if (!current || current.passwordHash !== account.passwordHash) return false;\n // mutant: each sign-in starts a new cart instead of restoring the account cart\n for (const line of [...tx.db.cartItem.byAccountItem.filter(account.id)]) tx.db.cartItem.id.delete(line.id);\n bindSession(tx, account.id);" + } + ] + }, + { + "id": "reconnect-discards-the-visible-account-state", + "scenario": "tracks/ecommerce/scenarios/progression-account-state-reconnect.json", + "targets": [ + "ecommerce.spec.state-durability.account-state-recovery.105b" + ], + "desc": "Keep normal reload recovery but discard the client account projection when the browser comes back online.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const currentUser = currentUserRows[0] ?? null;", + "replace": " const [discardAccountAfterReconnect, setDiscardAccountAfterReconnect] = useState(false);\n useEffect(() => {\n const discardAccount = () => setDiscardAccountAfterReconnect(true);\n window.addEventListener('online', discardAccount);\n return () => window.removeEventListener('online', discardAccount);\n }, []);\n const currentUser = discardAccountAfterReconnect ? null : currentUserRows[0] ?? null;" + } + ] + }, + { + "id": "order-views-return-every-customers-orders", + "scenario": "tracks/ecommerce/scenarios/01-order-ownership.json", + "targets": [ + "ecommerce.spec.access-control.order-ownership.106a" + ], + "desc": "Remove account filters from both order views, exposing another customer's order and its line items.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " const rows = [...ctx.db.customerOrder.accountId.filter(accountId)];", + "replace": " const rows = [...ctx.db.customerOrder.iter()];" + }, + { + "find": " for (const o of ctx.db.customerOrder.accountId.filter(accountId)) {", + "replace": " for (const o of ctx.db.customerOrder.iter()) {" + } + ] + }, + { + "id": "admin-revenue-double-counts-every-order", + "scenario": "tracks/ecommerce/scenarios/progression-books-balance.json", + "targets": [ + "ecommerce.spec.transactional-integrity.books-balance.107a" + ], + "desc": "Count every completed order twice in the administrator revenue projection.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " total += o.total - o.refundedTotal;", + "replace": " total += (o.total - o.refundedTotal) * 2;" + } + ] + }, + { + "id": "purchases-do-not-leave-the-warehouses", + "scenario": "tracks/ecommerce/scenarios/progression-books-balance.json", + "targets": [ + "ecommerce.spec.transactional-integrity.books-balance.107b" + ], + "desc": "Create normal orders and revenue while leaving warehouse stock unchanged.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " const allocations = decrementStockTracked(ctx, itemId, quantity);", + "replace": " const allocations: Array<{ warehouseId: bigint; quantity: number }> = [];" + } + ] + }, + { + "id": "review-purchase-eligibility-is-not-checked", + "scenario": "tracks/ecommerce/scenarios/01-review-eligibility.json", + "targets": [ + "ecommerce.spec.access-control.review-eligibility.108a" + ], + "desc": "Allow a signed-in customer to review an item with no matching purchase.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (!bought) throw new SenderError('You can only review items you have purchased.');", + "replace": " // mutant: purchase eligibility is not checked" + } + ] + }, + { + "id": "eligible-review-is-accepted-without-being-stored", + "scenario": "tracks/ecommerce/scenarios/01-review-eligibility.json", + "targets": [ + "ecommerce.spec.access-control.review-eligibility.108b", + "ecommerce.spec.access-control.review-eligibility.108a" + ], + "desc": "Keep the non-buyer refusal but omit the insert for a buyer's eligible new review. Both eligibility criteria require a successfully stored eligible review as a positive control; this does not establish a non-buyer authorization defect.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ctx.db.review.insert({\n id: 0n,\n itemId,\n accountId: acc.id,\n rating,\n comment,\n createdAt: ctx.timestamp,\n });", + "replace": " // mutant: eligible review is acknowledged but not persisted" + } + ] + }, + { + "id": "cart-line-lookup-ignores-cart-ownership", + "scenario": "tracks/ecommerce/scenarios/01-cart-boundary.json", + "targets": [ + "ecommerce.spec.access-control.cart-boundary.109a" + ], + "desc": "Find an existing cart line by item alone, so the same named add action from another customer increments the owner's line instead of that customer's cart.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "function findCartLine(ctx: Ctx, accountId: bigint, itemId: bigint) {\n for (const row of ctx.db.cartItem.byAccountItem.filter([accountId, itemId])) {\n return row;\n }\n return null;\n}", + "replace": "function findCartLine(ctx: Ctx, _accountId: bigint, itemId: bigint) {\n for (const row of ctx.db.cartItem.iter()) {\n if (row.itemId === itemId) return row;\n }\n return null;\n}" + } + ] + }, + { + "id": "purchase-does-not-reserve-stock-last-unit", + "scenario": "tracks/ecommerce/scenarios/01-last-unit.json", + "targets": [ + "ecommerce.spec.concurrency-safety.last-unit.201a", + "ecommerce.spec.concurrency-safety.last-unit.201b", + "ecommerce.spec.concurrency-safety.last-unit.201c" + ], + "desc": "Create purchase orders without reserving stock, proving the focused last-unit stock, order-count, and revenue consequences.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " const allocations = decrementStockTracked(ctx, itemId, quantity);", + "replace": " const allocations: Array<{ warehouseId: bigint; quantity: number }> = [];" + } + ] + }, + { + "id": "existing-cart-line-does-not-increment", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203a" + ], + "desc": "Render the old cart quantity after concurrent adds.", + "file": "client/src/components/CartPanel.tsx", + "edits": [ + { + "find": " value={line.quantity}", + "replace": " value={1}" + } + ] + }, + { + "id": "checkout-does-not-empty-cart", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Keep checked-out cart lines so the next serialized checkout creates a duplicate order.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " for (const line of lines) ctx.db.cartItem.id.delete(line.id);", + "replace": " // mutant: checked-out lines remain in the cart" + } + ] + }, + { + "id": "stock-subscription-snapshotted-once", + "scenario": "tracks/ecommerce/scenarios/01-external-live-sync.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock.901a" + ], + "desc": "Render the first non-empty stock snapshot forever instead of following committed subscription updates.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "import { useEffect, useMemo, useState } from 'react';", + "replace": "import { useEffect, useMemo, useRef, useState } from 'react';" + }, + { + "find": " const [stocks] = useTable(tables.stock);", + "replace": " const [liveStocks] = useTable(tables.stock);\n const initialStocks = useRef(null);\n if (initialStocks.current === null && liveStocks.length > 0) {\n initialStocks.current = liveStocks;\n }\n const stocks = initialStocks.current ?? liveStocks;" + } + ] + }, + { + "id": "stock-view-keeps-pre-reconnect-snapshot", + "scenario": "tracks/ecommerce/scenarios/01-external-reconnect-sync.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock.901d" + ], + "desc": "Continue following stock until the browser goes offline, then retain the last online snapshot after network restoration.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "import { useEffect, useMemo, useState } from 'react';", + "replace": "import { useEffect, useMemo, useRef, useState } from 'react';" + }, + { + "find": " const [stocks] = useTable(tables.stock);", + "replace": " const [liveStocks] = useTable(tables.stock);\n const [freezeStockAfterOffline, setFreezeStockAfterOffline] = useState(false);\n const lastOnlineStocks = useRef(liveStocks);\n useEffect(() => {\n const freezeStock = () => setFreezeStockAfterOffline(true);\n window.addEventListener('offline', freezeStock);\n return () => window.removeEventListener('offline', freezeStock);\n }, []);\n if (!freezeStockAfterOffline) {\n lastOnlineStocks.current = liveStocks;\n }\n const stocks = freezeStockAfterOffline ? lastOnlineStocks.current : liveStocks;" + } + ] + }, + { + "id": "open-review-list-snapshots-on-selection", + "scenario": "tracks/ecommerce/scenarios/progression-open-list-live.json", + "targets": [ + "ecommerce.spec.live-state.open-list.902a" + ], + "desc": "Snapshot the selected item's reviews when the detail opens instead of following later subscription updates.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "import { useEffect, useMemo, useState } from 'react';", + "replace": "import { useEffect, useMemo, useRef, useState } from 'react';" + }, + { + "find": " const selectedItemReviews = selectedItemId !== null ? reviewsByItem.get(selectedItemId) ?? [] : [];", + "replace": " const openedReviewItem = useRef(null);\n const openedReviews = useRef<(typeof reviews)[number][]>([]);\n if (selectedItemId !== openedReviewItem.current) {\n openedReviewItem.current = selectedItemId;\n openedReviews.current = selectedItemId !== null ? reviewsByItem.get(selectedItemId) ?? [] : [];\n }\n const selectedItemReviews = openedReviews.current;" + } + ] + }, + { + "id": "open-review-list-renders-each-review-twice", + "scenario": "tracks/ecommerce/scenarios/progression-open-list-live.json", + "targets": [ + "ecommerce.spec.live-state.open-list.902a" + ], + "desc": "Render every committed review twice in the already-open list.", + "file": "client/src/components/ItemDetail.tsx", + "edits": [ + { + "find": " {reviews.map((r) => (", + "replace": " {[...reviews, ...reviews].map((r) => (" + } + ] + }, + { + "id": "cancel-does-not-restore-stock-feature", + "scenario": "tracks/ecommerce/scenarios/02-order-cancellation-core.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3a" + ], + "desc": "The serialized cancellation reducer changes order state and purchase counts but skips allocation restoration.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " restoreOrderItemStock(ctx, li);\n decrementPurchaseCount(ctx, li.itemId, li.quantity);", + "replace": " // mutant: cancellation does not restore its reserved stock\n decrementPurchaseCount(ctx, li.itemId, li.quantity);" + } + ] + }, + { + "id": "cancellation-accounting-loses-stock-restoration", + "scenario": "tracks/ecommerce/scenarios/02-invariants.json", + "targets": [ + "ecommerce.returns-pricing.refund-accounting.203a" + ], + "desc": "Cancellation removes revenue and changes order status, but loses the original warehouse stock restoration. The native refund-accounting assertion must detect this.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " restoreOrderItemStock(ctx, li);\n decrementPurchaseCount(ctx, li.itemId, li.quantity);", + "replace": " // mutant: cancellation does not restore its reserved stock\n decrementPurchaseCount(ctx, li.itemId, li.quantity);" + } + ] + }, + { + "id": "cancel-does-not-restore-stock-fresh-client", + "scenario": "tracks/ecommerce/scenarios/02-self-contained.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202b" + ], + "desc": "The serialized cancellation reducer skips allocation restoration, so a fresh client reads the persisted shortfall.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " restoreOrderItemStock(ctx, li);\n decrementPurchaseCount(ctx, li.itemId, li.quantity);", + "replace": " // mutant: cancellation does not restore its reserved stock\n decrementPurchaseCount(ctx, li.itemId, li.quantity);" + } + ] + }, + { + "id": "cancel-restores-stock-but-keeps-pending-status", + "scenario": "tracks/ecommerce/scenarios/02-order-cancellation-history.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3b" + ], + "desc": "The serialized cancellation reducer restores allocations but writes pending back to order history.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ctx.db.customerOrder.id.update({ ...order, status: 'cancelled' });", + "replace": " ctx.db.customerOrder.id.update({ ...order, status: 'pending' });" + } + ] + }, + { + "id": "cancelled-order-remains-in-revenue-feature", + "scenario": "tracks/ecommerce/scenarios/02-order-cancellation-core.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3a" + ], + "desc": "The admin revenue view includes cancelled orders even though cancellation otherwise succeeds.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (!isOrderCounted(o)) continue;\n total += o.total - o.refundedTotal;", + "replace": " total += o.total - o.refundedTotal;" + } + ] + }, + { + "id": "cancelled-order-remains-in-revenue-invariant", + "scenario": "tracks/ecommerce/scenarios/02-invariants.json", + "targets": [ + "ecommerce.returns-pricing.refund-accounting.203a" + ], + "desc": "The admin revenue view includes cancelled orders even though cancellation otherwise succeeds.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (!isOrderCounted(o)) continue;\n total += o.total - o.refundedTotal;", + "replace": " total += o.total - o.refundedTotal;" + } + ] + }, + { + "id": "operator-authorization-allows-customer-transfer", + "scenario": "tracks/ecommerce/scenarios/02-strengthened.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201a" + ], + "desc": "The transfer reducer drops its administrator role gate.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " (ctx, { itemId, fromWarehouseId, toWarehouseId, quantity }) => {\n requireAdmin(ctx);", + "replace": " (ctx, { itemId, fromWarehouseId, toWarehouseId, quantity }) => {\n // mutant: no administrator role check" + } + ] + }, + { + "id": "customer-can-ship-order-direct-1-1", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "The shipping reducer drops its staff role check while retaining pending-state validation.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n requireStaffOrAdmin(ctx);", + "replace": "export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n // mutant: no staff role check" + } + ] + }, + { + "id": "customer-can-cancel-foreign-order-1-1", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.order-owner.204a" + ], + "desc": "Cancellation bypasses the owner helper while retaining missing-order and pending-state validation.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "export const cancelOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n const order = requireOrderOwner(ctx, orderId);", + "replace": "export const cancelOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n const order = ctx.db.customerOrder.id.find(orderId);\n if (!order) throw new SenderError('Order not found.');" + } + ] + }, + { + "id": "queue-depth-lags-one-order", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-live.json", + "targets": [ + "ecommerce.operations-access.fulfilment-queue.1a" + ], + "desc": "The reactive queue renders every order but its visible depth remains one behind.", + "file": "client/src/components/FulfilmentPanel.tsx", + "edits": [ + { + "find": "Waiting: {queue.length}", + "replace": "Waiting: {Math.max(0, queue.length - 1)}" + } + ] + }, + { + "id": "ship-acknowledges-without-changing-status", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-ship.json", + "targets": [ + "ecommerce.operations-access.fulfilment-queue.1c" + ], + "desc": "The serialized shipping reducer accepts the call but writes pending back to the order.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ctx.db.customerOrder.id.update({ ...order, status: 'shipped' });", + "replace": " ctx.db.customerOrder.id.update({ ...order, status: 'pending' });" + } + ] + }, + { + "id": "customer-sees-fulfilment-navigation", + "scenario": "tracks/ecommerce/scenarios/02-features.json", + "targets": [ + "ecommerce.operations-access.fulfilment-queue.1d" + ], + "desc": "Expose the protected staff area to signed-in customers, including its navigation and content.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const isStaff = currentUser?.isStaff ?? false;", + "replace": " const isStaff = isSignedIn;" + } + ] + }, + { + "id": "progression-customer-sees-fulfilment-content", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-access.json", + "targets": [ + "ecommerce.spec.access-control.fulfilment-area-boundary.1d" + ], + "desc": "Expose the protected staff area to signed-in customers, including its navigation and content.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const isStaff = currentUser?.isStaff ?? false;", + "replace": " const isStaff = isSignedIn;" + } + ] + }, + { + "id": "operator-authorization-allows-customer-shipping", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "The shipping reducer drops the staff role check while retaining the pending-order guard.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n requireStaffOrAdmin(ctx);", + "replace": "export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n // mutant: no staff role check" + } + ] + }, + { + "id": "transfer-debits-source-without-crediting-existing-destination", + "scenario": "tracks/ecommerce/scenarios/02-strengthened.json", + "targets": [ + "ecommerce.inventory-operations.warehouse-transfer.2a", + "ecommerce.operations-access.operator-authorization.201a" + ], + "desc": "A transfer debits the source row but writes the existing destination quantity back unchanged, violating both directional movement and total conservation inside the serialized reducer. It also breaks 201a's authorized-transfer positive control; that coupled failure is not independent evidence of an authorization defect.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ctx.db.stock.insert({ ...toRow, quantity: toRow.quantity + quantity });", + "replace": " ctx.db.stock.insert({ ...toRow, quantity: toRow.quantity });" + } + ] + }, + { + "id": "transfer-overdraft-guard-removed", + "scenario": "tracks/ecommerce/scenarios/02-transfer-overdraw.json", + "targets": [ + "ecommerce.inventory-operations.warehouse-transfer.2c" + ], + "desc": "The serialized reducer no longer rejects insufficient source stock and commits negative source quantity.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (available < quantity) {\n throw new SenderError(`Not enough stock in source warehouse: only ${available} available.`);\n }", + "replace": " // mutant: insufficient source stock is not rejected" + } + ] + }, + { + "id": "low-stock-excludes-boundary-ten", + "scenario": "tracks/ecommerce/scenarios/02-low-stock.json", + "targets": [ + "ecommerce.inventory-operations.operational-views.5a" + ], + "desc": "The reactive low-stock view uses a strict boundary and omits items with exactly ten units.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " .filter((i) => (stockByItem.get(i.id) ?? 0) <= LOW_STOCK_THRESHOLD)", + "replace": " .filter((i) => (stockByItem.get(i.id) ?? 0) < LOW_STOCK_THRESHOLD)" + } + ] + }, + { + "id": "category-totals-ignore-pending-purchases", + "scenario": "tracks/ecommerce/scenarios/02-operational-category-totals.json", + "targets": [ + "ecommerce.inventory-operations.operational-views.5b" + ], + "desc": "The category totals view includes only shipped orders, so a newly accepted pending purchase is absent.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " for (const order of ctx.db.customerOrder.iter()) {\n if (!isOrderCounted(order)) continue;", + "replace": " for (const order of ctx.db.customerOrder.iter()) {\n if (order.status !== 'shipped') continue;" + } + ] + }, + { + "id": "recommendations-ignore-pending-purchases", + "scenario": "tracks/ecommerce/scenarios/02-operational-recommendations.json", + "targets": [ + "ecommerce.inventory-operations.operational-views.5c" + ], + "desc": "The personal recommendation view derives categories only from shipped orders, so a new pending purchase has no influence.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " for (const order of ctx.db.customerOrder.accountId.filter(accountId)) {\n if (!isOrderCounted(order)) continue;", + "replace": " for (const order of ctx.db.customerOrder.accountId.filter(accountId)) {\n if (order.status !== 'shipped') continue;" + } + ] + }, + { + "id": "purchases-do-not-affect-best-sellers", + "scenario": "tracks/ecommerce/scenarios/02-operational-best-sellers.json", + "targets": [ + "ecommerce.inventory-operations.operational-views.5d" + ], + "desc": "Rank signed-out recommendations without purchase counts.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "const purchaseCountOf = (id: bigint) => ctx.db.itemStats.itemId.find(id)?.purchaseCount ?? 0;", + "replace": "const purchaseCountOf = (_id: bigint) => 0;" + } + ] + }, + { + "id": "queue-warehouse-reports-west", + "scenario": "tracks/ecommerce/scenarios/02-queue-warehouse.json", + "targets": [ + "ecommerce.operations-access.fulfilment-queue.1b" + ], + "desc": "The queue renders West for the deterministic Desk Lamp allocation even though the order reserved stock in East.", + "file": "client/src/components/FulfilmentPanel.tsx", + "edits": [ + { + "find": "{name}: {order.warehouseNames[i]}", + "replace": "{name}: West" + } + ] + }, + { + "id": "transfer-creates-stock-during-race", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202d" + ], + "desc": "The transfer reducer credits the destination one unit more than it debits from the source, so the item's total after a transfer racing a purchase is the starting total rather than one less. A stored conservation defect; it does not model a lost-update interleaving because SpacetimeDB reducer execution is atomically serialized.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ctx.db.stock.insert({ ...toRow, quantity: toRow.quantity + quantity });", + "replace": " ctx.db.stock.insert({ ...toRow, quantity: toRow.quantity + quantity + 1 });" + } + ] + }, + { + "id": "catalog-search-ignores-the-query", + "scenario": "tracks/ecommerce/scenarios/01-catalog-search.json", + "targets": [ + "ecommerce.feature.catalog.catalog-search.2d" + ], + "desc": "A non-empty catalog query filters out every product instead of matching names.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": ".filter(item => !q || item.name.toLowerCase().includes(q))", + "replace": ".filter(() => !q) // mutant: non-empty searches return no products" + } + ] + }, + { + "id": "admin-total-stock-is-not-rendered", + "scenario": "tracks/ecommerce/scenarios/01-warehouse-stock-live-staff.json", + "targets": [ + "ecommerce.spec.live-state.warehouse-stock.7c" + ], + "desc": "The staff stock total always renders zero after a warehouse restock.", + "file": "client/src/components/AdminPanel.tsx", + "edits": [ + { + "find": "{totalStockOf(item.id)}", + "replace": "{0}" + } + ] + }, + { + "id": "customers-can-schedule-restocks", + "scenario": "tracks/ecommerce/scenarios/03-deferred-access.json", + "targets": [ + "ecommerce.l3.deferred-access.scheduled-work-access.317a" + ], + "desc": "Scheduling a restock no longer checks that the caller is an administrator.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " (ctx, input) => {\n requireAdmin(ctx);\n const itemName = input.item.trim();\n const warehouseName = input.warehouse.trim();\n const item = [...ctx.db.item].find(row => row.name === itemName);\n const warehouse = [...ctx.db.warehouse].find(row => row.name === warehouseName);", + "replace": " (ctx, input) => {\n // mutant: any signed-in or anonymous caller can schedule work\n const itemName = input.item.trim();\n const warehouseName = input.warehouse.trim();\n const item = [...ctx.db.item].find(row => row.name === itemName);\n const warehouse = [...ctx.db.warehouse].find(row => row.name === warehouseName);" + } + ] + }, + { + "id": "scheduled-restock-execution-queue-is-process-local", + "scenario": "tracks/ecommerce/scenarios/03-server-time.json", + "targets": [ + "ecommerce.l3.server-time.server-time.312a" + ], + "desc": "Keep manual restock execution IDs only in the V8 process. Ordinary timers work, but restart loses the execution queue while pending rows remain. Isolate replacement can also lose this queue.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "export const scheduleRestock = spacetimedb.reducer(", + "replace": "const pendingRestockExecution = new Set();\n\nexport const scheduleRestock = spacetimedb.reducer(" + }, + { + "find": " ctx.db.scheduledRestock.insert({\n id: 0n,\n itemId: item.id,\n warehouseId: warehouse.id,\n quantity: input.quantity,\n dueMicros: nowMicros(ctx) + BigInt(input.delaySeconds) * SECOND,\n status: 'pending',\n reorderRuleId: undefined,\n });", + "replace": " const pending = ctx.db.scheduledRestock.insert({\n id: 0n,\n itemId: item.id,\n warehouseId: warehouse.id,\n quantity: input.quantity,\n dueMicros: nowMicros(ctx) + BigInt(input.delaySeconds) * SECOND,\n status: 'pending',\n reorderRuleId: undefined,\n });\n pendingRestockExecution.add(pending.id);" + }, + { + "find": " if (pending.status !== 'pending' || pending.dueMicros > now) continue;\n restoreStock(ctx, pending.itemId, pending.warehouseId, pending.quantity);", + "replace": " if (pending.status !== 'pending' || pending.dueMicros > now) continue;\n if (pending.reorderRuleId === undefined && !pendingRestockExecution.delete(pending.id)) continue;\n restoreStock(ctx, pending.itemId, pending.warehouseId, pending.quantity);" + } + ] + }, + { + "id": "reservation-is-delayed-past-the-durability-window", + "scenario": "tracks/ecommerce/scenarios/03-deferred-durability.json", + "targets": [ + "ecommerce.l3.deferred-durability.restart-survival.314a" + ], + "desc": "A reservation is persisted with a ten-minute lifetime instead of ninety seconds.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "const expiresMicros = nowMicros(ctx) + 90n * SECOND;", + "replace": "const expiresMicros = nowMicros(ctx) + 600n * SECOND;" + } + ] + }, + { + "id": "completed-restock-remains-pending", + "scenario": "tracks/ecommerce/scenarios/03-deferred-integrity.json", + "targets": [ + "ecommerce.l3.deferred-integrity.exactly-once.311a" + ], + "desc": "A completed restock remains pending and is applied again by later maintenance ticks.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.scheduledRestock.id.update({ ...pending, status: 'complete' });", + "replace": "ctx.db.scheduledRestock.id.update({ ...pending, status: 'pending' });" + } + ] + }, + { + "id": "reservation-expiry-restores-stock-twice", + "scenario": "tracks/ecommerce/scenarios/03-deferred-integrity.json", + "targets": [ + "ecommerce.l3.deferred-integrity.stock-conservation.313a" + ], + "desc": "Reservation expiry returns twice the quantity that was reserved.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "restoreStock(ctx, row.stockItemId || row.itemId, row.warehouseId, row.quantity);\n ctx.db.reservation.id.update({ ...row, expired: true });", + "replace": "restoreStock(ctx, row.stockItemId || row.itemId, row.warehouseId, row.quantity * 2);\n ctx.db.reservation.id.update({ ...row, expired: true });" + } + ] + }, + { + "id": "checkout-takes-reserved-stock-again", + "scenario": "tracks/ecommerce/scenarios/03-deferred-integrity.json", + "targets": [ + "ecommerce.l3.deferred-integrity.stock-conservation.314a" + ], + "desc": "Checkout decrements stock after the cart reservation already took it.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "const allocations = held.map(row => ({ warehouseId: row.warehouseId, quantity: row.quantity, stockItemId: row.stockItemId }));\n const orderItemRow", + "replace": "const allocations = decrementStockTracked(ctx, p.itemId, p.quantity);\n const orderItemRow" + } + ] + }, + { + "id": "reservation-does-not-decrement-stock", + "scenario": "tracks/ecommerce/scenarios/03-reservations.json", + "targets": [ + "ecommerce.l3.reservations.reservations.301a" + ], + "desc": "Creating a reservation leaves the public stock total unchanged.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.stock.insert({ ...row, quantity: row.quantity - allocation.quantity });", + "replace": "ctx.db.stock.insert({ ...row, quantity: row.quantity });" + } + ] + }, + { + "id": "reservation-timer-is-static", + "scenario": "tracks/ecommerce/scenarios/03-reservations.json", + "targets": [ + "ecommerce.l3.reservations.reservations.305a" + ], + "desc": "The cart always renders ninety seconds instead of a decreasing reservation timer.", + "file": "client/src/components/CartPanel.tsx", + "edits": [ + { + "find": "const seconds = Math.max(0, Number((reservation.expiresMicros - BigInt(Date.now()) * 1000n) / 1_000_000n));", + "replace": "const seconds = 90;" + } + ] + }, + { + "id": "checkout-leaves-cart-lines", + "scenario": "tracks/ecommerce/scenarios/03-reservations.json", + "targets": [ + "ecommerce.l3.reservations.reservations.306a" + ], + "desc": "Checkout creates an order but leaves the purchased lines in the cart.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "for (const line of lines) ctx.db.cartItem.id.delete(line.id);", + "replace": "for (const line of lines) void line;" + } + ] + }, + { + "id": "expired-reservation-is-still-marked-live", + "scenario": "tracks/ecommerce/scenarios/03-reservations.json", + "targets": [ + "ecommerce.l3.reservations.reservations.307a" + ], + "desc": "Expired reservations keep their live flag, so the cart does not mark them expired.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.reservation.id.update({ ...row, expired: true });", + "replace": "ctx.db.reservation.id.update({ ...row, expired: false });" + } + ] + }, + { + "id": "renewed-reservation-expires-too-soon", + "scenario": "tracks/ecommerce/scenarios/03-reservations.json", + "targets": [ + "ecommerce.l3.reservations.reservations.308a" + ], + "desc": "Renewed quantities receive only a twenty-second reservation window.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "reserveUnits(ctx, accountId, itemId, quantity);", + "replace": "reserveUnits(ctx, accountId, itemId, quantity);\n for (const renewed of findReservations(ctx, accountId, itemId)) {\n ctx.db.reservation.id.update({ ...renewed, expiresMicros: nowMicros(ctx) + 20n * SECOND });\n }" + } + ] + }, + { + "id": "pending-restock-timer-is-static", + "scenario": "tracks/ecommerce/scenarios/03-scheduled-restocks.json", + "targets": [ + "ecommerce.l3.scheduled-restocks.scheduled-restocks.302a" + ], + "desc": "The pending restock UI always renders ninety seconds instead of the server due time.", + "file": "client/src/components/ProgressionWorkbench.tsx", + "edits": [ + { + "find": "{Math.max(0, Number((row.dueMicros - BigInt(Date.now()) * 1000n) / 1_000_000n))}", + "replace": "{90}" + } + ] + }, + { + "id": "due-restock-omits-ledger-entry", + "scenario": "tracks/ecommerce/scenarios/03-scheduled-restock-apply.json", + "targets": [ + "ecommerce.l3.scheduled-restocks.scheduled-restocks.305a" + ], + "desc": "A due restock updates stock but does not create its stock ledger record.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.stockLedger.insert({\n id: 0n,\n itemId: pending.itemId,\n warehouseId: pending.warehouseId,\n quantity: pending.quantity,\n createdMicros: now,\n source: 'scheduled restock',\n });", + "replace": "// mutant: due restocks are not recorded in the ledger" + } + ] + }, + { + "id": "cancelled-restock-remains-pending", + "scenario": "tracks/ecommerce/scenarios/03-scheduled-restock-cancel.json", + "targets": [ + "ecommerce.l3.scheduled-restocks.scheduled-restocks.306a" + ], + "desc": "Cancelling a restock leaves it pending, so maintenance later applies it.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.scheduledRestock.id.update({ ...row, status: 'cancelled' });", + "replace": "ctx.db.scheduledRestock.id.update({ ...row, status: 'pending' });" + } + ] + }, + { + "id": "restart-restock-runs-early", + "scenario": "tracks/ecommerce/scenarios/03-server-time.json", + "targets": [ + "ecommerce.l3.server-time.server-time.312a" + ], + "desc": "A scheduled restock ignores its requested delay and becomes due after one second.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "dueMicros: nowMicros(ctx) + BigInt(input.delaySeconds) * SECOND,", + "replace": "dueMicros: nowMicros(ctx) + SECOND," + } + ] + }, + { + "id": "closed-browser-reservation-never-expires", + "scenario": "tracks/ecommerce/scenarios/03-server-time.json", + "targets": [ + "ecommerce.l3.server-time.server-time.313a" + ], + "desc": "Server maintenance skips reservation expiry when no customer browser is present.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "for (const row of [...ctx.db.reservation.iter()]) {\n if (row.expired || row.expiresMicros > now) continue;", + "replace": "for (const row of [...ctx.db.reservation.iter()].filter(() => false)) {\n if (row.expired || row.expiresMicros > now) continue;" + } + ] + }, + { + "id": "catalog-product-is-not-published", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-management.json", + "targets": [ + "ecommerce.progression.catalog-management.catalog-management.622a", + "ecommerce.progression.catalog-management.catalog-management.622b" + ], + "desc": "The public product card omits the submitted name.", + "file": "client/src/components/ItemCard.tsx", + "edits": [ + { + "find": "{item.name}", + "replace": "{item.name === 'Travel Mug' ? '' : item.name}" + } + ] + }, + { + "id": "catalog-variants-are-discarded", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-management.json", + "targets": [ + "ecommerce.progression.catalog-management.catalog-management.622b" + ], + "desc": "Catalog management discards every submitted product variant.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "for (const variantName of variants.split(',').map(value => value.trim()).filter(Boolean)) {\n ctx.db.itemVariant.insert({ id: 0n, itemId: product.id, name: variantName });\n }", + "replace": "void variants; // mutant: submitted variants are discarded" + } + ] + }, + { + "id": "profile-is-lost-on-fresh-account-login", + "scenario": "tracks/ecommerce/scenarios/progression-customer-profile.json", + "targets": [ + "ecommerce.spec.state-durability.customer-profile-reload.620a" + ], + "desc": "Delete a saved profile when its owner signs in from a new transport identity. Initial save, same-identity reload, and the independent privacy owner remain intact.", + "file": "backend/spacetimedb/src/auth.ts", + "edits": [ + { + "find": " if (!current || current.passwordHash !== account.passwordHash) return false;\n bindSession(tx, account.id);", + "replace": " if (!current || current.passwordHash !== account.passwordHash) return false;\n tx.db.customerProfile.accountId.delete(account.id);\n bindSession(tx, account.id);" + } + ] + }, + { + "id": "customer-profile-view-leaks-another-account", + "scenario": "tracks/ecommerce/scenarios/progression-customer-profile.json", + "targets": [ + "ecommerce.spec.access-control.customer-profile-privacy.620b" + ], + "desc": "The customer profile view returns the first stored profile without checking its owner.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "return accountId === null ? undefined : ctx.db.customerProfile.accountId.find(accountId) ?? undefined;", + "replace": "return accountId === null ? undefined : [...ctx.db.customerProfile.iter()][0];" + } + ] + }, + { + "id": "faceted-search-ignores-category", + "scenario": "tracks/ecommerce/scenarios/progression-faceted-filters.json", + "targets": [ + "ecommerce.progression.faceted-search.faceted-search.401a" + ], + "desc": "Faceted search applies price and stock filters but ignores the selected category.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": ".filter(item => !categoryFilter || categoryByItem.get(item.id) === categoryFilter)", + "replace": ".filter(() => true) // mutant: category filter is ignored" + } + ] + }, + { + "id": "active-search-uses-purchase-ranking", + "scenario": "tracks/ecommerce/scenarios/progression-search-ordering.json", + "targets": [ + "ecommerce.spec.search-ordering.search-ordering.402b" + ], + "desc": "Keep purchase ranking when search text or filters are active.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "const catalogItems = showingSearch ? filteredSearchResults : rankedItems;", + "replace": "const catalogItems = rankedItems;" + } + ] + }, + { + "id": "faceted-search-next-page-does-not-advance", + "scenario": "tracks/ecommerce/scenarios/progression-faceted-pagination.json", + "targets": [ + "ecommerce.progression.faceted-search.faceted-search.402a" + ], + "desc": "The next-page control keeps the search on its current page.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "onClick={() => setSearchPage(page => page + 1)}>Next", + "replace": "onClick={() => setSearchPage(page => page)}>Next" + } + ] + }, + { + "id": "managed-support-leaks-and-accepts-cross-account-replies", + "scenario": "tracks/ecommerce/scenarios/progression-managed-support-privacy.json", + "targets": [ + "ecommerce.spec.access-control.managed-support-privacy.613b" + ], + "desc": "Managed support tickets are visible across accounts and replayed replies are accepted.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": ".filter(row => isGuestTicketCreator(sender, row.creatorIdentity.toHexString(), row.accountId) ||\n !!actor && (actor.isAdmin || actor.isStaff || row.accountId === accountId))", + "replace": ".filter(() => true)" + }, + { + "file": "backend/spacetimedb/src/index.ts", + "find": "if (!actor.isAdmin && !actor.isStaff && ticket.accountId !== actor.id) {\n throw new SenderError('That support ticket is private.');\n }", + "replace": "// mutant: any signed-in account can access any support ticket" + } + ] + }, + { + "id": "managed-support-replies-are-empty", + "scenario": "tracks/ecommerce/scenarios/progression-managed-support-shared.json", + "targets": [ + "ecommerce.spec.live-state.managed-support.613a", + "ecommerce.progression.managed-support.managed-support.613c" + ], + "desc": "Managed support stores replies without their message body. Both the ordinary reply and shared-live reply assertions require the stored message body; neither can pass an empty reply.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "body: body.trim(),\n createdMicros: nowMicros(ctx),", + "replace": "body: '',\n createdMicros: nowMicros(ctx)," + } + ] + }, + { + "id": "managed-support-live-replies-stay-at-initial-snapshot", + "scenario": "tracks/ecommerce/scenarios/progression-managed-support-shared.json", + "targets": [ + "ecommerce.spec.live-state.managed-support.613a" + ], + "desc": "Keep the initial subscribed reply snapshot on each page. Reducers still save replies, and a reload shows them, but later replies do not reach the rendered conversation live.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const [supportReplyRows] = useTable(tables.visibleSupportReplies);", + "replace": " const [liveSupportReplyRows, supportRepliesReady] = useTable(tables.visibleSupportReplies);\n const supportReplyRows = useMemo(() => [...liveSupportReplyRows], [supportRepliesReady]);" + } + ] + }, + { + "id": "notification-preferences-are-not-saved", + "scenario": "tracks/ecommerce/scenarios/progression-notification-preferences.json", + "targets": [ + "ecommerce.spec.state-durability.notification-preferences-reload.630a", + "ecommerce.spec.access-control.notification-preferences-privacy.630b" + ], + "desc": "Saving notification preferences discards the selected values. It also breaks the owner's saved-choice positive control in 630b; that coupled failure is not evidence of a privacy defect.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "if (existing) ctx.db.notificationPreference.accountId.update(row);\n else ctx.db.notificationPreference.insert(row);", + "replace": "void existing;\n void row; // mutant: notification preferences are discarded" + } + ] + }, + { + "id": "notification-preferences-leak-across-accounts", + "scenario": "tracks/ecommerce/scenarios/progression-notification-preferences.json", + "targets": [ + "ecommerce.spec.access-control.notification-preferences-privacy.630b" + ], + "desc": "The preference view returns another account's first stored choice.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "const row = ctx.db.notificationPreference.accountId.find(accountId);\n return row ? { orderEnabled: row.orderEnabled, stockEnabled: row.stockEnabled } : undefined;", + "replace": "const row = [...ctx.db.notificationPreference.iter()][0];\n return row ? { orderEnabled: row.orderEnabled, stockEnabled: row.stockEnabled } : undefined;" + } + ] + }, + { + "id": "checkout-records-zero-payment", + "scenario": "tracks/ecommerce/scenarios/progression-core-business.json", + "targets": [ + "ecommerce.progression.payment-records.payment-records.623a" + ], + "desc": "Checkout records a paid payment with a zero amount.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.paymentRecord.insert({ id: 0n, orderId: order.id, amount: order.total, status: 'paid' });\n if (promo) {", + "replace": "ctx.db.paymentRecord.insert({ id: 0n, orderId: order.id, amount: 0, status: 'paid' });\n if (promo) {" + } + ] + }, + { + "id": "checkout-records-duplicate-payments", + "scenario": "tracks/ecommerce/scenarios/progression-core-business.json", + "targets": [ + "ecommerce.spec.transactional-integrity.payment-deduplication.623b" + ], + "desc": "Checkout inserts two payment records for one order.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.paymentRecord.insert({ id: 0n, orderId: order.id, amount: order.total, status: 'paid' });\n if (promo) {", + "replace": "ctx.db.paymentRecord.insert({ id: 0n, orderId: order.id, amount: order.total, status: 'paid' });\n ctx.db.paymentRecord.insert({ id: 0n, orderId: order.id, amount: order.total + 0.01, status: 'paid' });\n if (promo) {" + } + ] + }, + { + "id": "active-promotion-does-not-discount-checkout", + "scenario": "tracks/ecommerce/scenarios/progression-promotion-checkout.json", + "targets": [ + "ecommerce.progression.promotion-checkout.promotion-checkout-active.621a" + ], + "desc": "Checkout ignores an active promotion when it calculates the discount.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "const discount = promo ? total * (promo.discountPercent / 100) : 0;", + "replace": "const discount = 0;" + } + ] + }, + { + "id": "expired-promotion-is-accepted", + "scenario": "tracks/ecommerce/scenarios/progression-promotion-checkout.json", + "targets": [ + "ecommerce.progression.promotion-checkout.promotion-checkout-expired.621b" + ], + "desc": "Promotion application does not reject a promotion after its end time.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "if (!promo || promo.startMicros > now || promo.endMicros < now || promo.redemptions >= promo.usageLimit) {", + "replace": "if (!promo || promo.startMicros > now || promo.redemptions >= promo.usageLimit) {" + } + ] + }, + { + "id": "exhausted-promotion-is-accepted", + "scenario": "tracks/ecommerce/scenarios/progression-promotion-checkout.json", + "targets": [ + "ecommerce.progression.promotion-checkout.promotion-checkout-exhausted.621c" + ], + "desc": "Promotion application does not reject a promotion at its usage limit.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "if (!promo || promo.startMicros > now || promo.endMicros < now || promo.redemptions >= promo.usageLimit) {", + "replace": "if (!promo || promo.startMicros > now || promo.endMicros < now) {" + } + ] + }, + { + "id": "customers-can-create-promotions", + "scenario": "tracks/ecommerce/scenarios/progression-promotion-rules.json", + "targets": [ + "ecommerce.spec.access-control.promotion-management-boundary.620b" + ], + "desc": "Promotion creation does not require staff access.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " (ctx, input) => {\n requireStaffOrAdmin(ctx);\n if (input.discountPercent <= 0 || input.discountPercent > 100) {", + "replace": " (ctx, input) => {\n if (input.discountPercent <= 0 || input.discountPercent > 100) {" + } + ] + }, + { + "id": "promotion-rule-stores-the-wrong-discount", + "scenario": "tracks/ecommerce/scenarios/progression-promotion-rules.json", + "targets": [ + "ecommerce.progression.promotion-rules.promotion-rule-values.620a" + ], + "desc": "Promotion creation stores a one-percent discount instead of the submitted value.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.promotion.insert({ id: 0n, ...input, code: input.code.trim(), redemptions: 0 });", + "replace": "ctx.db.promotion.insert({ id: 0n, ...input, discountPercent: 1, code: input.code.trim(), redemptions: 0 });" + } + ] + }, + { + "id": "staff-cannot-open-staff-tools", + "scenario": "tracks/ecommerce/scenarios/progression-staff-access.json", + "targets": [ + "ecommerce.progression.staff-access.staff-access.601a" + ], + "desc": "Deny administrators entry to staff tools while keeping ordinary staff access working.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "{(isStaff || isAdmin) && (\n ({", + "replace": "return [...ctx.db.notification.iter()].map(row => ({" + } + ] + }, + { + "id": "support-history-is-lost-on-fresh-account-login", + "scenario": "tracks/ecommerce/scenarios/progression-support-history.json", + "targets": [ + "ecommerce.spec.state-durability.support-history-reload.612a" + ], + "desc": "Resolve customer support ownership by transport identity only, omitting the account ownership path. Initial submission, same-identity reload, and stored tickets remain intact; fresh account login cannot recover the history.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "!!actor && (actor.isAdmin || actor.isStaff || row.accountId === accountId))", + "replace": "!!actor && (actor.isAdmin || actor.isStaff || row.creatorIdentity.toHexString() === sender))" + } + ] + }, + { + "id": "support-history-leaks-across-customers", + "scenario": "tracks/ecommerce/scenarios/progression-support-history.json", + "targets": [ + "ecommerce.spec.access-control.support-history-privacy.612b", + "ecommerce.spec.access-control.support-history-logout.612d" + ], + "desc": "Return all support tickets, exposing them to other customers and signed-out visitors.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": ".filter(row => isGuestTicketCreator(sender, row.creatorIdentity.toHexString(), row.accountId) ||\n !!actor && (actor.isAdmin || actor.isStaff || row.accountId === accountId))", + "replace": ".filter(() => true)" + } + ] + }, + { + "id": "visitor-support-reference-is-hidden", + "scenario": "tracks/ecommerce/scenarios/progression-support-intake.json", + "targets": [ + "ecommerce.progression.support-intake.support-intake.610a" + ], + "desc": "A visitor can create a support ticket but the returned reference is not rendered.", + "file": "client/src/components/ProgressionWorkbench.tsx", + "edits": [ + { + "find": "
{supportReference}
", + "replace": "
" + } + ] + }, + { + "id": "support-assignment-is-discarded", + "scenario": "tracks/ecommerce/scenarios/progression-support-triage.json", + "targets": [ + "ecommerce.progression.support-triage.support-assignment.611a" + ], + "desc": "Support triage saves status and priority but discards the assignee.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.supportTicket.id.update({ ...ticket, assigneeId: nextAssigneeId, priority, status });", + "replace": "ctx.db.supportTicket.id.update({ ...ticket, assigneeId: undefined, priority, status });" + } + ] + }, + { + "id": "support-priority-is-discarded", + "scenario": "tracks/ecommerce/scenarios/progression-support-triage.json", + "targets": [ + "ecommerce.progression.support-triage.support-priority.611b" + ], + "desc": "Support triage always saves normal priority instead of the submitted value.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.supportTicket.id.update({ ...ticket, assigneeId: nextAssigneeId, priority, status });", + "replace": "ctx.db.supportTicket.id.update({ ...ticket, assigneeId: nextAssigneeId, priority: 'normal', status });" + } + ] + }, + { + "id": "support-status-is-discarded", + "scenario": "tracks/ecommerce/scenarios/progression-support-triage.json", + "targets": [ + "ecommerce.progression.support-triage.support-status.611c" + ], + "desc": "Support triage preserves the old status instead of the submitted value.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "ctx.db.supportTicket.id.update({ ...ticket, assigneeId: nextAssigneeId, priority, status });", + "replace": "ctx.db.supportTicket.id.update({ ...ticket, assigneeId: nextAssigneeId, priority, status: ticket.status });" + } + ] + }, + { + "id": "nonpositive-cart-quantity-is-treated-as-removal", + "scenario": "tracks/ecommerce/scenarios/01-cart-boundary.json", + "targets": [ + "ecommerce.spec.access-control.cart-boundary.109b" + ], + "desc": "Accept a negative quantity and remove the cart line instead of refusing the request.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (quantity < 1) throw new SenderError('Quantity must be at least 1.');\n const existing = findCartLine(ctx, acc.id, itemId);\n if (!existing) throw new SenderError('That item is not in your cart.');\n if (existing.bundlePrice > 0) throw new SenderError('Remove and re-add a whole bundle.');\n replaceReservation(ctx, acc.id, itemId, quantity);\n ctx.db.cartItem.id.update({ ...existing, quantity });", + "replace": " const existing = findCartLine(ctx, acc.id, itemId);\n if (!existing) throw new SenderError('That item is not in your cart.');\n if (quantity < 1) {\n ctx.db.cartItem.id.delete(existing.id);\n return;\n }\n if (existing.bundlePrice > 0) throw new SenderError('Remove and re-add a whole bundle.');\n replaceReservation(ctx, acc.id, itemId, quantity);\n ctx.db.cartItem.id.update({ ...existing, quantity });" + } + ] + }, + { + "id": "admin-restock-preserves-existing-stock", + "scenario": "tracks/ecommerce/scenarios/01-warehouse-stock-live-staff.json", + "targets": [ + "ecommerce.spec.live-state.warehouse-stock.7c" + ], + "desc": "Accept an administrator restock but write the existing quantity back unchanged.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ctx.db.stock.insert({ ...existing, quantity: existing.quantity + quantity });", + "replace": " ctx.db.stock.insert({ ...existing, quantity: existing.quantity });" + } + ] + }, + { + "id": "operator-authorization-allows-customer-price-change", + "scenario": "tracks/ecommerce/scenarios/02-strengthened.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201b" + ], + "desc": "The price reducer drops its administrator role gate.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " (ctx, { itemId, price }) => {\n requireAdmin(ctx);", + "replace": " (ctx, { itemId, price }) => {\n // mutant: no administrator role check" + } + ] + }, + { + "id": "fulfilment-queue-allows-customer-shipping", + "scenario": "tracks/ecommerce/scenarios/02-self-contained.json", + "targets": [ + "ecommerce.operations-access.fulfilment-queue.1e" + ], + "desc": "The shipping reducer drops the staff role check while retaining the pending-order guard.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n requireStaffOrAdmin(ctx);", + "replace": "export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n // mutant: no staff role check" + } + ] + }, + { + "id": "catalog-search-keeps-pre-change-price", + "scenario": "tracks/ecommerce/scenarios/02-live-price.json", + "targets": [ + "ecommerce.returns-pricing.price-history.4b" + ], + "desc": "The search result renderer caches each item's first visible price and ignores later live price updates.", + "file": "client/src/components/ItemCard.tsx", + "edits": [ + { + "find": "import { ItemRow } from '../types';", + "replace": "import { useRef } from 'react';\nimport { ItemRow } from '../types';" + }, + { + "find": " const outOfStock = stock <= 0;\n const lowStock = !outOfStock && stock <= 5;", + "replace": " const outOfStock = stock <= 0;\n const lowStock = !outOfStock && stock <= 5;\n // mutant: the card retains the first price it renders\n const firstPrice = useRef(item.price);" + }, + { + "find": " {formatMoney(item.price)}", + "replace": " {formatMoney(firstPrice.current)}" + } + ] + }, + { + "id": "open-cart-keeps-pre-change-price", + "scenario": "tracks/ecommerce/scenarios/progression-price-cart-checkout.json", + "targets": [ + "ecommerce.returns-pricing.price-history.4c" + ], + "desc": "The open-cart memo ignores reactive item-table price updates while checkout still reads the current server price.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " [cartRows, items, stockByItem]", + "replace": " [cartRows, stockByItem]" + } + ] + }, + { + "id": "catalog-price-rewrites-receipts", + "scenario": "tracks/ecommerce/scenarios/02-paid-price-history.json", + "targets": [ + "ecommerce.returns-pricing.price-history.4a" + ], + "desc": "Changing a catalog price cascades into saved order lines and recomputes historical order totals inside the reducer transaction.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ctx.db.item.id.update({ ...it, price });", + "replace": " ctx.db.item.id.update({ ...it, price });\n const repricedOrderIds = new Set();\n for (const line of ctx.db.orderItem.iter()) {\n if (line.itemId !== itemId) continue;\n ctx.db.orderItem.id.update({ ...line, unitPrice: price });\n repricedOrderIds.add(line.orderId);\n }\n for (const orderId of repricedOrderIds) {\n const historical = ctx.db.customerOrder.id.find(orderId);\n if (!historical) continue;\n let total = 0;\n for (const line of ctx.db.orderItem.orderId.filter(orderId)) total += line.unitPrice * line.quantity;\n ctx.db.customerOrder.id.update({ ...historical, total });\n }" + } + ] + }, + { + "id": "catalog-price-rewrites-earned-revenue", + "scenario": "tracks/ecommerce/scenarios/02-invariants.json", + "targets": [ + "ecommerce.returns-pricing.refund-accounting.203b" + ], + "desc": "Changing a catalog price cascades into saved order lines and recomputes historical order totals and revenue inside the reducer transaction.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ctx.db.item.id.update({ ...it, price });", + "replace": " ctx.db.item.id.update({ ...it, price });\n const repricedOrderIds = new Set();\n for (const line of ctx.db.orderItem.iter()) {\n if (line.itemId !== itemId) continue;\n ctx.db.orderItem.id.update({ ...line, unitPrice: price });\n repricedOrderIds.add(line.orderId);\n }\n for (const orderId of repricedOrderIds) {\n const historical = ctx.db.customerOrder.id.find(orderId);\n if (!historical) continue;\n let total = 0;\n for (const line of ctx.db.orderItem.orderId.filter(orderId)) total += line.unitPrice * line.quantity;\n ctx.db.customerOrder.id.update({ ...historical, total });\n }" + } + ] + }, + { + "id": "returned-line-marker-omitted", + "scenario": "tracks/ecommerce/scenarios/02-strengthened.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3c" + ], + "desc": "A returned order line keeps its persisted returned state, restored stock, and adjusted revenue but omits the visible returned marker.", + "file": "client/src/components/OrdersPanel.tsx", + "edits": [ + { + "find": "{item.returned && Returned}", + "replace": "{false && Returned}" + } + ] + }, + { + "id": "direct-review-access-is-not-checked", + "scenario": "tracks/ecommerce/scenarios/progression-review-access.json", + "targets": [ + "ecommerce.progression.review-access-specifications.review-eligibility-direct.618a" + ], + "desc": "The direct review action accepts a review from a customer who did not buy the item.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (!bought) throw new SenderError('You can only review items you have purchased.');", + "replace": " // mutant: purchase eligibility is not checked" + } + ] + }, + { + "id": "support-history-rows-are-hidden", + "scenario": "tracks/ecommerce/scenarios/progression-support-history.json", + "targets": [ + "ecommerce.progression.support-history.support-history.612c", + "ecommerce.spec.state-durability.support-history-reload.612a", + "ecommerce.spec.access-control.support-history-privacy.612b", + "ecommerce.spec.access-control.support-history-logout.612d" + ], + "desc": "Hide submitted support ticket rows while leaving the submission reference available. History, reload, privacy, and logout require the owner to see the ticket first. This breaks those positive observations; it does not create a privacy leak or remove stored data.", + "file": "client/src/components/ProgressionWorkbench.tsx", + "edits": [ + { + "find": "data-role=\"support-ticket\"", + "replace": "data-role=\"support-ticket\" style={{ display: \"none\" }}" + } + ] + }, + { + "id": "authorized-restock-does-not-change-stock", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.feature.warehouse-admin.admin-write.103a", + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Accept an administrator restock without changing warehouse stock.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (existing) {\n ctx.db.stock.by_item_warehouse.delete([itemId, warehouseId]);\n ctx.db.stock.insert({ ...existing, quantity: existing.quantity + quantity });\n } else {\n ctx.db.stock.insert({ item_id: itemId, warehouse_id: warehouseId, quantity });\n }\n", + "replace": " // mutant: accept restock without changing stock\n" + } + ] + }, + { + "id": "low-stock-threshold-is-two-units", + "scenario": "tracks/ecommerce/scenarios/02-low-stock.json", + "targets": [ + "ecommerce.inventory-operations.operational-views.5e", + "ecommerce.spec.live-state.inventory-dashboard.5a" + ], + "desc": "The dashboard lists only items with two units or fewer, so the seeded three-unit item is missing from the low-stock view. The live check in the same scenario opens with the identical observation and necessarily fails with it.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "const LOW_STOCK_THRESHOLD = 10;", + "replace": "const LOW_STOCK_THRESHOLD = 2;" + } + ] + }, + { + "id": "category-totals-count-only-since-the-dashboard-opened", + "scenario": "tracks/ecommerce/scenarios/02-operational-category-totals.json", + "targets": [ + "ecommerce.inventory-operations.operational-views.5f" + ], + "desc": "The category table shows units and revenue accumulated since the dashboard was opened instead of the stored totals, so a reload resets both to zero and the totals recorded before the reload are not reproduced. Live movement within one open dashboard is still correct, so the live check is unaffected.", + "file": "client/src/components/AdminPanel.tsx", + "edits": [ + { + "find": "import { useState } from 'react';", + "replace": "import { useRef, useState } from 'react';" + }, + { + "find": " return (\n
", + "replace": " const openingTotals = useRef | null>(null);\n if (openingTotals.current === null && categoryTotals.length > 0) {\n openingTotals.current = new Map(\n categoryTotals.map((cat): [bigint, { units: number; revenue: number }] => [\n cat.categoryId,\n { units: cat.unitsSold, revenue: cat.revenue },\n ])\n );\n }\n const sessionTotals = categoryTotals.map((cat) => {\n const opening = openingTotals.current?.get(cat.categoryId);\n return {\n ...cat,\n unitsSold: cat.unitsSold - (opening?.units ?? 0),\n revenue: cat.revenue - (opening?.revenue ?? 0),\n };\n });\n\n return (\n
" + }, + { + "find": " {categoryTotals.map((cat) => (", + "replace": " {sessionTotals.map((cat) => (" + } + ] + }, + { + "id": "profile-summary-ignores-a-profile-saved-this-session", + "scenario": "tracks/ecommerce/scenarios/progression-customer-profile.json", + "targets": [ + "ecommerce.progression.customer-profile.customer-profile.620c" + ], + "desc": "Hide the profile summary immediately after saving in the current view. Stored profile data and a reopened or reloaded view remain correct, so fresh-login durability and privacy positive controls remain observable.", + "file": "client/src/components/ProgressionWorkbench.tsx", + "edits": [ + { + "find": " const [profileName, setProfileName] = useState(profile?.name ?? '');", + "replace": " const [profileName, setProfileName] = useState(profile?.name ?? '');\n const [profileSavedHere, setProfileSavedHere] = useState(false);" + }, + { + "find": "onClick={async () => {\n setProfileSaveState('pending');\n try {\n if (!reducers) throw new Error('Not connected');\n await reducers.saveProfile({ name: profileName, address: profileAddress });\n setProfileSaveState('succeeded');\n } catch { setProfileSaveState('failed'); }\n }}", + "replace": "onClick={async () => {\n setProfileSavedHere(true);\n setProfileSaveState('pending');\n try {\n if (!reducers) throw new Error('Not connected');\n await reducers.saveProfile({ name: profileName, address: profileAddress });\n setProfileSaveState('succeeded');\n } catch { setProfileSaveState('failed'); }\n }}" + }, + { + "find": "
{profile?.name} {profile?.address}
", + "replace": "
{!profileSavedHere && <>{profile?.name} {profile?.address}}
" + } + ] + }, + { + "id": "stored-support-replies-are-hidden-after-reload", + "scenario": "tracks/ecommerce/scenarios/progression-managed-support-shared.json", + "targets": [ + "ecommerce.progression.managed-support.managed-support.613c" + ], + "desc": "Replies already stored when the page loads are hidden and only replies that arrive while the page is open are shown, so a reloaded customer or staff member cannot see the earlier exchange. The live shared-case check exchanges only new replies and is unaffected.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "import { useEffect, useMemo, useState } from 'react';", + "replace": "import { useEffect, useMemo, useRef, useState } from 'react';" + }, + { + "find": " const [supportReplyRows] = useTable(tables.visibleSupportReplies);", + "replace": " const [liveSupportReplyRows, supportRepliesReady] = useTable(tables.visibleSupportReplies);\n const openedSupportReplies = useRef | null>(null);\n if (openedSupportReplies.current === null && supportRepliesReady) {\n openedSupportReplies.current = new Set(\n liveSupportReplyRows.map((row) => `${row.ticketId}:${row.author}:${row.body}`)\n );\n }\n const supportReplyRows = liveSupportReplyRows.filter(\n (row) => !openedSupportReplies.current?.has(`${row.ticketId}:${row.author}:${row.body}`)\n );" + } + ] + }, + { + "id": "saving-notification-preferences-resets-the-toggles", + "scenario": "tracks/ecommerce/scenarios/progression-notification-preferences.json", + "targets": [ + "ecommerce.progression.notification-preferences.notification-preferences.630c" + ], + "desc": "Saving sends the chosen preferences but resets both toggles to off and stops the form from following the stored row for the rest of the session, so the saved choice cannot be seen until a reload. The reload and cross-account checks read the stored row on a fresh page and are unaffected.", + "file": "client/src/components/ProgressionWorkbench.tsx", + "edits": [ + { + "find": " useEffect(() => {\n setOrderEnabled(preferences?.orderEnabled ?? false);\n setStockEnabled(preferences?.stockEnabled ?? false);\n }, [preferences?.orderEnabled, preferences?.stockEnabled]);", + "replace": " const [preferencesSubmitted, setPreferencesSubmitted] = useState(false);\n useEffect(() => {\n if (preferencesSubmitted) return;\n setOrderEnabled(preferences?.orderEnabled ?? false);\n setStockEnabled(preferences?.stockEnabled ?? false);\n }, [preferences?.orderEnabled, preferences?.stockEnabled, preferencesSubmitted]);" + }, + { + "find": "onClick={async () => {\n setNotificationSaveState('pending');\n try {\n if (!reducers) throw new Error('Not connected');\n await reducers.saveNotificationPreferences({ orderEnabled, stockEnabled });\n setNotificationSaveState('succeeded');\n } catch { setNotificationSaveState('failed'); }\n }}", + "replace": "onClick={async () => {\n setNotificationSaveState('pending');\n try {\n if (!reducers) throw new Error('Not connected');\n await reducers.saveNotificationPreferences({ orderEnabled, stockEnabled });\n setPreferencesSubmitted(true); setOrderEnabled(false); setStockEnabled(false);\n setNotificationSaveState('succeeded');\n } catch { setNotificationSaveState('failed'); }\n }}" + } + ] + }, + { + "id": "saving-a-staff-role-snaps-the-input-back-to-the-stored-role", + "scenario": "tracks/ecommerce/scenarios/progression-staff-roles.json", + "targets": [ + "ecommerce.progression.staff-roles.staff-roles.621c" + ], + "desc": "Saving a role sends the new role to the server but snaps the visible input back to the role stored before the save for the rest of the session, so the administrator cannot see the assignment take. A reload renders the stored role, so the durability and boundary checks are unaffected.", + "file": "client/src/components/ProgressionWorkbench.tsx", + "edits": [ + { + "find": " ", + "replace": " " + } + ] + }, + { + "id": "fulfilment-queue-is-frozen-at-page-load", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-live.json", + "targets": [ + "ecommerce.spec.live-state.fulfilment-queue.1a" + ], + "desc": "The fulfilment queue renders the rows delivered with the page's initial subscription and ignores later updates, so an order placed while the queue is open never appears without a reload.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "import { useEffect, useMemo, useState } from 'react';", + "replace": "import { useEffect, useMemo, useRef, useState } from 'react';" + }, + { + "find": " const [queueRows] = useTable(tables.fulfilmentQueue);", + "replace": " const [liveQueueRows, queueReady] = useTable(tables.fulfilmentQueue);\n const openedQueueRows = useRef(null);\n if (openedQueueRows.current === null && queueReady) openedQueueRows.current = liveQueueRows;\n const queueRows = openedQueueRows.current ?? liveQueueRows;" + } + ] + }, + { + "id": "low-stock-list-is-frozen-at-page-load", + "scenario": "tracks/ecommerce/scenarios/02-low-stock.json", + "targets": [ + "ecommerce.spec.live-state.inventory-dashboard.5a" + ], + "desc": "The low-stock list is computed once from the first complete stock snapshot and never recomputed, so items no longer enter or leave it as stock is restocked or sold. The seeded low item is in that first snapshot, so the static listing check is unaffected.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "import { useEffect, useMemo, useState } from 'react';", + "replace": "import { useEffect, useMemo, useRef, useState } from 'react';" + }, + { + "find": " const lowStockItems = useMemo(\n () =>\n [...items]\n .filter((i) => (stockByItem.get(i.id) ?? 0) <= LOW_STOCK_THRESHOLD)\n .sort((a, b) => (stockByItem.get(a.id) ?? 0) - (stockByItem.get(b.id) ?? 0)),\n [items, stockByItem]\n );", + "replace": " const liveLowStockItems = useMemo(\n () =>\n [...items]\n .filter((i) => (stockByItem.get(i.id) ?? 0) <= LOW_STOCK_THRESHOLD)\n .sort((a, b) => (stockByItem.get(a.id) ?? 0) - (stockByItem.get(b.id) ?? 0)),\n [items, stockByItem]\n );\n const openedLowStockItems = useRef(null);\n if (openedLowStockItems.current === null && items.length > 0 && stocks.length > 0) {\n openedLowStockItems.current = liveLowStockItems;\n }\n const lowStockItems = openedLowStockItems.current ?? liveLowStockItems;" + } + ] + }, + { + "id": "category-totals-are-frozen-at-page-load", + "scenario": "tracks/ecommerce/scenarios/02-operational-category-totals.json", + "targets": [ + "ecommerce.spec.live-state.sales-dashboard.5b" + ], + "desc": "The category totals render the first non-empty row set received and ignore later updates, so a purchase does not move units or revenue while the dashboard is open. A reload receives the stored totals, so the reload check is unaffected.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "import { useEffect, useMemo, useState } from 'react';", + "replace": "import { useEffect, useMemo, useRef, useState } from 'react';" + }, + { + "find": " const [categoryTotalRows] = useTable(tables.categoryTotals);", + "replace": " const [liveCategoryTotalRows] = useTable(tables.categoryTotals);\n const openedCategoryTotalRows = useRef(null);\n if (openedCategoryTotalRows.current === null && liveCategoryTotalRows.length > 0) {\n openedCategoryTotalRows.current = liveCategoryTotalRows;\n }\n const categoryTotalRows = openedCategoryTotalRows.current ?? liveCategoryTotalRows;" + } + ] + }, + { + "id": "warehouse-totals-are-frozen-at-page-load", + "scenario": "tracks/ecommerce/scenarios/02-strengthened.json", + "targets": [ + "ecommerce.inventory-operations.warehouse-transfer.2a" + ], + "desc": "The per-warehouse totals are computed once from the first stock snapshot and never recomputed, so a transfer moves neither warehouse figure while the dashboard is open.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "import { useEffect, useMemo, useState } from 'react';", + "replace": "import { useEffect, useMemo, useRef, useState } from 'react';" + }, + { + "find": " const stockByWarehouse = useMemo(() => {\n const map = new Map();\n for (const row of stocks) {\n map.set(row.warehouseId, (map.get(row.warehouseId) ?? 0) + row.quantity);\n }\n return map;\n }, [stocks]);", + "replace": " const liveStockByWarehouse = useMemo(() => {\n const map = new Map();\n for (const row of stocks) {\n map.set(row.warehouseId, (map.get(row.warehouseId) ?? 0) + row.quantity);\n }\n return map;\n }, [stocks]);\n const openedStockByWarehouse = useRef(null);\n if (openedStockByWarehouse.current === null && liveStockByWarehouse.size > 0) {\n openedStockByWarehouse.current = liveStockByWarehouse;\n }\n const stockByWarehouse = openedStockByWarehouse.current ?? liveStockByWarehouse;" + } + ] + }, + { + "id": "transfer-skips-the-source-holding-check", + "scenario": "tracks/ecommerce/scenarios/02-transfer-overdraw.json", + "targets": [ + "ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c" + ], + "desc": "The serialized transfer reducer no longer checks that the source warehouse holds the requested quantity, so an overdraw is accepted instead of refused and the source quantity wraps below zero.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (available < quantity) {\n throw new SenderError(`Not enough stock in source warehouse: only ${available} available.`);\n }", + "replace": " // mutant: the source warehouse holding is not checked" + } + ] + }, + { + "id": "credit-checkout-ignores-wallet", + "desc": "A credit checkout pays entirely externally despite available credit.", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.feature.store-credit.store-credit-750.750a" + ], + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " const creditMinor = useCredit ? Math.min(wallet?.amountMinor ?? 0,totalMinor) : 0;", + "replace": " const creditMinor = 0;" + } + ] + }, + { + "id": "credit-grant-replay-increments-balance", + "desc": "Replaying a grant applies its credit to the wallet again.", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.spec.store-credit.production-752.752a" + ], + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (existing.amountMinor !== amountMinor) throw new SenderError('Reference identifies another grant.');\n return;", + "replace": " if (existing.amountMinor !== amountMinor) throw new SenderError('Reference identifies another grant.');\n const wallet = ctx.db.creditWallet.accountId.find(accountId)!;\n ctx.db.creditWallet.accountId.update({ ...wallet, amountMinor: wallet.amountMinor + amountMinor });\n return;" + } + ] + }, + { + "id": "customer-can-grant-credit", + "desc": "Customer authentication is accepted without staff authorization.", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.spec.store-credit.production-753.753a" + ], + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " (ctx, { accountId, amountMinor, reference }) => {\n requireStaffOrAdmin(ctx);", + "replace": " (ctx, { accountId, amountMinor, reference }) => {\n requireAccount(ctx);" + } + ] + }, + { + "id": "split-refund-does-not-restore-credit", + "desc": "The refund is recorded but its original wallet credit is not restored.", + "scenario": "tracks/ecommerce/scenarios/progression-split-tender-refunds.json", + "targets": [ + "ecommerce.feature.split-tender-refunds.split-tender-refunds-751.751a" + ], + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " refundOrderCredit(ctx, order, order.total);", + "replace": " // mutant: omit wallet restoration" + } + ] + }, + { + "id": "split-refund-duplicates-credit", + "desc": "A refund credits the wallet twice while recording one refund.", + "scenario": "tracks/ecommerce/scenarios/progression-split-tender-refunds.json", + "targets": [ + "ecommerce.spec.split-tender-refunds.production-756.756a" + ], + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "amountMinor: wallet.amountMinor + delta", + "replace": "amountMinor: wallet.amountMinor + delta * 2" + } + ] + }, + { + "id": "subscription-skips-due-purchase", + "desc": "Due deliveries are recorded as skipped although stock is available.", + "scenario": "tracks/ecommerce/scenarios/progression-subscriptions.json", + "targets": [ + "ecommerce.feature.subscriptions.subscriptions-760.760a" + ], + "file": "backend/spacetimedb/src/subscriptions.ts", + "edits": [ + { + "find": " const orderId = purchase(row.accountId, row.itemId, row.quantity, row.price);", + "replace": " const orderId: bigint | null = null;" + } + ] + }, + { + "id": "subscription-allows-foreign-cancellation", + "desc": "A customer can cancel another customer subscription.", + "scenario": "tracks/ecommerce/scenarios/progression-subscriptions.json", + "targets": [ + "ecommerce.spec.subscriptions.production-762.762a" + ], + "file": "backend/spacetimedb/src/subscriptions.ts", + "edits": [ + { + "find": " if (!row || row.accountId !== accountId) throw new SenderError('Subscription access denied.');", + "replace": " if (!row) throw new SenderError('Subscription access denied.');" + } + ] + }, + { + "id": "subscription-pause-is-not-recorded", + "desc": "Pause acknowledges the request but the subscription remains active.", + "scenario": "tracks/ecommerce/scenarios/progression-subscriptions.json", + "targets": [ + "ecommerce.spec.subscriptions.production-763.763a" + ], + "file": "backend/spacetimedb/src/subscriptions.ts", + "edits": [ + { + "find": " ctx.db.purchaseSubscription.id.update({ ...row, status: 'paused', pausedMicros: now });", + "replace": " ctx.db.purchaseSubscription.id.update({ ...row, status: 'active', pausedMicros: now });" + } + ] + }, + { + "id": "credit-checkout-retains-purchased-cart", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.spec.store-credit.production-754.754a" + ], + "desc": "The purchased cart remains available instead of being consumed by checkout.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " for (const line of lines) ctx.db.cartItem.id.delete(line.id);", + "replace": " // mutant: checked-out lines remain in the cart" + } + ] + }, + { + "id": "reconnection-erases-stored-credit", + "desc": "A new connection erases stored wallet credit; the fresh view after restart must catch the data loss.", + "file": "backend/spacetimedb/src/index.ts", + "scenario": "tracks/ecommerce/scenarios/progression-store-credit.json", + "targets": [ + "ecommerce.spec.store-credit.production-755.755a" + ], + "edits": [ + { + "find": "// --- views ---", + "replace": "export const onConnect = spacetimedb.clientConnected(ctx => { for (const row of ctx.db.creditWallet.iter()) ctx.db.creditWallet.accountId.update({ ...row, amountMinor: 0 }); });\n// --- views ---" + } + ] + }, + { + "id": "reconnection-cancels-pending-subscriptions", + "desc": "A new connection cancels pending subscriptions; restarting and reconnecting must preserve this work.", + "file": "backend/spacetimedb/src/index.ts", + "scenario": "tracks/ecommerce/scenarios/progression-subscriptions.json", + "targets": [ + "ecommerce.spec.subscriptions.production-761.761a" + ], + "edits": [ + { + "find": "// --- views ---", + "replace": "export const onConnect = spacetimedb.clientConnected(ctx => { for (const row of ctx.db.purchaseSubscription.iter()) if (row.status === 'active') ctx.db.purchaseSubscription.id.update({ ...row, status: 'cancelled' }); });\n// --- views ---" + } + ] + }, + { + "id": "bundle-definition-loses-component-quantity", + "scenario": "tracks/ecommerce/scenarios/progression-product-bundles.json", + "targets": [ + "ecommerce.feature.product-bundles.product-bundles.740a" + ], + "desc": "definition loses component quantity", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "return { ...component, itemId: String(product.id) };", + "replace": "return { ...component, quantity: 1, itemId: String(product.id) };" + } + ] + }, + { + "id": "bundle-catalog-write-allows-customers", + "scenario": "tracks/ecommerce/scenarios/progression-product-bundles.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-743.743a" + ], + "desc": "catalog write allows customers", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "const actor = requireStaffOrAdmin(ctx);\n if (!actor.isAdmin && ctx.db.staffRole.accountId.find(actor.id)?.role !== 'catalog') {", + "replace": "const actor = requireAccount(ctx);\n if (false) {" + } + ] + }, + { + "id": "bundle-checkout-price-not-snapshot", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-checkout.json", + "targets": [ + "ecommerce.feature.bundle-checkout.bundle-checkout.741a" + ], + "desc": "checkout price not snapshot", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "bundlePrice: product.price, bundleComponentsJson: definition.componentsJson", + "replace": "bundlePrice: product.price + 1, bundleComponentsJson: definition.componentsJson" + } + ] + }, + { + "id": "bundle-return-loses-original-components", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-returns.json", + "targets": [ + "ecommerce.feature.bundle-returns.bundle-returns.742a" + ], + "desc": "return loses original components", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "for (const line of bundles) {\n restoreOrderItemStock(ctx, line);", + "replace": "for (const line of bundles) {\n // mutant: original stock is not restored" + } + ] + }, + { + "id": "bundle-return-replay-restocks-again", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-returns.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-742.742b" + ], + "desc": "return replay restocks again", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": ".filter(row => row.isBundle && !row.returned);", + "replace": ".filter(row => row.isBundle);" + } + ] + }, + { + "id": "bundle-return-crosses-account-boundary", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-returns.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-748.748a" + ], + "desc": "return crosses account boundary", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "if (!order || order.accountId !== account.id || !['shipped', 'delivered'].includes(order.status))", + "replace": "if (!order || !['shipped', 'delivered'].includes(order.status))" + } + ] + }, + { + "id": "bundle-components-can-overdraw", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-checkout.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-744.744a", + "ecommerce.spec.bundle-integrity.bundle-745.745a" + ], + "desc": "components can overdraw", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "if (!allocations) throw new SenderError('Not enough stock to reserve.');", + "replace": "if (!allocations) { if (bundleId) return; throw new SenderError('Not enough stock to reserve.'); }" + } + ] + }, + { + "id": "bundle-checkout-reuses-reservation", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-checkout.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-747.747a" + ], + "desc": "checkout reuses reservation", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "for (const row of held) ctx.db.reservation.id.delete(row.id);\n processReorderRules(ctx, p.itemId);", + "replace": "// mutant: reservations survive checkout\n processReorderRules(ctx, p.itemId);" + }, + { + "find": "for (const line of lines) ctx.db.cartItem.id.delete(line.id);", + "replace": "// mutant: cart survives checkout" + } + ] + }, + { + "id": "bundle-expiry-does-not-release-components", + "scenario": "tracks/ecommerce/scenarios/progression-bundle-checkout.json", + "targets": [ + "ecommerce.spec.bundle-integrity.bundle-746.746a" + ], + "desc": "Expired bundle reservations retain their component stock after restart.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "if (row.expired || row.expiresMicros > now) continue;\n restoreStock(ctx, row.stockItemId || row.itemId, row.warehouseId, row.quantity);", + "replace": "if (row.expired || row.expiresMicros > now) continue;\n if (!row.stockItemId) restoreStock(ctx, row.itemId, row.warehouseId, row.quantity);" + } + ] + }, + { + "id": "return-after-support-refund-is-blocked", + "scenario": "tracks/ecommerce/scenarios/progression-support-return-interaction.json", + "targets": [ + "ecommerce.feature.split-tender-refunds.return-refund-interaction.757a" + ], + "desc": "Reject a valid physical return after a financial refund.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (target.returned) throw new SenderError('Item already returned.');", + "replace": " if (target.returned || order.refundedTotal > 0) throw new SenderError('Item already returned.');" + } + ] + }, + { + "id": "support-refund-after-return-pays-twice", + "scenario": "tracks/ecommerce/scenarios/progression-support-return-interaction.json", + "targets": [ + "ecommerce.feature.split-tender-refunds.return-refund-interaction.757b" + ], + "desc": "Pay the full order again after the physical return already refunded it.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " const amount = order.total - order.refundedTotal;", + "replace": " const amount = order.total;" + }, + { + "find": "refundedTotal: order.total, status: order.status", + "replace": "refundedTotal: order.refundedTotal + amount, status: order.status" + } + ] + }, + { + "id": "support-history-leaks-to-signed-out-visitors", + "scenario": "tracks/ecommerce/scenarios/progression-support-history.json", + "targets": [ + "ecommerce.spec.access-control.support-history-logout.612d" + ], + "desc": "Disclose support tickets to signed-out visitors while leaving signed-in ownership filtering unchanged.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": ".filter(row => isGuestTicketCreator(sender, row.creatorIdentity.toHexString(), row.accountId) ||", + "replace": ".filter(row => !actor || isGuestTicketCreator(sender, row.creatorIdentity.toHexString(), row.accountId) ||" + } + ] + }, + { + "id": "checkout-crash-integrity", + "scenario": "tracks/ecommerce/scenarios/progression-checkout-crash.json", + "targets": [ + "ecommerce.spec.state-durability.checkout-crash-integrity.910a" + ], + "desc": "A lifecycle callback changes a prepared or cleared cart to quantity two, leaving neither a legal rollback nor a complete checkout.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "// --- views ---", + "replace": "export const onDisconnect = spacetimedb.clientDisconnected((ctx) => { for (const row of ctx.db.cartItem.iter()) if ([...ctx.db.customerOrder.iter()].some(order => order.accountId === row.accountId)) ctx.db.cartItem.id.update({ ...row, quantity: 2 }); });\n// --- views ---" + } + ] + }, + { + "id": "checkout-crash-sibling-stock", + "scenario": "tracks/ecommerce/scenarios/progression-checkout-crash.json", + "targets": [ + "ecommerce.spec.state-durability.checkout-crash-integrity.910a" + ], + "desc": "A disconnect callback removes second-product stock once after a cart and prior order exist. A private ledger marker prevents repeated damage to later setup. The control tests stock observation after a crash, not the time at which the defect was injected.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "// --- views ---", + "replace": "export const onDisconnect = spacetimedb.clientDisconnected((ctx) => { if ([...ctx.db.stockLedger.iter()].some(row => row.source === 'stock fault control') || ![...ctx.db.cartItem.iter()].some(row => [...ctx.db.customerOrder.iter()].some(order => order.accountId === row.accountId))) return; const item = [...ctx.db.item.iter()].find(row => row.name === 'Coffee Grinder'); if (!item) return; const row = [...ctx.db.stock.by_item_warehouse.filter(item.id)][0]; if (row) { ctx.db.stock.by_item_warehouse.delete([row.item_id, row.warehouse_id]); ctx.db.stock.insert({ ...row, quantity: row.quantity - 1 }); ctx.db.stockLedger.insert({ id: 0n, itemId: row.item_id, warehouseId: row.warehouse_id, quantity: 1, createdMicros: ctx.timestamp.microsSinceUnixEpoch, source: 'stock fault control' }); } });\n// --- views ---" + } + ] + }, + { + "id": "checkout-crash-durability", + "scenario": "tracks/ecommerce/scenarios/progression-checkout-crash.json", + "targets": [ + "ecommerce.spec.state-durability.checkout-crash-durability.910b" + ], + "desc": "A lifecycle callback corrupts earlier order totals. The unchanged prepared snapshots and acknowledged baseline must expose the damage.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "// --- views ---", + "replace": "export const onDisconnect = spacetimedb.clientDisconnected((ctx) => { for (const order of ctx.db.customerOrder.iter()) if ([...ctx.db.cartItem.iter()].some(row => row.accountId === order.accountId)) ctx.db.customerOrder.id.update({ ...order, total: 0 }); });\n// --- views ---" + } + ] + }, + { + "id": "review-script-unsafe-render", + "scenario": "tracks/ecommerce/scenarios/progression-review-script.json", + "targets": [ + "ecommerce.progression.review-access-specifications.stored-review-script.9180a" + ], + "desc": "Render stored review text as HTML so it executes in another customer session.", + "file": "client/src/components/ItemDetail.tsx", + "edits": [ + { + "find": "
{r.comment}
", + "replace": "
" + } + ] + }, + { + "id": "review-script-reject-all", + "scenario": "tracks/ecommerce/scenarios/progression-review-script.json", + "targets": [ + "ecommerce.progression.review-access-specifications.stored-review-script.9180a" + ], + "desc": "Reject all nonempty reviews, including the required benign review; absence of script is insufficient.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "(ctx, { itemId, rating, comment }) => {", + "replace": "(ctx, { itemId, rating, comment }) => {\n if (comment.length > 0) throw new SenderError(\"Review rejected\");" + } + ] + }, + { + "id": "review-owner-reject-all", + "scenario": "tracks/ecommerce/scenarios/progression-review-access.json", + "targets": [ + "ecommerce.progression.review-access-specifications.review-eligibility-direct.618a" + ], + "desc": "Reject the legitimate buyer review as well as unauthorized reviews.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "(ctx, { itemId, rating, comment }) => {", + "replace": "(ctx, { itemId, rating, comment }) => {\n if (comment.length > 0) throw new SenderError(\"Reject all reviews\");" + } + ] + }, + { + "id": "password-verification-truncates-utf8", + "scenario": "tracks/ecommerce/scenarios/01-account-password.json", + "targets": [ + "ecommerce.spec.access-control.account-password.1c" + ], + "desc": "Hash and verify only the first 72 UTF-8 bytes, accepting a different password suffix.", + "file": "backend/spacetimedb/src/auth.ts", + "edits": [ + { + "find": "scrypt(password, salt, {", + "replace": "scrypt(new TextEncoder().encode(password).slice(0, 72), salt, {" + } + ] + }, + { + "id": "password-verification-rejects-every-login", + "scenario": "tracks/ecommerce/scenarios/01-account-password.json", + "targets": [ + "ecommerce.spec.access-control.account-password.1c" + ], + "desc": "Refuse even a correct password; refusal alone must not earn credit.", + "file": "backend/spacetimedb/src/auth.ts", + "edits": [ + { + "find": "const valid = !!account && sameHash(digest, account.passwordHash);", + "replace": "const valid = !!account && sameHash(digest, account.passwordHash) && false;" + } + ] + }, + { + "id": "order-cache-survives-account-switch", + "scenario": "tracks/ecommerce/scenarios/01-order-ownership.json", + "targets": [ + "ecommerce.spec.access-control.order-ownership.106a" + ], + "desc": "Persist order views in a browser cache that is not scoped to the account. Login reloads no longer clear the stale data.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const orderViews: OrderView[] = useMemo(", + "replace": " const freshOrderViews: OrderView[] = useMemo(" + }, + { + "find": " const revenue = revenueRows[0]?.total ?? 0;", + "replace": " const [orderCache] = useState(() => new Map(\n (JSON.parse(sessionStorage.getItem('order-cache') ?? '[]', (key, value) =>\n key === 'orderId' || key === 'itemId' ? BigInt(value) : key === 'createdAt' ? new Date(value) : value) as OrderView[])\n .map(order => [order.orderId, order])));\n for (const order of freshOrderViews) if (order.items.length) orderCache.set(order.orderId, order);\n const orderViews = [...orderCache.values()];\n sessionStorage.setItem('order-cache', JSON.stringify(orderViews, (_key, value) => typeof value === 'bigint' ? String(value) : value));\n\n const revenue = revenueRows[0]?.total ?? 0;" + } + ] + }, + { + "id": "checkout-records-wrong-total", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "Store an incorrect checkout total while retaining the normal cart and item lines. Native extra arguments remain rejected.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " total: total - discount,\n status: 'pending',", + "replace": " total: 1,\n status: 'pending'," + } + ] + }, + { + "id": "checkout-missing-sibling-line", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "Commit the cart checkout with one product line missing while retaining the other product and the full order total.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " for (const p of priced) {", + "replace": " for (const p of priced) {\n if (p.name === 'Coffee Grinder') continue;" + } + ] + }, + { + "id": "shipping-cancellation-cancel-shipped", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Cancellation accepts a shipped order and restores its stock after shipping.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "if (order.status !== 'pending') throw new SenderError('Order has already shipped.');", + "replace": "// Permit the invalid order transition for this control." + } + ] + }, + { + "id": "shipping-cancellation-resurrect-cancelled", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Shipping accepts a cancelled order after its stock was restored.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "if (order.status !== 'pending') throw new SenderError('Order is not pending.');", + "replace": "// Permit the invalid order transition for this control." + } + ] + }, + { + "id": "shipping-cancellation-combined-race", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Cancellation restores stock but records a shipped status, combining mutually exclusive effects.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "if (order.status !== 'pending') throw new SenderError('Order has already shipped.');", + "replace": "// Permit the invalid order transition for this control." + }, + { + "find": "ctx.db.customerOrder.id.update({ ...order, status: 'cancelled' });", + "replace": "ctx.db.customerOrder.id.update({ ...order, status: 'shipped' });" + } + ] + }, + { + "id": "shipping-cancellation-reject-both", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Both cancellation and shipping refuse pending orders; useful progress must remain required.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "if (order.status !== 'pending') throw new SenderError('Order has already shipped.');", + "replace": "if (order.status === 'pending') throw new SenderError('Cancellation disabled');" + }, + { + "find": "if (order.status !== 'pending') throw new SenderError('Order is not pending.');", + "replace": "if (order.status === 'pending') throw new SenderError('Shipping disabled');" + } + ] + }, + { + "id": "shipping-cancellation-queue-retains-removed-order", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "The live staff queue accepts new orders but retains rows after cancellation; the database state remains correct.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "import { useEffect, useMemo, useState } from 'react';", + "replace": "import { useEffect, useMemo, useRef, useState } from 'react';" + }, + { + "find": " const [queueRows] = useTable(tables.fulfilmentQueue);", + "replace": " const [liveQueueRows] = useTable(tables.fulfilmentQueue);\n const retainedQueueRows = useRef([]);\n if (liveQueueRows.length >= retainedQueueRows.current.length) retainedQueueRows.current = liveQueueRows;\n const queueRows = retainedQueueRows.current;" + } + ] + }, + { + "id": "opposing-transfer-ignored", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202d" + ], + "desc": "One valid opposing transfer is acknowledged without moving stock.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (quantity < 1) throw new SenderError('Transfer quantity must be at least 1.');", + "replace": " if (quantity < 1) throw new SenderError('Transfer quantity must be at least 1.');\n if (quantity === 7) return;" + } + ] + }, + { + "id": "opposing-transfer-rejected", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202d" + ], + "desc": "One valid opposing transfer is refused despite ample source stock.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (quantity < 1) throw new SenderError('Transfer quantity must be at least 1.');", + "replace": " if (quantity < 1) throw new SenderError('Transfer quantity must be at least 1.');\n if (quantity === 7) throw new SenderError('Transfer disabled');" + } + ] + }, + { + "id": "overlapping-cart-add-ignored", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "The overlapping second-product cart add is ignored while ordinary cart setup still works.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " const existing = findCartLine(ctx, acc.id, itemId);\n if (existing) {\n replaceReservation(ctx, acc.id, itemId, existing.quantity + 1);", + "replace": " if (acc.username.includes('checkout-overlap') && it.name === 'Coffee Grinder') return;\n const existing = findCartLine(ctx, acc.id, itemId);\n if (existing) {\n replaceReservation(ctx, acc.id, itemId, existing.quantity + 1);" + } + ] + }, + { + "id": "overlapping-cart-add-rejected", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "The overlapping second-product cart add is rejected while ordinary cart setup still works.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " const existing = findCartLine(ctx, acc.id, itemId);\n if (existing) {\n replaceReservation(ctx, acc.id, itemId, existing.quantity + 1);", + "replace": " if (acc.username.includes('checkout-overlap') && it.name === 'Coffee Grinder') throw new SenderError('Cart add disabled');\n const existing = findCartLine(ctx, acc.id, itemId);\n if (existing) {\n replaceReservation(ctx, acc.id, itemId, existing.quantity + 1);" + } + ] + }, + { + "id": "mixed-history-lost-transfer", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: lost-transfer. The stored-state comparison must catch the defect at its first effect.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ctx.db.stock.insert({ ...toRow, quantity: toRow.quantity + quantity });", + "replace": " ctx.db.stock.insert({ ...toRow, quantity: toRow.quantity });" + } + ] + }, + { + "id": "mixed-history-wrong-price", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: wrong-price. The stored-state comparison must catch the defect at its first effect.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " total: Math.round(price * 100) * quantity / 100,\n status: 'pending',", + "replace": " total: 1,\n status: 'pending'," + } + ] + }, + { + "id": "mixed-history-ignored-write", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: ignored-write. The stored-state comparison must catch the defect at its first effect.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " const existing = findCartLine(ctx, acc.id, itemId);\n if (existing) {\n replaceReservation(ctx, acc.id, itemId, existing.quantity + 1);", + "replace": " if (acc.username.includes('history-') && it.name === 'Coffee Grinder') return;\n const existing = findCartLine(ctx, acc.id, itemId);\n if (existing) {\n replaceReservation(ctx, acc.id, itemId, existing.quantity + 1);" + } + ] + }, + { + "id": "mixed-history-reject-valid-write", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: reject-valid-write. The stored-state comparison must catch the defect at its first effect.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " const existing = findCartLine(ctx, acc.id, itemId);\n if (existing) {\n replaceReservation(ctx, acc.id, itemId, existing.quantity + 1);", + "replace": " if (acc.username.includes('history-') && it.name === 'Coffee Grinder') throw new SenderError('Cart add disabled');\n const existing = findCartLine(ctx, acc.id, itemId);\n if (existing) {\n replaceReservation(ctx, acc.id, itemId, existing.quantity + 1);" + } + ] + }, + { + "id": "mixed-history-repeat-cancel", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: repeat-cancel. The stored-state comparison must catch the defect at its first effect.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (order.status !== 'pending') throw new SenderError('Order has already shipped.');", + "replace": " if (!['pending', 'cancelled'].includes(order.status)) throw new SenderError('Order has already shipped.');" + } + ] + }, + { + "id": "mixed-history-wrong-owner", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: wrong-owner. The stored-state comparison must catch the defect at its first effect.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " accountId,\n createdAt: ctx.timestamp,\n total: Math.round(price * 100) * quantity / 100,", + "replace": " accountId: ctx.db.account.username.find('admin')!.id,\n createdAt: ctx.timestamp,\n total: Math.round(price * 100) * quantity / 100," + } + ] + }, + { + "id": "checkout-empty-replay-creates-order", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Allow retrying the consumed empty cart to create another order after a lost reply.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (lines.length === 0) throw new SenderError('Your cart is empty.');", + "replace": "// Defect: an empty consumed cart may create another order." + } + ] + }, + { + "id": "checkout-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Reject every valid checkout; refusal cannot earn duplicate protection credit.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (lines.length === 0) throw new SenderError('Your cart is empty.');", + "replace": " if (lines.length === 0) throw new SenderError('Your cart is empty.');\n if (lines.length >= 0) throw new Error('Checkout disabled');" + } + ] + }, + { + "id": "checkout-replay-loses-prior-order", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Retrying the consumed cart deletes an earlier acknowledged order.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (lines.length === 0) throw new SenderError('Your cart is empty.');", + "replace": " if (lines.length === 0) { const prior = [...ctx.db.customerOrder.iter()].filter(order => order.accountId === acc.id).sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0)[0]; if (prior) ctx.db.customerOrder.id.delete(prior.id); return; }" + } + ] + }, + { + "id": "checkout-clears-cart-without-order", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Checkout returns success and empties the cart without recording an order.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (lines.length === 0) throw new SenderError('Your cart is empty.');", + "replace": " if (lines.length === 0) throw new SenderError('Your cart is empty.');\n if (lines.length > 0) { for (const line of lines) ctx.db.cartItem.id.delete(line.id); return; }" + } + ] + }, + { + "id": "revenue-aggregation-truncates-order-population", + "scenario": "tracks/ecommerce/scenarios/progression-books-balance.json", + "targets": [ + "ecommerce.spec.transactional-integrity.books-balance.107a" + ], + "desc": "Calculate revenue from only the first 256 stored orders while preserving all acknowledged purchases.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " let total = 0;\n for (const o of ctx.db.customerOrder.iter()) {", + "replace": " let total = 0;\n for (const o of [...ctx.db.customerOrder.iter()].slice(0, 256)) {" + } + ] + }, + { + "id": "catalog-truncates-created-population", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-volume.json", + "targets": [ + "ecommerce.spec.transactional-integrity.catalog-volume.622c" + ], + "desc": "Omit committed products after the first 256 search results.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": ".slice(searchPage * CATALOG_PAGE_SIZE,", + "replace": ".slice(0, 256).slice(searchPage * CATALOG_PAGE_SIZE," + } + ] + }, + { + "id": "catalog-discards-product-writes", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-volume.json", + "targets": [ + "ecommerce.spec.transactional-integrity.catalog-volume.622c" + ], + "desc": "Acknowledge catalog creation without committing a product.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " const product = ctx.db.item.insert({ id: 0n, name: name.trim(), price });", + "replace": " if (name.length >= 0) return; // mutant: successful reducer without product\n const product = ctx.db.item.insert({ id: 0n, name: name.trim(), price });" + } + ] + }, + { + "id": "catalog-rejects-all-product-writes", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-volume.json", + "targets": [ + "ecommerce.spec.transactional-integrity.catalog-volume.622c" + ], + "desc": "Reject every catalog creation while keeping sign-in and catalog reads available.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " const product = ctx.db.item.insert({ id: 0n, name: name.trim(), price });", + "replace": " if (price > 0) throw new SenderError('Catalog writes refused');\n const product = ctx.db.item.insert({ id: 0n, name: name.trim(), price });" + } + ] + }, + { + "id": "shipping-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "Reject legitimate staff shipping as well as customer requests.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n requireStaffOrAdmin(ctx);", + "replace": "export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n requireStaffOrAdmin(ctx);\n if (orderId >= 0n) throw new SenderError('Shipping refused.');" + } + ] + }, + { + "id": "cancel-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.order-owner.204a" + ], + "desc": "Reject legitimate cancellation as well as foreign requests.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "export const cancelOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n const order = requireOrderOwner(ctx, orderId);", + "replace": "export const cancelOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n const order = requireOrderOwner(ctx, orderId);\n if (orderId >= 0n) throw new SenderError('Cancellation refused');" + } + ] + }, + { + "id": "account-password-in-public-data", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Expose the actual submitted account password through data available to other clients.", + "edits": [ + { + "file": "backend/spacetimedb/src/auth.ts", + "find": " isAdmin: false, isStaff: false });\n bindSession(tx, account.id);", + "replace": " isAdmin: false, isStaff: false });\n const publicItem = tx.db.item.id.find(1n);\n if (publicItem) tx.db.item.id.update({ ...publicItem, name: publicItem.name + ' ' + password });\n bindSession(tx, account.id);" + } + ] + }, + { + "id": "account-disclosure-rejects-login", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Reject every valid login; absent disclosures alone must not earn a pass.", + "file": "backend/spacetimedb/src/auth.ts", + "edits": [ + { + "find": "const valid = !!account && sameHash(digest, account.passwordHash);", + "replace": "const valid = !!account && sameHash(digest, account.passwordHash) && false;" + } + ] + }, + { + "id": "signout-keeps-session", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Show a signed-out interface while retaining the browser credential and its purchase authority.", + "edits": [ + { + "file": "client/src/components/AuthWidget.tsx", + "find": " const [error, setError] = useState(readAuthError);", + "replace": " const [error, setError] = useState(readAuthError);\n const [signedOut, setSignedOut] = useState(false);" + }, + { + "file": "client/src/components/AuthWidget.tsx", + "find": " await connection.reducers.signOut({});\n clearToken(); location.reload();", + "replace": " setSignedOut(true); // mutant: hide the account without removing its authority" + }, + { + "file": "client/src/components/AuthWidget.tsx", + "find": " {currentUsername ? <>", + "replace": " {currentUsername && !signedOut ? <>" + } + ] + }, + { + "id": "query-like-password-bypasses-signin", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Interpret the query-like password as a login bypass while retaining normal password verification.", + "file": "backend/spacetimedb/src/auth.ts", + "edits": [ + { + "find": "const valid = !!account && sameHash(digest, account.passwordHash);", + "replace": "const valid = !!account && (password === \"' OR '1'='1\" || sameHash(digest, account.passwordHash));" + } + ] + }, + { + "id": "purchase-session-rejects-returning-login", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Refuse every returning login; blocking all access after logout must not pass the session check.", + "file": "backend/spacetimedb/src/auth.ts", + "edits": [ + { + "find": "const valid = !!account && sameHash(digest, account.passwordHash);", + "replace": "const valid = !!account && sameHash(digest, account.passwordHash) && false;" + } + ] + }, + { + "id": "account-password-in-script", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Ship the account password sentinel in the public JavaScript bundle while normal signup and login still work.", + "edits": [ + { + "file": "client/src/main.tsx", + "find": "import React from 'react';", + "replace": "import React from 'react';\nObject.assign(window, { debugPassword: 'sb-private-pw-76dc492ea153' });" + } + ] + }, + { + "id": "account-password-in-style", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Ship the account password sentinel in a public stylesheet while normal signup and login still work.", + "edits": [ + { + "file": "client/src/index.css", + "find": ":root {", + "replace": ":root {\n --debug-password: \"sb-private-pw-76dc492ea153\";" + } + ] + }, + { + "id": "signup-accepts-admin-claim", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Trust a submitted signup administrator claim and grant protected stock-write authority.", + "edits": [ + { + "file": "backend/spacetimedb/src/auth.ts", + "find": "{ name: t.string(), password: t.string(), salt: t.string() }, t.bool(),", + "replace": "{ name: t.string(), password: t.string(), salt: t.string(), claims: t.object('SignupClaims', { role: t.string() }) }, t.bool()," + }, + { + "file": "backend/spacetimedb/src/auth.ts", + "find": "(ctx, { name, password, salt }) => {", + "replace": "(ctx, { name, password, salt, claims }) => {" + }, + { + "file": "backend/spacetimedb/src/auth.ts", + "find": "isAdmin: false, isStaff: false });", + "replace": "isAdmin: claims.role === 'admin', isStaff: false });" + }, + { + "file": "client/src/auth.ts", + "find": "mode === 'signup' ? [name, password, salt] : [name, password]", + "replace": "mode === 'signup' ? [name, password, salt, { role: 'customer' }] : [name, password]" + } + ] + }, + { + "id": "signup-accepts-optional-admin-claim", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Trust an optional signup administrator claim and grant protected stock-write authority.", + "edits": [ + { + "file": "backend/spacetimedb/src/auth.ts", + "find": "{ name: t.string(), password: t.string(), salt: t.string() }, t.bool(),", + "replace": "{ name: t.string(), password: t.string(), salt: t.string(), claims: t.option(t.object('SignupClaims', { role: t.string() })) }, t.bool()," + }, + { + "file": "backend/spacetimedb/src/auth.ts", + "find": "(ctx, { name, password, salt }) => {", + "replace": "(ctx, { name, password, salt, claims }) => {" + }, + { + "file": "backend/spacetimedb/src/auth.ts", + "find": "isAdmin: false, isStaff: false });", + "replace": "isAdmin: claims?.role === 'admin', isStaff: false });" + }, + { + "file": "client/src/auth.ts", + "find": "mode === 'signup' ? [name, password, salt] : [name, password]", + "replace": "mode === 'signup' ? [name, password, salt, { some: { role: 'customer' } }] : [name, password]" + } + ] + }, + { + "id": "admin-boundary-rejects-all-signups", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Refuse every new registration; blocking useful signup must not pass the authority check.", + "edits": [ + { + "file": "backend/spacetimedb/src/auth.ts", + "find": "if (!validInput(name, password) || !/^[a-f0-9]{64}$/.test(salt)) return false;", + "replace": "if (!validInput(name, password) || !/^[a-f0-9]{64}$/.test(salt) || name.length > 0) return false;" + } + ] + }, + { + "id": "cart-inactivity-timeout-fifty-minutes", + "scenario": "tracks/ecommerce/scenarios/03-cart-expiration.json", + "targets": [ + "ecommerce.l3.cart-expiration.cart-expiration.304a" + ], + "desc": "Set the cart inactivity timeout to fifty minutes instead of five, so an abandoned cart is never moved to the expired cart while its item reservations still lapse.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "const expiresMicros = nowMicros(ctx) + 300n * SECOND;", + "replace": "const expiresMicros = nowMicros(ctx) + 3000n * SECOND;" + } + ] + }, + { + "id": "delivery-scheduled-an-hour-after-shipment", + "scenario": "tracks/ecommerce/scenarios/03-order-delivery.json", + "targets": [ + "ecommerce.l3.order-delivery.order-delivery.303a" + ], + "desc": "Schedule delivery sixty minutes after shipment instead of sixty seconds, so a shipped order stays shipped.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "const schedule = { orderId, dueMicros: nowMicros(ctx) + 60n * SECOND, completed: false };", + "replace": "const schedule = { orderId, dueMicros: nowMicros(ctx) + 60n * 60n * SECOND, completed: false };" + } + ] + }, + { + "id": "delivery-clock-advances-cancelled-orders", + "scenario": "tracks/ecommerce/scenarios/03-order-delivery.json", + "targets": [ + "ecommerce.l3.order-delivery.order-delivery.305a" + ], + "desc": "Start the delivery clock when an order is placed and let the delivery job advance any order that is not yet delivered, so a cancelled order becomes delivered.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " ctx.db.paymentRecord.insert({ id: 0n, orderId: order.id, amount: order.total, status: 'paid' });\n processReorderRules(ctx, itemId);", + "replace": " ctx.db.paymentRecord.insert({ id: 0n, orderId: order.id, amount: order.total, status: 'paid' });\n ctx.db.deliverySchedule.insert({ orderId: order.id, dueMicros: nowMicros(ctx) + 60n * SECOND, completed: false });\n processReorderRules(ctx, itemId);" + }, + { + "find": " if (order && order.status === 'shipped') {", + "replace": " if (order && order.status !== 'delivered') {" + } + ] + }, + { + "id": "reorder-rule-replayable-by-customer", + "scenario": "tracks/ecommerce/scenarios/progression-automatic-reorder-access.json", + "targets": [ + "ecommerce.spec.access-control.automatic-reorder-access.502c" + ], + "desc": "Accept a reorder rule from any signed-in account; only the interface hides the reorder tools from customers.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " requireStaffOrAdmin(ctx);\n const warehouse = [...ctx.db.warehouse.iter()]", + "replace": " requireAccount(ctx);\n const warehouse = [...ctx.db.warehouse.iter()]" + } + ] + }, + { + "id": "reorder-dedupe-checks-wrong-status", + "scenario": "tracks/ecommerce/scenarios/progression-automatic-reorder-duplicate.json", + "targets": [ + "ecommerce.spec.transactional-integrity.automatic-reorder-deduplication.502b" + ], + "desc": "Look for an existing automatic restock with status 'scheduled' instead of 'pending', so every further sale below the threshold schedules another restock.", + "file": "backend/spacetimedb/src/progression-policy.ts", + "edits": [ + { + "find": "return rows.some(row => row.reorderRuleId === ruleId && row.status === 'pending');", + "replace": "return rows.some(row => row.reorderRuleId === ruleId && row.status === 'scheduled');" + } + ] + }, + { + "id": "reorder-threshold-requires-stock-below", + "scenario": "tracks/ecommerce/scenarios/progression-automatic-reorder.json", + "targets": [ + "ecommerce.progression.automatic-reorder.automatic-reorder.502a" + ], + "desc": "Trigger an automatic restock only when stock falls below the threshold, so stock that reaches the threshold exactly schedules nothing.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "if (rule.itemId !== itemId || totalStock(ctx, itemId) > rule.threshold) continue;", + "replace": "if (rule.itemId !== itemId || totalStock(ctx, itemId) >= rule.threshold) continue;" + } + ] + }, + { + "id": "cart-restore-warning-shown-after-full-restore", + "scenario": "tracks/ecommerce/scenarios/progression-cart-recovery.json", + "targets": [ + "ecommerce.progression.cart-recovery.cart-recovery.503a" + ], + "desc": "Render the cart restore warning for every signed-in customer instead of only while expired items remain, so a full restore still shows a warning.", + "file": "client/src/components/ProgressionWorkbench.tsx", + "edits": [ + { + "find": "
Unavailable: {expiredCart.map(row => itemName(row.itemId)).join(', ')}
\n \n \n )}", + "replace": " \n \n )}\n {isSignedIn &&
Unavailable: {expiredCart.map(row => itemName(row.itemId)).join(', ')}
}" + } + ] + }, + { + "id": "cart-restore-drops-unavailable-items-silently", + "scenario": "tracks/ecommerce/scenarios/progression-cart-recovery.json", + "targets": [ + "ecommerce.progression.cart-recovery.cart-recovery.503b" + ], + "desc": "Clear each expired cart line before checking its stock, so an unavailable item is dropped without being named.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (findCartLine(ctx,acc.id,row.itemId)) continue;", + "replace": " ctx.db.expiredCartItem.id.delete(row.id);\n if (findCartLine(ctx,acc.id,row.itemId)) continue;" + }, + { + "find": " bundlePrice:row.bundlePrice,bundleComponentsJson:row.bundleComponentsJson});\n ctx.db.expiredCartItem.id.delete(row.id);", + "replace": " bundlePrice:row.bundlePrice,bundleComponentsJson:row.bundleComponentsJson});" + } + ] + }, + { + "id": "notifications-view-missing-owner-filter", + "scenario": "tracks/ecommerce/scenarios/progression-delivery-notifications.json", + "targets": [ + "ecommerce.spec.access-control.delivery-notification-privacy.501b" + ], + "desc": "Return every stored notification to each signed-in account instead of only the account owner's notifications.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "return [...ctx.db.notification.accountId.filter(accountId)].map(row => ({", + "replace": "return [...ctx.db.notification.iter()].map(row => ({" + } + ] + }, + { + "id": "return-button-shown-for-pending-order", + "scenario": "tracks/ecommerce/scenarios/progression-order-return-boundary.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3e" + ], + "desc": "Offer the return button on every order that is not cancelled, including one that has not shipped.", + "file": "client/src/components/OrdersPanel.tsx", + "edits": [ + { + "find": " {['shipped', 'delivered'].includes(order.status) && !item.returned && !item.isBundle && (", + "replace": " {order.status !== 'cancelled' && !item.returned && !item.isBundle && (" + } + ] + }, + { + "id": "support-order-link-skips-order-owner-check", + "scenario": "tracks/ecommerce/scenarios/progression-order-support-boundary.json", + "targets": [ + "ecommerce.spec.access-control.order-support-ownership.614b" + ], + "desc": "Attach any existing order to a support case the caller can access, without checking that the caller owns the order.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " const order = requireOrderOwner(ctx, orderId);\n ctx.db.supportTicket.id.update({ ...ticket, orderId: order.id });", + "replace": " const order = ctx.db.customerOrder.id.find(orderId);\n if (!order) throw new SenderError('Order not found.');\n ctx.db.supportTicket.id.update({ ...ticket, orderId: order.id });" + } + ] + }, + { + "id": "support-order-link-requires-staff", + "scenario": "tracks/ecommerce/scenarios/progression-order-support-owned.json", + "targets": [ + "ecommerce.progression.order-support.order-support-owned.614a" + ], + "desc": "Allow only staff to attach an order to a support case, so a customer cannot link their own order.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " const ticket = requireSupportAccess(ctx, ticketId).ticket;", + "replace": " requireStaffOrAdmin(ctx);\n const ticket = requireSupportAccess(ctx, ticketId).ticket;" + } + ] + }, + { + "id": "recommendations-capped-at-highest-price-paid", + "scenario": "tracks/ecommerce/scenarios/progression-personalized-recommendations.json", + "targets": [ + "ecommerce.progression.personalized-recommendations.personalized-recommendations.403a" + ], + "desc": "Recommend only items priced at or below the most the customer has paid for one item, so higher-priced items in the customer's categories are missing.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " const purchasedItemIds = new Set();", + "replace": " const purchasedItemIds = new Set();\n let maxPaid = 0;" + }, + { + "find": " purchasedItemIds.add(li.itemId);", + "replace": " purchasedItemIds.add(li.itemId);\n maxPaid = Math.max(maxPaid, li.unitPrice);" + }, + { + "find": " if (!it) continue;\n candidates.push(", + "replace": " if (!it || it.price > maxPaid) continue;\n candidates.push(" + } + ] + }, + { + "id": "recommendations-blend-latest-store-order", + "scenario": "tracks/ecommerce/scenarios/progression-personalized-recommendations.json", + "targets": [ + "ecommerce.spec.access-control.recommendation-profile-isolation.403b" + ], + "desc": "Add the category of the store's most recent order, placed by anyone, to each customer's recommendation categories, so another customer's purchase changes them.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (purchasedCategoryIds.size === 0) return [];", + "replace": " if (purchasedCategoryIds.size === 0) return [];\n let latestOrderId = 0n;\n for (const order of ctx.db.customerOrder.iter()) if (order.id > latestOrderId) latestOrderId = order.id;\n for (const li of ctx.db.orderItem.orderId.filter(latestOrderId)) {\n const link = ctx.db.itemCategory.itemId.find(li.itemId);\n if (link) purchasedCategoryIds.add(link.categoryId);\n }" + } + ] + }, + { + "id": "promotion-redemption-counted-on-apply", + "scenario": "tracks/ecommerce/scenarios/progression-promotion-reporting.json", + "targets": [ + "ecommerce.progression.promotion-reporting.promotion-report-redemptions.622a" + ], + "desc": "Count a redemption when the code is applied to a cart and again at checkout, so one order reports two redemptions.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (existing) ctx.db.cartPromotion.accountId.update(row);\n else ctx.db.cartPromotion.insert(row);", + "replace": " if (existing) ctx.db.cartPromotion.accountId.update(row);\n else ctx.db.cartPromotion.insert(row);\n ctx.db.promotion.id.update({ ...promo, redemptions: promo.redemptions + 1 });" + } + ] + }, + { + "id": "promotion-report-uses-pre-discount-revenue", + "scenario": "tracks/ecommerce/scenarios/progression-promotion-reporting.json", + "targets": [ + "ecommerce.progression.promotion-reporting.promotion-report-revenue.622b" + ], + "desc": "Report promotion revenue before the discount instead of the discounted amount paid.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": ".reduce((total, order) => total + order.total, 0),", + "replace": ".reduce((total, order) => total + order.total + order.discount, 0)," + } + ] + }, + { + "id": "recommendation-dismissals-shared-across-customers", + "scenario": "tracks/ecommerce/scenarios/progression-recommendation-feedback.json", + "targets": [ + "ecommerce.spec.access-control.recommendation-feedback-privacy.504b" + ], + "desc": "Apply every customer's dismissals when building each customer's recommendations.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " for (const row of ctx.db.recommendationDismissal.byAccountItem.filter(accountId)) {\n dismissedItemIds.add(row.itemId);", + "replace": " for (const row of ctx.db.recommendationDismissal.iter()) {\n dismissedItemIds.add(row.itemId);" + } + ] + }, + { + "id": "activity-history-shows-viewer-as-actor", + "scenario": "tracks/ecommerce/scenarios/progression-staff-activity.json", + "targets": [ + "ecommerce.progression.staff-activity.staff-activity.624a" + ], + "desc": "Show the account viewing the history as the actor of every entry instead of the account that made the change.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " actor: ctx.db.account.id.find(row.actorAccountId)?.username ?? 'unknown',", + "replace": " actor: actor.username," + } + ] + }, + { + "id": "activity-history-open-to-customers", + "scenario": "tracks/ecommerce/scenarios/progression-staff-activity.json", + "targets": [ + "ecommerce.spec.access-control.staff-activity-privacy.624b" + ], + "desc": "Return staff activity history to every signed-in account and show the Activity link whenever history is present.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (!actor || (!actor.isAdmin && !actor.isStaff)) return [];\n return [...ctx.db.staffActivity.iter()].map(row => ({", + "replace": " if (!actor) return [];\n return [...ctx.db.staffActivity.iter()].map(row => ({" + }, + { + "file": "client/src/components/ProgressionWorkbench.tsx", + "find": " {(isStaff || isAdmin) && }", + "replace": " {activity.length > 0 && }" + } + ] + }, + { + "id": "support-refund-allowed-for-case-owner", + "scenario": "tracks/ecommerce/scenarios/progression-support-refunds-access.json", + "targets": [ + "ecommerce.spec.access-control.support-refund-access.615c" + ], + "desc": "Let anyone with access to a support case issue its refund, including the customer who opened it.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " requireStaffOrAdmin(ctx);\n const ticket = ctx.db.supportTicket.id.find(ticketId);\n if (!ticket || ticket.orderId === undefined) throw new SenderError('The case has no linked order.');", + "replace": " const { ticket } = requireSupportAccess(ctx, ticketId);\n if (ticket.orderId === undefined) throw new SenderError('The case has no linked order.');" + } + ] + }, + { + "id": "support-refund-repeats-full-payout", + "scenario": "tracks/ecommerce/scenarios/progression-support-refunds-accounting.json", + "targets": [ + "ecommerce.spec.transactional-integrity.support-refund-accounting.615b" + ], + "desc": "Skip the already-refunded check and refund the full order total on every request, so a repeated refund records a second payout.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " if (order.status === 'cancelled' || ticket.refundTotal > 0) throw new SenderError('The order was already refunded.');\n const amount = order.total - order.refundedTotal;", + "replace": " if (order.status === 'cancelled') throw new SenderError('The order was already refunded.');\n const amount = order.total;" + } + ] + }, + { + "id": "support-refund-leaves-order-status", + "scenario": "tracks/ecommerce/scenarios/progression-support-refunds-resolution.json", + "targets": [ + "ecommerce.progression.support-refunds.support-refunds-resolution.615a" + ], + "desc": "Resolve the support case and record the refund without marking the order refunded.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "refundedTotal: order.total, status: order.status === 'pending' ? 'refunded' : order.status });", + "replace": "refundedTotal: order.total, status: order.status });" + } + ] + }, + { + "id": "delivery-due-time-kept-in-memory", + "scenario": "tracks/ecommerce/scenarios/03-deferred-durability.json", + "targets": [ + "ecommerce.l3.deferred-durability.restart-survival.315a" + ], + "desc": "The delivery job reads each due time from a module-level map filled when the order ships, so a restart forgets pending deliveries and a shipped order stays shipped.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "const SECOND = 1_000_000n;", + "replace": "const SECOND = 1_000_000n;\nconst deliveryDueMicros = new Map();" + }, + { + "find": " else ctx.db.deliverySchedule.insert(schedule);", + "replace": " else ctx.db.deliverySchedule.insert(schedule);\n deliveryDueMicros.set(orderId, schedule.dueMicros);" + }, + { + "find": "if (schedule.completed || schedule.dueMicros > now) continue;", + "replace": "const dueMicros = deliveryDueMicros.get(schedule.orderId);\n if (schedule.completed || dueMicros === undefined || dueMicros > now) continue;" + } + ] + }, + { + "id": "cart-deadlines-rearmed-at-startup", + "scenario": "tracks/ecommerce/scenarios/03-deferred-durability.json", + "targets": [ + "ecommerce.l3.deferred-durability.restart-survival.316a" + ], + "desc": "The first maintenance run after the module loads restarts every open cart deadline with a full five minutes, so a cart expires five minutes after the restart instead of near its original deadline.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "const SECOND = 1_000_000n;", + "replace": "const SECOND = 1_000_000n;\nlet cartTimersRestored = false;" + }, + { + "find": " for (const expiry of [...ctx.db.cartExpiry.iter()]) {", + "replace": " if (!cartTimersRestored) {\n cartTimersRestored = true;\n for (const expiry of [...ctx.db.cartExpiry.iter()]) {\n if (!expiry.expired) ctx.db.cartExpiry.accountId.update({ ...expiry, expiresMicros: now + 300n * SECOND });\n }\n }\n for (const expiry of [...ctx.db.cartExpiry.iter()]) {" + } + ] + }, + { + "id": "startup-replays-completed-shipments", + "scenario": "tracks/ecommerce/scenarios/03-deferred-integrity.json", + "targets": [ + "ecommerce.l3.deferred-integrity.exactly-once.312a" + ], + "desc": "The first maintenance run after the module loads replays shipment for every delivery schedule, including completed ones, so a delivered order is set back to shipped after a restart.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "const SECOND = 1_000_000n;", + "replace": "const SECOND = 1_000_000n;\nlet deliveriesRestored = false;" + }, + { + "find": " for (const schedule of [...ctx.db.deliverySchedule.iter()]) {", + "replace": " if (!deliveriesRestored) {\n deliveriesRestored = true;\n for (const schedule of [...ctx.db.deliverySchedule.iter()]) {\n const order = ctx.db.customerOrder.id.find(schedule.orderId);\n if (!order) continue;\n ctx.db.customerOrder.id.update({ ...order, status: 'shipped' });\n ctx.db.deliverySchedule.orderId.update({ ...schedule, dueMicros: now + 60n * SECOND, completed: false });\n }\n }\n for (const schedule of [...ctx.db.deliverySchedule.iter()]) {" + } + ] + }, + { + "id": "delivery-notification-preference-inverted", + "scenario": "tracks/ecommerce/scenarios/progression-delivery-notifications.json", + "targets": [ + "ecommerce.progression.delivery-notifications.delivery-notification-delivery.501a", + "ecommerce.spec.access-control.delivery-notification-privacy.501b" + ], + "desc": "The order-notification preference check is inverted, so a customer who turned order notifications on never receives a delivery notification.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": "if (kind === 'delivery' && preference && !preference.orderEnabled) return;", + "replace": "if (kind === 'delivery' && preference && preference.orderEnabled) return;" + } + ] + }, + { + "id": "recommendations-held-after-dismissal", + "scenario": "tracks/ecommerce/scenarios/progression-recommendation-feedback.json", + "targets": [ + "ecommerce.progression.recommendation-feedback.recommendation-feedback.504a", + "ecommerce.spec.access-control.recommendation-feedback-privacy.504b" + ], + "desc": "Dismissing a recommendation holds the list as it was when the dismissal was sent, so the dismissed item stays on screen until the page reloads.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const [searchPage, setSearchPage] = useState(0);", + "replace": " const [searchPage, setSearchPage] = useState(0);\n const [sentDismissalRows, setSentDismissalRows] = useState(null);" + }, + { + "find": "{[...recommendedRows].sort((a, b) => a.rank - b.rank).map((r) => (", + "replace": "{[...(sentDismissalRows ?? recommendedRows)].sort((a, b) => a.rank - b.rank).map((r) => (" + }, + { + "find": "onClick={() => conn?.reducers.dismissRecommendation({ itemId: r.itemId })}", + "replace": "onClick={() => { setSentDismissalRows(recommendedRows); conn?.reducers.dismissRecommendation({ itemId: r.itemId }); }}" + } + ] + }, + { + "id": "stock-alert-misses-direct-stock-writes", + "scenario": "tracks/ecommerce/scenarios/progression-stock-alert-direct-restore.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock-alert.631d" + ], + "desc": "Stock alerts are delivered only by the restock reducers, so stock written directly to the database never delivers its alert.", + "file": "backend/spacetimedb/src/index.ts", + "edits": [ + { + "find": " // Stock written directly to the database runs no reducer, so alerts also follow the stock rows.\n for (const alert of [...ctx.db.stockAlert.iter()]) {\n if (alert.fulfilled || totalStock(ctx, alert.itemId) <= 0) continue;\n const name = ctx.db.item.id.find(alert.itemId)?.name ?? 'Item';\n notify(ctx, alert.accountId, 'stock', `${name} is available again.`);\n ctx.db.stockAlert.id.update({ ...alert, fulfilled: true });\n }\n", + "replace": " // Stock alerts are delivered by the restock reducers.\n" + } + ] + } + ] +} diff --git a/tools/stack-bench/grader/mutations/supabase-ecommerce.json b/tools/stack-bench/grader/mutations/supabase-ecommerce.json new file mode 100644 index 00000000000..8980fd4c637 --- /dev/null +++ b/tools/stack-bench/grader/mutations/supabase-ecommerce.json @@ -0,0 +1,2390 @@ +{ + "schemaVersion": 3, + "fixtureSha256": "0f6a67cbb1922d02cee1750b2b802431fc6cc05e055748dd6885e9545c12eeef", + "backend": "supabase", + "track": "ecommerce", + "note": "Native Supabase selected-L3 defect controls. Authored coverage is not qualification; every baseline and mutant requires execution against this exact source hash. Restart and crash-recovery controls act through the start-up seed path that every application start runs; reconnect controls act on the real Realtime channel.", + "mutations": [ + { + "id": "signup-does-not-expose-created-account", + "scenario": "tracks/ecommerce/scenarios/01-account-create.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1a" + ], + "desc": "Every account view omits the username, so the created account is never shown as signed in, including after Realtime refreshes.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": "'username', account.username, 'roles', account.roles", + "replace": "'username', '', 'roles', account.roles" + } + ] + }, + { + "id": "session-token-not-persisted", + "scenario": "tracks/ecommerce/scenarios/01-account-reload.json", + "targets": [ + "ecommerce.spec.state-durability.session-reload.1e" + ], + "desc": "Fresh sign-in works, but each new document deletes the stored Auth session before the Supabase client starts, so reload cannot restore it.", + "file": "client/src/request.ts", + "edits": [ + { + "find": "const STORAGE_KEY = 'storefront-auth';", + "replace": "const STORAGE_KEY = 'storefront-auth';\nlocalStorage.removeItem(STORAGE_KEY);" + } + ] + }, + { + "id": "reload-hydrates-an-empty-cart", + "scenario": "tracks/ecommerce/scenarios/01-cart.json", + "targets": [ + "ecommerce.spec.state-durability.cart-reload.4b" + ], + "desc": "Page loads that start with a stored session discard the cart from shop_state; the first signup page and its working cart are unchanged.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "const CATALOG_PAGE_SIZE = 10;", + "replace": "const CATALOG_PAGE_SIZE = 10;\nconst discardHydratedCart = Boolean(sessionToken());" + }, + { + "find": " setCart(state.cart);", + "replace": " setCart(discardHydratedCart ? { items: [], total: 0 } : state.cart);" + } + ] + }, + { + "id": "warehouse-view-omits-one-location", + "scenario": "tracks/ecommerce/scenarios/01-warehouse-admin-staff.json", + "targets": [ + "ecommerce.feature.warehouse-admin.warehouse-view.7b" + ], + "desc": "The admin warehouse projection truncates the final item-location row.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " {overview.locations.map((loc) => (", + "replace": " {overview.locations.slice(0, -1).map((loc) => (" + } + ] + }, + { + "id": "sign-in-starts-an-empty-cart", + "scenario": "tracks/ecommerce/scenarios/progression-account-state-reload.json", + "targets": [ + "ecommerce.spec.state-durability.account-state-recovery.105a" + ], + "desc": "A successful password sign-in removes the account's stored cart lines through the native cart write, so a fresh sign-in after a restart finds an empty cart; sign-up, token-restored sessions and orders are unchanged.", + "file": "client/src/request.ts", + "edits": [ + { + "find": " session = data.session;\n return { token: data.session.access_token, user: (await mutate('shop_state')).user };", + "replace": " session = data.session;\n // Defect: each sign-in starts a new cart instead of restoring the account cart.\n if (flow === 'signIn') for (const line of (await mutate('shop_state')).cart.items) await mutate('update_cart_quantity', { itemId: line.itemId, quantity: 0 });\n return { token: data.session.access_token, user: (await mutate('shop_state')).user };" + } + ] + }, + { + "id": "duplicate-signup-reports-success", + "scenario": "tracks/ecommerce/scenarios/01-account-duplicate.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1b" + ], + "desc": "Sign-up gives Auth a fresh address for a taken username, and the new-user trigger re-points the existing account at the new Auth user, so the impostor is signed in as the existing account without its password.", + "file": "supabase/migrations/20260925000000_schema.sql", + "edits": [ + { + "find": " account_name := convert_from(decode(split_part(new.email, '@', 1), 'hex'), 'UTF8');", + "replace": " account_name := convert_from(decode(split_part(split_part(new.email, '@', 1), '.', 1), 'hex'), 'UTF8');" + }, + { + "find": " insert into order_account (auth_user_id, username) values (new.id, account_name);", + "replace": " insert into order_account (auth_user_id, username) values (new.id, account_name)\n on conflict (username) do update set auth_user_id = excluded.auth_user_id;" + }, + { + "file": "client/src/request.ts", + "find": " ? await supabase.auth.signUp(credentials)", + "replace": " ? await supabase.auth.signUp({ ...credentials, email: credentials.email.replace('@', `.${Date.now()}@`) })" + } + ] + }, + { + "id": "signin-skips-password-verification", + "scenario": "tracks/ecommerce/scenarios/01-account-password.json", + "targets": [ + "ecommerce.spec.access-control.account-password.1c" + ], + "desc": "Every account uses the same Auth password, so an incorrect typed password grants a session.", + "file": "client/src/request.ts", + "edits": [ + { + "find": "const credentials = { email: accountEmail(username), password };", + "replace": "const credentials = { email: accountEmail(username), password: 'same-password-for-everyone' };" + } + ] + }, + { + "id": "password-verification-truncates-utf8", + "scenario": "tracks/ecommerce/scenarios/01-account-password.json", + "targets": [ + "ecommerce.spec.access-control.account-password.1c" + ], + "desc": "Truncate a password before sending it to Auth, so a different suffix signs in instead of being refused.", + "file": "client/src/request.ts", + "edits": [ + { + "find": "const credentials = { email: accountEmail(username), password };", + "replace": "const credentials = { email: accountEmail(username), password: new TextDecoder().decode(new TextEncoder().encode(password).slice(0, 72)) };" + } + ] + }, + { + "id": "signout-keeps-current-account", + "scenario": "tracks/ecommerce/scenarios/01-account-signout.json", + "targets": [ + "ecommerce.feature.accounts.accounts.1d" + ], + "desc": "Sign-out neither ends the Auth session nor clears the current account.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " await signOut();\n clearSession();", + "replace": " // Defect: sign-out leaves the authenticated account intact." + } + ] + }, + { + "id": "signout-keeps-session", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Show a signed-out interface while keeping the Auth session and its purchase authority in the browser.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " await signOut();\n clearSession();", + "replace": " clearSession(); // Defect: hide the account without ending the Auth session." + }, + { + "find": " {currentUser ? (", + "replace": " {currentUser && token ? (" + } + ] + }, + { + "id": "purchase-session-rejects-returning-login", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "Refuse every returning login; blocking all access after logout must not pass the session check.", + "file": "client/src/request.ts", + "edits": [ + { + "find": " : await supabase.auth.signInWithPassword(credentials);", + "replace": " : await supabase.auth.signInWithPassword({ ...credentials, password: credentials.password + '-refused' });" + } + ] + }, + { + "id": "account-disclosure-rejects-login", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Reject every valid login; absent disclosures alone must not earn a pass.", + "file": "client/src/request.ts", + "edits": [ + { + "find": " : await supabase.auth.signInWithPassword(credentials);", + "replace": " : await supabase.auth.signInWithPassword({ ...credentials, password: credentials.password + '-refused' });" + } + ] + }, + { + "id": "account-password-in-public-data", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Sign-up stores the typed password in Auth user metadata and the new-user trigger appends it to a public catalog item name, delivering it to other clients.", + "file": "supabase/migrations/20260925000000_schema.sql", + "edits": [ + { + "find": " insert into order_account (auth_user_id, username) values (new.id, account_name);\n return new;", + "replace": " insert into order_account (auth_user_id, username) values (new.id, account_name);\n if new.raw_user_meta_data ? 'password' then\n update item set name = name || ' ' || (new.raw_user_meta_data ->> 'password') where id = (select min(id) from item);\n end if;\n return new;" + }, + { + "file": "client/src/request.ts", + "find": " ? await supabase.auth.signUp(credentials)", + "replace": " ? await supabase.auth.signUp({ ...credentials, options: { data: { password } } })" + } + ] + }, + { + "id": "account-password-in-script", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Ship the account password sentinel in the public JavaScript bundle while normal signup and login still work.", + "file": "client/src/main.tsx", + "edits": [ + { + "find": "import React from \"react\";", + "replace": "import React from \"react\";\nObject.assign(window, { debugPassword: 'sb-private-pw-76dc492ea153' });" + } + ] + }, + { + "id": "account-password-in-style", + "scenario": "tracks/ecommerce/scenarios/01-account-disclosure.json", + "targets": [ + "ecommerce.spec.access-control.account-disclosure.110a" + ], + "desc": "Ship the account password sentinel in a public stylesheet while normal signup and login still work.", + "file": "client/src/index.css", + "edits": [ + { + "find": ":root {", + "replace": ":root {\n --debug-password: \"sb-private-pw-76dc492ea153\";" + } + ] + }, + { + "id": "signup-accepts-admin-claim", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "The new-user trigger trusts a role claimed in sign-up user metadata and grants the administrator role, including its stock-write authority.", + "file": "supabase/migrations/20260925000000_schema.sql", + "edits": [ + { + "find": " insert into order_account (auth_user_id, username) values (new.id, account_name);", + "replace": " insert into order_account (auth_user_id, username, roles) values (new.id, account_name,\n case when new.raw_user_meta_data ->> 'role' = 'admin' then array['admin'] else '{}'::text[] end);" + } + ] + }, + { + "id": "admin-boundary-rejects-all-signups", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Keep fixture provisioning available, but the new-user trigger refuses every other registration; blocking useful signup must not pass the authority check.", + "file": "supabase/migrations/20260925000000_schema.sql", + "edits": [ + { + "find": " if account_name !~ '^[A-Za-z0-9-]{1,48}$' then\n raise exception 'Invalid username';\n end if;", + "replace": " if account_name !~ '^[A-Za-z0-9-]{1,48}$' then\n raise exception 'Invalid username';\n end if;\n if account_name not in ('admin', 'staff', 'staff2', 'customer') then\n raise exception 'Registration disabled';\n end if;" + } + ] + }, + { + "id": "reconnect-hydration-loses-account-state", + "scenario": "tracks/ecommerce/scenarios/progression-account-state-reconnect.json", + "targets": [ + "ecommerce.spec.state-durability.account-state-recovery.105b" + ], + "desc": "Network loss permanently removes the Realtime channel, so the account misses peer cart changes after reconnect.", + "file": "client/src/request.ts", + "edits": [ + { + "find": "setTimeout(followChanges);\n});", + "replace": "setTimeout(followChanges);\n});\nwindow.addEventListener('offline', () => { if (changes) supabase.removeChannel(changes); changes = null; }, { once: true });" + } + ] + }, + { + "id": "profile-data-is-lost-on-server-restart", + "scenario": "tracks/ecommerce/scenarios/progression-customer-profile.json", + "targets": [ + "ecommerce.spec.state-durability.customer-profile-reload.620a" + ], + "desc": "The start-up seed path clears stored customer profiles whenever the application restarts; the initial save and token reload remain valid.", + "file": "supabase/seed.sql", + "edits": [ + { + "find": " if exists (select 1 from public.item) then return; end if;", + "replace": " if exists (select 1 from public.item) then\n -- Defect: restarting the application resets customer profiles.\n update public.order_account set profile_name = '', profile_address = '';\n return;\n end if;" + } + ] + }, + { + "id": "purchased-review-ui-does-not-submit", + "scenario": "tracks/ecommerce/scenarios/01-review-eligibility.json", + "targets": [ + "ecommerce.spec.access-control.review-eligibility.108b" + ], + "desc": "The review form discards valid customer submissions. Direct authorized and unauthorized review calls retain their normal behavior.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const handleReviewSubmit = async (itemId: string, rating: number, comment: string) => {\n setReviewError(\"\");", + "replace": " const handleReviewSubmit = async (itemId: string, rating: number, comment: string) => {\n setReviewError(\"Review submission unavailable\");\n return;" + } + ] + }, + { + "id": "progression-customer-sees-fulfilment-content", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-access.json", + "targets": [ + "ecommerce.spec.access-control.fulfilment-area-boundary.1d" + ], + "desc": "Expose the protected staff area to signed-in customers, including its navigation and content.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " {(currentUser?.isStaff || currentUser?.isAdmin) && (\n ", + "replace": " setPreference(value => ({ ...value, order: !value.order }))} />" + }, + { + "find": "function SupportTicket({ ticket, user, act }: any) {", + "replace": "function OrderNotificationToggle({ on, onToggle }: { on: boolean; onToggle: () => void }) {\n const [shown] = useState(on); // Defect: the toggle keeps the state loaded when the card opened.\n return ;\n}\n\nfunction SupportTicket({ ticket, user, act }: any) {" + } + ] + }, + { + "id": "role-editor-snaps-back-to-stored-role", + "scenario": "tracks/ecommerce/scenarios/progression-staff-roles.json", + "targets": [ + "ecommerce.progression.staff-roles.staff-roles.621c" + ], + "desc": "Pressing Save persists the role but resets the dropdown to the role that was stored before the save, so the assignment is not visible until a reload.", + "file": "client/src/ProgressionPanel.tsx", + "edits": [ + { + "find": " ", + "replace": " " + } + ] + }, + { + "id": "review-script-unsafe-render", + "scenario": "tracks/ecommerce/scenarios/progression-review-script.json", + "targets": [ + "ecommerce.progression.review-access-specifications.stored-review-script.9180a" + ], + "desc": "Render stored review text as HTML so it executes in another customer session.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": "
{r.comment}
", + "replace": "
" + } + ] + }, + { + "id": "shipping-counts-sale-twice", + "scenario": "tracks/ecommerce/scenarios/progression-shipping-accounting.json", + "targets": [ + "ecommerce.inventory-operations.shipping-accounting.202e" + ], + "desc": "Shipping doubles the authoritative booked order total.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " update order_header set status = 'shipped' where id = \"orderId\" and status = 'pending';", + "replace": " update order_header set status = 'shipped', total = total * 2 where id = \"orderId\" and status = 'pending';" + } + ] + }, + { + "id": "checkout-retains-purchased-cart", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Checkout reads its cart without consuming it; concurrent calls create duplicate effects.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " delete from order_cart where account_id = me.id returning id, item_id, quantity", + "replace": " select id, item_id, quantity from order_cart where account_id = me.id" + } + ] + }, + { + "id": "checkout-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Reject every checkout, including the positive control; refusal alone is insufficient.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " return json_build_object('orderId', private.place_order(me.id, lines));", + "replace": " raise exception 'Checkout rejected';\n return json_build_object('orderId', private.place_order(me.id, lines));" + } + ] + }, + { + "id": "purchases-do-not-decrement-stock", + "scenario": "tracks/ecommerce/scenarios/01-last-unit.json", + "targets": [ + "ecommerce.spec.concurrency-safety.last-unit.201a", + "ecommerce.spec.concurrency-safety.last-unit.201b", + "ecommerce.spec.concurrency-safety.last-unit.201c" + ], + "desc": "Record purchases without decrementing stock, causing unchanged inventory, excess orders and excess revenue in the same purchase burst.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": "update stock set quantity = quantity - take", + "replace": "update stock set quantity = quantity" + } + ] + }, + { + "id": "purchase-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/01-last-unit.json", + "targets": [ + "ecommerce.spec.concurrency-safety.last-unit.201a", + "ecommerce.spec.concurrency-safety.last-unit.201b", + "ecommerce.spec.concurrency-safety.last-unit.201c" + ], + "desc": "Reject every purchase, leaving unchanged stock, zero successful orders and zero revenue in the required purchase burst.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " insert into order_header (account_id, total) values (buyer, 0) returning id into new_order;", + "replace": " raise exception 'Purchases rejected';\n insert into order_header (account_id, total) values (buyer, 0) returning id into new_order;" + } + ] + }, + { + "id": "order-history-not-owner-scoped", + "scenario": "tracks/ecommerce/scenarios/01-order-ownership.json", + "targets": [ + "ecommerce.spec.access-control.order-ownership.106a" + ], + "desc": "Order history exposes every customer's orders instead of only the signed-in customer's orders.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " from order_header entry where entry.account_id = me.id),", + "replace": " from order_header entry)," + } + ] + }, + { + "id": "cart-repeat-does-not-increment", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4a" + ], + "desc": "An existing cart line remains at its old quantity.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " set quantity = case when adding then order_cart.quantity + excluded.quantity else excluded.quantity end", + "replace": " set quantity = case when adding then excluded.quantity else excluded.quantity end" + } + ] + }, + { + "id": "checkout-leaves-cart-claimed", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "Checkout creates an order without emptying the cart.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " delete from order_cart where account_id = me.id returning id, item_id, quantity", + "replace": " select id, item_id, quantity from order_cart where account_id = me.id" + } + ] + }, + { + "id": "catalog-price-is-offset", + "scenario": "tracks/ecommerce/scenarios/01-catalog-values.json", + "targets": [ + "ecommerce.feature.catalog.catalog-values.2a" + ], + "desc": "Public catalog prices are seeded one unit too high.", + "file": "supabase/seed.sql", + "edits": [ + { + "find": "select name, category, price, description from catalogue returning id, name", + "replace": "select name, category, price + 1, description from catalogue returning id, name" + } + ] + }, + { + "id": "purchase-order-uses-zero-price", + "scenario": "tracks/ecommerce/scenarios/progression-purchasing.json", + "targets": [ + "ecommerce.feature.purchasing.purchase-order.3c" + ], + "desc": "A direct purchase records the item but stores a zero order total instead of the price paid.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " update order_header set total = booked where id = new_order;", + "replace": " update order_header set total = 0 where id = new_order;" + } + ] + }, + { + "id": "authorized-restock-does-not-change-stock", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.feature.warehouse-admin.admin-write.103a", + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Accept an administrator restock without changing warehouse stock.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " update stock set quantity = stock.quantity + admin_restock.quantity", + "replace": " update stock set quantity = stock.quantity" + } + ] + }, + { + "id": "purchases-do-not-affect-best-sellers", + "scenario": "tracks/ecommerce/scenarios/02-operational-best-sellers.json", + "targets": [ + "ecommerce.inventory-operations.operational-views.5d" + ], + "desc": "Rank signed-out recommendations without purchase counts.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " from private.catalog order by purchase_count desc, name limit 10) best;", + "replace": " from private.catalog order by name limit 10) best;" + } + ] + }, + { + "id": "initial-dashboard-load-omits-low-stock", + "scenario": "tracks/ecommerce/scenarios/02-low-stock.json", + "targets": [ + "ecommerce.inventory-operations.operational-views.5e", + "ecommerce.spec.live-state.inventory-dashboard.5a" + ], + "desc": "The dashboard reports no low-stock items, so an item already at ten units or fewer is missing; 5a's opening precondition re-asserts the same listing, so it necessarily regresses with 5e.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": "from private.catalog where stock <= 10),", + "replace": "from private.catalog where false)," + } + ] + }, + { + "id": "category-totals-skip-the-newest-order", + "scenario": "tracks/ecommerce/scenarios/02-operational-category-totals.json", + "targets": [ + "ecommerce.inventory-operations.operational-views.5f" + ], + "desc": "Category totals leave out the most recent order, so the reloaded dashboard shows a purchase only once a later order pushes it into the aggregate.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " where header.status <> 'cancelled' and not line.returned group by item.category) sold", + "replace": " where header.status <> 'cancelled' and not line.returned\n and header.id < (select max(id) from order_header) group by item.category) sold" + } + ] + }, + { + "id": "transfer-debits-source-without-crediting-existing-destination", + "scenario": "tracks/ecommerce/scenarios/02-strengthened.json", + "targets": [ + "ecommerce.inventory-operations.warehouse-transfer.2a", + "ecommerce.operations-access.operator-authorization.201a" + ], + "desc": "A transfer debits the source but credits zero to the destination. This breaks both movement/conservation assertions and the authorized-transfer positive observation inside the authorization check; that last failure is not evidence of an unauthorized transfer.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " update stock set quantity = stock.quantity + admin_transfer_stock.quantity", + "replace": " update stock set quantity = stock.quantity" + } + ] + }, + { + "id": "cancel-does-not-restore-stock-fresh-client", + "scenario": "tracks/ecommerce/scenarios/02-self-contained.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202b" + ], + "desc": "Cancellation changes order state but skips restoration, so a fresh client reads the persisted shortfall.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " update stock set quantity = stock.quantity + allocation.quantity", + "replace": " update stock set quantity = stock.quantity" + } + ] + }, + { + "id": "transfer-creates-stock-during-race", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202d" + ], + "desc": "A transfer creates one extra unit while a purchase runs.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " update stock set quantity = stock.quantity + admin_transfer_stock.quantity", + "replace": " update stock set quantity = stock.quantity + admin_transfer_stock.quantity + 1" + } + ] + }, + { + "id": "customer-can-cancel-scheduled-restock", + "scenario": "tracks/ecommerce/scenarios/03-deferred-access.json", + "targets": [ + "ecommerce.l3.deferred-access.scheduled-work-access.317a" + ], + "desc": "Scheduled-restock cancellation omits its administrator check.", + "file": "supabase/migrations/20260925000200_progression.sql", + "edits": [ + { + "find": " perform private.require_operator(true);\n update scheduled_restock set status = 'cancelled'", + "replace": " perform private.require_account();\n update scheduled_restock set status = 'cancelled'" + } + ] + }, + { + "id": "completed-restock-is-replayed", + "scenario": "tracks/ecommerce/scenarios/03-deferred-integrity.json", + "targets": [ + "ecommerce.l3.deferred-integrity.exactly-once.311a" + ], + "desc": "Completing the deferred work applies its stock effect twice.", + "file": "supabase/migrations/20260925000200_progression.sql", + "edits": [ + { + "find": " update stock set quantity = quantity + due.quantity", + "replace": " update stock set quantity = quantity + 2 * due.quantity" + } + ] + }, + { + "id": "due-restock-does-not-change-stock", + "scenario": "tracks/ecommerce/scenarios/03-scheduled-restock-apply.json", + "targets": [ + "ecommerce.l3.scheduled-restocks.scheduled-restocks.305a" + ], + "desc": "A due restock is recorded without adding stock.", + "file": "supabase/migrations/20260925000200_progression.sql", + "edits": [ + { + "find": " update stock set quantity = quantity + due.quantity", + "replace": " update stock set quantity = quantity" + } + ] + }, + { + "id": "cancelled-restock-remains-pending", + "scenario": "tracks/ecommerce/scenarios/03-scheduled-restock-cancel.json", + "targets": [ + "ecommerce.l3.scheduled-restocks.scheduled-restocks.306a" + ], + "desc": "Cancelling a restock is acknowledged but leaves it pending so it later applies.", + "file": "supabase/migrations/20260925000200_progression.sql", + "edits": [ + { + "find": " update scheduled_restock set status = 'cancelled' where id = \"restockId\" and status = 'pending';", + "replace": " perform 1 from scheduled_restock where id = \"restockId\" and status = 'pending';" + } + ] + }, + { + "id": "server-time-restock-never-completes", + "scenario": "tracks/ecommerce/scenarios/03-server-time.json", + "targets": [ + "ecommerce.l3.server-time.server-time.312a" + ], + "desc": "Scheduling ignores the requested delay and sets a due time a day away, so the restock never completes in time.", + "file": "supabase/migrations/20260925000200_progression.sql", + "edits": [ + { + "find": "now() + make_interval(secs => \"delaySeconds\")", + "replace": "now() + interval '1 day'" + } + ] + }, + { + "id": "scheduled-restock-never-becomes-due-after-restart", + "scenario": "tracks/ecommerce/scenarios/03-server-time.json", + "targets": [ + "ecommerce.l3.server-time.server-time.312a" + ], + "desc": "The start-up seed path mistakenly cancels pending deferred work whenever the application restarts; ordinary scheduling before a restart still completes.", + "file": "supabase/seed.sql", + "edits": [ + { + "find": " if exists (select 1 from public.item) then return; end if;", + "replace": " if exists (select 1 from public.item) then\n -- Defect: restarting the application cancels pending restocks.\n update public.scheduled_restock set status = 'cancelled' where status = 'pending';\n return;\n end if;" + } + ] + }, + { + "id": "ship-acknowledges-without-changing-status", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-ship.json", + "targets": [ + "ecommerce.operations-access.fulfilment-queue.1c" + ], + "desc": "Shipping returns success but writes pending back to the order, leaving both live views unchanged.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " update order_header set status = 'shipped' where id = \"orderId\" and status = 'pending';", + "replace": " update order_header set status = 'pending' where id = \"orderId\" and status = 'pending';" + } + ] + }, + { + "id": "customer-can-ship-order-direct-1-1", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "The shipping operation keeps authentication but drops its staff role gate.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " me order_account := private.require_operator(false);", + "replace": " me order_account := private.require_account();" + } + ] + }, + { + "id": "customer-can-cancel-foreign-order-1-1", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.order-owner.204a" + ], + "desc": "Cancellation retains authentication and pending-state validation but drops order ownership.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " select * into found_order from order_header where id = wanted and account_id = me.id for update;", + "replace": " select * into found_order from order_header where id = wanted for update;" + } + ] + }, + { + "id": "promotion-save-drops-bounded-values", + "scenario": "tracks/ecommerce/scenarios/progression-promotion-rules.json", + "targets": [ + "ecommerce.progression.promotion-rules.promotion-rule-values.620a" + ], + "desc": "Promotion creation drops its bounded discount and limit.", + "file": "supabase/migrations/20260925000200_progression.sql", + "edits": [ + { + "find": " values (create_promotion.code, \"discountPercent\", \"startMicros\", \"endMicros\", \"usageLimit\");", + "replace": " values (create_promotion.code, 0, \"startMicros\", \"endMicros\", 0);" + } + ] + }, + { + "id": "stock-alert-delivery-is-suppressed", + "scenario": "tracks/ecommerce/scenarios/progression-stock-alert-delivery.json", + "targets": [ + "ecommerce.progression.stock-alerts.stock-alert-delivery.631c" + ], + "desc": "A request is accepted and marked delivered, but restored stock does not deliver its alert.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " from delivered join item on item.id = delivered.item_id;", + "replace": " from delivered join item on item.id = delivered.item_id where false;" + } + ] + }, + { + "id": "support-intake-returns-no-reference", + "scenario": "tracks/ecommerce/scenarios/progression-support-intake.json", + "targets": [ + "ecommerce.progression.support-intake.support-intake.610a" + ], + "desc": "Support intake creates a ticket but stores and returns an empty reference.", + "file": "supabase/migrations/20260925000200_progression.sql", + "edits": [ + { + "find": " update support_ticket set reference = 'SUP-' || lpad(ticket.id::text, 6, '0') where id = ticket.id", + "replace": " update support_ticket set reference = '' where id = ticket.id" + } + ] + }, + { + "id": "support-triage-discards-updates", + "scenario": "tracks/ecommerce/scenarios/progression-support-triage.json", + "targets": [ + "ecommerce.progression.support-triage.support-assignment.611a", + "ecommerce.progression.support-triage.support-priority.611b", + "ecommerce.progression.support-triage.support-status.611c" + ], + "desc": "Support triage accepts but discards assignment, priority, and status updates.", + "file": "supabase/migrations/20260925000200_progression.sql", + "edits": [ + { + "find": " update support_ticket set assignee = coalesce(update_support.assignee, ''),\n priority = coalesce(update_support.priority, ''), status = coalesce(update_support.status, '')\n where id = \"ticketId\";", + "replace": " perform 1 from support_ticket where id = \"ticketId\";" + } + ] + }, + { + "id": "cancel-does-not-restore-stock-feature", + "scenario": "tracks/ecommerce/scenarios/02-order-cancellation-core.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3a" + ], + "desc": "Cancellation changes order state but skips restoration of its recorded warehouse allocations.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " update stock set quantity = stock.quantity + allocation.quantity", + "replace": " update stock set quantity = stock.quantity" + } + ] + }, + { + "id": "cancel-restores-stock-but-keeps-pending-status", + "scenario": "tracks/ecommerce/scenarios/02-order-cancellation-history.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3b" + ], + "desc": "Cancellation restores allocations but writes pending back to order history.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " update order_header set status = 'cancelled', refunded = total where id = target.id;", + "replace": " update order_header set status = 'pending', refunded = total where id = target.id;" + } + ] + }, + { + "id": "cancellation-accounting-loses-stock-restoration", + "scenario": "tracks/ecommerce/scenarios/02-invariants.json", + "targets": [ + "ecommerce.returns-pricing.refund-accounting.203a" + ], + "desc": "Cancellation removes revenue and changes order status, but loses the original warehouse stock restoration. The native refund-accounting assertion must detect this.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " update stock set quantity = stock.quantity + allocation.quantity", + "replace": " update stock set quantity = stock.quantity" + } + ] + }, + { + "id": "negative-cart-quantity-is-accepted", + "scenario": "tracks/ecommerce/scenarios/01-cart-boundary.json", + "targets": [ + "ecommerce.spec.access-control.cart-boundary.109b" + ], + "desc": "Cart quantity updates accept values below one.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " perform private.check_quantity(amount, not adding);", + "replace": " if amount >= 0 then perform private.check_quantity(amount, not adding); end if;" + }, + { + "file": "supabase/migrations/20260925000000_schema.sql", + "find": " quantity integer not null check (quantity > 0),\n unique (account_id, item_id)", + "replace": " quantity integer not null,\n unique (account_id, item_id)" + } + ] + }, + { + "id": "customer-can-create-promotion", + "scenario": "tracks/ecommerce/scenarios/progression-promotion-rules.json", + "targets": [ + "ecommerce.spec.access-control.promotion-management-boundary.620b" + ], + "desc": "Promotion creation omits its staff authorization check.", + "file": "supabase/migrations/20260925000200_progression.sql", + "edits": [ + { + "find": " perform private.require_operator(false);\n if coalesce(trim(code), '') = ''", + "replace": " perform private.require_account();\n if coalesce(trim(code), '') = ''" + } + ] + }, + { + "id": "unpurchased-review-is-accepted", + "scenario": "tracks/ecommerce/scenarios/01-review-eligibility.json", + "targets": [ + "ecommerce.spec.access-control.review-eligibility.108a" + ], + "desc": "The review operation bypasses its completed-purchase eligibility check.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " raise exception 'Purchase this item before reviewing' using errcode = '42501';", + "replace": " null; -- Defect: the purchase requirement is not enforced." + } + ] + }, + { + "id": "customer-signin-gains-staff-role", + "scenario": "tracks/ecommerce/scenarios/progression-staff-access.json", + "targets": [ + "ecommerce.spec.access-control.staff-area-boundary.601b" + ], + "desc": "Treat every signed-in account, including the seeded customer, as staff in authorization and public account state.", + "file": "supabase/migrations/20260925000000_schema.sql", + "edits": [ + { + "find": "select coalesce(roles && array['admin', 'staff', 'inventory'], false)", + "replace": "select roles is not null" + } + ] + }, + { + "id": "staff-can-assign-roles", + "scenario": "tracks/ecommerce/scenarios/progression-staff-roles.json", + "targets": [ + "ecommerce.spec.access-control.staff-role-boundary.621b", + "ecommerce.spec.access-control.staff-role-revocation.621d" + ], + "desc": "Staff can call the role assignment operation, including after administrator access is revoked.", + "file": "supabase/migrations/20260925000200_progression.sql", + "edits": [ + { + "find": " perform private.require_operator(true);\n if role is null or role not in ('staff', 'inventory', 'admin')", + "replace": " perform private.require_operator(false);\n if role is null or role not in ('staff', 'inventory', 'admin')" + } + ] + }, + { + "id": "stock-alerts-are-not-owner-scoped", + "scenario": "tracks/ecommerce/scenarios/progression-stock-alerts.json", + "targets": [ + "ecommerce.spec.access-control.stock-alert-privacy.631b" + ], + "desc": "Notification reads expose alerts from other accounts.", + "file": "supabase/migrations/20260925000200_progression.sql", + "edits": [ + { + "find": " order by id), '[]') from notification where account_id = me.id),", + "replace": " order by id), '[]') from notification)," + } + ] + }, + { + "id": "staff-can-use-direct-restock", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "The direct restock operation accepts staff instead of administrators only.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " perform private.require_operator(true);\n perform private.check_quantity(admin_restock.quantity);", + "replace": " perform private.require_operator(false);\n perform private.check_quantity(admin_restock.quantity);" + } + ] + }, + { + "id": "concurrent-cart-add-does-not-increment", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203a" + ], + "desc": "Concurrent cart adds overwrite the line quantity instead of incrementing it, so the existing cart remains checkable at checkout.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " set quantity = case when adding then order_cart.quantity + excluded.quantity else excluded.quantity end", + "replace": " set quantity = case when adding then excluded.quantity else excluded.quantity end" + } + ] + }, + { + "id": "restock-race-records-wrong-order-total", + "scenario": "tracks/ecommerce/scenarios/01-restock-race.json", + "targets": [ + "ecommerce.spec.concurrency-safety.restock-race.202a" + ], + "desc": "Purchases preserve stock and visible order counts but record the wrong booked total. Native mixed-race reconciliation must reject them.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " update order_header set total = booked where id = new_order;", + "replace": " update order_header set total = booked + 1 where id = new_order;" + } + ] + }, + { + "id": "profile-read-not-owner-scoped", + "scenario": "tracks/ecommerce/scenarios/progression-customer-profile.json", + "targets": [ + "ecommerce.spec.access-control.customer-profile-privacy.620b" + ], + "desc": "Profile reads expose another account's saved address.", + "file": "supabase/migrations/20260925000200_progression.sql", + "edits": [ + { + "find": " json_build_object('name', me.profile_name, 'address', me.profile_address) end,", + "replace": " (select json_build_object('name', acct.profile_name, 'address', acct.profile_address)\n from order_account acct where acct.profile_address <> '' order by acct.id limit 1) end," + } + ] + }, + { + "id": "support-read-not-owner-scoped", + "scenario": "tracks/ecommerce/scenarios/progression-support-history.json", + "targets": [ + "ecommerce.spec.access-control.support-history-privacy.612b", + "ecommerce.spec.access-control.support-history-logout.612d" + ], + "desc": "All customers and signed-out clients receive private support history.", + "file": "supabase/migrations/20260925000200_progression.sql", + "edits": [ + { + "find": " from support_ticket ticket where staff or ticket.account_id = me.id),", + "replace": " from support_ticket ticket)," + } + ] + }, + { + "id": "support-reply-not-owner-scoped", + "scenario": "tracks/ecommerce/scenarios/progression-managed-support-privacy.json", + "targets": [ + "ecommerce.spec.access-control.managed-support-privacy.613b" + ], + "desc": "A customer can reply to another customer's support case.", + "file": "supabase/migrations/20260925000200_progression.sql", + "edits": [ + { + "find": " if not exists (select 1 from support_ticket where id = \"ticketId\"\n and (account_id = me.id or private.staff_roles(me.roles))) then", + "replace": " if not exists (select 1 from support_ticket where id = \"ticketId\") then" + } + ] + }, + { + "id": "preference-read-not-owner-scoped", + "scenario": "tracks/ecommerce/scenarios/progression-notification-preferences.json", + "targets": [ + "ecommerce.spec.access-control.notification-preferences-privacy.630b" + ], + "desc": "Notification settings expose another customer's saved preference.", + "file": "supabase/migrations/20260925000200_progression.sql", + "edits": [ + { + "find": " 'preference', json_build_object('order', coalesce(me.order_notifications, false),\n 'stock', coalesce(me.stock_notifications, false)),", + "replace": " 'preference', coalesce((select json_build_object('order', acct.order_notifications, 'stock', acct.stock_notifications)\n from order_account acct where acct.order_notifications or acct.stock_notifications order by acct.id limit 1),\n json_build_object('order', false, 'stock', false))," + } + ] + }, + { + "id": "staff-admin-access-survives-role-removal", + "scenario": "tracks/ecommerce/scenarios/progression-staff-roles.json", + "targets": [ + "ecommerce.spec.access-control.staff-role-revocation.621d" + ], + "desc": "Role reassignment retains the previous administrator privilege.", + "file": "supabase/migrations/20260925000200_progression.sql", + "edits": [ + { + "find": " update order_account set roles = array[role] where id = \"accountId\";", + "replace": " update order_account set roles = array[role] || case when 'admin' = any(roles) and role <> 'admin' then array['admin'] else '{}'::text[] end where id = \"accountId\";" + } + ] + }, + { + "id": "review-eligibility-not-enforced", + "scenario": "tracks/ecommerce/scenarios/progression-review-access.json", + "targets": [ + "ecommerce.progression.review-access-specifications.review-eligibility-direct.618a" + ], + "desc": "The native review operation permits an account that never purchased the item.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " raise exception 'Purchase this item before reviewing' using errcode = '42501';", + "replace": " null; -- Defect: non-buyers may submit a review." + } + ] + }, + { + "id": "review-script-reject-all", + "scenario": "tracks/ecommerce/scenarios/progression-review-script.json", + "targets": [ + "ecommerce.progression.review-access-specifications.stored-review-script.9180a" + ], + "desc": "Reject every review, including benign text; rejecting all input is not safe rendering.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " if exists (select 1 from review where item_id = \"itemId\" and account_id = me.id) then", + "replace": " raise exception 'Reviews rejected';\n if exists (select 1 from review where item_id = \"itemId\" and account_id = me.id) then" + } + ] + }, + { + "id": "duplicate-review-accepted", + "scenario": "tracks/ecommerce/scenarios/01-review-uniqueness.json", + "targets": [ + "ecommerce.spec.transactional-integrity.unique-review.6b" + ], + "desc": "The same purchaser can record multiple reviews for one item: the duplicate check and its unique constraint are both gone.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " raise exception 'Already reviewed';", + "replace": " null; -- Defect: duplicate author reviews are accepted." + }, + { + "file": "supabase/migrations/20260925000000_schema.sql", + "find": " created_at timestamptz not null default now(),\n unique (item_id, account_id)\n);", + "replace": " created_at timestamptz not null default now()\n);" + } + ] + }, + { + "id": "external-stock-client-snapshot-open", + "scenario": "tracks/ecommerce/scenarios/01-external-live-sync.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock.901a" + ], + "desc": "Keep the first catalog values from shop_state instead of accepting later Realtime-driven stock changes.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setItems(state.items);", + "replace": " setItems(current => current.length ? current : state.items);" + } + ] + }, + { + "id": "external-stock-client-snapshot-reconnect", + "scenario": "tracks/ecommerce/scenarios/01-external-reconnect-sync.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock.901d" + ], + "desc": "Keep the first catalog values from shop_state instead of accepting later Realtime-driven stock changes.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setItems(state.items);", + "replace": " setItems(current => current.length ? current : state.items);" + } + ] + }, + { + "id": "review-comment-is-not-persisted", + "scenario": "tracks/ecommerce/scenarios/01-review-visibility.json", + "targets": [ + "ecommerce.feature.reviews.reviews.6a" + ], + "desc": "Review submission persists an empty comment rather than the customer's submitted text.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " values (\"itemId\", me.id, submit_review.rating, submit_review.comment);", + "replace": " values (\"itemId\", me.id, submit_review.rating, '');" + } + ] + }, + { + "id": "queue-ignores-live-fulfilment-updates", + "scenario": "tracks/ecommerce/scenarios/02-fulfilment-live.json", + "targets": [ + "ecommerce.spec.live-state.fulfilment-queue.1a" + ], + "desc": "The open staff queue ignores fulfilment state from shop_state, so a new order never appears in it.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setFulfilmentQueue(state.fulfilment || { orders: [], depth: 0 });", + "replace": " // Defect: the refreshed fulfilment view is ignored." + } + ] + }, + { + "id": "managed-support-live-refresh-keeps-stale-tickets", + "scenario": "tracks/ecommerce/scenarios/progression-managed-support-shared.json", + "targets": [ + "ecommerce.spec.live-state.managed-support.613a" + ], + "desc": "The progression subscription keeps the first support ticket snapshot. Saved replies remain visible on initial load, but another open client does not receive updates.", + "file": "client/src/ProgressionPanel.tsx", + "edits": [ + { + "find": " useEffect(() => subscribeProgression(next => {\n setState({ ...next, loadedToken: token });", + "replace": " useEffect(() => subscribeProgression(next => {\n setState((current: any) => ({ ...next, tickets: current.loadedToken === token ? current.tickets : next.tickets, loadedToken: token }));" + } + ] + }, + { + "id": "open-review-list-ignores-live-update", + "scenario": "tracks/ecommerce/scenarios/progression-open-list-live.json", + "targets": [ + "ecommerce.spec.live-state.open-list.902a" + ], + "desc": "The detail state retains its first review list while other item data stays current. New reviews appear on reload but not in the already-open view.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setItemDetails(state.details);", + "replace": " setItemDetails(previous => state.details.map((item: ItemDetailT) => ({ ...item, reviews: previous.find(old => old.id === item.id)?.reviews || item.reviews })));" + } + ] + }, + { + "id": "espresso-stock-row-ignores-live-updates", + "scenario": "tracks/ecommerce/scenarios/01-buying.json", + "targets": [ + "ecommerce.spec.live-state.purchase-stock.3b" + ], + "desc": "The live catalogue handler preserves a stale Espresso Machine stock projection while applying all other item updates.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setItems(state.items);", + "replace": " setItems(current => state.items.map((item: ItemT) => item.name === 'Espresso Machine' ? current.find(previous => previous.id === item.id) || item : item));" + } + ] + }, + { + "id": "purchase-counts-never-affect-ranking", + "scenario": "tracks/ecommerce/scenarios/01-core.json", + "targets": [ + "ecommerce.spec.live-state.ranking.2c" + ], + "desc": "The catalogue ranking ignores recorded purchases and therefore never promotes the bought item.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " coalesce((select sum(line.quantity) from public.order_line line\n join public.order_header header on header.id = line.order_id\n where line.item_id = item.id and not line.returned and header.status <> 'cancelled'), 0)::integer as purchase_count", + "replace": " 0 as purchase_count" + } + ] + }, + { + "id": "live-review-average-uses-an-extra-divisor", + "scenario": "tracks/ecommerce/scenarios/01-review-rating-live.json", + "targets": [ + "ecommerce.spec.live-state.rating.6c" + ], + "desc": "The review average divides the rating sum by one more review than actually exists.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " avg(review.rating) as average", + "replace": " sum(review.rating)::numeric / (count(*) + 1) as average" + } + ] + }, + { + "id": "live-admin-updates-keep-stale-category-totals", + "scenario": "tracks/ecommerce/scenarios/02-operational-category-totals.json", + "targets": [ + "ecommerce.spec.live-state.sales-dashboard.5b" + ], + "desc": "Live admin updates keep the category totals loaded at page load, so a purchase does not move units or revenue until a reload.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setAdminOverview(state.admin);", + "replace": " setAdminOverview(previous => state.admin && previous ? { ...state.admin, categories: previous.categories } : state.admin);" + } + ] + }, + { + "id": "shared-cart-live-events-ignored", + "scenario": "tracks/ecommerce/scenarios/01-cart.json", + "targets": [ + "ecommerce.spec.live-state.shared-cart.4c" + ], + "desc": "The cart accepts initial snapshots and updates after a local write, but an idle second session ignores remote cart changes.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const [cart, setCart] = useState({ items: [], total: 0 });", + "replace": " const [cart, setCart] = useState({ items: [], total: 0 });\n const cartSnapshotAccount = useRef(null);\n const cartWrittenHere = useRef(false);" + }, + { + "find": " setCart(state.cart);", + "replace": " const cartAccount = state.user?.id ?? \"anonymous\";\n if (cartSnapshotAccount.current !== cartAccount) cartWrittenHere.current = false;\n if (cartSnapshotAccount.current !== cartAccount || cartWrittenHere.current) setCart(state.cart);\n cartSnapshotAccount.current = cartAccount;" + }, + { + "find": " await mutate(\"add_to_cart\", { itemId, quantity: 1 });", + "replace": " cartWrittenHere.current = true;\n await mutate(\"add_to_cart\", { itemId, quantity: 1 });" + } + ] + }, + { + "id": "restock-does-not-increase-stock", + "scenario": "tracks/ecommerce/scenarios/01-warehouse-stock-live-staff.json", + "targets": [ + "ecommerce.spec.live-state.warehouse-stock.7c" + ], + "desc": "The restock operation accepts a positive quantity but increments authoritative stock by zero.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " update stock set quantity = stock.quantity + admin_restock.quantity", + "replace": " update stock set quantity = stock.quantity" + } + ] + }, + { + "id": "notification-preference-is-not-saved", + "scenario": "tracks/ecommerce/scenarios/progression-notification-preferences.json", + "targets": [ + "ecommerce.spec.state-durability.notification-preferences-reload.630a", + "ecommerce.spec.access-control.notification-preferences-privacy.630b" + ], + "desc": "Saved notification preferences remain disabled after reload. It also breaks the owner's saved-choice positive control in 630b; that coupled failure is not evidence of a privacy defect.", + "file": "supabase/migrations/20260925000200_progression.sql", + "edits": [ + { + "find": " update order_account set order_notifications = coalesce(\"order\", false),\n stock_notifications = coalesce(save_preferences.stock, false) where id = me.id;", + "replace": " update order_account set order_notifications = false, stock_notifications = false where id = me.id;" + } + ] + }, + { + "id": "role-assignment-drops-role", + "scenario": "tracks/ecommerce/scenarios/progression-staff-roles.json", + "targets": [ + "ecommerce.spec.state-durability.staff-role-reload.621a", + "ecommerce.spec.access-control.staff-role-boundary.621b", + "ecommerce.spec.access-control.staff-role-revocation.621d" + ], + "desc": "Inventory role assignments are saved as staff, including the authorized baselines for access-control checks; staff and administrator role transitions still work.", + "file": "supabase/migrations/20260925000200_progression.sql", + "edits": [ + { + "find": " update order_account set roles = array[role] where id = \"accountId\";", + "replace": " update order_account set roles = array[case when role = 'inventory' then 'staff' else role end] where id = \"accountId\";" + } + ] + }, + { + "id": "revenue-aggregation-ignores-order-totals", + "scenario": "tracks/ecommerce/scenarios/progression-books-balance.json", + "targets": [ + "ecommerce.spec.transactional-integrity.books-balance.107a" + ], + "desc": "The admin revenue aggregation counts every order as zero regardless of its stored total.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " 'revenue', (select coalesce(sum(total - refunded), 0) from order_header),", + "replace": " 'revenue', 0," + } + ] + }, + { + "id": "purchase-does-not-reduce-warehouse-stock", + "scenario": "tracks/ecommerce/scenarios/progression-books-balance.json", + "targets": [ + "ecommerce.spec.transactional-integrity.books-balance.107b" + ], + "desc": "Purchases create orders without reducing warehouse stock.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": "update stock set quantity = quantity - take", + "replace": "update stock set quantity = quantity" + } + ] + }, + { + "id": "direct-purchase-total-ignores-store-price", + "scenario": "tracks/ecommerce/scenarios/01-server-price.json", + "targets": [ + "ecommerce.spec.transactional-integrity.server-price.104a" + ], + "desc": "The direct purchase creates one order but books one unit of currency per item rather than the store's current price.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " booked := booked + product.price * amount;", + "replace": " booked := booked + 1 * amount;" + } + ] + }, + { + "id": "overdraw-transfer-is-accepted", + "scenario": "tracks/ecommerce/scenarios/02-transfer-overdraw.json", + "targets": [ + "ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c" + ], + "desc": "The source debit no longer requires sufficient quantity and the stock table no longer forbids negative quantities, so an overdrawn transfer succeeds and moves both warehouse totals.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " if available < admin_transfer_stock.quantity then raise exception 'Not enough stock'; end if;", + "replace": " -- Defect: a transfer may debit below zero." + }, + { + "file": "supabase/migrations/20260925000000_schema.sql", + "find": " quantity integer not null check (quantity >= 0),", + "replace": " quantity integer not null," + } + ] + }, + { + "id": "recommendations-ignore-pending-purchases", + "scenario": "tracks/ecommerce/scenarios/02-operational-recommendations.json", + "targets": [ + "ecommerce.inventory-operations.operational-views.5c" + ], + "desc": "Never use the customer's purchases to choose recommended categories.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " where header.account_id = me.id and header.status <> 'cancelled' and not line.returned)", + "replace": " where false)" + } + ] + }, + { + "id": "cart-add-line-lookup-not-owner-scoped", + "scenario": "tracks/ecommerce/scenarios/01-cart-boundary.json", + "targets": [ + "ecommerce.spec.access-control.cart-boundary.109a" + ], + "desc": "Cart line lookup ignores account ownership, so a second customer changes the first customer's item quantity.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " insert into order_cart (account_id, item_id, quantity) values (me.id, product, amount)\n on conflict (account_id, item_id) do update", + "replace": " -- Defect: an existing cart line for the item is found without checking its owner.\n update order_cart set quantity = case when adding then order_cart.quantity + amount else amount end\n where id = (select id from order_cart where item_id = product order by id limit 1)\n returning quantity into next_quantity;\n if not found then\n insert into order_cart (account_id, item_id, quantity) values (me.id, product, amount)\n on conflict (account_id, item_id) do update" + }, + { + "find": " returning quantity into next_quantity;\n if next_quantity >", + "replace": " returning quantity into next_quantity;\n end if;\n if next_quantity >" + } + ] + }, + { + "id": "direct-purchase-is-attributed-to-another-account", + "scenario": "tracks/ecommerce/scenarios/01-purchase-attribution.json", + "targets": [ + "ecommerce.spec.access-control.purchase-attribution.102a" + ], + "desc": "The purchase order is attributed to another account despite the real caller session.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": "values (buyer, 0) returning id into new_order;", + "replace": "values ((select id from order_account where id <> buyer order by id limit 1), 0) returning id into new_order;" + } + ] + }, + { + "id": "unauthenticated-purchase-defaults-to-admin", + "scenario": "tracks/ecommerce/scenarios/01-purchase-session.json", + "targets": [ + "ecommerce.spec.access-control.purchase-session.101a" + ], + "desc": "A signed-out direct buy defaults to the administrator account and commits a purchase.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " me order_account := private.require_account();\nbegin\n return json_build_object('orderId',\n private.place_order(me.id, jsonb_build_array(", + "replace": " me order_account := private.current_account();\nbegin\n -- Defect: a signed-out purchase is booked to the administrator account.\n if me.id is null then select * into me from order_account where 'admin' = any(roles) order by id limit 1; end if;\n return json_build_object('orderId',\n private.place_order(me.id, jsonb_build_array(" + } + ] + }, + { + "id": "signed-out-visitor-purchase-is-accepted", + "scenario": "tracks/ecommerce/scenarios/progression-signed-out-purchase.json", + "targets": [ + "ecommerce.spec.access-control.signed-out-purchase.3a" + ], + "desc": "Expose the purchase control while signed out and accept its anonymous buy as an administrator-owned purchase.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " me order_account := private.require_account();\nbegin\n return json_build_object('orderId',\n private.place_order(me.id, jsonb_build_array(", + "replace": " me order_account := private.current_account();\nbegin\n -- Defect: a signed-out purchase is booked to the administrator account.\n if me.id is null then select * into me from order_account where 'admin' = any(roles) order by id limit 1; end if;\n return json_build_object('orderId',\n private.place_order(me.id, jsonb_build_array(" + }, + { + "file": "client/src/App.tsx", + "find": " const isCustomer = !!currentUser && !currentUser.isAdmin && !currentUser.isStaff;", + "replace": " const isCustomer = !currentUser || (!currentUser.isAdmin && !currentUser.isStaff);" + } + ] + }, + { + "id": "staff-can-see-admin-navigation", + "scenario": "tracks/ecommerce/scenarios/01-warehouse-admin-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-area-boundary.7a" + ], + "desc": "Staff are incorrectly treated as administrators in reads, navigation, and writes.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": "'isAdmin', 'admin' = any(account.roles),", + "replace": "'isAdmin', account.roles && array['admin', 'staff']," + }, + { + "find": " admin boolean := coalesce('admin' = any(me.roles), false);", + "replace": " admin boolean := coalesce(me.roles && array['admin', 'staff'], false);" + }, + { + "find": "then 'admin' = any(me.roles) else", + "replace": "then me.roles && array['admin', 'staff'] else" + } + ] + }, + { + "id": "stock-alert-repeats-while-in-stock", + "scenario": "tracks/ecommerce/scenarios/progression-stock-alerts.json", + "targets": [ + "ecommerce.spec.transactional-integrity.stock-alert-deduplication.631a" + ], + "desc": "One restock delivers two independent notifications for the same pending stock alert.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " from delivered join item on item.id = delivered.item_id;", + "replace": " from delivered join item on item.id = delivered.item_id cross join generate_series(1, 2);" + } + ] + }, + { + "id": "last-unit-allows-negative-stock", + "scenario": "tracks/ecommerce/scenarios/01-last-unit.json", + "targets": [ + "ecommerce.spec.concurrency-safety.last-unit.201a", + "ecommerce.spec.concurrency-safety.last-unit.201b", + "ecommerce.spec.concurrency-safety.last-unit.201c" + ], + "desc": "Allocation permits an oversell and debits the first warehouse below zero; the stock table no longer forbids negative quantities.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " if (select coalesce(sum(quantity), 0) from stock where item_id = product) < wanted then\n raise exception 'Not enough stock';\n end if;", + "replace": " -- Defect: insufficient stock is accepted." + }, + { + "find": " where item_id = product and quantity > 0 order by warehouse_id loop", + "replace": " where item_id = product order by warehouse_id loop" + }, + { + "find": " take := least(holding.quantity, wanted);", + "replace": " take := wanted;" + }, + { + "file": "supabase/migrations/20260925000000_schema.sql", + "find": " quantity integer not null check (quantity >= 0),", + "replace": " quantity integer not null," + } + ] + }, + { + "id": "checkout-crash-integrity", + "scenario": "tracks/ecommerce/scenarios/progression-checkout-crash.json", + "targets": [ + "ecommerce.spec.state-durability.checkout-crash-integrity.910a" + ], + "desc": "A recovery worker started when the Realtime channel fails restores a wrong cart quantity through the data API, producing neither the prepared cart nor a completed checkout. The worker retries data API calls while the database is down; it does not wait for the Realtime reconnect backoff.", + "file": "client/src/request.ts", + "edits": [ + { + "find": " .subscribe();\n}", + "replace": " .subscribe(status => {\n if (status === 'SUBSCRIBED') subscribedOnce = true;\n else if (subscribedOnce && (status === 'CHANNEL_ERROR' || status === 'TIMED_OUT')) startRecovery();\n });\n}\nlet subscribedOnce = false, recoveryStarted = false;\nfunction startRecovery() {\n if (recoveryStarted) return;\n recoveryStarted = true;\n void (async () => {\n const deadline = Date.now() + 20000;\n while (true) {\n try {\n const current = await mutate('shop_state');\n const line = current.cart.items[0] || current.orders[0]?.items[0];\n if (!line) throw new Error('Crash control has no cart or prior order');\n await mutate('update_cart_quantity', { itemId: line.itemId, quantity: 2 });\n console.error('Crash control committed', Date.now());\n return;\n } catch (error) {\n if (Date.now() >= deadline) throw error;\n await new Promise(resolve => setTimeout(resolve, 25));\n }\n }\n })().catch(error => console.error('Crash control failed', String(error)));\n}" + } + ] + }, + { + "id": "checkout-crash-sibling-stock", + "scenario": "tracks/ecommerce/scenarios/progression-checkout-crash.json", + "targets": [ + "ecommerce.spec.state-durability.checkout-crash-integrity.910a" + ], + "desc": "A recovery worker started when the Realtime channel fails removes Coffee Grinder stock while leaving cart and order data intact. The second-product stock discrepancy must fail crash atomicity. The worker retries data API calls while the database is down; it does not wait for the Realtime reconnect backoff.", + "file": "client/src/request.ts", + "edits": [ + { + "find": " .subscribe();\n}", + "replace": " .subscribe(status => {\n if (status === 'SUBSCRIBED') subscribedOnce = true;\n else if (subscribedOnce && (status === 'CHANNEL_ERROR' || status === 'TIMED_OUT')) startRecovery();\n });\n}\nlet subscribedOnce = false, recoveryStarted = false;\nfunction startRecovery() {\n if (recoveryStarted) return;\n recoveryStarted = true;\n void (async () => {\n const deadline = Date.now() + 20000;\n while (true) {\n try {\n const current = await mutate('shop_state');\n const priorOrder = current.orders.find((order: any) => order.status === 'pending');\n if (!priorOrder) throw new Error('Crash control has no pending order');\n await mutate('cancel_order', { orderId: priorOrder.id });\n console.error('Crash control committed', Date.now());\n return;\n } catch (error) {\n if (Date.now() >= deadline) throw error;\n await new Promise(resolve => setTimeout(resolve, 25));\n }\n }\n })().catch(error => console.error('Crash control failed', String(error)));\n}" + }, + { + "file": "supabase/migrations/20260925000100_shop.sql", + "find": " for line in select * from order_line where order_id = target.id loop\n perform private.restore_line(line);\n end loop;\n update order_header set status = 'cancelled', refunded = total where id = target.id;", + "replace": " -- Defect: the recovery call removes sibling-product stock and changes nothing else.\n update stock set quantity = quantity - 1\n where item_id = (select id from item where name = 'Coffee Grinder')\n and warehouse_id = (select min(warehouse_id) from stock where item_id = (select id from item where name = 'Coffee Grinder'));" + } + ] + }, + { + "id": "checkout-crash-durability", + "scenario": "tracks/ecommerce/scenarios/progression-checkout-crash.json", + "targets": [ + "ecommerce.spec.state-durability.checkout-crash-durability.910b" + ], + "desc": "A recovery worker started when the Realtime channel fails corrupts the total of an earlier acknowledged order through the data API, without changing stock, allocations or cart state. The worker retries data API calls while the database is down; it does not wait for the Realtime reconnect backoff.", + "file": "client/src/request.ts", + "edits": [ + { + "find": " .subscribe();\n}", + "replace": " .subscribe(status => {\n if (status === 'SUBSCRIBED') subscribedOnce = true;\n else if (subscribedOnce && (status === 'CHANNEL_ERROR' || status === 'TIMED_OUT')) startRecovery();\n });\n}\nlet subscribedOnce = false, recoveryStarted = false;\nfunction startRecovery() {\n if (recoveryStarted) return;\n recoveryStarted = true;\n void (async () => {\n const deadline = Date.now() + 20000;\n while (true) {\n try {\n const current = await mutate('shop_state');\n const priorOrder = current.orders.find((order: any) => order.status === 'pending');\n if (!priorOrder) throw new Error('Crash control has no pending order');\n await mutate('cancel_order', { orderId: priorOrder.id });\n console.error('Crash control committed', Date.now());\n return;\n } catch (error) {\n if (Date.now() >= deadline) throw error;\n await new Promise(resolve => setTimeout(resolve, 25));\n }\n }\n })().catch(error => console.error('Crash control failed', String(error)));\n}" + }, + { + "file": "supabase/migrations/20260925000100_shop.sql", + "find": " for line in select * from order_line where order_id = target.id loop\n perform private.restore_line(line);\n end loop;\n update order_header set status = 'cancelled', refunded = total where id = target.id;", + "replace": " -- Defect: the recovery call rewrites an acknowledged order total.\n update order_header set total = 0 where id = target.id;" + } + ] + }, + { + "id": "restock-trusts-submitted-admin-role", + "scenario": "tracks/ecommerce/scenarios/01-admin-write-staff.json", + "targets": [ + "ecommerce.spec.access-control.warehouse-write-boundary.103b" + ], + "desc": "Trust an administrator role supplied by staff in the restock request. Normal staff requests are still refused.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": "create function public.admin_restock(\"itemId\" bigint, \"warehouseId\" bigint, quantity integer) returns void", + "replace": "create function public.admin_restock(\"itemId\" bigint, \"warehouseId\" bigint, quantity integer, role text default null) returns void" + }, + { + "find": " perform private.require_operator(true);\n perform private.check_quantity(admin_restock.quantity);", + "replace": " perform private.require_operator(admin_restock.role is distinct from 'admin');\n perform private.check_quantity(admin_restock.quantity);" + } + ] + }, + { + "id": "direct-purchase-trusts-submitted-price", + "scenario": "tracks/ecommerce/scenarios/01-server-price.json", + "targets": [ + "ecommerce.spec.transactional-integrity.server-price.104a" + ], + "desc": "Ordinary purchases use the catalog price; an extra price field overrides the booked order and line price.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": "create function public.buy_now(\"itemId\" bigint) returns json", + "replace": "create function public.buy_now(\"itemId\" bigint, price numeric default null) returns json" + }, + { + "find": "jsonb_build_object('item_id', \"itemId\", 'quantity', 1)", + "replace": "jsonb_build_object('item_id', \"itemId\", 'quantity', 1, 'price', buy_now.price)" + }, + { + "find": " if not found then raise exception 'Unknown item'; end if;\n insert into order_line", + "replace": " if not found then raise exception 'Unknown item'; end if;\n product.price := coalesce((entry ->> 'price')::numeric, product.price);\n insert into order_line" + } + ] + }, + { + "id": "order-cache-survives-account-switch", + "scenario": "tracks/ecommerce/scenarios/01-order-ownership.json", + "targets": [ + "ecommerce.spec.access-control.order-ownership.106a" + ], + "desc": "Merge order updates into a browser cache that is never cleared when the account changes. Separate-browser ownership still works.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " const [orders, setOrders] = useState([]);", + "replace": " const [orders, setStoredOrders] = useState([]);\n const setOrders: typeof setStoredOrders = update => setStoredOrders(previous => {\n const incoming = typeof update === 'function' ? update(previous) : update;\n return [...new Map([...previous, ...incoming].map(order => [order.id, order])).values()];\n });" + } + ] + }, + { + "id": "checkout-trusts-submitted-price", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "Accept an optional price field and overwrite the checkout total with it.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": "create function public.checkout() returns json", + "replace": "create function public.checkout(price numeric default null) returns json" + }, + { + "find": " lines jsonb;\n", + "replace": " lines jsonb;\n placed bigint;\n" + }, + { + "find": " return json_build_object('orderId', private.place_order(me.id, lines));", + "replace": " placed := private.place_order(me.id, lines);\n -- Defect: a submitted price overwrites the booked checkout total.\n if checkout.price is not null then update order_header set total = checkout.price where id = placed; end if;\n return json_build_object('orderId', placed);" + } + ] + }, + { + "id": "checkout-missing-sibling-line", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "Commit the cart checkout with its Coffee Grinder line missing while retaining the other product and the full order total.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " lines jsonb;\n", + "replace": " lines jsonb;\n placed bigint;\n" + }, + { + "find": " return json_build_object('orderId', private.place_order(me.id, lines));", + "replace": " placed := private.place_order(me.id, lines);\n -- Defect: the Coffee Grinder line is dropped after the order is booked.\n delete from order_allocation where order_line_id in (select line.id from order_line line\n join item on item.id = line.item_id where line.order_id = placed and item.name = 'Coffee Grinder');\n delete from order_line where order_id = placed and item_id = (select id from item where name = 'Coffee Grinder');\n return json_build_object('orderId', placed);" + } + ] + }, + { + "id": "shipping-cancellation-cancel-shipped", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Cancellation accepts a shipped order and restores its stock after shipping.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " if target.status <> 'pending' then raise exception 'Only pending orders can be cancelled'; end if;", + "replace": " if false then raise exception 'Only pending orders can be cancelled'; end if;" + } + ] + }, + { + "id": "shipping-cancellation-resurrect-cancelled", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Shipping accepts a cancelled order after its stock was restored.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " update order_header set status = 'shipped' where id = \"orderId\" and status = 'pending';", + "replace": " update order_header set status = 'shipped' where id = \"orderId\";" + } + ] + }, + { + "id": "shipping-cancellation-combined-race", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Cancellation restores stock but records a shipped status, combining mutually exclusive effects.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " if target.status <> 'pending' then raise exception 'Only pending orders can be cancelled'; end if;", + "replace": " if false then raise exception 'Only pending orders can be cancelled'; end if;" + }, + { + "find": " update order_header set status = 'cancelled', refunded = total where id = target.id;", + "replace": " update order_header set status = 'shipped', refunded = total where id = target.id;" + } + ] + }, + { + "id": "shipping-cancellation-reject-both", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "Both cancellation and shipping refuse pending orders; useful progress must remain required.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " if target.status <> 'pending' then raise exception 'Only pending orders can be cancelled'; end if;", + "replace": " if true then raise exception 'Only pending orders can be cancelled'; end if;" + }, + { + "find": " update order_header set status = 'shipped' where id = \"orderId\" and status = 'pending';", + "replace": " raise exception 'Order is not pending';\n update order_header set status = 'shipped' where id = \"orderId\" and status = 'pending';" + } + ] + }, + { + "id": "shipping-cancellation-queue-retains-removed-order", + "scenario": "tracks/ecommerce/scenarios/02-cancellation-queue.json", + "targets": [ + "ecommerce.returns-pricing.cancellation-and-return.3d" + ], + "desc": "The live staff queue accepts new orders but retains rows after cancellation; the database state remains correct.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": " setFulfilmentQueue(state.fulfilment || { orders: [], depth: 0 });", + "replace": " setFulfilmentQueue(previous => { const next = state.fulfilment || { orders: [], depth: 0 }; return next.orders.length < previous.orders.length ? previous : next; });" + } + ] + }, + { + "id": "opposing-transfer-ignored", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202d" + ], + "desc": "One valid opposing transfer is acknowledged without moving stock.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " if \"fromWarehouseId\" = \"toWarehouseId\" then raise exception 'Choose different warehouses'; end if;", + "replace": " if \"fromWarehouseId\" = \"toWarehouseId\" then raise exception 'Choose different warehouses'; end if;\n if admin_transfer_stock.quantity = 7 then return; end if;" + } + ] + }, + { + "id": "opposing-transfer-rejected", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202d" + ], + "desc": "One valid opposing transfer is refused despite ample source stock.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " if \"fromWarehouseId\" = \"toWarehouseId\" then raise exception 'Choose different warehouses'; end if;", + "replace": " if \"fromWarehouseId\" = \"toWarehouseId\" then raise exception 'Choose different warehouses'; end if;\n if admin_transfer_stock.quantity = 7 then raise exception 'Transfer disabled'; end if;" + } + ] + }, + { + "id": "overlapping-cart-add-ignored", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "The overlapping second-product cart add is ignored while ordinary cart setup still works.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " if not exists (select 1 from item where id = product) then raise exception 'Unknown item'; end if;", + "replace": " if not exists (select 1 from item where id = product) then raise exception 'Unknown item'; end if;\n if adding and me.username like '%checkout-overlap%' and (select name from item where id = product) = 'Coffee Grinder' then return; end if;" + } + ] + }, + { + "id": "overlapping-cart-add-rejected", + "scenario": "tracks/ecommerce/scenarios/progression-cart-checkout.json", + "targets": [ + "ecommerce.feature.cart-checkout.cart.4d" + ], + "desc": "The overlapping second-product cart add is rejected while ordinary cart setup still works.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " if not exists (select 1 from item where id = product) then raise exception 'Unknown item'; end if;", + "replace": " if not exists (select 1 from item where id = product) then raise exception 'Unknown item'; end if;\n if adding and me.username like '%checkout-overlap%' and (select name from item where id = product) = 'Coffee Grinder' then raise exception 'Cart add disabled'; end if;" + } + ] + }, + { + "id": "mixed-history-lost-transfer", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: lost-transfer. The stored-state comparison must catch the defect at its first effect.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " update stock set quantity = stock.quantity + admin_transfer_stock.quantity", + "replace": " update stock set quantity = stock.quantity" + } + ] + }, + { + "id": "mixed-history-wrong-price", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: wrong-price. The stored-state comparison must catch the defect at its first effect.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " booked := booked + product.price * amount;", + "replace": " booked := booked + 1 * amount;" + } + ] + }, + { + "id": "mixed-history-ignored-write", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: ignored-write. The stored-state comparison must catch the defect at its first effect.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " if not exists (select 1 from item where id = product) then raise exception 'Unknown item'; end if;", + "replace": " if not exists (select 1 from item where id = product) then raise exception 'Unknown item'; end if;\n if adding and me.username like '%history-%' and (select name from item where id = product) = 'Coffee Grinder' then return; end if;" + } + ] + }, + { + "id": "mixed-history-reject-valid-write", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: reject-valid-write. The stored-state comparison must catch the defect at its first effect.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " if not exists (select 1 from item where id = product) then raise exception 'Unknown item'; end if;", + "replace": " if not exists (select 1 from item where id = product) then raise exception 'Unknown item'; end if;\n if adding and me.username like '%history-%' and (select name from item where id = product) = 'Coffee Grinder' then raise exception 'Cart add disabled'; end if;" + } + ] + }, + { + "id": "mixed-history-repeat-cancel", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: repeat-cancel. The stored-state comparison must catch the defect at its first effect.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " if target.status <> 'pending' then raise exception 'Only pending orders can be cancelled'; end if;", + "replace": " if target.status not in ('pending', 'cancelled') then raise exception 'Only pending orders can be cancelled'; end if;" + } + ] + }, + { + "id": "mixed-history-wrong-owner", + "scenario": "tracks/ecommerce/scenarios/mixed-operation-history.json", + "targets": [ + "ecommerce.inventory-operations.stock-conservation.202f" + ], + "desc": "Mixed-history control: wrong-owner. The stored-state comparison must catch the defect at its first effect.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": "values (buyer, 0) returning id into new_order;", + "replace": "values ((select id from order_account where id <> buyer order by id limit 1), 0) returning id into new_order;" + } + ] + }, + { + "id": "checkout-empty-replay-creates-order", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Allow retrying the consumed empty cart to create another order after a lost reply.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " if jsonb_array_length(lines) = 0 then raise exception 'Cart is empty'; end if;", + "replace": " -- Defect: an empty consumed cart may create another order." + } + ] + }, + { + "id": "checkout-replay-loses-prior-order", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Retrying the consumed cart deletes an earlier acknowledged order.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " return json_build_object('orderId', private.place_order(me.id, lines));", + "replace": " if jsonb_array_length(lines) = 0 then\n -- Defect: retrying a consumed cart deletes the account's earliest order.\n delete from order_allocation where order_line_id in (select id from order_line\n where order_id = (select min(id) from order_header where account_id = me.id));\n delete from order_line where order_id = (select min(id) from order_header where account_id = me.id);\n delete from order_header where id = (select min(id) from order_header where account_id = me.id);\n return json_build_object('orderId', null);\n end if;\n return json_build_object('orderId', private.place_order(me.id, lines));" + } + ] + }, + { + "id": "checkout-clears-cart-without-order", + "scenario": "tracks/ecommerce/scenarios/01-duplicate-checkout.json", + "targets": [ + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b" + ], + "desc": "Checkout returns success and empties the cart without recording an order.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " return json_build_object('orderId', private.place_order(me.id, lines));", + "replace": " return json_build_object('orderId', null);" + } + ] + }, + { + "id": "revenue-aggregation-truncates-order-population", + "scenario": "tracks/ecommerce/scenarios/progression-books-balance.json", + "targets": [ + "ecommerce.spec.transactional-integrity.books-balance.107a" + ], + "desc": "Calculate revenue from only the first 256 stored orders while preserving all acknowledged purchases.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " 'revenue', (select coalesce(sum(total - refunded), 0) from order_header),", + "replace": " 'revenue', (select coalesce(sum(total - refunded), 0) from (select * from order_header order by id limit 256) first_orders)," + } + ] + }, + { + "id": "catalog-truncates-created-population", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-volume.json", + "targets": [ + "ecommerce.spec.transactional-integrity.catalog-volume.622c" + ], + "desc": "Omit committed products after the first 256 search results.", + "file": "client/src/App.tsx", + "edits": [ + { + "find": ".slice(searchPage * CATALOG_PAGE_SIZE,", + "replace": ".slice(0, 256).slice(searchPage * CATALOG_PAGE_SIZE," + } + ] + }, + { + "id": "catalog-discards-product-writes", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-volume.json", + "targets": [ + "ecommerce.spec.transactional-integrity.catalog-volume.622c" + ], + "desc": "Acknowledge catalog creation without committing a product.", + "file": "supabase/migrations/20260925000200_progression.sql", + "edits": [ + { + "find": " perform private.require_operator(true);\n if coalesce(trim(name), '') = '' or coalesce(trim(category), '') = ''", + "replace": " perform private.require_operator(true);\n return; -- Defect: the product write is acknowledged but never committed.\n if coalesce(trim(name), '') = '' or coalesce(trim(category), '') = ''" + } + ] + }, + { + "id": "catalog-rejects-all-product-writes", + "scenario": "tracks/ecommerce/scenarios/progression-catalog-volume.json", + "targets": [ + "ecommerce.spec.transactional-integrity.catalog-volume.622c" + ], + "desc": "Reject every catalog creation while keeping sign-in and catalog reads available.", + "file": "supabase/migrations/20260925000200_progression.sql", + "edits": [ + { + "find": " perform private.require_operator(true);\n if coalesce(trim(name), '') = '' or coalesce(trim(category), '') = ''", + "replace": " perform private.require_operator(true);\n raise exception 'Catalog writes refused';\n if coalesce(trim(name), '') = '' or coalesce(trim(category), '') = ''" + } + ] + }, + { + "id": "shipping-trusts-submitted-staff-role", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "Trust a staff role supplied by a customer; ordinary customers remain refused.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": "create function public.ship_order(\"orderId\" bigint) returns void", + "replace": "create function public.ship_order(\"orderId\" bigint, role text default null) returns void" + }, + { + "find": " me order_account := private.require_operator(false);", + "replace": " me order_account := case when ship_order.role = 'staff' then private.require_account() else private.require_operator(false) end;" + } + ] + }, + { + "id": "shipping-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.operator-authorization.201c" + ], + "desc": "Reject legitimate staff shipping as well as customer requests.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " if 'inventory' = any(me.roles) and not 'admin' = any(me.roles) then", + "replace": " raise exception 'Shipping refused';\n if 'inventory' = any(me.roles) and not 'admin' = any(me.roles) then" + } + ] + }, + { + "id": "cancel-trusts-submitted-owner", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.order-owner.204a" + ], + "desc": "Trust a username submitted by another signed-in customer as the order owner.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": "create function public.cancel_order(\"orderId\" bigint) returns void", + "replace": "create function public.cancel_order(\"orderId\" bigint, username text default null) returns void" + }, + { + "find": " target order_header := private.owned_order(\"orderId\");\n line order_line;\nbegin\n if target.status <> 'pending' then raise exception 'Only pending orders can be cancelled'; end if;", + "replace": " target order_header;\n line order_line;\nbegin\n -- Defect: a submitted username is trusted as the order owner.\n perform private.require_account();\n select o.* into target from order_header o join order_account a on a.id = o.account_id\n where o.id = \"orderId\" and a.username = cancel_order.username for update of o;\n if not found then target := private.owned_order(\"orderId\"); end if;\n if target.status <> 'pending' then raise exception 'Only pending orders can be cancelled'; end if;" + } + ] + }, + { + "id": "cancel-rejects-every-request", + "scenario": "tracks/ecommerce/scenarios/02-server-actions.json", + "targets": [ + "ecommerce.operations-access.order-owner.204a" + ], + "desc": "Reject legitimate cancellation as well as foreign requests.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": " if target.status <> 'pending' then raise exception 'Only pending orders can be cancelled'; end if;", + "replace": " raise exception 'Cancellation refused';\n if target.status <> 'pending' then raise exception 'Only pending orders can be cancelled'; end if;" + } + ] + }, + { + "id": "stock-alert-misses-direct-stock-writes", + "scenario": "tracks/ecommerce/scenarios/progression-stock-alert-direct-restore.json", + "targets": [ + "ecommerce.spec.external-data-sync.external-stock-alert.631d" + ], + "desc": "Stock alerts are delivered only by the restock functions, so stock written directly to the database never delivers its alert.", + "file": "supabase/migrations/20260925000100_shop.sql", + "edits": [ + { + "find": "create trigger stock_increase_alerts after update of quantity on public.stock\n for each row when (new.quantity > old.quantity) execute function private.notify_stock_increase();\n", + "replace": "-- Stock alerts are delivered by the restock functions.\n" + } + ] + } + ] +} diff --git a/tools/stack-bench/grader/response-loss.ts b/tools/stack-bench/grader/response-loss.ts new file mode 100644 index 00000000000..c254f7da87b --- /dev/null +++ b/tools/stack-bench/grader/response-loss.ts @@ -0,0 +1,108 @@ +import { createHash } from 'node:crypto'; +import { request, type BrowserContext, type WebSocketRoute } from 'playwright'; +import { evidenceNowMs } from '../src/evidence/evidence-timing.js'; +import { viteToken } from '../container/browser-network-proxy.js'; +import { connectBrowserSocket, installSocketDeliveryGate } from '../src/actions/browser-socket-delivery.js'; + +// Install before the actor opens its connection. This gate changes delivery, +// never request contents, database state, or the application's retry policy. +export async function installResponseLoss(context: BrowserContext) { + if (context.serviceWorkers().length) throw new Error('response loss cannot observe an existing service worker'); + const sockets = new Set<{ page: WebSocketRoute; server: WebSocketRoute }>(); + const pending = new Set>(); + const events: { kind: 'http-request' | 'http-response' | 'ws-send' | 'ws-drop'; + atMs: number; bytes?: number; sha256?: string; status?: number; path: string }[] = []; + const errors = new Set(); + context.on('serviceworker', () => { errors.add('service worker bypasses response interception'); }); + let state: 'ready' | 'armed' | 'finished' = 'ready', armedAtMs: number | null = null; + let truncated = false, release!: () => void; + const released = new Promise(resolve => { release = resolve; }); + const record = (kind: typeof events[number]['kind'], path: string, data?: string | Buffer, status?: number) => { + // Bounded evidence: an overflow is unmeasured, never a successful fault. + if (events.length === 256) { truncated = true; return; } + events.push({ kind, path, atMs: evidenceNowMs(), ...(data === undefined ? {} : { + bytes: Buffer.byteLength(data), sha256: createHash('sha256').update(data).digest('hex'), + }), ...(status === undefined ? {} : { status }) }); + }; + installSocketDeliveryGate(context, (page, server) => { + const pair = { page, server }, path = new URL(page.url()).pathname; + if (state !== 'finished') sockets.add(pair); + return { + sent(data) { if (state === 'armed') record('ws-send', path, data); }, + received(data) { + if (state !== 'armed') return true; + record('ws-drop', path, data); return false; + }, + closed() { sockets.delete(pair); }, + }; + }); + // The isolated actor loses write replies on all its HTTP paths and inbound + // WebSocket messages. This also covers SDK retries and proxy URLs unchanged. + await context.route('**/*', route => { + const task = (async () => { + if (state !== 'armed' || ['GET', 'HEAD', 'OPTIONS'].includes(route.request().method())) return route.continue(); + const path = new URL(route.request().url()).pathname; + record('http-request', path, route.request().postDataBuffer() ?? Buffer.alloc(0)); + // No redirects or transport retries: one intercepted write remains one write. + // A discarded reply must not update the browser's cookie jar. Forward the + // exact intercepted headers (including HttpOnly cookies) in a separate jar. + const api = await request.newContext(); + try { + const response = await api.fetch(route.request(), { headers: await route.request().allHeaders(), + maxRedirects: 0, maxRetries: 0, timeout: 10_000 }); + record('http-response', path, undefined, response.status()); + await released; + await route.abort('connectionreset'); + } finally { await api.dispose(); } + })().catch(() => { errors.add('HTTP response interception failed'); }); + pending.add(task); + void task.finally(() => pending.delete(task)); + return task; + }); + await context.routeWebSocket('**/*', async page => { + const url = new URL(page.url()), token = url.searchParams.get('token'); + // Closing Vite's verified tooling connection reloads the document. It is + // not an application reply; unknown tokens still take the fault path. + if (token && await viteToken(url.host, token)) { page.connectToServer(); return; } + const delivery = connectBrowserSocket(context, page), { server } = delivery; + page.onMessage(data => { + try { + delivery.send(data); + } catch { errors.add('WebSocket request forwarding failed'); } + }); + server.onMessage(data => { + try { + if (delivery.receive(data)) page.send(data); + } catch { errors.add('WebSocket response forwarding failed'); } + }); + page.onClose(async (code, reason) => { + delivery.closed(); + try { await server.close({ code, reason }); } + catch { errors.add('WebSocket server close failed'); } + }); + server.onClose(async (code, reason) => { + delivery.closed(); + try { await page.close({ code, reason }); } + catch { errors.add('WebSocket client close failed'); } + }); + }); + return { + arm() { + if (state !== 'ready') throw new Error('response-loss gate can be armed only once'); + state = 'armed'; armedAtMs = evidenceNowMs(); + }, + evidence() { return { state, armedAtMs, truncated, errors: [...errors], events: [...events] }; }, + async finish() { + if (state === 'finished') return; + state = 'finished'; + release(); + // Discard the lost reply permanently. New SDK connections can recover normally. + const closed = await Promise.allSettled([...sockets].flatMap(({ page, server }) => [ + page.close({ code: 1011, reason: 'connection interrupted' }), + server.close({ code: 1011, reason: 'connection interrupted' }), + ])); + if (closed.some(result => result.status === 'rejected')) errors.add('WebSocket fault cleanup failed'); + await Promise.all([...pending]); + }, + }; +} diff --git a/tools/stack-bench/grader/transport-frames.ts b/tools/stack-bench/grader/transport-frames.ts new file mode 100644 index 00000000000..064793fe2cf --- /dev/null +++ b/tools/stack-bench/grader/transport-frames.ts @@ -0,0 +1,207 @@ +import { brotliDecompressSync, gunzipSync } from 'node:zlib'; +import { createHash } from 'node:crypto'; +import type { Page, Request, Response } from 'playwright'; +import { inconclusive } from '../src/actions/actor-action-runtime.js'; + +const MAX_RECEIVED_BYTES = 8 * 1024 * 1024; + +export function urlDiagnostic(value: string) { + const url = new URL(value); + return { origin: url.origin, pathSha256: createHash('sha256').update(url.pathname).digest('hex') }; +} + +export function requestDiagnostic(request: Request) { + return { ...urlDiagnostic(request.url()), method: request.method(), resourceType: request.resourceType() }; +} + +function responseDiagnostic(response: Response, page: Page) { + return { ...requestDiagnostic(response.request()), + status: response.status(), contentType: response.headers()['content-type'] ?? '', + pageClosed: page.isClosed(), + failure: response.request().failure()?.errorText ?? null }; +} + +function bodyReadErrorCategory(error: unknown): string { + // Browser errors can include private URLs or payloads. Emit only fixed labels. + const message = error instanceof Error ? error.message : ''; + if (/evicted from inspector cache/i.test(message)) return 'body-evicted'; + if (/No resource with given identifier|No data found for (?:resource with )?given identifier/i.test(message)) return 'resource-unavailable'; + if (/Target page, context or browser has been closed|Session closed/i.test(message)) return 'target-closed'; + if (/net::ERR_BLOCKED_BY_ORB\b/.test(message)) return 'blocked-by-orb'; + if (/net::ERR_[A-Z_]+\b/.test(message)) return 'request-failed'; + if (/Protocol error \(/.test(message)) return 'protocol-error'; + return 'unknown'; +} + +// A SpacetimeDB server frame carries a one-byte compression tag ahead of the +// message: 0 none, 1 brotli, 2 gzip, and the SDK compresses by default. The +// message text is inline UTF-8 once decoded, so a substring search finds it +// without the harness knowing the wire format. Any other frame is kept as it +// arrived. +export function transportFrameText(payload: string | Buffer): string { + if (typeof payload === 'string') return payload; + const bytes = Buffer.from(payload); + if (bytes.length > 1) { + try { + if (bytes[0] === 1) return brotliDecompressSync(bytes.subarray(1)).toString('utf8'); + if (bytes[0] === 2) return gunzipSync(bytes.subarray(1)).toString('utf8'); + } catch { /* not a compressed SpacetimeDB frame */ } + } + return bytes.toString('utf8'); +} + +// Bounded evidence must never turn dropped data into a privacy pass. +export class ReceivedTransport { + readonly chunks: string[] = []; + // Each distinct delivery as it arrived, for checks that compare whole records. + readonly deliveries: { atMs: number; payload: string | Buffer }[] = []; + private bytes = 0; + // Kept apart from text evidence: dropped deliveries leave text observations unaffected. + deliveriesDropped = false; + private deliveryBytes = 0; + private readonly deliveryDigests = new Set(); + private readonly incompleteCounts = { byteLimit: 0, bodyReadFailures: 0, unsupportedStreams: 0, navigationInterrupted: 0 }; + incomplete = false; + pending = 0; + readonly pendingResponses = new Map(); + + constructor(private readonly limit = MAX_RECEIVED_BYTES) {} + + markIncomplete(reason: keyof ReceivedTransport['incompleteCounts']): void { + this.incomplete = true; + this.incompleteCounts[reason]++; + } + + record(payload: string | Buffer, cumulative = false): void { + this.recordDelivery(payload, cumulative); + const text = transportFrameText(payload); + // Receipt checks need presence, not frequency. Reloading an identical bundle + // adds no evidence and must not evict distinct data. + if (!text || (cumulative ? this.chunks.some(chunk => chunk.startsWith(text)) : this.chunks.includes(text))) return; + if (Buffer.byteLength(text) > this.limit) { + this.markIncomplete('byteLimit'); + return; + } + // A later snapshot can extend an earlier prefix. It already contains every + // marker in that prefix; retaining both would count the same bytes twice. + for (let i = cumulative ? this.chunks.length - 1 : -1; i >= 0; i--) { + if (!text.startsWith(this.chunks[i]!)) continue; + this.bytes -= Buffer.byteLength(this.chunks[i]!); + this.chunks.splice(i, 1); + } + this.chunks.push(text); + this.bytes += Buffer.byteLength(text); + while (this.bytes > this.limit) { + this.markIncomplete('byteLimit'); + this.bytes -= Buffer.byteLength(this.chunks.shift()!); + } + } + + // Raw bytes are kept so binary records can be decoded later. A repeat adds nothing; a growing + // snapshot replaces its own earlier prefix and keeps that prefix's arrival time. + private recordDelivery(payload: string | Buffer, cumulative: boolean): void { + const kept = typeof payload === 'string' ? payload : Buffer.from(payload); + const digest = createHash('sha256').update(typeof kept === 'string' ? `t${kept}` : Buffer.concat([Buffer.from('b'), kept])) + .digest('hex'); + if (this.deliveryDigests.has(digest)) return; + this.deliveryDigests.add(digest); + const size = Buffer.byteLength(kept); + if (size > this.limit) { this.deliveriesDropped = true; return; } + const prefix = cumulative && typeof kept === 'string' + ? this.deliveries.findIndex(item => typeof item.payload === 'string' && kept.startsWith(item.payload)) : -1; + if (prefix >= 0) { + this.deliveryBytes += size - Buffer.byteLength(this.deliveries[prefix]!.payload); + this.deliveries[prefix] = { atMs: this.deliveries[prefix]!.atMs, payload: kept }; + } else { + this.deliveries.push({ atMs: Date.now(), payload: kept }); + this.deliveryBytes += size; + } + while (this.deliveryBytes > this.limit) { + this.deliveriesDropped = true; + this.deliveryBytes -= Buffer.byteLength(this.deliveries.shift()!.payload); + } + } + + contains(needle: string, requireComplete = true): boolean { + if (this.chunks.some(chunk => chunk.includes(needle))) return true; + if (requireComplete && (this.incomplete || this.pending)) { + for (const [response, { page, startedAt }] of [...this.pendingResponses].slice(0, 8)) { + try { process.stderr.write(`transport body pending ${JSON.stringify({ + ...responseDiagnostic(response, page), elapsedMs: Date.now() - startedAt, + })}\n`); } catch { /* Diagnostics must not change the verdict. */ } + } + inconclusive('transport-incomplete', { + capture: { ...this.incompleteCounts, pendingBodies: this.pending, retainedBytes: this.bytes }, + }); + } + return false; + } +} + +export async function captureResponses(page: Page, received: ReceivedTransport, freshResponses = false, + http?: { covered(response: Response): Promise; completed: ReceivedTransport }): Promise { + let reportedBodyFailures = 0; + page.on('response', async response => { + const type = response.headers()['content-type'] ?? ''; + // Native EventSource messages are captured below without waiting for stream closure. + if (/text\/event-stream/.test(type)) return; + // Public scripts and styles can contain secrets too. Keep the same bounded, + // fail-closed capture for data, rendered pages, assets and error responses. + if (!/(application\/(json|[^;]+\+json|x-ndjson|(?:x-)?(?:java|ecma)script)|text\/(plain|html|css|(?:java|ecma)script))/i.test(type)) return; + if (Number(response.headers()['content-length']) > MAX_RECEIVED_BYTES) { + received.markIncomplete('byteLimit'); + return; + } + received.pending++; + received.pendingResponses.set(response, { page, startedAt: Date.now() }); + const body = response.text(); + // The proxy observes HTTP prefixes. Replay discovery still receives only + // complete bodies, independently of the privacy observation window. + void body.catch(() => {}); + try { + if (http && await http.covered(response)) { + void body.then(text => http.completed.record(text)).catch(() => {}); + return; + } + const text = await body; + received.record(text); + http?.completed.record(text); + } + catch (error) { + received.markIncomplete('bodyReadFailures'); + if (reportedBodyFailures++ < 8) { + try { + process.stderr.write(`transport body unavailable ${JSON.stringify({ + ...responseDiagnostic(response, page), + error: bodyReadErrorCategory(error), + })}\n`); + } catch { /* Diagnostics must not turn incomplete capture into a process failure. */ } + } + } + finally { received.pending--; received.pendingResponses.delete(response); } + }); + const session = await page.context().newCDPSession(page); + if (http) session.on('close', () => { + if (page.isClosed()) return; + received.markIncomplete('bodyReadFailures'); + try { process.stderr.write('transport capture session closed {"pageClosed":false}\n'); } catch { /* Diagnostics only. */ } + }); + session.on('Network.eventSourceMessageReceived', event => { + received.record(event.data); + http?.completed.record(event.data); + }); + session.on('Network.responseReceived', event => { + // Fetch streams have no EventSource events. Absence of an unseen body is not a pass. + if (event.response.mimeType === 'text/event-stream' && event.type !== 'EventSource') { + received.markIncomplete('unsupportedStreams'); + } + }); + await session.send('Network.enable'); + // Chromium discards decoded font bytes; a later memory-cache hit has no body + // to inspect. Privacy absence observations require fresh HTTP responses. + if (freshResponses) await session.send('Network.setCacheDisabled', { cacheDisabled: true }); + // Keep response bodies available when the app reloads immediately after reading them. + await session.send('Network.configureDurableMessages', { + maxTotalBufferSize: MAX_RECEIVED_BYTES, maxResourceBufferSize: MAX_RECEIVED_BYTES, + }); +} diff --git a/tools/stack-bench/linter/fixtures/agreed.html b/tools/stack-bench/linter/fixtures/agreed.html new file mode 100644 index 00000000000..c462174caf2 --- /dev/null +++ b/tools/stack-bench/linter/fixtures/agreed.html @@ -0,0 +1,4 @@ +agreement fixture + +4 + diff --git a/tools/stack-bench/linter/fixtures/divergent.html b/tools/stack-bench/linter/fixtures/divergent.html new file mode 100644 index 00000000000..5d95d68cd79 --- /dev/null +++ b/tools/stack-bench/linter/fixtures/divergent.html @@ -0,0 +1,12 @@ +divergence fixture + +
+ + + diff --git a/tools/stack-bench/linter/fixtures/mock-chat.html b/tools/stack-bench/linter/fixtures/mock-chat.html new file mode 100644 index 00000000000..2677b6feb48 --- /dev/null +++ b/tools/stack-bench/linter/fixtures/mock-chat.html @@ -0,0 +1,71 @@ + + +Linter fixture — mock chat + + +
+ + +
+ + + + + + diff --git a/tools/stack-bench/linter/fixtures/mock-shop.html b/tools/stack-bench/linter/fixtures/mock-shop.html new file mode 100644 index 00000000000..5034271fdc9 --- /dev/null +++ b/tools/stack-bench/linter/fixtures/mock-shop.html @@ -0,0 +1,297 @@ + +

Mock Shop

+ +
+ + + + + +
+ + + + + +
+ + + + + + + + + + + + + + diff --git a/tools/stack-bench/linter/fixtures/spec-accounts.html b/tools/stack-bench/linter/fixtures/spec-accounts.html new file mode 100644 index 00000000000..ddbb1ad5630 --- /dev/null +++ b/tools/stack-bench/linter/fixtures/spec-accounts.html @@ -0,0 +1,29 @@ + +
+ + + + + +
+ + diff --git a/tools/stack-bench/linter/lint.ts b/tools/stack-bench/linter/lint.ts new file mode 100644 index 00000000000..e3484971588 --- /dev/null +++ b/tools/stack-bench/linter/lint.ts @@ -0,0 +1,256 @@ +#!/usr/bin/env node +// Scenario-stage hooks require scenario setup and are not linted here. + +import { chromium } from 'playwright'; +import { attemptBrowserLaunchOptions } from '../container/browser-pipe.js'; +import type { Page } from 'playwright'; +import { readFileSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { parseArgs as parseNodeArgs } from 'node:util'; +import { loadTrack, DEFAULT_TRACK } from '../src/composition/tracks.js'; +import { emptyArtifactIdentities, writeArtifact } from '../src/evidence/artifacts.js'; +import { stableElementSelector } from '../src/actions/element-selector.js'; +import { harnessBrowserFailure } from '../src/evidence/harness-errors.js'; + +const CHECK_TIMEOUT = 5000; + +export interface LintHook { + id: string; + element: string; + stage: string; + check: 'visible' | 'attached'; + note: string; + revealedBy?: string; +} + +export interface LintResult { + id: string; + status: 'PASS' | 'FAIL' | 'BLOCKED' | 'SCENARIO' | 'HARNESS' | 'UNMEASURED'; + detail?: string; +} + +export interface LintArgs { + url?: string; + track: string; + level: number; + headed: boolean; + out?: string; + label?: string; + parentAttemptId?: string; + credentialAliases?: unknown; + hooks?: string[]; +} + +export interface LintWalkContext { + page: Page; + args: LintArgs; + hooks: LintHook[]; + byStage(stage: string): LintHook[]; + blocked(stage: string): void; + checkHook(page: Page, hook: LintHook, results: LintResult[]): Promise; + results: LintResult[]; + uniq: string; + tid(id: string): string; + CHECK_TIMEOUT: number; +} + +function parseArgs(argv: string[]): LintArgs { + const { values } = parseNodeArgs({ args: argv.slice(2), options: { + url: { type: 'string' }, track: { type: 'string' }, level: { type: 'string' }, + out: { type: 'string' }, label: { type: 'string' }, + 'parent-attempt-id': { type: 'string' }, 'credential-aliases-json': { type: 'string' }, + hook: { type: 'string', multiple: true }, 'selected-hooks': { type: 'boolean' }, + headed: { type: 'boolean' }, + } }); + const args: LintArgs = { url: values.url, track: values.track ?? DEFAULT_TRACK, + level: values.level === undefined ? 1 : Number(values.level), + headed: values.headed ?? false, out: values.out, label: values.label, + parentAttemptId: values['parent-attempt-id'], + credentialAliases: values['credential-aliases-json'] === undefined + ? undefined : JSON.parse(values['credential-aliases-json']), + hooks: values.hook ?? (values['selected-hooks'] ? [] : undefined) }; + if (!args.url || !Number.isInteger(args.level) || args.level < 1) { + console.error('Usage: node dist/linter/lint.js --url --level [--headed]'); + process.exit(2); + } + return args; +} + +export function selectHooks(hooks: LintHook[], selectedIds?: string[]): LintHook[] { + if (selectedIds === undefined) return hooks; + const remaining = new Set(selectedIds); + const selected = hooks.filter(hook => remaining.delete(hook.id)); + const unknown: LintHook[] = [...remaining].sort().map(id => ({ + id, + element: `the selected application control ${id}`, + stage: 'scenario', + check: 'visible', + note: 'checked by the selected feature suite', + })); + return [...selected, ...unknown]; +} + +export function loadHooks(level: number, track: { contracts: string }, selectedIds?: string[]): LintHook[] { + const CONTRACTS_DIR = track.contracts; + const files = readdirSync(CONTRACTS_DIR).filter(f => /^\d+-[a-z-]+\.json$/.test(f)).sort(); + const hooks = []; + for (const f of files) { + const contract = JSON.parse(readFileSync(join(CONTRACTS_DIR, f), 'utf8')) as { + level: number; hooks: LintHook[]; + }; + if (contract.level <= level) hooks.push(...contract.hooks); + } + if (hooks.length === 0 && selectedIds === undefined) { + console.error(`No contracts found for level ${level} in ${CONTRACTS_DIR}`); + process.exit(2); + } + return selectHooks(hooks, selectedIds); +} + +const tid = stableElementSelector; +const uniq = Date.now().toString(36).slice(-5); + +// Browser, protocol, selector, and walk-script faults measure nothing about the app; +// a navigation timeout is unmeasured (grader/README.md outcome rules). +const errorMessage = (error: unknown): string => error instanceof Error ? error.message : String(error ?? 'unknown error'); + +function faultStatus(error: unknown): 'FAIL' | 'HARNESS' | 'UNMEASURED' { + const message = errorMessage(error); + if (/^page\.goto: Timeout/.test(message)) return 'UNMEASURED'; + return harnessBrowserFailure(error) || error instanceof TypeError || error instanceof ReferenceError + || /Protocol error|while parsing .*selector|is not a valid selector/.test(message) ? 'HARNESS' : 'FAIL'; +} + +export async function checkHook(page: Page, hook: LintHook, results: LintResult[]): Promise { + const loc = page.locator(tid(hook.id)).first(); + try { + if (hook.revealedBy && !(await loc.count())) { + await page.locator(tid(hook.revealedBy)).first().click({ timeout: CHECK_TIMEOUT }); + } + await loc.waitFor({ + state: hook.check === 'visible' ? 'visible' : 'attached', + timeout: CHECK_TIMEOUT, + }); + results.push({ id: hook.id, status: 'PASS' }); + return true; + } catch (error) { + const status = faultStatus(error); + results.push({ + id: hook.id, + status, + detail: status !== 'FAIL' ? errorMessage(error).split(/\r?\n/, 1)[0] + : `no element matching ${tid(hook.id)} became ${hook.check} during contract stage ${JSON.stringify(hook.stage)}` + + (hook.revealedBy ? ` (after clicking ${tid(hook.revealedBy)})` : '') + + ` — expected: ${hook.element}`, + }); + return false; + } +} + +export function completeUnvisitedHooks(hooks: LintHook[], results: LintResult[]): LintResult[] { + const visited = new Set(results.map(result => result.id)); + for (const hook of hooks) { + if (visited.has(hook.id)) continue; + results.push(hook.stage === 'scenario' + ? { id: hook.id, status: 'SCENARIO', detail: hook.note } + : { id: hook.id, status: 'BLOCKED', + detail: `the core flow did not visit contract stage ${JSON.stringify(hook.stage)}` }); + } + return results; +} + +export function completeAbortedHooks(hooks: LintHook[], results: LintResult[], error: unknown): LintResult[] { + const visited = new Set(results.map(result => result.id)); + const detail = errorMessage(error) + .split(/\r?\n/).map(line => line.trim()).filter(Boolean).slice(0, 6).join(' ').slice(0, 800); + results.push({ id: 'core-flow', status: faultStatus(error), detail: `core flow aborted: ${detail}` }); + for (const hook of hooks) { + if (visited.has(hook.id)) continue; + if (hook.stage === 'scenario') { + results.push({ id: hook.id, status: 'SCENARIO', detail: hook.note }); + } else { + results.push({ id: hook.id, status: 'BLOCKED', detail: 'core flow aborted' }); + } + } + return results; +} + +async function run() { + const args = parseArgs(process.argv); + const track = loadTrack(args.track); + const hooks = loadHooks(args.level, track, args.hooks); + const byStage = (stage: string): LintHook[] => hooks.filter(h => h.stage === stage); + const results: LintResult[] = []; + const blocked = (stage: string): void => { + for (const h of hooks.filter(x => x.stage === stage)) { + results.push({ id: h.id, status: 'BLOCKED', detail: 'earlier core flow step failed' }); + } + }; + + if (hooks.some(hook => hook.stage !== 'scenario')) { + const browser = await chromium.launch({ headless: !args.headed, ...attemptBrowserLaunchOptions() }); + const page = await browser.newContext().then(c => c.newPage()); + page.setDefaultTimeout(CHECK_TIMEOUT); + + try { + // The core flow is the one part of linting that is entirely + // application-specific, so each track brings its own. + const { walk } = await import(pathToFileURL(track.walk).href) as { + walk(context: LintWalkContext): Promise; + }; + await walk({ page, args, hooks, byStage, blocked, checkHook, results, uniq, tid, CHECK_TIMEOUT }); + // Every lintable hook must record explicit evidence. + completeUnvisitedHooks(hooks, results); + } catch (err: unknown) { + console.error(`Core flow aborted: ${err instanceof Error ? err.message : String(err)}`); + completeAbortedHooks(hooks, results, err); + } finally { + await browser.close(); + } + } else { + completeUnvisitedHooks(hooks, results); + } + + const failures = results.filter(r => r.status !== 'PASS' && r.status !== 'SCENARIO'); + const report = { + label: args.label ?? null, + url: args.url, + level: args.level, + selectedHooks: args.hooks === undefined ? null : [...new Set(args.hooks)].sort(), + pass: failures.length === 0, + harness: results.some(r => r.status === 'HARNESS'), + unmeasured: results.some(r => r.status === 'UNMEASURED'), + counts: { + lintable: results.filter(r => r.status !== 'SCENARIO').length, + pass: results.filter(r => r.status === 'PASS').length, + fail: results.filter(r => r.status === 'FAIL').length, + blocked: results.filter(r => r.status === 'BLOCKED').length, + scenario: results.filter(r => r.status === 'SCENARIO').length, + }, + results, + }; + if (args.out) { + const id = `${args.parentAttemptId ?? args.label ?? 'lint'}-contract-lint`; + writeArtifact(args.out, { + kind: 'contract_lint', id, + attempt: { id, parentId: args.parentAttemptId ?? null }, + identities: emptyArtifactIdentities(), + payload: report, + }); + console.log(`\nLint report written to ${args.out}`); + } + for (const r of results) { + console.log(`${r.status.padEnd(9)} ${r.id}${r.detail ? ` — ${r.detail}` : ''}`); + } + console.log(failures.length === 0 + ? report.counts.pass > 0 + ? `\nAPPLICATION CONTRACT PASS (${report.counts.pass} interfaces)` + : report.counts.scenario > 0 + ? `\nAPPLICATION CONTRACT DEFERRED (${report.counts.scenario} interfaces checked during feature grading)` + : '\nNO STANDALONE INTERFACES SELECTED' + : `\nAPPLICATION CONTRACT FAIL (${failures.length} interfaces missing or blocked)`); + process.exit(failures.length === 0 ? 0 : 1); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) run(); diff --git a/tools/stack-bench/package-lock.json b/tools/stack-bench/package-lock.json new file mode 100644 index 00000000000..430532137f5 --- /dev/null +++ b/tools/stack-bench/package-lock.json @@ -0,0 +1,136 @@ +{ + "name": "@spacetimedb/stack-bench", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@spacetimedb/stack-bench", + "dependencies": { + "eventsource-parser": "3.1.1", + "playwright": "1.62.1", + "semver": "7.7.4", + "zod": "4.5.4" + }, + "devDependencies": { + "@types/node": "22.15.30", + "@types/semver": "7.8.0", + "typescript": "5.6.3" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@types/node": { + "version": "22.15.30", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.15.30.tgz", + "integrity": "sha512-6Q7lr06bEHdlfplU6YRbgG1SFBdlsfNC4/lX+SkhiTs0cpJkOElmWls8PxDFv4yY/xKb8Y6SO0OmSX4wgqTZbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/semver": { + "version": "7.8.0", + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.8.0.tgz", + "integrity": "sha512-1mAINjtQCXXeLkJ9ehXkwOcBpqtLxiVtKhpUf83DdRNdQKV0iXZpaHYqRr7nj+wvxuJzoAmAwXI+sCNMv1CzLQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/eventsource-parser": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", + "integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.62.1.tgz", + "integrity": "sha512-0M+L3LAD8/nm554LOla9Ayx0j0tmFZ0FBcoQ7F1VuVHpM/XpiC8RcDzBQB8W5+hA8L22THxELzeF+2WcUzvcLg==", + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.62.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.62.1.tgz", + "integrity": "sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/typescript": { + "version": "5.6.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.3.tgz", + "integrity": "sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/zod": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.5.4.tgz", + "integrity": "sha512-sC95tT5iHHH9gtpj6A81kh+NEaRAUFN+qlUPDUbRfOMvNf5QCBqsb3WgvnpVtK5Y+4UfA6KqufotuTvMGiTlsA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/tools/stack-bench/package.json b/tools/stack-bench/package.json new file mode 100644 index 00000000000..173e7504123 --- /dev/null +++ b/tools/stack-bench/package.json @@ -0,0 +1,68 @@ +{ + "name": "@spacetimedb/stack-bench", + "private": true, + "type": "module", + "engines": { + "node": ">=22" + }, + "scripts": { + "build": "npm run clean && tsc -p tsconfig.build.json && node scripts/build-spacetime-codec.mjs && node dist/scripts/copy-dashboard-assets.js", + "clean": "node --eval \"require('node:fs').rmSync('dist', { recursive: true, force: true })\"", + "lint": "eslint appliance commands container dashboard grader linter scripts src tests", + "typecheck": "tsc -p tsconfig.json --noEmit", + "bootstrap:browsers": "playwright install chromium", + "precheck:scenarios": "npm run build --silent", + "check:scenarios": "node dist/commands/check-scenarios.js --track chat && node dist/commands/check-scenarios.js --track ecommerce --recipe sequential-l1.json && node dist/commands/check-scenarios.js --track ecommerce --recipe sequential-l2.json && node dist/commands/check-scenarios.js --track ecommerce --recipe sequential-l3.json && node dist/commands/check-scenarios.js --track ecommerce --recipe progression-catalog.json", + "precheck:mutations": "npm run build", + "check:mutations": "node dist/commands/check-mutations.js", + "precheck:definition-snapshots": "npm run build --silent", + "check:definition-snapshots": "node dist/commands/definition-snapshots.js", + "precheck:composition": "npm run build --silent", + "check:composition": "node dist/commands/check-composition.js", + "precheck:prompts": "npm run build --silent", + "check:prompts": "node --test dist/tests/dependency-neutral-prompt.contract.js", + "precheck:calibration": "npm run build --silent", + "check:calibration": "node dist/commands/check-calibration.js", + "prerepin": "npm run build --silent", + "repin": "node dist/commands/repin.js", + "precheck:references": "npm run build --silent", + "check:references": "node dist/src/references/reference-fixtures.js", + "pregraph": "npm run build", + "graph": "node dist/commands/progression-graph.js tracks/ecommerce/progression/ecommerce.json", + "pack": "npm run build --silent && node dist/commands/composition-cli.js pack", + "recipe": "npm run build --silent && node dist/commands/composition-cli.js recipe", + "predashboard": "npm run build --silent", + "dashboard": "node dist/dashboard/dashboard-server.js", + "pretest": "npm run build --silent", + "test": "node --test --test-concurrency=4 dist/tests/*.test.js", + "pretest:dashboard": "npm run build --silent", + "test:dashboard": "node --test dist/tests/dashboard/*.test.js", + "pretest:all": "npm run build --silent", + "test:all": "node --test --test-concurrency=4 dist/tests/*.test.js dist/tests/dashboard/*.test.js dist/tests/*.contract.js", + "pretest:contracts": "npm run build --silent", + "test:contracts": "node --test --test-concurrency=4 dist/tests/*.contract.js", + "pretest:mutation-definitions": "npm run build --silent", + "test:mutation-definitions": "node --test --test-concurrency=4 dist/tests/*.mutation.js", + "pretest:integration": "npm run build --silent", + "test:integration": "node --test --test-concurrency=1 dist/tests/*.integration.js", + "pretest:container": "npm run build --silent", + "test:container": "node dist/commands/container-smoke.js", + "pretest:references": "npm run build --silent", + "test:references": "node dist/src/references/reference-build.js", + "pretest:faults": "npm run build --silent", + "test:faults": "node dist/commands/fault-injection.js", + "pretest:loop": "npm run build", + "test:loop": "node dist/commands/test-loop.js" + }, + "dependencies": { + "eventsource-parser": "3.1.1", + "playwright": "1.62.1", + "semver": "7.7.4", + "zod": "4.5.4" + }, + "devDependencies": { + "@types/node": "22.15.30", + "@types/semver": "7.8.0", + "typescript": "5.6.3" + } +} diff --git a/tools/stack-bench/reference-apps/README.md b/tools/stack-bench/reference-apps/README.md new file mode 100644 index 00000000000..9061da3cbe0 --- /dev/null +++ b/tools/stack-bench/reference-apps/README.md @@ -0,0 +1,51 @@ +# Reference apps + +A reference app is a correct store for one stack, written by hand to prove the +checks. It is a grading fixture, not a product example or a recommended design. +`registry.json` records each app's source hash and supported recipes. + +A reference app qualifies a check selection only when all of this holds: + +1. Dependencies install from committed lockfiles in the benchmark build image. +2. The app starts in Docker with run-specific ports and database names. +3. Every required check passes for the exact recipe. +4. The source contains no secrets, generated bindings, build output, + transcripts, grader output, or seeded-bug backups. +5. Each seeded bug has an exact source anchor and fails at an observation step + (`expect`, `dbExpect`, or `waitUntilAbsent`) with no unrelated failures. A + bug that breaks a click, sign-in, or restart instead is + `CAUGHT_OFF_ASSERTION` and does not count. +6. The registry records the qualified source hash. + +Compile success, a build pass, or an old full score does not qualify an app. +Only matching live evidence does. + +## Compile + +From `tools/stack-bench`, a model-free Docker compile of every app, or one: + +```bash +npm run test:references +npm run test:references -- --fixture +``` + +The compile copies the source into a temporary workspace and never edits the +registered app. + +## Qualify + +The live procedure, which runs each app, its seeded bugs, and the empty app on +the appliance and records the evidence in the calibration, is the +[qualify-checks skill](../skills/qualify-checks/SKILL.md). Its rules: + +- Set the recipe, depth, and repetition count explicitly from the calibration; + command defaults are not the policy. +- Write every artifact to a new path. Never edit an old artifact to match a + changed source or calibration. +- A changed source hash needs new evidence. Do not edit a registered app during + qualification. +- A code change outside the grading path can keep old evidence only through a + reviewed `qualificationReuse` decision in the calibration that names the old + and new executable hashes. +- Adding a stack does not invalidate another stack's evidence, but the new + stack needs its own complete reference and seeded-bug evidence. diff --git a/tools/stack-bench/reference-apps/ecommerce/convex/client/index.html b/tools/stack-bench/reference-apps/ecommerce/convex/client/index.html new file mode 100644 index 00000000000..8b77d8835cc --- /dev/null +++ b/tools/stack-bench/reference-apps/ecommerce/convex/client/index.html @@ -0,0 +1,12 @@ + + + + + + Storefront + + +
+ + + diff --git a/tools/stack-bench/reference-apps/ecommerce/convex/client/package-lock.json b/tools/stack-bench/reference-apps/ecommerce/convex/client/package-lock.json new file mode 100644 index 00000000000..931bd8e6d14 --- /dev/null +++ b/tools/stack-bench/reference-apps/ecommerce/convex/client/package-lock.json @@ -0,0 +1,1469 @@ +{ + "name": "client", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "client", + "version": "1.0.0", + "dependencies": { + "convex": "1.46.0", + "react": "^18.3.1", + "react-dom": "^18.3.1" + }, + "devDependencies": { + "@types/react": "^18.3.3", + "@types/react-dom": "^18.3.0", + "@vitejs/plugin-react": "^6.1.0", + "typescript": "^5.5.3", + "vite": "^8.2.2" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.0.tgz", + "integrity": "sha512-KuZrd2hRjz01y5JK9mEBSD3Vj3mbCvemhT466rSuJYeE/hjuBrHfjjcjMdTm/sz7au+++sdbJZJmuBwQLuw68A==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.0.tgz", + "integrity": "sha512-j67aezrPNYWJEOHUNLPj9maeJte7uSMM6gMoxfPC9hOg8N02JuQi/T7ewumf4tNvJadFkvLZMlAq73b9uwdMyQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.0.tgz", + "integrity": "sha512-CC3vt4+1xZrs97/PKDkl0yN7w8edvU2vZvAFGD16n9F0Cvniy5qvzRXjfO1l94efczkkQE6g1x0i73Qf5uthOQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.0.tgz", + "integrity": "sha512-wurMkF1nmQajBO1+0CJmcN17U4BP6GqNSROP8t0X/Jiw2ltYGLHpEksp9MpoBqkrFR3kv2/te6Sha26k3+yZ9Q==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.0.tgz", + "integrity": "sha512-uJOQKYCcHhg07DL7i8MzjvS2LaP7W7Pn/7uA0B5S1EnqAirJtbyw4yC5jQ5qcFjHK9l6o/MX9QisBg12kNkdHg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.0.tgz", + "integrity": "sha512-8mG6arH3yB/4ZXiEnXof5MK72dE6zM9cDvUcPtxhUZsDjESl9JipZYW60C3JGreKCEP+p8P/72r69m4AZGJd5g==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.0.tgz", + "integrity": "sha512-9FHtyO988CwNMMOE3YIeci+UV+x5Zy8fI2qHNpsEtSF83YPBmE8UWmfYAQg6Ux7Gsmd4FejZqnEUZCMGaNQHQw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.0.tgz", + "integrity": "sha512-zCMeMXI4HS/tXvJz8vWGexpZj2YVtRAihHLk1imZj4efx1BQzN76YFeKqlDr3bUWI26wHwLWPd3rwh6pe4EV7g==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.0.tgz", + "integrity": "sha512-t76XLQDpxgmq2cNXKTVEB7O7YMb42atj2Re2Haf45HkaUpjM2J0UuJZDuaGbPbamzZ7bawyGFUkodL+zcE+jvQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.0.tgz", + "integrity": "sha512-AS18v0V+vZiLJyi/4LphvBE+OIX682Pu7ZYNsdUHyUKSoRwdnOsMf6FDekwoAFKej14WAkOef3zAORJgAtXnlQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.0.tgz", + "integrity": "sha512-Mz1jxqm/kfgKkc/KLHC5qIujMvnnarD9ra1cEcrs7qshTUSksPihGrWHVG5+osAIQ68577Zpww7SGapmzSt4Nw==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.0.tgz", + "integrity": "sha512-QbEREjdJeIreIAbdG2hLU1yXm1uu+LTdzoq1KCo4G4pFOLlvIspBm36QrQOar9LFduavoWX2msNFAAAY9j4BDg==", + "cpu": [ + "loong64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.0.tgz", + "integrity": "sha512-sJz3zRNe4tO2wxvDpH/HYJilb6+2YJxo/ZNbVdtFiKDufzWq4JmKAiHy9iGoLjAV7r/W32VgaHGkk35cUXlNOg==", + "cpu": [ + "mips64el" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.0.tgz", + "integrity": "sha512-z9N10FBD0DCS2dmSABDBb5TLAyF1/ydVb+N4pi88T45efQ/w4ohr/F/QYCkxDPnkhkp6AIpIcQKQ8F0ANoA2JA==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.0.tgz", + "integrity": "sha512-pQdyAIZ0BWIC5GyvVFn5awDiO14TkT/19FTmFcPdDec94KJ1uZcmFs21Fo8auMXzD4Tt+diXu1LW1gHus9fhFQ==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.0.tgz", + "integrity": "sha512-hPlRWR4eIDDEci953RI1BLZitgi5uqcsjKMxwYfmi4LcwyWo2IcRP+lThVnKjNtk90pLS8nKdroXYOqW+QQH+w==", + "cpu": [ + "s390x" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.0.tgz", + "integrity": "sha512-1hBWx4OUJE2cab++aVZ7pObD6s+DK4mPGpemtnAORBvb5l/g5xFGk0vc0PjSkrDs0XaXj9yyob3d14XqvnQ4gw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.0.tgz", + "integrity": "sha512-6m0sfQfxfQfy1qRuecMkJlf1cIzTOgyaeXaiVaaki8/v+WB+U4hc6ik15ZW6TAllRlg/WuQXxWj1jx6C+dfy3w==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.0.tgz", + "integrity": "sha512-xbbOdfn06FtcJ9d0ShxxvSn2iUsGd/lgPIO2V3VZIPDbEaIj1/3nBBe1AwuEZKXVXkMmpr6LUAgMkLD/4D2PPA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.0.tgz", + "integrity": "sha512-fWgqR8uNbCQ/GGv0yhzttj6sU/9Z5/Sv/VGU3F5OuXK6J6SlriONKrQ7tNlwBrJZXRYk5jUhuWvF7GYzGguBZQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.0.tgz", + "integrity": "sha512-aCwlRdSNMNxkGGqQajMUza6uXzR/U0dIl1QmLjPtRbLOx3Gy3otfFu/VjATy4yQzo9yFDGTxYDo1FfAD9oRD2A==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.0.tgz", + "integrity": "sha512-nyvsBccxNAsNYz2jVFYwEGuRRomqZ149A39SHWk4hV0jWxKM0hjBPm3AmdxcbHiFLbBSwG6SbpIcUbXjgyECfA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.0.tgz", + "integrity": "sha512-Q1KY1iJafM+UX6CFEL+F4HRTgygmEW568YMqDA5UV97AuZSm21b7SXIrRJDwXWPzr8MGr75fUZPV67FdtMHlHA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.0.tgz", + "integrity": "sha512-W1eyGNi6d+8kOmZIwi/EDjrL9nxQIQ0MiGqe/AWc6+IaHloxHSGoeRgDRKHFISThLmsewZ5nHFvGFWdBYlgKPg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.0.tgz", + "integrity": "sha512-30z1aKL9h22kQhilnYkORFYt+3wp7yZsHWus+wSKAJR8JtdfI76LJ4SBdMsCopTR3z/ORqVu5L1vtnHZWVj4cQ==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.0.tgz", + "integrity": "sha512-aIitBcjQeyOhMTImhLZmtxfdOcuNRpwlPNmlFKPcHQYPhEssw75Cl1TSXJXpMkzaua9FUetx/4OQKq7eJul5Cg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.150.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.150.0.tgz", + "integrity": "sha512-rDS5/31E9HfPl/CIzGrn0DOlvBbXFseQ5URJ9sYMfstbKLD/c6Gm9vmRzRGDdAXyOIL4zmO37lc9RIwYqVruZw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/oxc-project" + } + }, + "node_modules/@rolldown/binding-android-arm-eabi": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.9.tgz", + "integrity": "sha512-tNISae1QEf/vkb3xkRcjV5SEdzPE97We5IVaa2Z8jSszQPZ8U60B/YCYpw4QI7VidYsBtKavczXf+DyDs9WGxw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.9.tgz", + "integrity": "sha512-YC8YsI30o606GTZi0VyzYlsDKFP8W61i/QzayHDkLbNEz/IShqAmTa+hsJRj13xTHA0H+6fk4b2UmGn+Q/cMlg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.9.tgz", + "integrity": "sha512-IwhlH3qK5urrY8hZiEgGkHKEFN901p/p2bjxCxJlr4GyNnF7wYpUvK+Y43uaRYuC4hpfjzbR3SJC3arX1jGvmw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.9.tgz", + "integrity": "sha512-XxpJfVzFh+jilRxIXUqcfYAYcunIc/XEzIizsOL1fcJee5Sf7H3mH8WlLmfHfluz5amqR88QQo9izKtmMlavAw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.9.tgz", + "integrity": "sha512-kSfvhmgeWyfkbT3p/1s5vSgboogoah2zkm9fX2zjg2hHxSV7T4KhMWRUUaRk4OXNqoD3QAUeRqLcs1aZOK4U1g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.9.tgz", + "integrity": "sha512-1RVzG17pxqbTfYLC352JlLt6kKLG+6Hr30n8DlIJqsnV5luUDd2Qdx9Ayw1Cabfyb1K9k0jXEZ7evxkRoT+uiw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.9.tgz", + "integrity": "sha512-BXqPvZ2drqVD+/Z8UpKwcs4Mp7grM+eGFku4CAEKrEtcbAsUpzREphK1sogCRZGreVPiMkiiBtw0n3TPteuqvw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.9.tgz", + "integrity": "sha512-11vWvo8YDwLzukt27J3aYDWU+gg2P7J+ZOmiJ0hkF5BXZDW7pVya7r40MXDy6ya0i9KamoENSVKIugvJNgFXIA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.9.tgz", + "integrity": "sha512-a1tijMkdwsIARtc0F39ApURROkf3NwqinI6TOiSSWCTR7dT96dffNvMUtDHnq64wKNTIZOIlzKrFvvFUznJiyw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.9.tgz", + "integrity": "sha512-x6SQNdAvv4c3hWqTMaWuawzMX9myaCs/yEmlGsxJzkdClnHW7FbrjQuSiRDhuSYzEYoEMhsaJy9qHG/XNemJPQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.9.tgz", + "integrity": "sha512-9s0AZ8BFK5/n7B/TBoa2yJE3gI3KURrbXcPBlsAsvjU4VeJKgE90y1YtNxyEUIcHPQkg6/yfF3qihUrcM/Kf0Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.9.tgz", + "integrity": "sha512-P7VWAmV+WdJluH7ovnRGoiv2i8To7GAZ+kGzfGup635cyL7SyYl3lSUaA3Gp5THf0n/Co5EyEqb2zbqq+nMOHQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.9.tgz", + "integrity": "sha512-1qixtsE4BK8h+yS3BfmZ09UhA7O/N4IACva6YBr7EBvCJraByTuRcgOTaiA62Tm0vey3UcKXLOaoGHtYmNGEVg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.9.tgz", + "integrity": "sha512-ok8IQjcEPs1AKZfuEUznVBrJw+gK4soq+bx8b1X2XoMqVClarc1q5JDmVtWXY1xfr6ZuHTAsPXHTgTrqKTZeww==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.9.tgz", + "integrity": "sha512-Ip2mXoU0hM0boq3Rf+ekuT653OROSo6aSYcPT1VHE4q52KvyxgFkQgrgb/IEsxOuvQ2fZZbs8khJAyCEPM24/g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/prop-types": { + "version": "15.7.15", + "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.15.tgz", + "integrity": "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/react": { + "version": "18.3.31", + "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.31.tgz", + "integrity": "sha512-vfEqpXTvwT91yhmwdfouStN2hSKwTvyRs8qpLfADyrq/kxDw0hZM7Wk9Ug1FELj8hIby+S/+kQCSRFF32nv2Qw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/prop-types": "*", + "csstype": "^3.2.2" + } + }, + "node_modules/@types/react-dom": { + "version": "18.3.7", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.7.tgz", + "integrity": "sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^18.0.0" + } + }, + "node_modules/@vitejs/plugin-react": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-6.1.1.tgz", + "integrity": "sha512-yxLaQV9gkhS8ezJqCM6+ndU7mDY6gqAg75NQ+0IjwEI8IYOmQCgkRwHKVSfWXW076DsqMo0Dk+0FK1U+M5RgFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@rolldown/pluginutils": "^1.0.1" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "peerDependencies": { + "@rolldown/plugin-babel": "^0.1.7 || ^0.2.0", + "babel-plugin-react-compiler": "^1.0.0", + "oxc-transform-react": "^0.145.0", + "vite": "^8.0.0" + }, + "peerDependenciesMeta": { + "@rolldown/plugin-babel": { + "optional": true + }, + "babel-plugin-react-compiler": { + "optional": true + }, + "oxc-transform-react": { + "optional": true + } + } + }, + "node_modules/convex": { + "version": "1.46.0", + "resolved": "https://registry.npmjs.org/convex/-/convex-1.46.0.tgz", + "integrity": "sha512-dhBwDQTZDOhi5SUGAX17EWNehCdNq6/UFrSL0eLZuGfMkdsQbCVWVRZCzbZ5CF+/pgvoGxtApbEoTSipqL4JTw==", + "license": "Apache-2.0", + "dependencies": { + "esbuild": "0.27.0", + "prettier": "^3.0.0", + "ws": "8.21.0" + }, + "bin": { + "convex": "bin/main.js" + }, + "engines": { + "node": ">=20.0.0", + "npm": ">=7.0.0" + }, + "peerDependencies": { + "@auth0/auth0-react": "^2.0.1", + "@clerk/clerk-react": "^4.12.8 || ^5.0.0", + "@clerk/react": "^6.4.3", + "react": "^18.0.0 || ^19.0.0-0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@auth0/auth0-react": { + "optional": true + }, + "@clerk/clerk-react": { + "optional": true + }, + "@clerk/react": { + "optional": true + }, + "react": { + "optional": true + } + } + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/esbuild": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.0.tgz", + "integrity": "sha512-jd0f4NHbD6cALCyGElNpGAOtWxSq46l9X/sWB0Nzd5er4Kz2YTm+Vl0qKFT9KUJvD8+fiO8AvoHhFvEatfVixA==", + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.0", + "@esbuild/android-arm": "0.27.0", + "@esbuild/android-arm64": "0.27.0", + "@esbuild/android-x64": "0.27.0", + "@esbuild/darwin-arm64": "0.27.0", + "@esbuild/darwin-x64": "0.27.0", + "@esbuild/freebsd-arm64": "0.27.0", + "@esbuild/freebsd-x64": "0.27.0", + "@esbuild/linux-arm": "0.27.0", + "@esbuild/linux-arm64": "0.27.0", + "@esbuild/linux-ia32": "0.27.0", + "@esbuild/linux-loong64": "0.27.0", + "@esbuild/linux-mips64el": "0.27.0", + "@esbuild/linux-ppc64": "0.27.0", + "@esbuild/linux-riscv64": "0.27.0", + "@esbuild/linux-s390x": "0.27.0", + "@esbuild/linux-x64": "0.27.0", + "@esbuild/netbsd-arm64": "0.27.0", + "@esbuild/netbsd-x64": "0.27.0", + "@esbuild/openbsd-arm64": "0.27.0", + "@esbuild/openbsd-x64": "0.27.0", + "@esbuild/openharmony-arm64": "0.27.0", + "@esbuild/sunos-x64": "0.27.0", + "@esbuild/win32-arm64": "0.27.0", + "@esbuild/win32-ia32": "0.27.0", + "@esbuild/win32-x64": "0.27.0" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "license": "MIT" + }, + "node_modules/lightningcss": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.33.0.tgz", + "integrity": "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.33.0", + "lightningcss-darwin-arm64": "1.33.0", + "lightningcss-darwin-x64": "1.33.0", + "lightningcss-freebsd-x64": "1.33.0", + "lightningcss-linux-arm-gnueabihf": "1.33.0", + "lightningcss-linux-arm64-gnu": "1.33.0", + "lightningcss-linux-arm64-musl": "1.33.0", + "lightningcss-linux-x64-gnu": "1.33.0", + "lightningcss-linux-x64-musl": "1.33.0", + "lightningcss-win32-arm64-msvc": "1.33.0", + "lightningcss-win32-x64-msvc": "1.33.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.33.0.tgz", + "integrity": "sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.33.0.tgz", + "integrity": "sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.33.0.tgz", + "integrity": "sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.33.0.tgz", + "integrity": "sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.33.0.tgz", + "integrity": "sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.33.0.tgz", + "integrity": "sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.33.0.tgz", + "integrity": "sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.33.0.tgz", + "integrity": "sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.33.0.tgz", + "integrity": "sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.33.0.tgz", + "integrity": "sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.33.0.tgz", + "integrity": "sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "license": "MIT", + "dependencies": { + "js-tokens": "^3.0.0 || ^4.0.0" + }, + "bin": { + "loose-envify": "cli.js" + } + }, + "node_modules/nanoid": { + "version": "3.3.19", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.19.tgz", + "integrity": "sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.28", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.28.tgz", + "integrity": "sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.18", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/prettier": { + "version": "3.9.8", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.9.8.tgz", + "integrity": "sha512-WRFq3Wn3WId7LLROfMLdH7xaFr2jR62wU8nLO6rQUOLOxNZUviyJQs1M0iIhLexSFy+L+w0ch66wtoO2jRjG0A==", + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/react": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", + "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", + "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0", + "scheduler": "^0.23.2" + }, + "peerDependencies": { + "react": "^18.3.1" + } + }, + "node_modules/rolldown": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.9.tgz", + "integrity": "sha512-hx/Pv0N1haXRb11qkfnK5MXB/iqr7i0yjWQqmO9uHqZpBgQSqzc8UsSnEpalsh+j1I8qQ2CkXAkJC8Br3dKSlg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.150.0", + "@rolldown/pluginutils": "^1.0.0" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm-eabi": "1.2.9", + "@rolldown/binding-android-arm64": "1.2.9", + "@rolldown/binding-darwin-arm64": "1.2.9", + "@rolldown/binding-darwin-x64": "1.2.9", + "@rolldown/binding-freebsd-x64": "1.2.9", + "@rolldown/binding-linux-arm-gnueabihf": "1.2.9", + "@rolldown/binding-linux-arm64-gnu": "1.2.9", + "@rolldown/binding-linux-arm64-musl": "1.2.9", + "@rolldown/binding-linux-ppc64-gnu": "1.2.9", + "@rolldown/binding-linux-s390x-gnu": "1.2.9", + "@rolldown/binding-linux-x64-gnu": "1.2.9", + "@rolldown/binding-linux-x64-musl": "1.2.9", + "@rolldown/binding-openharmony-arm64": "1.2.9", + "@rolldown/binding-win32-arm64-msvc": "1.2.9", + "@rolldown/binding-win32-x64-msvc": "1.2.9" + } + }, + "node_modules/scheduler": { + "version": "0.23.2", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz", + "integrity": "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/vite": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.3.0.tgz", + "integrity": "sha512-lhZBVvEHefgE+HQZC9O7EBJgCU/nVzFNl7vkS4RE0APtWLP02/8QVIkQtzBxPquh7lq5/78NHipTj7ODQ6XuyQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "lightningcss": "^1.33.0", + "picomatch": "^4.0.7", + "postcss": "^8.5.28", + "rolldown": "~1.2.6", + "tinyglobby": "^0.2.17" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "@vitejs/devtools": "^0.7.1", + "esbuild": "^0.27.0 || ^0.28.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "@vitejs/devtools": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/ws": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + } + } +} diff --git a/tools/stack-bench/reference-apps/ecommerce/convex/client/package.json b/tools/stack-bench/reference-apps/ecommerce/convex/client/package.json new file mode 100644 index 00000000000..b2cb2fbc428 --- /dev/null +++ b/tools/stack-bench/reference-apps/ecommerce/convex/client/package.json @@ -0,0 +1,23 @@ +{ + "name": "client", + "version": "1.0.0", + "private": true, + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview" + }, + "dependencies": { + "react": "^18.3.1", + "react-dom": "^18.3.1", + "convex": "1.46.0" + }, + "devDependencies": { + "@types/react": "^18.3.3", + "@types/react-dom": "^18.3.0", + "@vitejs/plugin-react": "^6.1.0", + "typescript": "^5.5.3", + "vite": "^8.2.2" + } +} diff --git a/tools/stack-bench/reference-apps/ecommerce/convex/client/src/App.tsx b/tools/stack-bench/reference-apps/ecommerce/convex/client/src/App.tsx new file mode 100644 index 00000000000..39ad76e47db --- /dev/null +++ b/tools/stack-bench/reference-apps/ecommerce/convex/client/src/App.tsx @@ -0,0 +1,1235 @@ +import React, { useEffect, useMemo, useRef, useState, useCallback } from "react"; +import { mutate, authenticate, validateSession, subscribeState, signOut, TOKEN_KEY } from "./request"; +import { ProgressionPanel } from "./ProgressionPanel"; + +const CATALOG_PAGE_SIZE = 10; + +interface ItemT { + id: string; + name: string; + price: number; + description?: string; + category: string; + stock: number; + purchaseCount: number; + variants?: string[]; +} + +interface ReviewT { + id: string; + itemId: string; + userId: string; + username: string; + rating: number; + comment: string; + createdAt: string; +} + +interface ItemDetailT { + id: string; + name: string; + price: number; + description: string; + stock: number; + reviews: ReviewT[]; + average: number; +} + +interface CartLineT { + itemId: string; + name: string; + price: number; + stock: number; + quantity: number; +} + +interface CartT { + items: CartLineT[]; + total: number; + promotionCode?: string; + discount?: number; +} + +interface OrderLineT { + itemId: string; + name: string; + price: number; + quantity: number; + returned?: boolean; + warehouseNames?: string[]; +} + +interface OrderT { + id: string; + items: OrderLineT[]; + total: number; + status: "pending" | "shipped" | "delivered" | "cancelled" | "refunded"; + discount?: number; + refundTotal?: number; + createdAt: string; +} + +interface UserT { + id: string; + username: string; + isAdmin: boolean; + isStaff: boolean; + roles?: string[]; +} + +interface AdminLocationT { + id: string; + itemId: string; + itemName: string; + warehouseId: string; + warehouseName: string; + quantity: number; +} + +interface CategoryTotalT { + category: string; + units: number; + revenue: number; +} + +interface AdminOverviewT { + items: Array<{ id: string; name: string; price: number; stock: number; category: string }>; + warehouses: Array<{ id: string; name: string; total: number }>; + locations: AdminLocationT[]; + revenue: number; + categories: CategoryTotalT[]; + lowStock: Array<{ id: string; name: string; stock: number }>; + queueDepth: number; +} + +interface FulfilmentQueueT { + orders: OrderT[]; + depth: number; +} + +function useTransientError(): [string, (msg: string) => void] { + const [message, setMessage] = useState(""); + const timer = useRef | null>(null); + const show = useCallback((msg: string) => { + setMessage(msg); + if (timer.current) clearTimeout(timer.current); + timer.current = setTimeout(() => setMessage(""), 5000); + }, []); + return [message, show]; +} + + +export default function App() { + const [token, setToken] = useState(() => localStorage.getItem(TOKEN_KEY)); + const [currentUser, setCurrentUser] = useState(null); + const [initializing, setInitializing] = useState(true); + const [items, setItems] = useState([]); + const [searchQuery, setSearchQuery] = useState(""); + const [categoryFilter, setCategoryFilter] = useState(""); + const [minimumPrice, setMinimumPrice] = useState(""); + const [maximumPrice, setMaximumPrice] = useState(""); + const [inStockOnly, setInStockOnly] = useState(false); + const [searchPage, setSearchPage] = useState(0); + const [cart, setCart] = useState({ items: [], total: 0 }); + // Keep floating panels mutually exclusive so navigation remains reachable. + const [activeView, setActiveView] = useState<"cart" | "orders" | "admin" | "fulfilment" | null>(null); + const cartOpen = activeView === "cart"; + const ordersOpen = activeView === "orders"; + const [orders, setOrders] = useState([]); + const adminOpen = activeView === "admin"; + const [adminOverview, setAdminOverview] = useState(null); + const fulfilmentOpen = activeView === "fulfilment"; + const [fulfilmentQueue, setFulfilmentQueue] = useState({ orders: [], depth: 0 }); + const [recommended, setRecommended] = useState([]); + const [selectedItemId, setSelectedItemId] = useState(null); + const [itemDetails, setItemDetails] = useState([]); + const itemDetail = itemDetails.find(item => item.id === selectedItemId) || null; + + const [buyError, showBuyError] = useTransientError(); + const [orderError, showOrderError] = useTransientError(); + + + const saveSession = (tok: string, user: UserT) => { + setToken(tok); + setCurrentUser(user); + }; + + const clearSession = () => { + localStorage.removeItem(TOKEN_KEY); + setToken(null); + setCurrentUser(null); + setCart({ items: [], total: 0 }); + setOrders([]); + setAdminOverview(null); + setFulfilmentQueue({ orders: [], depth: 0 }); + setActiveView(null); + }; + + // Validate a restored session once. Native subscriptions own application data. + useEffect(() => { + if (!token) return; + validateSession().catch(clearSession); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + useEffect(() => subscribeState((state) => { + setInitializing(false); + setItems(state.items); + setCurrentUser(state.user); + setCart(state.cart); + setOrders(state.orders); + setAdminOverview(state.admin); + setFulfilmentQueue(state.fulfilment || { orders: [], depth: 0 }); + setRecommended(state.recommended); + setItemDetails(state.details); + }), [token]); + + // Escape closes whichever overlay is open. + useEffect(() => { + const onKeyDown = (e: KeyboardEvent) => { + if (e.key !== "Escape") return; + if (selectedItemId) setSelectedItemId(null); + else if (activeView) setActiveView(null); + }; + window.addEventListener("keydown", onKeyDown); + return () => window.removeEventListener("keydown", onKeyDown); + }, [selectedItemId, activeView]); + + const handleSignUp = async (username: string, password: string) => { + const data = await authenticate(username, password, "signUp"); + saveSession(data.token, data.user); + }; + + const handleSignIn = async (username: string, password: string) => { + const data = await authenticate(username, password, "signIn"); + saveSession(data.token, data.user); + }; + + const handleSignOut = async () => { + await signOut(); + clearSession(); + }; + + const handleBuyNow = async (itemId: string) => { + try { + await mutate("api:buy_now", { itemId }); + } catch (err: any) { + showBuyError(err.message); + } + }; + + const handleAddToCart = async (itemId: string) => { + try { + await mutate("api:add_to_cart", { itemId, quantity: 1 }); + } catch (err: any) { + showBuyError(err.message); + } + }; + + const handleQuantityChange = async (itemId: string, quantity: number) => { + try { + await mutate("api:update_cart_quantity", { itemId, quantity }); + } catch (err: any) { + showBuyError(err.message); + } + }; + + const handleCheckout = async () => { + try { + await mutate("api:checkout"); + } catch (err: any) { + showBuyError(err.message); + } + }; + + + const openOrders = () => setActiveView("orders"); + const openAdmin = () => setActiveView("admin"); + const openFulfilment = () => setActiveView("fulfilment"); + + const [reviewError, setReviewError] = useState(""); + const handleReviewSubmit = async (itemId: string, rating: number, comment: string) => { + setReviewError(""); + try { + await mutate("api:submit_review", { itemId, rating, comment }); + } catch (err: any) { + setReviewError(err.message); + } + }; + + const handleRestock = async (itemId: string, warehouseId: string, quantity: number) => { + try { + await mutate("api:admin_restock", { itemId, warehouseId, quantity }); + } catch (err: any) { + showBuyError(err.message); + } + }; + + const handleTransfer = async (itemId: string, fromWarehouseId: string, toWarehouseId: string, quantity: number) => { + try { + await mutate("api:admin_transfer_stock", { itemId, fromWarehouseId, toWarehouseId, quantity }); + } catch (err: any) { + showOrderError(err.message); + } + }; + + const handlePriceChange = async (itemId: string, price: number) => { + try { + await mutate("api:admin_change_price", { itemId, price }); + } catch (err: any) { + showOrderError(err.message); + } + }; + + const handleShipOrder = async (orderId: string) => { + try { + await mutate("api:ship_order", { orderId }); + } catch (err: any) { + showOrderError(err.message); + throw err; + } + }; + + const handleCancelOrder = async (orderId: string) => { + try { + await mutate("api:cancel_order", { orderId }); + } catch (err: any) { + showOrderError(err.message); + } + }; + + const handleReturnItem = async (orderId: string, itemId: string) => { + try { + await mutate("api:return_order_item", { orderId, itemId }); + } catch (err: any) { + showOrderError(err.message); + } + }; + + const filteredItems = useMemo(() => { + const q = searchQuery.trim().toLowerCase(); + const min = minimumPrice === "" ? -Infinity : Number(minimumPrice); + const max = maximumPrice === "" ? Infinity : Number(maximumPrice); + const filtering = Boolean(q || categoryFilter || minimumPrice || maximumPrice || inStockOnly); + return items.filter((it) => (!q || it.name.toLowerCase().includes(q)) + && (!categoryFilter || it.category === categoryFilter) + && it.price >= min && it.price <= max && (!inStockOnly || it.stock > 0)) + .sort((a, b) => (filtering ? 0 : b.purchaseCount - a.purchaseCount) || a.name.localeCompare(b.name)); + }, [items, searchQuery, categoryFilter, minimumPrice, maximumPrice, inStockOnly]); + const searchResults = filteredItems.slice(searchPage * CATALOG_PAGE_SIZE, + searchPage * CATALOG_PAGE_SIZE + CATALOG_PAGE_SIZE); + + const cartCount = cart.items.reduce((s, l) => s + l.quantity, 0); + const selectedItem = items.find((it) => it.id === selectedItemId) || null; + const isCustomer = !!currentUser && !currentUser.isAdmin && !currentUser.isStaff; + + return ( +
+ {initializing && ( +
+
+
Connecting to Storefront...
+
+ )} + +
+

+ Storefront +

+ + { setSearchQuery(e.target.value); setSearchPage(0); }} + onKeyDown={(e) => { + if (e.key === "Escape") setSearchQuery(""); + }} + /> +
+
+ + {currentUser && ( + + )} + {currentUser?.isAdmin && ( + + )} + {(currentUser?.isStaff || currentUser?.isAdmin) && ( + + )} + {currentUser ? ( + <> + + {currentUser.username} + + + + ) : ( + + )} +
+
+ +
+
+ {buyError && ( +
+ {buyError} +
+ )} + +
+
+ + { setMinimumPrice(e.target.value); setSearchPage(0); }} /> + { setMaximumPrice(e.target.value); setSearchPage(0); }} /> + +
+

Catalog

+
+
+ {searchResults.map((item) => ( + setSelectedItemId(item.id)} + onBuy={() => handleBuyNow(item.id)} + onAddToCart={() => handleAddToCart(item.id)} + /> + ))} +
+
+
+ + +
+
+ +
+

Recommended for you

+
+ {recommended.length === 0 ? ( +
Nothing recommended yet
+ ) : ( + recommended.map((item, index) => ( +
+ {index + 1} + setSelectedItemId(item.id)} onBuy={() => handleBuyNow(item.id)} + onAddToCart={() => handleAddToCart(item.id)} /> + +
+ )) + )} +
+
+ + {!activeView && (currentUser?.isStaff || currentUser?.isAdmin) && + } +
+ + {selectedItem && ( + { + setSelectedItemId(null); + setReviewError(""); + }} + onBuy={() => handleBuyNow(selectedItem.id)} + onAddToCart={() => handleAddToCart(selectedItem.id)} + onSubmitReview={(rating, comment) => handleReviewSubmit(selectedItem.id, rating, comment)} + /> + )} +
+ +
setActiveView(null)} /> +
+
+

Cart

+ +
+
+ {cart.items.length === 0 ? ( +
+ Your cart is empty +
+ ) : ( + <> + {cart.items.map((line) => ( + handleQuantityChange(line.itemId, qty)} + onRemove={() => handleQuantityChange(line.itemId, 0)} + /> + ))} +
+ Total + ${cart.total.toFixed(2)} +
+ + + )} +
+
+ +
setActiveView(null)} /> +
+
+

Order history

+ +
+ {orderError && ( +
+ {orderError} +
+ )} +
+ {orders.length === 0 ? ( +
You haven't placed any orders yet
+ ) : ( + orders.map((order) => ( +
+
+ {new Date(order.createdAt).toLocaleString()} + {order.status !== "cancelled" && order.items.length > 0 && order.items.every(line => line.returned) ? "returned" : order.status} +
+
{order.items.map(l => {l.name} ×{l.quantity}{l.returned ? " (returned)" : ""} )}
+
+ ${order.total.toFixed(2)} +
+
{Number(order.discount || 0).toFixed(2)}
+
{Number(order.refundTotal || 0).toFixed(2)}
+
+ {order.status === "pending" && ( + + )} + {["shipped", "delivered"].includes(order.status) && + order.items + .filter((l) => !l.returned) + .map((l) => ( + + ))} +
+
+ )) + )} +
+
+ + {fulfilmentOpen && (currentUser?.isStaff || currentUser?.isAdmin) && ( + setActiveView(null)} + onShip={handleShipOrder} orderError={orderError}> + + + )} + + {adminOpen && currentUser?.isAdmin && ( + setActiveView(null)} + onRestock={handleRestock} + onTransfer={handleTransfer} + onPriceChange={handlePriceChange} + orderError={orderError} + > + + + )} +
+ ); +} + +function ItemCard({ + item, + isCustomer, + onOpen, + onBuy, + onAddToCart, + testId = "item-card", +}: { + item: ItemT; + isCustomer: boolean; + onOpen: () => void; + onBuy: () => void; + onAddToCart: () => void; + testId?: string | null; +}) { + const outOfStock = item.stock === 0; + const [submitState, setSubmitState] = useState('idle'); + const requestAlert = async (event: React.MouseEvent) => { + event.stopPropagation(); + setSubmitState('pending'); + try { + await mutate("progression:stockAlert", { itemId: item.id }); + setSubmitState('succeeded'); + } catch { setSubmitState('failed'); } + }; + return ( +
+
+ {item.name} +
+
+ + ${item.price.toFixed(2)} + +
+
+ 0 && item.stock <= 5 ? " low" : ""}`} data-role="item-stock"> + {item.stock} + + {outOfStock && ( + + Out of stock + + )} +
+ {isCustomer && ( +
e.stopPropagation()}> + + +
+ )} + {(item.variants || []).map(variant => + {variant})} + {isCustomer && outOfStock && } +
+ ); +} + +function ItemDetailPanel({ + item, + detail, + isCustomer, + reviewError, + onClose, + onBuy, + onAddToCart, + onSubmitReview, +}: { + item: ItemT; + detail: ItemDetailT | null; + isCustomer: boolean; + reviewError: string; + onClose: () => void; + onBuy: () => void; + onAddToCart: () => void; + onSubmitReview: (rating: number, comment: string) => void; +}) { + const [rating, setRating] = useState(5); + const [comment, setComment] = useState(""); + const outOfStock = item.stock === 0; + + const submit = (e: React.FormEvent) => { + e.preventDefault(); + onSubmitReview(rating, comment); + setComment(""); + }; + + return ( +
+
+

{item.name}

+ +
+
+ + ${item.price.toFixed(2)} + + 0 && item.stock <= 5 ? "item-stock low" : "item-stock"}> + {item.stock} in stock + +
+ {outOfStock && ( + + Out of stock + + )} +

{detail?.description || "Loading description..."}

+ + {isCustomer && ( +
+ + +
+ )} + +

+ Reviews — average {(detail?.average ?? 0).toFixed(1)} +

+ {!detail || detail.reviews.length === 0 ? ( +
No reviews yet
+ ) : ( + detail.reviews.map((r) => ( +
+
+ {r.username} + {"★".repeat(r.rating)} +
+
{r.comment}
+
+ )) + )} + + {isCustomer && ( +
+ + setComment(e.target.value)} + /> + +
+ )} + {reviewError && ( +
+ {reviewError} +
+ )} +
+ ); +} + +function CartLineRow({ + line, + onQuantityChange, + onRemove, +}: { + line: CartLineT; + onQuantityChange: (qty: number) => void; + onRemove: () => void; +}) { + const [value, setValue] = useState(String(line.quantity)); + + useEffect(() => { + setValue(String(line.quantity)); + }, [line.quantity]); + + const commit = () => { + const qty = Number(value); + if (Number.isInteger(qty) && qty >= 1 && qty !== line.quantity) { + onQuantityChange(qty); + } else { + setValue(String(line.quantity)); + } + }; + + return ( +
+ {line.name} + setValue(e.target.value)} + onBlur={commit} + onKeyDown={(e) => { + if (e.key === "Enter") { + e.preventDefault(); + commit(); + } + }} + /> + ${(line.price * line.quantity).toFixed(2)} + +
+ ); +} + +function AuthBox({ + onSignUp, + onSignIn, +}: { + onSignUp: (username: string, password: string) => Promise; + onSignIn: (username: string, password: string) => Promise; +}) { + const [signUpUsername, setSignUpUsername] = useState(""); + const [signUpPassword, setSignUpPassword] = useState(""); + const [signUpError, setSignUpError] = useState(""); + + const [showSignIn, setShowSignIn] = useState(false); + const [signInUsername, setSignInUsername] = useState(""); + const [signInPassword, setSignInPassword] = useState(""); + const [signInError, setSignInError] = useState(""); + + const submitSignUp = async (e: React.FormEvent) => { + e.preventDefault(); + setSignUpError(""); + try { + await onSignUp(signUpUsername.trim(), signUpPassword); + } catch (err: any) { + setSignUpError(err.message); + } + }; + + const submitSignIn = async (e: React.FormEvent) => { + e.preventDefault(); + setSignInError(""); + try { + await onSignIn(signInUsername.trim(), signInPassword); + } catch (err: any) { + setSignInError(err.message); + } + }; + + return ( +
+
+ setSignUpUsername(e.target.value)} + /> + setSignUpPassword(e.target.value)} + /> + + {signUpError && ( +
+ {signUpError} +
+ )} +
+ + {showSignIn && ( +
+ setSignInUsername(e.target.value)} + /> + setSignInPassword(e.target.value)} + /> + + {signInError && ( +
+ {signInError} +
+ )} +
+ )} +
+ ); +} + +function AdminPanel({ + overview, + onClose, + onRestock, + onTransfer, + onPriceChange, + orderError, + children, +}: { + overview: AdminOverviewT | null; + onClose: () => void; + onRestock: (itemId: string, warehouseId: string, quantity: number) => void; + onTransfer: (itemId: string, fromWarehouseId: string, toWarehouseId: string, quantity: number) => void; + onPriceChange: (itemId: string, price: number) => void; + orderError: string; + children?: React.ReactNode; +}) { + const [restockValues, setRestockValues] = useState>({}); + const [globalRestock, setGlobalRestock] = useState({ item: "", warehouse: "", quantity: "" }); + const [priceValues, setPriceValues] = useState>({}); + const [transferValues, setTransferValues] = useState< + Record + >({}); + + if (!overview) { + return ( +
+
+

Admin

+ +
+
Loading admin data...
+
+ ); + } + + const warehouses = overview.warehouses; + + const transferFor = (itemId: string) => + transferValues[itemId] || { from: warehouses[0]?.id || "", to: warehouses[1]?.id || warehouses[0]?.id || "", qty: "" }; + + return ( +
+
+

Admin

+ +
+ + {orderError && ( +
+ {orderError} +
+ )} + +
+ Total revenue: ${overview.revenue.toFixed(2)} +
+ +
+ setGlobalRestock(value => ({ ...value, item: event.target.value }))} /> + setGlobalRestock(value => ({ ...value, warehouse: event.target.value }))} /> + setGlobalRestock(value => ({ ...value, quantity: event.target.value }))} /> + +
+ +
+
+

Items

+ {overview.items.map((it) => { + const transfer = transferFor(it.id); + return ( +
+ {it.name} + {it.stock} +
+ setPriceValues((prev) => ({ ...prev, [it.id]: e.target.value }))} + /> + +
+
+ + + + setTransferValues((prev) => ({ ...prev, [it.id]: { ...transferFor(it.id), qty: e.target.value } })) + } + /> + +
+
+ ); + })} +
+
+

Warehouses

+
+ {overview.warehouses.map((w) => ( + + {w.name} — {w.total} + + ))} +
+

Stock by warehouse

+ {overview.locations.map((loc) => ( +
+ + {loc.itemName} @ {loc.warehouseName} + + {loc.quantity} +
{ + e.preventDefault(); + const qty = Number(restockValues[loc.id]); + if (Number.isInteger(qty) && qty >= 1) { + onRestock(loc.itemId, loc.warehouseId, qty); + setRestockValues((prev) => ({ ...prev, [loc.id]: "" })); + } + }}> + setRestockValues((prev) => ({ ...prev, [loc.id]: e.target.value }))} + /> + +
+
+ ))} +
+
+ +
+
+

Low stock

+
+ {overview.lowStock.length === 0 ? ( +
Nothing is running low
+ ) : ( + overview.lowStock.map((it) => ( +
+ {it.name} + {it.stock} +
+ )) + )} +
+
+
+

Category totals

+ {overview.categories.map((c) => ( +
+ {c.category} + {c.units} + ${c.revenue.toFixed(2)} +
+ ))} +
+
+ {children} +
+ ); +} + +function FulfilmentPanel({ + queue, + onClose, + onShip, + orderError, + children, +}: { + queue: FulfilmentQueueT; + onClose: () => void; + onShip: (orderId: string) => Promise; + orderError: string; + children?: React.ReactNode; +}) { + const [submitState, setSubmitState] = useState('idle'); + const ship = async (orderId: string) => { + setSubmitState('pending'); + try { await onShip(orderId); setSubmitState('succeeded'); } + catch { setSubmitState('failed'); } + }; + return ( +
+
+

Fulfilment queue

+ +
+ + {orderError && ( +
+ {orderError} +
+ )} + +
+ Orders waiting: {queue.depth} +
+ + {queue.orders.length === 0 ? ( +
Nothing waiting to ship
+ ) : ( + queue.orders.map((order) => ( +
+
+ {new Date(order.createdAt).toLocaleString()} +
+
{order.items.map((l) => `${l.name} ×${l.quantity}`).join(", ")}
+
+ {order.items.map((l, idx) => ( + + {(l.warehouseNames || []).join(", ") || "Unknown"} + + ))} +
+ +
+ )) + )} + {children} +
+ ); +} diff --git a/tools/stack-bench/reference-apps/ecommerce/convex/client/src/ProgressionPanel.tsx b/tools/stack-bench/reference-apps/ecommerce/convex/client/src/ProgressionPanel.tsx new file mode 100644 index 00000000000..769e535a93b --- /dev/null +++ b/tools/stack-bench/reference-apps/ecommerce/convex/client/src/ProgressionPanel.tsx @@ -0,0 +1,285 @@ +import { mutate, subscribeProgression } from "./request"; +import { useEffect, useState } from "react"; + +type User = { username: string; isAdmin: boolean; isStaff: boolean; roles?: string[] }; +type Item = { id: string; name: string }; +type Order = { id: string; items: Array<{ name: string }>; total: number }; + + +function nameFor(items: Item[], id: unknown) { + return items.find(item => item.id === String(id))?.name || String(id || "Unknown item"); +} + +export function ProgressionPanel({ token, user, items, orders, onSignIn, staffOnly = false }: { + token: string | null; + user: User | null; + items: Item[]; + orders: Order[]; + onSignIn: (username: string, password: string) => Promise; + staffOnly?: boolean; +}) { + const [state, setState] = useState({ tickets: [], promotions: [], notifications: [], + scheduledRestocks: [], ledger: [], staffUsers: [] }); + const [error, setError] = useState(""); + const [staffName, setStaffName] = useState(""); + const [staffPassword, setStaffPassword] = useState(""); + const [profileOpen, setProfileOpen] = useState(false); + const [supportOpen, setSupportOpen] = useState(false); + const [notificationsOpen, setNotificationsOpen] = useState(false); + const [profileName, setProfileName] = useState(""); + const [profileSaveState, setProfileSaveState] = useState("idle"); + const [notificationSaveState, setNotificationSaveState] = useState("idle"); + const [profileAddress, setProfileAddress] = useState(""); + const [supportEmail, setSupportEmail] = useState(""); + const [supportSubject, setSupportSubject] = useState(""); + const [supportMessage, setSupportMessage] = useState(""); + const [supportReference, setSupportReference] = useState(""); + const [preference, setPreference] = useState({ order: false, stock: false }); + + useEffect(() => { + setPreference({ order: !!state.preference?.order, stock: !!state.preference?.stock }); + }, [user?.username, state.preference?.order, state.preference?.stock]); + + useEffect(() => { + setProfileName(state.profile?.name || ""); + setProfileAddress(state.profile?.address || ""); + }, [user?.username, state.profile?.name, state.profile?.address]); + + useEffect(() => subscribeProgression(next => { + setState({ ...next, loadedToken: token }); + }), [token]); + + + const act = async (name: string, args: Record = {}, onComplete?: (state: "succeeded" | "failed") => void) => { + setError(""); + try { + const result = await mutate(name, args); + onComplete?.("succeeded"); + return result; + } catch (err: any) { + onComplete?.("failed"); + setError(err.message); + return null; + } + }; + + if (staffOnly) { + if (!(user?.isStaff || user?.isAdmin)) return null; + return ; + } + + const submitSupport = async () => { + const result = await act("progression:submitSupport", { + email: supportEmail, subject: supportSubject, message: supportMessage, + }); + if (result) setSupportReference(result.ticket.reference); + }; + + const saveProfile = () => { + setProfileSaveState("pending"); + return act("progression:saveProfile", { name: profileName, address: profileAddress }, setProfileSaveState); + }; + + return
+ {user && <> + {profileSaveState} + {notificationSaveState} + } + {!user ?
+

Staff sign in

+ setStaffName(event.target.value)} placeholder="Username" /> + setStaffPassword(event.target.value)} placeholder="Password" /> + +
: {user.username}} + + + + {profileOpen && user &&
+

Profile

+ setProfileName(event.target.value)} placeholder="Name" /> + setProfileAddress(event.target.value)} placeholder="Address" /> + +

{state.profile?.address || ""}

+
} + + {supportOpen &&
+

Support

+ setSupportEmail(event.target.value)} placeholder="Email" /> + setSupportSubject(event.target.value)} placeholder="Subject" /> +
${mode === 'confirmation' ? '

Saved

' : ''}'; + const dialog=document.querySelector('dialog'); + const refresh=async()=>{ + const rows=await (await fetch('/history?user='+encodeURIComponent(user))).json(); + document.querySelector('#history').replaceChildren(...rows.map(row=>{ + const node=document.createElement('div'); node.dataset.role='support-ticket'; node.textContent=row.subject; return node; + })); + }; + const open=()=>{if(mode==='confirmation')document.querySelector('#support-surface').hidden=false;else dialog.showModal();location.hash='support';void refresh();}; + document.querySelector('#support-link').onclick=open; + document.querySelector('#overlay-close').onclick=()=>{dialog.close();history.replaceState(null,'','/');}; + document.querySelector('#support-submit').onclick=async()=>{ + const result=await fetch('/ticket',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({user,subject:document.querySelector('#support-subject').value})}); + await result.json(); + if(result.ok){document.querySelector('#support-reference').textContent='Saved';await refresh();} + }; + if(user){ + document.querySelector('#current-user').onclick=()=>document.querySelector('nav').hidden=false; + document.querySelector('#signout').onclick=()=>{localStorage.removeItem('user');location.reload();}; + }else for(const kind of ['signup','signin']) document.querySelector('#'+kind+'-submit').onclick=()=>{ + localStorage.setItem('user',document.querySelector('#'+kind+'-username').value);location.reload(); + }; + if(location.hash==='#support'&&mode!=='direct'&&mode!=='menu'){open();if(mode==='confirmation')dialog.showModal();} + `); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const browser = await chromium.launch({ headless: true }); + try { + for (const fixture of ['direct', 'menu', 'restored-dialog', 'confirmation', 'lost-history', 'private-leak', 'logout-leak', 'wrong-account', 'reject-all']) { + await t.test(fixture, async () => { + mode = fixture; tickets.length = 0; + const definition = JSON.parse(readFileSync(join(STACK_BENCH_ROOT, + 'tracks/ecommerce/scenarios/progression-support-history.json'), 'utf8')); + const selected = definition.features[0]; + for (const criterion of selected.criteria) { + criterion.steps = criterion.steps.filter((step: { do: string }) => step.do !== 'restartBackend'); + } + for (const step of [...selected.setup, ...selected.criteria.flatMap((criterion: { steps: Record[] }) => criterion.steps)]) { + if ('within' in step || ['click', 'fill', 'expect', 'expectNotReceived'].includes(step.do)) step.within = 500; + if ('settleMs' in step) step.settleMs = 0; + } + const grade = await gradeFeature(browser, compileScenarioDefinition(definition).features[0]!, + { url, level: 2, headed: false, selectedCheckKeys: [], nullControl: false }, + { runId: `support-navigation-${fixture}`, roomName: name => name, url, actions: [], + spacetime: null, backend: 'postgres', nullControl: false, defaultWithin: 500 }); + evidence.push({ fixture, grade, tickets: [...tickets] }); + assert.equal(grade.setupEvidence.status, fixture === 'reject-all' ? 'failed' : 'passed', JSON.stringify(grade)); + if (fixture === 'reject-all') { + assert.equal((grade.setupEvidence.actions.at(-1)?.evidence as ActionEvidence).action.id, 'expect'); + assert.ok(grade.criteria.every(criterion => criterion.evidence.status === 'blocked')); + } else if (['direct', 'menu', 'restored-dialog', 'confirmation'].includes(fixture)) { + assert.ok(grade.criteria.every(criterion => criterion.evidence.status === 'passed'), JSON.stringify(grade)); + } else { + const id = fixture === 'private-leak' ? '612b' : fixture === 'logout-leak' ? '612d' : '612c'; + const result = grade.criteria.find(criterion => criterion.id === id)!.evidence; + assert.equal(result.status, 'failed', JSON.stringify(grade)); + assert.equal((result.actions.at(-1)?.evidence as ActionEvidence).action.id, fixture === 'wrong-account' ? 'ensureSignedIn' : 'expect', JSON.stringify(result)); + } + }); + } + } finally { + await browser.close(); server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + if (process.env.STACK_BENCH_SUPPORT_NAVIGATION_EVIDENCE) { + const { writeFileSync } = await import('node:fs'); + writeFileSync(process.env.STACK_BENCH_SUPPORT_NAVIGATION_EVIDENCE, JSON.stringify({ + rerun: 'node --test --test-name-pattern="support history reaches" dist/tests/account-navigation.integration.js', + limitation: 'restartBackend omitted; real backend durability requires registered qualification controls', evidence, + }, null, 2)); + } + } +}); + +test('managed support live status observes a distinct change without reopening the observer', async () => { + const feature = compileScenarioDefinition(JSON.parse(readFileSync(join(STACK_BENCH_ROOT, + 'tracks/ecommerce/scenarios/progression-managed-support-shared.json'), 'utf8'))).features[0]!; + const criterion = feature.criteria.find(criterion => criterion.id === '613a')!; + const last = criterion.steps.findIndex(step => step.do === 'expect' && step.testid === 'support-status' + && step.contains === 'in progress'); + const steps = criterion.steps.slice(0, last + 1).filter(step => !['support-reply', 'support-reply-submit'].includes(step.testid ?? '')); + const browser = await chromium.launch({ headless: true }); + try { + for (const initial of ['open', 'in progress']) for (const live of [false, true]) { + const context = await browser.newContext(); + let savedStatus = initial; + let ownerLoads = 0; + const owner = await context.newPage(); + const staff = await context.newPage(); + await owner.route('http://support.test/**', route => { + ownerLoads++; + return route.fulfill({ contentType: 'text/html', body: `managed-shared-owner + + ` }); + }); + await staff.exposeFunction('saveStatus', async (status: string) => { + savedStatus = status; + await staff.locator('#support-status').evaluate((element, value) => { element.textContent = value; }, status); + if (live) await owner.locator('#support-status').evaluate((element, value) => { element.textContent = value; }, status); + }); + await staff.setContent(`
Shared managed case + ${initial} + +
`); + await owner.goto('http://support.test/'); + const actors = new Map(([['owner', owner], ['staff', staff]] as const).map(([name, page]) => [String(name), { + page, loc: (id: string, options?: { contains?: string; scope?: { testid: string; contains?: string } }) => { + const root = options?.scope ? page.locator(stableElementSelector(options.scope.testid)) + .filter({ hasText: options.scope.contains }).first() : page; + let locator = root.locator(stableElementSelector(id)).filter({ visible: true }); + if (options?.contains) locator = locator.filter({ hasText: options.contains }); + return locator.first(); + }, + }])); + const service = { defaultWithin: 200, scopedUser: (name: string) => name, + expand: (value: string) => value, testId: stableElementSelector, + sleep: (ms: number) => new Promise(resolve => setTimeout(resolve, Math.min(ms, 20))) }; + const capabilities = { actors: { get: (name: string) => actors.get(name) }, + 'browser-interaction': service, 'browser-observation': service }; + let status = ''; + for (const step of steps) { + const result = await executeAction(ACTION_REGISTRY, step.do, + { ...step, ...(step.testid ? { within: 200 } : {}) }, { capabilities }); + status = result.status; + if (status !== 'passed') break; + } + assert.equal(status, live ? 'passed' : 'failed', `${initial}/live=${live}`); + assert.equal(ownerLoads, 2, 'observer loads only for its baseline, never after the live mutation'); + await context.close(); + } + } finally { await browser.close(); } +}); + +test('cart and recommendation probes leave blocking overlays before the next catalog action', async () => { + const read = (file: string) => compileScenarioDefinition(JSON.parse(readFileSync(join(STACK_BENCH_ROOT, + 'tracks/ecommerce/scenarios', file), 'utf8'))).features[0]!.criteria[0]!.steps; + const cases = [ + { steps: read('progression-cart-checkout.json'), item: 'Headphones', action: 'add-to-cart' }, + { steps: read('02-operational-recommendations.json').slice(0, 4), item: 'Bluetooth Speaker', action: 'buy-now' }, + ]; + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage(); + for (const item of cases) for (const overlay of [false, true]) for (const broken of [false, true]) { + await page.setContent(` +
${item.item}
+ + + `); + const actor = { page, loc: (id: string, options?: { contains?: string; scope?: { testid: string; contains?: string } }) => { + const root = options?.scope ? page.locator(stableElementSelector(options.scope.testid)) + .filter({ hasText: options.scope.contains }).first() : page; + let locator = root.locator(stableElementSelector(id)).filter({ visible: true }); + if (options?.contains) locator = locator.filter({ hasText: options.contains }); + return locator.first(); + } }; + const service = { defaultWithin: 150, expand: (value: string) => value, testId: stableElementSelector, + sleep: (ms: number) => new Promise(resolve => setTimeout(resolve, Math.min(ms, 20))) }; + const capabilities = { actors: { get: () => actor }, 'browser-interaction': service, + 'browser-observation': service, clock: service }; + let status = ''; + for (const step of item.steps) { + const result = await executeAction(ACTION_REGISTRY, step.do, + { ...step, ...(step.testid ? { within: 150 } : {}) }, { capabilities }); + status = result.status; + if (status !== 'passed') break; + } + assert.equal(status, broken ? 'failed' : 'passed', `${item.action}/overlay=${overlay}/broken=${broken}`); + } + } finally { await browser.close(); } +}); + +test('checkout observations reopen the committed cart after redirects and reloads', async t => { + // Failure cases precede the scenario fix: immediate/delayed redirect, an open + // toggle, and an account menu behind confirmation. Wrong/missing counts fail. + // This fixture measures browser navigation only. Existing native mutation + // controls retain responsibility for the complete dbExpectCheckout invariant. + const load = (file: string) => compileScenarioDefinition(JSON.parse(readFileSync(join(STACK_BENCH_ROOT, + 'tracks/ecommerce/scenarios', file), 'utf8'))).features[0]!; + const duplicate = load('01-duplicate-checkout.json').criteria.find(c => c.id === '203b')!.steps; + const crash = load('progression-checkout-crash.json').setup; + const cases = [ + { name: 'normal', source: duplicate, before: 'control-before', layouts: ['redirect', 'delayed', 'open', 'menu-confirmation', 'wrong', 'missing'] }, + { name: 'recovered', source: duplicate, before: 'recovered-before', layouts: ['redirect'] }, + { name: 'crash-setup', source: crash, before: 'recovered-before', layouts: ['delayed', 'open'] }, + { name: 'lost-reply-reload', source: duplicate, before: null, layouts: ['menu-confirmation', 'wrong', 'missing'] }, + ]; + const evidence: unknown[] = []; + t.after(() => { + if (process.env.STACK_BENCH_CART_NAVIGATION_EVIDENCE) + writeFileSync(process.env.STACK_BENCH_CART_NAVIGATION_EVIDENCE, JSON.stringify(evidence, null, 2) + '\n'); + }); + const browser = await chromium.launch({ headless: true }); + t.after(() => browser.close()); + for (const item of cases) for (const layout of item.layouts) await t.test(`${item.name}/${layout}`, async () => { + const proof = item.before ? item.source.findIndex(s => s.do === 'dbExpectCheckout' && s.before === item.before) : -1; + const start = item.before ? item.source.slice(0, proof).findLastIndex(s => s.do === 'click' && s.testid === 'checkout-submit') + : item.source.findIndex(s => s.do === 'loseCheckoutResponse') + 1; + const count = item.source.findIndex((s, i) => i > start && s.do === 'expectNumber' && s.testid === 'cart-count' && s.equals === 0); + assert(start >= 0 && count > start); + const steps = item.source.slice(start, Math.max(count, proof) + 1); + let committed = item.before === null, loads = 0, writes = 0, toggles = 0; + let recordCommit!: () => void, deliverReply!: () => void; + const commit = new Promise(resolve => { recordCommit = resolve; }); + const reply = new Promise(resolve => { deliverReply = resolve; }); + if (committed) recordCommit(); + const context = await browser.newContext(); + const page = await context.newPage(); + page.setDefaultTimeout(250); + await page.exposeFunction('recordToggle', () => { toggles++; }); + await page.route('http://checkout.test/**', async route => { + if (route.request().method() === 'POST') { + writes++; committed = true; recordCommit(); + if (layout === 'delayed') await reply; + // Reload deliberately abandons the old document's pending reply. + await route.fulfill({ contentType: 'application/json', body: '{}' }); + return; + } + loads++; + const open = !committed || layout === 'open'; + await route.fulfill({ contentType: 'text/html', body: ` + + + +
0 + ${committed && layout === 'missing' ? '' : `${committed && layout !== 'wrong' ? 0 : 1}`} +
+ + ${committed && layout === 'menu-confirmation' ? '' : ''}` }); + }); + try { + await page.goto('http://checkout.test/'); + const actor = { page, loc: (id: string) => page.locator(stableElementSelector(id)).filter({ visible: true }).first() }; + const service = { defaultWithin: 250, expand: (value: string) => value, testId: stableElementSelector, + sleep: (ms: number) => new Promise(resolve => setTimeout(resolve, Math.min(ms, 20))) }; + const capabilities = { actors: { get: () => actor }, 'browser-interaction': service, 'browser-observation': service }; + const actions: ActionEvidence[] = []; + for (const step of steps) { + if (step.do === 'dbExpectCheckout') { await commit; continue; } + if (step.do === 'ensureSignedIn') continue; // The fixture remains signed in on every load. + if (step.do === 'expectNumber' && layout === 'delayed') { + // Hold the reply across any visibility guard. Its redirect races the + // count observation, unless the scenario has replaced the old document. + if (loads === 1) assert.equal(await page.locator('#cart-count').innerText(), '1'); + deliverReply(); + } + const result = await executeAction(ACTION_REGISTRY, step.do, + { ...step, ...(step.testid ? { within: 250 } : {}) }, { capabilities }); + actions.push(result); + if (result.status !== 'passed') break; + } + const expected = ['wrong', 'missing'].includes(layout) ? 'failed' : 'passed'; + evidence.push({ name: item.name, layout, expected, loads, writes, toggles, actions }); + assert.equal(actions.at(-1)?.status, expected, JSON.stringify(evidence.at(-1))); + if (layout === 'open') assert.equal(toggles, 0, 'do not close an already open cart'); + if (expected === 'failed') assert.equal(actions.at(-1)?.action.id, 'expectNumber', 'invalid counts must reach the value assertion'); + assert.equal(writes, item.before === null ? 0 : 1, 'navigation cannot issue another checkout'); + } finally { deliverReply(); await context.close(); } + }); +}); + +test('declared subview openers accept inline content and tabs without accepting broken views', async () => { + const read = (name: string) => compileScenarioDefinition(JSON.parse(readFileSync( + join(STACK_BENCH_ROOT, 'tracks/ecommerce/scenarios', name), 'utf8'))); + const support = read('progression-support-triage.json').features[0]!; + const reviews = read('01-review-visibility.json').features[0]!; + const sales = read('02-operational-category-totals.json').features[0]!; + const cases = [ + { opener: 'support-queue-link', target: 'support-assignee', + step: support.setup.find(step => step.testid === 'support-queue-link')! }, + { opener: 'review-toggle', target: 'review-rating', + step: reviews.criteria[0]!.steps.find(step => step.testid === 'review-toggle')! }, + { opener: 'sales-link', target: 'category-row', + step: sales.criteria[0]!.steps.find(step => step.testid === 'sales-link')! }, + ]; + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage(); + page.setDefaultTimeout(150); + const actor = { page, loc: (id: string) => page.locator(stableElementSelector(id)).filter({ visible: true }).first() }; + const service = { defaultWithin: 150, expand: (value: string) => value, + testId: stableElementSelector, sleep: (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) }; + const capabilities = { actors: { get: () => actor }, 'browser-interaction': service, 'browser-observation': service }; + for (const item of cases) { + assert(item.step, `missing navigation step for ${item.opener}`); + for (const layout of ['inline', 'tab', 'broken'] as const) { + await page.setContent(`${item.opener === 'support-queue-link' ? 'Personal ticket' : ''} +
Expected content
`); + const check = { do: 'expect', actor: item.step.actor, testid: item.target, contains: 'Expected content', within: 100 }; + if (layout === 'tab') assert.equal((await executeAction(ACTION_REGISTRY, 'expect', check, { capabilities })).status, + 'failed', 'the old direct-access assumption fails for a valid closed tab'); + const opened = await executeAction(ACTION_REGISTRY, 'click', item.step, { capabilities }); + assert.equal(opened.status, 'passed', opened.summary ?? undefined); + const result = await executeAction(ACTION_REGISTRY, 'expect', check, { capabilities }); + assert.equal(result.status, layout === 'broken' ? 'failed' : 'passed', `${item.opener}: ${layout}`); + assert.equal(await page.getAttribute('body', 'data-clicked'), layout === 'inline' ? null : 'yes'); + } + } + // The live totals observer must stay on the same view after the purchase. + const live = sales.criteria.find(criterion => criterion.id === '5b')!.steps; + const purchase = live.findIndex(step => step.testid === 'buy-now'); + assert(purchase > 0); + assert(!live.slice(purchase + 1).some(step => step.do === 'click' || step.do === 'reload')); + } finally { await browser.close(); } +}); + +test('a navigation link that renders under an opened account menu is reached through overlay-close', async () => { + const roles = compileScenarioDefinition(JSON.parse(readFileSync( + join(STACK_BENCH_ROOT, 'tracks/ecommerce/scenarios/progression-staff-roles.json'), 'utf8'))).features[0]!; + const steps = roles.criteria.find(criterion => criterion.id === '621d')!.steps; + const reveal = steps.findIndex(step => step.testid === 'current-user' && step.do === 'click'); + const [menu, admin] = [steps[reveal]!, steps[reveal + 1]!]; + assert.equal(admin.testid, 'admin-link'); + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage(); + const actor = { page, loc: (id: string) => page.locator(stableElementSelector(id)).filter({ visible: true }).first() }; + const service = { defaultWithin: 1000, expand: (value: string) => value, + testId: stableElementSelector, sleep: (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) }; + const capabilities = { actors: { get: () => actor }, 'browser-interaction': service, 'browser-observation': service }; + for (const closable of [true, false]) { + // The admin link appears after the role loads, under a full-page account menu. + await page.setContent(` + + + `); + assert.equal((await executeAction(ACTION_REGISTRY, 'click', menu, { capabilities })).status, 'passed'); + await page.waitForSelector('#admin-link'); + const result = await executeAction(ACTION_REGISTRY, 'click', admin, { capabilities }); + assert.equal(result.status, closable ? 'passed' : 'failed', result.summary ?? undefined); + assert.equal(await page.getAttribute('body', 'data-admin'), closable ? '1' : null); + } + } finally { await browser.close(); } +}); + +test('restock setup uses an open form and waits for delayed navigation without accepting broken forms', async () => { + // Failure cases: closing an inline form, skipping a closed/delayed drawer, + // accepting a broken opener or a form blocked by another modal. + const names = ['03-deferred-access.json', '03-deferred-integrity.json', + '03-scheduled-restock-apply.json', '03-scheduled-restock-cancel.json', '03-scheduled-restocks.json']; + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage({ viewport: { width: 800, height: 600 } }); + page.setDefaultTimeout(250); + const actor = { page, loc: (id: string) => page.locator(stableElementSelector(id)).filter({ visible: true }).first() }; + let sleeps = 0; + const service = { defaultWithin: 700, expand: (value: string) => value, testId: stableElementSelector, + sleep: async (ms: number) => { sleeps++; await new Promise(resolve => setTimeout(resolve, ms)); } }; + const capabilities = { actors: { get: () => actor }, 'browser-interaction': service }; + for (const name of names) { + const feature = compileScenarioDefinition(JSON.parse(readFileSync(join(STACK_BENCH_ROOT, + 'tracks/ecommerce/scenarios', name), 'utf8'))).features[0]!; + const step = feature.setup.find(step => step.testid === 'restocks-link')!; + for (const layout of ['inline', 'delayed-inline', 'tab', 'delayed-tab', 'drawer', 'broken', 'blocked']) { + await page.setContent(` + ${layout.endsWith('inline') || layout === 'blocked' ? '' : ``} +
+
+ ${layout === 'blocked' ? 'Blocked' : ''} + `); + sleeps = 0; + const opened = await executeAction(ACTION_REGISTRY, 'click', step, { capabilities }); + assert.equal(opened.status, 'passed', `${name}: ${layout}: ${opened.summary}`); + if (layout === 'inline') assert.equal(sleeps, 0, 'an already open form needs no optional-link wait'); + const filled = await executeAction(ACTION_REGISTRY, 'fill', + { do: 'fill', actor: 'admin', testid: 'schedule-restock-item', text: 'Webcam' }, { capabilities }); + const submitted = filled.status !== 'passed' ? filled : await executeAction(ACTION_REGISTRY, 'click', + { do: 'click', actor: 'admin', testid: 'schedule-restock-submit', within: 250 }, { capabilities }); + const invalid = layout === 'broken' || layout === 'blocked'; + assert.equal(submitted.status, invalid ? 'failed' : 'passed', `${name}: ${layout}: ${submitted.summary}`); + assert.equal(await page.getAttribute('body', 'data-submitted'), invalid ? null : 'yes'); + assert.equal(await page.getAttribute('body', 'data-clicked'), + ['tab', 'delayed-tab', 'drawer', 'broken'].includes(layout) ? 'yes' : null); + } + } + } finally { await browser.close(); } +}); + +test('promotions follow the staff path and delivery setup returns from persistent settings', async () => { + const read = (name: string) => compileScenarioDefinition(JSON.parse(readFileSync( + join(STACK_BENCH_ROOT, 'tracks/ecommerce/scenarios', name), 'utf8'))).features[0]!; + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage(); + page.setDefaultTimeout(150); + const actor = { page, loc: (id: string) => page.locator(stableElementSelector(id)).filter({ visible: true }).first() }; + const service = { defaultWithin: 150, expand: (value: string) => value, + testId: stableElementSelector, sleep: (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) }; + const capabilities = { actors: { get: () => actor }, 'browser-interaction': service }; + for (const file of ['progression-promotion-checkout.json', 'progression-promotion-reporting.json']) { + await page.setContent(` + `); + const steps = read(file).setup.slice(1, 4); + assert.deepEqual(steps.map(step => step.testid), ['staff-link', 'promotions-link', 'promotion-code']); + const old = await executeAction(ACTION_REGISTRY, 'click', { do: 'click', actor: 'staff', testid: 'promotions-link' }, { capabilities }); + assert.equal(old.status, 'failed'); + for (const step of steps) assert.equal((await executeAction(ACTION_REGISTRY, step.do, step, { capabilities })).status, 'passed'); + assert.notEqual(await page.locator('#promotion-code').inputValue(), ''); + } + const delivery = read('progression-delivery-notifications.json').setup; + const saved = delivery.findIndex(step => step.testid === 'notification-save'); + assert.deepEqual(delivery.slice(saved + 1, saved + 3).map(step => step.testid), ['overlay-close', 'catalog-link']); + for (const overlay of [false, true]) { + await page.setContent(` + +
+ ${overlay ? '' : ''}
`); + const buy = { do: 'click', actor: 'owner', testid: 'buy-now' }; + assert.equal((await executeAction(ACTION_REGISTRY, 'click', buy, { capabilities })).status, 'failed'); + for (const step of delivery.slice(saved + 1, saved + 3)) + assert.equal((await executeAction(ACTION_REGISTRY, 'click', step, { capabilities })).status, 'passed'); + assert.equal((await executeAction(ACTION_REGISTRY, 'click', buy, { capabilities })).status, 'passed'); + assert.equal(await page.getAttribute('body', 'data-bought'), 'yes'); + } + } finally { await browser.close(); } +}); + +test('conditional navigation opens closed drawers and preserves inline or animated open panels', async () => { + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage({ viewport: { width: 800, height: 600 } }); + for (const layout of ['closed-drawer', 'open-drawer', 'animated-open', 'below-fold', 'clipped'] as const) { + await page.setContent(` +
Keyboard
`); + const actor = { page, loc: (id: string) => page.locator(stableElementSelector(id)).filter({ visible: true }).first() }; + assert.equal(await actor.loc('order-item').isVisible(), true, 'all layouts reproduce Playwright visibility'); + const result = await executeAction(ACTION_REGISTRY, 'click', { + do: 'click', actor: 'customer', testid: 'orders-toggle', unlessVisible: 'order-item', within: 1000, + }, { capabilities: { actors: { get: () => actor }, 'browser-interaction': { + defaultWithin: 1000, expand: (value: string) => value, testId: stableElementSelector, + } } }); + assert.equal(result.status, 'passed', result.summary ?? undefined); + assert.equal(await page.locator('body').getAttribute('data-clicked'), ['closed-drawer', 'clipped'].includes(layout) ? 'true' : null); + await actor.loc('cancel-order').click({ timeout: 1000 }); + if (layout === 'below-fold') assert((await page.locator('#panel').boundingBox())!.y < 600, 'inline content was scrolled into view'); + } + // After a reload a browser can still be animating to the restored scroll position when the + // check runs. A link at the top of the page is there to click once the page holds still. + await page.setContent(`
+ Admin
`); + await page.evaluate(() => { + let top = 0; + const step = () => { scrollTo({ top: top += 60, behavior: 'instant' }); if (top < 1500) requestAnimationFrame(step); }; + requestAnimationFrame(step); + }); + await page.waitForFunction(() => scrollY > 100); + const moving = { page, loc: (id: string) => page.locator(stableElementSelector(id)).filter({ visible: true }).first() }; + const reached = await executeAction(ACTION_REGISTRY, 'click', { + do: 'click', actor: 'admin', testid: 'current-user', ifAvailable: true, unlessVisible: 'admin-link', + }, { capabilities: { actors: { get: () => moving }, 'browser-interaction': { + defaultWithin: 5000, expand: (value: string) => value, testId: stableElementSelector, + } } }); + assert.deepEqual(reached.observation, { clicked: false, testid: 'current-user', visible: 'admin-link' }); + assert.equal(await page.locator('body').getAttribute('data-clicked'), null); + // The staff role check after reload keeps an open role panel visible and opens a closed one. + const source = join(STACK_BENCH_ROOT, 'tracks/ecommerce/scenarios/progression-staff-roles.json'); + const roles = compileScenarioDefinition(JSON.parse(readFileSync(source, 'utf8')), { source }).features[0]! + .criteria.find(criterion => criterion.id === '621a')!; + const entry = roles.steps.findIndex(step => step.testid === 'admin-link'); + assert(entry >= 0); + for (const open of [true, false]) { + await page.setContent(` +
+
`); + const actor = { page, loc: (id: string, options?: { scope?: { testid: string } }) => { + const root = options?.scope ? page.locator(stableElementSelector(options.scope.testid)) : page; + return root.locator(stableElementSelector(id)); + } }; + const capability = { defaultWithin: 300, expand: (value: string) => value, + testId: stableElementSelector, sleep: async () => {} }; + for (const step of roles.steps.slice(entry, entry + 2)) { + const result = await executeAction(ACTION_REGISTRY, step.do, + { ...step, ...(step.do === 'expect' ? { within: 300 } : {}) }, + { capabilities: { actors: { get: () => actor }, 'browser-interaction': capability, + 'browser-observation': capability } }); + assert.equal(result.status, 'passed', `621a open=${open}: ${result.summary}`); + } + } + } finally { await browser.close(); } +}); + +test('return observation accepts a line marker only within the selected order', async () => { + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage(); + for (const returned of [false, true]) { + await page.setContent(`
Keyboard + shipped${returned ? 'Returned' : ''}
+
Desk Lamp Returned
`); + const actor = { page, loc: (id: string, options?: { contains?: string }) => + page.locator(stableElementSelector(id)).filter({ hasText: options?.contains, visible: true }).first() }; + const result = await executeAction(ACTION_REGISTRY, 'expect', { + do: 'expect', actor: 'customer', testid: 'order-item', contains: 'Keyboard', + containsText: 'returned', ignoreCase: true, within: 50, + }, { capabilities: { actors: { get: () => actor }, 'browser-observation': { + defaultWithin: 50, expand: (value: string) => value, testId: stableElementSelector, + sleep: (ms: number) => page.waitForTimeout(ms), + } } }); + assert.equal(result.status, returned ? 'passed' : 'failed', result.summary ?? undefined); + } + } finally { await browser.close(); } +}); + +test('purchase attribution requires a working private history, not a blank view', async () => { + const source = join(STACK_BENCH_ROOT, 'tracks/ecommerce/scenarios/01-purchase-attribution.json'); + const feature = compileScenarioDefinition(JSON.parse(readFileSync(source, 'utf8')), { source }).features[0]!; + const steps = feature.criteria[0]!.steps.filter(step => step.actor === 'victim' && step.do === 'expect'); + assert.equal(steps.length, 2); + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage(); + for (const items of [[], ['Coffee Grinder'], ['Coffee Grinder', 'Desk Lamp']]) { + await page.setContent(items.map(item => `
${item}
`).join('')); + const actor = { page, loc: (id: string, options?: { contains?: string }) => page.locator(stableElementSelector(id), + { hasText: options?.contains }).filter({ visible: true }).first() }; + const results = []; + for (const step of steps) results.push(await executeAction(ACTION_REGISTRY, 'expect', { ...step, within: 50 }, { capabilities: { + actors: { get: () => actor }, 'browser-observation': { defaultWithin: 50, + expand: (value: string) => value, testId: stableElementSelector, + sleep: async () => new Promise(resolve => setTimeout(resolve, 1)) }, + } })); + assert.equal(results.every(result => result.status === 'passed'), items.length === 1, + `history ${JSON.stringify(items)} must pass only when own order is visible and other order is absent`); + } + } finally { await browser.close(); } +}); + +test('role assignment targets the account ID despite role text in every dropdown', async () => { + const source = join(STACK_BENCH_ROOT, 'tracks/ecommerce/scenarios/progression-staff-roles.json'); + const feature = compileScenarioDefinition(JSON.parse(readFileSync(source, 'utf8')), { source }).features[0]!; + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage(); + await page.setContent(['customer', 'staff'].map((name, index) => ` +
+ ${name} + +
`).join('')); + assert.equal(await page.locator('[data-role="staff-role-row"]').filter({ hasText: 'staff' }).count(), 2, + 'the fixture must reproduce the ambiguous dropdown text'); + const actor = { page, loc: (id: string, options?: { contains?: string; scope?: { testid: string; contains?: string } }) => { + const root = options?.scope + ? page.locator(stableElementSelector(options.scope.testid), { hasText: options.scope.contains }).first() : page; + return root.locator(stableElementSelector(id), { hasText: options?.contains }).first(); + } }; + const capabilities = { actors: { get: () => actor }, 'browser-interaction': { + defaultWithin: 1000, expand: (value: string) => value, sleep: async () => {}, testId: stableElementSelector, + } }; + for (const step of feature.setup.filter(step => step.do === 'fill' || step.testid === 'staff-role-save')) { + const result = await executeAction(ACTION_REGISTRY, step.do, step, { capabilities }); + assert.equal(result.status, 'passed', result.summary ?? undefined); + } + assert.equal(await page.locator('#staff-role-account-staff').getAttribute('data-saved'), 'inventory'); + assert.equal(await page.locator('#staff-role-account-customer').getAttribute('data-saved'), null); + const replay = feature.criteria.flatMap(criterion => criterion.steps).find(step => step.do === 'replayAs')!; + const target = replay.namedTarget as { testid: string; contains?: string; attribute: string }; + assert.equal(target.contains, undefined); + assert.equal(await actor.loc(target.testid).getAttribute(target.attribute), '2'); + } finally { await browser.close(); } +}); + +test('signup and signin reach hidden, direct and shared-dialog forms but reject missing hooks and failed signup', async () => { + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage(); + for (const [action, layout] of [['signUp', 'inline'], ['signUp', 'direct'], ['signUp', 'shared-dialog'], + ['signUp', 'missing-hook'], ['signUp', 'rejected'], ['signIn', 'hidden'], ['signIn', 'inline']] as const) { + const revealed = action === 'signUp' && layout !== 'inline'; + await page.setContent(` +
+ + +
+
+ + +
+ ${revealed ? `` : ''} + + + `); + const actor = { page, loc: (id: string) => page.locator(`#${id}`) }; + const result = await executeAction(ACTION_REGISTRY, action, + { do: action, actor: 'shopper', name: 'Alice' }, { + capabilities: { + actors: { get: () => actor }, + 'browser-interaction': { defaultWithin: 300, scopedUser: (name: string) => `${name}-scope`, + testId: (id: string) => `#${id}` }, + }, + }); + if (layout === 'missing-hook' || layout === 'rejected') { + assert.equal(result.status, 'failed', layout); + assert.equal(await page.locator('#current-user').isVisible(), false); + continue; + } + assert.equal(result.status, 'passed', result.summary ?? JSON.stringify(result)); + assert.equal((result.observation as { authenticationPath: string }).authenticationPath, 'local-form'); + assert.equal(await page.locator('#current-user').innerText(), 'Alice-scope'); + assert.equal(await page.locator(action === 'signUp' ? '#signup-password' : '#signin-password').inputValue(), 'pw-Alice-scope'); + assert.equal(await page.evaluate(() => Reflect.get(window, 'signInClicks')), + layout === 'shared-dialog' || layout === 'hidden' ? 1 : 0); + } + } finally { await browser.close(); } +}); + +// Failure cases precede the signup-flow correction: registration need not create a session. +// A later login must use the same saved credentials, and must not erase a refused signup. +test('account creation accepts explicit login without masking refused or wrong-account signup', async () => { + const browser = await chromium.launch({ headless: true }); + let mode = 'auto', signupPending = false; + const accounts = new Map(); + const requests: { path: string; username: string; role?: string; signupPending: boolean }[] = []; + const evidence: unknown[] = []; + const server = createServer(async (request, response) => { + response.setHeader('Content-Type', 'application/json'); + if (request.method === 'POST') { + let text = ''; for await (const chunk of request) text += String(chunk); + const body = JSON.parse(text) as { username: string; password: string; role?: string }; + requests.push({ path: request.url!, username: body.username, role: body.role, signupPending }); + if (request.url === '/signup') { + signupPending = true; + if (mode === 'delayed-manual') await new Promise(resolve => setTimeout(resolve, 150)); + signupPending = false; + if (mode === 'patched-silent-refusal') { response.writeHead(409).end('{}'); return; } + if (mode === 'reject' || mode === 'rejected-observed' || mode === 'patched-reject' || accounts.has(body.username)) { + response.writeHead(409).end(JSON.stringify({ error: 'Registration refused' })); return; + } + if (mode === 'missing') { response.end('{}'); return; } + accounts.set(body.username, { password: body.password, role: body.role ?? 'customer' }); + response.end(JSON.stringify(mode === 'auto' || mode === 'wrong-account' + ? { user: mode === 'wrong-account' ? 'other-account' : body.username } + : mode === 'missing-identity-observed' ? { activeWithoutIdentity: true } : {})); return; + } + if (request.url === '/signin') { + const account = accounts.get(body.username); + if (!account || account.password !== body.password) { + response.writeHead(401).end(JSON.stringify({ error: 'Invalid credentials' })); return; + } + response.end(JSON.stringify({ user: body.username })); return; + } + if (request.url === '/privileged') { + const account = accounts.get(body.username); + response.writeHead(account?.role === 'admin' ? 200 : 403).end('{}'); return; + } + } + response.setHeader('Content-Type', 'text/html'); + response.end(`
+
+ +
+
+ + `); + }); + try { + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + for (mode of ['patched-silent-refusal', 'missing-identity-observed', 'auto', 'manual', 'delayed-manual', 'reject', 'missing', 'duplicate', 'wrong-account', + 'manual-observed', 'rejected-observed', 'patched-manual', 'patched-reject', 'restored-correct-account', 'restored-wrong-account']) { + accounts.clear(); requests.length = 0; signupPending = false; + if (mode === 'duplicate') accounts.set('Alice-scope', { password: 'fixture-password', role: 'customer' }); + const page = await browser.newPage(); + try { + await page.goto(url); + const actor = { page, loc: (id: string, options?: { contains?: string }) => { + const locator = page.locator(`#${id}`); + return options?.contains ? locator.filter({ hasText: options.contains }) : locator; + } }; + const interaction = { + defaultWithin: 250, scopedUser: (name: string) => `${name}-scope`, testId: (id: string) => `#${id}`, + expand: (value: unknown) => value, + sleep: (ms: number) => new Promise(resolve => setTimeout(resolve, ms)), + }; + const capabilities = { actors: { get: () => actor }, 'browser-interaction': interaction, + 'browser-observation': interaction }; + const result = await executeAction(ACTION_REGISTRY, 'signUp', { + do: 'signUp', actor: 'shopper', name: 'Alice', password: 'fixture-password', + ...(mode === 'duplicate' ? { expectFailure: true } : {}), + ...(mode.endsWith('-observed') ? { awaitSignedIn: false } : {}), + ...(mode.startsWith('patched-') ? { requestPatch: { fields: { role: 'admin' } } } : {}), + }, { capabilities }); + // expectFailure submits only; observe the actual refusal before reading server receipts. + if (mode === 'duplicate') await page.locator('#auth-error').waitFor({ state: 'visible', timeout: 1000 }); + const loginRequests = requests.filter(r => r.path === '/signin'); + const observation = { mode, result, requests: [...requests], accountCreated: accounts.has('Alice-scope'), + currentUser: await page.locator('#current-user').innerText(), privilegedStatus: null as number | null }; + evidence.push(observation); + assert.equal(requests.filter(r => r.path === '/signup').length, 1, `${mode}: do not retry registration`); + assert.equal(loginRequests.some(r => r.signupPending), false, `${mode}: visible login form is not signup completion`); + if (mode.startsWith('restored-')) { + assert.equal(accounts.get('Alice-scope')?.password, 'fixture-password'); + assert.equal(loginRequests.length, 0, 'a restored session must not submit another login'); + assert.equal(observation.currentUser, mode === 'restored-wrong-account' ? 'other-account' : 'Alice-scope'); + assert.equal(result.status, mode === 'restored-wrong-account' ? 'failed' : 'passed', + `restored identity is an app observation, not a harness exception: ${JSON.stringify(result)}`); + } else if (mode === 'patched-silent-refusal') { + assert.equal(result.status, 'failed', 'a known rejected patch must not gain an error observation from a later login'); + assert.equal(accounts.size, 0); + assert.equal(loginRequests.length, 0, 'never sign in after a proved HTTP registration refusal'); + assert.equal(await page.locator('#auth-error').isVisible(), false); + } else if (mode === 'reject' || mode === 'missing') { + assert.equal(result.status, 'failed', `${mode}: ${JSON.stringify(result)}`); + assert.equal(accounts.size, 0); + assert.equal(loginRequests.length, mode === 'reject' ? 0 : 1); + } else if (mode === 'missing-identity-observed') { + assert.equal(result.status, 'passed', JSON.stringify(result)); + assert.equal(accounts.get('Alice-scope')?.password, 'fixture-password'); + assert.equal(loginRequests.length, 0, 'no sign-in navigation when the app exposes no sign-in control'); + assert.equal(await page.locator('#signin-username').isVisible(), false); + const identity = await executeAction(ACTION_REGISTRY, 'expect', { + do: 'expect', actor: 'shopper', testid: 'current-user', contains: 'Alice-scope', within: 100, + }, { capabilities }); + assert.equal(identity.status, 'failed', 'missing active identity must fail at the scored observation'); + } else if (mode === 'rejected-observed') { + assert.equal(result.status, 'passed', JSON.stringify(result)); + assert.equal(accounts.size, 0); + assert.equal(loginRequests.length, 0); + const identity = await executeAction(ACTION_REGISTRY, 'expect', { + do: 'expect', actor: 'shopper', testid: 'current-user', contains: 'Alice-scope', within: 100, + }, { capabilities }); + assert.equal(identity.status, 'failed', 'signup refusal remains caught by the scored identity observation'); + } else if (mode === 'duplicate' || mode === 'patched-reject') { + assert.equal(result.status, 'passed', `${mode}: ${JSON.stringify(result)}`); + assert.equal(loginRequests.length, 0, `${mode}: refusal must not be followed by login`); + assert.equal(await page.locator('#auth-error').isVisible(), true); + assert.equal(await page.locator('#current-user').isVisible(), false); + if (mode === 'patched-reject') assert.equal(accounts.size, 0); + } else { + assert.equal(result.status, 'passed', `${mode}: ${JSON.stringify(result)}`); + assert.equal(accounts.get('Alice-scope')?.password, 'fixture-password'); + assert.equal(loginRequests.length, mode === 'auto' || mode === 'wrong-account' ? 0 : 1); + const identity = await executeAction(ACTION_REGISTRY, 'expect', { + do: 'expect', actor: 'shopper', testid: 'current-user', contains: 'Alice-scope', within: 100, + }, { capabilities }); + assert.equal(identity.status, mode === 'wrong-account' ? 'failed' : 'passed', JSON.stringify(identity)); + if (mode === 'patched-manual') { + assert.equal(requests.find(r => r.path === '/signup')?.role, 'admin'); + assert.equal(loginRequests[0]?.role, undefined, 'patch must be removed before ordinary login'); + assert.ok((result.observation as { requestPatch?: unknown }).requestPatch, 'retain original interception receipt'); + observation.privilegedStatus = await page.evaluate(async () => (await fetch('/privileged', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ username: 'Alice-scope' }), + })).status); + assert.equal(observation.privilegedStatus, 200, 'a trusted signup role remains observable by the later authority probe'); + } + } + } finally { await page.close(); } + } + } finally { + await browser.close(); + await new Promise(resolve => server.close(() => resolve())); + if (process.env.STACK_BENCH_SIGNUP_EVIDENCE) { + const { writeFileSync } = await import('node:fs'); + writeFileSync(process.env.STACK_BENCH_SIGNUP_EVIDENCE, JSON.stringify({ + rerun: 'STACK_BENCH_SIGNUP_EVIDENCE= node --test --test-name-pattern="account creation accepts explicit login" dist/tests/account-navigation.integration.js', + evidence, + }, null, 2)); + } + } +}); + +test('mixed history reads the committed cart before preparing its next server operation', async t => { + // Direct harness writes bypass UI callbacks. These real scenario slices must + // refresh the document before using cart rows as input to the next operation. + const source = join(STACK_BENCH_ROOT, 'tracks/ecommerce/scenarios/mixed-operation-history.json'); + const all = compileScenarioDefinition(JSON.parse(readFileSync(source, 'utf8'))).features[0]!.criteria[0]!.steps; + const reads = all.flatMap((step, index) => step.do === 'expectNumber' && step.testid === 'cart-quantity' ? [index] : []); + assert.equal(reads.length, 3); + const browser = await chromium.launch({ headless: true }); + const evidence: unknown[] = []; + try { + for (const [position, end] of reads.entries()) for (const layout of ['correct', 'confirmation', 'restored-cart', 'wrong', 'missing', 'restored-missing']) { + await t.test(`cart preparation ${position + 1}/${layout}`, async () => { + const start = all.slice(0, end).findLastIndex(step => step.do === 'dbExpectOperation') + 1; + const steps = all.slice(start, end + 1); + const actorName = all[end]!.actor!; + const user = actorName === 'b' ? 'history-17-b' : 'history-17-a'; + let stored = 0, loads = 0, writes = 0; + const page = await browser.newPage(); + const actions: ActionEvidence[] = []; + try { + await page.route('http://history-cart.test/**', async route => { + if (route.request().method() === 'POST') { + writes++; stored = layout === 'wrong' ? 2 : 1; + await route.fulfill({ contentType: 'application/json', body: '{}' }); + return; + } + loads++; + const restored = stored > 0 && ['restored-cart', 'restored-missing'].includes(layout); + const confirmation = stored > 0 && layout === 'confirmation'; + const cartTag = restored ? 'dialog' : 'section'; + await route.fulfill({ contentType: 'text/html', body: ` + ${user} + + <${cartTag} id="cart" ${restored ? '' : 'hidden'}>64 + ${stored && !['missing', 'restored-missing'].includes(layout) ? `
Coffee Grinder
` : 'Empty cart'} + ${restored ? '' : ''} + + ${confirmation ? '' : ''} + ${restored || confirmation ? `` : ''}` }); + }); + await page.goto('http://history-cart.test/'); + // This is a real HTTP write, with no app event handler to update its cache. + await page.evaluate(async () => { await fetch('/cart-add', { method: 'POST' }); }); + assert.equal(stored, layout === 'wrong' ? 2 : 1); + const actor = { page, name: actorName, + loc: (id: string, options?: { contains?: string; scope?: { testid: string; contains?: string } }) => { + const root = options?.scope ? page.locator(stableElementSelector(options.scope.testid)) + .filter({ hasText: options.scope.contains }).first() : page; + const loc = root.locator(stableElementSelector(id)).filter({ visible: true }); + return (options?.contains ? loc.filter({ hasText: options.contains }) : loc).first(); + } }; + const service = { defaultWithin: 250, testId: stableElementSelector, + scopedUser: (name: string) => name, expand: (value: string) => value, + sleep: (ms: number) => new Promise(resolve => setTimeout(resolve, Math.min(ms, 20))) }; + const capabilities = { actors: { get: () => actor }, + 'browser-interaction': service, 'browser-observation': service }; + for (const step of steps) { + const result = await executeAction(ACTION_REGISTRY, step.do, + { ...step, ...(step.testid ? { within: 250 } : {}) }, { capabilities }); + actions.push(result); + if (result.status !== 'passed') break; + } + assert.equal(actions.at(-1)?.action.id, 'expectNumber', 'reach the original cart quantity assertion'); + assert.equal(actions.at(-1)?.status, ['wrong', 'missing', 'restored-missing'].includes(layout) ? 'failed' : 'passed'); + assert.equal(writes, 1, 'observation must not add another cart item'); + } finally { + await page.close(); + evidence.push({ position: position + 1, layout, steps, expected: ['wrong', 'missing', 'restored-missing'].includes(layout) ? 'failed' : 'passed', + stored, loads, writes, actions, pageClosed: page.isClosed() }); + } + }); + } + } finally { + await browser.close(); + if (process.env.STACK_BENCH_HISTORY_CART_EVIDENCE) { + const { createHash } = await import('node:crypto'); + const hash = (file: string | URL) => createHash('sha256').update(readFileSync(file)).digest('hex'); + writeFileSync(process.env.STACK_BENCH_HISTORY_CART_EVIDENCE, JSON.stringify({ + rerun: 'STACK_BENCH_HISTORY_CART_EVIDENCE= node --test --test-name-pattern="mixed history reads" dist/tests/account-navigation.integration.js', + fixture: 'Server cart read only on document load; direct POST commits quantity 1 or 2. Closed cart, restored cart, restored confirmation, and missing row variants.', + source, sourceSha256: hash(source), testSha256: hash(new URL(import.meta.url)), + driverSha256: hash(new URL('../src/actions/browser-action-executors.js', import.meta.url)), + browserClosed: !browser.isConnected(), evidence, + }, null, 2) + '\n'); + } + } +}); + +test('restored operational dialogs reach the stored value and order assertions', async t => { + const browser = await chromium.launch({ headless: true }); + const evidence: unknown[] = []; + try { + for (const kind of ['admin', 'staff', 'orders'] as const) { + const source = join(STACK_BENCH_ROOT, 'tracks/ecommerce/scenarios', + kind === 'admin' ? '01-restock-race.json' : '02-fulfilment-ship.json'); + const feature = compileScenarioDefinition(JSON.parse(readFileSync(source, 'utf8')), { source }).features[0]!; + const all = kind === 'admin' ? feature.setup! : feature.criteria[0]!.steps; + const actorName = kind === 'orders' ? 'customer' : kind; + const start = all.findIndex(step => step.do === 'reload' && step.actor === actorName); + const observation = kind === 'admin' ? 'admin-stock' : kind === 'staff' ? 'queue-item' : 'order-status'; + const end = all.findIndex((step, index) => index > start && step.actor === actorName && step.testid === observation); + assert.ok(start >= 0 && end > start, 'the real scenario must contain the reload and readback'); + const reloadSteps = all.slice(start, end + 1).filter(step => step.actor === actorName); + const initial = feature.setup.filter(step => step.actor === actorName); + const signedIn = initial.findIndex(step => step.do === 'signIn'); + const opened = initial.findIndex((step, index) => index > signedIn + && step.do === 'click' && step.testid === `${kind}-link`); + for (const phase of kind === 'orders' ? ['reload'] : ['reload', 'sign-in-result']) { + // Sign-in may render the destination directly. Exercise its actual setup + // entry pair, then the same real readback used by the reload case. + const steps = phase === 'reload' ? reloadSteps + : [...initial.slice(signedIn + 1, opened + 1), all[end]!]; + if (phase !== 'reload') assert.ok(signedIn >= 0 && opened > signedIn); + for (const defective of [false, true]) await t.test(`${kind}/${phase}: ${defective ? 'wrong stored value' : 'correct restored dialog'}`, async () => { + const page = await browser.newPage(); + const actions: ActionEvidence[] = []; + try { + // The route restores an open native dialog. Required entry links and the + // account are still present behind it; the target surface is ready inside. + const content = kind === 'admin' + ? `
Bluetooth Speaker${defective ? 100 : 99}
` + : kind === 'staff' + ? `
${defective ? 'Different item' : 'Keyboard'}
` + : `
Keyboard${defective ? 'pending' : 'shipped'}
`; + await page.route('http://restored-tools.test/**', route => route.fulfill({ contentType: 'text/html', body: ` + + + + ${content} + ` })); + await page.goto('http://restored-tools.test/'); + const actor = { page, name: actorName, + loc: (id: string, options?: { contains?: string; scope?: { testid: string; contains?: string } }) => { + const root = options?.scope ? page.locator(stableElementSelector(options.scope.testid)) + .filter({ hasText: options.scope.contains }).first() : page; + const loc = root.locator(stableElementSelector(id)).filter({ visible: true }); + return (options?.contains ? loc.filter({ hasText: options.contains }) : loc).first(); + } }; + const capability = { defaultWithin: 200, testId: stableElementSelector, + scopedUser: (name: string) => name, expand: (value: string) => value, + recorded: new Map([['stored-before-serial-purchase', 100]]), + sleep: async () => {} }; + const capabilities = { actors: { get: () => actor }, + 'browser-interaction': capability, 'browser-observation': capability }; + let last: ActionEvidence | undefined; + for (const [index, step] of steps.entries()) { + last = await executeAction(ACTION_REGISTRY, step.do, + { ...step, ...(step.testid ? { within: 200 } : {}) }, { capabilities }); + actions.push(last); + if (index < steps.length - 1) assert.equal(last.status, 'passed', + `${kind} must reach its readback; ${step.do}/${step.testid ?? ''}: ${last.summary}`); + if (last.status !== 'passed') break; + } + assert.equal(last?.status, defective ? 'failed' : 'passed', last?.summary ?? undefined); + } finally { + await page.close(); + const { createHash } = await import('node:crypto'); + evidence.push({ kind, phase, defective, source, sourceSha256: createHash('sha256') + .update(readFileSync(source)).digest('hex'), steps, expected: defective ? 'failed' : 'passed', + actions, pageClosed: page.isClosed() }); + } + }); + } + } + } finally { + await browser.close(); + if (process.env.STACK_BENCH_OPERATIONAL_NAVIGATION_EVIDENCE) { + const { writeFileSync } = await import('node:fs'); + const { createHash } = await import('node:crypto'); + writeFileSync(process.env.STACK_BENCH_OPERATIONAL_NAVIGATION_EVIDENCE, JSON.stringify({ + rerun: 'STACK_BENCH_OPERATIONAL_NAVIGATION_EVIDENCE= node --test --test-name-pattern="restored operational dialogs" dist/tests/account-navigation.integration.js', + fixture: 'Native restored dialog; admin stock 99/100, fulfilment Keyboard/Different item, order shipped/pending', + testSha256: createHash('sha256').update(readFileSync(new URL(import.meta.url))).digest('hex'), + driverSha256: createHash('sha256').update(readFileSync(new URL('../src/actions/browser-action-executors.js', import.meta.url))).digest('hex'), + browserClosed: !browser.isConnected(), evidence, + }, null, 2)); + } + } +}); diff --git a/tools/stack-bench/tests/action-contract.contract.ts b/tools/stack-bench/tests/action-contract.contract.ts new file mode 100644 index 00000000000..12b825d081d --- /dev/null +++ b/tools/stack-bench/tests/action-contract.contract.ts @@ -0,0 +1,247 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { ACTION_REGISTRY, actionPlugin } from '../src/actions/action-catalog.js'; +import { + ActionApplicationFailure, + ActionHarnessFailure, + ActionInconclusive, + createActionRegistry, + executeAction, +} from '../src/actions/action-contract.js'; +import type { + ActionImplementation, + ActionPlugin, +} from '../src/actions/action-contract.js'; +import { ACTION_IDS } from '../src/composition/definition-compiler.js'; + +interface FixtureStep { + do: string; + branches?: FixtureStep[][]; +} + +interface FixtureCriterion { + steps: FixtureStep[]; +} + +interface FixtureFeature { + setup: FixtureStep[]; + criteria: FixtureCriterion[]; +} + +interface FixtureDefinition { + features: FixtureFeature[]; +} + +const fixture = JSON.parse(readFileSync( + join(STACK_BENCH_ROOT, 'tests', 'fixtures', 'definitions', 'all-actions.json'), + 'utf8')) as FixtureDefinition; + +function steps(value: FixtureDefinition): FixtureStep[] { + const out: FixtureStep[] = []; + const visit = (step: FixtureStep): void => { + out.push(step); + (step.branches ?? []).flat().forEach(visit); + }; + for (const feature of value.features) { + [...feature.setup, ...feature.criteria.flatMap(criterion => criterion.steps)].forEach(visit); + } + return out; +} + +function plugin(overrides: Partial = {}): ActionPlugin { + return { + id: 'fakeAction', version: '1.0.0', + category: 'transport', + compile: (input: unknown) => { + if (typeof input !== 'object' || input === null || !('do' in input) + || input.do !== 'fakeAction') throw new Error('wrong fake input'); + return structuredClone(input); + }, + capabilities: ['clock'], + timeoutMs: 250, + sensitivity: ['public'], + execute: () => ({ ok: true }), + ...overrides, + }; +} + +function context(overrides: Record = {}) { + return { capabilities: { clock: {}, hidden: 'must not leak' }, ...overrides }; +} + +function registry(implementation: ActionImplementation, overrides: Partial = {}) { + return createActionRegistry([plugin({ ...overrides, execute: implementation })]); +} + +test('every scenario action has a complete versioned runtime contract', () => { + assert.deepEqual(ACTION_REGISTRY.ids, ACTION_IDS); + const representative = new Map(steps(fixture).map(step => [step.do, step])); + for (const id of ACTION_IDS) { + const action = ACTION_REGISTRY.get(id); + assert.match(action.version, /^\d+\.\d+\.\d+$/); + assert(action.capabilities.length > 0, id); + assert(action.timeoutMs > 0, id); + assert(action.sensitivity.every(value => typeof value === 'string'), id); + assert.doesNotThrow(() => action.compile(representative.get(id)), id); + } + // Credential sensitivity drives evidence redaction. + for (const id of ['ensureSignedIn', 'signIn', 'signUp']) { + assert(ACTION_REGISTRY.get(id).sensitivity.includes('credential'), id); + } +}); + +test('duplicate, unknown, malformed, and incomplete registrations fail at startup', () => { + assert.throws(() => createActionRegistry([plugin(), plugin()]), /duplicate action registration/); + assert.throws(() => createActionRegistry([plugin()], { expectedIds: ['fakeAction', 'missing'] }), + /missing missing/); + assert.throws(() => createActionRegistry([plugin(), { ...plugin(), id: 'surprise' }], + { expectedIds: ['fakeAction'] }), /unknown surprise/); + assert.throws(() => ACTION_REGISTRY.get('notReal'), /unknown registered action/); + assert.throws(() => actionPlugin('notReal'), /unknown action/); +}); + +test('a successful action sees only declared capabilities and returns structured evidence', async () => { + const result = await executeAction(registry(({ capabilities }) => { + assert.deepEqual(Object.keys(capabilities), ['clock']); + return { ok: true, value: 7 }; + }), 'fakeAction', { do: 'fakeAction' }, context()); + assert.equal(result.status, 'passed'); + assert.equal(result.code, 'completed'); + assert.deepEqual(result.observation, { ok: true, value: 7 }); + assert.equal(result.action.id, 'fakeAction'); + assert.equal(result.timing.deadlineMs, 250); +}); + +test('action evidence remains coherent if an injected wall clock moves backward', async () => { + const ticks = [100, 95]; + const result = await executeAction(registry(() => ({ ok: true })), 'fakeAction', + { do: 'fakeAction' }, context(), { now: () => ticks.shift() ?? 0 }); + assert.deepEqual(result.timing, { + startedAtMs: 100, + completedAtMs: 100, + durationMs: 0, + deadlineMs: 250, + }); +}); + +test('application failure and inconclusive results stay distinct', async () => { + const failed = await executeAction(registry(() => { + throw new ActionApplicationFailure('subject rejected assertion', + { observation: { actual: 2 }, expected: { value: 1 } }); + }), 'fakeAction', { do: 'fakeAction' }, context()); + assert.equal(failed.status, 'failed'); + assert.deepEqual(failed.observation, { actual: 2 }); + assert.deepEqual(failed.expected, { value: 1 }); + const inconclusive = await executeAction(registry(() => { + throw new ActionInconclusive('transport cannot be observed'); + }), 'fakeAction', { do: 'fakeAction' }, context()); + assert.equal(inconclusive.status, 'inconclusive'); + assert.equal(inconclusive.code, 'inconclusive'); +}); + +test('deadline, cancellation, and unclassified exceptions are fail-closed evidence', async () => { + let cleanedUp = false; + const never: ActionImplementation = ({ signal }) => new Promise((_resolve, reject) => + signal.addEventListener('abort', () => setTimeout(() => { + cleanedUp = true; + reject(signal.reason); + }, 5), { once: true })); + const timedOut = await executeAction(registry(never, { timeoutMs: 20 }), 'fakeAction', + { do: 'fakeAction' }, context()); + assert.equal(timedOut.status, 'harness_failure'); + assert.equal(timedOut.code, 'deadline_exceeded'); + assert.equal(timedOut.retryable, true); + assert.equal(cleanedUp, true, 'timeout evidence must wait for implementation cleanup'); + + const controller = new AbortController(); + const cancelledPromise = executeAction(registry(never), 'fakeAction', + { do: 'fakeAction' }, context({ signal: controller.signal })); + controller.abort('operator cancelled'); + const cancelled = await cancelledPromise; + assert.equal(cancelled.status, 'inconclusive'); + assert.equal(cancelled.code, 'cancelled'); + + const crashed = await executeAction(registry(() => { throw new TypeError('unexpected bug'); }), + 'fakeAction', { do: 'fakeAction' }, context()); + assert.equal(crashed.status, 'harness_failure'); + assert.equal(crashed.code, 'unclassified_exception'); +}); + +test('deadline evidence retains a drained observation without making the action pass', async () => { + for (const rejects of [false, true]) { + for (const observation of [{ responses: 1, unknown: 1 }, new Date()]) { + const result = await executeAction(registry(({ signal }) => new Promise((resolve, reject) => { + signal.addEventListener('abort', () => rejects + ? reject(new ActionHarnessFailure('partial fault', { observation })) : resolve(observation), { once: true }); + }), { timeoutMs: 10 }), 'fakeAction', { do: 'fakeAction' }, context()); + assert.equal(result.code, 'deadline_exceeded'); + assert.equal(result.status, 'harness_failure'); + assert.deepEqual(result.observation, observation instanceof Date ? null : observation); + } + } +}); + +test('invalid input, missing services, and malformed observations never become passes', async () => { + const validRegistry = registry(() => ({ ok: true })); + const invalid = await executeAction(validRegistry, 'fakeAction', { do: 'wrong' }, context()); + assert.equal(invalid.code, 'invalid_input'); + const missingCapability = await executeAction(validRegistry, 'fakeAction', { do: 'fakeAction' }, + { capabilities: {} }); + assert.equal(missingCapability.code, 'missing_capability'); + assert.equal(missingCapability.status, 'harness_failure'); + for (const observation of [new Date(), { recovery: undefined }, [undefined]]) { + const malformed = await executeAction(registry(() => observation), 'fakeAction', + { do: 'fakeAction' }, context()); + assert.equal(malformed.code, 'invalid_evidence'); + assert.equal(malformed.status, 'harness_failure'); + } + const cyclic: { ok: boolean; self?: unknown } = { ok: true }; + cyclic.self = cyclic; + const nonSerializable = await executeAction(registry(() => cyclic), 'fakeAction', + { do: 'fakeAction' }, context()); + assert.equal(nonSerializable.code, 'invalid_evidence'); +}); + + +test('deadline closes owned browser work before draining an operation that ignores abort', { timeout: 1000 }, async () => { + const events: string[] = []; + let rejectOperation!: (reason: Error) => void; + const run = executeAction(registry(() => new Promise((_resolve, reject) => { + rejectOperation = reject; + events.push('started'); + }), { timeoutMs: 20 }), 'fakeAction', { do: 'fakeAction' }, { + capabilities: { clock: {} }, + onAbort: async () => { + events.push('owned browser closed'); + rejectOperation(new Error('connection closed')); + }, + }); + const result = await run; + events.push('returned'); + assert.equal(result.code, 'deadline_exceeded'); + assert.deepEqual(events, ['started', 'owned browser closed', 'returned']); +}); + + +for (const teardown of ['rejects', 'hangs'] as const) { + test(`failed browser teardown ${teardown} returns structured evidence within cleanup grace`, { timeout: 1000 }, async () => { + let stopped = false; + const result = await executeAction(registry(() => new Promise(() => {}), { timeoutMs: 10 }), + 'fakeAction', { do: 'fakeAction' }, { + capabilities: { clock: {} }, + onAbort: async () => { + stopped = true; + if (teardown === 'rejects') throw new Error('browser close failed'); + await new Promise(() => {}); + }, + }, { setTimer: (callback, ms) => setTimeout(callback, ms === 5000 ? 20 : ms) }); + assert.equal(stopped, true); + assert.equal(result.status, 'harness_failure'); + assert.equal(result.code, 'cancellation_cleanup_failed'); + assert.equal(result.retryable, false); + }); +} diff --git a/tools/stack-bench/tests/action-findings.test.ts b/tools/stack-bench/tests/action-findings.test.ts new file mode 100644 index 00000000000..894f01d70df --- /dev/null +++ b/tools/stack-bench/tests/action-findings.test.ts @@ -0,0 +1,175 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { ActionApplicationFailure, ActionInconclusive } from '../src/actions/action-contract.js'; +import { FAILED_FINDING_KINDS, INCONCLUSIVE_FINDING_KINDS, finding, findingStatus, + isFinding, renderFinding, renderRepairFinding } from '../src/actions/action-findings.js'; +import type { Finding, FindingKind } from '../src/actions/action-findings.js'; +import { fail, inconclusive } from '../src/actions/actor-action-runtime.js'; +import { assertAgentVisibleText } from '../src/composition/agent-visible-contract.js'; + +const FINDING_KINDS = [...FAILED_FINDING_KINDS, ...INCONCLUSIVE_FINDING_KINDS].sort(); + +// One sample per kind. The sample values are the shapes an executor has: +// contract names, actor labels, action ids, numbers, statuses. `detail` is +// the raw text a human may read and must never be rendered. +const DETAIL = 'RAW-DETAIL locator.click: Timeout 5000ms exceeded [data-testid="secret"]'; +const SAMPLES: { [K in FindingKind]: Finding } = { + 'control-missing': finding('control-missing', { control: 'buy-now' }), + 'control-present': finding('control-present', { control: 'admin-link' }), + 'control-available': finding('control-available', { control: 'buy-now', actor: 'visitor' }), + 'control-not-ready': finding('control-not-ready', { control: 'buy-now', actors: ['a', 'b'] }), + 'control-blocked': finding('control-blocked', { control: 'buy-now', detail: DETAIL }), + 'control-empty': finding('control-empty', { control: 'order-total' }), + 'control-unreadable': finding('control-unreadable', { control: 'stock', actors: ['a'] }), + 'value-mismatch': finding('value-mismatch', { control: 'support-status' }), + 'text-unexpected': finding('text-unexpected', { control: 'support-ticket' }), + 'text-missing': finding('text-missing', { control: 'order-item', matchingText: 'Keyboard', + observed: 'Keyboard · shipped', expected: 'returned' }), + 'value-unstable': finding('value-unstable', { control: 'stock' }), + 'clients-disagree': finding('clients-disagree', { control: 'stock', actors: ['a', 'b'] }), + 'number-missing': finding('number-missing', { control: 'order-total' }), + 'number-mismatch': finding('number-mismatch', { control: 'order-total', observed: 9, + expected: { equals: 12 } }), + 'count-mismatch': finding('count-mismatch', { control: 'order-item', observed: 2, expected: 1 }), + 'order-mismatch': finding('order-mismatch', { control: 'message-item', actors: ['a', 'b'] }), + 'entries-missing': finding('entries-missing', { expected: 10, missing: 2, duplicated: 1 }), + 'actors-with-control': finding('actors-with-control', { control: 'ticket', observed: 2, expected: 1 }), + 'too-many-per-actor': finding('too-many-per-actor', { control: 'ticket', maxEach: 1 }), + 'clicks-failed': finding('clicks-failed', { control: 'buy-now', failed: 1, total: 3, detail: DETAIL }), + 'choice-missing': finding('choice-missing', { control: 'frequency', detail: DETAIL }), + 'page-timeout': finding('page-timeout', { control: 'buy-now', detail: DETAIL }), + 'page-crashed': finding('page-crashed', { detail: DETAIL }), + 'grading-time-limit': finding('grading-time-limit', { limitMs: 2_220_000 }), + 'page-error': finding('page-error', { detail: DETAIL }), + 'app-control-failed': finding('app-control-failed', { mode: 'start', target: 'app-server', detail: DETAIL }), + 'script-failed': finding('script-failed', { script: 'scripts/seed.js', detail: DETAIL }), + 'script-invalid': finding('script-invalid', { script: '../outside.js' }), + 'action-failed': finding('action-failed', { action: 'click' }), + 'call-refused': finding('call-refused', { action: 'buy', actor: 'buyer', status: 404, + operation: { reducer: 'buy_now', path: '/api/buy', method: 'POST' } }), + 'call-accepted': finding('call-accepted', { action: 'buy', actor: 'guest', status: 200, required: 'refused' }), + 'call-error': finding('call-error', { action: 'buy', actor: 'guest', status: 500, required: 'refused', + operation: null }), + 'concurrent-calls-mismatch': finding('concurrent-calls-mismatch', { action: 'checkout', expected: 1, + accepted: 2, fired: 2, detail: DETAIL }), + 'interface-missing': finding('interface-missing', { control: 'admin-row', action: 'restock', + attribute: 'data-action-input' }), + 'interface-invalid': finding('interface-invalid', { action: 'restock', attribute: 'data-action-input', + missing: ['itemId'], detail: DETAIL }), + 'replay-accepted': finding('replay-accepted', { actor: 'customer', status: 200, action: 'restock' }), + 'replay-error': finding('replay-error', { status: 500 }), + 'forgery-accepted': finding('forgery-accepted', { field: 'userId', status: 201 }), + 'forgery-error': finding('forgery-error', { status: null }), + 'message-delivered': finding('message-delivered', { actor: 'other' }), + 'stock-interface-missing': finding('stock-interface-missing', { detail: DETAIL }), + 'assertion-without-action': finding('assertion-without-action', { action: 'replayAs' }), + 'observation-window-missed': finding('observation-window-missed', { detail: DETAIL }), + 'unknown-action': finding('unknown-action', { action: 'refund' }), + 'action-without-parameters': finding('action-without-parameters', { action: 'refund' }), + 'no-session': finding('no-session', { actor: 'buyer', action: 'buy' }), + 'unresolved-action': finding('unresolved-action', { action: 'buy' }), + 'replay-unavailable': finding('replay-unavailable', { actor: 'customer', detail: DETAIL }), + 'route-unproven': finding('route-unproven', { actor: 'customer', action: 'scheduleRestock' }), + 'forgery-unverifiable': finding('forgery-unverifiable', { actor: 'customer', detail: DETAIL }), + 'not-observed': finding('not-observed', { actor: 'owner' }), + 'transport-incomplete': finding('transport-incomplete', {}), + 'nothing-contended': finding('nothing-contended', { detail: DETAIL }), + 'network-not-interrupted': finding('network-not-interrupted', { actor: 'buyer', detail: DETAIL }), + 'no-backend-control': finding('no-backend-control', { target: 'backend-runtime' }), + 'control-refused': finding('control-refused', { target: 'app-server' }), + 'database-write-failed': finding('database-write-failed', { detail: DETAIL }), + 'stock-read-unavailable': finding('stock-read-unavailable', { detail: DETAIL }), + 'unsupported-backend': finding('unsupported-backend', { backend: 'stub' }), + 'app-directory-unknown': finding('app-directory-unknown', {}), + 'invalid-input': finding('invalid-input', { detail: DETAIL }), +}; + +test('every kind renders one agent-visible sentence and never its detail', () => { + assert.deepEqual(Object.keys(SAMPLES).sort(), [...FINDING_KINDS]); + for (const kind of FINDING_KINDS) { + const sentence = renderFinding(SAMPLES[kind]); + assert.ok(sentence.length > 8, kind); + assert.doesNotMatch(sentence, /RAW-DETAIL|data-testid|Timeout|\d+ms|undefined/, kind); + assert.doesNotThrow(() => assertAgentVisibleText(sentence), kind); + assert.doesNotThrow(() => assertAgentVisibleText(renderRepairFinding(SAMPLES[kind])), kind); + assert.ok(isFinding(SAMPLES[kind]), kind); + } +}); + +test('repair findings retain measured quantities without changing evidence or prescribing a fix', () => { + for (const kind of ['number-mismatch', 'count-mismatch', 'entries-missing', + 'actors-with-control', 'too-many-per-actor', 'clicks-failed', 'concurrent-calls-mismatch'] as const) { + const original = structuredClone(SAMPLES[kind]); + assert.match(renderRepairFinding(original), /\d/); + assert.match(renderFinding(original), /\d/); + assert.deepEqual(original, SAMPLES[kind]); + } + assert.match(renderRepairFinding(SAMPLES['number-mismatch']), /order-total.*reads.*expected/); + assert.match(renderRepairFinding(finding('number-mismatch', { + control: 'stock', observed: 52, expected: { atMost: 50 }, + })), /stock.*52.*at most 50/); + assert.match(renderRepairFinding(SAMPLES['call-refused']), /HTTP 404/); + assert.match(renderFinding(SAMPLES['call-refused']), /names the buy_now reducer or POST \/api\/buy/); + assert.match(renderFinding(SAMPLES['forgery-error']), /returned no server response/); + assert.match(renderRepairFinding(SAMPLES['entries-missing']), /missing.*duplicated/); + assert.doesNotMatch(renderRepairFinding(finding('stock-interface-missing', { + missingRow: 'warehouse', + })), /check the|links/); +}); + +test('the catalog partitions into application failures and unmeasured outcomes', () => { + assert.equal(new Set(FINDING_KINDS).size, FINDING_KINDS.length); + for (const kind of FAILED_FINDING_KINDS) assert.equal(findingStatus(SAMPLES[kind]), 'failed'); + for (const kind of INCONCLUSIVE_FINDING_KINDS) assert.equal(findingStatus(SAMPLES[kind]), 'inconclusive'); + assert.equal(isFinding({ kind: 'control-not-ready', fields: { control: 'buy-now' } }), false); + assert.equal(isFinding({ kind: 'control-not-ready', fields: { control: 'buy-now', + actors: ['buyer'], extra: true } }), false); + assert.equal(isFinding({ kind: 'call-refused', fields: { action: 'buy', actor: 'buyer', status: 403, + operation: { reducer: null, path: '/api/buy', method: 1 } } }), false); + assert.equal(isFinding({ kind: 'control-not-ready', fields: { control: 'buy-now', + actors: ['buyer'] } }), true); + assert.equal(isFinding({ kind: 'made-up', fields: {} }), false); + assert.equal(isFinding({ kind: 'control-missing' }), false); +}); + +test('executors fail with a finding, and the message is its rendering', () => { + assert.throws(() => fail('count-mismatch', { control: 'order-item', observed: 2, expected: 1 }), + (error: unknown) => error instanceof ActionApplicationFailure + && error.message === '2 order-item entries shown, expected 1' + && isFinding(error.details.finding) && error.details.finding.kind === 'count-mismatch'); + assert.throws(() => inconclusive('not-observed', { actor: 'owner' }), + (error: unknown) => error instanceof ActionInconclusive + && error.message === 'the expected data could not be observed reaching owner' + && isFinding(error.details.finding) && error.details.finding.kind === 'not-observed'); +}); + +test('missing rows and filtered controls give useful feedback without exposing probe text', () => { + assert.match(renderRepairFinding(finding('stock-interface-missing', { + missingRow: 'warehouse', item: 'Desk Lamp', warehouse: 'East', detail: DETAIL, + })), /warehouse "East" row was not found/); + const stock = renderRepairFinding(finding('stock-interface-missing', { + missingRow: 'stock', item: 'Desk Lamp', warehouse: 'East', detail: DETAIL, + })); + assert.match(stock, /item "Desk Lamp" and warehouse "East"/); + assert.doesNotMatch(stock, /RAW-DETAIL|check the|links/); + assert.match(renderFinding(finding('stock-interface-missing', { missingRow: 'warehouse', detail: DETAIL })), /required warehouse row was not found/); + const text = renderFinding(finding('control-missing', { control: 'admin-warehouse-item', filtered: true })); + assert.match(text, /matching the requested entry/); + assert.doesNotMatch(text, /RAW-DETAIL/); + assert.match(renderRepairFinding(finding('control-missing', { control: 'cart-item', filtered: true, + matchingText: 'Keyboard', scope: 'cart-panel', scopeText: 'My cart' })), /My cart.*Keyboard/); + assert.match(renderRepairFinding(finding('value-mismatch', { control: 'status', observed: 'Pending', + expected: 'Approved' })), /shows "Pending", expected "Approved"/); + assert.match(renderRepairFinding(finding('text-unexpected', { control: 'ticket', matchedText: 'Private note' })), + /shows "Private note" that must not appear/); + assert.match(renderRepairFinding(finding('choice-missing', { control: 'frequency', requestedChoice: 'Weekly' })), + /required choice "Weekly"/); + assert.doesNotMatch(renderRepairFinding(finding('value-mismatch', { control: 'status', + observed: 'password=DO_NOT_DISCLOSE', expected: 'Approved' })), /DO_NOT_DISCLOSE/); + assert.equal(isFinding({ kind: 'value-mismatch', fields: { control: 'status', observed: 'x'.repeat(161) } }), false); + assert.match(renderRepairFinding(finding('value-mismatch', { control: 'latency', observed: '10ms', expected: '5ms' })), + /"10ms", expected "5ms"/); + assert.match(renderRepairFinding(finding('value-mismatch', { control: 'text', observed: ' a\nb ', expected: 'a b' })), + /" a\\nb ", expected "a b"/); +}); diff --git a/tools/stack-bench/tests/actor-transport-action-executors.test.ts b/tools/stack-bench/tests/actor-transport-action-executors.test.ts new file mode 100644 index 00000000000..8e4cfb9c3db --- /dev/null +++ b/tools/stack-bench/tests/actor-transport-action-executors.test.ts @@ -0,0 +1,2079 @@ +import assert from 'node:assert/strict'; +import { recordConvexSession } from '../src/stacks/backends/convex-browser-session.js'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { createServer } from 'node:http'; +import { createHmac } from 'node:crypto'; +import { once } from 'node:events'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { runInNewContext } from 'node:vm'; +import { chromium } from 'playwright'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { ACTION_REGISTRY } from '../src/actions/action-catalog.js'; +import { classifyResponseContract, tamperedSessionCredentials } from '../src/actions/named-action-runtime.js'; +import { executeAction } from '../src/actions/action-contract.js'; +import { createNamedActionsCapability } from '../src/actions/actor-transport-action-executors.js'; + +type UnknownRecord = Record; +type NamedOptions = Parameters[0]; +type Calls = ReturnType; +type Verification = readonly ['structural' | 'unverified' | 'verified', string]; + +interface ServiceOverrides { + readonly url?: string; + readonly actions?: NamedOptions['actions']; + readonly appRoot?: string | null; + readonly backend?: string; + readonly fetchImpl?: NamedOptions['fetchImpl']; + readonly sleep?: (milliseconds: number, signal: AbortSignal) => Promise; + readonly spacetime?: { uri: string; mod: string }; +} + +interface ProvidedServices { + readonly capabilities: Readonly>; + readonly calls: Calls; + readonly verification: Verification[]; +} + +interface CapturedRequest { + readonly options: UnknownRecord; + readonly url: string; +} + +const record = (value: unknown): UnknownRecord => { + assert(value !== null && typeof value === 'object'); + return value as UnknownRecord; +}; + +const namedResponse = (status: number, ok: boolean) => ({ + status, + ok, + text: async (): Promise => '', +}); + +function services( + actors: ReadonlyMap, + overrides: ServiceOverrides = {}, +): ProvidedServices { + for (const value of actors.values()) { + const actor = record(value); + actor.record ??= () => {}; + actor.context ??= { cookies: async () => [] }; + actor.page ??= {}; + record(actor.page).evaluate ??= async () => []; + for (const write of (actor.writes ?? []) as UnknownRecord[]) { + write.url ??= `${overrides.backend === 'spacetime' ? overrides.spacetime?.uri : 'http://app.test'}/api/session`; + } + } + const verification: Verification[] = []; + let calls: Calls = null; + const sleep = overrides.sleep + ?? (async (_milliseconds: number, _signal: AbortSignal): Promise => {}); + const browser = { + defaultWithin: 5000, + expand: (value: string) => value === '{room:test}' ? 'test-scoped' : value, + hyphenatedScopedUser: (name: string) => `${name}-scope`, + roomName: (room: string) => `${room}-scope`, + scopedUser: (name: string) => `${name}scope`, + sleep, + testId: (id: string) => `[data-testid="${id}"]`, + }; + const named = createNamedActionsCapability({ + actions: overrides.actions ?? [{ id: 'checkout', path: '/api/checkout', reducer: 'checkout', args: [] }], + backend: overrides.backend ?? 'postgres', + url: overrides.url ?? 'http://app.test', + spacetime: overrides.spacetime, + lastCalls: { get: () => calls, set: value => { calls = value; } }, + sleep, + fetchImpl: overrides.fetchImpl ?? (async () => ({ status: 200, ok: true, text: async () => '' })), + now: (() => { let value = 10; return () => value++; })(), + }); + return { + capabilities: { + actors: { get: (name: string) => actors.get(name) }, + 'application-files': { root: overrides.appRoot ?? null, expand: browser.expand }, + 'browser-interaction': browser, + 'named-actions': named, + subprocess: { sleep }, + 'transport-observation': { + defaultWithin: 5000, + expand: browser.expand, + sleep, + verification: { + structural: (message: string) => { verification.push(['structural', message]); }, + unverified: (message: string) => { verification.push(['unverified', message]); }, + verified: (message: string) => { verification.push(['verified', message]); }, + }, + }, + }, + get calls() { return calls; }, + verification, + }; +} + +async function run(input: UnknownRecord, provided: ProvidedServices) { + const action = String(input.do); + return executeAction(ACTION_REGISTRY, action, input, { + capabilities: provided.capabilities, + }); +} + +test('a parameterless action needs no DOM input; parameterized actions still do', async () => { + let calls = 0; + const provided = services(new Map([['guest', { name: 'guest' }]]), { + fetchImpl: async (url, options) => { + assert.equal(url, 'http://app.test/api/checkout'); + assert.deepEqual(JSON.parse(String(options.body)), {}); + calls++; + return namedResponse(200, true); + }, + }); + const input = { do: 'callAction', actor: 'guest', action: 'checkout', authentication: 'none', + namedAction: { id: 'checkout', path: '/api/checkout', reducer: 'checkout', args: [] } }; + const result = await run(input, provided); + assert.equal(result.status, 'passed', JSON.stringify(result)); + assert.equal((await run({ ...input, namedAction: { ...input.namedAction, args: [1] } }, provided)).status, + 'harness_failure'); + assert.equal(calls, 1); +}); + +test('live session hooks override captured credentials after signout and account changes', async () => { + // An array is an app without the hook that keeps these session tokens in storage and has no captured Authorization. + for (const current of [null, 'current-session', 'broken', 'no-hook', ['illicit-session'], ['one-token', 'another-token']]) { + let calls = 0; + const stored = Array.isArray(current) ? current : []; + const storage = { length: stored.length, key: (index: number) => `session-${index}`, + getItem: (key: string) => stored[Number(key.slice('session-'.length))] ?? null }; + const actor = { name: 'buyer', writes: [{ url: 'http://app.test/buy', headers: { + ...(Array.isArray(current) ? {} : { authorization: 'Bearer old-session' }), 'x-csrf-token': 'current-context', + } }], context: { cookies: async () => [] }, page: { + evaluate: async (callback: () => unknown) => + runInNewContext(`(${callback.toString()})()`, { localStorage: storage, sessionStorage: { length: 0 }, + window: current === 'no-hook' || Array.isArray(current) ? {} : { + getSessionToken: () => { if (current === 'broken') throw new Error('broken hook'); return current; }, + } }), + } }; + const broken = current === 'broken' || stored.length > 1; + const provided = services(new Map([['buyer', actor]]), { fetchImpl: async (_url, options) => { + calls++; + assert.equal(options.headers?.['x-csrf-token'], 'current-context'); + assert.equal(options.headers?.Authorization ?? options.headers?.authorization, + current === null ? undefined : `Bearer ${Array.isArray(current) ? current[0] : current === 'no-hook' ? 'old-session' : current}`); + return namedResponse(401, false); + } }); + const result = await run({ do: 'callAction', actor: 'buyer', action: 'buy', authentication: 'optional', settleMs: 0, + namedAction: { id: 'buy', path: '/buy', reducer: 'buy_now', args: [] } }, provided); + assert.equal(result.status, broken ? 'inconclusive' : 'passed', JSON.stringify(result)); + assert.equal(calls, broken ? 0 : 1); + } +}); + +test('session tampering reaches the server, preserves context, requires a valid matching control and restores access', async () => { + const jwt = 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.ABCD1234'; + const credentials: Record[] = [{ Authorization: `Bearer ${jwt}` }, { Cookie: 'sid=0123456789abcdef' }]; + for (const credential of credentials) { + const [key, valid] = Object.entries(credential)[0]!; + let requests = 0, writes = 0; + const server = createServer((req, res) => { + requests++; + assert.equal(req.headers.origin, 'http://app.test'); + assert.equal(req.headers.referer, 'http://app.test/'); + if (req.headers[key.toLowerCase()] !== valid) { res.writeHead(401).end('refused'); return; } + writes++; res.end('{}'); + }); + server.listen(0, '127.0.0.1'); await once(server, 'listening'); + try { + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}/buy`; + const headers: Record = { ...credential, Origin: 'http://app.test', Referer: 'http://app.test/' }; + const actor = { name: 'buyer', record() {}, writes: [{ url: 'http://app.test/buy', headers }], + page: { evaluate: async () => [] }, + context: { cookies: async () => key === 'Cookie' ? [{ name: 'sid', value: headers.Cookie!.slice(4) }] : [] } }; + const provided = services(new Map([['buyer', actor]]), { fetchImpl: (_url, options) => fetch(url, options) }); + const input = { do: 'callAction', actor: 'buyer', action: 'buy', settleMs: 0, + namedAction: { id: 'buy', path: '/buy', reducer: 'buy_now', args: [] } }; + const attack = { ...input, authentication: 'tampered-session' }; + const unproven = await run(attack, provided); + assert.equal(unproven.status, 'inconclusive', JSON.stringify(unproven)); + assert.equal(requests, 0); + assert.equal((await run(input, provided)).status, 'passed'); + assert.equal((await run(attack, provided)).status, 'passed'); + assert.equal((await run({ do: 'expectActionOutcome', actor: 'buyer', outcome: 'refused' }, provided)).status, 'passed'); + assert.equal(writes, 1); + assert.equal((await run(input, provided)).status, 'passed'); + assert.equal(writes, 2); + headers[key] = `${valid}changed`; + assert.equal((await run(attack, provided)).status, 'inconclusive', 'a stale positive control proves nothing'); + assert.equal(requests, 3); + } finally { server.closeAllConnections(); await new Promise(resolve => server.close(() => resolve())); } + } + const changed = tamperedSessionCredentials({ Authorization: `Bearer ${jwt}`, 'x-csrf-token': 'context' }, 'buyer'); + assert.deepEqual(changed.Authorization!.split('.').slice(0, 2), `Bearer ${jwt}`.split('.').slice(0, 2)); + assert.equal(changed['x-csrf-token'], 'context'); + assert.notEqual(changed.Authorization, `Bearer ${jwt}`); + const ambiguous: Record[] = [ + { Authorization: 'Bearer token', Cookie: 'sid=token' }, + { Cookie: 'sid=token; extra=other' }, { Cookie: 'sid=token', 'x-csrf-token': 'token' }, + { Cookie: 'csrf=token' }, { Authorization: 'Basic dGVzdA==' }, + { Authorization: 'Bearer token', 'x-auth-token': 'other' }, + ]; + for (const headers of ambiguous) assert.throws(() => tamperedSessionCredentials(headers, 'buyer'), /could not issue the replay/); + for (const cookies of [[{ name: 'sid', value: 'token' }], [{ name: 'theme', value: 'dark' }]]) { + let requests = 0; + const actor = { name: 'buyer', writes: [{ headers: { authorization: `Bearer ${jwt}` } }], + context: { cookies: async () => cookies } }; + const provided = services(new Map([['buyer', actor]]), { fetchImpl: async () => { + requests++; return namedResponse(200, true); + } }); + const input = { do: 'callAction', actor: 'buyer', action: 'checkout', settleMs: 0, + namedAction: { id: 'checkout', path: '/api/checkout', reducer: 'checkout', args: [] } }; + const positive = await run(input, provided); + assert.equal(positive.status, 'passed', JSON.stringify(positive)); + assert.equal((await run({ ...input, authentication: 'tampered-session' }, provided)).status, 'inconclusive'); + assert.equal(requests, 1, 'neither a fallback credential nor an unrelated cookie can silently earn a result'); + } + for (const status of [401, 403]) { + const classified = classifyResponseContract({ responseContract: 'convex-mutation' }, { status, text: 'unauthorized' }); + assert.equal(classified.refusalKind, 'access'); assert.equal(classified.ok, false); + } +}); + +// Failure cases: normal dotted names and encoded signed cookies must reach the +// server, while extra cookies and CSRF context must not become forged-session proof. +test('browser session cookies preserve format, prove controls and refuse forged values', async t => { + const browser = await chromium.launch({ headless: true }); + t.after(() => browser.close()); + for (const mode of ['dotted-name', 'signed-value', 'dotted-signed', 'accept-forgery', 'extra-cookie', 'csrf-header']) { + await t.test(mode, async () => { + const name = mode === 'signed-value' ? 'sid' : 'storefront.sid'; + const session = 'server-issued-session'; + const signed = `s:${session}.${createHmac('sha256', 'test-session-secret').update(session).digest('base64')}`; + const value = mode === 'dotted-name' ? session : encodeURIComponent(signed); + const observed: string[] = []; + let purchases = 0; + const server = createServer((req, res) => { + if (req.method === 'GET') { + res.setHeader('Set-Cookie', [`${name}=${value}; HttpOnly; Path=/; SameSite=Lax`, + ...(mode === 'extra-cookie' ? ['locale=en; Path=/'] : [])]); + res.setHeader('Content-Type', 'text/html'); + res.end(''); return; + } + const cookie = req.headers.cookie ?? ''; + observed.push(cookie); + const current = cookie.split('; ').find(pair => pair.startsWith(`${name}=`))?.slice(name.length + 1); + const accepted = current !== undefined && (mode === 'accept-forgery' || decodeURIComponent(current) === + (mode === 'dotted-name' ? session : signed)); + if (accepted) purchases++; + res.writeHead(accepted ? 201 : 401).end('{}'); + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + const context = await browser.newContext(); + try { + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const page = await context.newPage(); + await page.goto(`${url}/login`); + const original = await context.cookies(url); + assert.equal(original.find(cookie => cookie.name === name)?.value, value); + const actor = { name: 'buyer', context, page, writes: [{ url: `${url}/buy`, headers: { + Origin: url, Referer: `${url}/`, ...(mode === 'csrf-header' ? { 'x-csrf-token': 'unchanged-context' } : {}), + } }] }; + const provided = services(new Map([['buyer', actor]]), { url, fetchImpl: fetch }); + const input = { do: 'callAction', actor: 'buyer', action: 'buy', settleMs: 0, + namedAction: { id: 'buy', path: '/buy', reducer: 'buy_now', args: [] } }; + const control = { ...input, authentication: 'session-control' }; + const attack = { ...input, authentication: 'tampered-session' }; + assert.equal((await run(control, provided)).status, 'inconclusive'); + assert.equal(observed.length, 0, 'a valid prior request is required'); + assert.equal((await run(input, provided)).status, 'passed'); + assert.equal((await run({ do: 'expectActionOutcome', actor: 'buyer', outcome: 'accepted' }, provided)).status, 'passed'); + const isolated = await run(control, provided); + if (mode === 'extra-cookie' || mode === 'csrf-header') { + assert.equal(isolated.status, 'inconclusive'); + assert.equal((await run(attack, provided)).status, 'inconclusive'); + assert.equal(observed.length, 1, 'ambiguous credentials must not be altered or sent'); + assert.equal(purchases, 1); return; + } + assert.equal(isolated.status, 'passed', JSON.stringify(isolated)); + assert.equal(purchases, 2); + assert.equal((await run(attack, provided)).status, 'passed'); + assert.equal((await run({ do: 'expectActionOutcome', actor: 'buyer', outcome: 'refused' }, provided)).status, + mode === 'accept-forgery' ? 'failed' : 'passed'); + assert.equal(purchases, mode === 'accept-forgery' ? 3 : 2, + 'stored effects distinguish a vulnerable cookie app from a refused forgery'); + assert(observed[2]!.startsWith(`${name}=`), 'the cookie name must not change'); + assert.notEqual(observed[2], observed[1], 'the altered value must reach the server'); + assert.equal((await run(input, provided)).status, 'passed'); + assert.equal(purchases, mode === 'accept-forgery' ? 4 : 3); + assert.deepEqual(await context.cookies(url), original, 'the browser retains its original session'); + assert.equal(observed[3], observed[0]); + } finally { + await context.close(); + server.closeAllConnections(); await new Promise(resolve => server.close(() => resolve())); + } + }); + } +}); + +test('mixed credentials need a measured isolated control; cookie and CSRF dependencies stay inconclusive', async t => { + for (const mode of ['bearer', 'cookie-only', 'requires-cookie', 'accept-forgery', 'commit-then-refuse', 'reject-all', 'response-loss']) { + await t.test(mode, async () => { + const token = 'server-issued-session-0123456789'; + let purchases = 0, requests = 0; + const observed: { cookie: string | undefined; authorization: string | undefined }[] = []; + const server = createServer((req, res) => { + if (req.url === '/login') { + res.setHeader('Set-Cookie', [`sid=${token}; HttpOnly; Path=/`, 'locale=en; Path=/']); + res.end(JSON.stringify({ token })); return; + } + requests++; + observed.push({ cookie: req.headers.cookie, authorization: req.headers.authorization }); + assert.equal(req.headers.origin, 'http://app.test'); + assert.equal(req.headers['x-csrf-token'], 'unchanged-context'); + const bearer = req.headers.authorization === `Bearer ${token}`; + const cookie = req.headers.cookie?.includes(`sid=${token}`) === true; + if (mode === 'response-loss' && !cookie) { res.destroy(); return; } + const accepted = mode === 'reject-all' ? false : mode === 'cookie-only' ? cookie + : mode === 'requires-cookie' ? bearer && cookie + : ['accept-forgery', 'commit-then-refuse'].includes(mode) ? !!req.headers.authorization : bearer; + if (accepted) purchases++; + res.writeHead(accepted && !(mode === 'commit-then-refuse' && !bearer) ? 200 : 401).end('{}'); + }); + server.listen(0, '127.0.0.1'); await once(server, 'listening'); + try { + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const login = await fetch(`${url}/login`, { method: 'POST' }); + const cookies = login.headers.getSetCookie().map(value => { + const [name, cookieValue] = value.split(';')[0]!.split('='); + return { name: name!, value: cookieValue! }; + }); + const credentials = await login.json() as { token: string }; + const headers: Record = { Origin: 'http://app.test', 'x-csrf-token': 'unchanged-context' }; + if (mode !== 'cookie-only') headers.Authorization = `Bearer ${credentials.token}`; + const actor = { name: 'buyer', writes: [{ url: 'http://app.test/buy', headers }], + context: { cookies: async () => cookies }, page: { evaluate: async () => [] } }; + const provided = services(new Map([['buyer', actor]]), { fetchImpl: (_url, options) => fetch(`${url}/buy`, options) }); + const input = { do: 'callAction', actor: 'buyer', action: 'buy', settleMs: 0, + namedAction: { id: 'buy', path: '/buy', reducer: 'buy_now', args: [] } }; + const control = { ...input, authentication: 'session-control' }; + assert.equal((await run(control, provided)).status, 'inconclusive', 'isolation requires a full-credential control'); + assert.equal(requests, 0); + assert.equal((await run(input, provided)).status, 'passed'); + const positive = await run({ do: 'expectActionOutcome', actor: 'buyer', outcome: 'accepted' }, provided); + if (mode === 'reject-all') { + assert.equal(positive.status, 'failed', 'reject-all must fail the original positive control'); + assert.equal((await run(control, provided)).status, 'inconclusive'); + assert.equal(requests, 1); return; + } + assert.equal(positive.status, 'passed'); + const isolated = await run(control, provided); + if (['cookie-only', 'requires-cookie', 'response-loss'].includes(mode)) { + assert.equal(isolated.status, 'inconclusive', JSON.stringify(isolated)); + assert.equal(purchases, 1); + assert.equal(requests, mode === 'cookie-only' ? 1 : 2); + assert.equal((await run({ ...input, authentication: 'tampered-session' }, provided)).status, 'inconclusive'); + assert.equal(requests, mode === 'cookie-only' ? 1 : 2, 'no forgery after an unproven isolated control'); + return; + } + assert.equal(isolated.status, 'passed'); + assert.equal(purchases, 2, 'isolation is a real purchase, not an auth discovery request hidden from accounting'); + assert.equal(observed[1]!.cookie, undefined); + const beforeForgery = purchases; + assert.equal((await run({ ...input, authentication: 'tampered-session' }, provided)).status, 'passed'); + assert.equal(observed[2]!.cookie, undefined, 'no valid fallback credential can mask the forgery'); + assert.notEqual(observed[2]!.authorization, observed[1]!.authorization); + const refusal = await run({ do: 'expectActionOutcome', actor: 'buyer', outcome: 'refused' }, provided); + assert.equal(refusal.status, mode === 'accept-forgery' ? 'failed' : 'passed'); + assert.equal(purchases - beforeForgery, mode === 'bearer' ? 0 : 1, + 'stored effects must catch commit-then-refuse even though the refusal assertion passes'); + assert.equal((await run(input, provided)).status, 'passed'); + assert.equal(purchases, beforeForgery + (mode === 'bearer' ? 1 : 2)); + assert.equal(observed[3]!.cookie, `sid=${token}; locale=en`, 'the original browser session is unchanged'); + } finally { server.closeAllConnections(); await new Promise(resolve => server.close(() => resolve())); } + }); + } +}); + +test('shipping accounting waits for the staff response before reading accounting', async () => { + const scenario = JSON.parse(readFileSync(join(STACK_BENCH_ROOT, + 'tracks/ecommerce/scenarios/progression-shipping-accounting.json'), 'utf8')); + const steps = scenario.features[0].criteria[0].steps as UnknownRecord[]; + const index = steps.findIndex(step => step.do === 'callAction' && step.action === 'ship'); + assert(index >= 0); + let release!: () => void; + const response = new Promise(resolve => { release = resolve; }); + let submitted!: () => void; + const started = new Promise(resolve => { submitted = resolve; }); + const staff = { name: 'staff', writes: [{ headers: { authorization: 'Bearer staff' } }] }; + const customer = { name: 'customer', loc: () => ({ waitFor: async () => {}, + getAttribute: async () => JSON.stringify({ orderId: '42' }) }) }; + const provided = services(new Map([['staff', staff], ['customer', customer]]), { + fetchImpl: async (url, options) => { + assert.equal(url, 'http://app.test/api/fulfilment/ship'); + assert.equal(options.headers?.authorization, 'Bearer staff'); + assert.deepEqual(JSON.parse(String(options.body)), { orderId: '42' }); + submitted(); + await response; + return namedResponse(200, true); + }, + }); + let finished = false; + const pending = run(steps[index]!, provided).then(result => { finished = true; return result; }); + await started; + assert.equal(finished, false, 'the action cannot finish while its response is pending'); + release(); + assert.equal((await pending).status, 'passed'); + assert.equal(steps[index + 1]!.do, 'expectActionOutcome'); + assert.equal((await run(steps[index + 1]!, provided)).status, 'passed'); + assert.equal(steps[index + 2]!.do, 'reload'); +}); + +test('one named server action maps DOM input symmetrically and verifies its outcome', async () => { + const requests: CapturedRequest[] = []; + const source = { + name: 'source', + loc: (testid: string, options: UnknownRecord) => { + assert.equal(testid, 'item-card'); + assert.deepEqual(options, { contains: 'Desk Lamp' }); + return { + waitFor: async () => {}, + getAttribute: async (attribute: string) => { + assert.equal(attribute, 'data-action-input'); + return JSON.stringify({ itemId: 'item-42' }); + }, + }; + }, + }; + const guest = { name: 'guest' }; + const buyer = { name: 'buyer', writes: [{ headers: { authorization: 'Bearer buyer' } }] }; + const provided = services(new Map([['source', source], ['guest', guest], ['buyer', buyer]]), { + actions: [{ id: 'buy', path: '/api/items/:item/buy', reducer: 'buy_now', args: [0], + params: [{ name: 'itemId', in: 'path', placeholder: ':item' }] }], + fetchImpl: async (url, options) => { + requests.push({ url, options: options as unknown as UnknownRecord }); + return options.headers?.authorization === 'Bearer buyer' ? namedResponse(200, true) : namedResponse(401, false); + }, + }); + + const input = { testid: 'item-card', contains: 'Desk Lamp', attribute: 'data-action-input' }; + const called = await run({ do: 'callAction', actor: 'guest', from: 'source', action: 'buy', input, + authentication: 'none', settleMs: 0 }, provided); + assert.equal(called.status, 'passed'); + assert.equal(record(called.observation).action, 'buy'); + assert.equal(record(called.observation).accepted, false); + assert.equal(record(called.observation).status, 401); + const request = requests[0]; + assert(request); + assert.equal(request.url, 'http://app.test/api/items/item-42/buy'); + assert.deepEqual(JSON.parse(String(request.options.body)), {}); + assert.equal(Object.hasOwn(record(request.options.headers), 'Authorization'), false); + + // Until the same request succeeds for an authorized caller, the 401 may be a malformed call. + const unproven = await run({ do: 'expectActionOutcome', actor: 'guest', outcome: 'refused' }, provided); + assert.equal(unproven.status, 'inconclusive'); + assert.equal(unproven.finding?.kind, 'route-unproven'); + const authorized = await run({ do: 'callAction', actor: 'buyer', from: 'source', action: 'buy', input, settleMs: 0 }, + provided); + assert.equal(record(authorized.observation).accepted, true); + assert.equal(requests[1]?.url, request.url); + + const checked = await run({ do: 'expectActionOutcome', actor: 'guest', outcome: 'refused' }, provided); + assert.equal(checked.status, 'passed'); + assert.equal(record(checked.observation).classification, 'verified'); + assert.deepEqual(provided.verification.map(([kind]) => kind), ['verified']); +}); + +test('an action input on the one element inside its control is read from there', async () => { + const row = (inner: number) => ({ name: 'admin', loc: () => ({ + waitFor: async () => {}, + getAttribute: async () => null, + locator: (selector: string) => { + assert.equal(selector, '[data-action-input]'); + return { count: async () => inner, getAttribute: async () => JSON.stringify({ itemId: 1 }) }; + }, + }) }); + const action: NonNullable[number] = { id: 'restock', path: '/api/admin/restock', + reducer: 'admin_restock', args: [0], params: [{ name: 'itemId', in: 'body' }] }; + const call = (inner: number, fetchImpl?: NamedOptions['fetchImpl']) => run({ do: 'callAction', actor: 'admin', + action: 'restock', input: { testid: 'row', attribute: 'data-action-input' }, authentication: 'none', settleMs: 0 }, + services(new Map([['admin', row(inner)]]), { actions: [action], fetchImpl })); + const nested = await call(1, async (_url, options) => { + assert.deepEqual(JSON.parse(String(options.body)), { itemId: 1 }); + return namedResponse(200, true); + }); + assert.equal(nested.status, 'passed'); + // Two inner inputs leave the row's own value to stand, so the read cannot pick one. + const ambiguous = await call(2); + assert.equal(ambiguous.finding?.kind, 'interface-missing'); +}); + +test('a control that never appears is a page timeout, not a missing attribute', async () => { + let clock = 0; + const absent = { name: 'customer', loc: () => ({ + waitFor: async () => { + clock = Number.MAX_SAFE_INTEGER; + throw Object.assign(new Error('waiting for the control timed out'), { name: 'TimeoutError' }); + }, + getAttribute: async () => assert.fail('an absent control has no attribute to read'), + }) }; + const provided = services(new Map([['customer', absent]]), { actions: [{ + id: 'restock', path: '/api/admin/restock', reducer: 'admin_restock', args: [0], + params: [{ name: 'itemId', in: 'body' }] }] }); + const clocked = { ...provided, capabilities: { ...provided.capabilities, + 'named-actions': { ...record(provided.capabilities['named-actions']), now: () => clock } } }; + const result = await run({ do: 'callAction', actor: 'customer', action: 'restock', + input: { testid: 'row', attribute: 'data-action-input' }, authentication: 'none' }, clocked as typeof provided); + assert.equal(result.status, 'failed'); + assert.deepEqual(result.finding, { kind: 'page-timeout', fields: { control: 'row' } }); +}); + +test('named action input uses declared defaults and a missing route is not mistaken for a refusal', async () => { + const actor = (input: UnknownRecord) => ({ + name: 'customer', + loc: () => ({ waitFor: async () => {}, getAttribute: async () => JSON.stringify(input) }), + }); + const action: NonNullable[number] = { + id: 'restock', path: '/api/admin/restock', reducer: 'admin_restock', args: [0, 0, 1], + params: [{ name: 'itemId', in: 'body' }, { name: 'warehouseId', in: 'body' }, + { name: 'quantity', in: 'body' }], + }; + const withDefault = services(new Map([ + ['customer', actor({ itemId: 1, warehouseId: 2 })], + ]), { actions: [action], fetchImpl: async (_url, options) => { + assert.deepEqual(JSON.parse(String(options.body)), { itemId: 1, warehouseId: 2, quantity: 1 }); + return namedResponse(200, true); + } }); + const calledWithDefault = await run({ do: 'callAction', actor: 'customer', action: 'restock', + input: { testid: 'row', attribute: 'data-action-input' }, authentication: 'none' }, withDefault); + assert.equal(calledWithDefault.status, 'passed'); + + // Declared fields are all the attribute supplies; another value it carries + // cannot replace the scenario's test value. + const scoped = { ...action, args: [0, 0, 7] }; + const fields = { testid: 'row', attribute: 'data-action-input', fields: ['itemId', 'warehouseId'] }; + const withFields = services(new Map([ + ['customer', actor({ itemId: 1, warehouseId: 2, quantity: 3 })], + ]), { actions: [scoped], fetchImpl: async (_url, options) => { + assert.deepEqual(JSON.parse(String(options.body)), { itemId: 1, warehouseId: 2, quantity: 7 }); + return namedResponse(200, true); + } }); + const calledWithFields = await run({ do: 'callAction', actor: 'customer', action: 'restock', + input: fields, authentication: 'none' }, withFields); + assert.equal(calledWithFields.status, 'passed'); + const withoutField = services(new Map([['customer', actor({ itemId: 1 })]]), { + actions: [scoped], fetchImpl: async () => assert.fail('a call without a declared field must not be sent'), + }); + const missingField = await run({ do: 'callAction', actor: 'customer', action: 'restock', + input: fields, authentication: 'none' }, withoutField); + assert.equal(missingField.status, 'failed'); + assert.match(missingField.summary ?? '', /warehouseId/); + + const unexpected = services(new Map([ + ['customer', actor({ itemId: 1, warehouseId: 2, quantity: 3, extra: true })], + ]), { actions: [action] }); + const rejectedInput = await run({ do: 'callAction', actor: 'customer', action: 'restock', + input: { testid: 'row', attribute: 'data-action-input' }, authentication: 'none' }, unexpected); + assert.equal(rejectedInput.status, 'failed'); + assert.match(rejectedInput.summary ?? '', /unexpected extra/); + + const route = { name: 'route', actionCall: { action: 'restock', accepted: true, status: 200 } }; + const missing = services(new Map([ + ['customer', actor({ itemId: 1, warehouseId: 2, quantity: 3 })], ['route', route], + ]), { + actions: [action], fetchImpl: async () => namedResponse(404, false), + }); + await run({ do: 'callAction', actor: 'customer', action: 'restock', + input: { testid: 'row', attribute: 'data-action-input' }, authentication: 'none' }, missing); + const checked = await run({ do: 'expectActionOutcome', actor: 'customer', outcome: 'refused' }, missing); + assert.equal(checked.status, 'failed'); + assert.match(checked.summary ?? '', /does not meet the access-error status contract/); + assert.doesNotMatch(checked.summary ?? '', /was accepted|instead of refus/); + // A 404 names the operation the application interface requires, so a repair + // round can create the missing endpoint instead of chasing authorization. + assert.match(checked.summary ?? '', /the admin_restock reducer/); + assert.match(checked.summary ?? '', /POST \/api\/admin\/restock/); + + const privateResource = await run({ do: 'expectActionOutcome', actor: 'customer', outcome: 'refused', + routeProvenBy: 'route' }, missing); + assert.equal(privateResource.status, 'passed'); + assert.equal(record(privateResource.observation).status, 404); + + route.actionCall.action = 'different-action'; + const unrelatedProof = await run({ do: 'expectActionOutcome', actor: 'customer', outcome: 'refused', + routeProvenBy: 'route' }, missing); + assert.equal(unrelatedProof.status, 'failed'); +}); + +// A refusal is evidence only for a route this run has seen succeed. The same +// caller's earlier accepted request proves it as well as another actor's does. +test('a refusal counts only after the same action was accepted in this run', async () => { + for (const [first, then, expected] of [ + ['checkout', 403, 'passed'], ['checkout', 404, 'passed'], ['checkout', 500, 'failed'], + [null, 403, 'inconclusive'], [null, 404, 'failed'], ['restock', 403, 'inconclusive'], ['restock', 404, 'failed'], + ] as const) { + const provided = services(new Map([['customer', { name: 'customer', + writes: [{ headers: { authorization: 'Bearer customer' } }] }]]), { + fetchImpl: async (url, options) => options.headers?.authorization === 'Bearer customer' && url.endsWith(`/${first}`) + ? namedResponse(200, true) : namedResponse(then, false), + }); + const call = (action: string) => ({ do: 'callAction', actor: 'customer', action, settleMs: 0, + namedAction: { id: action, path: `/api/${action}`, reducer: action, args: [] } }); + if (first) assert.equal(record((await run(call(first), provided)).observation).accepted, true); + const refused = await run({ ...call('checkout'), authentication: 'none' }, provided); + assert.equal(record(refused.observation).status, then); + for (const outcome of ['refused', 'application-refused']) { + const checked = await run({ do: 'expectActionOutcome', actor: 'customer', outcome }, provided); + assert.equal(checked.status, expected, `${first} then ${then}: ${outcome}`); + if (expected === 'inconclusive') assert.equal(checked.finding?.kind, 'route-unproven'); + } + } +}); + +test('validation refusal accepts only deliberate application rejection statuses', async () => { + for (const [outcome, status, expected] of [ + ['validation-refused', 400, 'passed'], ['validation-refused', 409, 'passed'], ['validation-refused', 422, 'passed'], + ['validation-refused', 403, 'failed'], ['validation-refused', 500, 'failed'], + // Generic client errors do not prove a named action was refused for authorization. + ['refused', 400, 'passed'], ['refused', 409, 'passed'], ['refused', 422, 'passed'], + ] as const) { + const actor = { name: 'customer', + actionCall: { action: 'cart-set-quantity', accepted: false, status } }; + const owner = { name: 'owner', actionCall: { action: 'cart-set-quantity', accepted: true, status: 200 } }; + const provided = services(new Map([['customer', actor], ['owner', owner]])); + const checked = await run({ do: 'expectActionOutcome', actor: 'customer', outcome, + ...(outcome === 'refused' ? { routeProvenBy: 'owner' } : {}) }, provided); + assert.equal(checked.status, expected, checked.summary ?? undefined); + if (outcome === 'refused') { + const unproven = await run({ do: 'expectActionOutcome', actor: 'customer', outcome }, provided); + assert.equal(unproven.status, 'inconclusive', `${status} without a proven route`); + assert.equal(unproven.finding?.kind, 'route-unproven'); + } + } +}); + +test('business outcomes allow deliberate refusals but never turn server errors into completed work', async () => { + for (const outcome of ['completed', 'application-refused']) { + for (const status of [0, 200, 400, 401, 403, 404, 409, 422, 500, 530]) { + const actor = { name: 'customer', actionCall: { action: 'review', status, + accepted: status === 200, applicationRejected: status === 530 } }; + const owner = { name: 'owner', actionCall: { action: 'review', status: 200, accepted: true } }; + const provided = services(new Map([['customer', actor], ['owner', owner]])); + const result = await run({ do: 'expectActionOutcome', actor: 'customer', outcome }, provided); + const allowed = [400, 401, 403, 409, 422, 530].includes(status) + || (outcome === 'completed' && status === 200); + // A refusal of an operation never proven to work cannot measure the business rule. + const unproven = allowed && outcome === 'application-refused'; + assert.equal(result.status, !allowed ? 'failed' : unproven ? 'inconclusive' : 'passed', `${outcome}: ${status}`); + if (unproven) assert.equal(result.finding?.kind, 'route-unproven'); + // With the route proven, a 404 is the application hiding the resource; errors still fail. + const proven = await run({ do: 'expectActionOutcome', actor: 'customer', outcome, routeProvenBy: 'owner' }, provided); + assert.equal(proven.status, allowed || status === 404 ? 'passed' : 'failed', `${outcome} proven: ${status}`); + } + const customer = { name: 'customer', + actionCall: { action: 'review', status: 404, accepted: false } }; + const owner = { name: 'owner', + actionCall: { action: 'review', status: 200, accepted: true } }; + const provided = services(new Map([['customer', customer], ['owner', owner]])); + assert.equal((await run({ do: 'expectActionOutcome', actor: 'customer', outcome, + routeProvenBy: 'owner' }, provided)).status, 'passed'); + owner.actionCall.action = 'buy'; + assert.equal((await run({ do: 'expectActionOutcome', actor: 'customer', outcome, + routeProvenBy: 'owner' }, provided)).status, 'failed'); + } +}); + +test('purchase and restock privacy refusals require their successful control', async () => { + for (const [file, id] of [['01-purchase-session.json', '101a'], ['01-admin-write-staff.json', '103b']]) { + const scenario = JSON.parse(readFileSync(`tracks/ecommerce/scenarios/${file}`, 'utf8')); + const feature = scenario.features.find((feature: { criteria: { id: string }[] }) => + feature.criteria.some(criterion => criterion.id === id)); + const steps = [...feature.setup, ...feature.criteria.find((criterion: { id: string }) => criterion.id === id).steps]; + const refusal = steps.find(step => step.do === 'expectActionOutcome' + && ['refused', 'application-refused'].includes(step.outcome)); + const accepted = steps.findIndex(step => step.do === 'expectActionOutcome' + && step.outcome === 'accepted' && step.actor === refusal.routeProvenBy); + assert(accepted >= 0 && accepted < steps.indexOf(refusal)); + const control = steps.slice(0, accepted).findLast(step => step.do === 'callAction'); + const caller = { name: refusal.actor, actionCall: { action: control.action, status: 404, accepted: false } }; + const owner = { name: refusal.routeProvenBy, actionCall: { action: control.action, status: 200, accepted: true } }; + const provided = services(new Map([[caller.name, caller], [owner.name, owner]])); + assert.equal((await run(refusal, provided)).status, 'passed'); + owner.actionCall.accepted = false; + owner.actionCall.status = 404; + assert.equal((await run(refusal, provided)).status, 'failed'); + } +}); + +test('browser-origin calls need a matching positive control and a real browser observer', async () => { + let requests = 0; + const caller = { name: 'buyer', writes: [{ headers: { authorization: 'Bearer private' } }] }; + const provided = services(new Map([['buyer', caller]]), { fetchImpl: async () => { + requests++; return namedResponse(200, true); + } }); + const input = { do: 'callAction', actor: 'buyer', action: 'checkout', settleMs: 0, + namedAction: { id: 'checkout', path: '/api/checkout', reducer: 'checkout', args: [] } }; + assert.equal((await run({ ...input, browserOrigin: 'same-site' }, provided)).status, 'inconclusive'); + assert.equal(requests, 0); + assert.equal((await run(input, provided)).status, 'passed'); + assert.equal((await run({ ...input, browserOrigin: 'same-site' }, provided)).status, 'inconclusive'); + assert.equal(requests, 1, 'missing browser observation cannot fall back to a server fetch'); + assert.equal((await run({ do: 'expectActionOutcome', actor: 'buyer', outcome: 'accepted' }, provided)).status, 'inconclusive'); +}); + +test('restock role claims reach the real write with staff credentials and stored-state checks', async () => { + const scenario = JSON.parse(readFileSync('tracks/ecommerce/scenarios/01-admin-write-staff.json', 'utf8')); + const criterion = scenario.features[0].criteria.find((c: UnknownRecord) => c.id === '103b'); + assert.equal(criterion.points, 2); + const steps = criterion.steps as UnknownRecord[]; + const calls = steps.filter(step => step.do === 'callAction' && ['admin', 'staff'].includes(String(step.actor))); + assert.equal(calls.length, 4); + const attack = steps.indexOf(calls[2]!); + assert.equal(steps[attack - 1]!.do, 'dbRecordStock'); + assert.deepEqual(steps.slice(attack + 1, attack + 4).map(step => [step.do, step.outcome ?? step.plus]), [ + ['expectActionOutcome', 'completed'], ['dbExpectStock', 0], ['expectActionOutcome', 'application-refused'], + ]); + assert.equal(steps.at(-1)!.plus, 1, 'normal administrator access must still work after the claim'); + for (const backend of ['postgres', 'mongodb', 'spacetime']) { + const requests: { url: string; body: string; authorization: string | undefined }[] = []; + const provided = services(new Map(['admin', 'staff'].map(name => [name, { + name, writes: [{ headers: { authorization: `Bearer ${name}` } }], + loc: () => ({ waitFor: async () => {}, getAttribute: async () => + JSON.stringify({ itemId: '9007199254740993', warehouseId: '17', quantity: 1 }) }), + }])), { backend, spacetime: { uri: 'http://native.test', mod: 'shop' }, + fetchImpl: async (url, options) => { + requests.push({ url, body: options.body!, authorization: options.headers?.authorization }); + return namedResponse(403, false); + } }); + for (const call of calls) assert.equal((await run(call, provided)).status, 'passed'); + assert.deepEqual(requests.map(request => request.authorization), + ['Bearer admin', 'Bearer staff', 'Bearer staff', 'Bearer admin']); + assert(requests.every(request => request.url === requests[0]!.url), 'the claim must keep the original operation'); + if (backend === 'spacetime') { + assert.equal(requests[2]!.body, '[9007199254740993,17,1,"admin"]'); + assert.equal(requests[0]!.body, '[9007199254740993,17,1]'); + } else { + assert.deepEqual(JSON.parse(requests[2]!.body), { + itemId: '9007199254740993', warehouseId: '17', quantity: 1, role: 'admin', + }); + assert.equal(JSON.parse(requests[0]!.body).role, undefined); + } + } +}); + +test('shipping role and cancellation owner claims retain the caller and check fresh state before classifying refusal', async () => { + const scenario = JSON.parse(readFileSync('tracks/ecommerce/scenarios/02-server-actions.json', 'utf8')); + for (const [id, [caller, claimant], final] of [ + [201, ['staff', 'customer'], 'shipped'], [204, ['owner', 'other'], 'cancelled'], + ] as const) { + const feature = scenario.features.find((f: UnknownRecord) => f.id === id); + const owner = feature.setup.find((s: UnknownRecord) => s.do === 'signUp' && s.actor === 'owner'); + if (id === 204) assert.equal(owner.exact, true); + const criterion = feature.criteria[0]; + const steps = criterion.steps as UnknownRecord[], calls = steps.filter(step => step.do === 'callAction'); + assert.equal(criterion.points, 2); + assert.equal(calls.length, 4); + const attack = steps.indexOf(calls[2]!); + assert.equal(steps[attack + 1]!.outcome, 'completed'); + const refused = steps.findIndex((step, i) => i > attack && step.outcome === 'application-refused'); + assert(steps.slice(attack, refused).some(step => step.do === 'reload')); + assert.equal(steps[refused - 1]!.statusText, 'pending'); + assert.equal(steps.at(-1)!.statusText, final); + for (const backend of ['postgres', 'mongodb', 'spacetime']) { + const requests: { body: string; authorization: string | undefined }[] = []; + const provided = services(new Map([caller, claimant].map(name => [name, { name, + writes: [{ headers: { authorization: `Bearer ${name}` } }], + loc: () => ({ waitFor: async () => {}, getAttribute: async () => '{"orderId":"9007199254740993"}' }), + }])), { backend, spacetime: { uri: 'http://native.test', mod: 'shop' }, + fetchImpl: async (_url, options) => { requests.push({ body: options.body!, authorization: options.headers?.authorization }); return namedResponse(403, false); } }); + for (const call of calls) assert.equal((await run(call, provided)).status, 'passed'); + assert.deepEqual(requests.map(r => r.authorization), [caller, claimant, claimant, caller].map(name => `Bearer ${name}`)); + if (id === 201) { + if (backend === 'spacetime') assert.equal(requests[2]!.body, '[9007199254740993,"staff"]'); + else assert.deepEqual(JSON.parse(requests[2]!.body), { orderId: '9007199254740993', role: 'staff' }); + } else if (backend === 'spacetime') assert.equal(requests[2]!.body, `[9007199254740993,${JSON.stringify(owner.name)}]`); + else assert.deepEqual(JSON.parse(requests[2]!.body), { username: owner.name }); + } + } +}); + +test('fixed checkout price claims reach the native transport without inventing a DOM hook', async () => { + const scenario = JSON.parse(readFileSync(join(STACK_BENCH_ROOT, 'tracks/ecommerce/scenarios/progression-cart-checkout.json'), 'utf8')); + const steps = scenario.features[0].criteria.find((c: UnknownRecord) => c.id === '4d').steps as UnknownRecord[]; + const call = steps.find(step => step.do === 'callAction')!; + assert(steps.findIndex(step => step.do === 'dbExpectCheckout') < steps.indexOf(call)); + assert.equal(steps[steps.indexOf(call) + 2]!.do, 'dbExpectCheckout'); + assert.equal(steps[steps.indexOf(call) + 2]!.actor, 'checkout'); + for (const backend of ['postgres', 'mongodb', 'spacetime']) { + const requests: string[] = []; + const provided = services(new Map([['checkout', { name: 'checkout', + writes: [{ headers: { authorization: 'Bearer buyer' } }], + loc: () => { throw new Error('fixed body has no DOM input'); }, + }]]), { backend, spacetime: { uri: 'http://native.test', mod: 'shop' }, + fetchImpl: async (_url, options) => { + assert.equal(options.headers?.authorization, 'Bearer buyer'); + requests.push(options.body!); + return namedResponse(400, false); + } }); + assert.equal((await run(call, provided)).status, 'passed'); + assert.deepEqual(requests.map(body => JSON.parse(body)), backend === 'spacetime' ? [[1]] : [{ price: 1 }]); + } +}); + +test('price claims reach the purchase transport without replacing the normal purchase controls', async () => { + const scenario = JSON.parse(readFileSync(join(STACK_BENCH_ROOT, 'tracks/ecommerce/scenarios/01-server-price.json'), 'utf8')); + const criterion = scenario.features[0].criteria[0]; + assert.equal(criterion.points, 2); + const steps = criterion.steps as UnknownRecord[]; + const calls = steps.filter(step => step.do === 'callAction'); + assert.equal(calls.length, 4); + const attack = steps.indexOf(calls[2]!); + assert.deepEqual(steps.slice(attack + 1, attack + 3).map(step => [step.do, step.outcome]), + [['expectActionOutcome', 'completed'], ['dbExpectPurchase', undefined]]); + assert.equal(steps.at(-2)!.outcome, 'accepted'); + for (const backend of ['postgres', 'mongodb', 'spacetime']) { + const requests: { url: string; body: string; authorization: string | undefined }[] = []; + const provided = services(new Map([['buyer', { + name: 'buyer', writes: [{ headers: { authorization: 'Bearer buyer' } }], + loc: () => ({ waitFor: async () => {}, getAttribute: async () => JSON.stringify({ itemId: '9007199254740993' }) }), + }]]), { backend, spacetime: { uri: 'http://native.test', mod: 'shop' }, + fetchImpl: async (url, options) => { + requests.push({ url, body: options.body!, authorization: options.headers?.authorization }); + return namedResponse(400, false); + } }); + for (const call of calls) assert.equal((await run(call, provided)).status, 'passed'); + assert(requests.every(request => request.authorization === 'Bearer buyer')); + assert(requests.every(request => request.url === requests[0]!.url)); + if (backend === 'spacetime') { + assert.equal(requests[2]!.body, '[9007199254740993,1]'); + assert.equal(requests[3]!.body, '[9007199254740993]'); + } else { + assert.deepEqual(JSON.parse(requests[2]!.body), { price: 1 }); + assert.equal(JSON.parse(requests[3]!.body).price, undefined); + } + } +}); + +test('an invalid Spacetime u64 input fails before transport and cannot prove refusal', async () => { + let requests = 0; + const customer = { + name: 'customer', + loc: () => ({ waitFor: async () => {}, + getAttribute: async () => JSON.stringify({ itemId: '-1' }) }), + }; + const provided = services(new Map([['customer', customer]]), { + backend: 'spacetime', + spacetime: { uri: 'http://127.0.0.1:3000', mod: 'shop' }, + actions: [{ id: 'buy', path: '/api/items/:id/buy', reducer: 'buy_now', args: [0], + params: [{ name: 'itemId', in: 'path', placeholder: ':id', wireType: 'u64' }] }], + fetchImpl: async () => { requests += 1; return namedResponse(400, false); }, + }); + const called = await run({ do: 'callAction', actor: 'customer', action: 'buy', + input: { testid: 'item-card', attribute: 'data-buy-input' }, + authentication: 'none', settleMs: 0 }, provided); + assert.equal(called.status, 'failed'); + assert.match(called.summary ?? '', /input for the buy action is not valid/); + assert.equal(requests, 0); + + const checked = await run({ do: 'expectActionOutcome', actor: 'customer', outcome: 'refused' }, provided); + // An assertion whose action never ran measures the scenario, not the app. + assert.equal(checked.status, 'inconclusive'); + assert.match(checked.summary ?? '', /no callAction ran/); +}); + +test('account setup preserves scoped credentials and classifies browser failures', async () => { + const calls: unknown[][] = []; + let actualUser = 'Alicescope'; + const locator = (purpose: string) => ({ + first() { return this; }, + or() { return this; }, + filter() { return this; }, + isVisible: async () => true, + fill: async (value: string) => { calls.push([purpose, 'fill', value]); }, + inputValue: async () => actualUser, + innerText: async () => actualUser, + click: async () => { calls.push([purpose, 'click']); }, + waitFor: async (options: unknown) => { calls.push([purpose, 'waitFor', options]); }, + }); + const actor = { + loc: (id: string) => { + assert.notEqual(id, 'signup-toggle', 'a visible signup form does not need a toggle'); + return locator(`[data-testid="${id}"]`); + }, + page: { locator: (selector: string) => locator(selector) }, + }; + const passed = await run({ do: 'signUp', actor: 'a', name: 'Alice' }, + services(new Map([['a', actor]]))); + assert.equal(passed.status, 'passed'); + assert.equal(record(passed.observation).user, 'Alicescope'); + assert(calls.some(call => call[2] === 'pw-Alicescope')); + actualUser = 'Alice'; calls.length = 0; + const truncated = await run({ do: 'signUp', actor: 'a', name: 'Alice' }, + services(new Map([['a', actor]]))); + assert.equal(truncated.status, 'inconclusive'); + assert.match(JSON.stringify(truncated.finding), /input changed the requested username/); + assert(!calls.some(call => call[1] === 'click')); + // A refusal-tolerant signup must still deliver the exact account name, or it measures nothing. + const refusalTolerant = await run({ do: 'signUp', actor: 'a', name: 'Alice', expectFailure: true }, + services(new Map([['a', actor]]))); + assert.equal(refusalTolerant.status, 'inconclusive'); + assert.equal(refusalTolerant.finding?.kind, 'invalid-input'); + assert.match(JSON.stringify(refusalTolerant.finding), /input changed the requested username/); + assert(!calls.some(call => call[1] === 'click')); + + const timeout = Object.assign(new Error('locator.fill: timed out'), { name: 'TimeoutError' }); + const timedOutActor = { page: { locator: () => ({ first() { return this; }, + isVisible: async () => true, + fill: async () => { throw timeout; } }) } }; + const timedOut = await run({ do: 'signUp', actor: 'a', name: 'Alice' }, + services(new Map([['a', timedOutActor]]))); + assert.equal(timedOut.status, 'failed'); + assert.equal(timedOut.code, 'application_failure'); + + const buggyActor = { page: { locator: () => ({ first() { return this; }, + isVisible: async () => true, + fill: async () => { throw new TypeError('executor bug'); } }) } }; + const bug = await run({ do: 'signUp', actor: 'a', name: 'Alice' }, + services(new Map([['a', buggyActor]]))); + assert.equal(bug.status, 'harness_failure'); + assert.equal(bug.code, 'unclassified_exception'); +}); + +test('awaitSignedIn false leaves the signed-in view to the following expect', async () => { + const timeout = Object.assign(new Error('locator.waitFor: timed out'), { name: 'TimeoutError' }); + for (const action of ['signIn', 'signUp']) { + const prefix = action === 'signIn' ? 'signin' : 'signup'; + for (const signedInViewAppears of [true, false]) { + const calls: unknown[][] = []; + const locator = (purpose: string) => ({ + first() { return this; }, + or() { return this; }, + filter() { return this; }, + isVisible: async () => true, + fill: async (value: string) => { calls.push([purpose, 'fill', value]); }, + inputValue: async () => 'Alicescope', + innerText: async () => 'Alicescope', + click: async () => { calls.push([purpose, 'click']); }, + waitFor: async (options: unknown) => { + calls.push([purpose, 'waitFor', options]); + if (purpose.includes('current-user') && !signedInViewAppears) throw timeout; + }, + }); + const actor = { + loc: (id: string) => locator(`[data-testid="${id}"]`), + page: { locator: (selector: string) => locator(selector) }, + }; + const awaited = await run({ do: action, actor: 'a', name: 'Alice' }, + services(new Map([['a', actor]]))); + assert.equal(awaited.status, signedInViewAppears ? 'passed' : 'failed', action); + assert(calls.some(call => call[0] === '[data-testid="current-user"]' && call[1] === 'waitFor'), action); + calls.length = 0; + const submitted = await run({ do: action, actor: 'a', name: 'Alice', awaitSignedIn: false }, + services(new Map([['a', actor]]))); + // The refusal must land on the next expect, not on this interaction step. + assert.equal(submitted.status, 'passed', action); + assert.deepEqual(submitted.observation, { user: 'Alicescope', authenticationPath: 'local-form', submitted: true }); + assert(calls.some(call => call[0] === `[data-testid="${prefix}-submit"]` && call[1] === 'click'), action); + } + const compile = ACTION_REGISTRY.get(action).compile; + assert.doesNotThrow(() => compile({ do: action, actor: 'a', name: 'Alice', awaitSignedIn: false })); + assert.throws(() => compile({ do: action, actor: 'a', name: 'Alice', awaitSignedIn: 'no' }), /awaitSignedIn/); + } + assert.throws(() => ACTION_REGISTRY.get('ensureSignedIn') + .compile({ do: 'ensureSignedIn', actor: 'a', name: 'Alice', awaitSignedIn: false }), /awaitSignedIn/); +}); + +test('account restoration does not accept the wrong signed-in user', async () => { + const currentUser = { + first() { return this; }, + isVisible: async () => true, + innerText: async () => 'signed in as another-user', + }; + const actor = { + name: 'browser-a', + page: { locator: () => currentUser }, + }; + const result = await run({ do: 'ensureSignedIn', actor: 'a', name: 'alice' }, + services(new Map([['a', actor]]))); + assert.equal(result.status, 'failed'); + assert.match(result.summary ?? '', /different account/); +}); + +test('an unreplayable WebSocket write or missing transport evidence cannot earn server-side forgery credit', async () => { + for (const lastWsWrite of [{ event: 'send_message', body: { content: 'hello' } }, null]) { + const actor = { name: 'a', lastWrite: null, lastWsWrite }; + const provided = services(new Map([['a', actor], ['victim', {}]])); + const forged = await run({ do: 'forgeWrite', actor: 'a', fromActor: 'victim', settleMs: 0 }, provided); + assert.equal(forged.status, 'passed'); + assert.deepEqual(forged.observation, { attempted: false, classification: 'unverified' }); + + const checked = await run({ do: 'expectForgeryRejected', actor: 'a' }, provided); + assert.equal(checked.status, 'inconclusive'); + assert.match(checked.summary ?? '', /could not verify the forgery refusal/); + assert.deepEqual(provided.verification.map(([kind]) => kind), ['unverified']); + } +}); + +test('a negative delivery check observes its full window before passing', async () => { + const delays: number[] = []; + const actor = { name: 'outsider', wasSent: () => false }; + const provided = services(new Map([['outsider', actor]]), { + sleep: async milliseconds => { delays.push(milliseconds); }, + }); + const checked = await run({ do: 'expectNotReceived', actor: 'outsider', contains: 'secret', + within: 1234 }, provided); + assert.equal(checked.status, 'passed'); + assert.deepEqual(delays, [1234]); +}); + +test('a negative delivery check judges a window that has settled, and only that', async () => { + for (const pending of [0, 1]) { + const verdicts: unknown[][] = []; + const actor = { name: 'outsider', pendingReceived: pending, syncReceived: async () => {}, + wasSent: (_needle: string, requireComplete = true, settled = false) => { + if (requireComplete) verdicts.push([requireComplete, settled]); + return false; + } }; + const provided = services(new Map([['outsider', actor]]), { sleep: async () => undefined }); + const checked = await run({ do: 'expectNotReceived', actor: 'outsider', contains: 'secret', within: 1 }, provided); + assert.equal(checked.status, 'passed'); + // Nothing pending when the window closed: a request the app starts afterwards is outside it. + // Something still pending: the verdict stays strict. + assert.deepEqual(verdicts, [[true, pending === 0]]); + } +}); + +test('a private write is recorded from what is new to its owner and must not reach another actor', async () => { + const json = (value: unknown) => JSON.stringify(value); + const publicItem = json({ id: 3, name: 'Air Purifier', stock: 0 }); + const alert = json({ id: 7, accountId: 42, itemId: 3, delivered: false }); + const tick = (n: number) => json({ id: 1, now: n }); + const owner = { name: 'owner', pendingReceived: 0, deliveryMarks: new Map(), wasSent: () => false, + deliveries: [{ atMs: 1, payload: publicItem }] as { atMs: number; payload: string }[] }; + let clock = 1; + const sleep = async () => { clock++; owner.deliveries.push({ atMs: Date.now(), payload: tick(clock) }); }; + const actors = new Map([['owner', owner]]); + const provided = services(actors, { sleep }); + assert.equal((await run({ do: 'markDelivery', actor: 'owner', as: 'alert', phase: 'start' }, provided)).status, 'passed'); + // The write: the owner receives its alert, an unchanged public item and another clock tick. + owner.deliveries.push({ atMs: Date.now() + 1, payload: alert }, { atMs: Date.now() + 1, payload: publicItem }); + const ended = await run({ do: 'markDelivery', actor: 'owner', as: 'alert', phase: 'end' }, provided); + assert.deepEqual(ended.observation, { phase: 'end', privateRecords: 1 }, 'only the alert belongs to the write'); + for (const [received, expected] of [[[publicItem, tick(99)], 'passed'], [[publicItem, alert], 'failed']] as const) { + actors.set('other', { name: 'other', pendingReceived: 0, wasSent: () => false, + deliveries: received.map(payload => ({ atMs: 5, payload })) }); + const checked = await run({ do: 'expectNotDelivered', actor: 'other', from: 'owner', as: 'alert', within: 1 }, provided); + assert.equal(checked.status, expected, checked.summary ?? undefined); + } + actors.set('other', { name: 'other', pendingReceived: 0, wasSent: () => false, + deliveries: [{ atMs: 5, payload: Buffer.from([9, 0xff, 0x80]) }] }); + assert.equal((await run({ do: 'expectNotDelivered', actor: 'other', from: 'owner', as: 'alert', within: 1 }, provided)).status, + 'inconclusive', 'unreadable data is not absence'); + // A write the owner never saw as a record leaves nothing to compare; the visible checks remain. + owner.deliveryMarks.set('silent', { startedAtMs: 0, noisyKinds: new Set(), seen: new Set(), privateRecords: [] }); + const silent = await run({ do: 'expectNotDelivered', actor: 'other', from: 'owner', as: 'silent', within: 1 }, provided); + assert.equal(silent.status, 'passed'); + assert.deepEqual(silent.observation, { measured: false }); +}); + +test('a delivery check is inconclusive when application traffic is not observable', async () => { + const actor = { name: 'owner', wasSent: () => false }; + const provided = services(new Map([['owner', actor]]), { + sleep: async () => undefined, + }); + const checked = await run({ do: 'expectReceived', actor: 'owner', contains: 'private message', + within: 1 }, provided); + assert.equal(checked.status, 'inconclusive'); +}); + +test('replay retargeting maps nested entity ids by field and relationship depth', async () => { + const requests: CapturedRequest[] = []; + const actor = (name: string, received: string[], writes: UnknownRecord[]) => ({ + name, + received, + writes, + page: { + request: { fetch: async (url: string, options: UnknownRecord) => { + requests.push({ url, options }); + return { status: () => 200, ok: () => true }; + } }, + }, + }); + const staff = actor('staff', [[ + 'data: {"orders":[{"_id":"order-desk","userId":"staff-user",', + 'data: "items":[{"itemId":"item-desk","name":"Desk Lamp"}]}]}', + '', + ].join('\n')], [{ + url: 'http://app.test/api/fulfilment/order-desk/ship', method: 'POST', + headers: { authorization: 'Bearer staff-token' }, body: null, + }]); + const customer = actor('customer', [JSON.stringify({ order: { + _id: 'order-webcam', userId: 'customer-user', + items: [{ itemId: 'item-webcam', name: 'Webcam' }], + } })], [{ + url: 'http://app.test/api/items/item-webcam/buy', method: 'POST', + headers: { authorization: 'Bearer customer-token' }, body: null, + }]); + const provided = services(new Map([ + ['staff', staff], + ['customer', customer], + ])); + const replayed = await run({ do: 'replayAs', actor: 'customer', from: 'staff', match: 'ship', + swap: { find: 'Desk Lamp', with: 'Webcam' }, settleMs: 0 }, provided); + assert.equal(replayed.status, 'passed'); + assert.deepEqual(replayed.observation, + { attempted: true, accepted: true, status: 200 }); + const request = requests[0]; + assert(request); + assert.equal(request.url, 'http://app.test/api/fulfilment/order-webcam/ship'); + assert.equal(record(request.options.headers).authorization, 'Bearer customer-token'); + + const rejected = await run({ do: 'expectReplayRejected', actor: 'customer' }, provided); + assert.equal(rejected.status, 'failed'); + assert.match(rejected.summary ?? '', /who must be refused, was accepted/); +}); + +test('replay decodes Socket.IO entities and uses the target actor browser cookie', async () => { + const requests: CapturedRequest[] = []; + const actor = (name: string, received: string[], writes: UnknownRecord[], sid: string) => ({ + name, + received, + writes, + context: { cookies: async () => [{ name: 'sid', value: sid }] }, + page: { + evaluate: async () => null, + request: { fetch: async (url: string, options: UnknownRecord) => { + requests.push({ url, options }); + return { status: () => 403, ok: () => false }; + } }, + }, + }); + const staff = actor('staff', [JSON.stringify({ queue: [{ + id: 41, items: [{ itemId: 7, name: 'Desk Lamp' }], + }] })], [{ + url: 'http://app.test/api/fulfilment/ship', method: 'POST', + headers: { 'content-type': 'application/json' }, body: { orderId: 41 }, accepted: true, + }], 'staff-session'); + const customer = actor('customer', [ + JSON.stringify({ items: [{ id: 8, name: 'Webcam' }] }), + `42["orders:update",${JSON.stringify({ orders: [{ + id: 52, items: [{ orderItemId: 61, itemId: 8, name: 'Webcam' }], + }] })}]`, + ], [{ + url: 'http://app.test/api/items/8/buy', method: 'POST', + headers: { 'content-type': 'application/json' }, body: null, + }], 'customer-session'); + const provided = services(new Map([ + ['staff', staff], + ['customer', customer], + ])); + + const replayed = await run({ do: 'replayAs', actor: 'customer', from: 'staff', match: 'ship', + swap: { find: 'Desk Lamp', with: 'Webcam' }, settleMs: 0 }, provided); + assert.equal(replayed.status, 'passed'); + assert.deepEqual(replayed.observation, + { attempted: true, accepted: false, status: 403 }); + const request = requests[0]; + assert(request); + assert.deepEqual(JSON.parse(String(request.options.data)), { orderId: 52 }); + assert.match(String(record(request.options.headers).cookie), /sid=customer-session/); + + const rejected = await run({ do: 'expectReplayRejected', actor: 'customer' }, provided); + assert.equal(rejected.status, 'passed'); + assert.equal(record(rejected.observation).classification, 'verified'); + + // A source write the server did not accept does not prove the route works. + staff.writes[0]!.accepted = false; + assert.equal((await run({ do: 'replayAs', actor: 'customer', from: 'staff', match: 'ship', + swap: { find: 'Desk Lamp', with: 'Webcam' }, settleMs: 0 }, provided)).status, 'passed'); + assert.deepEqual(JSON.parse(String(requests[1]?.options.data)), { orderId: 52 }); + const unproven = await run({ do: 'expectReplayRejected', actor: 'customer' }, provided); + assert.equal(unproven.status, 'inconclusive'); + assert.equal(unproven.finding?.kind, 'route-unproven'); + assert.deepEqual(provided.verification.map(([kind]) => kind), ['verified']); +}); + +test('replay uses an authenticated named action when the source write is an opaque WebSocket call', async () => { + const requests: CapturedRequest[] = []; + const session = (token: string) => ({ context: { cookies: async () => [] }, + page: { evaluate: async (callback: () => unknown) => Function('window', + `return (${callback.toString()})()`)({ getSessionToken: () => token }) } }); + const source = { name: 'staff', writes: [], received: [], ...session('eyJstaff.token.value'), + lastWsWrite: { event: 'binary reducer call', body: {} }, + loc: (testid: string, options: UnknownRecord) => { + assert.equal(testid, 'order-item'); + assert.deepEqual(options, { contains: 'test-scoped' }); + return { + waitFor: async (value: unknown) => + assert.deepEqual(value, { state: 'visible', timeout: 5000 }), + getAttribute: async (attribute: string) => { + assert.equal(attribute, 'data-entity-id'); + return '52'; + }, + }; + } }; + const customer = { name: 'customer', writes: [], received: [], ...session('eyJcustomer.token.value') }; + const provided = services(new Map([ + ['staff', source], + ['customer', customer], + ]), { + backend: 'spacetime', + actions: [{ id: 'ship', path: '/api/fulfilment/ship', reducer: 'ship_order', args: [0] }], + spacetime: { uri: 'http://127.0.0.1:3000', mod: 'shop' }, + fetchImpl: async (url, options) => { + requests.push({ url, options: options as unknown as UnknownRecord }); + return record(options.headers).Authorization === 'Bearer eyJstaff.token.value' + ? namedResponse(200, true) : namedResponse(530, false); + }, + }); + + const replay = { do: 'replayAs', from: 'staff', match: 'ship', + namedAction: { id: 'ship', path: '/api/fulfilment/ship', reducer: 'ship_order', args: [0] }, + namedTarget: { testid: 'order-item', contains: '{room:test}', + attribute: 'data-entity-id', valueType: 'number' }, settleMs: 0 }; + const replayed = await run({ ...replay, actor: 'customer', swap: { find: '52', with: '53' } }, provided); + assert.equal(replayed.status, 'passed'); + assert.deepEqual(replayed.observation, + { attempted: true, accepted: false, status: 530, namedAction: 'ship' }); + const request = requests[0]; + assert(request); + assert.equal(request.url, 'http://127.0.0.1:3000/v1/database/shop/call/ship_order'); + assert.equal(request.options.body, '[53]'); + assert.equal(record(request.options.headers).Authorization, 'Bearer eyJcustomer.token.value'); + + // The refusal counts only after the same named action succeeds for the authorized staff member. + const unproven = await run({ do: 'expectReplayRejected', actor: 'customer' }, provided); + assert.equal(unproven.status, 'inconclusive'); + assert.equal(unproven.finding?.kind, 'route-unproven'); + const control = await run({ ...replay, actor: 'staff' }, provided); + assert.deepEqual(control.observation, { attempted: true, accepted: true, status: 200, namedAction: 'ship' }); + assert.equal(requests[1]?.options.body, '[52]'); + + const rejected = await run({ do: 'expectReplayRejected', actor: 'customer' }, provided); + assert.equal(rejected.status, 'passed'); + assert.equal(record(rejected.observation).classification, 'verified'); +}); + +test('named replay can replace a declared literal without an undeclared UI attribute', async () => { + const requests: CapturedRequest[] = []; + const source = { name: 'staff', writes: [], received: [], + lastWsWrite: { event: 'binary reducer call', body: {} } }; + const customer = { + name: 'customer', writes: [], received: [], + context: { cookies: async () => [] }, + page: { evaluate: async () => ['customer-token'] }, + }; + const provided = services(new Map([ + ['staff', source], + ['customer', customer], + ]), { + backend: 'spacetime', + spacetime: { uri: 'http://127.0.0.1:3000', mod: 'shop' }, + fetchImpl: async (url, options) => { + requests.push({ url, options: options as unknown as UnknownRecord }); + return namedResponse(530, false); + }, + }); + + const replayed = await run({ do: 'replayAs', actor: 'customer', from: 'staff', match: 'SAVE10', + swap: { find: 'SAVE10', with: 'HACK10' }, + namedAction: { id: 'createPromotion', path: '/api/promotions', reducer: 'create_promotion', + args: ['SAVE10', 10] }, settleMs: 0 }, provided); + assert.equal(replayed.status, 'passed'); + assert.equal(requests[0]?.options.body, '["HACK10",10]'); +}); + +test('a missing or malformed declared replay target is an application failure', async () => { + for (const [value, message] of [[null, /exposes no data-entity-id/], + ['not-an-id', /data-entity-id for the ship action is not valid/]] as const) { + const source = { name: 'staff', writes: [], received: [], + loc: () => ({ waitFor: async () => undefined, getAttribute: async () => value, + locator: () => ({ count: async () => 0 }) }) }; + const customer = { name: 'customer', writes: [], received: [] }; + const provided = services(new Map([ + ['staff', source], + ['customer', customer], + ])); + const replayed = await run({ do: 'replayAs', actor: 'customer', from: 'staff', match: 'ship', + namedAction: { id: 'ship', path: '/api/ship', reducer: 'ship', args: [0] }, + namedTarget: { testid: 'order', attribute: 'data-entity-id', valueType: 'number' }, + settleMs: 0 }, provided); + assert.equal(replayed.status, 'failed'); + assert.match(replayed.summary ?? '', message); + } +}); + +test('a replay refusal accepts expected application errors but not missing routes or transport faults', async () => { + // Match native application rejection semantics. A response proves refusal + // only; scenarios must still prove the protected state and useful access. + for (const status of [0, 302, 404, 503]) { + const actor = { + name: 'customer', + replay: { accepted: false, status, method: 'POST', url: '/ship' }, + }; + const provided = services(new Map([['customer', actor]])); + const checked = await run({ do: 'expectReplayRejected', actor: 'customer' }, provided); + assert.equal(checked.status, 'failed'); + assert.match(checked.summary ?? '', /does not meet the access-error status contract/); + assert.doesNotMatch(checked.summary ?? '', /was accepted|instead of refus/); + assert.equal(provided.verification.length, 0); + } + // A refusal measures authorization only for a route proven to work, such as + // a replay of a write the source actor completed successfully. + for (const status of [400, 401, 403, 409, 422]) { + for (const routeProven of [true, false]) { + const provided = services(new Map([['customer', { name: 'customer', replay: { + accepted: false, status, method: 'POST', url: '/ship', routeProven, + } }]])); + const checked = await run({ do: 'expectReplayRejected', actor: 'customer' }, provided); + assert.equal(checked.status, routeProven ? 'passed' : 'inconclusive', String(status)); + if (!routeProven) assert.equal(checked.finding?.kind, 'route-unproven'); + assert.equal(provided.verification.length, routeProven ? 1 : 0); + } + } + // A private-resource replay may explicitly treat not found as refusal, and a + // proven route makes not found the application hiding the resource. + for (const [allowNotFound, routeProven, expected] of [ + [true, true, 'passed'], [false, true, 'passed'], [true, false, 'inconclusive'], [false, false, 'failed'], + ] as const) { + const privateResource = services(new Map([['customer', { name: 'customer', + replay: { accepted: false, status: 404, method: 'POST', url: '/support/1/replies', routeProven } }]])); + assert.equal((await run({ do: 'expectReplayRejected', actor: 'customer', allowNotFound }, + privateResource)).status, expected, JSON.stringify({ allowNotFound, routeProven })); + } + for (const status of [200, 400, 401, 403, 404, 500]) { + const classified = classifyResponseContract({ responseContract: 'convex-mutation' }, { status, text: '{}' }); + const actor = { name: 'customer', replay: { ...classified, accepted: classified.ok, status, method: 'POST', url: '/api/mutation', + routeProven: true } }; + const provided = services(new Map([['customer', actor]])); + assert.equal((await run({ do: 'expectReplayRejected', actor: 'customer', allowNotFound: true }, provided)).status, + [401, 403].includes(status) ? 'passed' : 'failed', String(status)); + actor.replay.complete = false; + assert.equal((await run({ do: 'expectReplayRejected', actor: 'customer' }, provided)).status, 'inconclusive'); + } +}); + +test('an idempotent replay permits success or deliberate validation refusal, never a transport failure', async () => { + for (const status of [0, 200, 204, 302, 400, 401, 403, 404, 409, 422, 500, 530]) { + const actor = { name: 'staff', replay: { accepted: status >= 200 && status < 300, + status, method: 'POST', url: '/refund', applicationRejected: status === 530 } }; + const provided = services(new Map([['staff', actor]])); + const checked = await run({ do: 'expectReplayCompleted', actor: 'staff' }, provided); + assert.equal(checked.status, + [200, 204, 400, 409, 422, 530].includes(status) ? 'passed' : 'failed', String(status)); + assert.equal((await run({ do: 'expectReplayCompleted', actor: 'staff', requireAccepted: true }, provided)).status, + actor.replay.accepted ? 'passed' : 'failed'); + if (actor.replay.accepted) { + assert.equal((await run({ do: 'expectReplayRejected', actor: 'staff' }, provided)).status, + 'failed', 'idempotency acceptance must not weaken authorization checks'); + } + } + const absent = services(new Map([['staff', { name: 'staff' }]])); + assert.equal((await run({ do: 'expectReplayCompleted', actor: 'staff' }, absent)).status, + 'inconclusive'); +}); + +test('only an explicit authorization response proves a forged-write refusal', async () => { + for (const status of [400, 404, 422, 500]) { + const actor = { + name: 'attacker', + forge: { accepted: false, status, tamperedField: 'userId', reason: 'tampered request sent' }, + }; + const provided = services(new Map([['attacker', actor]])); + const checked = await run({ do: 'expectForgeryRejected', actor: 'attacker' }, provided); + assert.equal(checked.status, 'failed'); + assert.match(checked.summary ?? '', /does not meet the access-error status contract/); + assert.doesNotMatch(checked.summary ?? '', /was accepted|instead of refus/); + assert.equal(provided.verification.length, 0); + } +}); + +test('a missing numeric literal makes the server-side replay check inconclusive', async () => { + const requests: unknown[][] = []; + const buyer = { + name: 'buyer', + received: [JSON.stringify({ items: [{ _id: 'item-espresso', name: 'Espresso Machine', price: 449 }] })], + writes: [{ + url: 'http://app.test/api/items/item-espresso/buy', method: 'POST', + headers: { authorization: 'Bearer buyer-token' }, body: null, + }], + page: { request: { fetch: async (...args: unknown[]) => { requests.push(args); } } }, + }; + const provided = services(new Map([['buyer', buyer]])); + const replayed = await run({ do: 'replayAs', actor: 'buyer', from: 'buyer', match: 'buy', + swap: { find: '449', with: '1' }, settleMs: 0 }, provided); + assert.equal(replayed.status, 'inconclusive'); + assert.match(replayed.summary ?? '', /could not issue the replay as/); + assert.equal(requests.length, 0); +}); + +test('named calls scope browser cookies and captured request context to the action destination', async () => { + const seen: CapturedRequest[] = []; + const scoped: string[] = []; + const actor = (name: string) => ({ + name, + writes: [ + { url: 'http://app.test/api/cart', method: 'POST', body: {}, + headers: { cookie: 'stale=must-not-replay', 'x-csrf-token': `${name}-csrf`, + origin: 'http://app.test', referer: 'http://app.test/cart' } }, + { url: 'http://provider.test/token', method: 'POST', body: {}, + headers: { authorization: 'Bearer provider-private', cookie: 'provider=private', + 'x-csrf-token': 'provider-private', origin: 'http://provider.test' } }, + ], + context: { cookies: async (url: string) => { + scoped.push(url); + assert.equal(url, 'http://app.test/api/checkout'); + // The browser owns cookie domain/path/secure matching; an unscoped read + // would also expose provider and /account-only cookies. + return [{ name: 'sid', value: name }]; + } }, + page: { evaluate: async () => null }, + }); + const provided = services(new Map([['a', actor('a')], ['b', actor('b')]]), { + fetchImpl: async (url, options) => { + seen.push({ url, options: options as unknown as UnknownRecord }); + const headers = record(options.headers); + const user = String(headers.Cookie).slice(4); + const valid = headers.origin === 'http://app.test' + && headers['x-csrf-token'] === `${user}-csrf` && !headers.authorization; + return namedResponse(valid ? 200 : 403, valid); + }, + }); + for (const step of [ + { do: 'callAction', actor: 'a', action: 'checkout', settleMs: 0, + namedAction: { id: 'checkout', path: '/api/checkout', reducer: 'checkout', args: [] } }, + { do: 'expectActionOutcome', actor: 'a', outcome: 'accepted' }, + { do: 'callConcurrently', actors: ['a', 'b'], action: 'checkout', settleMs: 0 }, + { do: 'expectCallOutcomes', accepted: 2 }, + ]) { + const result = await run(step, provided); + assert.equal(result.status, 'passed', JSON.stringify(result)); + } + assert.equal(scoped.length, 3); + assert.equal(seen.length, 3); + assert(!JSON.stringify(seen).includes('provider-private')); + assert(!JSON.stringify(seen).includes('stale=must-not-replay')); + assert.equal(provided.calls?.action, 'checkout'); + assert.equal(provided.calls?.fired, 2); + assert.equal((await run({ do: 'expectCallOutcomes' }, provided)).status, 'passed', + 'idempotent success does not require a specific accepted-response count'); + const mismatch = await run({ do: 'expectCallOutcomes', accepted: 1 }, provided); + assert.equal(mismatch.status, 'failed'); + assert.match(mismatch.summary ?? '', /calls were accepted, expected 1/); +}); + +test('cross-account replay uses caller CSRF context and cannot credit missing caller context', async () => { + for (const callerContext of [true, false]) { + let requests = 0; + const owner = { name: 'owner', received: [], writes: [{ + url: 'http://app.test/api/orders/1/cancel', method: 'POST', body: {}, accepted: true, + headers: { cookie: 'sid=owner', 'x-csrf-token': 'owner-csrf', origin: 'http://app.test' }, + }] }; + const other = { name: 'other', received: [], writes: callerContext ? [{ + url: 'http://app.test/api/cart', method: 'POST', body: {}, + headers: { cookie: 'sid=other', 'x-csrf-token': 'other-csrf', origin: 'http://app.test' }, + }] : [], context: { cookies: async (url: string) => { + assert.equal(url, 'http://app.test/api/orders/1/cancel'); + return [{ name: 'sid', value: 'other' }]; + } }, page: { evaluate: async () => null, request: { fetch: async (_url: string, options: UnknownRecord) => { + requests++; + const headers = record(options.headers); + assert.equal(headers.cookie, 'sid=other'); + assert.equal(headers['x-csrf-token'], 'other-csrf'); + assert.equal(headers.origin, 'http://app.test'); + // A valid caller request reaches the ownership check and is refused. + return { status: () => 403, ok: () => false }; + } } } }; + const provided = services(new Map([['owner', owner], ['other', other]])); + const result = await run({ do: 'replayAs', actor: 'other', from: 'owner', match: 'cancel', settleMs: 0 }, provided); + assert.equal(result.status, callerContext ? 'passed' : 'inconclusive', JSON.stringify(result)); + assert.equal(requests, callerContext ? 1 : 0); + if (callerContext) assert.equal((await run({ do: 'expectReplayRejected', actor: 'other' }, provided)).status, 'passed'); + } +}); + +test('concurrent calls classify every result before counting accepted requests', async () => { + for (const backend of ['postgres', 'spacetime']) { + for (const status of [0, 400, 401, 403, 404, 409, 422, 500, 530]) { + const actors = new Map(['a', 'b'].map(name => [name, { name, + writes: [{ headers: { authorization: `Bearer ${name}-session` } }] }])); + let calls = 0; + const provided = services(actors, { backend, + spacetime: { uri: 'http://127.0.0.1:3000', mod: 'shop' }, + fetchImpl: async url => { + // A reducer called without arguments first reads its declared parameters. + if (url.endsWith('/schema?version=9')) return namedResponse(404, false); + if (calls++ === 0) return namedResponse(200, true); + if (status === 0) throw new Error('connection lost'); + return namedResponse(status, false); + } }); + assert.equal((await run({ do: 'callConcurrently', actors: ['a', 'b'], + action: 'checkout', settleMs: 0 }, provided)).status, 'passed'); + const checked = await run({ do: 'expectCallOutcomes', accepted: 1 }, provided); + assert.equal(checked.status, status === 0 ? 'inconclusive' : [400, 409, 422].includes(status) + || (backend === 'spacetime' && status === 530) ? 'passed' : 'failed', + `${backend}: ${status}`); + } + } + const provided = services(new Map()); + const named = provided.capabilities['named-actions'] as ReturnType; + // A missing outcome must not pass, and an unknown outcome outranks an error response in either order. + for (const [statuses, accepted] of [[[200], 1], [[0, 500], undefined], [[500, 0], undefined]] as const) { + named.lastCalls.set({ action: 'checkout', fired: 2, ms: 1, + outcomes: statuses.map(status => ({ name: 'a', status, ok: status === 200, text: '' })) }); + assert.equal((await run({ do: 'expectCallOutcomes', ...(accepted === undefined ? {} : { accepted }) }, provided)).status, + 'inconclusive', statuses.join(',')); + } +}); + +test('bounded checkout cohorts issue every request and retain client timing and transport outcomes', async () => { + const actors = new Map(['a', 'b'].map(name => [name, { name, + writes: [{ headers: { authorization: `Bearer ${name}-session` } }] }])); + for (const requests of [1, 4, 16, 64]) { + let dispatched = 0; + let release!: () => void; + const allDispatched = new Promise(resolve => { release = resolve; }); + const provided = services(actors, { fetchImpl: async () => { + dispatched++; + if (dispatched === requests) release(); + await allDispatched; + return namedResponse(200, true); + } }); + const result = await run({ do: 'callConcurrently', actors: ['a', 'b'], + action: 'checkout', requests, settleMs: 0 }, provided); + assert.equal(result.status, 'passed'); + const evidence = record(result.observation); + assert.equal(dispatched, requests); + assert.equal(evidence.responses, requests); + assert.equal(evidence.transportErrors, 0); + assert.equal(evidence.timeouts, 0); + assert.match(String(evidence.timingScope), /not server execution overlap/); + for (const [index, outcome] of provided.calls!.outcomes.entries()) { + assert.equal(outcome.requestIndex, index + 1); + assert.equal(outcome.name, index % 2 ? 'b' : 'a'); + assert.equal(outcome.transport, 'response'); + assert(outcome.completedAtMs! >= outcome.startedAtMs!); + assert.equal(outcome.durationMs, outcome.completedAtMs! - outcome.startedAtMs!); + } + } + let calls = 0; + const provided = services(actors, { fetchImpl: async (_url, options) => { + if (calls++ === 0) throw new Error('sensitive transport internals'); + return new Promise((_resolve, reject) => options.signal!.addEventListener('abort', + () => reject(options.signal!.reason), { once: true })); + } }); + const keepAlive = setInterval(() => {}, 20); + try { + const result = await run({ do: 'callConcurrently', actors: ['a', 'b'], + action: 'checkout', requests: 2, requestTimeoutMs: 10, settleMs: 0 }, provided); + assert.equal(record(result.observation).responses, 0); + assert.equal(record(result.observation).transportErrors, 1); + assert.equal(record(result.observation).timeouts, 1); + assert.doesNotMatch(JSON.stringify(result), /sensitive transport internals/); + assert.equal((await run({ do: 'expectCallOutcomes' }, provided)).status, 'inconclusive'); + } finally { clearInterval(keepAlive); } +}); + +test('named writes expose refused response bodies to the existing privacy check', async () => { + const received: string[] = []; + const actor = { name: 'guest', record: (text: string) => received.push(text), + wasSent: (needle: string) => received.some(text => text.includes(needle)) }; + const provided = services(new Map([['guest', actor]]), { + fetchImpl: async () => ({ ok: false, status: 403, text: async () => 'private-owner-address' }), + }); + assert.equal((await run({ do: 'callAction', actor: 'guest', authentication: 'none', action: 'checkout', + namedAction: { id: 'checkout', path: '/api/checkout', reducer: 'checkout', args: [] } }, provided)).status, 'passed'); + assert.equal((await run({ do: 'expectNotReceived', actor: 'guest', contains: 'private-owner-address' }, provided)).status, 'failed'); +}); + +test('cancelled cohorts drain requests and retain their history in action evidence', async () => { + for (const duringSettle of [false, true]) { + const controller = new AbortController(); + let drained = false; + let calls = 0; + const actors = new Map(['a', 'b'].map(name => [name, { name, + writes: [{ headers: { authorization: `Bearer ${name}-session` } }] }])); + const provided = services(actors, { + fetchImpl: async (_url, options) => { + if (calls++ === 0 || duringSettle) return namedResponse(200, true); + return new Promise((_resolve, reject) => { + options.signal!.addEventListener('abort', () => setTimeout(() => { + drained = true; + reject(options.signal!.reason); + }, 5), { once: true }); + controller.abort('test cancellation'); + }); + }, + sleep: async () => { controller.abort('test cancellation'); throw controller.signal.reason; }, + }); + const result = await executeAction(ACTION_REGISTRY, 'callConcurrently', { + do: 'callConcurrently', actors: ['a', 'b'], action: 'checkout', settleMs: 1, + }, { capabilities: provided.capabilities, signal: controller.signal, onAbort: async () => {} }); + assert.equal(result.code, 'cancelled'); + assert.equal(result.status, 'inconclusive'); + assert.equal(calls, 2); + assert.equal(drained, !duringSettle); + const history = record(result.observation); + assert.deepEqual(history.outcomes, provided.calls!.outcomes); + assert.equal(history.responses, duringSettle ? 2 : 1); + assert.equal(history.cancelled, duringSettle ? 0 : 1); + assert.doesNotMatch(JSON.stringify(result), /Bearer|session/); + } +}); + +test('a committed operation with a truncated HTTP success body remains unknown', { timeout: 10000 }, async () => { + let committed = 0; + const server = createServer((_request, response) => { + committed++; + response.writeHead(200, { 'Content-Type': 'application/json' }); + if (committed === 1) response.end('{"accepted":true}'); + else response.write('{"accepted":'); // Commit occurred, but the reply never finishes. + }); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + try { + const address = server.address(); + assert(address && typeof address !== 'string'); + const actors = new Map(['a', 'b'].map(name => [name, { name, + writes: [{ headers: { authorization: `Bearer ${name}-session` } }] }])); + const provided = services(actors, { fetchImpl: (url, options) => fetch( + url.replace('http://app.test', `http://127.0.0.1:${address.port}`), options) }); + const result = await run({ do: 'callConcurrently', actors: ['a', 'b'], action: 'checkout', + requestTimeoutMs: 1000, settleMs: 0 }, provided); + assert.equal(committed, 2); + assert.equal(record(result.observation).responses, 1); + assert.equal(record(result.observation).timeouts, 1); + assert.equal((await run({ do: 'expectCallOutcomes', accepted: 1 }, provided)).status, 'inconclusive'); + } finally { + const closed = new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())); + server.closeAllConnections(); + await closed; + } +}); + +test('purchase bursts reuse validated dynamic action inputs across stack transports', async () => { + for (const backend of ['postgres', 'mongodb', 'spacetime']) { + const requests: CapturedRequest[] = []; + let raw = '{"itemId":"9007199254740993"}'; + const actors = new Map(['a', 'b'].map(name => [name, { name, + writes: [{ headers: { authorization: `Bearer ${name}-session` } }], + loc: () => ({ waitFor: async () => {}, getAttribute: async () => raw }), + }])); + const provided = services(actors, { backend, + spacetime: { uri: 'http://127.0.0.1:3000', mod: 'shop' }, + fetchImpl: async (url, options) => { + requests.push({ url, options: options as unknown as UnknownRecord }); + return namedResponse(200, true); + } }); + const action = { do: 'callConcurrently', actors: ['a', 'b'], action: 'buy', + namedAction: { id: 'buy', path: '/api/items/:id/buy', reducer: 'buy_now', args: [0], + params: [{ name: 'itemId', in: 'path', placeholder: ':id', wireType: 'u64' }] }, + input: { testid: 'item-card', contains: 'Keyboard', attribute: 'data-buy-input' }, + requests: 4, settleMs: 0 }; + assert.equal((await run(action, provided)).status, 'passed', backend); + assert.equal(requests.length, 4); + for (const request of requests) { + if (backend === 'spacetime') assert.equal(request.options.body, '[9007199254740993]'); + else assert.match(request.url, /\/api\/items\/9007199254740993\/buy$/); + } + raw = '{"itemId":1,"unexpected":2}'; + assert.equal((await run(action, provided)).status, 'failed'); + assert.equal(requests.length, 4, 'malformed interface cannot dispatch another request'); + } +}); + +test('mixed groups prepare every input before release and keep each operation and buyer in the history', async () => { + let reads=0; + const requests: CapturedRequest[]=[]; + const actors=new Map(['a','admin'].map(name=>[name,{name, + writes:[{headers:{authorization:`Bearer ${name}`}}], + loc:()=>({waitFor:async()=>{},getAttribute:async()=>{reads++;return name==='a'?'{"itemId":"1"}':'{"itemId":"1","warehouseId":"2","quantity":2}';}}), + }])); + const provided=services(actors,{fetchImpl:async(url,options)=>{ + assert.equal(reads,2,'no mutation before all inputs are prepared'); + requests.push({url,options:options as unknown as UnknownRecord});return namedResponse(200,true); + }}); + const call={do:'callConcurrently',action:'buy',actors:['a'],requests:4,settleMs:0, + namedAction:{id:'buy',path:'/api/items/:id/buy',reducer:'buy_now',args:[0],params:[{name:'itemId',in:'path',placeholder:':id',wireType:'u64'}]}, + input:{testid:'item',attribute:'data-buy-input'},alongside:[{action:'restock',actors:['admin'],requests:4,delayMs:5, + namedAction:{id:'restock',path:'/api/admin/restock',reducer:'admin_restock',args:[0,0,1],params:[{name:'itemId',in:'body',wireType:'u64'},{name:'warehouseId',in:'body',wireType:'u64'},{name:'quantity',in:'body'}]}, + input:{testid:'stock',attribute:'data-restock-input'}}]}; + const result=await run(call,provided); + assert.equal(result.status,'passed',JSON.stringify(result));assert.equal(requests.length,8); + const outcomes=record(result.observation).outcomes as UnknownRecord[]; + assert.equal(outcomes.length,8); + for(const row of outcomes){ + assert.equal(row.name,row.action==='buy'?'a':'admin'); + assert.equal(row.dispatched,true); + assert.deepEqual(row.values,row.action==='buy'?{itemId:'1'}:{itemId:'1',warehouseId:'2',quantity:2}); + assert(Number(row.startedAtMs)>=Number(row.scheduledAtMs)); + } + for (const invalid of [{body:{}}, {actors:['admin','admin']}, {input:{}}, {alongside:[]}, + {namedAction:{...call.alongside[0]!.namedAction,id:'wrong'}}]) { + const rejected=await run({...call,alongside:[{...call.alongside[0],...invalid}]},provided); + assert.equal(rejected.status,'harness_failure'); + assert.equal(requests.length,8,'invalid nested input cannot dispatch mutations'); + } +}); + +test('named calls keep actor credentials separate in storage and in-memory accessors', async () => { + for (const inMemory of [false, true]) { + const requests: CapturedRequest[] = []; + const storage = (entries: ReadonlyArray) => ({ + length: entries.length, + key: (index: number) => entries[index]?.[0] ?? null, + getItem: (key: string) => + entries.find(([candidate]) => candidate === key)?.[1] ?? null, + }); + const actor = (name: string) => ({ + name, + context: { cookies: async () => [{ name: 'ui', value: name }] }, + page: { evaluate: async (browserFunction: () => unknown) => Function('localStorage', 'sessionStorage', 'window', + `return (${browserFunction.toString()})()`)( + storage(inMemory ? [] : [['theme', 'dark'], ['pgshop_token', `${name}-opaque-session-token-value`]]), + storage([]), inMemory ? { getSessionToken: () => `${name}-opaque-session-token-value` } : {}) }, + }); + const provided = services(new Map([ + ['a', actor('a')], + ['b', actor('b')], + ]), { + fetchImpl: async (url, options) => { + requests.push({ url, options: options as unknown as UnknownRecord }); + return { status: 200, ok: true, text: async () => '' }; + }, + }); + const called = await run({ do: 'callConcurrently', actors: ['a', 'b'], + action: 'checkout', settleMs: 0 }, provided); + assert.equal(called.status, 'passed'); + assert.equal(requests.length, 2); + const firstRequest = requests[0]; + const secondRequest = requests[1]; + assert(firstRequest && secondRequest); + assert.equal(record(firstRequest.options.headers).Authorization, + 'Bearer a-opaque-session-token-value'); + assert.equal(record(firstRequest.options.headers).Cookie, 'ui=a'); + assert.equal(record(secondRequest.options.headers).Authorization, + 'Bearer b-opaque-session-token-value'); + assert.equal(record(secondRequest.options.headers).Cookie, 'ui=b'); + } +}); + +test('unobserved session tokens cannot issue a replay or earn authorization credit', async () => { + for (const getSessionToken of [undefined, () => null, () => '', () => 42, + () => 'invalid\r\nheader', () => { throw new Error('private-token-sentinel'); }]) { + let requests = 0; + const actor = { name: 'customer', writes: [], received: [], + context: { cookies: async () => [] }, + page: { evaluate: async (callback: () => unknown) => { + const storage = { length: getSessionToken ? 1 : 0, key: () => 'auth_token', + getItem: () => 'stale-storage-token' }; + return Function('window', 'localStorage', 'sessionStorage', + `return (${callback.toString()})()`)({ getSessionToken }, storage, storage); + } } }; + const provided = services(new Map([ + ['customer', actor], ['staff', { name: 'staff', writes: [], received: [] }], + ]), { fetchImpl: async () => { requests++; return namedResponse(403, false); } }); + const replayed = await run({ do: 'replayAs', actor: 'customer', from: 'staff', match: 'ship', + namedAction: { id: 'ship', path: '/api/ship', reducer: 'ship_order', args: [52] }, + settleMs: 0 }, provided); + assert.equal(replayed.status, 'inconclusive'); + assert.match(replayed.summary ?? '', /could not issue the replay/); + if (getSessionToken) assert.match(JSON.stringify(replayed), /getSessionToken/); + assert.doesNotMatch(JSON.stringify(replayed), /private-token-sentinel|stale-storage-token/); + assert.equal(requests, 0); + const checked = await run({ do: 'expectReplayRejected', actor: 'customer' }, provided); + assert.equal(checked.status, 'inconclusive'); + } +}); + +test('missing named actions and application roots stay inconclusive', async () => { + const actor = { context: { cookies: async () => [{ name: 'sid', value: 'a' }] }, + page: { evaluate: async () => null } }; + const missingAction = await run({ do: 'callConcurrently', actors: ['a', 'b'], + action: 'missing', settleMs: 0 }, + services(new Map([['a', actor], ['b', actor]]), { actions: [] })); + assert.equal(missingAction.status, 'inconclusive'); + assert.match(missingAction.summary ?? '', /track names no missing action/); + + const missingRoot = await run({ do: 'runScript', script: 'backoffice.mjs', args: [] }, + services(new Map())); + assert.equal(missingRoot.status, 'inconclusive'); + assert.match(missingRoot.summary ?? '', /application directory is unknown/); +}); + +test('an application-owned script timeout is a scored application failure', async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-script-timeout-')); + try { + writeFileSync(join(root, 'slow.mjs'), 'await new Promise(resolve => setTimeout(resolve, 10000));\n'); + const result = await run({ do: 'runScript', script: 'slow.mjs', args: [], timeoutMs: 20 }, + services(new Map(), { appRoot: root })); + assert.equal(result.status, 'failed'); + assert.match(result.summary ?? '', /failed|timed out/i); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('named calls override one declared parameter from another actor without changing the caller', async () => { + for (const backend of ['postgres', 'mongodb', 'spacetime']) { + for (const doAction of ['callAction', 'callConcurrently']) { + const requests: CapturedRequest[] = []; + let targetId: string | null = '202'; + const owner = { name: 'owner', writes: [{ headers: { Authorization: 'Bearer owner' } }], loc: () => ({ waitFor: async () => {}, + getAttribute: async () => JSON.stringify({ caseId: '101', orderId: '303' }) }) }; + const other = { name: 'other', writes: [{ headers: { Authorization: 'Bearer other' } }], + loc: () => ({ waitFor: async () => {}, getAttribute: async () => targetId, + locator: () => ({ count: async () => 0 }) }) }; + const provided = services(new Map([['owner', owner], ['other', other]]), { + backend, spacetime: { uri: 'http://127.0.0.1:3000', mod: 'shop' }, + actions: [{ id: 'link', path: '/api/cases/{caseId}/order', reducer: 'link_support_order', + args: [0, 0], params: [{ name: 'caseId', in: 'path', placeholder: '{caseId}', wireType: 'u64' }, + { name: 'orderId', in: 'body', wireType: 'u64' }] }], + fetchImpl: async (url, options) => { + requests.push({ url, options: options as unknown as UnknownRecord }); + return namedResponse(200, true); + }, + }); + const selector = { actor: 'other', testid: 'support-ticket', attribute: 'data-entity-id' }; + const input = { do: doAction, ...(doAction === 'callAction' ? { actor: 'other' } : { actors: ['other', 'owner'], requests: 1 }), from: 'owner', action: 'link', + input: { testid: 'support-link-order', attribute: 'data-action-input', overrides: { caseId: selector } }, + settleMs: 0 }; + const called = await run(input, provided); + assert.equal(called.status, 'passed', JSON.stringify({ backend, doAction, called })); + const request = requests[0]!; + assert.equal(record(request.options.headers).authorization, 'Bearer other'); + if (backend === 'spacetime') { + assert.deepEqual(JSON.parse(String(request.options.body)), [202, 303]); + } else { + assert.equal(request.url, 'http://app.test/api/cases/202/order'); + assert.deepEqual(JSON.parse(String(request.options.body)), { orderId: '303' }); + } + const rejected = await run({ ...input, input: { ...input.input, overrides: { unknown: selector } } }, provided); + assert.equal(rejected.status, 'failed'); + assert.match(JSON.stringify(rejected.finding), /not declared/); + targetId = null; + assert.equal((await run(input, provided)).status, 'failed'); + assert.equal(requests.length, 1, 'invalid override must not send a request'); + } + } +}); + + +test('staff-role replay changes the role without changing the HTTP route or the reducer target', async () => { + const scenario = JSON.parse(readFileSync(join(STACK_BENCH_ROOT, + 'tracks/ecommerce/scenarios/progression-staff-roles.json'), 'utf8')); + const steps = scenario.features[0].criteria.find((c: { id: string }) => c.id === '621b').steps; + const replay = steps.find((step: { do: string; actor: string }) => step.do === 'replayAs' && step.actor === 'staff'); + // The administrator's accepted replay of the same named action proves the route the staff refusal uses. + const control = steps.find((step: { do: string; actor: string; namedAction?: { id: string } }) => step.do === 'replayAs' + && step.actor === 'replayAdmin' && step.namedAction?.id === replay.namedAction.id); + assert(steps.indexOf(control) < steps.indexOf(replay)); + assert.deepEqual(steps[steps.indexOf(control) + 1], { do: 'expectReplayCompleted', actor: 'replayAdmin', requireAccepted: true }); + for (const backend of ['postgres', 'mongodb', 'spacetime']) { + const requests: CapturedRequest[] = []; + const source = { name: 'replayAdmin', received: [], + writes: backend === 'spacetime' ? [{ + url: 'http://app.test/api/me', method: 'GET', headers: { authorization: 'Bearer admin-token' }, body: null, + }] : [{ + url: 'http://app.test/api/staff/42/role', method: 'PUT', + headers: { authorization: 'Bearer admin-token' }, body: { role: 'staff' }, + }], + loc: () => ({ waitFor: async () => undefined, getAttribute: async () => '42' }), + }; + const staff = { name: 'staff', received: [], writes: [{ + url: 'http://app.test/api/me', method: 'GET', + headers: { authorization: 'Bearer staff-token' }, body: null, + }], page: { request: { fetch: async (url: string, options: UnknownRecord) => { + requests.push({ url, options }); + return { status: () => 403, ok: () => false }; + } } } }; + const provided = services(new Map([['staff', staff], ['replayAdmin', source]]), { + backend, spacetime: { uri: 'http://app.test', mod: 'shop' }, + fetchImpl: async (url, options) => { + requests.push({ url, options: options as unknown as UnknownRecord }); + return record(options.headers).authorization === 'Bearer admin-token' ? namedResponse(200, true) : namedResponse(403, false); + }, + }); + assert.equal(record((await run({ ...control, settleMs: 0 }, provided)).observation).accepted, true); + assert.equal((await run({ ...replay, settleMs: 0 }, provided)).status, 'passed'); + assert.equal(requests.length, 2); + assert.equal(record(requests[1]!.options.headers).authorization, 'Bearer staff-token'); + assert.equal(requests[1]!.url, requests[0]!.url); + if (backend === 'spacetime') { + assert.equal(requests[1]!.options.body, '[42,"inventory"]'); + } else { + assert.equal(requests[1]!.url, 'http://app.test/api/staff/42/role'); + assert.equal(requests[1]!.options.body, '{"role":"inventory"}'); + } + assert.equal((await run({ do: 'expectReplayRejected', actor: 'staff' }, provided)).status, 'passed'); + } + const rejected = steps.findIndex((step: { do: string }) => step.do === 'expectReplayRejected'); + const after = steps.slice(rejected + 1); + assert.equal(after[0].do, 'reload'); + assert.equal(after.at(-1).value, 'staff', 'a denied response must leave the persisted role unchanged'); + assert.equal(after.at(-1).in.testid, 'staff-role-account-staff2'); +}); + + +test('role revocation uses declared transitions despite earlier captured role writes', async () => { + const scenario = JSON.parse(readFileSync(join(STACK_BENCH_ROOT, + 'tracks/ecommerce/scenarios/progression-staff-roles.json'), 'utf8')); + const steps = scenario.features[0].criteria.find((c: { id: string }) => c.id === '621d').steps; + const calls = steps.filter((step: { do: string }) => step.do === 'replayAs'); + for (const backend of ['postgres', 'mongodb', 'spacetime']) { + const requests: CapturedRequest[] = []; + const actor = (name: string) => ({ name, received: [], writes: [{ + url: 'http://app.test/api/staff/42/role', method: 'PUT', + headers: { authorization: `Bearer ${name}-token` }, body: { role: 'staff' }, + }], loc: (id: string) => ({ waitFor: async () => undefined, getAttribute: async () => id === 'staff-role-account-staff2' ? '43' : '42' }) }); + const provided = services(new Map([ + ['roleAdmin', actor('roleAdmin')], ['promotedStaff', actor('promotedStaff')], + ]), { backend, spacetime: { uri: 'http://app.test', mod: 'shop' }, + fetchImpl: async (url, options) => { + requests.push({ url, options: options as unknown as UnknownRecord }); + return namedResponse(200, true); + }, + }); + for (const step of calls) assert.equal((await run(step, provided)).status, 'passed'); + assert.equal(requests.length, calls.length); + assert.deepEqual(requests.map(request => { + const body = JSON.parse(String(request.options.body)); + return backend === 'spacetime' ? body[1] : body.role; + }), ['admin', 'inventory', 'staff', 'staff', 'inventory']); + assert.deepEqual(requests.map(request => backend === 'spacetime' + ? JSON.parse(String(request.options.body))[0] : Number(request.url.match(/staff\/(\d+)\/role/)![1])), + [42, 43, 43, 42, 43], 'the other staff account is reset before the same-session negative write'); + assert.deepEqual(requests.map(request => record(request.options.headers).authorization + ?? record(request.options.headers).Authorization), calls.map((step: { actor: string }) => `Bearer ${step.actor}-token`)); + } +}); + + +test('native envelopes govern single, concurrent and replay outcomes, including body loss', async () => { + for (const session of ['bearer', 'argument']) + for (const reply of ['accepted', 'refused', 'schema-refused', 'unhandled', 'malformed', 'body-loss', 'native400']) { + const nativeActor = (name: string, token: string) => { + const actor = { name, received: [], writes: [], context: { cookies: async () => [] }, + page: { evaluate: async () => [token] } }; + recordConvexSession(actor.page, 'ws://native.test/api/1.0.0/sync', JSON.stringify(session === 'bearer' + ? { type: 'Authenticate', tokenType: 'User', value: token } + : { type: 'Mutation', args: [{ session: token }] })); + return actor; + }; + // The owner's session proves the operation works; every buyer request gets the reply under test. + const provided = services(new Map([['buyer', nativeActor('buyer', 'real-browser-token')], + ['owner', nativeActor('owner', 'owner-browser-token')]])); + const prior = record(provided.capabilities['named-actions']); + const status = reply === 'native400' ? 400 : 200; + const text = reply === 'accepted' ? '{"status":"success","value":null}' + : reply === 'refused' ? '{"status":"error","errorMessage":"refused","errorData":null}' + : reply === 'schema-refused' ? JSON.stringify({ status: 'error', errorMessage: '[Request ID: c3c0e4b69f8972e5] Server Error\nArgumentValidationError: extra field' }) + : reply === 'unhandled' ? '{"status":"error","errorMessage":"missing function"}' : '{}'; + const native = { ...provided, capabilities: { ...provided.capabilities, 'named-actions': { + ...prior, + request: () => ({ url: 'http://native.test/api/mutation', method: 'POST', body: '{"path":"api:checkout","args":{}}', responseContract: 'convex-mutation' }), + classifyResponse: (request: Parameters[0], response: Parameters[1]) => + classifyResponseContract(request, response), + fetch: async (_url: string, options: { headers: Record; body: string }) => { + const owner = [options.headers.Authorization, JSON.parse(options.body).args.session].some(value => + value?.endsWith('owner-browser-token')); + const token = owner ? 'owner-browser-token' : 'real-browser-token'; + assert.equal(options.headers.Authorization, session === 'bearer' ? `Bearer ${token}` : undefined); + assert.equal(JSON.parse(options.body).args.session, session === 'argument' ? token : undefined); + if (owner) return { status: 200, ok: true, text: async () => '{"status":"success","value":null}' }; + return { status, ok: status === 200, text: async () => { if (reply === 'body-loss') throw new Error('lost'); return text; } }; + }, + } } }; + const call = await run({ do: 'callAction', actor: 'buyer', action: 'checkout', + namedAction: { id: 'checkout', path: '/api/checkout', reducer: 'checkout', args: [] }, settleMs: 0 }, native); + assert.equal(call.status, 'passed', JSON.stringify({ reply, call })); + const outcome = await run({ do: 'expectActionOutcome', actor: 'buyer', outcome: 'completed' }, native); + assert.equal(outcome.status, ['accepted', 'refused', 'schema-refused'].includes(reply) ? 'passed' + : reply === 'body-loss' ? 'inconclusive' : 'failed', reply); + const concurrentCall = await run({ do: 'callConcurrently', action: 'checkout', actors: ['buyer'], requests: 2, settleMs: 0 }, native); + assert.equal(concurrentCall.status, 'passed', JSON.stringify({ reply, concurrentCall })); + const concurrent = await run({ do: 'expectCallOutcomes' }, native); + assert.equal(concurrent.status, outcome.status, JSON.stringify({ reply, concurrent })); + await run({ do: 'replayAs', actor: 'buyer', from: 'buyer', match: 'checkout', + namedAction: { id: 'checkout', path: '/api/checkout', reducer: 'checkout', args: [] }, settleMs: 0 }, native); + const replay = await run({ do: 'expectReplayCompleted', actor: 'buyer' }, native); + assert.equal(replay.status, outcome.status, reply); + if (reply === 'schema-refused') { + for (const check of [{ do: 'expectActionOutcome', actor: 'buyer', outcome: 'refused' }, { do: 'expectReplayRejected', actor: 'buyer' }]) { + const unproven = await run(check, native); + assert.equal(unproven.status, 'inconclusive', check.do); + assert.equal(unproven.finding?.kind, 'route-unproven'); + } + const control = await run({ do: 'callAction', actor: 'owner', action: 'checkout', + namedAction: { id: 'checkout', path: '/api/checkout', reducer: 'checkout', args: [] }, settleMs: 0 }, native); + assert.equal(record(control.observation).accepted, true); + assert.equal((await run({ do: 'expectActionOutcome', actor: 'buyer', outcome: 'refused' }, native)).status, 'passed'); + assert.equal((await run({ do: 'expectReplayRejected', actor: 'buyer' }, native)).status, 'passed'); + } + } +}); + + +test('native POST queries cannot be forged as writes, and mutation arguments stay inside their envelope', async () => { + for (const withMutation of [false, true]) { + const query = { url: 'http://native.test/api/query', method: 'POST', headers: {}, + body: { path: 'api:list', args: { userId: 'buyer' } } }; + const mutation = { ...query, url: 'http://native.test/api/mutation', body: { path: 'api:write', args: [{ userId: 'buyer', message: 'old' }] } }; + const sent: unknown[] = []; + const actor = { name: 'buyer', writes: withMutation ? [mutation, query] : [query], lastWrite: query, + page: { request: { fetch: async (url: string, options: UnknownRecord) => { + assert.equal(url, mutation.url); sent.push(JSON.parse(String(options.data))); + return { status: () => 200, ok: () => true, text: async () => '{"status":"error","errorMessage":"denied","errorData":null}' }; + } } } }; + const victim = { name: 'victim', lastWrite: { ...mutation, body: { path: 'api:write', args: [{ userId: 'victim' }] } } }; + const provided = services(new Map([['buyer', actor], ['victim', victim]])); + const prior = record(provided.capabilities['named-actions']); + const native = { ...provided, capabilities: { ...provided.capabilities, 'named-actions': { ...prior, + classifyResponse: (request: Parameters[0], response: Parameters[1]) => + classifyResponseContract({ ...request, responseContract: request.url?.endsWith('/query') ? 'convex-query' : 'convex-mutation' }, response), + } } }; + const result = await run({ do: 'forgeWrite', actor: 'buyer', fromActor: 'victim', text: 'new', settleMs: 0 }, native); + assert.equal(result.status, 'passed', JSON.stringify(result)); + assert.equal(sent.length, withMutation ? 1 : 0); + if (withMutation) { + assert.deepEqual(sent[0], { path: 'api:write', args: [{ userId: 'victim', message: 'new' }] }); + assert.equal((await run({ do: 'expectForgeryRejected', actor: 'buyer' }, native)).status, 'passed'); + } else assert.equal((await run({ do: 'expectForgeryRejected', actor: 'buyer' }, native)).status, 'inconclusive'); + } +}); + +test('an accepted forged write is decided by its stored effect, not the status', async () => { + const actor = { name: 'attacker', + forge: { accepted: true, status: 200, tamperedField: 'userId', reason: 'tampered request sent' } }; + const provided = services(new Map([['attacker', actor]])); + const checked = await run({ do: 'expectForgeryRejected', actor: 'attacker' }, provided); + assert.equal(checked.status, 'passed'); + assert.deepEqual(provided.verification.map(([kind]) => kind), ['unverified']); +}); + +test('platform request headers reach single and concurrent calls under the caller bearer token', async () => { + const seen: UnknownRecord[] = []; + const actor = (token: string | null) => ({ name: token ?? 'guest', writes: [], context: { cookies: async () => [] }, + page: { evaluate: async (callback: () => unknown) => runInNewContext(`(${callback.toString()})()`, + { localStorage: { length: 0 }, sessionStorage: { length: 0 }, window: { getSessionToken: () => token } }) } }); + const provided = services(new Map([['a', actor('token-a')], ['b', actor('token-b')], ['guest', actor(null)]]), { + fetchImpl: async (_url, options) => { seen.push(record(options.headers)); return namedResponse(200, true); }, + }); + const named = record(provided.capabilities['named-actions']); + const platform = { ...named, request: () => ({ url: 'http://platform.test/rest/v1/rpc/checkout', method: 'POST', + body: '{}', headers: { apikey: 'project-key', Authorization: 'Bearer project-key' } }) }; + const capabilities = { ...provided.capabilities, 'named-actions': platform }; + const namedAction = { id: 'checkout', path: '/api/checkout', reducer: 'checkout', args: [] }; + for (const step of [ + { do: 'callAction', actor: 'a', action: 'checkout', namedAction, settleMs: 0 }, + { do: 'callAction', actor: 'guest', action: 'checkout', namedAction, authentication: 'none', settleMs: 0 }, + { do: 'callConcurrently', actors: ['a', 'b'], action: 'checkout', settleMs: 0 }, + ]) { + const result = await executeAction(ACTION_REGISTRY, step.do, step, { capabilities }); + assert.equal(result.status, 'passed', JSON.stringify(result)); + } + assert.deepEqual(seen.map(headers => [headers.apikey, headers.Authorization]), [ + ['project-key', 'Bearer token-a'], ['project-key', 'Bearer project-key'], + ['project-key', 'Bearer token-a'], ['project-key', 'Bearer token-b'], + ]); + assert(seen.every(headers => Object.keys(headers).filter(key => /^authorization$/i.test(key)).length === 1)); +}); diff --git a/tools/stack-bench/tests/agent-adapters.test.ts b/tools/stack-bench/tests/agent-adapters.test.ts new file mode 100644 index 00000000000..ac076e63554 --- /dev/null +++ b/tools/stack-bench/tests/agent-adapters.test.ts @@ -0,0 +1,302 @@ +import assert from 'node:assert/strict'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { runAgent } from '../commands/bench.js'; +import { parseBenchArguments } from '../commands/bench-arguments.js'; +import { killTree } from '../src/runtime/platform.js'; +import { runBounded } from '../src/runtime/bounded-process.js'; +import { createAgentVisibleTaskRequest, createBoundRecipeTaskRequest } from '../src/composition/recipe-selection.js'; +import { requireRecipeRelease } from '../src/composition/recipe-release.js'; +import { loadTrack } from '../src/composition/tracks.js'; + +import { AGENT_ADAPTER_SCHEMA_VERSION, agentRecipeIdentity, agentRequestArgv, + createAgentAdapterRegistry, defineAgentAdapter } + from '../src/agents/agent-adapter-contract.js'; +import { agentSessionFailure, validateAgentResult } + from '../src/agents/agent-result-contract.js'; +import { AGENT_ADAPTER_REGISTRY, agentAdapterIdentity } + from '../src/agents/agent-adapters.js'; +import type { AgentRequest } from '../src/agents/agent-adapter-contract.js'; +import type { PricingAuthority } from '../src/evidence/pricing-authority.js'; + +const request: AgentRequest = { mode: 'build', level: 1, app: 'C:\\bench\\app', backend: 'stub', + track: 'loop', runIndex: 0, model: 'deterministic', guidance: 'prescribed', skills: null }; +const pricing: PricingAuthority = { unit: 'USD-per-million-tokens', rates: { + input: 3, output: 15, cacheWrite5m: 3.75, cacheWrite1h: 6, cacheRead: 0.3, +} }; + +test('production framing follows every coding adapter but not model-free controls', () => { + for (const id of AGENT_ADAPTER_REGISTRY.ids) { + const adapter = AGENT_ADAPTER_REGISTRY.get(id); + const routed = adapter.provider === 'openrouter'; + // Grok Build and Antigravity requests must declare the output cap the broker prices. + const capped = routed || adapter.provider === 'xai' || adapter.provider === 'google'; + const input = { ...request, pricing, maxBudgetUsd: 12.5, + ...(routed ? { providerRoute: 'openai' } : {}), ...(capped ? { maxOutputTokens: 8192 } : {}) }; + const enabled = agentRequestArgv(adapter, { ...input, productionQuality: true }); + const legacy = agentRequestArgv(adapter, input); + assert.equal(enabled.includes('--no-production-quality'), false); + assert.equal(legacy.includes('--no-production-quality'), Boolean(adapter.provider)); + const flag = (name: string) => legacy.includes(name) ? legacy[legacy.indexOf(name) + 1] : undefined; + assert.equal(flag('--provider-route'), routed ? 'openai' : undefined); + assert.equal(flag('--max-output-tokens'), capped ? '8192' : undefined); + assert.equal(flag('--max-budget-usd'), adapter.costLimit === 'native' ? '12.5' : undefined); + assert.equal(flag('--pricing-json'), JSON.stringify(pricing)); + for (const mode of ['build', 'upgrade', 'resume', 'fix'] as const) { + if (adapter.modes.includes(mode)) assert.doesNotThrow(() => agentRequestArgv(adapter, { ...input, mode })); + else assert.throws(() => agentRequestArgv(adapter, { ...input, mode }), /does not support mode/); + } + } +}); + +test('adapter identity binds grading credentials as well as the executable', () => { + const adapter = AGENT_ADAPTER_REGISTRY.get('deterministic'); + assert.notEqual(agentAdapterIdentity(adapter).sha256, + agentAdapterIdentity({ ...adapter, gradesWithFixtureCredentials: true }).sha256); +}); + +test('a supervisor-owned deadline still requires and obeys cancellation', async () => { + await assert.rejects(runBounded(process.execPath,[],{ timeoutMs:null,stdio:'ignore' }),/owner abort signal/); + const owner = new AbortController(); + const timer = setTimeout(() => owner.abort(),100); + try { + const result = await runBounded(process.execPath,['-e','setInterval(()=>{},1000)'], { + timeoutMs:null,signal:owner.signal,stdio:'ignore', + }); + assert.equal(result.cancelled,true); + assert.equal(result.timedOut,false); + assert.equal(result.ok,false); + } finally { clearTimeout(timer); } +}); + +test('an external entry point receives exact visible tasks and shares one remaining budget across modes', async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-external agent-')); + try { + const entrypoint = join(root, 'agent.mjs'); + const received = join(root, 'received.jsonl'); + writeFileSync(entrypoint, `import {appendFileSync} from 'node:fs'; +const argv=process.argv.slice(2), args={}; +for(let i=0;i JSON.parse(line)); + assert.equal(deliveries.length,4,'budget checks must fail before starting another process'); + for (const delivery of deliveries) { + assert.equal(delivery.app,root); + assert.equal(delivery.model,'external model'); + assert.equal(delivery.provider,undefined); + assert.deepEqual(JSON.parse(delivery['recipe-task-json']),visible); + assert.deepEqual(JSON.parse(delivery['skills-json']),[]); + assert.deepEqual(JSON.parse(delivery['guidance-document-json']),args.guidanceDocument); + } + } finally { rmSync(root,{ recursive:true,force:true }); } +}); + +for (const exitsEarly of [false, true]) test(`standalone adapter deadline stops ${exitsEarly ? 'an orphaned child' : 'an unresponsive agent and its child'}`, { + skip: exitsEarly && process.platform !== 'linux' ? 'Linux process-group cleanup' : false, +}, async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-adapter-deadline-')); + const entrypoint = join(root, 'agent.mjs'); + const marker = join(root, 'pids.json'); + writeFileSync(entrypoint, `import {spawn} from 'node:child_process'; +import {writeFileSync} from 'node:fs'; +process.on('SIGTERM',()=>{}); +const child=spawn(process.execPath,['-e',"process.on('SIGTERM',()=>{});setInterval(()=>{},1000)"],{stdio:'inherit',detached:${!exitsEarly}}); +writeFileSync(${JSON.stringify(marker)},JSON.stringify([process.pid,child.pid])); +${exitsEarly ? 'process.exit(0);' : 'setInterval(()=>{},1000);'} +`); + const adapter = defineAgentAdapter({ ...AGENT_ADAPTER_REGISTRY.get('deterministic'), + id: 'external-test', entrypoint, deadlineMs: 2000 }); + const args = { ...parseBenchArguments(['node','bench','--backend','stub','--agent-adapter','deterministic']), + model: 'external-test', recipeTasks: new Map(), recipeBindings: new Map() }; + let watchdogUsed = false; + const watchdog = setTimeout(() => { + watchdogUsed = true; + if (existsSync(marker)) for (const pid of JSON.parse(readFileSync(marker,'utf8'))) killTree(pid); + }, 10000); + try { + await assert.rejects(runAgent(args, adapter, 'build', 1, root), /agent deadline exceeded/); + assert.equal(watchdogUsed, false, 'the adapter deadline must stop the complete process tree'); + const pids = JSON.parse(readFileSync(marker,'utf8')) as number[]; + for (const pid of pids) { + // Linux can retain a killed child as a zombie until its parent is reaped. + if (process.platform === 'linux' && existsSync(`/proc/${pid}/stat`) + && /\) Z /.test(readFileSync(`/proc/${pid}/stat`,'utf8'))) continue; + assert.throws(() => process.kill(pid, 0), { code: 'ESRCH' }); + } + } finally { + clearTimeout(watchdog); + if (existsSync(marker)) for (const pid of JSON.parse(readFileSync(marker,'utf8'))) killTree(pid); + rmSync(root, { recursive: true, force: true }); + } +}); + +test('a campaign-bound task supplies the exact recipe when no explicit recipe exists', () => { + const recipeTask = { schemaVersion: 3, + recipe: { id: 'ecommerce.sequential-l1' }, + selection: {}, task: {} }; + assert.equal(agentRecipeIdentity(null, recipeTask), 'ecommerce.sequential-l1'); + assert.throws(() => agentRecipeIdentity('ecommerce.sequential-l2', recipeTask), + /does not match bound task/); +}); + +test('completion validation rejects wrong identity and malformed usage', () => { + const nativeRequest: AgentRequest = { ...request, + adapterCostLimit: 'native', maxBudgetUsd: 1, pricing }; + const receipt = { schemaVersion: 3, source: 'credential-broker', exact: true, estimatedRequests: 0, + estimatedByReason: { 'no-usage': 0, 'response-aborted': 0, 'upstream-error': 0 }, model: request.model, + maxBudgetUsd: 1, costUsd: 0.0018, cliCostUsd: 0.0018, calculatedCostUsd: 0.0018, + usage: { input: 100, output: 100, cacheWrite5m: 0, cacheWrite1h: 0, cacheRead: 0 }, + pricingRates: { input: 3, output: 15, cacheWrite5m: 3.75, cacheWrite1h: 6, + cacheRead: 0.3 }, + complete: true, reconciled: true, error: null }; + const valid = { appDir: request.app, mode: 'build', level: 1, ok: true, + sessionId: 'session-1', costUsd: 0.0018, tokens: 3, outputTokens: 1, turns: 1, + promptBytes: 20, durationMs: 10, setup: { isolation: { mode: 'test' }, + resources: { buildContainerMemory: { + currentBytes: 100, peakBytes: 200, limitBytes: 400, + }, memoryProbeError: null } }, + usage: { input: 1, output: 1, cacheWrite: 1, cacheRead: 0 }, + costReceipts: [{ invocation: 1, receipt }] }; + const normalized = validateAgentResult(valid, nativeRequest); + assert.equal(normalized.backend, request.backend); + assert.equal(normalized.model, request.model); + assert.equal(validateAgentResult({ ...valid, productionQuality: true }, + { ...nativeRequest, productionQuality: true }).productionQuality, true); + assert.throws(() => validateAgentResult({ ...valid, productionQuality: 'true' }, nativeRequest), /productionQuality/); + assert.throws(() => validateAgentResult({ ...valid, productionQuality: true }, nativeRequest), /does not match/); + assert.deepEqual(normalized.transcript, { kind: 'provider-session', id: 'session-1' }); + assert.deepEqual(normalized.costReceipts, valid.costReceipts); + assert.deepEqual(normalized.setup.resources, valid.setup.resources); + const resources = { ...valid.setup.resources, + buildContainerMemory: { ...valid.setup.resources.buildContainerMemory, + oomEvents: 2, oomKillEvents: 1 }, + buildContainerPids: { current: 7, peak: 512, limit: 512, limitEvents: 129 } }; + assert.deepEqual(validateAgentResult({ ...valid, setup: { resources } }, nativeRequest) + .setup.resources, resources); + assert.throws(() => validateAgentResult({ ...valid, setup: { resources: { ...resources, + buildContainerPids: { ...resources.buildContainerPids, limitEvents: -1 } } } }, nativeRequest), + /limitEvents/); + assert.equal(normalized.costComplete, true); + // Codex reports tokens, not dollar charges; the broker is the cost authority. + assert.equal(validateAgentResult({ ...valid, costReceipts: [{ invocation: 1, + receipt: { ...receipt, cliCostUsd: null } }] }, nativeRequest).costComplete, true); + const routedRequest = { ...nativeRequest, providerRoute: 'openai' }; + const routedReceipt = { ...receipt, costSource: 'provider-reported', provider: 'openrouter', + providerRoute: 'openai', providerReportedCostUsd: receipt.costUsd, + upstreamProviders: ['OpenAI'], cliCostUsd: null, calculatedCostUsd: null }; + const routedResult = { ...valid, costReceipts: [{ invocation: 1, receipt: routedReceipt }] }; + assert.equal(validateAgentResult(routedResult, routedRequest).costComplete, true); + assert.throws(() => validateAgentResult(routedResult, { ...routedRequest, providerRoute: 'other' }), + /provider route/); + assert.throws(() => validateAgentResult({ ...routedResult, costReceipts: [{ invocation: 1, + receipt: { ...routedReceipt, providerReportedCostUsd: 0 } }] }, routedRequest), /cost evidence/); + const noLedger = validateAgentResult({ ...routedResult, ok: false, costReceipts: [{ invocation: 1, + receipt: { ...routedReceipt, providerRoute: undefined, providerReportedCostUsd: undefined, + upstreamProviders: undefined, exact: false, complete: false, reconciled: false, + error: 'ledger unavailable' } }] }, routedRequest); + assert.equal(noLedger.costComplete, false); + assert.throws(() => validateAgentResult({ ...valid, appDir: 'C:\\other' }, nativeRequest), /appDir/); + assert.throws(() => validateAgentResult({ ...valid, usage: { ...valid.usage, input: -1 } }, nativeRequest), + /usage.input/); + assert.throws(() => validateAgentResult({ ...valid, sessionId: undefined }, nativeRequest), /sessionId/); + assert.throws(() => validateAgentResult({ ...valid, + costReceipts: [{ invocation: 1, receipt: { ...receipt, model: 'wrong' } }] }, nativeRequest), + /costReceipts/); + assert.throws(() => validateAgentResult({ ...valid, costReceipts: [] }, nativeRequest), + /requires complete reconciled broker cost proof/); + assert.doesNotThrow(() => validateAgentResult({ ...valid, costUsd: 0.00185 }, nativeRequest)); + assert.throws(() => validateAgentResult({ ...valid, costUsd: 0.01 }, nativeRequest), + /requires complete reconciled broker cost proof/); + assert.throws(() => validateAgentResult(valid, { ...nativeRequest, + pricing: { ...pricing, rates: { ...pricing.rates, output: 12 } } }), + /requires complete reconciled broker cost proof/); + assert.throws(() => validateAgentResult({ ...valid, + costReceipts: [{ invocation: 1, receipt: { ...receipt, complete: false, + reconciled: false, error: 'incomplete' } }] }, nativeRequest), + /complete reconciled broker cost proof/); + const failed = validateAgentResult({ ...valid, ok: false, + costReceipts: [{ invocation: 1, receipt: { ...receipt, complete: false, + reconciled: false, error: 'incomplete' } }] }, nativeRequest); + assert.equal(failed.costComplete, false); + assert.equal(validateAgentResult({ ...valid, ok: false, unaccountedInvocations: 1 }, nativeRequest).costComplete, + false, 'an invocation without a receipt leaves the session cost unknown'); + const failedReceipt = failed.costReceipts[0]; + assert(failedReceipt); + assert.equal(failedReceipt.receipt.error, 'incomplete'); + assert.doesNotThrow(() => validateAgentResult({ ...valid, costUsd: 0, costReceipts: [] }, + { ...request, adapterCostLimit: 'non-billable' })); + assert.equal(validateAgentResult({ ...valid, costUsd: 0, costReceipts: [] }, + { ...request, adapterCostLimit: 'unsupported' }).costComplete, false); +}); + +test('provider failures stay separate from harness failures', () => { + assert.equal(agentSessionFailure({ ok: true, sessionId: 'session-1' }), null); + assert.deepEqual(agentSessionFailure({ ok: false, sessionId: 'session-2', + providerMetadata: { failureCode: 'provider-session-error' } }), { + kind: 'provider_failure', phase: 'coding-session', reason: 'provider-session-error', + provider: null, + appFailures: [], inconclusive: [], harnessFailures: [], + }); + const missingSession = agentSessionFailure({ ok: true, sessionId: null }); + assert(missingSession); + assert.equal(missingSession.reason, 'coding session did not run'); + const failedSession = agentSessionFailure({ ok: false, sessionId: null, + providerMetadata: { failureCode: 'coding-session-no-output', + diagnostic: 'coding session failed: permission denied' } }); + assert(failedSession); + assert.equal(failedSession.kind, 'harness_failure'); + assert.equal(failedSession.reason, 'coding session failed: permission denied'); + const capped = agentSessionFailure({ ok: false, sessionId: 'session-3', + providerMetadata: { failureCode: 'broker-budget', diagnostic: 'session budget cannot cover the next request' } }); + assert.equal(capped?.phase, 'cost-cap'); + assert.match(capped?.reason ?? '', /^Cost cap reached: session budget/); +}); + +test('malformed and duplicate agent adapters fail at registry construction', () => { + const source = { schemaVersion: AGENT_ADAPTER_SCHEMA_VERSION, id: 'fake', version: '1.0.0', + entrypoint: 'fake.mjs', modes: ['build'], deadlineMs: 1000, + defaultModel: 'fake-model', apiKeyEnvironmentVariable: null, + credentialEnvironmentVariables: [], credentialFiles: [], outboundDestinations: [], + requiredExecutables: [], + credentialStatusCommand: null, usesStackSkills: false, gradesWithFixtureCredentials: false, + costLimit: 'unsupported' }; + assert.equal(defineAgentAdapter(source).id, 'fake'); + assert.throws(() => createAgentAdapterRegistry([source, source]), /duplicate/); + assert.throws(() => defineAgentAdapter({ ...source, version: 'latest' }), /version/); + assert.throws(() => defineAgentAdapter({ ...source, modes: ['unknown'] }), /modes/); + assert.throws(() => defineAgentAdapter({ ...source, credentialFiles: ['..\\secret'] }), /credentialFiles/); + assert.throws(() => defineAgentAdapter({ ...source, + credentialEnvironmentVariables: ['not-valid'] }), /credentialEnvironmentVariables/); + assert.throws(() => defineAgentAdapter({ ...source, outboundDestinations: ['http://insecure.example'] }), + /outboundDestinations/); + assert.throws(() => defineAgentAdapter({ ...source, requiredExecutables: ['../claude'] }), + /requiredExecutables/); + assert.throws(() => defineAgentAdapter({ ...source, credentialStatusCommand: ['claude', 'bad\narg'] }), + /credentialStatusCommand/); + assert.throws(() => defineAgentAdapter({ ...source, command: 'node' }), /command is unknown/); +}); diff --git a/tools/stack-bench/tests/agent-credentials.test.ts b/tools/stack-bench/tests/agent-credentials.test.ts new file mode 100644 index 00000000000..0237250f44a --- /dev/null +++ b/tools/stack-bench/tests/agent-credentials.test.ts @@ -0,0 +1,38 @@ +import assert from 'node:assert/strict'; +import { resolve } from 'node:path'; +import test from 'node:test'; + +import { applyAgentCredential } from '../src/agents/agent-credentials.js'; +import type { AgentCredentialArgs } from '../src/agents/agent-credentials.js'; + +const paid = { id: 'paid', apiKeyEnvironmentVariable: 'PROVIDER_API_KEY' }; +const modelFree = { id: 'model-free', apiKeyEnvironmentVariable: null }; + +test('credential resolution reads only the selected adapter secret file', () => { + for (const { args, env, path } of [ + { args: {}, env: { PROVIDER_API_KEY_FILE: '/selected/key', ANTHROPIC_API_KEY_FILE: '/unrelated/key' }, + path: '/selected/key' }, + { args: {}, env: { STACK_BENCH_API_KEY_FILE: 'relative-key' }, path: 'relative-key' }, + ] as { args: AgentCredentialArgs; env: Record; path: string }[]) { + let readPath: string | null = null; + applyAgentCredential(args, paid, { env, read: file => { readPath = file; return 'selected-secret\n'; } }); + assert.equal(readPath, resolve(path)); + assert.equal(args.apiKey, 'selected-secret'); + assert.equal(args.apiKeyFile, resolve(path)); + } +}); + +test('model-free adapters ignore unrelated provider secrets but reject explicit credentials', () => { + const args: AgentCredentialArgs = {}; + applyAgentCredential(args, modelFree, + { env: { ANTHROPIC_API_KEY_FILE: '/mounted/by-appliance' } }); + assert.deepEqual(args, {}); + assert.throws(() => applyAgentCredential({}, modelFree, + { env: { STACK_BENCH_API_KEY_FILE: '/generic/key' }, read: () => 'secret' }), + /does not accept an API key/); +}); + +test('credential resolution rejects empty selected credentials', () => { + assert.throws(() => applyAgentCredential({}, paid, + { env: { PROVIDER_API_KEY_FILE: '/key' }, read: () => '\n' }), /is empty/); +}); diff --git a/tools/stack-bench/tests/agent-database.test.ts b/tools/stack-bench/tests/agent-database.test.ts new file mode 100644 index 00000000000..18617fc186a --- /dev/null +++ b/tools/stack-bench/tests/agent-database.test.ts @@ -0,0 +1,121 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { ensureDatabase, hostServiceAddress } from '../commands/agent.js'; +import { codingSessionFailure } from '../src/agents/coding-session-retry.js'; +import { createBackendLease, writeBackendLease } from '../src/runtime/backend-lease.js'; +import type { BackendLease } from '../src/runtime/backend-lease.js'; +import { loadTrack } from '../src/composition/tracks.js'; + +const track = loadTrack('ecommerce'); + +function withLease(backend: string, run: (lease: BackendLease) => Result): Result { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-agent-db-')); + const path = join(root, 'lease.json'); + const common = { runId: `agent-${backend}-run0-test`, backend, track: track.name, runIndex: 0 }; + const lease = backend === 'spacetime' + ? createBackendLease({ ...common, serverUri: 'http://127.0.0.1:3210', + module: 'app-ecom-run0', dataDir: join(root, 'data') }) + : createBackendLease({ ...common, database: 'app_ecom_run0', + container: { name: `stack-bench-${backend}`, id: backend.repeat(8) } }); + lease.state = 'active'; + writeBackendLease(path, lease); + const previousPath = process.env.STACK_BENCH_LEASE; + const previousToken = process.env.STACK_BENCH_LEASE_TOKEN; + process.env.STACK_BENCH_LEASE = path; + process.env.STACK_BENCH_LEASE_TOKEN = lease.ownershipToken; + try { return run(lease); } + finally { + if (previousPath === undefined) delete process.env.STACK_BENCH_LEASE; + else process.env.STACK_BENCH_LEASE = previousPath; + if (previousToken === undefined) delete process.env.STACK_BENCH_LEASE_TOKEN; + else process.env.STACK_BENCH_LEASE_TOKEN = previousToken; + rmSync(root, { recursive: true, force: true }); + } +} + +test('a failed PostgreSQL create is accepted only when the exact database exists', () => { + withLease('postgres', lease => { + const calls: Array<{ args: readonly string[]; options: { timeout?: number } }> = []; + const createError = new Error('docker daemon unavailable'); + const exec = (_command: string, args: readonly string[], options: { timeout?: number }) => { + calls.push({ args, options }); + const container = lease.resources.container; + assert(container); + if (args[0] === 'inspect') return `${container.id}\n`; + if (args.some(arg => String(arg).startsWith('CREATE DATABASE'))) throw createError; + if (args.includes('-tAc')) return ''; + return ''; + }; + assert.throws(() => ensureDatabase('postgres', 0, null, track, false, { exec, lease }), + error => error === createError); + assert.equal(calls.every(call => call.options.timeout === 120_000), true); + }); +}); + +test('a PostgreSQL or MongoDB wipe failure aborts a supposedly clean build', () => { + for (const backend of ['postgres', 'mongodb']) withLease(backend, lease => { + const exec = (_command: string, args: readonly string[]) => { + const container = lease.resources.container; + assert(container); + if (args[0] === 'inspect') return `${container.id}\n`; + if (backend === 'mongodb' || args.includes('dropdb')) throw new Error('wipe failed'); + return ''; + }; + assert.throws(() => ensureDatabase(backend, 0, null, track, true, { exec, lease }), + /could not wipe app_ecom_run0/); + }); +}); + +test('Spacetime cleanup ignores absence but rejects authorization and transport failures', () => { + withLease('spacetime', lease => { + const absent = Object.assign(new Error('404 Not Found'), { stderr: '404 Not Found' }); + assert.doesNotThrow(() => ensureDatabase('spacetime', 0, null, track, true, + { exec: () => { throw absent; }, stdbBin: 'spacetime-test', lease })); + const currentCliAbsent = Object.assign(new Error('command failed'), + { stderr: Buffer.from('Error: failed to find database `app-ecom-run0`.\n') }); + assert.doesNotThrow(() => ensureDatabase('spacetime', 0, null, track, true, + { exec: () => { throw currentCliAbsent; }, stdbBin: 'spacetime-test', lease })); + const unauthorized = Object.assign(new Error('401 Unauthorized'), { stderr: '401 Unauthorized' }); + assert.throws(() => ensureDatabase('spacetime', 0, null, track, true, + { exec: () => { throw unauthorized; }, stdbBin: 'spacetime-test', lease }), + /could not delete prior module/); + }); +}); + +test('Convex and Supabase setup requires the owned platform and prepares no database', () => { + const lease = createBackendLease({ runId: 'agent-platform-run0-test', backend: 'convex', + track: track.name, runIndex: 0, serverUri: 'http://127.0.0.1:13410' }); + const exec = () => { throw new Error('platform setup must not run database commands'); }; + for (const backend of ['convex', 'supabase']) { + lease.resources.container = null; + assert.throws(() => ensureDatabase(backend, 0, null, track, true, { exec, lease }), + new RegExp(`${backend} application setup requires its active owned deployment`)); + lease.resources.container = { name: `stack-bench-${backend}`, id: 'a'.repeat(64), owned: true }; + assert.equal(ensureDatabase(backend, 0, null, track, true, { exec, lease }), undefined); + } +}); + +test('exit 137 is reported as a kill without guessing that it was OOM', () => { + const failed = codingSessionFailure({ status: 1, + stdout: Buffer.from('provider stdout detail'), + stderr: Buffer.from('provider rejected the session') }); + assert.match(failed, /coding session failed \(exit 1\)/); + assert.match(failed, /inner stdout tail:\nprovider stdout detail/); + assert.match(failed, /inner stderr tail:\nprovider rejected the session/); + const detail = codingSessionFailure({ status: 137, + stderr: Buffer.from('STACK_BENCH_CODING_PROCESS_DIAGNOSTIC {"status":137,"cgroupMemory":"oom_kill 0"}') }); + assert.match(detail, /forcibly killed/); + assert.match(detail, /coding-process diagnostic/); + assert.doesNotMatch(detail, /out of memory|OOM failure/i); + assert.match(detail, /oom_kill 0/); +}); + +test('appliance uses the configured host-service address', () => { + assert.equal(hostServiceAddress({ STACK_BENCH_APPLIANCE: '1' }), '127.0.0.1'); + assert.equal(hostServiceAddress({}), 'host.docker.internal'); + assert.equal(hostServiceAddress({ STACK_BENCH_APPLIANCE: '1', + STACK_BENCH_HOST_ALIAS: 'explicit-host' }), 'explicit-host'); +}); diff --git a/tools/stack-bench/tests/agent-guidance.contract.ts b/tools/stack-bench/tests/agent-guidance.contract.ts new file mode 100644 index 00000000000..9dcc8e29719 --- /dev/null +++ b/tools/stack-bench/tests/agent-guidance.contract.ts @@ -0,0 +1,34 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { readBackendGuidanceDocument } from '../commands/agent.js'; +import { normalizePromptText } from '../src/agents/agent-materials.js'; +import { sha256 } from '../src/evidence/provenance.js'; + +const documentPath = 'backends/postgres.md'; +const absolute = join(STACK_BENCH_ROOT, documentPath); +const bytes = Buffer.from(normalizePromptText(readFileSync(absolute, 'utf8')), 'utf8'); +const identity = { + path: documentPath, + sha256: sha256(bytes), + bytes: bytes.length, + applicationInterface: 'http' as const, +}; + +test('campaign guidance loads only the exact content-identified document', () => { + assert.equal(readBackendGuidanceDocument(identity, 'unused.md'), bytes.toString('utf8')); + assert.throws(() => readBackendGuidanceDocument({ ...identity, sha256: '0'.repeat(64) }, + 'unused.md'), /changed after compilation/); + assert.throws(() => readBackendGuidanceDocument({ ...identity, bytes: bytes.length + 1 }, + 'unused.md'), /changed after compilation/); +}); + +test('campaign guidance rejects malformed and out-of-root paths', () => { + assert.throws(() => readBackendGuidanceDocument({ ...identity, path: '..\\secret.md' }, + 'unused.md'), /identity is invalid/); + assert.throws(() => readBackendGuidanceDocument({ ...identity, path: '../../Cargo.toml' }, + 'unused.md'), /escapes the Stack Bench root/); +}); diff --git a/tools/stack-bench/tests/agent-materials.test.ts b/tools/stack-bench/tests/agent-materials.test.ts new file mode 100644 index 00000000000..ba0af320b59 --- /dev/null +++ b/tools/stack-bench/tests/agent-materials.test.ts @@ -0,0 +1,38 @@ +import assert from 'node:assert/strict'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { agentSkillPaths, normalizePromptText, readAgentSkillDocuments, + selectAgentSkills } from '../src/agents/agent-materials.js'; + +test('stack defaults and explicit agent skill selections resolve predictably', () => { + assert.deepEqual(selectAgentSkills(['typescript-server'], null), ['typescript-server']); + assert.deepEqual(selectAgentSkills(['typescript-server'], []), []); + assert.deepEqual(selectAgentSkills([], ['typescript-server']), ['typescript-server']); + assert.throws(() => selectAgentSkills(['../private'], null), /invalid/); + assert.throws(() => agentSkillPaths('/repo/tools/stack-bench', ['same', 'same']), /invalid/); +}); + +test('benchmark workflows resolve separately from public SDK skills in selected order', () => { + const ids = ['typescript-server', 'spacetime-dev', 'spacetime-managed-dev']; + const paths = [join('/repo', 'skills', 'typescript-server', 'SKILL.md'), + join('/repo', 'tools', 'stack-bench', 'backends', 'workflows', 'spacetime-dev.md'), + join('/repo', 'tools', 'stack-bench', 'backends', 'workflows', 'spacetime-managed-dev.md')]; + assert.deepEqual(agentSkillPaths('/repo/tools/stack-bench', ids), paths); + assert.deepEqual(agentSkillPaths('/opt/stack-bench', ids), [ + join('/skills', 'typescript-server', 'SKILL.md'), + join('/opt/stack-bench/backends/workflows', 'spacetime-dev.md'), + join('/opt/stack-bench/backends/workflows', 'spacetime-managed-dev.md'), + ]); + assert.equal(readAgentSkillDocuments('/repo/tools/stack-bench', ids, { + read: path => `---\nname: ignored\n---\n${ids[paths.indexOf(path)]}`, + }), ids.join('\n\n---\n\n')); +}); + +test('prompt material is identical across platform line endings', () => { + assert.equal(normalizePromptText('one\r\ntwo\rthree\n'), 'one\ntwo\nthree\n'); + const text = readAgentSkillDocuments('/repo/tools/stack-bench', ['typescript-server'], { + read: () => '---\r\nname: ignored\r\n---\r\nreference\r\nline\r\n', + }); + assert.equal(text, 'reference\nline\n'); +}); diff --git a/tools/stack-bench/tests/agent-runtime-identity.contract.ts b/tools/stack-bench/tests/agent-runtime-identity.contract.ts new file mode 100644 index 00000000000..0c7ca35470b --- /dev/null +++ b/tools/stack-bench/tests/agent-runtime-identity.contract.ts @@ -0,0 +1,103 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { dbName, moduleName } from '../src/composition/tracks.js'; +import { loadTrack } from '../src/composition/tracks.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { CODING_CONTAINER_CONTROL_DIR, CODING_CONTAINER_PROCESS_IDENTITY, + codingContainerAgentCommand, codingContainerAgentEnvironment, + codingContainerAgentExecOptions, codingContainerTranscriptHandoffCommands } + from '../src/runtime/coding-container-policy.js'; +import { leasedDatabaseEnvironment, STACK_ADAPTER_REGISTRY } + from '../src/stacks/stack-adapters.js'; +import { POSTGRES_APPLICATION_IDENTITY, attemptDatabaseIdentity, attemptDatabaseUrl } + from '../src/stacks/hosted-database-identity.js'; +import { spacetimeBuildContainerPlan } from '../src/stacks/stack-agent-operations.js'; + +const FORBIDDEN_IDENTITY = /stackbench|stack[-_ ]bench|benchmark|harness|test|grader/i; + +test('transcript handoff gives only the controller group read access', () => { + assert.deepEqual(codingContainerTranscriptHandoffCommands(42), [ + ['chown', '-R', '10001:42', '/home/developer/.claude/projects/-app'], + ['chmod', '-R', 'u+rwX,g+rX,o-rwx', '/home/developer/.claude/projects/-app'], + ]); +}); + +test('the materialized agent runtime uses neutral application identities', () => { + const track = { ...loadTrack('ecommerce'), slug: 'ecom' }; + const database = dbName(track, 4); + const spacetimePlan = spacetimeBuildContainerPlan({ + repo: 'C:/product-source', appDir: 'C:/workspace/app', + env: { STACK_BENCH_RELEASE_DEPS_VOLUME: 'release-dependencies' }, + }); + const visibleRuntime = { + environment: codingContainerAgentEnvironment(), + execOptions: codingContainerAgentExecOptions(), + execCommand: codingContainerAgentCommand('claude', ['--print']), + controlDirectory: CODING_CONTAINER_CONTROL_DIR, + processIdentity: CODING_CONTAINER_PROCESS_IDENTITY, + database, + module: moduleName(track, 4), + postgres: leasedDatabaseEnvironment(STACK_ADAPTER_REGISTRY.get('postgres'), { + database, networkMode: 'bridge', + }), + postgresHostNetwork: leasedDatabaseEnvironment(STACK_ADAPTER_REGISTRY.get('postgres'), { + database, networkMode: 'host', + }), + mongodb: leasedDatabaseEnvironment(STACK_ADAPTER_REGISTRY.get('mongodb'), { + database, networkMode: 'bridge', + }), + spacetime: leasedDatabaseEnvironment(STACK_ADAPTER_REGISTRY.get('spacetime'), { + database: null, networkMode: 'host', + }), + spacetimeConfigTarget: (() => { + const mount = spacetimePlan.mounts[0]; + assert(mount); + return mount.target; + })(), + }; + + assert.doesNotMatch(JSON.stringify(visibleRuntime), FORBIDDEN_IDENTITY); + assert.deepEqual(visibleRuntime.environment, + { HOME: '/home/developer', USER: 'developer' }); + assert.equal(visibleRuntime.postgres.DATABASE_URL, + 'postgresql://appuser:local-app-password@host.docker.internal:6532/app_ecom_run4'); + assert.equal(visibleRuntime.mongodb.DATABASE_URL, + 'mongodb://host.docker.internal:6537/app_ecom_run4?replicaSet=rs0&directConnection=true'); + assert.equal(visibleRuntime.postgresHostNetwork.DATABASE_URL, + 'postgresql://appuser:local-app-password@127.0.0.1:6532/app_ecom_run4'); + assert.deepEqual(visibleRuntime.spacetime, {}); + assert.equal(visibleRuntime.module, 'app-ecom-run4'); +}); + +test('source PostgreSQL defaults and private attempt URLs keep neutral application identities', () => { + const expected = POSTGRES_APPLICATION_IDENTITY; + for (const relativePath of ['docker-compose.yaml']) { + const compose = readFileSync(join(STACK_BENCH_ROOT, relativePath), 'utf8'); + assert.match(compose, new RegExp(`POSTGRES_USER: ${expected.user}\\b`)); + assert.match(compose, new RegExp(`POSTGRES_PASSWORD: ${expected.password}\\b`)); + assert.match(compose, new RegExp(`POSTGRES_DB: ${expected.defaultDatabase}\\b`)); + assert.match(compose, + new RegExp(`pg_isready -U ${expected.user} -d ${expected.defaultDatabase}`)); + } + const token = 'a'.repeat(32); + const identity = attemptDatabaseIdentity(token); + const postgres = new URL(attemptDatabaseUrl({ backend: 'postgres', database: 'app_run1', ownershipToken: token })); + const mongo = new URL(attemptDatabaseUrl({ backend: 'mongodb', database: 'app_run1', ownershipToken: token })); + for (const url of [postgres, mongo]) { + assert.equal(url.username, expected.user); + assert.equal(url.hostname, '127.0.0.1'); + assert.equal(url.password, identity.password); + assert.equal(url.pathname, '/app_run1'); + assert.doesNotMatch(url.href, FORBIDDEN_IDENTITY); + } + assert.equal(postgres.port, '5432'); + assert.equal(mongo.port, '27017'); + assert.equal(mongo.searchParams.get('replicaSet'), 'rs0'); + assert.equal(mongo.searchParams.get('directConnection'), 'true'); + assert.notEqual(identity.password, attemptDatabaseIdentity('b'.repeat(32)).password); + assert.notEqual(identity.password, identity.adminPassword); + assert.doesNotMatch(JSON.stringify(expected), FORBIDDEN_IDENTITY); +}); diff --git a/tools/stack-bench/tests/agent-task-selection.contract.ts b/tools/stack-bench/tests/agent-task-selection.contract.ts new file mode 100644 index 00000000000..96965f7edcb --- /dev/null +++ b/tools/stack-bench/tests/agent-task-selection.contract.ts @@ -0,0 +1,303 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, readdirSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { agentScenarioPaths } from '../commands/agent.js'; +import { agentVisibleContractText, contractInterfaceNames } + from '../src/composition/agent-visible-contract.js'; +import { resolveGuidanceProfile } from '../src/campaigns/condition-compiler.js'; +import { createAgentVisibleTaskRequest, createBoundRecipeTaskRequest, + createRecipeTaskRequest, isModularRecipeTaskRequest } from '../src/composition/recipe-selection.js'; +import { requireRecipeRelease as resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { loadTrack } from '../src/composition/tracks.js'; + +const ROOT = STACK_BENCH_ROOT; +const AGENT = join(ROOT, 'dist', 'commands', 'agent.js'); +const binding = resolveRecipeRelease(loadTrack('ecommerce'), 1); + +type UnknownRecord = Record; +type VisibleTaskRequest = { task: { sha256: string }; selection: { requested: { checks: string[] } } }; + +const record = (value: unknown): value is UnknownRecord => + value !== null && typeof value === 'object' && !Array.isArray(value); + +function assertVisibleTaskRequest(value: unknown): asserts value is VisibleTaskRequest { + if (!record(value) || !record(value.task) || typeof value.task.sha256 !== 'string' + || !record(value.selection) || !record(value.selection.requested) + || !Array.isArray(value.selection.requested.checks) + || value.selection.requested.checks.some(check => typeof check !== 'string')) { + throw new Error('agent-visible task request is invalid'); + } +} + +function commandStderr(error: unknown): string { + if (!record(error) || !('stderr' in error)) return ''; + return String(error.stderr); +} + +test('selected contract linting uses only declared interface names', () => { + assert.deepEqual(contractInterfaceNames([ + '| Element ID | Element |', + '|---|---|', + '| `staff-link` | staff link |', + '| `staff-signin-submit` | sign-in button |', + 'The password is `stackbench-staff-2026`.', + 'The attribute is `data-action-input`.', + ].join('\n')), ['staff-link', 'staff-signin-submit']); +}); + +test('agent-facing documents reject internal evaluation language', () => { + for (const accepted of ['Expose the account controls.', 'Keep the contest action.']) { + assert.equal(agentVisibleContractText(accepted), accepted); + } + for (const source of [ + 'Stack Bench checks this.', + 'The grader checks this.', + 'The automated harness checks this.', + 'Expose this test action.', + 'Use data-testid="account-name".', + ]) { + assert.throws(() => agentVisibleContractText(source), /contains internal language/); + } +}); + +test('agent-visible contracts include only the selected stack section', () => { + const source = [ + 'HTTP contract', + 'reducer contract', + 'supabase contract', + ].join('\n'); + assert.equal(agentVisibleContractText(source, {}, 'http').trim(), 'HTTP contract'); + assert.equal(agentVisibleContractText(source, {}, 'reducer').trim(), 'reducer contract'); + assert.equal(agentVisibleContractText(source, {}, 'supabase').trim(), 'supabase contract'); + assert.throws(() => agentVisibleContractText(source), /requires a selected application interface/); + assert.equal(agentVisibleContractText('wrong', {}, 'http'), ''); + assert.throws(() => agentVisibleContractText('wrong', {}, 'http'), + /invalid markers/); + assert.throws(() => agentVisibleContractText('broken', {}, 'http'), + /invalid markers/); + assert.throws(() => agentVisibleContractText([ + 'outer', + 'nested', + '', + ].join('\n'), {}, 'http'), /invalid markers/); +}); + +test('every contract interface section belongs to an interface a guidance profile selects', () => { + const declared = new Set(readdirSync(join(ROOT, 'conditions', 'guidance')) + .flatMap(file => Object.values(JSON.parse(readFileSync(join(ROOT, 'conditions', 'guidance', file), 'utf8')) + .applicationInterfaces as Record))); + const contracts = join(ROOT, 'tracks', 'ecommerce', 'contracts'); + for (const file of readdirSync(contracts)) { + for (const [, id] of readFileSync(join(contracts, file), 'utf8').matchAll(//g)) { + assert(declared.has(id!), `${file} has a section for undeclared interface ${id}`); + } + } +}); + +function printPrompt(app: string, request: unknown, extraArgs: readonly string[] = []): string { + return execFileSync(process.execPath, [AGENT, '--mode', 'build', '--backend', 'postgres', + '--track', 'ecommerce', '--level', '1', '--app', app, + '--recipe-task-json', JSON.stringify(request), ...extraArgs, '--print-prompt'], { + encoding: 'utf8', stdio: 'pipe', + env: { ...process.env, STACK_BENCH_IMAGE: 'prompt-review-does-not-use-docker' }, + }); +} + +function printFixPrompt(app: string, request: unknown): string { + return execFileSync(process.execPath, [AGENT, '--mode', 'fix', '--backend', 'postgres', + '--track', 'ecommerce', '--level', '1', '--app', app, + '--recipe-task-json', JSON.stringify(request), '--print-prompt'], { + encoding: 'utf8', stdio: 'pipe', + env: { ...process.env, STACK_BENCH_IMAGE: 'prompt-review-does-not-use-docker' }, + }); +} + +function printStandalonePrompt(app: string, extraArgs: readonly string[] = []): string { + return execFileSync(process.execPath, [AGENT, '--mode', 'build', '--backend', 'postgres', + '--track', 'ecommerce', '--level', '1', '--app', app, ...extraArgs, '--print-prompt'], { + encoding: 'utf8', stdio: 'pipe', + env: { ...process.env, STACK_BENCH_IMAGE: 'prompt-review-does-not-use-docker' }, + }); +} + +test('pack selection changes the real model prompt and exact task identity', () => { + const app = mkdtempSync(join(tmpdir(), 'stack-bench-selected-task-')); + try { + const selected = createBoundRecipeTaskRequest(binding, + { featureIds: ['ecommerce.feature.accounts'] }); + const visible = createAgentVisibleTaskRequest(binding, selected); + const prompt = printPrompt(app, visible); + assert.match(prompt, /## Accounts/); + assert.match(prompt, /`data-role`/); + assert.doesNotMatch(prompt, /## Reviews/); + assert.doesNotMatch(prompt, /## Cart/); + assert.doesNotMatch(prompt, + /harness|runner|testing hooks|automated verification|test interface contract|\bhooks\b/i); + assert.doesNotMatch(prompt, /check-app|contract check|automated verification|data-testid/i); + assert.deepEqual(selected.selection.promptPacks, ['ecommerce.feature.accounts']); + + const tampered = structuredClone(visible); + assertVisibleTaskRequest(tampered); + tampered.task.sha256 = '0'.repeat(64); + assert.throws(() => printPrompt(app, tampered), error => + /recipe task changed after request resolution/.test(commandStderr(error))); + } finally { rmSync(app, { recursive: true, force: true }); } +}); + +test('repair prompts retain the selected feature request and public interface', () => { + const app = mkdtempSync(join(tmpdir(), 'stack-bench-repair-task-')); + try { + const selected = createBoundRecipeTaskRequest(binding, + { featureIds: ['ecommerce.feature.accounts'] }); + const prompt = printFixPrompt(app, createAgentVisibleTaskRequest(binding, selected)); + assert.match(prompt, /## Accounts/); + assert.match(prompt, /## Application interface/); + assert.doesNotMatch(prompt, /## Reviews|## Cart/); + assert.doesNotMatch(prompt, /grader|harness|automated verification|score/i); + } finally { rmSync(app, { recursive: true, force: true }); } +}); + +test('pack dependencies become requested task scope while checks only narrow measurement', () => { + const cart = createBoundRecipeTaskRequest(binding, + { featureIds: ['ecommerce.feature.cart-checkout'] }); + assert.deepEqual(cart.selection.promptPacks, [ + 'ecommerce.feature.accounts', + 'ecommerce.feature.cart', + 'ecommerce.feature.catalog-items', + 'ecommerce.feature.checkout', + ]); + assert.match(cart.task.requirementText, /signed-in customer/); + assert.match(cart.task.requirementText, /## Cart/); + assert.match(cart.task.requirementText, /## Checkout/); + + const oneKey = binding.release.checkCatalog.find(check => + check.packId === 'ecommerce.feature.accounts'); + assert(oneKey); + const full = createBoundRecipeTaskRequest(binding); + const checkOnly = createBoundRecipeTaskRequest(binding, { checkKeys: [oneKey.stableKey] }); + assert.deepEqual(checkOnly.selection.promptPacks, full.selection.promptPacks); + assert.equal(checkOnly.selection.checks.length, 1); + assert.equal(checkOnly.task.requirementText, full.task.requirementText); +}); + +test('ordinary runs select scored checks while test-development checks require exact selection', () => { + const ordinary = createRecipeTaskRequest(binding); + assert.equal(ordinary.selection.checks.length, 45); + assert.equal(ordinary.selection.checks.every(check => check.points > 0), true); + assert.equal(ordinary.selection.completeness, 'full'); + + const candidate = binding.release.checkCatalog.find(check => check.points === 0); + assert(candidate); + const development = createRecipeTaskRequest(binding, { checkKeys: [candidate.stableKey] }); + assert.deepEqual(development.selection.checks.map(check => check.stableKey), [candidate.stableKey]); + assert.equal(development.selection.scoredPoints, 0); + assert.equal(development.selection.completeness, 'subset'); + + const featureIds = binding.release.components.packs + .filter(pack => pack.moduleType === 'feature').map(pack => pack.id); + const expectedSpecifications = binding.release.components.packs + .filter(pack => pack.moduleType === 'specification').map(pack => pack.id); + const scoped = createBoundRecipeTaskRequest(binding, { featureIds, expectedSpecifications }); + const exact = createBoundRecipeTaskRequest(binding, { featureIds, expectedSpecifications, + checkKeys: binding.release.checkCatalog.map(check => check.stableKey) }); + assert.equal(exact.selection.checks.length, 47); + assert.equal(exact.selection.scoredPoints, 57); + assert.equal(exact.selection.checks.filter(check => check.points === 0).length, 2); + assert.deepEqual(exact.selection.promptPacks, scoped.selection.promptPacks); +}); + +test('selected pack prompts contain only their own framework-neutral named actions', () => { + const candidate = resolveRecipeRelease(loadTrack('ecommerce'), 1, + 'ecommerce.sequential-l1'); + const neutral = resolveGuidanceProfile('neutral', ['postgres']); + const app = mkdtempSync(join(tmpdir(), 'stack-bench-candidate-task-')); + try { + const identity = createBoundRecipeTaskRequest(candidate, + { featureIds: ['ecommerce.feature.accounts'] }); + const document = neutral.documents.postgres; + const skills = neutral.skills.postgres; + assert(document); + assert(skills); + const prompt = printPrompt(app, createAgentVisibleTaskRequest(candidate, identity), [ + '--guidance', neutral.mode, + '--guidance-document-json', JSON.stringify(document), + '--skills-json', JSON.stringify(skills.ids), + ]); + assert.match(prompt, /application's real authentication path/); + assert.doesNotMatch(prompt, /POST \/api\/auth\/(?:signup|signin)/); + assert.doesNotMatch(prompt, /\bExpress\b|socket\.io|Drizzle|Prisma/i); + assert.doesNotMatch(identity.task.requirementText, + /POST \/api\/checkout|POST \/api\/admin\/restock/); + + const cart = createBoundRecipeTaskRequest(candidate, + { featureIds: ['ecommerce.feature.cart-checkout'] }); + assert.match(cart.task.contractText, /POST \/api\/checkout/); + assert.match(cart.task.contractText, /application's real authentication path/); + assert.doesNotMatch(cart.task.contractText, /POST \/api\/admin\/restock/); + } finally { rmSync(app, { recursive: true, force: true }); } +}); + +test('the real unprescribed prompt withholds every expected quality specification', () => { + const modular = resolveRecipeRelease(loadTrack('ecommerce'), 1, + 'ecommerce.sequential-l1'); + const features = modular.release.components.packs + .filter(pack => pack.moduleType === 'feature').map(pack => pack.id); + const expectedSpecifications = modular.release.components.packs + .filter(pack => pack.moduleType === 'specification') + .map(pack => pack.id); + const task = createBoundRecipeTaskRequest(modular, { + featureIds: features, + expectedSpecifications, + }); + const app = mkdtempSync(join(tmpdir(), 'stack-bench-modular-task-')); + try { + const visible = createAgentVisibleTaskRequest(modular, task); + assert(isModularRecipeTaskRequest(task)); + const prompt = printPrompt(app, visible); + assert.match(prompt, /## Accounts/); + assert.match(prompt, /## Cart/); + assert.match(prompt, /## Checkout/); + assert.match(prompt, /## Warehouse administration/); + assert.doesNotMatch(prompt, /## Access control:|## State durability:|## Live state:|## Concurrency safety:|## Transactional integrity:/); + assert.doesNotMatch(prompt, /server-enforced authority|survives a page reload|only one customer can receive the last unit|historical order prices do not change/); + assert.doesNotMatch(JSON.stringify(visible), /ecommerce\.spec/); + assert.equal(task.selection.scoredPoints, 57); + assert(task.selection.scoredChecks.some(check => check.treatment === 'expected')); + assertVisibleTaskRequest(visible); + assert.deepEqual(visible.selection.requested.checks, []); + } finally { rmSync(app, { recursive: true, force: true }); } +}); + +test('agent provenance uses the exact recipe execution instead of level suites', () => { + const track = loadTrack('ecommerce'); + const modular = resolveRecipeRelease(track, 1, 'ecommerce.sequential-l1'); + const paths = agentScenarioPaths(track, 1, modular); + + assert.deepEqual(paths.map(path => path.replaceAll('\\', '/').split('/scenarios/')[1]), + modular.execution.map(execution => execution.source?.split('scenarios/')[1])); + assert(paths.some(path => path.endsWith('01-last-unit.json'))); + assert.equal(paths.some(path => path.endsWith('01-contention.json')), false); +}); + +test('a standalone recipe selects its exact prompt and cannot disagree with a bound task', () => { + const app = mkdtempSync(join(tmpdir(), 'stack-bench-standalone-recipe-')); + try { + const promoted = printStandalonePrompt(app); + const candidate = printStandalonePrompt(app, + ['--recipe', 'ecommerce.sequential-l1']); + assert.equal(candidate, promoted); + assert.match(candidate, /data-buy-input/); + assert.match(promoted, /data-buy-input/); + assert.match(promoted, /POST \/api\/items\/:id\/buy/); + + const request = createRecipeTaskRequest(binding).request; + assert.throws(() => printPrompt(app, request, + ['--recipe', 'ecommerce.sequential-l2']), error => + /does not match bound task/.test(commandStderr(error))); + } finally { rmSync(app, { recursive: true, force: true }); } +}); diff --git a/tools/stack-bench/tests/agy-login.test.ts b/tools/stack-bench/tests/agy-login.test.ts new file mode 100644 index 00000000000..c82a4725d88 --- /dev/null +++ b/tools/stack-bench/tests/agy-login.test.ts @@ -0,0 +1,62 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { ensureFreshAgyLogin, readAgyLogin } from '../src/agents/agy-login.js'; + +const MINUTE = 60_000; +const NOW = Date.parse('2026-10-09T12:00:00Z'); + +function login(expiresAt: number, access = 'ya29.access-1', subject = 'user-1') { + const claims = Buffer.from(JSON.stringify({ iss: 'https://accounts.google.com', aud: 'client-1', sub: subject, + email: 'kept@example.com' })).toString('base64url'); + return JSON.stringify({ token: { access_token: access, token_type: 'Bearer', refresh_token: '1//refresh-1', + expiry: new Date(expiresAt).toISOString() }, auth_method: 'consumer', id_token: `header.${claims}.signature` }); +} + +async function withLogin(text: string, body: (path: string) => Promise): Promise { + const root = mkdtempSync(join(tmpdir(), 'agy-login-')); + const path = join(root, 'agy_auth'); + writeFileSync(path, `${text}\n`, { mode: 0o600 }); + try { await body(path); } finally { rmSync(root, { recursive: true, force: true }); } +} + +test('an Antigravity login with enough time left is used as it is', async () => { + await withLogin(login(NOW + 50 * MINUTE), async path => { + const result = await ensureFreshAgyLogin(path, 40 * MINUTE, + () => assert.fail('no renewal while enough time remains'), { now: () => NOW }); + assert.equal(result.token, 'ya29.access-1'); + }); +}); + +test('an expiring Antigravity login is renewed by the CLI, written in place, and keeps its account identity', async () => { + await withLogin(login(NOW + 10 * MINUTE), async path => { + const before = readAgyLogin(readFileSync(path, 'utf8')); + const renewals: string[] = []; + const renew = async (text: string) => { renewals.push(text); return login(NOW + 60 * MINUTE, 'ya29.access-2'); }; + const results = await Promise.all([1, 2].map(() => ensureFreshAgyLogin(path, 40 * MINUTE, renew, { now: () => NOW, waitMs: 5 }))); + assert.equal(renewals.length, 1, 'concurrent sessions renew once'); + assert.deepEqual(results.map(result => result.token), ['ya29.access-2', 'ya29.access-2']); + assert.equal(results[0]!.identity, before.identity); + assert.equal(readAgyLogin(readFileSync(path, 'utf8')).token, 'ya29.access-2'); + }); +}); + +test('a renewal that changes the account or does not extend the sign-in is refused and leaves the file', async () => { + for (const renewed of [login(NOW + 60 * MINUTE, 'ya29.other', 'user-2'), login(NOW + 20 * MINUTE, 'ya29.short')]) { + await withLogin(login(NOW + 10 * MINUTE), async path => { + await assert.rejects(ensureFreshAgyLogin(path, 40 * MINUTE, async () => renewed, { now: () => NOW }), + /sign in with agy again/); + assert.equal(readAgyLogin(readFileSync(path, 'utf8')).token, 'ya29.access-1'); + }); + } +}); + +test('only a Google account sign-in from agy is accepted', () => { + assert.throws(() => readAgyLogin('{"token":{}}'), /not a Google account sign-in/); + assert.throws(() => readAgyLogin('not json'), /JSON file agy writes/); + const other = JSON.parse(login(NOW)); + other.id_token = `h.${Buffer.from(JSON.stringify({ iss: 'https://example.com', sub: 'x' })).toString('base64url')}.s`; + assert.throws(() => readAgyLogin(JSON.stringify(other)), /no Google account identity/); +}); diff --git a/tools/stack-bench/tests/antigravity-protocol.test.ts b/tools/stack-bench/tests/antigravity-protocol.test.ts new file mode 100644 index 00000000000..df0e576f91c --- /dev/null +++ b/tools/stack-bench/tests/antigravity-protocol.test.ts @@ -0,0 +1,147 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { chmodSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { ANTIGRAVITY_LAUNCHER, antigravityArguments, antigravityFinishedAnswer, parseAntigravityResult } from '../src/agents/antigravity-protocol.js'; + +const SESSION = '86eb1dda-1452-4d14-ab41-9afdc260bdd2'; +// A stream's steps end with the model's finished response unless `steps` says otherwise. +const stream = (result: Record | null, steps = ['agent_response']) => [ + { event: 'init', conversation_id: SESSION, init: { model: 'gemini-3.8-flash', cwd: '/app' } }, + ...steps.map((type, index) => ({ event: 'step_update', + step_update: { conversation_id: SESSION, step_index: index + 1, state: 'DONE', step_type: type } })), + ...(result ? [{ event: 'result', result: { conversation_id: SESSION, ...result } }] : []), +].map(event => JSON.stringify(event)).join('\n'); +const usage = (input: number, output: number, thinking: number, cached: number) => + ({ input_tokens: input, output_tokens: output, thinking_tokens: thinking, cache_read_tokens: cached, + total_tokens: input + output }); + +test('Antigravity reads the prompt as a stream message, pins model and effort, and resumes by conversation', () => { + assert.deepEqual(antigravityArguments({ model: 'gemini-3.8-flash', effort: 'medium', resumeSession: null }), + ['--print=', '--input-format', 'stream-json', '--output-format', 'stream-json', '--model', 'gemini-3.8-flash', + '--effort', 'medium', '--dangerously-skip-permissions']); + assert.deepEqual(antigravityArguments({ model: 'gemini-3.8-flash', effort: 'high', resumeSession: SESSION }).slice(-2), + ['--conversation', SESSION]); + // The CLI's name for Gemini 3.1 Pro covers both API models; the effort selects one. + const pro = antigravityArguments({ model: 'gemini-3.1-pro-preview-customtools', effort: 'high', resumeSession: null }); + assert.equal(pro[pro.indexOf('--model') + 1], 'gemini-3.1-pro'); +}); + +test('the launcher passes the prompt unchanged as one stream message and selects API-key mode', { + skip: process.platform === 'win32' ? 'needs a POSIX shell' : false, +}, () => { + const root = mkdtempSync(join(tmpdir(), 'antigravity-launcher-')); + try { + // A stand-in agy that records its stdin and arguments. + writeFileSync(join(root, 'agy'), `#!/bin/sh\ncat > "${root}/stdin"\nprintf '%s\\n' "$@" > "${root}/args"\n`); + chmodSync(join(root, 'agy'), 0o755); + const prompt = 'Build the "store".\nUse $HOME and `backticks` and \'quotes\'.\n'; + execFileSync('sh', ['-c', ANTIGRAVITY_LAUNCHER, 'agy', '--model', 'gemini-3.8-flash'], { input: prompt, + env: { PATH: `${root}:${process.env.PATH}`, HOME: root } }); + assert.deepEqual(JSON.parse(readFileSync(join(root, 'stdin'), 'utf8')), + { event: 'user', message: { role: 'user', content: prompt } }); + assert.equal(readFileSync(join(root, 'args'), 'utf8'), '--model\ngemini-3.8-flash\n'); + assert.deepEqual(JSON.parse(readFileSync(join(root, '.gemini', 'antigravity-cli', 'settings.json'), 'utf8')), + { modelProvider: 'gemini', enableTelemetry: false }); + // On the account route the CLI signs in with a stand-in carrying the broker's session token. + // The stand-in records what it holds itself and what a command it runs, as the CLI does, sees. + writeFileSync(join(root, 'agy'), `#!/bin/sh\ncat > /dev/null\nprintf '%s' "\${GEMINI_API_KEY-unset}" > "${root}/key"\n` + + `printf '%s %s' "\${HTTPS_PROXY-unset}" "\${SSL_CERT_FILE:+trusted}" > "${root}/own"\n` + + `bash -c 'sh -c "printf \\"%s %s %s\\" \\"\\\${HTTPS_PROXY-unset}\\" \\"\\\${SSL_CERT_FILE-unset}\\" \\"\\\${BASH_ENV-unset}\\""' > "${root}/command"\n`); + execFileSync('sh', ['-c', ANTIGRAVITY_LAUNCHER, 'agy'], { input: prompt, + env: { PATH: `${root}:${process.env.PATH}`, HOME: root, CLOUD_CODE_URL: 'http://127.0.0.1:1', GEMINI_API_KEY: 'session-1', + HTTPS_PROXY: 'http://127.0.0.1:1', BROKER_CA_CERT: 'CERTIFICATE' } }); + assert.equal(readFileSync(join(root, 'own'), 'utf8'), 'http://127.0.0.1:1 trusted', 'the CLI keeps its proxy and trust'); + assert.equal(readFileSync(join(root, 'command'), 'utf8'), 'unset unset unset', 'a command and its children see neither'); + const state = join(root, '.gemini', 'antigravity-cli'); + assert.deepEqual(JSON.parse(readFileSync(join(state, 'settings.json'), 'utf8')), { enableTelemetry: false }); + const standIn = JSON.parse(readFileSync(join(state, 'antigravity-oauth-token'), 'utf8')); + assert.equal(standIn.token.access_token, 'session-1'); + assert.ok(Date.parse(standIn.token.expiry) > Date.parse('2090-01-01')); + assert.equal(readFileSync(join(root, 'key'), 'utf8'), 'unset'); + // The launcher's files sit in its own directory, not among the agent's scratch files. + assert.deepEqual(readdirSync(join(root, '.config', 'antigravity-session')).sort(), ['ca.pem', 'command-env', 'message.json']); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +// The final usage of a live L1 session: input without the cache reads, output with thinking. +test('an Antigravity result maps to the shared result as reported', () => { + const result = parseAntigravityResult(stream({ status: 'SUCCESS', response: 'done\n', num_turns: 1, + usage: usage(732018, 74097, 35246, 10839770) })); + assert.deepEqual(result, { type: 'result', session_id: SESSION, is_error: false, result: 'done\n', num_turns: 1, + usage: { input_tokens: 732018, output_tokens: 74097, cache_read_input_tokens: 10839770, + cache_creation_input_tokens: 0 } }); +}); + +test('a resumed Antigravity conversation reports only this invocation', () => { + const prior = { input_tokens: 600, output_tokens: 80, cache_read_input_tokens: 400, cache_creation_input_tokens: 0 }; + const resumed = parseAntigravityResult(stream({ status: 'SUCCESS', response: 'ok', num_turns: 2, + usage: usage(1600, 160, 40, 900) }), prior); + assert.deepEqual(resumed.usage, { input_tokens: 1000, output_tokens: 80, cache_read_input_tokens: 500, + cache_creation_input_tokens: 0 }); + assert.equal(parseAntigravityResult(stream({ status: 'SUCCESS', response: 'ok', num_turns: 2, + usage: usage(100, 1, 0, 0) }), prior).is_error, true); +}); + +// Seen live: the resumed level-3 session finished, and its result repeated the 429 it was resumed after. +test('a resumed Antigravity conversation that finishes is not failed by the error it was resumed after', () => { + const prior = { input_tokens: 600, output_tokens: 80, cache_read_input_tokens: 400, cache_creation_input_tokens: 0 }; + const repeated = { status: 'ERROR', error: 'Error 429, RESOURCE_EXHAUSTED', response: 'UPGRADE_COMPLETE\n', num_turns: 2, + usage: usage(1600, 160, 40, 900) }; + const resumed = parseAntigravityResult(stream(repeated), prior); + assert.equal(resumed.is_error, false); + assert.equal(resumed.result, 'UPGRADE_COMPLETE\n'); + assert.equal(parseAntigravityResult(stream(repeated, ['agent_response', 'error_message']), prior).is_error, true); + // Seen live at level 3: a first invocation retried past a 429, finished, and still named it. + const retried = parseAntigravityResult(stream(repeated, ['tool', 'error_message', 'tool', 'agent_response'])); + assert.equal(retried.is_error, false); + assert.equal(retried.result, 'UPGRADE_COMPLETE\n'); +}); + +// Seen live: the agent started the app as a background command, so the CLI never exited or +// streamed its DEPLOY_COMPLETE; stopped, it reported "interrupted" with its usage. +test('an Antigravity session stopped after its finished answer returns that answer', () => { + const root = mkdtempSync(join(tmpdir(), 'antigravity-answer-')); + try { + const logs = join(root, 'brain', SESSION, '.system_generated', 'logs'); + mkdirSync(logs, { recursive: true }); + const entry = (fields: Record) => JSON.stringify({ source: 'MODEL', type: 'PLANNER_RESPONSE', + status: 'DONE', ...fields }); + const write = (...entries: string[]) => writeFileSync(join(logs, 'transcript.jsonl'), `${entries.join('\n')}\n`); + write(entry({ content: 'DEPLOY_COMPLETE soon', tool_calls: [{ name: 'run_command' }] })); + assert.equal(antigravityFinishedAnswer(root, 'DEPLOY_COMPLETE', 0), null, 'a response with a tool call is not final'); + write(entry({ tool_calls: [{ name: 'run_command' }] }), entry({ content: 'DEPLOY_COMPLETE' })); + assert.equal(antigravityFinishedAnswer(root, 'DEPLOY_COMPLETE', 0), 'DEPLOY_COMPLETE'); + assert.equal(antigravityFinishedAnswer(root, 'UPGRADE_COMPLETE', 0), null); + assert.equal(antigravityFinishedAnswer(root, 'DEPLOY_COMPLETE', Date.now() + 60_000), null, 'an earlier session is not this one'); + } finally { rmSync(root, { recursive: true, force: true }); } + const interrupted = { status: 'ERROR', error: 'interrupted', response: '', num_turns: 1, usage: usage(1000, 80, 40, 400) }; + // The stream stalls at the command that is still running. + const ended = `${stream(interrupted, ['tool'])}\n${JSON.stringify({ event: 'session_ended_after_answer', response: 'DEPLOY_COMPLETE' })}`; + const result = parseAntigravityResult(ended); + assert.equal(result.is_error, false); + assert.equal(result.result, 'DEPLOY_COMPLETE'); + assert.deepEqual(result.usage, { input_tokens: 1000, output_tokens: 80, cache_read_input_tokens: 400, + cache_creation_input_tokens: 0 }); + assert.equal(parseAntigravityResult(stream(interrupted, ['tool'])).is_error, true, 'an interruption alone is an error'); + // Seen live at level 3: stopped after UPGRADE_COMPLETE, the CLI repeated a 503 it had retried past. + const repeated = { ...interrupted, error: 'API error (attempt 1): Error 503, Status: UNAVAILABLE' }; + const afterRetry = parseAntigravityResult( + `${stream(repeated, ['tool'])}\n${JSON.stringify({ event: 'session_ended_after_answer', response: 'UPGRADE_COMPLETE' })}`); + assert.equal(afterRetry.is_error, false); + assert.equal(afterRetry.result, 'UPGRADE_COMPLETE'); +}); + +test('Antigravity errors, bad usage and missing results are provider errors', () => { + const failed = parseAntigravityResult(stream({ status: 'ERROR', error: 'quota exceeded', response: '', + num_turns: 0, usage: usage(0, 0, 0, 0) }, ['tool', 'error_message'])); + assert.equal(failed.is_error, true); + assert.match(String(failed.result), /quota exceeded/); + assert.equal(parseAntigravityResult(stream({ status: 'SUCCESS', response: 'x', num_turns: 1, + usage: { input_tokens: -1, output_tokens: 1, cache_read_tokens: 0 } })).is_error, true); + const missing = parseAntigravityResult(stream(null)); + assert.equal(missing.is_error, true); + assert.match(String(missing.result), /no result/); +}); diff --git a/tools/stack-bench/tests/artifacts.test.ts b/tools/stack-bench/tests/artifacts.test.ts new file mode 100644 index 00000000000..c3ae8e31a42 --- /dev/null +++ b/tools/stack-bench/tests/artifacts.test.ts @@ -0,0 +1,271 @@ +import assert from 'node:assert/strict'; +import { existsSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { ARTIFACT_SCHEMA_VERSION, createArtifact, readArtifact, readArtifactPayload, + readRunJson, writeArtifact, writeRunJson } from '../src/evidence/artifacts.js'; +import { createCheckEvidence } from '../src/evidence/check-evidence.js'; + +test('run artifacts are atomic and identify the producing run', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-artifact-')); + try { + const path = join(root, 'run.json'); + const mode = { id: 'sequential', version: '1.0.0' }; + const featureCatalog = { id: 'catalog', sha256: 'a'.repeat(64) }; + writeRunJson(path, { id: 'run-a', mode, featureCatalog, levels: [] }); + const artifact = readRunJson(path, 'run-a'); + assert.equal(artifact.id, 'run-a'); + assert.deepEqual(artifact.mode, mode); + assert.deepEqual(artifact.featureCatalog, featureCatalog); + assert.equal(artifact.artifactSchemaVersion, ARTIFACT_SCHEMA_VERSION); + assert.throws(() => readRunJson(path, 'run-b'), /belongs to run-a/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('an invalid replacement cannot overwrite an existing run artifact', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-artifact-')); + try { + const path = join(root, 'run.json'); + writeRunJson(path, { id: 'run-a', status: 'complete' }); + assert.throws(() => writeRunJson(path, { id: '', complete: false }), /non-empty id/); + assert.equal(readRunJson(path, 'run-a').status, 'complete'); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('a synced artifact replacement leaves only the committed file', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-artifact-')); + try { + const path = join(root, 'run.json'); + writeRunJson(path, { id: 'run-a', status: 'running' }); + writeRunJson(path, { id: 'run-a', status: 'complete' }); + assert.equal(readRunJson(path, 'run-a').status, 'complete'); + assert.deepEqual(readdirSync(root), ['run.json']); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('unknown artifact schemas fail rather than being guessed', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-artifact-')); + try { + const path = join(root, 'run.json'); + writeFileSync(path, JSON.stringify({ artifactSchemaVersion: 999, id: 'future-run' })); + assert.throws(() => readRunJson(path, 'future-run'), /unsupported schema 999/); + assert.throws(() => writeArtifact(path, { artifactSchemaVersion: 1, + kind: 'benchmark_run', id: 'old-envelope' }), /unsupported schema 1/); + assert.throws(() => writeRunJson(path, { artifactSchemaVersion: 999, id: 'future-run' }), + /schema 999 is not supported/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('v2 envelopes preserve exact identities, timestamps, and attempt ancestry', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-artifact-')); + try { + const path = join(root, 'grade.json'); + const digest = 'a'.repeat(64); + writeArtifact(path, { + kind: 'grade', id: 'grade-child', + attempt: { id: 'grade-child', parentId: 'bundle-parent' }, + timestamps: { startedAt: '2026-08-12T12:00:00.000Z', completedAt: '2026-08-12T12:00:01.000Z' }, + identities: { + recipe: { id: 'ecommerce-l1', sha256: digest }, + fixture: { id: 'standard', sha256: digest }, + calibration: { id: 'l1-standard', sha256: digest }, + packs: [{ id: 'auth', sha256: digest }], + stackAdapter: { id: 'postgres', sha256: digest }, + }, + payload: { total: 1, max: 1, features: [] }, + }); + const artifact = readArtifact(path, { expectedKind: 'grade', expectedId: 'grade-child' }); + assert.equal(artifact.attempt.parentId, 'bundle-parent'); + assert.equal(artifact.identities.recipe?.sha256, digest); + assert.equal(artifact.identities.packs[0]?.sha256, digest); + assert.equal(artifact.timestamps.completedAt, '2026-08-12T12:00:01.000Z'); + assert.deepEqual(readArtifactPayload(path).features, []); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('partial and invalid attempts are representable without invented scores', () => { + const artifact = createArtifact({ + kind: 'benchmark_run', id: 'partial-run', + attempt: { id: 'partial-run', parentId: null }, + timestamps: { startedAt: '2026-08-12T12:00:00.000Z', completedAt: null }, + payload: { status: 'invalid', outcome: { kind: 'harness_failure', reason: 'agent exited' } }, + }); + assert.equal(artifact.timestamps.completedAt, null); + assert.equal(artifact.payload.status, 'invalid'); + assert.equal('score' in artifact.payload, false); + assert.equal('totals' in artifact.payload, false); +}); + +test('campaign extension provenance is strict and records completed rechecks', () => { + const seed = { fromDepth: 2, sourceSha256: 'a'.repeat(64), sourceFiles: 4, + parent: { campaignId: 'parent', campaignSha256: 'b'.repeat(64), + attemptId: 'attempt-1', executionId: 'execution-1', runId: 'run-1', + runSha256: 'c'.repeat(64) }, validatedDepths: [1, 2] }; + assert.doesNotThrow(() => createArtifact({ kind: 'benchmark_run', id: 'extension', + payload: { progressionSeed: seed } })); + assert.throws(() => createArtifact({ kind: 'benchmark_run', id: 'bad-extension', + payload: { progressionSeed: { ...seed, validatedDepths: [2, 1] } } }), + /validatedDepths is invalid/); +}); + +test('unknown kinds, fields, malformed payloads, and backward timestamps fail closed', () => { + assert.throws(() => createArtifact({ kind: 'mystery', id: 'x' }), /unknown kind/); + assert.throws(() => createArtifact({ kind: 'benchmark_run', id: 'x', + payload: { outcome: { kind: 'typo' } } }), /outcome\.kind is invalid/); + assert.throws(() => createArtifact({ kind: 'grade', id: 'x', payload: { features: {} } }), + /payload\.features must be an array/); + assert.throws(() => createArtifact({ kind: 'grade', id: 'x', + timestamps: { startedAt: '2026-08-12T12:00:01.000Z', completedAt: '2026-08-12T12:00:00.000Z' } }), + /precedes/); + const valid = createArtifact({ kind: 'grade', id: 'x', + payload: { total: 0, max: 0, features: [] } }); + assert.throws(() => writeArtifact('unused.json', { ...valid, surprise: true }), /surprise is unknown/); + const missingIdentities: Partial = structuredClone(valid); + delete missingIdentities.identities; + assert.throws(() => writeArtifact('unused.json', missingIdentities), /identities is required/); + const incompleteTimestamps: Omit & { + timestamps: Partial; + } = structuredClone(valid); + delete incompleteTimestamps.timestamps.completedAt; + assert.throws(() => writeArtifact('unused.json', incompleteTimestamps), /timestamps is incomplete/); + const missingIdentitySlot: Omit & { + identities: Partial; + } = structuredClone(valid); + delete missingIdentitySlot.identities.experiment; + assert.throws(() => writeArtifact('unused.json', missingIdentitySlot), /identities\.experiment is required/); + const incompleteEngineIdentity = structuredClone(valid); + const engine = incompleteEngineIdentity.identities.engine; + assert(engine); + delete (engine as Partial).sha256; + assert.throws(() => writeArtifact('unused.json', incompleteEngineIdentity), + /identities\.engine\.sha256 is required/); + assert.throws(() => createArtifact({ kind: 'grade', id: 'x', identities: { engine: null } }), + /identities\.engine is required/); + const retiredIdentityFields = { id: 'old', sha256: 'a'.repeat(64), version: '1.0.0' }; + for (const identities of [ + { recipe: retiredIdentityFields }, + { fixture: retiredIdentityFields }, + { calibration: retiredIdentityFields }, + { packs: [retiredIdentityFields] }, + ]) { + assert.throws(() => createArtifact({ kind: 'grade', id: 'x', identities }), + /version is unknown/); + } + const observedPayload = { observation: 'observed', source: { sha256: 'a'.repeat(64) }, + suites: {}, totals: {}, selection: { observation: 'observed', scoredPoints: 0, + observedPoints: 1 } }; + assert.doesNotThrow(() => createArtifact({ kind: 'grade_bundle', id: 'observed', + payload: observedPayload })); + assert.throws(() => createArtifact({ kind: 'grade_bundle', id: 'unbound-observed', payload: { + ...observedPayload, source: undefined, + } }), /first-build SHA-256/); + assert.throws(() => createArtifact({ kind: 'grade_bundle', id: 'scored-observed', payload: { + ...observedPayload, selection: { ...observedPayload.selection, scoredPoints: 1 }, + } }), /contribute zero score/); + assert.doesNotThrow(() => createArtifact({ kind: 'grade_bundle', id: 'source-bound-scored', + payload: { observation: 'scored', source: { sha256: 'b'.repeat(64) }, suites: {}, totals: {} } })); + assert.throws(() => createArtifact({ kind: 'reference_qualification', id: 'incomplete-reference' }), + /payload\.fixture is required/); + const referencePayload = { fixture: 'standard', requiredRepetitions: 1, runs: [] }; + assert.throws(() => createArtifact({ kind: 'reference_qualification', id: 'bad-runner', payload: { + ...referencePayload, + runner: { schemaVersion: 1, mode: 'desktop', platform: 'win32', architecture: 'x64' }, + } }), /runner\.mode is invalid/); + assert.throws(() => createArtifact({ kind: 'reference_qualification', id: 'future-runner', payload: { + ...referencePayload, + runner: { schemaVersion: 2, mode: 'appliance', platform: 'linux', architecture: 'x64' }, + } }), /runner\.schemaVersion must be 1/); + assert.doesNotThrow(() => createArtifact({ kind: 'null_control', id: 'typed-runner', payload: { + runner: { schemaVersion: 1, mode: 'appliance', platform: 'linux', architecture: 'x64', + dockerEngineVersion: '29.1.2', dockerOs: 'linux', dockerArchitecture: 'x86_64', + kernelVersion: '6.8.0-test', cpuCount: 8, memoryBytes: 16_000_000_000 }, + } })); + assert.doesNotThrow(() => createArtifact({ kind: 'reference_qualification', id: 'base-runner', payload: { + ...referencePayload, + runner: { schemaVersion: 1, mode: 'appliance', platform: 'linux', architecture: 'x64' }, + } })); + assert.throws(() => createArtifact({ kind: 'reference_qualification', id: 'partial-runner', payload: { + ...referencePayload, + runner: { schemaVersion: 1, mode: 'appliance', platform: 'linux', architecture: 'x64', + dockerEngineVersion: '29.1.2' }, + } }), /runner\.dockerOs must be a non-empty string/); +}); + +test('mutation shard artifacts bind exact coordinates to the exact result set', () => { + const results = [{ id: 'broken-auth', status: 'CAUGHT' }, + { id: 'broken-owner', status: 'CAUGHT' }]; + assert.doesNotThrow(() => createArtifact({ kind: 'mutation_control', id: 'shard', payload: { + shard: { index: 1, count: 3, mutationIds: ['broken-owner', 'broken-auth'] }, results, + } })); + assert.throws(() => createArtifact({ kind: 'mutation_control', id: 'bad-index', payload: { + shard: { index: 3, count: 3, mutationIds: ['broken-auth'] }, results: [results[0]], + } }), /coordinates are invalid/); + assert.throws(() => createArtifact({ kind: 'mutation_control', id: 'wrong-result', payload: { + shard: { index: 1, count: 3, mutationIds: ['other'] }, results: [results[0]], + } }), /unique subset of the assigned mutation IDs/); + assert.doesNotThrow(() => createArtifact({ kind: 'mutation_control', id: 'partial-shard', payload: { + shard: { index: 1, count: 3, mutationIds: ['broken-auth', 'broken-owner'] }, + results: [results[0]], checkpoint: { status: 'incomplete' }, + } })); + assert.throws(() => createArtifact({ kind: 'mutation_control', id: 'partial-marked-complete', payload: { + shard: { index: 1, count: 3, mutationIds: ['broken-auth', 'broken-owner'] }, + results: [results[0]], checkpoint: { status: 'complete' }, + } }), /complete mutation_control.*match the exact result set/); + assert.throws(() => createArtifact({ kind: 'mutation_control', id: 'duplicate-partial-result', payload: { + shard: { index: 1, count: 3, mutationIds: ['broken-auth', 'broken-owner'] }, + results: [results[0], results[0]], checkpoint: { status: 'incomplete' }, + } }), /unique subset of the assigned mutation IDs/); + assert.throws(() => createArtifact({ kind: 'mutation_control', id: 'unknown-shard-field', payload: { + shard: { index: 1, count: 3, mutationIds: ['broken-auth'], worker: true }, results: [results[0]], + } }), /worker is unknown/); +}); + +test('public artifacts reject secret-bearing fields before touching disk', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-artifact-')); + try { + const path = join(root, 'run.json'); + assert.throws(() => writeArtifact(path, { kind: 'benchmark_run', id: 'secret-run', + payload: { backendLease: { leaseToken: 'do-not-persist' } } }), /secret-bearing/); + assert.throws(() => writeArtifact(path, { kind: 'benchmark_run', id: 'secret-run', + payload: { backendLease: { api_key: 'do-not-persist' } } }), /secret-bearing/); + assert.equal(existsSync(path), false); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('typed grade evidence is required and obsolete projection fields are rejected', () => { + const evidence = createCheckEvidence({ status: 'failed', code: 'application_failure', phase: 'assertion', + summary: 'not observed', startedAtMs: 1, completedAtMs: 2 }); + const setupEvidence = createCheckEvidence({ status: 'passed', code: 'completed', phase: 'setup', + startedAtMs: 1, completedAtMs: 2 }); + const criterion = { id: 'works', desc: 'works', points: 1, evidence }; + assert.doesNotThrow(() => createArtifact({ kind: 'grade', id: 'typed-grade', + payload: { total: 0, max: 1, + features: [{ id: 1, name: 'feature', score: 0, max: 1, + setupEvidence, criteria: [criterion] }] } })); + assert.throws(() => createArtifact({ kind: 'grade', id: 'obsolete-grade', + payload: { total: 0, max: 1, features: [{ id: 1, name: 'feature', score: 0, + max: 1, setupEvidence, + criteria: [{ ...criterion, passed: false }] }] } }), /passed is obsolete/); + assert.throws(() => createArtifact({ kind: 'grade', id: 'missing-evidence', + payload: { total: 0, max: 1, + features: [{ id: 1, name: 'feature', score: 0, max: 1, + setupEvidence, criteria: [{ id: 'works' }] }] } }), + /evidence is required/); +}); + +test('active readers reject unversioned files without inferring their kind', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-artifact-')); + try { + const bundle = join(root, 'bundle.json'); + writeFileSync(bundle, JSON.stringify({ suites: {}, totals: {} })); + assert.throws(() => readArtifact(bundle, { expectedKind: 'grade_bundle' }), + /unsupported schema missing/); + const unknown = join(root, 'unknown.json'); + writeFileSync(unknown, JSON.stringify({ kind: 'future_thing', id: 'x' })); + assert.throws(() => readArtifact(unknown), /unsupported schema missing/); + const staleRun = join(root, 'run.json'); + writeFileSync(staleRun, JSON.stringify({ id: 'old-run' })); + assert.throws(() => readRunJson(staleRun, 'current-run'), /unsupported schema missing/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/tools/stack-bench/tests/auth-interception.integration.ts b/tools/stack-bench/tests/auth-interception.integration.ts new file mode 100644 index 00000000000..63eb65834ce --- /dev/null +++ b/tools/stack-bench/tests/auth-interception.integration.ts @@ -0,0 +1,79 @@ +import assert from 'node:assert/strict'; +import { createServer } from 'node:http'; +import { once } from 'node:events'; +import { createRequire } from 'node:module'; +import { writeFileSync } from 'node:fs'; +import test from 'node:test'; +import { chromium } from 'playwright'; +import { gradeFeature } from '../grader/grade.js'; +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; + +// Integration gap, specified before scoping interception: credential patches must +// reach the socket in setup, ordinary criteria, replacement pages and fresh actors. +// A server that trusts the supplied role must fail; refusing elevation must pass. +test('selected authentication probes survive browser lifecycle and expose privilege escalation', async () => { + const { wsServer } = createRequire(import.meta.url)('playwright-core/lib/utilsBundle'); + const server = createServer((_request, response) => { + response.setHeader('Content-Type', 'text/html'); + response.end(` + `); + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const sockets = new wsServer({ server }); + let defective = false; + const received: { role?: string }[] = []; + sockets.on('connection', (socket: { on(event: string, fn: (data: Buffer) => void): void; send(data: string): void }) => { + socket.on('message', data => { + const message = JSON.parse(String(data)), account = message.args[0]; + received.push(account); + socket.send(JSON.stringify({ type: 'ActionResponse', requestId: message.requestId, success: true, + result: { username: account.username, role: defective ? account.role ?? 'user' : 'user' } })); + }); + }); + const browser = await chromium.launch({ headless: true }); + const evidence: unknown[] = []; + try { + for (const phase of ['setup', 'criterion', 'replacement', 'fresh', 'race'] as const) for (defective of [false, true]) { + received.length = 0; + const actor = phase === 'fresh' ? 'claimant-fresh' : 'claimant'; + const signup = { do: 'signUp', actor, name: 'claimant', password: 'fixture-password', + requestPatch: { fields: { role: 'admin' } } }; + const prepare = phase === 'replacement' + ? [{ do: 'closeClient', actor }, { do: 'openClient', actor, settleMs: 0 }] + : phase === 'fresh' ? [{ do: 'freshClient', actor: 'claimant' }] : []; + const submit = phase === 'race' ? [{ do: 'race', settleMs: 0, + branches: [[signup], [{ do: 'expect', actor, testid: 'signup-username' }]] }] : [signup]; + const feature = compileScenarioDefinition({ schemaVersion: 1, track: 'ecommerce', level: 1, features: [{ + id: 1, name: 'authentication boundary', actors: ['claimant'], setup: phase === 'setup' ? [signup] : [], + criteria: [{ id: 'authority', desc: 'signup cannot choose administrator authority', points: 1, + steps: [...prepare, ...(phase === 'setup' ? [] : submit), + { do: 'expect', actor, testid: 'authority', contains: 'user', within: 300 }] }], + }] }).features[0]!; + const result = await gradeFeature(browser, feature, + { url, level: 1, headed: false, selectedCheckKeys: [], nullControl: false }, + { runId: 'auth-interception', roomName: name => name, url, actions: [], spacetime: null, + backend: 'postgres', nullControl: false, defaultWithin: 1000 }); + evidence.push({ phase, defective, result }); + assert.equal(result.setupEvidence.status, 'passed'); + assert.deepEqual(received.map(account => account.role), ['admin'], 'the native server must receive the patch once'); + assert.equal(result.criteria[0]!.evidence.status, defective ? 'failed' : 'passed', JSON.stringify(result)); + } + } finally { + await browser.close(); sockets.close(); + await new Promise(resolve => server.close(() => resolve())); + if (process.env.STACK_BENCH_AUTH_EVIDENCE) writeFileSync(process.env.STACK_BENCH_AUTH_EVIDENCE, + JSON.stringify({ rerun: 'STACK_BENCH_AUTH_EVIDENCE= node --test dist/tests/auth-interception.integration.js', evidence }, null, 2)); + } +}); diff --git a/tools/stack-bench/tests/auth-request-patch.test.ts b/tools/stack-bench/tests/auth-request-patch.test.ts new file mode 100644 index 00000000000..6c9292f3999 --- /dev/null +++ b/tools/stack-bench/tests/auth-request-patch.test.ts @@ -0,0 +1,1112 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { createServer } from 'node:http'; +import { once } from 'node:events'; +import { createRequire } from 'node:module'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { chromium } from 'playwright'; +import { installAuthWebSocketCapture, patchAuthRequest, withAuthRequestPatch, withAuthWriteInventory, withWriteCompletion, + withAuthWriteTarget, stopAuthWriteInventory, type AuthWrite } from '../src/actions/auth-request-patch.js'; +import { createBackendLease, writeBackendLease } from '../src/runtime/backend-lease.js'; +import { installSpacetimeWriteCapture } from '../src/stacks/backends/spacetime-browser-session.js'; +import { supabaseAuthRequestPatch } from '../src/stacks/backends/supabase-operations.js'; +import { convexAuthReadEndpoints } from '../src/stacks/backends/convex-operations.js'; +import { ActionApplicationFailure, ActionHarnessFailure, ActionInconclusive } from '../src/actions/action-contract.js'; +import { installResponseLoss } from '../grader/response-loss.js'; + +test('UI write completion drains delayed HTTP effects, accepts no write, and rejects lost responses', async () => { + let release: (() => void) | undefined, received!: () => void, writes = 0; + const server = createServer(async (req, res) => { + if (req.method !== 'POST') { res.end(''); return; } + for await (const _chunk of req) { /* Consume either form or JSON input. */ } + writes++; + if (req.url === '/lost') { res.writeHead(200, { 'content-type': 'application/json' }); res.write('{'); } + release = () => { + release = undefined; + if (req.url === '/lost') res.destroy(); + else res.writeHead(req.url === '/invalid-json' ? 400 : req.url === '/accepted-pending' ? 202 : 200).end('{}'); + }; + received(); + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage(); + await installAuthWebSocketCapture(page); + await page.goto(`http://127.0.0.1:${(server.address() as { port: number }).port}`); + assert.deepEqual((await withWriteCompletion(page, async () => {})).writes, []); + for (const mode of ['json', 'form', 'invalid-json', 'accepted-pending', 'lost']) { + const request = new Promise(resolve => { received = resolve; }); + await page.evaluate(mode => { + document.querySelector('#buy')!.onclick = () => { + void fetch(['lost', 'invalid-json', 'accepted-pending'].includes(mode) ? '/' + mode : '/buy', { method: 'POST', + headers: { 'content-type': mode === 'form' ? 'application/x-www-form-urlencoded' : 'application/json' }, + body: mode === 'form' ? 'item=keyboard' : mode === 'invalid-json' ? '{' : '{"item":"keyboard"}' }).catch(() => {}); + }; + }, mode); + const completion = withWriteCompletion(page, () => page.locator('#buy').click()); + await request; + assert.equal(await Promise.race([completion.then(() => 'done', () => 'failed'), + new Promise(resolve => setTimeout(() => resolve('pending'), 100))]), 'pending'); + release!(); + if (['lost', 'accepted-pending'].includes(mode)) await assert.rejects(completion, ActionInconclusive); + else assert.equal((await completion).writes.length, 1); + } + // A callback dispatched after the capture returns is outside its boundary. + // This does not establish the absence of every possible future timer. + await page.evaluate(() => { document.querySelector('#buy')!.onclick = () => { + Object.assign(window, { laterWrite: () => fetch('/buy', { method: 'POST' }).catch(() => {}) }); + }; }); + assert.equal((await withWriteCompletion(page, () => page.locator('#buy').click())).writes.length, 0); + const request = new Promise(resolve => { received = resolve; }); + const later = page.evaluate(() => (window as unknown as { laterWrite(): Promise }).laterWrite()); + await request; release!(); await later; + assert.equal(writes, 6); + } finally { + release?.(); await browser.close(); server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + } +}); + +// Signup reconnects can carry authority in Socket.IO CONNECT data. Heartbeats +// are transport traffic; an application event without an acknowledgement is not. +test('signup reconnect probes include Socket.IO CONNECT claims and exclude heartbeat traffic', async t => { + const { wsServer } = createRequire(import.meta.url)('playwright-core/lib/utilsBundle'); + const accounts = new Map(); + const polls = new Map(); + const connects = new Map(); + let defective = false, stock = 0; + const connect = (user: string, frame: string) => { + const claims = frame.length > 2 ? JSON.parse(frame.slice(2)) : {}; + if (accounts.has(user) && defective && claims.role === 'admin') accounts.set(user, 'admin'); + return `40${JSON.stringify({ sid: `namespace-${user}` })}`; + }; + const server = createServer(async (req, res) => { + const user = /user=([^;]+)/.exec(req.headers.cookie ?? '')?.[1] ?? ''; + const address = new URL(req.url!, 'http://fixture.test'); + if (address.searchParams.get('transport') === 'polling') { + const sid = address.searchParams.get('sid'); + res.setHeader('content-type', 'text/plain'); + if (!sid) { + const id = `poll-${user}`; polls.set(id, []); + res.end(`0${JSON.stringify({ sid: id, upgrades: ['websocket'], pingInterval: 25000, pingTimeout: 20000, maxPayload: 1000000 })}`); return; + } + if (req.method === 'POST') { + let body = ''; for await (const chunk of req) body += chunk; + for (const frame of body.split('\x1e')) if (frame.startsWith('40')) polls.get(sid)!.push(connect(user, frame), '2'); + res.end('ok'); return; + } + res.end(polls.get(sid)!.splice(0).join('\x1e') || '6'); return; + } + if (req.url === '/signup') { + for await (const _chunk of req) { /* Consume the ordinary registration. */ } + accounts.set(user, 'customer'); res.end('ok'); return; + } + if (req.url === '/restock') { + if (accounts.get(user) !== 'admin') { res.writeHead(403).end(); return; } + stock++; res.end('ok'); return; + } + res.end('signup'); + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const sockets = new wsServer({ server }); + sockets.on('connection', (socket: { on(event: string, fn: (raw: Buffer) => void): void; send(data: string): void }, + request: { url: string; headers: { cookie?: string } }) => { + const user = /user=([^;]+)/.exec(request.headers.cookie ?? '')?.[1] ?? ''; + if (!new URL(request.url, url).searchParams.has('sid')) { + socket.send(`0${JSON.stringify({ sid: user, upgrades: [], pingInterval: 25000, pingTimeout: 20000, maxPayload: 1000000 })}`); + } + socket.on('message', raw => { + const frame = String(raw); + if (frame.startsWith('40')) { + connects.set(user, (connects.get(user) ?? 0) + 1); + if (user === 'overlap' && connects.get(user) === 1) { socket.send('6'); return; } + if (user === 'missing-ack' && connects.get(user)! > 1) { socket.send('6'); return; } + socket.send(connect(user, frame)); + socket.send('2'); + } else if (frame === '2probe') socket.send('3probe'); + else if (frame === '5') socket.send('2'); + else if (frame === '3' || frame.startsWith('42')) socket.send('6'); + }); + }); + const browser = await chromium.launch({ headless: true }); + try { + const fresh = async (user: string, query = '') => { + const context = await browser.newContext(), page = await context.newPage(); + await installAuthWebSocketCapture(page); await page.goto(url); + await page.evaluate(async ({ url, user, query }) => { + document.cookie = `user=${user}`; + const socket = new WebSocket(`${url.replace('http:', 'ws:')}/socket.io/?EIO=4&transport=websocket${query}`); + Object.assign(window, { fixtureSocket: socket }); + await new Promise(resolve => socket.addEventListener('message', event => { + if (String(event.data).startsWith('0')) socket.send('40'); + if (event.data === '2') socket.send('3'); + if (event.data === '6') resolve(); + })); + }, { url, user, query }); + return page; + }; + const submit = (page: Awaited>, auth: boolean, unknownEvent = false, polling = false) => page.evaluate(async ({ auth, unknownEvent, polling }) => { + const socket = (window as unknown as { fixtureSocket: WebSocket }).fixtureSocket; + await fetch('/signup', { method: 'POST', headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ username: 'customer', password: 'fixture-password' }) }); + if (polling) { + socket.send('41'); socket.close(); + const base = '/socket.io/?EIO=4&transport=polling'; + const open = JSON.parse((await (await fetch(base)).text()).slice(1)); + const path = `${base}&sid=${open.sid}`; + await fetch(path, { method: 'POST', body: auth ? '40{"token":"fixture-session"}' : '40' }); + const reply = await (await fetch(path)).text(); + if (!reply.startsWith('40')) throw new Error('Missing CONNECT receipt'); + await fetch(path, { method: 'POST', body: '3' }); + const upgrade = new WebSocket(`${location.origin.replace('http:', 'ws:')}/socket.io/?EIO=4&transport=websocket&sid=${open.sid}`); + await new Promise(resolve => { + upgrade.addEventListener('open', () => upgrade.send('2probe')); + upgrade.addEventListener('message', event => { + if (event.data === '3probe') upgrade.send('5'); + if (event.data === '2') upgrade.send('3'); + if (event.data === '6') resolve(); + }); + }); + return; + } + const reply = new Promise(resolve => { + const listener = (event: MessageEvent) => { if (event.data === '6') { socket.removeEventListener('message', listener); resolve(); } }; + socket.addEventListener('message', listener); + }); + if (unknownEvent) socket.send('42["finish",{"role":"customer"}]'); + else { socket.send('41'); socket.send(auth ? '40{"token":"fixture-session"}' : '40'); } + await reply; + }, { auth, unknownEvent, polling }); + let lastWrites: readonly AuthWrite[] = []; + for (const polling of [false, true]) for (const auth of [false, true]) await t.test(`${polling ? 'polling upgrade' : 'websocket'} ${auth ? 'token' : 'empty'} CONNECT`, async () => { + const ordinary = await fresh(`ordinary-${polling}-${auth}`); + const baseline = await withAuthWriteInventory(ordinary, () => submit(ordinary, auth, false, polling)); + lastWrites = baseline.writes; + assert.equal(baseline.writes.length, 2, 'the HTTP write and CONNECT are targets; disconnect and heartbeat are not'); + assert(!JSON.stringify(baseline).includes('fixture-session')); + for (defective of [false, true]) for (const index of [0, 1]) { + const page = await fresh(`probe-${polling}-${auth}-${defective}-${index}`), before = stock; + const result = await withAuthRequestPatch(page, 'customer', 'fixture-password', { fields: { role: 'admin' } }, + () => submit(page, auth, false, polling), undefined, 'signup', { writes: baseline.writes, index }); + assert(result.requestPatch.status === 200 || result.requestPatch.success === true); + const status = await page.evaluate(async () => (await fetch('/restock', { method: 'POST' })).status); + assert.equal(status, defective && index === 1 ? 200 : 403); + assert.equal(stock - before, defective && index === 1 ? 1 : 0); + await page.context().close(); + } + await ordinary.context().close(); + }); + const crossTransport = await fresh('cross-transport'); + await withAuthRequestPatch(crossTransport, 'customer', 'fixture-password', { fields: { role: 'admin' } }, + () => submit(crossTransport, true), undefined, 'signup', { writes: lastWrites, index: 1 }); + const overlap = await fresh('overlap'); + await assert.rejects(withAuthWriteInventory(overlap, () => overlap.evaluate(async () => { + await fetch('/signup', { method: 'POST', headers: { 'content-type': 'application/json' }, body: '{}' }); + const socket = (window as unknown as { fixtureSocket: WebSocket }).fixtureSocket; + await new Promise(resolve => { + socket.addEventListener('message', event => { if (event.data === '6') resolve(); }); + socket.send('40'); + }); + })), ActionInconclusive, 'a delayed pre-capture receipt must not prove a later overlapping CONNECT'); + const changedRoute = await fresh('changed-route', '&mode=promote'); + await assert.rejects(withAuthRequestPatch(changedRoute, 'customer', 'fixture-password', { fields: { role: 'admin' } }, + () => submit(changedRoute, true), undefined, 'signup', { writes: lastWrites, index: 1 }), ActionInconclusive); + const unacknowledged = await fresh('unacknowledged'); + await assert.rejects(withAuthWriteInventory(unacknowledged, () => submit(unacknowledged, false, true)), ActionInconclusive); + const missingAck = await fresh('missing-ack'); + await assert.rejects(withAuthWriteInventory(missingAck, () => submit(missingAck, false)), ActionInconclusive); + const duplicate = await fresh('duplicate'); + await assert.rejects(withAuthWriteInventory(duplicate, () => duplicate.evaluate(async () => { + const socket = (window as unknown as { fixtureSocket: WebSocket }).fixtureSocket; + await new Promise(resolve => { + socket.addEventListener('message', event => { if (event.data === '6') resolve(); }); + socket.send('41'); socket.send('40'); socket.send('40'); + }); + })), ActionInconclusive); + const unknownSid = await fresh('unknown-sid'); + await assert.rejects(withAuthWriteInventory(unknownSid, () => unknownSid.evaluate(async () => { + const socket = new WebSocket(`${location.origin.replace('http:', 'ws:')}/socket.io/?EIO=4&transport=websocket&sid=unknown`); + await new Promise(resolve => { + socket.addEventListener('open', () => socket.send('2probe')); + socket.addEventListener('message', () => resolve()); + }); + })), ActionInconclusive); + } finally { + await browser.close(); sockets.close(); await new Promise(resolve => server.close(() => resolve())); + } +}); + +// A later write can begin while an earlier response body is still draining. +// Reaching the end of the earlier body cannot prove the later write's receipt. +test('signup inventory drains CONNECT receipts added while an HTTP body is pending', async () => { + const { wsServer } = createRequire(import.meta.url)('playwright-core/lib/utilsBundle'); + let finishBody: (() => void) | undefined, acknowledge: (() => void) | undefined; + let receivedConnect!: () => void; + const connected = new Promise(resolve => { receivedConnect = resolve; }); + const server = createServer(async (req, res) => { + if (req.url !== '/signup') { res.end('signup'); return; } + for await (const _chunk of req) { /* Consume the write before sending headers. */ } + res.writeHead(200, { 'content-type': 'application/json' }); + res.write('{"ok":'); + finishBody = () => { finishBody = undefined; res.end('true}'); }; + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const sockets = new wsServer({ server }); + sockets.on('connection', (socket: { on(event: string, fn: (raw: Buffer) => void): void; send(data: string): void }) => { + socket.send(`0${JSON.stringify({ sid: 'drain', upgrades: [], pingInterval: 25000, pingTimeout: 20000 })}`); + socket.on('message', raw => { + if (String(raw) === '40') { + acknowledge = () => { acknowledge = undefined; socket.send('40{"sid":"namespace-drain"}'); }; + receivedConnect(); + } + }); + }); + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage(); + await installAuthWebSocketCapture(page); await page.goto(url); + await page.evaluate(async url => { + const socket = new WebSocket(`${url.replace('http:', 'ws:')}/socket.io/?EIO=4&transport=websocket`); + Object.assign(window, { fixtureSocket: socket }); + await new Promise(resolve => socket.addEventListener('message', () => resolve(), { once: true })); + }, url); + let submitted!: () => void; + const headersReceived = new Promise(resolve => { submitted = resolve; }); + const inventory = withAuthWriteInventory(page, async () => { + await page.evaluate(async () => { + // fetch resolves at headers; the evaluator still waits for the body. + await fetch('/signup', { method: 'POST', headers: { 'content-type': 'application/json' }, body: '{}' }); + }); + submitted(); + }); + await headersReceived; + await page.evaluate(() => (window as unknown as { fixtureSocket: WebSocket }).fixtureSocket.send('40')); + await connected; + finishBody!(); + const outcome = await Promise.race([ + inventory.then(() => 'measured', () => 'rejected'), + new Promise(resolve => setTimeout(() => resolve('waiting'), 150)), + ]); + assert.equal(outcome, 'waiting', 'inventory must wait for the CONNECT receipt added during the HTTP drain'); + acknowledge!(); + assert.equal((await inventory).writes.length, 2); + } finally { + finishBody?.(); acknowledge?.(); + await browser.close(); sockets.close(); await new Promise(resolve => server.close(() => resolve())); + } +}); + +// Only the exact authenticated backend endpoint has Convex's read semantics. +// A same-named app route, another origin/port, query suffix, or method must remain +// in the inventory and receive its own probe instead of inheriting an exemption. +test('signup inventory excludes only the exact leased Convex query POST', async () => { + const received: { url: string; method: string; role: unknown }[] = []; + const serve = () => createServer(async (req, res) => { + res.setHeader('Access-Control-Allow-Origin', '*'); + res.setHeader('Access-Control-Allow-Headers', 'content-type'); + res.setHeader('Access-Control-Allow-Methods', 'POST, PUT'); + if (req.method === 'OPTIONS') { res.writeHead(204).end(); return; } + if (req.method === 'GET') { res.end('signup'); return; } + let raw = ''; for await (const chunk of req) raw += chunk; + received.push({ url: req.url!, method: req.method!, role: raw ? JSON.parse(raw).role : null }); + res.end('ok'); + }).listen(0, '127.0.0.1'); + const backend = serve(), app = serve(); + await Promise.all([once(backend, 'listening'), once(app, 'listening')]); + const origin = (server: typeof app) => `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const lease = createBackendLease({ runId: 'convex-query-capture', backend: 'convex', track: 'ecommerce', + runIndex: 0, serverUri: origin(backend) }); + lease.state = 'active'; + const endpoints = convexAuthReadEndpoints(lease); + assert.deepEqual(endpoints, [`${origin(backend)}/api/query`]); + assert.throws(() => convexAuthReadEndpoints({ ...lease, state: 'released' }), /lease/i); + assert.throws(() => convexAuthReadEndpoints({ ...lease, backend: 'postgres' }), /lease/i); + const browser = await chromium.launch({ headless: true }); + try { + for (const [url, method, exempt] of [ + [`${origin(backend)}/api/query`, 'POST', true], + [`${origin(app)}/api/query`, 'POST', false], + [`${origin(backend).replace('127.0.0.1', 'localhost')}/api/query`, 'POST', false], + [`${origin(backend)}/api/query?write=true`, 'POST', false], + [`${origin(backend)}/nested/api/query`, 'POST', false], + [`${origin(backend)}/api/query`, 'PUT', false], + ] as const) { + const context = await browser.newContext(), page = await context.newPage(); await page.goto(origin(app)); + const submit = () => page.evaluate(async ({ url, method }) => { + await fetch('/signup', { method: 'POST', headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ username: 'ordinary', password: 'fixture-password' }) }); + await fetch(url, { method, headers: { 'content-type': 'application/json' }, body: '{}' }); + }, { url, method }); + const baseline: { writes: readonly AuthWrite[] } = await withAuthWriteInventory(page, submit, endpoints); + assert.equal(baseline.writes.length, exempt ? 1 : 2, `${method} ${url}`); + received.length = 0; + const index = exempt ? 0 : 1; + await withAuthRequestPatch(page, 'ordinary', 'fixture-password', { fields: { role: 'admin' } }, submit, + undefined, 'signup', { writes: baseline.writes, index, readEndpoints: endpoints }); + assert.deepEqual(received.map(request => request.role), exempt ? ['admin', undefined] : [undefined, 'admin']); + await context.close(); + } + } finally { + await browser.close(); + await Promise.all([backend, app].map(server => new Promise(resolve => server.close(() => resolve())))); + } +}); + +// Failure cases fixed before implementation: an unrelated profile write must not +// hide a bodyless account finalizer, and a missing/changed target is unmeasured. +test('each signup write is patched once, including a bodyless account finalizer', async () => { + const accounts = new Map(), calls: { path: string; role?: string }[] = []; + let defective = false, refuseClaims = false, stock = 10; + const server = createServer(async (req, res) => { + if (req.method !== 'POST') { res.end('accounts'); return; } + let raw = ''; for await (const chunk of req) raw += chunk; + const body = raw ? JSON.parse(raw) : {}; + const user = /user=([^;]+)/.exec(req.headers.cookie ?? '')?.[1] ?? ''; + calls.push({ path: req.url!, ...(body.role ? { role: body.role } : {}) }); + if (refuseClaims && req.url === '/profile' && body.role === 'admin') { res.writeHead(400).end(); return; } + if (req.url === '/nested/stdb/v1/identity/websocket-token') accounts.set(user, defective ? body.role ?? 'customer' : 'customer'); + if (req.url === '/restock') { + if (accounts.get(user) !== 'admin') { res.writeHead(403).end(); return; } + stock++; + } + res.end('ok'); + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const browser = await chromium.launch({ headless: true }); + try { + const fresh = async (user: string) => { + const context = await browser.newContext(), page = await context.newPage(); + await page.goto(url); await page.evaluate(user => { document.cookie = `user=${user}`; }, user); + return page; + }; + const submit = (page: Awaited>, mode = 'normal') => page.evaluate(async mode => { + const profile = () => fetch('/profile', { method: 'POST', headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ opaquePassword: 'fixture-derived-secret', role: 'customer' }) }); + if (!(await profile()).ok && mode === 'retry') await profile(); + if (mode === 'missing') return; + if (mode === 'extra') await fetch('/profile', { method: 'POST', headers: { 'content-type': 'application/json' }, body: '{}' }); + await fetch(mode === 'changed' ? '/other-finalizer' : '/nested/stdb/v1/identity/websocket-token', { method: 'POST' }); + if (mode === 'duplicate') await fetch('/nested/stdb/v1/identity/websocket-token', { method: 'POST' }); + if (mode === 'after') await fetch('/claimed-follow-up', { method: 'POST', headers: { 'content-type': 'application/json' }, body: '{}' }); + }, mode); + const baselinePage = await fresh('ordinary'); + const baseline = await withAuthWriteInventory(baselinePage, () => submit(baselinePage)); + assert.equal(baseline.writes.length, 2); + assert(!JSON.stringify(baseline).includes('fixture-derived-secret')); + const restock = (page: typeof baselinePage) => page.evaluate(async () => (await fetch('/restock', { method: 'POST' })).status); + assert.equal(await restock(baselinePage), 403); + accounts.set('admin', 'admin'); assert.equal(await restock(await fresh('admin')), 200); + for (defective of [false, true]) for (const index of [0, 1]) { + const page = await fresh(`probe-${defective}-${index}`), before = stock; + calls.length = 0; + const result = await withAuthRequestPatch(page, 'unused', 'unused', { fields: { role: 'admin' } }, + () => submit(page), undefined, 'signup', { writes: baseline.writes, index }); + assert.equal(result.requestPatch.status, 200); + assert.deepEqual(calls.map(call => call.role), index === 0 ? ['admin', undefined] : ['customer', 'admin']); + assert.equal(await restock(page), defective && index === 1 ? 200 : 403); + assert.equal(stock - before, defective && index === 1 ? 1 : 0); + await page.context().close(); + } + for (const mode of ['missing', 'changed', 'extra', 'duplicate']) { + const page = await fresh(mode); + // A sequence the fresh client cannot repeat keeps both inventories and may run once more. + await assert.rejects(withAuthRequestPatch(page, 'unused', 'unused', { fields: { role: 'admin' } }, + () => submit(page, mode), undefined, 'signup', { writes: baseline.writes, index: 1 }), (error: ActionInconclusive) => { + const observation = error.details.observation as { baseline: unknown; target: number; writes: unknown[] }; + assert.equal(error.details.retryable, true, mode); + assert.deepEqual([observation.baseline, observation.target], [baseline.writes, 1]); + assert(observation.writes.length > 0); + return true; + }); + await page.context().close(); + } + // The claim may change what the app does after the patched target, so a later + // different write is recorded, not refused. A repeat of the target (here the + // second profile write) would be unpatched and could undo the claim. + for (const [mode, index] of [['changed', 0], ['after', 0], ['after', 1]] as const) { + const page = await fresh(`${mode}-after-target-${index}`); + calls.length = 0; + const result = await withAuthRequestPatch(page, 'unused', 'unused', { fields: { role: 'admin' } }, + () => submit(page, mode), undefined, 'signup', { writes: baseline.writes, index }); + assert.equal(result.requestPatch.status, 200, `${mode} ${index}`); + assert.deepEqual(calls.map(call => call.role), index === 0 + ? ['admin', undefined, ...(mode === 'after' ? [undefined] : [])] : ['customer', 'admin', undefined]); + assert.equal(calls.at(-1)!.path, mode === 'after' ? '/claimed-follow-up' : '/other-finalizer'); + await page.context().close(); + } + const repeated = await fresh('repeated-target'); + await assert.rejects(withAuthRequestPatch(repeated, 'unused', 'unused', { fields: { role: 'admin' } }, + () => submit(repeated, 'extra'), undefined, 'signup', { writes: baseline.writes, index: 0 }), ActionInconclusive); + await repeated.context().close(); + // A refused claim stored nothing, so the app may send the same write again. + refuseClaims = true; calls.length = 0; + const retried = await fresh('retried-refusal'); + const refused = await withAuthRequestPatch(retried, 'unused', 'unused', { fields: { role: 'admin' } }, + () => submit(retried, 'retry'), undefined, 'signup', { writes: baseline.writes, index: 0 }); + assert.equal(refused.requestPatch.status, 400); + assert.deepEqual(calls.map(call => call.role), ['admin', 'customer', undefined]); + await retried.context().close(); + refuseClaims = false; + const completing = await fresh('completing'); + await withAuthWriteTarget(completing, { writes: baseline.writes, index: 0 }, () => + withAuthRequestPatch(completing, 'unused', 'unused', { fields: { role: 'admin' } }, + () => submit(completing, 'missing'), undefined, 'signup', undefined, async () => { + await completing.evaluate(() => fetch('/nested/stdb/v1/identity/websocket-token', { method: 'POST' }).then(() => {})); + stopAuthWriteInventory(completing); + await completing.evaluate(() => fetch('/separate-signin', { method: 'POST' }).then(() => {})); + })); + await completing.context().close(); + const delayed = await fresh('delayed'); + const delayedResult = await withAuthRequestPatch(delayed, 'unused', 'unused', { fields: { role: 'admin' } }, async () => { + await submit(delayed, 'missing'); + await delayed.evaluate(() => { setTimeout(() => { void fetch('/nested/stdb/v1/identity/websocket-token', { method: 'POST' }); }, 30); }); + }, undefined, 'signup', { writes: baseline.writes, index: 1 }); + assert.equal(delayedResult.requestPatch.status, 200, 'capture must stay active for the selected later write'); + await delayed.context().close(); + } finally { await browser.close(); await new Promise(resolve => server.close(() => resolve())); } +}); + +test('mixed native and HTTP signup probes expose either source of authority', async t => { + const codec = await import(new URL('../src/stacks/spacetime-wire-codec.js', import.meta.url).href); + const { wsServer } = createRequire(import.meta.url)('playwright-core/lib/utilsBundle'); + const encode = (type: { serialize(writer: unknown, value: unknown): void }, value: unknown) => { + const writer = new codec.BinaryWriter(128); type.serialize(writer, value); return Buffer.from(writer.getBuffer()); + }; + const readString = codec.AlgebraicType.makeDeserializer({ tag: 'String' }); + const writeString = codec.AlgebraicType.makeSerializer({ tag: 'String' }); + const accounts = new Map(), profiles = new Map(); + let mode = 'correct', stock = 10; + const heldReceipts = new Map void>(); + let nativeReceived: ((id: number) => void) | undefined; + const calls: { transport: string; role: string }[] = []; + const server = createServer(async (req, res) => { + if (req.url?.startsWith('/v1/database/auth/schema')) { + res.setHeader('content-type', 'application/json'); + res.end(JSON.stringify({ reducers: [{ name: 'profile_save', params: { elements: + ['name', 'digest', 'role'].map(name => ({ name: { some: name }, algebraic_type: { String: [] } })) } }, + { name: 'tick', params: { elements: [] } }] })); return; + } + if (req.method !== 'POST') { res.end('accounts'); return; } + let raw = ''; for await (const chunk of req) raw += chunk; + const user = /user=([^;]+)/.exec(req.headers.cookie ?? '')?.[1] ?? ''; + if (req.url === '/restock') { + if (accounts.get(user) !== 'admin') { res.writeHead(403).end(); return; } + stock++; res.end(); return; + } + const role = raw ? JSON.parse(raw).role : 'customer'; + calls.push({ transport: 'http', role }); + accounts.set(user, mode === 'http-defect' ? role : mode === 'native-defect' ? profiles.get(user)! : 'customer'); + res.end(); + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const sockets = new wsServer({ server }); + sockets.on('connection', (socket: { on(event: string, fn: (raw: Buffer) => void): void; send(data: Buffer): void; close(): void }) => { + const send = (value: unknown) => socket.send(Buffer.concat([Buffer.from([0]), encode(codec.ServerMessage, value)])); + send({ tag: 'InitialConnection', value: { identity: { __identity__: 1n }, connectionId: { __connection_id__: 1n }, token: 'fixture-token' } }); + socket.on('message', raw => { + const message = codec.ClientMessage.deserialize(new codec.BinaryReader(raw)); + if (['create_product', 'tick'].includes(message.value.reducer)) { + send({ tag: 'ReducerResult', value: { requestId: message.value.requestId, + timestamp: { __timestamp_micros_since_unix_epoch__: 1n }, result: { tag: 'OkEmpty' } } }); + return; + } + const reader = new codec.BinaryReader(message.value.args); + const [user, , role] = [readString(reader), readString(reader), readString(reader)]; + calls.push({ transport: 'native', role }); profiles.set(user, role); + if (mode === 'missing-terminal') { socket.close(); return; } + const acknowledge = () => send({ tag: 'ReducerResult', value: { requestId: message.value.requestId, + timestamp: { __timestamp_micros_since_unix_epoch__: 1n }, result: { tag: 'OkEmpty' } } }); + if (mode === 'append-native') { + heldReceipts.set(message.value.requestId, acknowledge); nativeReceived?.(message.value.requestId); return; + } + acknowledge(); + }); + }); + const dir = mkdtempSync(join(tmpdir(), 'auth-write-probe-')), leasePath = join(dir, 'lease.json'); + const previous = { path: process.env.STACK_BENCH_LEASE, token: process.env.STACK_BENCH_LEASE_TOKEN }; + const lease = createBackendLease({ runId: 'auth-write-probe', backend: 'spacetime', track: 'ecommerce', runIndex: 0, + serverUri: url, module: 'auth', dataDir: join(dir, 'data') }); + lease.state = 'active'; writeBackendLease(leasePath, lease); + process.env.STACK_BENCH_LEASE = leasePath; process.env.STACK_BENCH_LEASE_TOKEN = lease.ownershipToken; + const browser = await chromium.launch({ headless: true }); + try { + const fresh = async (user: string) => { + const context = await browser.newContext(), page = await context.newPage(); + await installAuthWebSocketCapture(page); await installSpacetimeWriteCapture(page); await page.goto(url); + await page.evaluate(async ({ url, user }) => { + document.cookie = `user=${user}`; + const socket = new WebSocket(`${url.replace('http:', 'ws:')}/nested/v1/database/auth/subscribe`, 'v3.bsatn.spacetimedb'); + Object.assign(window, { fixtureSocket: socket }); + await new Promise(resolve => socket.addEventListener('message', resolve, { once: true })); + }, { url, user }); + return page; + }; + const submit = async (page: Awaited>, user: string, omitFinalizer = false) => { + const writer = new codec.BinaryWriter(128); + for (const value of [user, 'opaque-derived-password', 'customer']) writeString(writer, value); + const bytes = encode(codec.ClientMessage, { tag: mode === 'procedure' ? 'CallProcedure' : 'CallReducer', value: { + ...(mode === 'procedure' ? { procedure: 'profile_save' } : { reducer: 'profile_save' }), + requestId: 1, flags: 0, args: writer.getBuffer() } }); + return page.evaluate(async ({ bytes, omitFinalizer }) => { + const socket = (window as unknown as { fixtureSocket: WebSocket }).fixtureSocket; + const reply = new Promise(resolve => { + socket.addEventListener('message', resolve, { once: true }); socket.addEventListener('close', resolve, { once: true }); + }); + socket.send(new Uint8Array(bytes)); await reply; + if (!omitFinalizer) await fetch('/nested/stdb/v1/identity/websocket-token', { method: 'POST' }); + }, { bytes: [...bytes], omitFinalizer }); + }; + for (const inventoryCall of [withAuthWriteInventory, withWriteCompletion]) await t.test(`${inventoryCall.name} waits for a reducer receipt appended during drain`, async () => { + mode = 'append-native'; + const page = await fresh(mode); + let firstReceived!: () => void, secondReceived!: () => void, submitted!: () => void; + const first = new Promise(resolve => { firstReceived = resolve; }); + const second = new Promise(resolve => { secondReceived = resolve; }); + const submission = new Promise(resolve => { submitted = resolve; }); + nativeReceived = id => { if (id === 1) firstReceived(); if (id === 2) secondReceived(); }; + const sendWrite = async (id: number) => { + const writer = new codec.BinaryWriter(128); + for (const value of ['append-native', 'opaque-derived-password', 'customer']) writeString(writer, value); + const bytes = encode(codec.ClientMessage, { tag: 'CallReducer', value: { + reducer: 'profile_save', requestId: id, flags: 0, args: writer.getBuffer() } }); + await page.evaluate(bytes => (window as unknown as { fixtureSocket: WebSocket }).fixtureSocket.send(new Uint8Array(bytes)), [...bytes]); + }; + try { + const inventory = inventoryCall(page, async () => { + await sendWrite(1); await first; submitted(); + }); + await submission; + await sendWrite(2); await second; + heldReceipts.get(1)!(); heldReceipts.delete(1); + const outcome = await Promise.race([ + inventory.then(() => 'measured', () => 'rejected'), + new Promise(resolve => setTimeout(() => resolve('waiting'), 150)), + ]); + assert.equal(outcome, 'waiting', 'an ACK for the first reducer must not prove the second reducer'); + heldReceipts.get(2)!(); heldReceipts.delete(2); + assert.equal((await inventory).writes.length, 2); + } finally { + for (const acknowledge of heldReceipts.values()) acknowledge(); + heldReceipts.clear(); nativeReceived = undefined; + await page.context().close(); mode = 'correct'; + } + }); + await t.test('a completion check waits for a reducer whose arguments it cannot decode', async () => { + const page = await fresh('undeclared'); + // Not in the schema, and an array argument the patch codec does not read. + const bytes = encode(codec.ClientMessage, { tag: 'CallReducer', value: { + reducer: 'create_product', requestId: 7, flags: 0, args: new Uint8Array([1, 0, 0, 0, 3, 0, 0, 0, 77, 117, 103]) } }); + const send = () => page.evaluate(async bytes => { + const socket = (window as unknown as { fixtureSocket: WebSocket }).fixtureSocket; + // A refused write is never sent, so the app waits only so long for its reply. + const reply = new Promise(resolve => { socket.addEventListener('message', resolve, { once: true }); setTimeout(resolve, 2000); }); + socket.send(new Uint8Array(bytes)); await reply; + }, [...bytes]); + try { + assert.equal((await withWriteCompletion(page, send)).writes.length, 1); + // A probe patches arguments, so its inventory still refuses a write it cannot read. + await assert.rejects(withAuthWriteInventory(page, send), ActionInconclusive); + } finally { await page.context().close(); } + }); + const baselinePage = await fresh('baseline'); + const baseline = await withAuthWriteInventory(baselinePage, () => submit(baselinePage, 'baseline')); + assert.deepEqual(baseline.writes.map(write => write.transport), ['spacetime-websocket', 'http']); + assert(!JSON.stringify(baseline).includes('opaque-derived-password')); + await t.test('a timer call without parameters does not disturb the signup sequence', { timeout: 30_000 }, async () => { + const page = await fresh('ticking'); + const tick = encode(codec.ClientMessage, { tag: 'CallReducer', value: { + reducer: 'tick', requestId: 9, flags: 0, args: new Uint8Array() } }); + try { + const result = await withAuthRequestPatch(page, 'ticking', 'unsubmitted-password', { fields: { role: 'admin' } }, async () => { + await page.evaluate(bytes => (window as unknown as { fixtureSocket: WebSocket }).fixtureSocket.send(new Uint8Array(bytes)), [...tick]); + await submit(page, 'ticking'); + }, undefined, 'signup', { writes: baseline.writes, index: 0 }); + assert.equal(result.requestPatch.transport, 'spacetime-websocket'); + } finally { await page.context().close(); } + }); + const restock = (page: typeof baselinePage) => page.evaluate(async () => (await fetch('/restock', { method: 'POST' })).status); + assert.equal(await restock(baselinePage), 403); + accounts.set('admin', 'admin'); assert.equal(await restock(await fresh('admin')), 200); + for (mode of ['correct', 'http-defect', 'native-defect']) for (const index of [0, 1]) { + const user = `${mode}-${index}`, page = await fresh(user), before = stock; calls.length = 0; + const result = await withAuthRequestPatch(page, user, 'unsubmitted-password', { fields: { role: 'admin' } }, + () => submit(page, user), undefined, 'signup', { writes: baseline.writes, index }); + assert.equal(result.requestPatch.transport, index === 0 ? 'spacetime-websocket' : undefined); + assert.deepEqual(calls.map(call => call.role), index === 0 ? ['admin', 'customer'] : ['customer', 'admin']); + const elevated = mode === 'http-defect' && index === 1 || mode === 'native-defect' && index === 0; + assert.equal(await restock(page), elevated ? 200 : 403); assert.equal(stock - before, elevated ? 1 : 0); + await page.context().close(); + } + for (mode of ['unreached', 'missing-terminal', 'procedure']) { + const page = await fresh(mode); + await assert.rejects(withAuthRequestPatch(page, mode, 'unused', { fields: { role: 'admin' } }, + () => submit(page, mode, true), undefined, 'signup', { writes: baseline.writes, index: 1 }), ActionInconclusive); + await page.context().close(); + } + mode = 'procedure'; + const procedure = await fresh(mode); + await assert.rejects(withAuthWriteInventory(procedure, () => submit(procedure, mode)), ActionInconclusive); + mode = 'missing-terminal'; + const incomplete = await fresh(mode); + await assert.rejects(withWriteCompletion(incomplete, () => submit(incomplete, mode, true)), ActionInconclusive); + } finally { + await browser.close(); sockets.close(); await new Promise(resolve => server.close(() => resolve())); + if (previous.path === undefined) delete process.env.STACK_BENCH_LEASE; else process.env.STACK_BENCH_LEASE = previous.path; + if (previous.token === undefined) delete process.env.STACK_BENCH_LEASE_TOKEN; else process.env.STACK_BENCH_LEASE_TOKEN = previous.token; + rmSync(dir, { recursive: true, force: true }); + } +}); + +test('credential patches preserve native envelopes and reject ambiguous matches', () => { + const credentials = { username: 'customer', password: 'secret' }; + const fields = JSON.parse('{"isAdmin":true,"__proto__":{"polluted":true}}'); + const claims = [{ name: 'name' }, { name: 'password' }, { name: 'salt' }, { name: 'claims', fields: ['isAdmin', 'nonce', '__proto__'] }]; + for (const body of [credentials, { path: 'auth:signIn', args: [{ provider: 'password', params: credentials }] }, + ['customer', 'secret', 'salt', { isAdmin: false, nonce: 4 }]]) { + const before = JSON.stringify(body); + const changed = patchAuthRequest(body, 'customer', 'secret', { fields }, Array.isArray(body) ? claims : undefined)!; + assert(changed.body.includes('"isAdmin":true')); + assert(changed.body.includes('"__proto__":')); + assert.equal(JSON.stringify(body), before); + assert.equal(Object.getPrototypeOf(credentials), Object.prototype); + } + assert.equal(patchAuthRequest({ other: 'secret' }, 'customer', 'secret', { fields }), null); + // Positional arguments take a field only where the interface declares it. + const named = (...names: string[]) => names.map(name => ({ name })); + const positional = ['customer', 'secret', 'salt', false]; + assert.equal(patchAuthRequest(positional, 'customer', 'secret', { fields: { isAdmin: true } }, + named('name', 'password', 'salt', 'is_admin'))!.body, '["customer","secret","salt",true]'); + assert.deepEqual(patchAuthRequest(positional.slice(0, 3), 'customer', 'secret', { fields: { role: 'admin' } }, + named('name', 'password', 'salt')), { body: '["customer","secret","salt"]', shape: 'positional', absentParameters: ['role'] }); + assert.throws(() => patchAuthRequest(positional.slice(0, 3), 'customer', 'secret', { fields }), /interface parameters/); + // A declared top-level parameter wins over a trailing object that does not declare the field. + const mixed = ['customer', 'secret', 'shopper', { nonce: 'keep' }]; + assert.equal(patchAuthRequest(mixed, 'customer', 'secret', { fields: { role: 'admin' } }, + [...named('name', 'password', 'role'), { name: 'metadata', fields: ['nonce'] }])!.body, + '["customer","secret","admin",{"nonce":"keep"}]'); + assert.throws(() => patchAuthRequest(mixed, 'customer', 'secret', { fields: { role: 'admin' } }), /interface parameters/, + 'a trailing object is never chosen just because it is last'); + assert.throws(() => patchAuthRequest(['customer', 'secret', { role: 'x' }, { role: 'y' }], 'customer', 'secret', + { fields: { role: 'admin' } }, [...named('name', 'password'), { name: 'a', fields: ['role'] }, { name: 'b', fields: ['role'] }]), + /Ambiguous/); + // A type that could hide the field leaves its location unknown; an exact top-level target is still proven. + assert.throws(() => patchAuthRequest(['customer', 'secret', { some: { role: 'x' } }], 'customer', 'secret', + { fields: { role: 'admin' } }, [...named('name', 'password'), { name: 'claims', open: true }]), /location unknown/); + assert.equal(patchAuthRequest(['customer', 'secret', 'shopper', {}], 'customer', 'secret', { fields: { role: 'admin' } }, + [...named('name', 'password', 'role'), { name: 'claims', open: true }])!.body, '["customer","secret","admin",{}]'); + assert.throws(() => patchAuthRequest([credentials, credentials], 'customer', 'secret', { fields }), /Multiple/); + assert.throws(() => patchAuthRequest({ ...credentials, repeated: 'customer' }, 'customer', 'secret', { fields }), /Ambiguous/); +}); + +test('credential patches can replace the located password without guessing its key', () => { + for (const body of [ + { username: 'customer', password: 'secret' }, + { path: 'auth:signIn', args: [{ provider: 'password', params: { username: 'customer', password: 'secret' } }] }, + ['customer', 'secret'], + ]) { + const changed = patchAuthRequest(body, 'customer', 'secret', { password: "' OR '1'='1" })!; + assert.match(changed.body, /' OR '1'='1/); + assert.doesNotMatch(changed.body, /secret/); + } + assert.throws(() => patchAuthRequest({}, 'customer', 'secret', {}), /request change/); +}); + +test('Convex credential probes preserve the live socket and require its matching response', async t => { + // Use the WebSocket server bundled with the pinned Playwright dependency. + const { wsServer } = createRequire(import.meta.url)('playwright-core/lib/utilsBundle'); + const server = createServer((_req, res) => res.end('auth')).listen(0, '127.0.0.1'); + await once(server, 'listening'); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const ws = new wsServer({ server }); + const received: unknown[] = []; + let effects = 0; + let mode: 'accept' | 'reject' | 'disconnect' | 'held' = 'accept'; + const held: { request: { type: string; requestId: number; args: Record[] }; + send(frame: object): void; close(): void }[] = []; + ws.on('connection', (socket: { on(event: string, callback: (data: Buffer) => void): void; + send(data: string): void; close(): void }) => socket.on('message', raw => { + const request = JSON.parse(String(raw)); received.push(request); + if (mode === 'disconnect') { socket.close(); return; } + if (mode === 'held') { + held.push({ request, send: frame => socket.send(JSON.stringify(frame)), close: () => socket.close() }); return; + } + socket.send(JSON.stringify({ type: `${request.type}Response`, requestId: request.requestId + 1, success: mode !== 'accept', result: null })); + const accepted = mode === 'accept'; + setTimeout(() => { + if (accepted) effects++; + socket.send(JSON.stringify({ type: `${request.type}Response`, requestId: request.requestId, success: accepted, result: null })); + }, 150); + })); + const browser = await chromium.launch({ headless: true }); + try { + for (mode of ['accept', 'reject', 'disconnect'] as const) { + const page = await browser.newPage(); + let sentFrames = 0; + page.on('websocket', socket => socket.on('framesent', () => sentFrames++)); + await installAuthWebSocketCapture(page); + await page.goto(url); + await page.evaluate(async url => { + const socket = new WebSocket(url.replace('http:', 'ws:') + '/api/1.0.0/sync'); + Object.assign(window, { testSocket: socket }); + await new Promise(resolve => socket.addEventListener('open', resolve, { once: true })); + }, url); + const submit = () => page.evaluate(async () => { + const socket = (window as unknown as { testSocket: WebSocket }).testSocket; + const done = new Promise(resolve => { + socket.addEventListener('message', event => { + if (JSON.parse(String(event.data)).requestId === 7) resolve(undefined); + }); + socket.addEventListener('close', resolve, { once: true }); + }); + socket.send(JSON.stringify({ type: 'Action', requestId: 7, udfPath: 'auth:signup', + args: [{ username: 'customer', password: 'secret', nonce: 'unchanged' }] })); + await done; + }); + const result = withAuthRequestPatch(page, 'customer', 'secret', { fields: { role: 'admin' } }, submit); + if (mode === 'disconnect') await assert.rejects(result, ActionInconclusive); + else { + const receipt = (await result).requestPatch; + assert.equal(receipt.transport, 'convex-websocket'); + assert.equal(receipt.success, mode === 'accept'); + assert.equal(receipt.status, undefined); + assert(!JSON.stringify(receipt).includes('secret')); + await submit(); // Outside the probe, the native request remains unchanged. + assert.equal((received.at(-1) as { args: { role?: string }[] }).args[0]!.role, undefined); + const before = effects; + const completed = withWriteCompletion(page, () => page.evaluate(() => { + (window as unknown as { testSocket: WebSocket }).testSocket.send(JSON.stringify({ + type: 'Mutation', requestId: 8, udfPath: 'shop:buy', args: [{ itemId: 'keyboard' }] })); + })); + assert.equal(await Promise.race([completed.then(() => 'done'), + new Promise(resolve => setTimeout(() => resolve('waiting'), 50))]), 'waiting'); + assert.equal((await completed).writes[0]!.transport, 'convex-websocket'); + assert.equal(effects - before, mode === 'accept' ? 1 : 0); + const previous = mode; + mode = 'disconnect'; + await assert.rejects(withWriteCompletion(page, submit), ActionInconclusive); + mode = previous; + } + const changed = received.find(value => (value as { args: { role?: string }[] }).args[0]?.role === 'admin'); + assert.deepEqual(changed, { type: 'Action', requestId: 7, udfPath: 'auth:signup', + args: [{ username: 'customer', password: 'secret', nonce: 'unchanged', role: 'admin' }] }); + received.length = 0; + await page.context().close(); + assert(sentFrames > 0, 'routing must preserve passive request observation'); + } + mode = 'held'; + for (const variant of ['completion', 'inventory', 'missing', 'two-sockets', 'selected-patch'] as const) { + await t.test(`overlapping writes: ${variant}`, async () => { + held.length = 0; + const page = await browser.newPage(); + await installAuthWebSocketCapture(page); await page.goto(url); + await page.evaluate(async twoSockets => { + const sockets = Array.from({ length: twoSockets ? 2 : 1 }, () => + new WebSocket(location.origin.replace('http:', 'ws:') + '/api/1.0.0/sync')); + Object.assign(window, { fixtureSockets: sockets }); + await Promise.all(sockets.map(socket => new Promise(resolve => socket.addEventListener('open', resolve, { once: true })))); + }, variant === 'two-sockets'); + const submit = () => page.evaluate(() => { + const sockets = (window as unknown as { fixtureSockets: WebSocket[] }).fixtureSockets; + sockets[0]!.send(JSON.stringify({ type: 'Action', requestId: 20, udfPath: 'auth:signup', + args: [{ username: 'customer', password: 'secret' }] })); + sockets.at(-1)!.send(JSON.stringify({ type: 'Mutation', requestId: sockets.length === 2 ? 20 : 21, + udfPath: 'profile:create', args: [{ username: 'customer' }] })); + }); + const reply = (index: number, success = true) => { + const write = held[index]!; + write.send({ type: `${write.request.type}Response`, requestId: write.request.requestId, success }); + }; + const waitForBoth = async () => { + for (let n = 0; n < 100 && held.length < 2; n++) await new Promise(resolve => setTimeout(resolve, 10)); + assert.equal(held.length, 2, 'the observer must forward both overlapping writes'); + // Separate sockets can reach the server in either order. + held.sort((left, right) => left.request.type.localeCompare(right.request.type)); + }; + const capture = variant === 'inventory' || variant === 'selected-patch' + ? withAuthWriteInventory(page, submit) : withWriteCompletion(page, submit); + // Handle rejection at once; close the browser even if an assertion fails. + const observed = capture.then(value => ({ value }), error => ({ error })); + try { + await waitForBoth(); + held[0]!.send({ type: 'ActionResponse', requestId: 999, success: true }); + held[0]!.send({ type: 'MutationResponse', requestId: 20, success: true }); + reply(1); + assert.equal(await Promise.race([observed.then(() => 'settled'), + new Promise(resolve => setTimeout(() => resolve('pending'), 75))]), 'pending', + 'an unrelated or later response cannot complete the earlier write'); + if (variant === 'missing') { + held[0]!.close(); + const result = await observed; + assert('error' in result && result.error instanceof ActionInconclusive); + } else { + reply(0); + const result = await observed; + assert('value' in result); + assert.equal(result.value.writes.length, 2); + if (variant === 'selected-patch') { + const baseline = result.value.writes; + held.length = 0; + const probe = withAuthWriteTarget(page, { writes: baseline, index: 0 }, () => + withAuthRequestPatch(page, 'customer', 'secret', { fields: { role: 'admin' } }, submit)); + const patched = probe.then(value => ({ value }), error => ({ error })); + await waitForBoth(); + assert.equal(held[0]!.request.args[0]!.role, 'admin'); + assert.equal(held[1]!.request.args[0]!.role, undefined); + reply(1, false); reply(0, true); + const outcome = await patched; + assert('value' in outcome); + assert.equal(outcome.value.requestPatch.success, true, 'only the selected request supplies the patch receipt'); + } + } + t.diagnostic(JSON.stringify({ fixture: 'convex-overlap', variant, forwarded: held.map(x => x.request.requestId), + expected: variant === 'missing' ? 'inconclusive' : 'complete', cleanup: 'page closed in finally' })); + } finally { await page.context().close(); await observed; } + }); + } + mode = 'accept'; + const page = await browser.newPage(); + await installAuthWebSocketCapture(page); + const gate = await installResponseLoss(page.context()); + await page.goto(url); + await page.evaluate(async url => { + const socket = new WebSocket(url.replace('http:', 'ws:') + '/api/1.0.0/sync'); + Object.assign(window, { testSocket: socket, replies: 0 }); + socket.onmessage = () => { (window as unknown as { replies: number }).replies++; }; + await new Promise(resolve => socket.addEventListener('open', resolve, { once: true })); + }, url); + gate.arm(); + await page.evaluate(() => (window as unknown as { testSocket: WebSocket }).testSocket.send(JSON.stringify({ + type: 'Mutation', requestId: 9, args: [{}], udfPath: 'shop:checkout' }))); + for (let n = 0; n < 100 && !gate.evidence().events.some(e => e.kind === 'ws-drop'); n++) { + await new Promise(resolve => setTimeout(resolve, 20)); + } + assert(gate.evidence().events.some(e => e.kind === 'ws-drop')); + assert.equal(await page.evaluate(() => (window as unknown as { replies: number }).replies), 0); + await gate.finish(); + assert.deepEqual(gate.evidence().errors, []); + await page.context().close(); + } finally { + await browser.close(); ws.close(); server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + } +}); + +test('real browser credential patch reaches the native request and keeps uncertain delivery unmeasured', async () => { + const received: { path: string; body: unknown; cookie?: string; authorization?: string; nonce?: string }[] = []; + const server = createServer(async (req, res) => { + if (req.method === 'GET') { res.writeHead(200, { 'Content-Type': 'text/html' }).end('auth'); return; } + let raw = ''; for await (const chunk of req) raw += chunk; + assert.equal(Buffer.byteLength(raw), Number(req.headers['content-length'])); + received.push({ path: req.url!, body: JSON.parse(raw), cookie: req.headers.cookie, + authorization: req.headers.authorization, nonce: req.headers['x-nonce'] as string }); + if (req.url === '/lost') { req.socket.destroy(); return; } + if (req.url === '/redirect') { res.writeHead(307, { Location: '/signup' }).end(); return; } + res.writeHead(200, { 'Content-Type': 'application/json', 'Set-Cookie': 'sid=created; Path=/; HttpOnly' }).end('{"accepted":true}'); + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const browser = await chromium.launch({ headless: true }); + try { + const context = await browser.newContext(); + await context.addCookies([{ name: 'sid', value: 'guest', url }]); + const page = await context.newPage(); await page.goto(url); + const body = { path: 'auth:signIn', args: [{ provider: 'password', params: { username: 'customer', password: 'secret', flow: 'signUp' } }] }; + const submit = (path = '/signup', data: unknown = body, contentType = 'application/json') => page.evaluate(async ({ path, data, contentType }) => { + try { return { status: (await fetch(path, { method: 'POST', headers: { 'Content-Type': contentType, Authorization: 'Bearer native-token', 'X-Nonce': 'kept' }, + body: typeof data === 'string' ? data : JSON.stringify(data) })).status }; } + catch { return { status: 0 }; } + }, { path, data, contentType }); + const run = (fn: () => Promise) => withAuthRequestPatch(page, 'customer', 'secret', { fields: { isAdmin: true } }, fn); + const result = await run(() => submit()); + assert.equal(result.requestPatch.status, 200); + assert.match(result.requestPatch.bodySha256, /^[a-f0-9]{64}$/); + assert.deepEqual(received[0], { path: '/signup', cookie: 'sid=guest', authorization: 'Bearer native-token', nonce: 'kept', + body: { path: 'auth:signIn', args: [{ provider: 'password', params: { username: 'customer', password: 'secret', flow: 'signUp', isAdmin: true } }] } }); + assert.equal((await context.cookies()).find(c => c.name === 'sid')?.value, 'created'); + assert(!JSON.stringify(result).includes('secret')); + for (const fn of [() => submit('/lost'), () => submit('/redirect'), () => submit('/signup', { other: 'value' }), + () => submit('/signup', [body, body]), () => submit('/signup', 'username=customer&password=secret', 'application/x-www-form-urlencoded'), + async () => { await submit(); return submit(); }]) { + await assert.rejects(run(fn), ActionInconclusive); + } + const redirects = received.filter(r => r.path === '/redirect'); assert.equal(redirects.length, 1); + for (const failure of [new ActionHarnessFailure('browser setup failed'), + new Error('locator.fill: Unexpected token in selector'), + new Error('Target page, context or browser has been closed'), + new ActionInconclusive('input dispatch was not confirmed'), undefined, false, null]) { + await assert.rejects(run(async () => { throw failure; }), error => { + assert.equal(error, failure); return true; + }); + } + // Incomplete capture cannot turn a browser-control failure into an app defect. + const timeout = Object.assign(new Error('locator.click: Timeout 1000ms exceeded'), { name: 'TimeoutError' }); + await assert.rejects(run(async () => { throw timeout; }), ActionInconclusive); + const appFailure = new ActionApplicationFailure('application control was missing'); + await assert.rejects(run(async () => { throw appFailure; }), ActionInconclusive); + await assert.rejects(run(async () => { await submit(); throw appFailure; }), error => { + assert.equal(error, appFailure); return true; + }); + // The route handler must be removed even after failed measurement. + await submit(); assert.equal(JSON.stringify(received.at(-1)!.body), JSON.stringify(body)); + await context.close(); + } finally { + await browser.close(); server.closeAllConnections(); await new Promise(resolve => server.close(() => resolve())); + } +}); + +test('a SpacetimeDB signup patch places authority by the module schema and never guesses', async () => { + // An object parameter is [name, its fields]; the schema declares it through the typespace, as a module does. + // [name, type] gives a raw schema type, with any referenced types in `typespace`. + let parameters: (string | [string, string[] | object])[] | null = ['name', 'password', 'salt']; + let typespace: unknown[] = []; + const calls: unknown[] = []; + const server = createServer(async (req, res) => { + if (req.url?.startsWith('/v1/database/shop/schema')) { + if (!parameters) { res.writeHead(404).end(); return; } + const types: unknown[] = [...typespace]; + const elements = parameters.map(parameter => typeof parameter === 'string' + ? { name: { some: parameter }, algebraic_type: { String: [] } } + : !Array.isArray(parameter[1]) ? { name: { some: parameter[0] }, algebraic_type: parameter[1] } + : { name: { some: parameter[0] }, algebraic_type: { Ref: types.push({ Product: { elements: parameter[1].map(field => + ({ name: { some: field }, algebraic_type: { String: [] } })) } }) - 1 } }); + res.writeHead(200, { 'Content-Type': 'application/json' }).end(JSON.stringify({ typespace: { types }, reducers: [], + misc_exports: [{ Procedure: { name: 'sign_up', params: { elements } } }] })); + return; + } + if (req.method === 'POST') { + let raw = ''; for await (const chunk of req) raw += chunk; + calls.push(JSON.parse(raw)); + res.writeHead(200, { 'Content-Type': 'application/json' }).end('true'); return; + } + res.writeHead(200, { 'Content-Type': 'text/html' }).end('shop'); + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage(); await page.goto(url); + const signUp = (args: unknown[]) => withAuthRequestPatch(page, 'claimant', 'secret', { fields: { role: 'admin' } }, + () => page.evaluate(async args => { + try { return (await fetch('/v1/database/shop/call/sign_up', { method: 'POST', + headers: { 'Content-Type': 'application/json', Authorization: 'Bearer native' }, body: JSON.stringify(args) })).status; } + catch { return 0; } + }, args)); + // The reference interface has no authority parameter: the schema proves it, and the request goes as sent. + const correct = await signUp(['claimant', 'secret', 'salt']); + assert.deepEqual(calls.at(-1), ['claimant', 'secret', 'salt']); + assert.deepEqual(correct.requestPatch.absentParameters, ['role']); + // A defective interface that accepts a role receives the claim where it declared it. + parameters = ['name', 'password', 'salt', 'role']; + await signUp(['claimant', 'secret', 'salt', 'customer']); + assert.deepEqual(calls.at(-1), ['claimant', 'secret', 'salt', 'admin']); + // So does one that declares the role inside a claims object. + parameters = ['name', 'password', 'salt', ['claims', ['role']]]; + await signUp(['claimant', 'secret', 'salt', { role: 'customer' }]); + assert.deepEqual(calls.at(-1), ['claimant', 'secret', 'salt', { role: 'admin' }]); + // A declared role parameter receives the claim even when a trailing object follows it. + parameters = ['name', 'password', 'role', ['metadata', ['nonce']]]; + await signUp(['claimant', 'secret', 'customer', { nonce: 'keep' }]); + assert.deepEqual(calls.at(-1), ['claimant', 'secret', 'admin', { nonce: 'keep' }]); + // Without parameter names the probe is unmeasured, not sent in a guessed shape. + parameters = null; + await assert.rejects(signUp(['claimant', 'secret', 'salt', { role: 'customer' }]), ActionInconclusive); + assert.equal(calls.length, 4); + // Optional claims: the role goes inside the object the app sent. + const field = (name: string, algebraic_type: object) => ({ name: { some: name }, algebraic_type }); + typespace = [{ Sum: { variants: [field('some', { Product: { elements: [field('role', { String: [] }), + field('nonce', { String: [] })] } }), field('none', { Product: { elements: [] } })] } }]; + parameters = ['name', 'password', ['claims', { Ref: 0 }]]; + await signUp(['claimant', 'secret', { some: { role: 'customer', nonce: 'keep' } }]); + assert.deepEqual(calls.at(-1), ['claimant', 'secret', { some: { role: 'admin', nonce: 'keep' } }]); + // Types the reader cannot see into are not proof of absence, and an absent option is not built: + // each leaves the probe unmeasured and sends nothing. + for (const [claims, argument] of [[{ Ref: 0 }, { none: [] }], [{ Ref: 9 }, { role: 'customer' }], + [{ Product: { elements: [field('settings', { Product: { elements: [field('role', { String: [] })] } })] } }, + { settings: { role: 'customer' } }]] as const) { + parameters = ['name', 'password', ['claims', claims]]; + await assert.rejects(signUp(['claimant', 'secret', argument]), ActionInconclusive); + } + assert.equal(calls.length, 5); + // A page whose own submit throws on the aborted request is still unmeasured, not a harness failure. + parameters = ['name', 'password', ['claims', { Ref: 9 }]]; + await assert.rejects(withAuthRequestPatch(page, 'claimant', 'secret', { fields: { role: 'admin' } }, + () => page.evaluate(() => fetch('/v1/database/shop/call/sign_up', { method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(['claimant', 'secret', { some: { role: 'customer' } }]) }).then(response => response.status))), + ActionInconclusive); + assert.equal(calls.length, 5); + // An optional plain value cannot hold the field, so absence stays measurable. + parameters = ['name', 'password', ['nickname', { Sum: { variants: [field('some', { String: [] }), + field('none', { Product: { elements: [] } })] } }]]; + const optional = await signUp(['claimant', 'secret', { some: 'pat' }]); + assert.deepEqual(calls.at(-1), ['claimant', 'secret', { some: 'pat' }]); + assert.deepEqual(optional.requestPatch.absentParameters, ['role']); + } finally { + await browser.close(); + server.close(); + } +}); + + +test('a platform password endpoint patch reaches the cross-origin request only through the stack hook', async () => { + const received: { path: string; body: unknown; apikey?: string }[] = []; + const gateway = createServer(async (req, res) => { + const cors = { 'Access-Control-Allow-Origin': String(req.headers.origin ?? '*'), + 'Access-Control-Allow-Headers': 'apikey, authorization, content-type', 'Access-Control-Allow-Methods': 'POST' }; + if (req.method === 'OPTIONS') { res.writeHead(204, cors).end(); return; } + let raw = ''; for await (const chunk of req) raw += chunk; + received.push({ path: req.url!, body: JSON.parse(raw), apikey: req.headers.apikey as string }); + res.writeHead(200, { ...cors, 'Content-Type': 'application/json' }).end('{"access_token":"token"}'); + }).listen(0, '127.0.0.1'); + const app = createServer((_req, res) => res.writeHead(200, { 'Content-Type': 'text/html' }).end('shop')) + .listen(0, '127.0.0.1'); + await Promise.all([once(gateway, 'listening'), once(app, 'listening')]); + const origin = (server: typeof app) => `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const lease = createBackendLease({ runId: 'auth-patch', backend: 'supabase', track: 'ecommerce', runIndex: 0, + serverUri: origin(gateway), database: 'postgres' }); + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage(); await page.goto(origin(app)); + // The application derives both credentials; neither typed value appears in the request. + const email = `${Buffer.from('claimant').toString('hex')}@accounts.invalid`; + const digest = createHash('sha256').update('secret').digest('hex'); + const signUp = () => page.evaluate(async ({ url, email, digest }) => (await fetch(`${url}/auth/v1/signup`, { + method: 'POST', headers: { 'Content-Type': 'application/json;charset=UTF-8', apikey: 'anon' }, + body: JSON.stringify({ email, password: digest, data: { username: 'claimant' } }) })).status, + { url: origin(gateway), email, digest }); + const result = await withAuthRequestPatch(page, 'claimant', 'secret', { fields: { role: 'admin' } }, signUp, + supabaseAuthRequestPatch(lease)); + assert.equal(result.requestPatch.shape, 'supabase-auth'); + assert.equal(result.requestPatch.status, 200); + assert.deepEqual(received.at(-1), { path: '/auth/v1/signup', apikey: 'anon', + body: { email, password: digest, role: 'admin', data: { username: 'claimant', role: 'admin' } } }); + assert(!JSON.stringify(result).includes(digest)); + // Without the stack hook the request holds neither typed credential, so nothing is proven or changed. + await assert.rejects(withAuthRequestPatch(page, 'claimant', 'secret', { fields: { role: 'admin' } }, signUp), + ActionInconclusive); + assert.deepEqual(received.at(-1)!.body, { email, password: digest, data: { username: 'claimant' } }); + // An invalid change is refused before any hook sees the request. + await assert.rejects(withAuthRequestPatch(page, 'claimant', 'secret', {}, signUp, supabaseAuthRequestPatch(lease)), + ActionInconclusive); + assert.equal(received.length, 2); + // A query-like password reaches password sign-in in place of the application's derived value. + const signIn = () => page.evaluate(async ({ url, email, digest }) => (await fetch(`${url}/auth/v1/token?grant_type=password`, { + method: 'POST', headers: { 'Content-Type': 'application/json', apikey: 'anon' }, + body: JSON.stringify({ email, password: digest }) })).status, { url: origin(gateway), email, digest }); + await withAuthRequestPatch(page, 'kim', 'not-kims-password', { password: "' OR '1'='1" }, signIn, + supabaseAuthRequestPatch(lease)); + assert.deepEqual(received.at(-1), { path: '/auth/v1/token?grant_type=password', apikey: 'anon', + body: { email, password: "' OR '1'='1" } }); + } finally { + await browser.close(); + gateway.close(); app.close(); + } +}); diff --git a/tools/stack-bench/tests/backend-control.test.ts b/tools/stack-bench/tests/backend-control.test.ts new file mode 100644 index 00000000000..c5598d52e49 --- /dev/null +++ b/tools/stack-bench/tests/backend-control.test.ts @@ -0,0 +1,362 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { captureApplicationDiagnostics, controlAppServer, controlBackendRuntime, drainApplicationDatabase, + parseRuntimeControlSpec } + from '../src/runtime/backend-control.js'; +import { createBackendLease, writeBackendLease } from '../src/runtime/backend-lease.js'; +import { STACK_ADAPTER_REGISTRY } from '../src/stacks/stack-adapters.js'; +import { controlHostedAppServer, HOSTED_START_TIMEOUT_MS, hostedLaunchCommand, hostedRecordedProcessStopScript, + hostedStopScript } + from '../src/stacks/hosted-lifecycle.js'; +import type { TextCommandOptions } from '../src/runtime/command-executor.js'; +import { attemptDatabaseUrl } from '../src/stacks/hosted-database-identity.js'; + +interface RecordedCommand { + argv: string[]; + options: TextCommandOptions; +} + +test('crash database drain requires observed zero work and preserves errors or an expired deadline', async () => { + for (const backend of ['postgres', 'mongodb']) { + const id = 'a'.repeat(64); + const lease = createBackendLease({ runId: `drain-${backend}`, backend, track: 'ecommerce', runIndex: 0, + database: 'app_ecom_run0', container: { name: 'database', id } }); + lease.resources.network = { name: 'attempt', id: 'b'.repeat(64), namespaceContainerId: id, + hostAddresses: ['172.20.0.1'], services: [], firewallSha256: null, firewallInstalledAt: null }; + const signal = new AbortController().signal; + const counts = ['1', '0']; + const receipt = await drainApplicationDatabase(lease, Date.now() + 5000, signal, (_command, args) => { + if (args[0] === 'inspect') return id; + assert.equal(args[0], 'exec'); + if (backend === 'postgres') assert.match(args.at(-1)!, /pg_stat_activity.*pid<>pg_backend_pid/); + else { + assert(args.some(arg => arg.includes('maxPoolSize=1'))); + assert.match(args.at(-1)!, /idleSessions:true/); + assert.match(args.at(-1)!, /connectionId:\{\$ne:self\}/); + // Only idle sessions end; a running commit is still waited for. + assert.match(args.at(-1)!, /op\.type === 'idleSession' && op\.lsid/); + assert.match(args.at(-1)!, /killSessions: idle/); + } + return counts.shift()!; + }); + assert.equal(receipt.settled, true); + assert.deepEqual(receipt.samples.map(sample => sample.pending), [1, 0]); + const expired = await drainApplicationDatabase(lease, Date.now() - 1, signal, () => id); + assert.equal(expired.settled, false); + assert.deepEqual(expired.samples, []); + const deadline = Date.now() + 20; + const timedOut = await drainApplicationDatabase(lease, deadline, signal, (_command, args) => { + if (args[0] === 'inspect') return id; + while (Date.now() < deadline) { /* Simulate a probe using its remaining deadline. */ } + throw Object.assign(new Error('private command details'), { code: 'ETIMEDOUT' }); + }); + assert.equal(timedOut.settled, false); + assert.deepEqual(timedOut.samples, []); + await assert.rejects(drainApplicationDatabase(lease, Date.now() + 5000, signal, (_command, args) => { + if (args[0] === 'inspect') return id; + throw Object.assign(new Error('private command details'), { code: 'ETIMEDOUT' }); + }), /could not observe pending database work/); + const cancelled = new AbortController(); + await assert.rejects(drainApplicationDatabase(lease, Date.now() + 5000, cancelled.signal, (_command, args) => { + if (args[0] === 'inspect') return id; + cancelled.abort(); + return '1'; + }), /abort/i); + for (const output of ['', '-1', 'NaN', '9007199254740992']) { + await assert.rejects(drainApplicationDatabase(lease, Date.now() + 5000, signal, + (_command, args) => args[0] === 'inspect' ? id : output), /invalid database work count/); + } + let samples = 0; + await assert.rejects(drainApplicationDatabase(lease, Date.now() + 5000, signal, (_command, args) => { + if (args[0] === 'inspect') return id; + if (samples++ === 0) return '1'; + throw new Error('command failed with private database credentials'); + }), error => { + assert(error instanceof Error && error.message === 'could not observe pending database work'); + assert('databaseDrain' in error); + assert.deepEqual((error.databaseDrain as { samples: Array<{ pending: number }> }).samples.map(row => row.pending), [1]); + return true; + }); + } +}); + +test('runtime control input is typed at the serialized boundary', () => { + const input = { backend: 'mongodb', app: '/app', port: 6301, probe: '/api/items' }; + assert.deepEqual(parseRuntimeControlSpec(input), input); + assert.throws(() => parseRuntimeControlSpec({ ...input, port: null }), + /runtime control spec is incomplete/); + assert.throws(() => parseRuntimeControlSpec({ ...input, port: '6301' }), + /runtime control spec is incomplete/); +}); + +test('backend control refuses without an authenticated lease', async () => { + const priorPath = process.env.STACK_BENCH_LEASE; + const priorToken = process.env.STACK_BENCH_LEASE_TOKEN; + delete process.env.STACK_BENCH_LEASE; + delete process.env.STACK_BENCH_LEASE_TOKEN; + try { + await assert.rejects(controlBackendRuntime({ backend: 'mongodb', app: '.', port: 6101, probe: '/api/items' }), + /STACK_BENCH_LEASE is required/); + } finally { + if (priorPath === undefined) delete process.env.STACK_BENCH_LEASE; + else process.env.STACK_BENCH_LEASE = priorPath; + if (priorToken === undefined) delete process.env.STACK_BENCH_LEASE_TOKEN; + else process.env.STACK_BENCH_LEASE_TOKEN = priorToken; + } +}); + +test('clean-source application start uses the private attempt database URL', async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-clean-source-')); + const leasePath = join(root, 'lease.json'); + const priorPath = process.env.STACK_BENCH_LEASE; + const priorToken = process.env.STACK_BENCH_LEASE_TOKEN; + writeFileSync(join(root, 'start.sh'), '#!/bin/sh\nexec app\n'); + try { + for (const backend of ['postgres', 'mongodb', 'spacetime']) { + const anchor = 'a'.repeat(64); + const id = 'c'.repeat(64); + const database = 'app_ecom_run0'; + const lease = createBackendLease(backend === 'spacetime' + ? { runId: 'clean-source-spacetime', backend, track: 'ecommerce', runIndex: 0, + serverUri: 'http://127.0.0.1:3310', module: 'shop-run-1', dataDir: join(root, 'spacetime-data') } + : { runId: `clean-source-${backend}`, backend, track: 'ecommerce', runIndex: 0, database, + container: { name: 'database', id: anchor } }); + lease.state = 'active'; + const namespace = backend === 'spacetime' ? null : anchor; + lease.resources.network = { name: 'attempt', id: 'b'.repeat(64), namespaceContainerId: namespace, + hostAddresses: ['172.20.0.1'], services: [], firewallSha256: null, firewallInstalledAt: null }; + lease.resources.buildContainer = { name: 'build', id, owned: true, running: true, + networkMode: namespace ? `container:${namespace}` : 'bridge', image: `sha256:${'d'.repeat(64)}`, + resourceLimits: { cpuCount: 2, memoryBytes: 1024, memorySwapBytes: 1024, pids: 32 } }; + writeBackendLease(leasePath, lease); + process.env.STACK_BENCH_LEASE = leasePath; + process.env.STACK_BENCH_LEASE_TOKEN = lease.ownershipToken; + const launchReached = new Error('launch captured without starting Docker'); + let launch: readonly string[] | undefined; + let launchEnvironment: NodeJS.ProcessEnv | undefined; + await assert.rejects(controlAppServer({ backend, app: root, port: 65534, probe: '' }, 'start', { + exec: (_command, args, options) => { + if (args[0] === 'inspect') return id; + if (args[0] === 'exec' && args.includes('-d')) { + launch = args; + launchEnvironment = options.env; + throw launchReached; + } + return ''; + }, + }), error => error === launchReached); + assert(launch); + if (backend === 'spacetime') { + assert(!launch.includes('DATABASE_URL')); + assert(launch.includes('VITE_MODULE_NAME') && launch.includes('VITE_SPACETIMEDB_URI')); + assert.equal(launchEnvironment?.VITE_MODULE_NAME, 'shop-run-1'); + assert.equal(launchEnvironment?.VITE_SPACETIMEDB_URI, 'http://127.0.0.1:3310'); + } else { + assert(launch.includes('DATABASE_URL')); + assert.equal(launchEnvironment?.DATABASE_URL, attemptDatabaseUrl({ backend, database, + ownershipToken: lease.ownershipToken })); + } + assert.equal(launchEnvironment?.VITE_PORT, '65534'); + assert.equal(launchEnvironment?.APP_WARM_START, '1'); + assert(!launch.some(value => value.includes('local-app-password'))); + // A first start from clean source is not a warm one: the flag is absent, not empty. + await assert.rejects(controlAppServer({ backend, app: root, port: 65534, probe: '' }, 'start', { + cleanSource: true, + exec: (_command, args, options) => { + if (args[0] === 'inspect') return id; + if (args[0] === 'exec' && args.includes('-d')) { launch = args; launchEnvironment = options.env; throw launchReached; } + return ''; + }, + }), error => error === launchReached); + assert.equal(launchEnvironment?.APP_WARM_START, undefined); + assert(!launch.includes('APP_WARM_START')); + assert.equal(launchEnvironment?.VITE_PORT, '65534'); + } + } finally { + if (priorPath === undefined) delete process.env.STACK_BENCH_LEASE; + else process.env.STACK_BENCH_LEASE = priorPath; + if (priorToken === undefined) delete process.env.STACK_BENCH_LEASE_TOKEN; + else process.env.STACK_BENCH_LEASE_TOKEN = priorToken; + rmSync(root, { recursive: true, force: true }); + } +}); + +test('restart diagnostics are copied only from the exact leased build container', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-backend-log-')); + const leasePath = join(root, 'lease.json'); + const output = join(root, 'restart.log'); + const lease = createBackendLease({ runId: 'diagnostics-test', backend: 'mongodb', + track: 'ecommerce', runIndex: 0, database: 'app_ecom_run0', + container: { name: 'database', id: 'd'.repeat(64) } }); + lease.state = 'active'; + lease.resources.buildContainer = { name: 'leased-build', id: 'a'.repeat(64), + running: true, owned: true, image: `sha256:${'b'.repeat(64)}`, + resourceLimits: { cpuCount: 2, memoryBytes: 1024, memorySwapBytes: 1024, pids: 32 } }; + writeBackendLease(leasePath, lease); + const priorPath = process.env.STACK_BENCH_LEASE; + const priorToken = process.env.STACK_BENCH_LEASE_TOKEN; + process.env.STACK_BENCH_LEASE = leasePath; + process.env.STACK_BENCH_LEASE_TOKEN = lease.ownershipToken; + const buildContainer = lease.resources.buildContainer; + assert(buildContainer); + const calls: RecordedCommand[] = []; + const exec = (command: string, args: readonly string[], options: TextCommandOptions): string => { + calls.push({ argv: [command, ...args], options }); + if (args[0] === 'inspect') return `${buildContainer.id}\n`; + return '===== /run/application/restart-mongodb-6723.log =====\nserver failed clearly\n'; + }; + try { + assert.deepEqual(captureApplicationDiagnostics(output, { exec }), { captured: true, path: output }); + assert.match(readFileSync(output, 'utf8'), /server failed clearly/); + const capture = calls[1]; + assert(capture); + assert.deepEqual(capture.argv.slice(0, 4), ['docker', 'exec', buildContainer.id, 'sh']); + assert.match(capture.argv.at(-1) ?? '', /reference-application\.log/); + assert.match(capture.argv.at(-1) ?? '', /restart-\*\.log/); + } finally { + if (priorPath === undefined) delete process.env.STACK_BENCH_LEASE; + else process.env.STACK_BENCH_LEASE = priorPath; + if (priorToken === undefined) delete process.env.STACK_BENCH_LEASE_TOKEN; + else process.env.STACK_BENCH_LEASE_TOKEN = priorToken; + rmSync(root, { recursive: true, force: true }); + } +}); + +test('hosted application stop targets safe process groups and exact group-1 listeners', () => { + const command = hostedStopScript(6301); + assert.ok((command.match(/lsof -ti tcp:6301 -sTCP:LISTEN/g)?.length ?? 0) >= 3, + 'listener ownership must be reacquired before TERM, before KILL, and during final verification'); + assert.match(command, /ps -o pgid=/); + assert.match(command, /\/bin\/kill -TERM -- "-\$pgid"/); + assert.match(command, /\/bin\/kill -TERM "\$pid"/); + assert.match(command, /\/bin\/kill -KILL -- "-\$pgid"/); + assert.match(command, /\/bin\/kill -KILL "\$pid"/); + assert.match(command, /self_pgid=/); + assert.match(command, /init_pgid=/); + assert.match(command, /"\$pgid" = 1/); + assert.match(command, /direct="\$direct \$pid"/); + assert.throws(() => hostedStopScript('6301; rm -rf /'), /invalid hosted application port/); + assert.throws(() => hostedRecordedProcessStopScript('/app/server.pid'), + /invalid hosted application process record/); +}); + +test('hosted application launch supports generated and fixed reference apps', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-hosted-launch-')); + try { + writeFileSync(join(root, 'start.sh'), '#!/usr/bin/env bash\nset -euo pipefail\nexec app\n'); + assert.deepEqual(hostedLaunchCommand(root), { + directory: '.', command: '/bin/bash ./start.sh', + }); + rmSync(join(root, 'start.sh')); + writeFileSync(join(root, 'package.json'), JSON.stringify({ + scripts: { start: 'node index.js', dev: 'node --watch index.js' }, + })); + assert.deepEqual(hostedLaunchCommand(root), { + directory: '.', command: '/usr/local/bin/npm run start', + }); + writeFileSync(join(root, 'package.json'), JSON.stringify({ scripts: { dev: 'vite' } })); + assert.throws(() => hostedLaunchCommand(root), error => + error instanceof Error + && error.message === 'app has no start.sh or npm start script' + && 'code' in error && error.code === 'generated_app_not_restartable'); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('hosted application control inspects and stops listeners as the application user', async () => { + const id = 'a'.repeat(64); + const calls: Array<{ command: string; args: readonly string[]; options: TextCommandOptions }> = []; + const exec = (command: string, args: readonly string[], options: TextCommandOptions): string => { + calls.push({ command, args, options }); + if (args[0] === 'inspect') return `${id}\n`; + return ''; + }; + + await controlHostedAppServer({ + adapterId: 'mongodb', + lease: { resources: { buildContainer: { name: 'leased-build', id, owned: true } } }, + app: '.', + port: 65534, + probe: '/', + mode: 'stop', + handoffWorkspace: true, + exec, + }); + + const recordedStop = calls[1]; + const stop = calls[2]; + assert(recordedStop && stop); + assert.deepEqual(recordedStop.args.slice(0, 3), ['exec', id, 'sh']); + assert.match(recordedStop.args.at(-1) ?? '', /restart-mongodb-65534\.pid/); + assert.deepEqual(stop.args.slice(0, 7), [ + 'exec', '--user', '10001:10001', '-e', 'HOME=/home/developer', '-e', 'USER=developer', + ]); + assert.equal(stop.args[7], id); + assert.match(stop.args.at(-1) ?? '', /lsof -ti tcp:65534 -sTCP:LISTEN/); + assert(calls.some(call => call.args.slice(0, 3).join(' ') === `exec ${id} chown`)); +}); + +test('application control rejects unsupported modes before touching a container', async () => { + let calls = 0; + await assert.rejects(controlHostedAppServer({ + adapterId: 'mongodb', + lease: { resources: { buildContainer: null } }, + app: '.', + port: 6301, + probe: '/', + mode: 'invalid' as 'restart', + exec: () => { calls += 1; return ''; }, + }), /unsupported application control mode invalid/); + assert.equal(calls, 0); + + assert.throws(() => STACK_ADAPTER_REGISTRY.get('spacetime').lifecycle.control!({ + adapterId: 'spacetime', lease: {} as never, app: '.', port: 6301, probe: '/', mode: 'stop', + }), /unsupported SpacetimeDB control mode stop/); +}); + +test('hosted application start fails as soon as the launch exits without a listener and keeps its log', async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-hosted-exit-')); + const id = 'b'.repeat(64); + writeFileSync(join(root, 'start.sh'), '#!/bin/sh\nexec app\n'); + const log = ['[start.sh] Installing dependencies', 'npm ERR! network request failed', + 'npm ERR! A complete log of this run can be found in: /home/developer/.npm/_logs/x.log']; + const launches: string[] = []; + const exec = (_command: string, args: readonly string[]): string => { + if (args[0] === 'inspect') return `${id}\n`; + if (args[0] === 'exec' && args[2] === 'tail') return `${log.join('\n')}\n`; + const script = String(args.at(-1) ?? ''); + if (args[0] === 'exec' && args.includes('-d')) launches.push(script); + if (/\/proc\/\$pid/.test(script) || /-sTCP:LISTEN\)"\s*\]/.test(script)) { + throw Object.assign(new Error('exit 1'), { status: 1 }); + } + return ''; + }; + try { + for (const [adapterId, restarts] of [['postgres', 1], ['spacetime', 2]] as const) { + launches.length = 0; + for (let restart = 0; restart < restarts; restart++) { + const startedAt = Date.now(); + await assert.rejects(controlHostedAppServer({ + adapterId, + lease: { resources: { buildContainer: { name: 'leased-build', id, owned: true } } }, + app: root, + port: 65533, + probe: '/', + mode: 'start', + exec, + }), error => error instanceof Error + && error.message.startsWith(`${adapterId} application exited before it listened on port 65533: `) + && error.message.includes('npm ERR! network request failed') + && 'code' in error && error.code === 'generated_app_not_restartable' + && 'startLog' in error && error.startLog === log.join('\n')); + assert.ok(Date.now() - startedAt < HOSTED_START_TIMEOUT_MS / 10); + } + const logs = launches.map(script => script.match(/restart-[a-z]+-65533-[a-f0-9-]+\.log/)?.[0]); + assert.equal(logs.length, restarts); + assert(logs.every(Boolean)); + assert.equal(new Set(logs).size, restarts, 'restarts must not overwrite earlier process logs'); + } + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/tools/stack-bench/tests/backend-lease-processes.integration.ts b/tools/stack-bench/tests/backend-lease-processes.integration.ts new file mode 100644 index 00000000000..6623e48da6e --- /dev/null +++ b/tools/stack-bench/tests/backend-lease-processes.integration.ts @@ -0,0 +1,105 @@ +import assert from 'node:assert/strict'; +import { execFileSync, spawn } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { createServer } from 'node:http'; +import type { Server } from 'node:http'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { compiledEntrypoint } from '../src/package-root.js'; +import { createBackendLease, readBackendLease, writeBackendLease } + from '../src/runtime/backend-lease.js'; +import { releaseBackendLease } from '../src/runtime/backend-teardown.js'; +import { processIdentity } from '../src/runtime/platform.js'; + +// Lease rules that need a real second process: an owned runtime process to +// stop, and a lock held by another controller. + +async function listen(server: Server): Promise { + await new Promise((resolve, reject) => + server.listen(0, '127.0.0.1', resolve).once('error', reject)); + const address = server.address(); + assert(address && typeof address !== 'string'); + return address.port; +} + +async function close(server: Server): Promise { + await new Promise((resolve, reject) => + server.close(error => error ? reject(error) : resolve())); +} + +function processStderr(error: unknown): string { + if (error === null || typeof error !== 'object' || !('stderr' in error)) return ''; + return String(error.stderr ?? ''); +} + +test('starting Spacetime cleanup requires and stops the exact launched process', async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-starting-release-')); + const path = join(root, 'lease.json'); + const ambiguousPath = join(root, 'ambiguous.json'); + const first = createServer(); + const ownedPort = await listen(first); + await close(first); + const second = createServer(); + const ambiguousPort = await listen(second); + await close(second); + const child = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], + { detached: true, stdio: 'ignore' }); + child.unref(); + try { + const lease = createBackendLease({ runId: 'starting-owned', backend: 'spacetime', track: 'chat', + runIndex: 0, serverUri: `http://127.0.0.1:${ownedPort}`, module: 'starting-owned', + dataDir: join(root, 'owned-data') }); + lease.state = 'starting'; + lease.resources.launchedProcess = child.pid ? processIdentity(child.pid) : null; + writeBackendLease(path, lease); + assert.equal(releaseBackendLease(path, lease.ownershipToken), true); + assert.equal(readBackendLease(path, { token: lease.ownershipToken }).state, 'released'); + + const ambiguous = createBackendLease({ runId: 'starting-ambiguous', backend: 'spacetime', + track: 'chat', runIndex: 1, serverUri: `http://127.0.0.1:${ambiguousPort}`, + module: 'starting-ambiguous', dataDir: join(root, 'ambiguous-data') }); + ambiguous.state = 'starting'; + writeBackendLease(ambiguousPath, ambiguous); + assert.equal(releaseBackendLease(ambiguousPath, ambiguous.ownershipToken), false); + assert.equal(readBackendLease(ambiguousPath, { token: ambiguous.ownershipToken }).state, 'starting'); + } finally { + try { child.kill('SIGKILL'); } catch { /* already stopped by lease teardown */ } + rmSync(root, { recursive: true, force: true }); + } +}); + +test('appliance controllers share locks and require recovery after an owner exits', + { skip: process.platform !== 'linux' ? 'Kernel flock requires Linux' : false }, async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-shared-lock-')); + const helper = compiledEntrypoint('tests', 'fixtures', 'resource-lock-process.js'); + const owner = spawn(process.execPath, [helper, root, 'first-controller', 'hold'], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + try { + await new Promise((resolve, reject) => { + let stderr = ''; + owner.stderr.on('data', chunk => { stderr += chunk; }); + owner.stdout.once('data', chunk => { + if (String(chunk).includes('acquired')) resolve(); + else reject(new Error(`lock helper returned unexpected output: ${chunk}`)); + }); + owner.once('error', reject); + owner.once('exit', code => reject(new Error(`lock helper exited ${code}: ${stderr}`))); + }); + assert.throws(() => execFileSync(process.execPath, + [helper, root, 'second-controller'], { encoding: 'utf8', stdio: 'pipe' }), + error => /already leased by first-controller/.test(processStderr(error))); + + owner.kill('SIGTERM'); + await new Promise(resolve => owner.once('exit', () => resolve())); + assert.throws(() => execFileSync(process.execPath, + [helper, root, 'second-controller'], { encoding: 'utf8', stdio: 'pipe' }), + error => /remains leased by first-controller; run authenticated recovery/ + .test(processStderr(error))); + } finally { + if (owner.exitCode === null) owner.kill('SIGKILL'); + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/tools/stack-bench/tests/backend-lease.test.ts b/tools/stack-bench/tests/backend-lease.test.ts new file mode 100644 index 00000000000..9a540886339 --- /dev/null +++ b/tools/stack-bench/tests/backend-lease.test.ts @@ -0,0 +1,556 @@ +import { activateHosted } from '../src/stacks/hosted-lifecycle.js'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, utimesSync, writeFileSync } from 'node:fs'; +import { createServer } from 'node:http'; +import type { Server } from 'node:http'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import test from 'node:test'; +import { runBounded } from '../src/runtime/bounded-process.js'; +import { + CAPACITY_WAIT_RECEIPT_ENV, + readCapacityWait, + createBackendLease, + acquireResourceLocks, + backendResourceLockKeys, + publicBackendLease, + readBackendLease, + resourceLockScope, + updateBackendLease, + writeBackendLease, + GRADING_WAIT_RECEIPT_ENV, + acquireGradingSlot, + readGradingWaits, + recordedGradingWaitMs, + releaseResourceLocks, +} from '../src/runtime/backend-lease.js'; +import { dockerNetworkMissing, handoffBuildWorkspace, releaseBackendLease, stopLeasedContainer } from '../src/runtime/backend-teardown.js'; +import { ATTEMPT_CREATION_LABEL } from '../src/runtime/container-identity.js'; + +async function listen(server: Server): Promise { + await new Promise((resolve, reject) => + server.listen(0, '127.0.0.1', resolve).once('error', reject)); + const address = server.address(); + assert(address && typeof address !== 'string'); + return address.port; +} + +async function close(server: Server): Promise { + await new Promise((resolve, reject) => + server.close(error => error ? reject(error) : resolve())); +} + +function fixture() { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-lease-')); + const path = join(root, 'lease.json'); + const lease = createBackendLease({ + runId: 'chat-spacetime-run0-test', backend: 'spacetime', track: 'chat', runIndex: 0, + serverUri: 'http://127.0.0.1:3210', module: 'app-run0', dataDir: join(root, 'data'), + }); + return { root, path, lease }; +} + +test('lease reads require the matching token, backend, and active state', () => { + const f = fixture(); + try { + f.lease.state = 'active'; + writeBackendLease(f.path, f.lease); + assert.equal(readBackendLease(f.path, { + token: f.lease.ownershipToken, backend: 'spacetime', active: true, + }).runId, f.lease.runId); + assert.throws(() => readBackendLease(f.path, { token: 'wrong' }), /token does not match/); + assert.throws(() => readBackendLease(f.path, { backend: 'postgres' }), /not postgres/); + } finally { rmSync(f.root, { recursive: true, force: true }); } +}); + +test('lease ownership tokens are stored with private filesystem modes', t => { + if (process.platform === 'win32') return t.skip('POSIX modes are not enforced on Windows'); + const f = fixture(); + try { + writeBackendLease(f.path, f.lease); + assert.equal(statSync(f.root).mode & 0o777, 0o700); + assert.equal(statSync(f.path).mode & 0o777, 0o600); + } finally { rmSync(f.root, { recursive: true, force: true }); } +}); + +test('leases reject unknown lifecycle states and malformed process identities', () => { + const f = fixture(); + assert.throws(() => writeBackendLease(f.path, { ...f.lease, state: 'guessing' }), + /unknown state guessing/); + assert.throws(() => writeBackendLease(f.path, { ...f.lease, state: 'starting', + resources: { ...f.lease.resources, launchedProcess: { pid: -1, startMarker: '1' } } }), + /launchedProcess must be a process identity/); + assert.throws(() => writeBackendLease(f.path, { ...f.lease, + resources: { ...f.lease.resources, listenerProcesses: [{ pid: 1, startMarker: 'bad' }] } }), + /listenerProcesses must contain only process identities/); + const invalidContainer = { name: 'build', id: 'container-id', image: 'image-id', + owned: true, networkMode: 'ambient' }; + assert.throws(() => writeBackendLease(f.path, { ...f.lease, + resources: { ...f.lease.resources, buildContainer: invalidContainer } }), + /buildContainer.networkMode is invalid/); + const missingLimits = { ...invalidContainer, networkMode: 'bridge' }; + assert.throws(() => writeBackendLease(f.path, { ...f.lease, + resources: { ...f.lease.resources, buildContainer: missingLimits } }), + /buildContainer.resourceLimits is invalid/); + const invalidLimits = { ...missingLimits, resourceLimits: { + cpuCount: 2, memoryBytes: 4096, memorySwapBytes: 2048, pids: 512 } }; + assert.throws(() => writeBackendLease(f.path, { ...f.lease, + resources: { ...f.lease.resources, buildContainer: invalidLimits } }), + /buildContainer.resourceLimits is invalid/); +}); + +test('public lease evidence hashes rather than exposes the ownership token', () => { + const f = fixture(); + try { + f.lease.resources.buildContainer = { name: 'build', id: 'container-id', image: 'image-id', + owned: true, networkMode: 'bridge', resourceLimits: { + cpuCount: 2, memoryBytes: 4096, memorySwapBytes: 4096, pids: 512, + } }; + f.lease.campaignDelegation = { path: '/private/delegation.json', token: 'private-child-token' }; + const publicLease = publicBackendLease(f.lease); + assert.equal('ownershipToken' in publicLease, false); + assert.equal('campaignDelegation' in publicLease, false); + assert.match(publicLease.ownership.markerSha256, /^[0-9a-f]{64}$/); + assert(publicLease.resources.buildContainer); + assert.deepEqual(publicLease.resources.buildContainer.resourceLimits, + { cpuCount: 2, memoryBytes: 4096, memorySwapBytes: 4096, pids: 512 }); + } finally { rmSync(f.root, { recursive: true, force: true }); } +}); + +test('Spacetime and Convex leases reject non-loopback and portless targets', () => { + for (const base of [ + { runId: 'unsafe', backend: 'spacetime', track: 'chat', runIndex: 0, module: 'app-run0', dataDir: tmpdir() }, + { runId: 'unsafe', backend: 'convex', track: 'ecommerce', runIndex: 0 }, + ]) { + assert.throws(() => createBackendLease({ ...base, serverUri: 'https://production.example:443' }), + /must use http/); + assert.throws(() => createBackendLease({ ...base, serverUri: 'http://localhost' }), + /explicit loopback port/); + } +}); + +test('private Convex lifecycle releases before first creation intent and preserves refusal', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-convex-release-')); + const path = join(root, 'lease.json'); + const lease = createBackendLease({ runId: 'convex-before-start', backend: 'convex', + track: 'ecommerce', runIndex: 0, serverUri: 'http://127.0.0.1:14310' }); + try { + writeBackendLease(path, lease); + assert.throws(() => releaseBackendLease(path, 'wrong-token'), /ownership token does not match/); + assert.equal(releaseBackendLease(path, lease.ownershipToken, { hostTeardown: () => false }), false); + assert.equal(readBackendLease(path).state, 'created'); + assert.equal(releaseBackendLease(path, lease.ownershipToken), true); + assert.equal(readBackendLease(path).state, 'released'); + assert.equal(releaseBackendLease(path, lease.ownershipToken), true); + assert.throws(() => createBackendLease({ ...lease, serverUri: 'http://127.0.0.1:14310', + container: { name: 'ambient', id: 'ambient' } }), /requires an owned container/); + const sharedPath = join(root, 'shared.json'); + const shared = createBackendLease({ + runId: 'chat-postgres-run0-supervisor', backend: 'postgres', track: 'chat', runIndex: 0, + database: 'app_supervisor', container: { name: 'unused-postgres', id: 'unused-id' }, + }); + shared.state = 'active'; + writeBackendLease(sharedPath, shared); + assert.equal(releaseBackendLease(sharedPath, shared.ownershipToken), true); + const released = readBackendLease(sharedPath, { token: shared.ownershipToken }); + assert.equal(released.state, 'released'); + assert(released.releasedAt); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('private host teardown refusal retains resource locks', { skip: process.platform !== 'linux' }, () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-convex-refused-')); + const path = join(root, 'lease.json'); + const lease = createBackendLease({ runId: 'convex-refused', backend: 'convex', + track: 'ecommerce', runIndex: 0, serverUri: 'http://127.0.0.1:14310' }); + try { + const lock = acquireResourceLocks({ root, keys: ['port:14310'], lease })[0]!; + lease.resources.locks.push(lock); + writeBackendLease(path, lease); + assert.equal(releaseBackendLease(path, lease.ownershipToken, { hostTeardown: () => false }), false); + assert(existsSync(lock.path)); + assert.equal(readBackendLease(path).resources.locks[0]?.releasedAt, undefined); + assert.equal(releaseBackendLease(path, lease.ownershipToken), true); + assert(!existsSync(lock.path)); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('build workspace handback stops private-namespace writers before changing permissions', () => { + const id = 'a'.repeat(64); + const recorded: string[][] = []; + handoffBuildWorkspace(id, (_command, args) => { + recorded.push([...args]); + return args[0] === 'inspect' ? JSON.stringify({ state: { Status: 'running', Running: true }, pidMode: '' }) : ''; + }); + assert.equal(recorded.length, 4); + assert.equal(recorded[0]?.at(-1), id); + assert.deepEqual(recorded[1]?.slice(0, 6), ['exec', '--user', '0:0', id, 'sh', '-ec']); + assert.match(recorded[1]?.at(-1) ?? '', /pkill -KILL -u 10001/); + assert.match(recorded[1]?.at(-1) ?? '', /application writers did not stop/); + assert.deepEqual(recorded.slice(2).map(args => args.slice(4)), [ + ['chown', '-R', `10001:${process.getgid?.() ?? 0}`, '/app'], + ['chmod', '-R', 'u+rwX,g+rwX,o-rwx', '/app'], + ]); + for (const runtime of [ + { state: { Status: 'exited', Running: false }, pidMode: '' }, + { state: { Status: 'running', Running: true }, pidMode: 'host' }, + { state: { Status: 'running', Running: true }, pidMode: `container:${'b'.repeat(64)}` }, + ]) { + assert.throws(() => handoffBuildWorkspace(id, (_command, args) => { + assert.equal(args[0], 'inspect', 'unsafe containers must never receive a signal'); + return JSON.stringify(runtime); + }), /workspace handback requires/); + } + handoffBuildWorkspace(id, (_command, args) => { + assert.equal(args[0], 'inspect'); + return JSON.stringify({ state: { Status: 'created', Running: false }, pidMode: '' }); + }); +}); + +test('network teardown recognizes both Docker missing-network responses without hiding other failures', () => { + for (const message of [ + 'Error response from daemon: No such network: sb-owned-network', + `Error response from daemon: network ${'a'.repeat(64)} not found`, + ]) { + assert(dockerNetworkMissing(new Error(message))); + assert(dockerNetworkMissing({ stderr: Buffer.from(message) })); + } + for (const message of [ + 'Cannot connect to the Docker daemon', 'permission denied', + 'network sb-owned-network has active endpoints', 'No such container: owned-container', + ]) assert.equal(dockerNetworkMissing(new Error(message)), false); +}); + +test('container teardown retries transient Docker removal failures', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-container-retry-')); + const path = join(root, 'lease.json'); + try { + const lease = createBackendLease({ + runId: 'container-retry', backend: 'postgres', track: 'chat', runIndex: 0, + database: 'container_retry', container: { name: 'owned-container', id: 'owned-id' }, + }); + lease.state = 'active'; + lease.resources.buildContainer = { + name: 'owned-build', id: 'owned-build-id', image: 'build-image', owned: true, + running: true, networkMode: 'bridge', + resourceLimits: { cpuCount: 2, memoryBytes: 4096, memorySwapBytes: 4096, pids: 512 }, + }; + writeBackendLease(path, lease); + const refused = { + inspect: () => 'owned-build-id', + handoffWorkspace: () => { throw new Error('workspace handback failed'); }, + remove: () => assert.fail('a container must remain available until workspace handback succeeds'), + wait: () => undefined, + }; + assert.throws(() => stopLeasedContainer(path, lease.ownershipToken, refused), /workspace handback failed/); + assert.equal(readBackendLease(path, { token: lease.ownershipToken }) + .resources.buildContainer?.workspaceHandedBackAt, undefined); + assert.throws(() => stopLeasedContainer(path, lease.ownershipToken, { + ...refused, inspect: () => { throw new Error('No such container: owned-build-id'); }, + }), /disappeared before workspace handback/); + assert.equal(stopLeasedContainer(path, lease.ownershipToken, { + ...refused, inspect: () => 'replacement-id', + }), false); + assert.equal(readBackendLease(path, { token: lease.ownershipToken }).resources.buildContainer?.running, true); + let attempts = 0; + const delays: number[] = []; + assert.equal(stopLeasedContainer(path, lease.ownershipToken, { + inspect: () => 'owned-build-id', + handoffWorkspace: () => assert.equal(attempts, 0), + remove: () => { + assert(readBackendLease(path, { token: lease.ownershipToken }).resources.buildContainer?.workspaceHandedBackAt); + attempts += 1; + if (attempts < 3) throw new Error('Docker daemon is temporarily unavailable'); + }, + wait: delay => delays.push(delay), + }), true); + assert.equal(attempts, 3); + assert.deepEqual(delays, [0, 250, 750]); + assert.equal(readBackendLease(path, { token: lease.ownershipToken }) + .resources.buildContainer?.running, false); + updateBackendLease(path, { token: lease.ownershipToken }, next => { + delete next.resources.buildContainer!.removedAt; + next.resources.buildContainer!.running = true; + return next; + }); + assert.equal(stopLeasedContainer(path, lease.ownershipToken, { + ...refused, inspect: () => { throw new Error('No such container: owned-build-id'); }, + }), true, 'handback evidence closes the crash gap between removal and its lease update'); + updateBackendLease(path, { token: lease.ownershipToken }, next => { + delete next.resources.buildContainer!.removedAt; + next.resources.buildContainer!.running = true; + return next; + }); + assert.equal(stopLeasedContainer(path, lease.ownershipToken, { + inspect: () => 'owned-build-id', + handoffWorkspace: () => undefined, + remove: () => { throw new Error('No such container: owned-build-id'); }, + wait: () => undefined, + }), true, 'an exact container removed after inspection is already gone'); + assert.equal(readBackendLease(path, { token: lease.ownershipToken }) + .resources.buildContainer?.running, false); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('pre-activation Spacetime cleanup releases only when its leased port stayed empty', async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-created-release-')); + const emptyPath = join(root, 'empty.json'); + const occupiedPath = join(root, 'occupied.json'); + const emptyServer = createServer(); + const emptyPort = await listen(emptyServer); + await close(emptyServer); + const occupied = createServer((_request, response) => response.end('foreign')); + const occupiedPort = await listen(occupied); + try { + const empty = createBackendLease({ runId: 'created-empty', backend: 'spacetime', track: 'chat', + runIndex: 0, serverUri: `http://127.0.0.1:${emptyPort}`, module: 'created-empty', + dataDir: join(root, 'empty-data') }); + writeBackendLease(emptyPath, empty); + assert.equal(releaseBackendLease(emptyPath, empty.ownershipToken), true); + assert.equal(readBackendLease(emptyPath, { token: empty.ownershipToken }).state, 'released'); + + const blocked = createBackendLease({ runId: 'created-occupied', backend: 'spacetime', track: 'chat', + runIndex: 1, serverUri: `http://127.0.0.1:${occupiedPort}`, + module: 'created-occupied', dataDir: join(root, 'occupied-data') }); + writeBackendLease(occupiedPath, blocked); + assert.equal(releaseBackendLease(occupiedPath, blocked.ownershipToken), false); + assert.equal(readBackendLease(occupiedPath, { token: blocked.ownershipToken }).state, 'created'); + assert.equal((await fetch(`http://127.0.0.1:${occupiedPort}`)).status, 200); + } finally { + occupied.closeAllConnections(); + await close(occupied); + rmSync(root, { recursive: true, force: true }); + } +}); + +test('bench and reference leases use the same canonical slot and backend keys', () => { + const input = { backend: 'spacetime', track: 'ecommerce', runIndex: 0, + serverUri: 'http://127.0.0.1:3310', module: 'app-ecom-run0', + dataDir: tmpdir() }; + const bench = createBackendLease({ ...input, runId: 'bench' }); + const reference = createBackendLease({ ...input, runId: 'reference' }); + const preparedKeys = ['listener:http://127.0.0.1:3310']; + const expected = [ + 'listener:http://127.0.0.1:3310', + 'port:3310', + 'port:6473', + 'slot:ecommerce:spacetime:run0', + ]; + assert.deepEqual(backendResourceLockKeys(bench, { vite: 6473, express: null }, preparedKeys), expected); + assert.deepEqual(backendResourceLockKeys(reference, { vite: 6473, express: null }, preparedKeys), expected); +}); + +test('resource lock scope is shared only for the appliance', () => { + const temporaryDirectory = join(tmpdir(), 'scope-test'); + assert.deepEqual(resourceLockScope({}, { temporaryDirectory }), { + root: join(temporaryDirectory, 'stack-bench-resource-locks'), + }); + assert.deepEqual(resourceLockScope({ STACK_BENCH_APPLIANCE: '1' }), { + root: '/var/lib/stack-bench/controller-home/resource-locks', + }); + assert.throws(() => resourceLockScope({ STACK_BENCH_RESOURCE_LOCK_DIR: 'relative' }), + /must be an absolute path/); +}); + +test('a different PID start marker cannot authorize reclamation across controller namespaces', { skip: process.platform !== 'linux' ? 'Kernel flock requires the Linux appliance' : false }, () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-lock-pid-reuse-')); + const key = 'shared-listener'; + const digest = createHash('sha256').update(key).digest('hex'); + const path = join(root, `${digest}.lock.json`); + const current = createBackendLease({ runId: 'current', backend: 'stub', track: 'loop', runIndex: 1 }); + try { + for (const owner of [ + { ownerPid: process.pid, ownerStartMarker: 'not-this-process' }, + { ownerPid: 2_147_483_646, ownerStartMarker: null }, + ]) { + writeFileSync(path, JSON.stringify({ version: 1, key, runId: 'dead-owner', ...owner, + ownershipMarkerSha256: 'dead-owner', acquiredAt: new Date().toISOString() })); + const before = readFileSync(path, 'utf8'); + assert.throws(() => acquireResourceLocks({ root, keys: [key], lease: current }), /run authenticated recovery before reuse/); + assert.equal(readFileSync(path, 'utf8'), before); + } + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('resource-free stub activation stays resource-free inside the appliance', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-stub-activation-')); + const previous = process.env.STACK_BENCH_APPLIANCE; + try { + process.env.STACK_BENCH_APPLIANCE = '1'; + const lease = createBackendLease({ runId: 'stub-appliance', backend: 'stub', track: 'loop', runIndex: 0 }); + const path = join(root, 'lease.json'); + writeBackendLease(path, lease); + activateHosted({ leasePath: path, leaseToken: lease.ownershipToken, lease, + ports: { vite: 1, express: null, dbPort: null } }); + const activated = readBackendLease(path, { token: lease.ownershipToken, active: true }); + assert.equal(activated.resources.container, null); + assert.equal(activated.resources.network, undefined); + assert.deepEqual(activated.resources.locks, []); + } finally { + if (previous === undefined) delete process.env.STACK_BENCH_APPLIANCE; + else process.env.STACK_BENCH_APPLIANCE = previous; + rmSync(root, { recursive: true, force: true }); + } +}); + +test('a supervised child waiting for host capacity does not spend its timeout', async () => { + const root = mkdtempSync(join(tmpdir(), 'capacity-wait-')); + const receipt = join(root, 'capacity-wait.json'); + const lease = new URL('../src/runtime/backend-lease.js', import.meta.url).href; + const child = `import { writeCapacityWait } from ${JSON.stringify(lease)}; + const startedAt = Date.now(); + writeCapacityWait({ startedAt, resumedAt: null }); + await new Promise(resolve => setTimeout(resolve, 4000)); + writeCapacityWait({ startedAt, resumedAt: Date.now() }); + await new Promise(resolve => setTimeout(resolve, 300));`; + try { + const result = await runBounded(process.execPath, ['--input-type=module', '-e', child], { + stdio: 'ignore', timeoutMs: 2500, env: { ...process.env, [CAPACITY_WAIT_RECEIPT_ENV]: receipt }, + pauseIntervals: () => { const wait = readCapacityWait(receipt); return wait ? [wait] : []; }, terminate: pid => process.kill(pid, 'SIGKILL'), + }); + assert.equal(result.error, null); + assert.equal(result.timedOut, false); + assert.equal(result.ok, true); + assert(result.pausedMs! >= 3900, String(result.pausedMs)); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +function platformLease(root: string) { + const anchor = 'a'.repeat(64); + const lease = createBackendLease({ runId: 'supabase-services', backend: 'supabase', track: 'ecommerce', runIndex: 0, + serverUri: 'http://127.0.0.1:13410', database: 'postgres' }); + lease.state = 'active'; + lease.resources.container = { name: 'sb-anchor-backend', id: anchor, image: `sha256:${'d'.repeat(64)}`, + owned: true, networkMode: 'b'.repeat(64) }; + lease.resources.network = { name: 'sb-anchor-network', id: 'b'.repeat(64), namespaceContainerId: anchor, + hostAddresses: ['172.20.0.1'], services: [], firewallSha256: null, firewallInstalledAt: null }; + const service = (role: string, digit: string) => ({ name: `sb-anchor-service-${role}`, id: digit.repeat(64), + image: `sha256:${'e'.repeat(64)}`, owned: true, networkMode: `container:${anchor}` }); + lease.resources.serviceContainers = { gateway: service('gateway', '1'), auth: service('auth', '2') }; + lease.resources.browserContainer = { ...service('browser', '3'), name: 'sb-anchor-browser' }; + const intent = (name: string, digit: string) => ({ name, creationToken: digit.repeat(32) }); + lease.resources.creationIntents = { network: intent('sb-anchor-network', '4'), backend: intent('sb-anchor-backend', '5'), + 'service-auth': intent('sb-anchor-service-auth', '6'), 'service-gateway': intent('sb-anchor-service-gateway', '7'), + 'service-realtime': intent('sb-anchor-service-realtime', '8'), browser: intent('sb-anchor-browser', '9') }; + const path = join(root, 'lease.json'); + writeBackendLease(path, lease); + return { path, lease, anchor }; +} + +test('platform service containers are owned, exact, and inside the anchor namespace', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-services-')); + try { + const { path, lease, anchor } = platformLease(root); + assert.equal(readBackendLease(path).resources.serviceContainers?.auth?.networkMode, `container:${anchor}`); + assert.equal(publicBackendLease(lease).resources.serviceContainers?.gateway?.name, 'sb-anchor-service-gateway'); + const invalid = (change: (next: typeof lease) => void, message: RegExp) => { + const next = structuredClone(lease); + change(next); + assert.throws(() => writeBackendLease(join(root, 'invalid.json'), next), message); + }; + invalid(next => { next.resources.serviceContainers!.auth!.networkMode = `container:${'c'.repeat(64)}`; }, + /serviceContainers.auth is outside the leased network namespace/); + invalid(next => { next.resources.serviceContainers!.auth!.owned = false; }, /serviceContainers.auth must identify an owned/); + invalid(next => { next.resources.serviceContainers!.auth!.image = 'supabase/gotrue:latest'; }, /must identify an owned/); + invalid(next => { next.resources.serviceContainers = { 'Auth-1': next.resources.serviceContainers!.auth! }; }, + /map service roles/); + invalid(next => { (next.resources.creationIntents as Record)['service-'] = { name: 'x', creationToken: '1'.repeat(32) }; }, + /creation intent is invalid/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('teardown removes platform services before their namespace anchor, including unrecorded creations', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-services-')); + try { + const { path, lease, anchor } = platformLease(root); + const removed: string[] = []; + const present = new Set([...Object.values(lease.resources.serviceContainers!).map(container => container.id), + lease.resources.browserContainer!.id, anchor]); + const names = new Map([...Object.values(lease.resources.serviceContainers!), lease.resources.browserContainer!, + lease.resources.container!].map(container => [container.name, container.id])); + const unrecorded = 'f'.repeat(64); + const attempt: string[][] = []; + assert.equal(releaseBackendLease(path, lease.ownershipToken, { + hostTeardown: () => true, + docker: { + inspect: name => { + const id = names.get(name); + if (!id || !present.has(id)) throw new Error(`No such object: ${name}`); + return id; + }, + handoffWorkspace: () => assert.fail('no build container'), + remove: id => { removed.push(id); present.delete(id); }, + wait: () => undefined, + }, + attempt: args => { + attempt.push(args); + const name = args.at(-1)!; + if (args[0] === 'network' && args[1] === 'inspect') { + return JSON.stringify([{ Id: lease.resources.network!.id, Labels: { [ATTEMPT_CREATION_LABEL]: '4'.repeat(32) }, Containers: {} }]); + } + if (args[0] === 'container' && name === 'sb-anchor-service-realtime') { + return JSON.stringify([{ Id: unrecorded, Config: { Labels: { [ATTEMPT_CREATION_LABEL]: '8'.repeat(32) } } }]); + } + if (args[0] === 'container') throw new Error(`No such container: ${name}`); + return ''; + }, + }), true); + assert.deepEqual(removed, [lease.resources.browserContainer!.id, lease.resources.serviceContainers!.auth!.id, + lease.resources.serviceContainers!.gateway!.id, anchor]); + const order = attempt.filter(args => args[0] === 'container').map(args => args.at(-1)); + assert.deepEqual(order, ['sb-anchor-browser', 'sb-anchor-service-auth', 'sb-anchor-service-gateway', + 'sb-anchor-service-realtime', 'sb-anchor-backend']); + assert.deepEqual(attempt.filter(args => args[0] === 'rm'), [['rm', '-f', '--volumes', unrecorded]]); + assert.deepEqual(attempt.at(-1), ['network', 'rm', lease.resources.network!.id]); + const released = readBackendLease(path); + assert.equal(released.state, 'released'); + assert(Object.values(released.resources.serviceContainers!).every(container => container.removedAt && !container.running)); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('grading slots go to waiters in arrival order, skip a dead waiter, and report each wait', { skip: process.platform !== 'linux' ? 'Kernel flock requires the Linux appliance' : false }, async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-grading-slot-')); + const run = (runId: string, runIndex: number) => { + const lease = createBackendLease({ runId, backend: 'convex', track: 'ecommerce', runIndex, + serverUri: `http://127.0.0.1:${14310 + runIndex}` }); + const path = join(root, `${runId}.json`); + writeBackendLease(path, lease); + return { lease, path, receipt: join(root, `${runId}-waits.json`), env: { STACK_BENCH_RESOURCE_LOCK_DIR: root, + STACK_BENCH_GRADING_CAPACITY: '1', STACK_BENCH_LEASE: path, STACK_BENCH_LEASE_TOKEN: lease.ownershipToken, + [GRADING_WAIT_RECEIPT_ENV]: join(root, `${runId}-waits.json`) } }; + }; + const tick = () => delay(10); + try { + const holder = run('holder', 0), first = run('first', 1), second = run('second', 2); + holder.lease.resources.locks.push(...acquireResourceLocks({ root, keys: ['grading:0'], lease: holder.lease })); + // A waiter that died before these leaves an old ticket at the head of the queue. + mkdirSync(join(root, 'grading-queue')); + writeFileSync(join(root, 'grading-queue', '000000000000001-dead'), ''); + utimesSync(join(root, 'grading-queue', '000000000000001-dead'), new Date(0), new Date(0)); + const order: string[] = []; + const firstSlot = acquireGradingSlot(first.env, tick).then(slot => { order.push('first'); return slot; }); + while (!readGradingWaits(first.receipt).length) await tick(); + const secondSlot = acquireGradingSlot(second.env, tick).then(slot => { order.push('second'); return slot; }); + while (!readGradingWaits(second.receipt).length) await tick(); + releaseResourceLocks(holder.lease); + const granted = await firstSlot; + await delay(100); + assert.deepEqual(order, ['first'], 'the earlier waiter is served first'); + assert.deepEqual(readBackendLease(first.path).resources.locks.map(lock => lock.key), ['grading:0']); + granted.release(); + granted.release(); + (await secondSlot).release(); + assert.deepEqual(order, ['first', 'second']); + for (const waiter of [first, second]) { + assert.deepEqual(readBackendLease(waiter.path).resources.locks, []); + const [wait] = readGradingWaits(waiter.receipt); + assert.equal(recordedGradingWaitMs(waiter.receipt), wait!.resumedAt! - wait!.startedAt); + } + assert.deepEqual(readdirSync(join(root, 'grading-queue')), []); + const free = await acquireGradingSlot(first.env, () => assert.fail('a free slot never waits')); + free.release(); + assert.equal(readGradingWaits(first.receipt).length, 1); + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/tools/stack-bench/tests/bench-policy.test.ts b/tools/stack-bench/tests/bench-policy.test.ts new file mode 100644 index 00000000000..21e64d07860 --- /dev/null +++ b/tools/stack-bench/tests/bench-policy.test.ts @@ -0,0 +1,604 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { pendingRunSnapshot, auditFailureSummary, gradeArgv, parseAgentProcessResult } + from '../commands/bench.js'; +import { finalizeRunTotals } + from '../src/evidence/benchmark-run.js'; +import { formatLevelSummary } from '../src/evidence/evidence-presentation.js'; +import type { BenchmarkRunRecord, RunLevelRecord, RunSessionRecord, RunTotalsInput } from '../src/evidence/benchmark-run.js'; +import { parseBenchArguments } from '../commands/bench-arguments.js'; +import { pristineMutationBaselinePath } from '../src/evidence/mutation-control.js'; + +test('new direct runs default to production framing with an explicit opt-out', () => { + const argv = ['node', 'bench', '--backend', 'postgres']; + assert.equal(parseBenchArguments(argv).productionQuality, true); + assert.equal(parseBenchArguments([...argv, '--no-production-quality']).productionQuality, false); +}); +import { clearPrivateGradingEvidence, privateGradingDirectory, levelGradeIsUsable, repairEvidenceDecision, + repairHistoryEntry, repairProgressState, repairRegressionDecision, + restorePrivateGradingEvidence } + from '../src/evidence/repair-evidence.js'; +import { finalPackageEvidenceRequired, preserveFinalPackageEvidence, sourceBoundFirstBuildOutcome } + from '../src/runtime/source-checkpoint.js'; +import { materializationAppFailure, materializeAcceptedSource, restoreRepairSource } + from '../src/runtime/source-materialization.js'; +import { dependencyLevelRepairRecords, dependencyRepairBudget, dependencyRepairRecords } + from '../src/progression/dependency-mode.js'; +import { loadTrack, RUN_INDEX_CAP } from '../src/composition/tracks.js'; +import { writeArtifact } from '../src/evidence/artifacts.js'; +import { createCheckEvidence } from '../src/evidence/check-evidence.js'; +import { hashAppSource } from '../src/runtime/source-snapshot.js'; +import type { GradeBundlePayload } from '../src/evidence/benchmark-run.js'; +import { compiledEntrypoint } from '../src/package-root.js'; +import { agentSessionFailure } from '../src/agents/agent-result-contract.js'; + +test('billable agent runs require the Docker appliance', () => { + const env = { ...process.env }; + delete env.STACK_BENCH_APPLIANCE; + const result = spawnSync(process.execPath, + [compiledEntrypoint('commands', 'bench.js'), '--backend', 'stub'], + { encoding: 'utf8', env }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /agent adapter claude-code requires the Docker appliance/); +}); + +test('a nonzero agent process preserves its valid provider-failure result', () => { + const request = { app: '/app', mode: 'build' as const, level: 1, backend: 'spacetime', + track: 'ecommerce', runIndex: 0, model: 'provider-model', guidance: 'neutral', + adapterCostLimit: 'non-billable' as const }; + const raw = { + appDir: '/app', mode: 'build', level: 1, ok: false, sessionId: 'session-1', + costUsd: 0, tokens: 10, outputTokens: 2, turns: 1, promptBytes: 20, durationMs: 100, + setup: {}, usage: { input: 8, output: 2, cacheWrite: 0, cacheRead: 0 }, costReceipts: [], + providerMetadata: { failureCode: 'provider-usage-receipt-missing' }, + }; + const result = parseAgentProcessResult(`${JSON.stringify(raw)}\n`, '', new Error('exit code 3'), + request); + assert.equal(result.ok, false); + assert.equal(agentSessionFailure(result)?.kind, 'provider_failure'); + assert.throws(() => parseAgentProcessResult( + `${JSON.stringify({ ...raw, ok: true })}\n`, '', new Error('exit code 3'), request), + /failed after reporting success/); +}); + +test('accepted source is materialized through the application lifecycle before grading', async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-materialized-source-')); + try { + const source = join(root, 'source'); + const app = join(root, 'app'); + mkdirSync(join(app, 'node_modules'), { recursive: true }); + mkdirSync(source, { recursive: true }); + writeFileSync(join(source, 'index.js'), 'export const value = "accepted";\n'); + writeFileSync(join(source, 'start.sh'), '#!/bin/sh\n'); + writeFileSync(join(app, 'index.js'), 'export const value = "coding runtime";\n'); + writeFileSync(join(app, '.server.pid'), '100\n'); + writeFileSync(join(app, 'node_modules', 'state'), 'stale\n'); + const modes: string[] = []; + const application = { backend: 'postgres', app, port: 6573, probe: '' }; + await materializeAcceptedSource(source, app, application, + async (_spec, mode = 'restart') => { + modes.push(mode); + if (mode === 'stop' && modes.length === 1) { + assert.match(readFileSync(join(app, 'index.js'), 'utf8'), /coding runtime/); + } + if (mode === 'start') { + assert.match(readFileSync(join(app, 'index.js'), 'utf8'), /accepted/); + assert.equal(existsSync(join(app, 'node_modules')), false); + writeFileSync(join(app, '.server.pid'), '200\n'); + } + }); + assert.deepEqual(modes, ['stop', 'start']); + assert.equal(readFileSync(join(app, '.server.pid'), 'utf8'), '200\n'); + assert.equal(hashAppSource(app).sha256, hashAppSource(source).sha256); + await assert.rejects(materializeAcceptedSource(source, app, application, async (_spec, mode) => { + if (mode === 'start') writeFileSync(join(app, 'index.js'), 'changed during start\n'); + }), /differs from its accepted snapshot/); + assert.match(readFileSync(join(app, 'index.js'), 'utf8'), /accepted/); + rmSync(join(source, 'start.sh')); + const missingContractModes: string[] = []; + let missingContract: unknown = null; + try { + await materializeAcceptedSource(source, app, application, async (_spec, mode) => { + missingContractModes.push(mode ?? 'restart'); + }); + } catch (error) { + missingContract = error; + } + assert.deepEqual(missingContractModes, ['stop']); + assert.equal(materializationAppFailure(missingContract)?.kind, 'app_failure'); + + writeFileSync(join(source, 'start.sh'), '#!/bin/sh\n'); + let cleanupStops = 0; + await assert.rejects(materializeAcceptedSource(source, app, application, async (_spec, mode) => { + if (mode === 'start') writeFileSync(join(app, 'index.js'), 'changed during start\n'); + if (mode === 'stop' && ++cleanupStops === 2) throw new Error('stop failed'); + }), /could not stop and restore/); + assert.match(readFileSync(join(app, 'index.js'), 'utf8'), /accepted/); + + const failedStartModes: string[] = []; + let failedStart: unknown = null; + try { + await materializeAcceptedSource(source, app, application, async (_spec, mode) => { + failedStartModes.push(mode ?? 'restart'); + if (mode === 'start') { + mkdirSync(join(app, 'dist')); + throw Object.assign(new Error( + 'npm install failed for DATABASE_URL=mongodb://user:secret@database:27017/app'), + { code: 'generated_app_not_restartable' }); + } + }); + } catch (error) { + failedStart = error; + } + assert.deepEqual(failedStartModes, ['stop', 'start', 'stop']); + assert.equal(existsSync(join(app, 'node_modules')), false); + assert.equal(existsSync(join(app, 'dist')), false); + const failedStartOutcome = materializationAppFailure(failedStart); + assert.equal(failedStartOutcome?.kind, 'app_failure'); + assert.match(failedStartOutcome?.reason ?? '', /npm install failed/); + assert.doesNotMatch(failedStartOutcome?.reason ?? '', /secret/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('repair rollback restores schema before startup and stops on reset failure for every stack', async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-repair-schema-')); + try { + const source = join(root, 'accepted'); + mkdirSync(source); + writeFileSync(join(source, 'schema.txt'), 'accepted'); + writeFileSync(join(source, 'start.sh'), '#!/bin/sh\n'); + for (const backend of ['spacetime', 'postgres', 'mongodb']) { + const app = join(root, backend); + mkdirSync(app); + writeFileSync(join(app, 'schema.txt'), 'rejected'); + const application = { backend, app, port: 6573, probe: '' }; + let databaseSchema = 'rejected'; + let stopped = false; + const events: string[] = []; + const lifecycle: Parameters[3] = async (_spec, mode) => { + events.push(mode ?? 'restart'); + if (mode === 'stop') stopped = true; + if (mode === 'start') { + assert.equal(stopped, true); + assert.equal(readFileSync(join(app, 'schema.txt'), 'utf8'), 'accepted'); + if (databaseSchema !== 'accepted') throw new Error('schema migration required'); + stopped = false; + } + }; + await assert.rejects(materializeAcceptedSource(source, app, application, lifecycle), + /schema migration required/); + events.length = 0; + await restoreRepairSource(source, app, application, lifecycle, request => { + assert.deepEqual(request, { backend, app }); + assert.equal(stopped, true); + assert.equal(readFileSync(join(app, 'schema.txt'), 'utf8'), 'accepted'); + events.push('reset'); + databaseSchema = 'accepted'; + }); + assert.deepEqual(events, ['stop', 'reset', 'start']); + assert.equal(hashAppSource(app).sha256, hashAppSource(source).sha256); + events.length = 0; + await assert.rejects(restoreRepairSource(source, app, application, lifecycle, () => { + events.push('reset'); + throw new Error('database reset failed'); + }), /database reset failed/); + assert.deepEqual(events, ['stop', 'reset', 'stop']); + assert.equal(stopped, true); + assert.equal(hashAppSource(app).sha256, hashAppSource(source).sha256); + } + } finally { rmSync(root, { recursive: true, force: true }); } +}); +test('final package preservation verifies both source and grading before success', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-final-package-')); + try { + const app = join(root, 'app'); + const output = join(root, 'output'); + mkdirSync(app, { recursive: true }); + mkdirSync(privateGradingDirectory(app), { recursive: true }); + mkdirSync(output, { recursive: true }); + writeFileSync(join(app, 'index.js'), 'export const ready = true;\n'); + const source = hashAppSource(app); + writeArtifact(join(privateGradingDirectory(app), 'bundle.json'), { + kind: 'grade_bundle', id: 'final-grade', payload: { + observation: 'scored', source: { sha256: source.sha256 }, + suites: {}, totals: { score: 1, max: 1 }, + selection: { sha256: 'a'.repeat(64) }, + }, + }); + + const evidence = preserveFinalPackageEvidence({ appDir: app, outputDir: output }); + assert.equal(evidence.source.sha256, source.sha256); + assert.equal(existsSync(join(output, 'grading', 'bundle.json')), true); + + rmSync(join(privateGradingDirectory(app), 'bundle.json')); + assert.throws(() => preserveFinalPackageEvidence({ appDir: app, outputDir: output }), + /mandatory result package evidence.*final grader produced no bundle/); + assert.equal(existsSync(join(output, 'source', 'index.js')), true, + 'source evidence remains available when grading preservation fails'); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('hostile app-local grade output cannot replace accepted private evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-private-grade-')); + try { + const app = join(root, 'app'); + const output = join(root, 'result'); + mkdirSync(app); + writeFileSync(join(app, 'index.js'), 'export {};'); + const source = hashAppSource(app); + const grading = privateGradingDirectory(app); + mkdirSync(grading); + writeArtifact(join(grading, 'bundle.json'), { + kind: 'grade_bundle', id: 'trusted-grade', payload: { + source: { sha256: source.sha256 }, totals: { score: 0, max: 1 }, + }, + }); + const attacker = spawnSync(process.execPath, ['-e', ` + const fs = require('node:fs'); + fs.mkdirSync('stack-bench', { recursive: true }); + fs.writeFileSync('stack-bench/bundle.json', JSON.stringify({ totals: { score: 1, max: 1 } })); + `], { cwd: app, encoding: 'utf8' }); + assert.equal(attacker.status, 0, attacker.stderr); + preserveFinalPackageEvidence({ appDir: app, outputDir: output }); + assert.deepEqual(readFileSync(join(output, 'grading', 'bundle.json')), + readFileSync(join(grading, 'bundle.json'))); + const argv = gradeArgv({ backend: 'postgres', track: 'ecommerce', runIndex: 0, + media: false }, app, 'http://localhost:3000', 'isolation', 1, + loadTrack('ecommerce'), 'test'); + assert.equal(argv[argv.indexOf('--out') + 1], grading); + assert.throws(() => privateGradingDirectory(app, join(app, 'stack-bench')), /outside/); + const alias = join(root, 'alias'); + symlinkSync(app, alias, 'junction'); + assert.throws(() => privateGradingDirectory(app, join(alias, 'new-grades')), /outside/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('an interrupted run keeps its root failure even after an earlier grade', () => { + const levels = [{ graded: true }, { graded: false }]; + assert.equal(finalPackageEvidenceRequired({ kind: 'provider_failure' }, levels), false); + assert.equal(finalPackageEvidenceRequired({ kind: 'harness_failure' }, levels), false); + assert.equal(finalPackageEvidenceRequired({ kind: 'app_failure' }, levels), true); +}); + +test('a missing first-build source is a harness failure', () => { + const passed = { outcome: { kind: 'passed' }, totals: { score: 1, max: 1 }, suites: {} }; + assert.equal(sourceBoundFirstBuildOutcome(passed, { sha256: 'a'.repeat(64) }).kind, 'passed'); + assert.deepEqual(sourceBoundFirstBuildOutcome(passed, null), { + kind: 'harness_failure', + phase: 'first-build-source', + reason: 'the first-build source could not be preserved and verified', + appFailures: [], + inconclusive: [], + harnessFailures: ['the first-build source could not be preserved and verified'], + }); +}); + +test('direct runs default to ten repairs while an explicit budget still wins', () => { + assert.equal(parseBenchArguments(['node', 'bench', '--backend', 'postgres']).repairs, 10); + assert.equal(parseBenchArguments(['node', 'bench', '--backend', 'postgres', + '--repairs', '4']).repairs, 4); +}); + +test('bench arguments reject partial and out-of-range run indexes', () => { + assert.throws(() => parseBenchArguments(['node', 'bench', '--backend', 'postgres', + '--run-index', '1junk']), /--run-index must be an integer/); + assert.throws(() => parseBenchArguments(['node', 'bench', '--backend', 'postgres', + '--run-index', String(RUN_INDEX_CAP + 1)]), /--run-index must be an integer from 0 through/); + assert.throws(() => parseBenchArguments(['node', 'bench', '--backend', 'postgres', + '--pricing-json', '{}']), /--pricing-json/); + const sharded = parseBenchArguments(['node', 'bench', '--backend', 'postgres', + '--mutation-shard-index', '1', '--mutation-shard-count', '3']); + assert.equal(sharded.mutationShardIndex, 1); + assert.equal(sharded.mutationShardCount, 3); + assert.throws(() => parseBenchArguments(['node', 'bench', '--backend', 'postgres', + '--mutation-shard-index', '1']), /must be supplied together/); +}); + +test('progression level usability follows its stricter evidence result', () => { + assert.equal(levelGradeIsUsable({ kind: 'app_failure' }), true); + assert.equal(levelGradeIsUsable({ kind: 'app_failure', inconclusive: ['feature/check'] }), false); + assert.equal(levelGradeIsUsable( + { kind: 'app_failure', inconclusive: ['unrelated/check'] }, { outcome: 'conclusive' }), true); + assert.equal(levelGradeIsUsable({ kind: 'inconclusive' }), false); + assert.equal(levelGradeIsUsable({ kind: 'provider_failure' }), false); + assert.equal(levelGradeIsUsable({ kind: 'app_failure' }, { outcome: 'inconclusive' }), false); + assert.equal(levelGradeIsUsable({ kind: 'app_failure' }, { outcome: 'conclusive' }), true); +}); + +test('dependency repair accounting uses repairs, not grading observations', () => { + const action = { type: 'repair', repair: { remaining: 3 } }; + assert.equal(dependencyRepairBudget(action, 0), 3); + assert.equal(dependencyRepairBudget(action, 1), 4); + assert.throws(() => dependencyRepairBudget({ type: 'build', repair: { remaining: null } }, 0), + /valid repair action/); + + const state = { + definition: { nodes: [ + { id: 'accounts', level: 1 }, + { id: 'catalog', level: 1 }, + { id: 'recovery', level: 2 }, + ] }, + nodes: { + accounts: { repairs: { used: 1 }, + exhaustedAtLevel: null, exhaustionReason: null }, + catalog: { repairs: { used: 5 }, + exhaustedAtLevel: 1, exhaustionReason: 'feature-repairs-exhausted' }, + recovery: { repairs: { used: 1 }, + exhaustedAtLevel: null, exhaustionReason: null }, + }, + attempts: [ + { level: 1, repair: { depth: 1, nodeIds: ['accounts', 'recovery'] } }, + { level: 2, repair: { depth: 2, nodeIds: ['recovery'] } }, + { level: 2, repair: null }, + ], + }; + assert.deepEqual(dependencyRepairRecords(state, 1, ['recovery']), [ + { nodeId: 'accounts', used: 1, exhaustionReason: null }, + { nodeId: 'catalog', used: 5, exhaustionReason: 'feature-repairs-exhausted' }, + { nodeId: 'recovery', used: 1, exhaustionReason: null }, + ]); + // The level record covers the level's own nodes plus the nodes a repair at + // that depth touched, and nothing else. + assert.deepEqual(dependencyLevelRepairRecords(state, 1), [ + { nodeId: 'accounts', used: 1, exhaustionReason: null }, + { nodeId: 'catalog', used: 5, exhaustionReason: 'feature-repairs-exhausted' }, + { nodeId: 'recovery', used: 1, exhaustionReason: null }, + ]); + assert.deepEqual(dependencyLevelRepairRecords(state, 2), [ + { nodeId: 'recovery', used: 1, exhaustionReason: null }, + ]); +}); + +test('resumed dependency costs separate prior, current, and cumulative execution usage', () => { + const run: RunTotalsInput = { levels: [ + { level: 1, graded: true, score: 1, max: 1, buildCostUsd: 4, + sessionTotals: { sessions: 1, tokens: 10, outputTokens: 2, turns: 1, durationMs: 100 } }, + { level: 2, graded: true, score: 1, max: 1, repairCostUsd: 2, + sessionTotals: { sessions: 1, tokens: 5, outputTokens: 1, turns: 1, durationMs: 50 } }, + ], progressionResume: { inheritedLevels: [1], + priorTotals: { costUsd: 4, costComplete: true } } }; + const totals = finalizeRunTotals(run, 1_000, { now: 3_000 }); + assert.equal(totals.priorExecutionCostUsd, 4); + assert.equal(totals.currentExecutionCostUsd, 2); + assert.equal(totals.cumulativeCostUsd, 6); + assert.equal(totals.costUsd, 6); + assert.equal(totals.costComplete, true); + assert.equal(totals.currentExecutionCostComplete, true); + assert.equal(finalizeRunTotals(run, 1_000, { now: 3_000, costComplete: false }).currentExecutionCostComplete, false); +}); + +test('audit failures retain the exit code and stderr needed for diagnosis', () => { + const error = Object.assign(new Error('Command failed: leak audit'), { + status: 7, + stderr: "node:fs:441\nError: EACCES: permission denied, open '/run/transcript.jsonl'", + }); + assert.equal(auditFailureSummary(error), + "Command failed: leak audit (exit 7; stderr: Error: EACCES: permission denied, open '/run/transcript.jsonl')"); +}); + +test('repair preparation removes raw grading evidence but keeps the app and bug report', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-repair-isolation-')); + try { + mkdirSync(privateGradingDirectory(root), { recursive: true }); + writeFileSync(join(privateGradingDirectory(root), 'bundle.json'), '{"private":true}\n'); + writeFileSync(join(root, 'BUG_REPORT.md'), '# Behaviour only\n'); + writeFileSync(join(root, 'app.js'), 'export {};\n'); + clearPrivateGradingEvidence(root); + assert.equal(existsSync(privateGradingDirectory(root)), false); + assert.equal(existsSync(join(root, 'BUG_REPORT.md')), true); + assert.equal(existsSync(join(root, 'app.js')), true); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('repair progress pauses only on repeated findings without a score gain', () => { + const bundle = (score: number, failures: string[]): GradeBundlePayload => ({ + totals: { score, max: 10, contractPass: true }, + suites: {}, outcome: { kind: 'app_failure', phase: 'grading', + appFailures: failures, inconclusive: [], harnessFailures: [] } }); + let state = repairProgressState(null, bundle(5, ['a'])); + state = repairProgressState(state, bundle(5, ['a'])); + assert.equal(state.stalledRounds, 1); + state = repairProgressState(state, bundle(5, ['b'])); + assert.equal(state.stalledRounds, 0); + state = repairProgressState(state, bundle(6, ['b'])); + assert.equal(state.stalledRounds, 0); + state = repairProgressState(state, bundle(6, ['b'])); + assert.equal(state.stalledRounds, 1); +}); + +test('repair history reports score movement and exact remaining findings', () => { + const before = { totals: { score: 4, max: 6 }, suites: {}, outcome: { + kind: 'app_failure', phase: 'grading', appFailures: ['suite/a'] } }; + const after = { totals: { score: 5, max: 6 }, suites: { lint: { results: [ + { id: 'review-average', status: 'FAIL' }, + ] } }, outcome: { kind: 'app_failure', phase: 'grading', + appFailures: ['contract-lint', 'suite/b'] } }; + assert.deepEqual(repairHistoryEntry(2, before, after, 'kept'), { + round: 2, beforeScore: 4, beforeMax: 6, afterScore: 5, afterMax: 6, + result: 'kept', remainingFailures: ['suite/b', 'testing-interface/review-average'], + }); +}); + +test('mutation-only execution is restricted to model-free reference runs', () => { + assert.throws(() => parseBenchArguments(['node', 'bench', '--backend', 'postgres', + '--reference-mutation-only']), /requires a mutation-bound reference fixture/); + const args = parseBenchArguments(['node', 'bench', '--backend', 'postgres', '--repairs', '0', + '--agent-adapter', 'reference-fixture', '--app', 'fixture', '--mutations', 'mutations.json', + '--reference-mutation-only', '--mutation-baseline-bundle', 'baseline.json']); + assert.equal(args.referenceMutationOnly, true); + assert(args.mutationBaselineBundle); + assert.match(args.mutationBaselineBundle, /baseline\.json$/); + assert.throws(() => parseBenchArguments(['node', 'bench', '--backend', 'postgres', + '--mutation-baseline-bundle', 'baseline.json']), /require --mutations/); + assert.throws(() => parseBenchArguments(['node', 'bench', '--backend', 'postgres', + '--mutations', 'mutations.json', '--mutation-baseline-bundle', 'baseline.json']), + /internal reference mutation option/); +}); + +test('mutation control reuses the existing clean grade when it is present', () => { + const args = { out: 'results/run', levelList: [1, 3] }; + assert.equal(pristineMutationBaselinePath(args, () => true), + join('results/run', 'first-build-l3-grading', 'bundle.json')); + assert.equal(pristineMutationBaselinePath(args, () => false), null); + assert.equal(pristineMutationBaselinePath({ ...args, referenceMutationOnly: true }), null); +}); + +test('the first repair that makes an unstartable app gradeable is never rolled back', () => { + const after = { suites: {}, totals: { score: 35, max: 58 } }; + for (const phase of ['application-restart', 'application-readiness']) { + const before = { outcome: { kind: 'app_failure', phase }, + suites: {}, totals: { score: 0, max: 58 } }; + assert.equal(repairEvidenceDecision(before, after).action, 'keep-setup-repair'); + } + assert.equal(repairEvidenceDecision({ suites: {} }, after).action, 'rollback-no-comparison'); +}); + +test('repair comparison respects declared scope without hiding missing or regressed evidence', () => { + const criterion = (id: string, status: 'passed' | 'failed', points = 1) => ({ + id, stableKey: `check.${id}`, points, + evidence: createCheckEvidence({ status, code: 'test_result', phase: 'assertion', + startedAtMs: 1, completedAtMs: 2 }), + }); + const bundle = (criteria: ReturnType[]) => ({ + suites: { features: { features: [{ id: 'work', criteria }] } }, + }); + const before = bundle([criterion('retained', 'passed'), criterion('repair', 'failed'), + criterion('outside', 'passed')]); + const after = { ...bundle([criterion('retained', 'passed'), criterion('repair', 'passed')]), + selection: { checks: [{ stableKey: 'check.retained' }, { stableKey: 'check.repair' }] } }; + for (const decide of [repairEvidenceDecision, repairRegressionDecision]) { + assert.equal(decide(before, after).action, 'keep'); + assert.equal(decide(before, { ...after, + ...bundle([criterion('repair', 'passed')]) }).action, 'rollback-regression'); + assert.equal(decide(before, { ...after, + ...bundle([criterion('retained', 'failed'), criterion('repair', 'passed')]) + }).action, 'rollback-regression'); + assert.equal(decide(before, { ...after, selection: undefined }).action, 'rollback-regression'); + } + const traded = repairEvidenceDecision(bundle([criterion('a', 'passed', 2), criterion('b', 'failed', 3)]), + bundle([criterion('a', 'failed', 2), criterion('b', 'passed', 3)])); + assert.equal(traded.action, 'rollback-regression'); + assert.deepEqual(traded.shared.regressions, ['check.a']); + const earlier = bundle([criterion('pass', 'passed'), criterion('fail', 'failed')]); + assert.equal(repairRegressionDecision(earlier, bundle([criterion('pass', 'passed')])).action, 'keep'); + assert.equal(repairRegressionDecision(earlier, bundle([criterion('pass', 'failed')])).action, + 'rollback-regression'); +}); + +test('repair rollback restores the accepted grading evidence without another grade', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-repair-grade-rollback-')); + try { + const app = join(root, 'app'); + const snapshot = join(root, 'snapshot'); + mkdirSync(app, { recursive: true }); + mkdirSync(privateGradingDirectory(app), { recursive: true }); + mkdirSync(snapshot, { recursive: true }); + writeFileSync(join(privateGradingDirectory(app), 'bundle.json'), 'new grade\n'); + writeFileSync(join(snapshot, 'bundle.json'), 'accepted grade\n'); + + restorePrivateGradingEvidence(app, snapshot); + + assert.equal(readFileSync(join(privateGradingDirectory(app), 'bundle.json'), 'utf8'), + 'accepted grade\n'); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('reference qualification does not recover inconclusive suites, while paid grading can', () => { + const track = loadTrack('ecommerce'); + for (const backend of ['spacetime', 'postgres', 'mongodb', 'convex']) { + for (const agentAdapter of ['reference-fixture', 'codex']) { + const args = { backend, agentAdapter, track: 'ecommerce', runIndex: 0, media: false }; + const argv = gradeArgv(args, '/app', 'http://localhost:6573', 'qualification', 1, + track, 'attempt', { sourceSha256: 'a'.repeat(64) }); + assert.equal(argv.includes('--retry-inconclusive'), agentAdapter === 'codex', + `${backend}/${agentAdapter}: qualification must preserve the first inconclusive result`); + } + } +}); + +test('grading binds scored evidence to the selected application source', () => { + const track = loadTrack('ecommerce'); + const sourceSha256 = 'a'.repeat(64); + const argv = gradeArgv({ backend: 'spacetime', track: 'ecommerce', runIndex: 0, + media: false }, '/app', 'http://localhost:6481', 'spacetime-l1', 1, track, 'attempt', + { sourceSha256 }); + const index = argv.indexOf('--source-sha256'); + assert(index > 0); + assert.equal(argv[index + 1], sourceSha256); +}); + +test('grading records a materialization defect through the complete grading path', () => { + const track = loadTrack('ecommerce'); + const applicationFailure = { kind: 'app_failure', phase: 'application-restart', + reason: 'application startup changed the accepted source', + appFailures: ['application-restart'], inconclusive: [], harnessFailures: [] } as const; + const argv = gradeArgv({ backend: 'postgres', track: 'ecommerce', runIndex: 0, + media: false }, '/app', 'http://localhost:6573', 'postgres-l1', 1, track, 'attempt', + { applicationFailure }); + const index = argv.indexOf('--application-failure-json'); + assert(index > 0); + assert.deepEqual(JSON.parse(argv[index + 1] ?? ''), applicationFailure); +}); + +test('later-level grading receives prior selected checks as regression scope', () => { + const track = loadTrack('ecommerce'); + const args = { backend: 'postgres', track: 'ecommerce', runIndex: 0, media: false, + recipeTasks: new Map([ + [1, { request: { schemaVersion: 3 }, selection: { scoredChecks: [ + { stableKey: 'prior/a' }, { stableKey: 'prior/b' }, + ] } }], + [2, { request: { schemaVersion: 3 }, selection: { scoredChecks: [ + { stableKey: 'current/c' }, + ] } }], + ]) }; + const argv = gradeArgv(args, '/app', 'http://localhost:6573', 'postgres-l2', 2, + track, 'attempt'); + const index = argv.indexOf('--regression-checks-json'); + assert(index > 0); + const regressionChecks = argv[index + 1]; + assert(regressionChecks); + assert.deepEqual(JSON.parse(regressionChecks), ['prior/a', 'prior/b']); + const dependency = { ...args, progression: { identity: { policy: 'dependency-graph' } } }; + assert.equal(gradeArgv(dependency, '/app', 'http://localhost:6573', 'postgres-l2', 2, + track, 'attempt').includes('--regression-checks-json'), false); +}); + +test('level summary names early stopping without claiming budget exhaustion', () => { + const summary = formatLevelSummary({ level: 2, graded: true, score: 65, max: 70, + repair: { status: 'incomplete', stopReason: 'repeated-findings', used: 2, limit: 5 } }); + assert(summary.includes('stopped: repeated findings')); + assert(!summary.includes('budget exhausted')); + const ungraded = formatLevelSummary({ level: 1, graded: false, + error: 'coding-session-failed', buildCostUsd: 1.25, durationMs: 4_400 }); + assert(ungraded.includes('NOT GRADED')); + assert(ungraded.includes('stopped: coding session failed')); + assert(!ungraded.includes('budget exhausted')); +}); + + +test('pending paid sessions survive a stop without changing accepted level ownership', () => { + const paid = (costUsd: number) => ({ costUsd, costComplete: true, costReceipts: [], sessionId: null, durationMs: 0, usage: null, providerThrottle: null, resources: null, tokens: null, outputTokens: null, turns: null, promptBytes: null, thinking: null, transcript: null, provenance: null, providerMetadata: null }) as RunSessionRecord; + const baseline = { level: 2, graded: true, score: 70, max: 70, selection: null, + outcome: { kind: 'passed' }, buildCostUsd: 2, buildSessions: [paid(2)] } as RunLevelRecord; + const run = { id: 'pending', startedAt: new Date().toISOString(), levels: [baseline] } as BenchmarkRunRecord; + const pending = { level: 3, graded: false, score: null, max: null, selection: null, + outcome: { kind: 'ungraded' }, buildCostUsd: 3, buildSessions: [paid(3)], + repairCostUsd: 5, repairSessions: [paid(5)], repairs: 1 } as RunLevelRecord; + const snapshot = pendingRunSnapshot(run, pending, false); + assert.equal(snapshot.totals!.costUsd, 10); + assert.equal(snapshot.totals!.costComplete, false); + assert.equal(snapshot.levels[1]!.repairSessions![0]!.costUsd, 5); + assert.equal(run.levels.length, 1); + assert.equal(run.totals, undefined); + const sameDepth = pendingRunSnapshot(run, { ...pending, level: 2 }, true); + assert.equal(sameDepth.levels.length, 1); + assert.equal(sameDepth.levels[0]!.buildSessions!.length, 2); + assert.equal(sameDepth.totals!.costUsd, 10); +}); diff --git a/tools/stack-bench/tests/binary-provenance.contract.ts b/tools/stack-bench/tests/binary-provenance.contract.ts new file mode 100644 index 00000000000..c5610f884c3 --- /dev/null +++ b/tools/stack-bench/tests/binary-provenance.contract.ts @@ -0,0 +1,129 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import test from 'node:test'; + +import { assertBinarySourceUnchanged, createBinaryProvenance, RUST_BUILDER_IMAGE, + verifyBinaryProvenance } from '../container/binary-provenance.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { binarySourceIdentity, SOURCE_IDENTITY_SCHEME } + from '../src/releases/release-source.js'; + +const SOURCE = { identityScheme: SOURCE_IDENTITY_SCHEME, + revision: 'a'.repeat(40), sha256: 'b'.repeat(64), files: 12 } as const; + +function fixture(): string { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-binaries-')); + const bin = join(root, 'container', 'bin'); + mkdirSync(bin, { recursive: true }); + writeFileSync(join(bin, 'spacetimedb-cli'), Buffer.from([0x7f, 0x45, 0x4c, 0x46, 1])); + writeFileSync(join(bin, 'spacetimedb-standalone'), Buffer.from([0x7f, 0x45, 0x4c, 0x46, 2])); + return root; +} + +function record(root: string) { + const manifest = createBinaryProvenance(root, SOURCE); + writeFileSync(join(root, 'container', 'spacetimedb-binaries.json'), + `${JSON.stringify(manifest, null, 2)}\n`); + return manifest; +} + +test('binary provenance verifies both exact Linux binaries and their source identity', () => { + const root = fixture(); + try { + const manifest = record(root); + assert.equal(manifest.builderImage, RUST_BUILDER_IMAGE); + assert.deepEqual(verifyBinaryProvenance(root, { sourceSha256: SOURCE.sha256 }), manifest); + + writeFileSync(join(root, 'container', 'bin', 'spacetimedb-cli'), + Buffer.from([0x7f, 0x45, 0x4c, 0x46, 3])); + assert.throws(() => verifyBinaryProvenance(root, { sourceSha256: SOURCE.sha256 }), + /spacetimedb-cli checksum does not match provenance/); + + writeFileSync(join(root, 'container', 'bin', 'spacetimedb-cli'), + Buffer.from([0x7f, 0x45, 0x4c, 0x46, 1])); + writeFileSync(join(root, 'container', 'bin', 'spacetimedb-standalone'), + Buffer.from([0x7f, 0x45, 0x4c, 0x46, 4])); + assert.throws(() => verifyBinaryProvenance(root, { sourceSha256: SOURCE.sha256 }), + /spacetimedb-standalone checksum does not match provenance/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('binary provenance rejects stale source and an unbuilt clean checkout', () => { + const root = fixture(); + try { + record(root); + assert.throws(() => verifyBinaryProvenance(root, { sourceSha256: 'c'.repeat(64) }), + /do not match the selected release source/); + const invalid = record(root); + invalid.source.files = 0; + writeFileSync(join(root, 'container', 'spacetimedb-binaries.json'), JSON.stringify(invalid)); + assert.throws(() => verifyBinaryProvenance(root, { sourceSha256: SOURCE.sha256 }), + /source file count is invalid/); + writeFileSync(join(root, 'container', 'spacetimedb-binaries.json'), JSON.stringify({ + schemaVersion: 1, status: 'unbuilt', + })); + assert.throws(() => verifyBinaryProvenance(root, { sourceSha256: SOURCE.sha256 }), + /build-linux-cli\.sh/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('binary source identity uses canonical Git content across checkout line endings', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-binary-source-')); + const cargo = join(root, 'Cargo.toml'); + const main = join(root, 'crates', 'cli', 'src', 'main.rs'); + mkdirSync(dirname(main), { recursive: true }); + writeFileSync(join(root, '.gitattributes'), '* text=auto\n'); + writeFileSync(cargo, '[workspace]\n'); + writeFileSync(main, 'fn main() {}\n'); + const git = (args: string[]): string => execFileSync('git', ['-C', root, ...args], { + encoding: 'utf8', windowsHide: true, + }); + try { + git(['init', '--quiet']); + git(['config', 'user.email', 'stack-bench@example.invalid']); + git(['config', 'user.name', 'Stack Bench']); + git(['config', 'core.autocrlf', 'false']); + git(['add', '.gitattributes', 'Cargo.toml', 'crates/cli/src/main.rs']); + git(['commit', '--quiet', '-m', 'fixture']); + const before = binarySourceIdentity(root); + + git(['config', 'core.autocrlf', 'true']); + rmSync(cargo); + rmSync(main); + git(['checkout', '--', 'Cargo.toml', 'crates/cli/src/main.rs']); + assert.match(readFileSync(main, 'utf8'), /\r\n/); + assert.equal(git(['status', '--porcelain=v1', '--', 'Cargo.toml', 'crates']).trim(), ''); + assert.equal(binarySourceIdentity(root).sha256, before.sha256); + + writeFileSync(main, 'fn main() { println!("changed"); }\n'); + assert.throws(() => binarySourceIdentity(root), + /binary source paths are not clean/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('binary provenance refuses a source change during the build', () => { + assert.doesNotThrow(() => assertBinarySourceUnchanged(SOURCE, { ...SOURCE })); + assert.throws(() => assertBinarySourceUnchanged(SOURCE, + { ...SOURCE, sha256: 'c'.repeat(64) }), /source changed during the build/); + assert.throws(() => assertBinarySourceUnchanged(SOURCE, + { ...SOURCE, revision: 'd'.repeat(40) }), /source changed during the build/); + assert.throws(() => assertBinarySourceUnchanged(SOURCE, + { ...SOURCE, files: SOURCE.files + 1 }), /source changed during the build/); + assert.throws(() => createBinaryProvenance('unused', + { ...SOURCE, files: 0 }), /file count must be a positive integer/); +}); + +test('controller verifies recorded binaries before it installs them', () => { + const dockerfile = readFileSync(join(STACK_BENCH_ROOT, 'appliance', + 'Controller.Dockerfile'), 'utf8'); + assert.match(dockerfile, /COPY --from=binary-build \/binaries\//); + assert.match(dockerfile, /binary-provenance\.js record-snapshot/); + const verify = dockerfile.indexOf('binary-provenance.js verify'); + const install = dockerfile.indexOf('install -m 0555 container/bin/spacetimedb-cli'); + assert.equal(verify >= 0 && install > verify, true); + assert.doesNotMatch(dockerfile, /^COPY .*container\/bin\/spacetimedb-/m); + assert.match(dockerfile, new RegExp(RUST_BUILDER_IMAGE.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))); +}); diff --git a/tools/stack-bench/tests/browser-action-executors.test.ts b/tools/stack-bench/tests/browser-action-executors.test.ts new file mode 100644 index 00000000000..609ada56a46 --- /dev/null +++ b/tools/stack-bench/tests/browser-action-executors.test.ts @@ -0,0 +1,1027 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { chromium, errors } from 'playwright'; + +import { ACTION_REGISTRY } from '../src/actions/action-catalog.js'; +import { executeAction } from '../src/actions/action-contract.js'; +import { compileActionInput } from '../src/composition/definition-compiler.js'; +import { parseRenderedNumber } from '../src/actions/browser-action-executors.js'; + +type UnknownRecord = Record; +interface ServiceOverrides { + readonly browser?: UnknownRecord; + readonly clockSleep?: (milliseconds: number) => Promise; +} +interface ProvidedServices { + readonly capabilities: Record; + readonly recorded: Map; +} + +function services(actor: unknown, overrides: ServiceOverrides = {}): ProvidedServices { + const recorded = new Map(); + const browser = { + defaultWithin: 5000, + expand: (value: string | undefined) => value === '{room:test}' ? 'test-scoped' : value, + recorded: { + get: (key: string) => recorded.get(key), + set: (key: string, value: number) => recorded.set(key, value), + }, + sleep: async () => {}, + testId: (id: string) => `[data-testid="${id}"]`, + ...overrides.browser, + }; + return { + capabilities: { + // Stub actors without a page have no collapsed sections to open. + actors: { get: (name: string) => name !== 'a' ? undefined : actor && typeof actor === 'object' && !('page' in actor) + ? { ...actor, page: { locator: () => ({ evaluateAll: async () => {} }) } } : actor }, + 'browser-interaction': browser, + 'browser-observation': browser, + clock: { sleep: overrides.clockSleep ?? (async () => {}) }, + }, + recorded, + }; +} + +async function run( + input: UnknownRecord & { readonly do: string }, + provided: ProvidedServices, +) { + return executeAction(ACTION_REGISTRY, input.do, input, { + capabilities: provided.capabilities, + }); +} + +// A redraw can settle within the count deadline. Missing rows, lasting duplicates +// and late results must fail; hidden rows and rows outside the scope do not count. +test('count observations wait for a rendered list without accepting missing or duplicate rows', async t => { + const browser = await chromium.launch({ headless: true }); + const observations: unknown[] = []; + t.after(async () => { + await browser.close(); + if (process.env.STACK_BENCH_COUNT_EVIDENCE) { + const { writeFileSync } = await import('node:fs'); + writeFileSync(process.env.STACK_BENCH_COUNT_EVIDENCE, + JSON.stringify({ browser: browser.version(), observations }, null, 2)); + } + }); + for (const mode of ['delayed', 'missing', 'duplicates', 'late', 'scoped'] as const) { + await t.test(mode, async () => { + const page = await browser.newPage(); + try { + await page.setContent(`
Selected queue
Keyboard
+
+
Other queueKeyboard outside
`); + let redrawStarted = false; + const actor = { page, loc: () => { + const locator = page.locator('[data-testid="queue"]').filter({ hasText: 'Selected queue' }) + .locator('[data-testid="row"]').filter({ visible: true }).first(); + return new Proxy(locator, { get(target, key) { + if (key === 'waitFor') return async (options: Parameters[0]) => { + await target.waitFor(options); + if (redrawStarted) return; + redrawStarted = true; + // Place the real DOM redraw between visibility and observation deterministically. + await page.evaluate(mode => { + const rows = document.querySelector('#rows')!; + if (mode === 'scoped') return; + rows.innerHTML = mode === 'duplicates' + ? 'KeyboardKeyboard' : ''; + if (mode === 'delayed' || mode === 'late') setTimeout(() => { + rows.innerHTML = 'Keyboard'; + }, mode === 'delayed' ? 100 : 1500); + }, mode); + }; + const value = Reflect.get(target, key); + return typeof value === 'function' ? value.bind(target) : value; + } }); + } }; + const started = Date.now(); + const result = await run({ do: 'expect', actor: 'a', testid: 'row', count: 1, + in: { testid: 'queue', contains: 'Selected queue' }, within: 600 }, services(actor, { + browser: { sleep: (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) }, + })); + observations.push({ mode, elapsedMs: Date.now() - started, status: result.status, + finding: result.finding ?? null, summary: result.summary ?? null }); + assert.equal(result.status, mode === 'delayed' || mode === 'scoped' ? 'passed' : 'failed', result.summary ?? mode); + if (mode === 'missing' || mode === 'duplicates' || mode === 'late') assert.equal(result.finding?.kind, 'count-mismatch'); + } finally { await page.close(); } + }); + } +}); + +// Value reads must use the authored observation budget when a visible control redraws. +// A value which arrives after that budget, or stays wrong, must not pass. +test('number observations use the authored budget across a control redraw', async t => { + const browser = await chromium.launch({ headless: true }); + const observations: unknown[] = []; + t.after(async () => { + await browser.close(); + if (process.env.STACK_BENCH_NUMBER_READ_EVIDENCE) { + const { writeFileSync } = await import('node:fs'); + writeFileSync(process.env.STACK_BENCH_NUMBER_READ_EVIDENCE, + JSON.stringify({ browser: browser.version(), observations }, null, 2)); + } + }); + // A final redraw must not erase a value already observed inside the budget. + for (const mode of ['within-budget', 'after-budget', 'wrong-value', 'final-redraw', 'input', 'textarea', 'select'] as const) { + await t.test(mode, async () => { + const page = await browser.newPage(); + try { + page.setDefaultTimeout(100); + await page.setContent('
East
0
' + + '
West20
', { timeout: 5000 }); + await page.evaluate(mode => { + const holder = document.querySelector('#value')!; + if (mode === 'input') holder.innerHTML = ''; + if (mode === 'textarea') { + holder.innerHTML = ''; + holder.querySelector('textarea')!.value = '20'; + } + if (mode === 'select') holder.innerHTML = ''; + }, mode); + let redrawStarted = false; + const actor = { page, loc: () => { + const locator = page.locator('[data-testid="warehouse"]').filter({ hasText: 'East' }) + .locator('[data-testid="stock"]').filter({ visible: true }).first(); + return new Proxy(locator, { get(target, key) { + if (key === 'evaluate' && mode === 'final-redraw') return async (...args: Parameters) => { + const value = await target.evaluate(...args); + await page.locator('#value').evaluate(element => { element.innerHTML = ''; }); + return value; + }; + if (key === 'waitFor') return async (options: Parameters[0]) => { + await target.waitFor(options); + if (redrawStarted) return; + redrawStarted = true; + await page.evaluate(mode => { + const value = document.querySelector('#value')!; + if (mode !== 'within-budget' && mode !== 'after-budget') return; + value.innerHTML = ''; + setTimeout(() => { + value.innerHTML = '20'; + value.setAttribute('data-restored-at', String(Date.now())); + }, + mode === 'within-budget' ? 350 : 1500); + }, mode); + }; + const value = Reflect.get(target, key); + return typeof value === 'function' ? value.bind(target) : value; + } }); + } }; + const started = Date.now(); + const result = await run({ do: 'expectNumber', actor: 'a', testid: 'stock', equals: 20, + in: { testid: 'warehouse', contains: 'East' }, within: 1000 }, services(actor, { + browser: { sleep: (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) }, + })); + const elapsedMs = Date.now() - started; + // Independently witness restoration even if the grader has already failed early. + if (mode === 'within-budget') await page.locator('#value [data-testid="stock"]').waitFor({ timeout: 2000 }); + const restoredAt = await page.locator('#value').getAttribute('data-restored-at', { timeout: 2000 }); + const evidence = process.env.STACK_BENCH_NUMBER_READ_EVIDENCE; + if (evidence) await page.screenshot({ path: `${evidence}.${mode}.png`, timeout: 5000 }); + observations.push({ mode, locatorTimeoutMs: 100, withinMs: 1000, + restoredAfterMs: mode === 'within-budget' ? 350 : mode === 'after-budget' ? 1500 : null, + observedRestoreAfterMs: restoredAt ? Number(restoredAt) - started : null, + elapsedMs, status: result.status, finding: result.finding ?? null, summary: result.summary ?? null }); + if (mode === 'within-budget') assert(restoredAt && Number(restoredAt) - started < 1000); + assert.equal(result.status, ['after-budget', 'wrong-value', 'final-redraw'].includes(mode) ? 'failed' : 'passed', result.summary ?? mode); + if (mode === 'wrong-value' || mode === 'final-redraw') { + assert.equal(result.finding?.kind, 'number-mismatch'); + assert.equal(result.finding?.fields.observed, 0); + } + } finally { await page.close(); } + }); + } +}); + +test('script canary detects execution after DOM removal and rejects missing observers', async () => { + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage(); + const provided = services({ page }); + assert.equal((await run({ do: 'expectNoScriptExecution', actor: 'a' }, provided)).status, 'inconclusive'); + assert.equal((await run({ do: 'armScriptCanary', actor: 'a' }, provided)).status, 'passed'); + const payload = ''; + await page.setContent('
'); + await page.locator('main').evaluate((element, text) => { element.textContent = text; }, payload); + assert.equal((await run({ do: 'expectNoScriptExecution', actor: 'a' }, provided)).status, 'passed'); + const delivered = page.waitForEvent('console', { predicate: message => message.text() === 'canary-delivered', timeout: 5000 }); + await page.setContent(payload); + await delivered; + await page.setContent('
replacement document
'); + assert.equal((await run({ do: 'expectNoScriptExecution', actor: 'a' }, provided)).status, 'failed'); + await page.evaluate(() => { + delete (globalThis as unknown as { __stackBenchScriptCanary?: unknown }).__stackBenchScriptCanary; + }); + assert.equal((await run({ do: 'expectNoScriptExecution', actor: 'a' }, provided)).status, 'inconclusive'); + } finally { await browser.close(); } +}); + +test('UI failures retain bounded observations but exclude passwords and unproven missing choices', async () => { + for (const sensitive of [false, true]) { + const result = await run({ do: 'expectNumber', actor: 'a', testid: 'stock', equals: 99, within: 1, + in: { testid: sensitive ? 'secret' : 'item-card', contains: 'Bluetooth Speaker' } }, + services({ page: { locator: () => ({ filter: () => ({ count: async () => 1 }) }) }, + loc: () => ({ waitFor: async () => {}, + evaluate: async (read: (element: { tagName: string; innerText: string }) => unknown) => read({ tagName: 'DIV', innerText: '100' }), + innerText: async () => '100' }) })); + assert.equal(result.finding?.kind, 'number-mismatch'); + if (sensitive) assert.doesNotMatch(result.summary ?? '', /Bluetooth Speaker/); + else assert.match(result.summary ?? '', /entry matching "Bluetooth Speaker"/); + } + for (const operation of ['expect', 'expectNumber']) { + const result = await run({ do: operation, actor: 'a', testid: 'stock', + ...(operation === 'expect' ? { contains: 'Keyboard' } : {}), + in: { testid: 'warehouse', contains: 'East' }, ...(operation === 'expectNumber' ? { equals: 1 } : {}) }, + services({ loc: () => ({ waitFor: async () => { throw new errors.TimeoutError('not visible'); } }) })); + assert.equal(result.finding?.kind, 'control-missing'); + assert.match(result.summary ?? '', operation === 'expect' ? /East.*Keyboard/ : /East/); + } + for (const password of [false, true]) { + const result = await run({ do: 'expect', actor: 'a', testid: 'field', value: 'Approved', within: 1 }, + services({ loc: () => ({ waitFor: async () => {}, + evaluate: async (read: (element: { tagName: string; value: string }) => unknown) => + read({ tagName: 'INPUT', value: password ? 'RAW_PASSWORD' : 'Pending' }), + getAttribute: async () => password ? 'password' : 'text' }) })); + assert.equal(result.finding?.kind, 'value-mismatch'); + if (password) assert.doesNotMatch(JSON.stringify(result.finding), /RAW_PASSWORD|Approved/); + else assert.match(result.summary ?? '', /Pending.*Approved/); + } + for (const present of [false, true]) { + const select = { tagName: 'SELECT', options: [{ value: 'weekly', label: present ? 'Weekly' : 'Daily' }] }; + const result = await run({ do: 'fill', actor: 'a', testid: 'frequency', text: 'Weekly' }, services({ loc: () => ({ + waitFor: async () => {}, count: async () => 1, evaluate: async (read: (element: typeof select) => unknown) => read(select), + selectOption: async () => { throw new errors.TimeoutError('locator.selectOption: Timeout exceeded'); }, + }) })); + assert.equal(result.finding?.kind, present ? 'page-timeout' : 'choice-missing'); + if (!present) assert.match(result.summary ?? '', /required choice "Weekly"/); + } +}); + +test('observation and selection errors cannot become missing-control findings', async () => { + for (const operation of ['expect', 'expectNumber', 'waitUntilAbsent', 'fill']) { + const result = await run({ do: operation, actor: 'a', testid: 'field', + ...(operation === 'expectNumber' ? { equals: 1 } : {}), + ...(operation === 'fill' ? { text: 'Weekly' } : {}) }, services({ loc: () => ({ + waitFor: async () => { throw new Error('locator.waitFor: invalid selector'); }, + }) })); + assert.equal(result.status, 'harness_failure', `${operation}: ${result.summary}`); + assert.equal(result.finding, null); + } + for (const stage of ['select', 'options']) { + let reads = 0; + const result = await run({ do: 'fill', actor: 'a', testid: 'field', text: 'Weekly' }, + services({ loc: () => ({ waitFor: async () => {}, + evaluate: async () => { + if (reads++ === 0) return 'SELECT'; + throw new Error('locator.evaluate: observation script failed'); + }, + selectOption: async () => { + if (stage === 'select') throw new Error('locator.selectOption: Protocol error'); + throw new errors.TimeoutError('locator.selectOption: Timeout exceeded'); + }, + }) })); + assert.equal(result.status, 'harness_failure', result.summary ?? undefined); + assert.equal(result.finding, null); + } +}); + +test('message order requires observed messages and compares the merged sender sequence', async () => { + for (const [left, right, expected] of [ + [[], [], 'failed'], + [['AA-1', 'BB-1'], ['AA-1', 'BB-1'], 'passed'], + [['AA-1', 'BB-1'], ['BB-1', 'AA-1'], 'failed'], + ] as const) { + const provided = services({}); + provided.capabilities.actors = { get: (name: string) => ({ page: { + locator: () => ({ allInnerTexts: async () => name === 'a' ? left : right }), + } }) }; + const result = await run({ do: 'expectOrderMatches', actors: ['a', 'b'], prefix: '(?:AA|BB)' }, provided); + assert.equal(result.status, expected, result.summary ?? undefined); + } + // Ordered text is an explicit implementation-neutral observation. + const items = { + filter: () => items, + allInnerTexts: async () => ['Coffee Grinder', 'Air Purifier'], + }; + const scope = { filter: () => scope, first: () => scope, locator: () => items }; + const ordered = await run({ do: 'expectSequence', actor: 'a', testid: 'item-name', + in: { testid: 'item-list' }, equals: ['Coffee Grinder', 'Air Purifier'] }, services({ page: { locator: () => scope } })); + assert.equal(ordered.status, 'passed'); + assert.deepEqual((ordered.observation as { values: string[] }).values, ['Coffee Grinder', 'Air Purifier']); +}); + +test('optional clicks operate enabled controls and skip unavailable controls', async () => { + for (const [visible, enabled] of [[false, false], [true, false], [true, true]]) { + let clicks = 0, timeout: unknown; + const provided = services({ loc: () => ({ isVisible: async () => visible, + isDisabled: async () => !enabled, click: async (options: { timeout: number }) => { clicks += 1; timeout = options.timeout; } }) }); + const result = await run({ do: 'click', actor: 'a', testid: 'buy-now', ifAvailable: true, within: 1 }, provided); + assert.equal(result.status, 'passed'); + assert.equal(clicks, visible && enabled ? 1 : 0); + // The short wait is for the control to appear; a control that is there gets the ordinary click time. + if (clicks) assert.equal(timeout, 5000); + } +}); + +test('interaction scopes expand scoped values and can match separate text fragments without assuming punctuation', async () => { + const received: unknown[] = []; + let clickOptions: unknown; + const actor = { loc: (_testid: string, options: unknown) => { + received.push(options); + return { click: async (options: unknown) => { clickOptions = options; } }; + } }; + const expanded = await run({ do: 'click', actor: 'a', testid: 'open', + contains: '{room:test}', in: { testid: 'row', contains: '{room:test}' }, settleMs: 5 }, services(actor)); + assert.equal(expanded.status, 'passed'); + assert.deepEqual(received, [{ contains: 'test-scoped', scope: { testid: 'row', contains: 'test-scoped' } }]); + assert.deepEqual(clickOptions, { timeout: 5000 }); + + const result = await run({ do: 'click', actor: 'a', testid: 'save', + in: { testid: 'row', containsAll: ['Mirrorless Camera', 'East'] } }, services(actor)); + assert.equal(result.status, 'passed'); + const scope = (received.at(-1) as { scope?: { testid: string; contains: unknown } }).scope; + assert(scope && scope.contains instanceof RegExp); + assert.equal(scope.testid, 'row'); + assert(scope.contains.test('Mirrorless Camera @ East')); + assert(scope.contains.test('East: Mirrorless Camera')); + assert.equal(scope.contains.test('Mirrorless Camera @ West'), false); +}); + +test('fill adapts values to date input types', async () => { + const values: Array<[string, string]> = []; + const locator = (type: string) => ({ + waitFor: async () => {}, + count: async () => 1, + evaluate: async () => 'INPUT', + getAttribute: async (name: string) => name === 'type' ? type : null, + fill: async (value: string) => { values.push([type, value]); }, + }); + const actor = { loc: (testid: string) => locator(testid) }; + + for (const testid of ['datetime-local', 'date', 'text']) { + const result = await run({ do: 'fill', actor: 'a', testid, text: '2020-01-01' }, services(actor)); + assert.equal(result.status, 'passed'); + } + + assert.deepEqual(values, [ + ['datetime-local', '2020-01-01T00:00'], + ['date', '2020-01-01'], + ['text', '2020-01-01'], + ]); + + values.length = 0; + await run({ do: 'fill', actor: 'a', testid: 'date', text: '2099-12-31T23:59' }, services(actor)); + assert.deepEqual(values, [['date', '2099-12-31']]); +}); + +test('stock observations accept an explicit zero-stock state without relaxing other numeric controls', async () => { + let rendered = ' Out of stock '; + const locator = { + waitFor: async () => {}, + isVisible: async () => true, + evaluate: async (read: (element: { tagName: string; innerText: string }) => unknown) => read({ tagName: 'SPAN', innerText: rendered }), + innerText: async () => rendered, + }; + const provided = services({ loc: () => locator }); + provided.capabilities.actors = { get: () => ({ loc: () => locator }) }; + const recorded = await run({ do: 'recordNumber', actor: 'a', testid: 'item-stock', as: 'stock' }, provided); + assert.equal(recorded.status, 'passed'); + assert.equal(provided.recorded.get('stock'), 0); + assert.equal((await run({ do: 'expectNumber', actor: 'a', testid: 'item-stock', equals: 0 }, provided)).status, 'passed'); + assert.equal((await run({ do: 'expectAgreement', actors: ['a', 'b'], testid: 'item-stock', numeric: true }, provided)).status, 'passed'); + assert.equal((await run({ do: 'recordNumber', actor: 'a', testid: 'order-total', as: 'total' }, provided)).status, 'failed'); + rendered = 'Stock unavailable'; + assert.equal((await run({ do: 'recordNumber', actor: 'a', testid: 'item-stock', as: 'stock' }, provided)).status, 'failed'); + rendered = 'Stock: -1'; + await run({ do: 'recordNumber', actor: 'a', testid: 'item-stock', as: 'stock' }, provided); + assert.equal(provided.recorded.get('stock'), -1); +}); + +test('an observation mismatch is application evidence, not a harness crash', async () => { + const locator = { + waitFor: async () => {}, + innerText: async () => 'contains private value', + }; + const actor = { loc: () => locator }; + const result = await run({ do: 'expect', actor: 'a', testid: 'status', + notContains: 'private value' }, services(actor)); + assert.equal(result.status, 'failed'); + assert.equal(result.code, 'application_failure'); + assert.match(result.summary ?? '', /shows "private value" that must not appear/); + assert.doesNotMatch(JSON.stringify(result.finding), /contains private value/); +}); + +test('a visible but blank field does not satisfy a non-empty assertion', async () => { + let rendered = ' '; + const locator = { + waitFor: async () => {}, + evaluate: async (read: (element: { tagName: string; innerText: string }) => unknown) => read({ tagName: 'DIV', innerText: rendered }), + innerText: async () => rendered, + }; + const actor = { loc: () => locator }; + const blank = await run({ do: 'expect', actor: 'a', testid: 'warehouse', nonEmpty: true }, + services(actor)); + assert.equal(blank.status, 'failed'); + assert.equal(blank.code, 'application_failure'); + assert.match(blank.summary ?? '', /control is empty/); + + rendered = 'East'; + const populated = await run({ do: 'expect', actor: 'a', testid: 'warehouse', nonEmpty: true }, + services(actor)); + assert.equal(populated.status, 'passed'); +}); + +test('a page that navigates while its sections are opened is observed, not a harness failure', async () => { + for (const message of ['locator.evaluateAll: Execution context was destroyed, most likely because of a navigation', + 'browser disconnected']) { + const actor = { loc: () => ({ waitFor: async () => {} }), + page: { locator: () => ({ evaluateAll: async () => { throw new Error(message); } }) } }; + const result = await run({ do: 'expect', actor: 'a', testid: 'current-user' }, services(actor)); + assert.equal(result.status, message.includes('navigation') ? 'passed' : 'harness_failure', result.summary ?? undefined); + } +}); + +test('absence checks do not pass before a late element appears', async () => { + let checks = 0; + const actor = { loc: () => ({ isVisible: async () => ++checks > 1 }) }; + const result = await run({ do: 'expect', actor: 'a', testid: 'private-row', + absent: true, within: 100 }, services(actor)); + assert.equal(result.status, 'failed'); + assert.match(result.summary ?? '', /was shown when it must not be/); +}); + +test('unavailable checks do not pass before a control becomes enabled', async () => { + for (const enabled of [true, false]) { + let checks = 0; + const locator = { + filter() { return this; }, + first() { return this; }, + isVisible: async () => true, + isDisabled: async () => !enabled || ++checks === 1, + getAttribute: async () => null, + }; + const actor = { page: { locator: () => locator } }; + const result = await run({ do: 'expectUnavailable', actor: 'a', testid: 'admin', + within: 100 }, services(actor)); + if (enabled) { + assert.equal(result.status, 'failed'); + assert.match(result.summary ?? '', /stayed available to/); + } else { + assert.equal(result.status, 'passed'); + assert.deepEqual(result.observation, { unavailable: true, reason: 'disabled' }); + } + } +}); + +test('missing values do not satisfy agreement across actors', async () => { + const actor = { loc: () => ({ isVisible: async () => false, innerText: async () => '' }) }; + const provided = services(actor, { browser: { + sleep: async () => new Promise(resolve => setTimeout(resolve, 2)), + } }); + provided.capabilities.actors = { get: (name: string) => + name === 'a' || name === 'b' ? actor : undefined }; + const result = await run({ do: 'expectAgreement', actors: ['a', 'b'], + testid: 'total', within: 1 }, provided); + assert.equal(result.status, 'failed'); + assert.match(result.summary ?? '', /missing or unreadable/); +}); + +test('expect can verify a persisted form value or an element attribute', async () => { + const field = { waitFor: async () => {}, + evaluate: async (read: (element: { tagName: string; value: string }) => unknown) => read({ tagName: 'INPUT', value: 'staff' }) }; + const persisted = await run({ do: 'expect', actor: 'a', testid: 'support-assignee', + value: 'staff' }, services({ loc: () => field })); + assert.equal(persisted.status, 'passed'); + assert.deepEqual(persisted.observation, { visible: true, value: 'staff' }); + + const locator = { + waitFor: async () => {}, + getAttribute: async (name: string) => name === 'data-state' ? 'on' : null, + }; + const actor = { loc: () => locator }; + const result = await run({ do: 'expect', actor: 'a', testid: 'notification-preference', + attribute: 'data-state', value: 'on' }, services(actor)); + assert.equal(result.status, 'passed'); + assert.deepEqual(result.observation, { visible: true, attribute: 'data-state', value: 'on' }); +}); + +test('element counts use visible observations instead of hidden duplicate markup', async () => { + const matches = { + filter: () => ({ count: async () => 2 }), + }; + const actor = { page: { locator: () => matches } }; + const result = await run({ do: 'expectElementCount', actor: 'a', testid: 'row', + contains: 'item', equals: 2, within: 1 }, services(actor)); + assert.equal(result.status, 'passed'); + assert.deepEqual(result.observation, { count: 2 }); +}); + +test('element counts can be scoped to a matching parent', async () => { + const child = { filter: () => ({ count: async () => 1 }) }; + const parent = { filter: () => parent, first: () => parent, locator: () => child }; + const actor = { page: { locator: () => parent } }; + const result = await run({ do: 'expectElementCount', actor: 'a', testid: 'payment-record', + in: { testid: 'order-item', contains: 'Desk Lamp' }, equals: 1, within: 1 }, services(actor)); + assert.equal(result.status, 'passed'); + assert.deepEqual(result.observation, { count: 1 }); +}); + +test('relative element counts preserve existing rows and accept an empty baseline', async () => { + for (const initial of [0, 3]) { + let count = initial; + const loc = { filter: () => loc, count: async () => count }; + const provided = services({ page: { locator: () => loc } }); + assert.equal((await run({ do: 'recordNumber', actor: 'a', testid: 'row', + as: 'before', count: true }, provided)).status, 'passed'); + assert.equal(provided.recorded.get('before'), initial); + count++; + assert.equal((await run({ do: 'expectElementCount', actor: 'a', testid: 'row', + relativeTo: 'before', plus: 1 }, provided)).status, 'passed'); + count++; + assert.equal((await run({ do: 'expectElementCount', actor: 'a', testid: 'row', + relativeTo: 'before', plus: 1, within: 1 }, provided)).status, 'failed'); + } +}); + +test('relative count assertions reject missing records and invalid count targets', async () => { + const loc = { filter: () => loc, count: async () => 0 }; + const provided = services({ page: { locator: () => loc } }); + const input = { do: 'expectElementCount', actor: 'a', testid: 'row', relativeTo: 'before' }; + const missing = await run(input, provided); + assert.equal(missing.status, 'inconclusive'); + assert.equal(missing.code, 'inconclusive'); + assert.equal(missing.finding?.kind, 'assertion-without-action'); + for (const base of [-1, 0.5, Infinity, Number.MAX_SAFE_INTEGER + 1]) { + provided.recorded.set('before', base); + assert.equal((await run(input, provided)).status, 'harness_failure'); + } + for (const fields of [{}, { equals: 0, relativeTo: 'before' }, { equals: 0, plus: 1 }, + { relativeTo: 'before', plus: 0.5 }, { equals: -1 }]) { + assert.throws(() => compileActionInput({ do: 'expectElementCount', actor: 'a', + testid: 'row', ...fields })); + } +}); + +test('unactionable controls are application evidence while crashes and code bugs remain harness failures', async () => { + const timeout = Object.assign(new Error('locator.click: element was never actionable'), + { name: 'TimeoutError' }); + const timedOut = await run({ do: 'click', actor: 'a', testid: 'submit' }, + services({ loc: () => ({ click: async () => { throw timeout; } }) })); + assert.equal(timedOut.status, 'failed'); + assert.equal(timedOut.code, 'application_failure'); + + const crashed = await run({ do: 'click', actor: 'a', testid: 'submit' }, + services({ loc: () => ({ click: async () => { + throw new Error('locator.click: Target page, context or browser has been closed'); + } }) })); + assert.equal(crashed.status, 'harness_failure'); + assert.equal(crashed.code, 'unclassified_exception'); + + const bug = await run({ do: 'click', actor: 'a', testid: 'submit' }, + services({ loc: () => ({ click: async () => { throw new TypeError('executor bug'); } }) })); + assert.equal(bug.status, 'harness_failure'); + assert.equal(bug.code, 'unclassified_exception'); +}); + +test('a click that removes its target before timing out is inconclusive and is not repeated', async () => { + const { chromium } = await import('playwright'); + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage(); + await page.setContent(``); + let clicks = 0; + const result = await run({ do: 'click', actor: 'a', testid: 'close', within: 1000 }, + services({ loc: () => ({ click: async (options: Parameters['click']>[0]) => { + clicks += 1; + await page.locator('#close').click(options); + } }) })); + assert.equal(result.status, 'inconclusive', result.summary ?? undefined); + assert.equal(clicks, 1); + assert.equal(await page.locator('#close').count(), 0); + assert.equal(await page.locator('body').getAttribute('data-clicks'), '1'); + assert.match(JSON.stringify(result.observation), /performing click action/); + } finally { await browser.close(); } +}); + +test('a hit-target interceptor that prevents the click is blocked-control evidence', async () => { + const browser = await chromium.launch({ headless: true }); + let observation: unknown; + try { + const page = await browser.newPage(); + await page.setContent(` + `); + let calls = 0; + let detail = ''; + const result = await run({ do: 'click', actor: 'a', testid: 'submit', within: 1000 }, + services({ loc: () => ({ click: async (options: Parameters['click']>[0]) => { + calls += 1; + try { await page.locator('#submit').click(options); } + catch (error) { detail = String(error); throw error; } + } }) })); + const delivered = await page.locator('body').evaluate(body => ({ + pointerdown: body.dataset.down ?? null, click: body.dataset.clicks ?? null, + })); + observation = { fixture: 'cover appears on pointerover, after the initial hit-target check', + expected: 'failed/control-blocked with no delivered pointerdown or click', + result, calls, detail, delivered }; + assert.match(detail, /performing click action/); + assert.match(detail, /intercepts pointer events/); + assert.doesNotMatch(detail, /click action done/); + assert.deepEqual(delivered, { pointerdown: null, click: null }); + assert.equal(calls, 1); + assert.equal(result.status, 'failed'); + assert.equal(result.finding?.kind, 'control-blocked'); + } finally { + if (process.env.STACK_BENCH_CLICK_EVIDENCE) { + const { writeFileSync } = await import('node:fs'); + writeFileSync(process.env.STACK_BENCH_CLICK_EVIDENCE, JSON.stringify({ + command: 'node --test --test-name-pattern="a hit-target interceptor|a click that removes" dist/tests/browser-action-executors.test.js', + browser: browser.version(), observation, + }, null, 2)); + } + await browser.close(); + } +}); + +test('an intercepted attempt cannot erase a later delivered click timeout', async () => { + const browser = await chromium.launch({ headless: true }); + let observation: unknown; + try { + const page = await browser.newPage(); + await page.setContent(` + `); + let calls = 0; + let detail = ''; + const result = await run({ do: 'click', actor: 'a', testid: 'submit', within: 2000 }, + services({ loc: () => ({ click: async (options: Parameters['click']>[0]) => { + calls += 1; + try { await page.locator('#submit').click(options); } + catch (error) { detail = String(error); throw error; } + } }) })); + const delivered = await page.locator('body').getAttribute('data-clicks'); + observation = { fixture: 'one transient pointerover cover, then a delivered click blocks its completion', + expected: 'inconclusive with exactly one delivered click and one locator invocation', + result, calls, detail, delivered }; + assert.match(detail, /performing click action[\s\S]*intercepts pointer events[\s\S]*performing click action/); + assert.equal(delivered, '1'); + assert.equal(await page.locator('#submit').count(), 0); + assert.equal(calls, 1); + assert.equal(result.status, 'inconclusive'); + } finally { + if (process.env.STACK_BENCH_CLICK_EVIDENCE) { + const { writeFileSync } = await import('node:fs'); + writeFileSync(`${process.env.STACK_BENCH_CLICK_EVIDENCE}.mixed.json`, JSON.stringify({ + command: 'node --test --test-name-pattern="an intercepted attempt" dist/tests/browser-action-executors.test.js', + browser: browser.version(), observation, + }, null, 2)); + } + await browser.close(); + } +}); + + +test('relative number bounds use recorded values and report the resolved bound', async () => { + assert.equal(parseRenderedNumber('Stock: 1,024 left'), 1024); + assert.equal(parseRenderedNumber('$12.50'), 12.5); + assert.equal(parseRenderedNumber('none'), null); + let value: number | string = 'Total: 1,024'; + const provided = services({ loc: () => ({ waitFor: async () => {}, + evaluate: async (read: (element: { tagName: string; innerText: string }) => unknown) => read({ tagName: 'SPAN', innerText: String(value) }), + innerText: async () => String(value) }) }); + assert.equal((await run({ do: 'recordNumber', actor: 'a', testid: 'total', as: 'before' }, provided)).status, 'passed'); + assert.equal(provided.recorded.get('before'), 1024); + value = 'Total: 1,027'; + const compared = await run({ do: 'expectNumber', actor: 'a', testid: 'total', relativeTo: 'before', plus: 3 }, provided); + assert.equal(compared.status, 'passed'); + assert.deepEqual(compared.observation, { value: 1027 }); + + value = 70; + provided.recorded.set('initial', 72); + const step = { do: 'expectNumber', actor: 'a', testid: 'timer', relativeTo: 'initial', plus: -1, within: 1 }; + assert.equal((await run({ ...step, comparison: 'atMost' }, provided)).status, 'passed'); + value = 74; + assert.equal((await run({ ...step, plus: 1, comparison: 'atLeast' }, provided)).status, 'passed'); + value = 72; + const failed = await run({ ...step, comparison: 'atMost' }, provided); + assert.equal(failed.status, 'failed'); + assert.match(JSON.stringify(failed), /"atMost":71/); + assert.doesNotMatch(JSON.stringify(failed), /"equals":71/); + const missing = await run({ ...step, relativeTo: 'missing', comparison: 'atMost' }, provided); + assert.equal(missing.status, 'inconclusive'); +}); + + +test('optional navigation waits for delayed controls or inline content', async () => { + for (const inline of [false, true]) { + let ready = false; + let clicks = 0; + const provided = services({ loc: (id: string) => ({ + isVisible: async () => ready && (inline ? id === 'low-stock-item' : id === 'low-stock-link'), + isDisabled: async () => false, + evaluateAll: async () => true, + click: async () => { clicks += 1; }, + }) }, { browser: { sleep: async () => { ready = true; } } }); + const result = await run({ do: 'click', actor: 'a', testid: 'low-stock-link', + ifAvailable: true, unlessVisible: 'low-stock-item', within: 1000 }, provided); + assert.equal(result.status, 'passed', result.summary ?? undefined); + assert.equal(clicks, inline ? 0 : 1); + } +}); + +test('navigation retries a replaced destination without repeating clicks or hiding errors', async () => { + for (const failure of ['once', 'always', 'unrelated']) { + let reads = 0; + let clicks = 0; + const provided = services({ loc: () => ({ + isVisible: async () => true, + evaluateAll: async () => { + reads += 1; + if (failure === 'unrelated') throw new Error('browser disconnected'); + if (failure === 'always' || reads === 1) throw new Error('Element is not attached to the DOM'); + return true; + }, + click: async () => { clicks += 1; }, + }) }, { browser: { sleep: async (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) } }); + const result = await run({ do: 'click', actor: 'a', testid: 'low-stock-link', + unlessVisible: 'low-stock-item', within: failure === 'always' ? 30 : 1000 }, provided); + assert.equal(result.status, failure === 'once' ? 'passed' : 'harness_failure'); + assert.equal(clicks, 0); + assert.equal(reads === 1, failure === 'unrelated'); + } +}); + +test('unreadable destinations never trigger a blind toggle click', async () => { + for (const optional of [false, true]) { + let clicks = 0; + const provided = services({ loc: (id: string) => ({ + isVisible: async () => id === 'order-item', + isDisabled: async () => false, + evaluateAll: async () => { throw Object.assign(new Error('observation timed out'), { name: 'TimeoutError' }); }, + click: async () => { clicks += 1; throw Object.assign(new Error('required control missing'), { name: 'TimeoutError' }); }, + }) }); + const result = await run({ do: 'click', actor: 'a', testid: 'orders-toggle', + unlessVisible: 'order-item', ifAvailable: optional, within: 10 }, provided); + assert.equal(result.status, 'failed'); + assert.equal(clicks, 0); + } +}); + +test('covered navigation accepts a destination that finishes loading without another click', async () => { + const { chromium } = await import('playwright'); + const browser = await chromium.launch({ headless: true }); + try { + for (const mode of ['delayed', 'absent', 'button', 'removed']) { + const page = await browser.newPage(); + try { + await page.setContent(` + ${mode === 'removed' ? '
Audio
' : `Loading`}`); + const loc = (id: string) => { + const locator = page.locator(`#${id}`); + if (mode !== 'removed' || id !== 'category-row') return locator; + // Catalog navigation can remove the old destination between two reads. + return new Proxy(locator, { get(target, key) { + if (key === 'isVisible') return async () => { + const visible = await target.isVisible(); + await page.locator('#category-row').evaluate(element => element.remove()); + return visible; + }; + const value = Reflect.get(target, key); + return typeof value === 'function' ? value.bind(target) : value; + } }); + }; + const result = await run({ do: 'click', actor: 'a', testid: 'sales-link', + unlessVisible: 'category-row', ifAvailable: true, within: 600 }, + services({ loc }, { + browser: { sleep: async (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) }, + })); + assert.equal(result.status, mode === 'absent' ? 'failed' : 'passed', result.summary ?? undefined); + assert.equal(await page.locator('#sales-link').getAttribute('data-clicks'), + mode === 'button' || mode === 'removed' ? '1' : null); + if (mode === 'absent') assert.equal(result.finding?.kind, 'control-blocked'); + } finally { await page.close(); } + } + } finally { await browser.close(); } +}); + +test('covered navigation preserves cancellation and browser failures', async () => { + for (const cancelled of [false, true]) { + const controller = new AbortController(); + let reads = 0; + let clicks = 0; + const provided = services({ loc: () => ({ + isVisible: async () => { reads += 1; return reads > 1; }, + isDisabled: async () => false, + evaluateAll: async () => true, + click: async () => { + clicks += 1; + if (cancelled) controller.abort('cancelled by test'); + throw Object.assign(new Error(cancelled ? 'intercepts pointer events' : 'browser disconnected'), + { name: cancelled ? 'TimeoutError' : 'Error' }); + }, + }) }); + const result = await executeAction(ACTION_REGISTRY, 'click', { + do: 'click', actor: 'a', testid: 'sales-link', unlessVisible: 'category-row', within: 100, + }, { capabilities: provided.capabilities, signal: controller.signal }); + assert.equal(result.status, cancelled ? 'inconclusive' : 'harness_failure'); + if (cancelled) assert.equal(result.code, 'cancelled'); + assert.equal(reads, 1); + assert.equal(clicks, 1); + } +}); + +test('reload timeouts fail the app while proven browser crashes stay harness failures', async () => { + for (const [message, expected] of [ + ['page.reload: Timeout 20000ms exceeded', 'failed'], + ['page.reload: Target crashed', 'harness_failure'], + ]) { + const provided = services({ loc: () => ({}), page: { reload: async () => { + throw Object.assign(new Error(message), { name: 'TimeoutError' }); + } } }); + const result = await run({ do: 'reload', actor: 'a', settleMs: 0 }, provided); + assert.equal(result.status, expected); + } +}); + +test('containsText polls the selected item without requiring an exact status value', async () => { + const step = { do: 'expect', actor: 'a', testid: 'order-item', contains: 'Keyboard', + containsText: 'returned', ignoreCase: true, within: 1 }; + assert.doesNotThrow(() => compileActionInput(step)); + for (const conflict of [{ value: 'returned' }, { attribute: 'data-state' }, { absent: true }, { containsText: '' }]) { + assert.throws(() => compileActionInput({ ...step, ...conflict })); + } + for (const returned of [false, true]) { + let ready = false; + const provided = services({ loc: (_id: string, options: { contains: string }) => { + assert.equal(options.contains, 'Keyboard'); + return { waitFor: async () => {}, + evaluate: async (read: (element: { tagName: string; innerText: string }) => unknown) => + read({ tagName: 'ARTICLE', innerText: `Keyboard · shipped${ready && returned ? ' · Returned' : ''}` }), + getAttribute: async () => null, + innerText: async () => `Keyboard · shipped${ready && returned ? ' · Returned' : ''}` }; + } }, { browser: { sleep: async () => { ready = true; } } }); + const result = await run(step, provided); + assert.equal(result.status, returned ? 'passed' : 'failed'); + if (!returned) assert.match(result.summary!, /Keyboard.*does not contain "returned"/); + } +}); + +test('failed disappearance identifies the matched entry and scope', async () => { + const provided = services({ loc: () => ({ waitFor: async () => { throw new errors.TimeoutError('Timeout'); } }) }); + const result = await run({ do: 'waitUntilAbsent', actor: 'a', testid: 'item-card', + contains: 'Coffee Grinder', in: { testid: 'search-results' }, within: 1 }, provided); + assert.equal(result.status, 'failed'); + assert.match(result.summary!, /Coffee Grinder/); + assert.match(result.summary!, /search-results/); +}); + +test('statusText accepts only boundary list decoration and keeps other values strict', async t => { + const browser = await chromium.launch({ headless: true }); + const observations: unknown[] = []; + t.after(async () => { + await browser.close(); + if (process.env.STACK_BENCH_STATUS_TEXT_EVIDENCE) { + const { writeFileSync } = await import('node:fs'); + writeFileSync(process.env.STACK_BENCH_STATUS_TEXT_EVIDENCE, + JSON.stringify({ browser: browser.version(), observations }, null, 2)); + } + }); + const cases = [ + ['plain', 'pending', 'passed'], + ['middle-dot', ' · pending', 'passed'], + ['bullet-and-case', '\u00a0• PENDING ·\u00a0', 'passed'], + ['negation', 'not pending', 'failed'], + ['mixed-state', 'pending/shipped', 'failed'], + ['word-prefix', 'Status: pending', 'failed'], + ['number-prefix', '1 pending', 'failed'], + ['semantic-icon', '❌ pending', 'failed'], + ['unknown-symbol', '? pending', 'failed'], + ['wrong-state', '· shipped', 'failed'], + ['internal-separator', 'pen·ding', 'failed'], + ['empty-decoration', ' · • ', 'failed'], + ] as const; + for (const [name, text, expected] of cases) { + await t.test(name, async () => { + const page = await browser.newPage(); + try { + await page.setContent('
Keyboard
' + + '
Otherpending
'); + await page.locator('[data-testid="order"]').first().locator('[data-testid="order-status"]').evaluate( + (node, value) => { node.textContent = value; }, text); + const actor = { page, loc: (id: string, options: { scope?: { contains?: string } }) => { + assert.equal(options.scope?.contains, 'Keyboard'); + return page.locator('[data-testid="order"]').filter({ hasText: 'Keyboard' }) + .locator(`[data-testid="${id}"]`); + } }; + const result = await run({ do: 'expect', actor: 'a', testid: 'order-status', statusText: 'pending', + ignoreCase: true, in: { testid: 'order', contains: 'Keyboard' }, within: 250 }, services(actor)); + observations.push({ name, text, expected, status: result.status, finding: result.finding ?? null, + observation: result.observation }); + assert.equal(result.status, expected, result.summary ?? name); + const visibleText = await page.locator('[data-testid="order"]').first() + .locator('[data-testid="order-status"]').innerText(); + if (expected === 'failed') { + assert.equal(result.finding?.kind, 'value-mismatch'); + assert.equal(result.finding?.fields.observed, visibleText); + } else assert.deepEqual(result.observation, { visible: true, statusText: 'pending', observedText: visibleText }); + } finally { await page.close(); } + }); + } + for (const [name, markup, step] of [ + ['strict-text', '· pending', { value: 'pending' }], + ['strict-input', '', { value: 'pending' }], + ['strict-protocol', 'pending', + { value: 'pending', attribute: 'data-state' }], + ] as const) { + await t.test(name, async () => { + const page = await browser.newPage(); + try { + await page.setContent(markup); + const result = await run({ do: 'expect', actor: 'a', testid: 'field', ...step, within: 250 }, + services({ page, loc: () => page.locator('[data-testid="field"]') })); + observations.push({ name, status: result.status, finding: result.finding ?? null }); + assert.equal(result.status, 'failed'); + assert.equal(result.finding?.kind, 'value-mismatch'); + } finally { await page.close(); } + }); + } +}); + +test('statusText is explicit and cannot replace input or protocol expectations', () => { + const step = { do: 'expect', actor: 'a', testid: 'order-status', statusText: 'pending', ignoreCase: true }; + assert.doesNotThrow(() => compileActionInput(step)); + for (const conflict of [{ value: 'pending' }, { containsText: 'pending' }, { attribute: 'data-state' }, + { absent: true }, { statusText: '' }]) { + assert.throws(() => compileActionInput({ ...step, ...conflict })); + } +}); + +test('statusText preserves existing input, textarea and select status values', async t => { + const browser = await chromium.launch({ headless: true }); + t.after(() => browser.close()); + for (const [name, markup, machineValue, expected] of [ + ['readonly-input', '', 'pending', 'passed'], + ['readonly-textarea', '', 'pending', 'passed'], + ['select', '', 'pending', 'passed'], + ['select-machine-id', '', 'p', 'passed'], + ['select-wrong-label', '', 'pending', 'failed'], + ] as const) await t.test(name, async () => { + const page = await browser.newPage(); + try { + await page.setContent(markup); + const locator = page.locator('[data-testid="order-status"]'); + const provided = services({ page, loc: () => locator }); + const step = { do: 'expect', actor: 'a', testid: 'order-status', within: 250 }; + assert.equal((await run({ ...step, value: machineValue }, provided)).status, 'passed'); + const result = await run({ ...step, statusText: 'pending' }, provided); + assert.equal(result.status, expected, result.summary ?? name); + if (expected === 'passed') { + assert.deepEqual(result.observation, { visible: true, statusText: 'pending', observedText: 'pending' }); + } else { + assert.equal(result.finding?.kind, 'value-mismatch'); + assert.equal(result.finding?.fields.observed, 'shipped'); + } + await locator.evaluate(node => { (node as HTMLInputElement).value = 'shipped'; }); + assert.equal((await run({ ...step, statusText: 'pending' }, provided)).status, 'failed'); + } finally { await page.close(); } + }); +}); + +test('statusText rejects non-text input values and mixed selected states', async t => { + const browser = await chromium.launch({ headless: true }); + t.after(() => browser.close()); + for (const type of ['checkbox', 'radio', 'password']) await t.test(type, async () => { + const page = await browser.newPage(); + try { + await page.setContent(``); + const provided = services({ page, loc: () => page.locator('[data-testid="order-status"]') }); + const step = { do: 'expect', actor: 'a', testid: 'order-status', within: 250 }; + assert.equal((await run({ ...step, value: 'pending' }, provided)).status, 'passed'); + assert.equal((await run({ ...step, statusText: 'pending' }, provided)).status, 'failed'); + } finally { await page.close(); } + }); + await t.test('multiple-selected-states', async () => { + const page = await browser.newPage(); + try { + await page.setContent(''); + const provided = services({ page, loc: () => page.locator('[data-testid="order-status"]') }); + const step = { do: 'expect', actor: 'a', testid: 'order-status', within: 250 }; + assert.equal((await run({ ...step, value: 'pending' }, provided)).status, 'passed'); + assert.equal((await run({ ...step, statusText: 'pending' }, provided)).status, 'failed'); + } finally { await page.close(); } + }); +}); diff --git a/tools/stack-bench/tests/browser-evidence.test.ts b/tools/stack-bench/tests/browser-evidence.test.ts new file mode 100644 index 00000000000..8a901c2d0fc --- /dev/null +++ b/tools/stack-bench/tests/browser-evidence.test.ts @@ -0,0 +1,15 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { settledLocatorCount } from '../src/evidence/browser-evidence.js'; + +test('a crashed page is never converted into an element count of zero', async () => { + // An optional locator timeout is a healthy absence. + const timeout = Object.assign(new Error('not visible'), { name: 'TimeoutError' }); + assert.equal(await settledLocatorCount({ waitFor: async () => { throw timeout; }, count: async () => 0 }, 10), 0); + const crash = new Error('locator.waitFor: Page crashed'); + const locator = { waitFor: async () => { throw crash; }, count: async () => 0 }; + await assert.rejects(() => settledLocatorCount(locator, 10), /Page crashed/); + // Count failures propagate after a locator becomes visible. + const closed = { waitFor: async () => {}, count: async () => { throw new Error('Target closed'); } }; + await assert.rejects(() => settledLocatorCount(closed, 10), /Target closed/); +}); diff --git a/tools/stack-bench/tests/browser-request.integration.ts b/tools/stack-bench/tests/browser-request.integration.ts new file mode 100644 index 00000000000..ef67c1103f6 --- /dev/null +++ b/tools/stack-bench/tests/browser-request.integration.ts @@ -0,0 +1,199 @@ +import assert from 'node:assert/strict'; +import { once } from 'node:events'; +import { writeFileSync } from 'node:fs'; +import { createServer } from 'node:http'; +import test from 'node:test'; +import { chromium, type APIRequestContext } from 'playwright'; +import { ActionInconclusive } from '../src/actions/action-contract.js'; +import type { Finding } from '../src/actions/action-findings.js'; +import { registerBrowserRequest, withBrowserRequest } from '../src/actions/browser-request.js'; + +function unavailable(error: unknown, reason: RegExp): boolean { + assert(error instanceof ActionInconclusive); + const finding = error.details.finding as Finding; + assert(finding.kind === 'replay-unavailable'); + assert.match(String(finding.fields.detail), reason); + return true; +} + +// Failure cases: a browser-only proxy breaks controller API calls; an old cookie +// snapshot uses the wrong account; copying the whole jar erases concurrent cookies; +// rotation/deletion or explicit forged headers change replay semantics; failed +// callbacks leak request contexts. Evaluator replies must stay outside page capture. +test('evaluator requests preserve actor cookies without the browser-only proxy', async t => { + const requests: { path: string; cookie: string; userAgent: string }[] = []; + const server = createServer((req, res) => { + requests.push({ path: req.url ?? '', cookie: req.headers.cookie ?? '', userAgent: req.headers['user-agent'] ?? '' }); + res.setHeader('Content-Type', 'application/json'); + if (req.url === '/reset') { req.socket.destroy(); return; } + if (req.url === '/rotate') res.setHeader('Set-Cookie', [ + 'session=rotated; Path=/; HttpOnly; SameSite=Lax', 'remove=; Path=/; Max-Age=0', + ]); + if (req.url === '/route-cookie' || req.url === '/drop-cookie') { + res.setHeader('Set-Cookie', 'route-session=accepted; Path=/; Max-Age=60; HttpOnly; SameSite=Lax'); + } + if (req.url === '/body') { + res.write('{"value":'); + setTimeout(() => res.end('"complete-evaluator-echo"}'), 40); + return; + } + res.end(JSON.stringify({ cookie: req.headers.cookie ?? '', userAgent: req.headers['user-agent'] ?? '' })); + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + t.after(() => { server.closeAllConnections(); server.close(); }); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const browser = await chromium.launch({ headless: true, executablePath: chromium.executablePath() }); + t.after(() => browser.close()); + const a = await browser.newContext({ proxy: { server: 'http://127.0.0.1:1', bypass: '<-loopback>' } }); + const b = await browser.newContext({ proxy: { server: 'http://127.0.0.1:1', bypass: '<-loopback>' } }); + const ordinary = await browser.newContext(); + t.after(async () => { await Promise.all([a.close(), b.close(), ordinary.close()]); }); + const page = await a.newPage(); + const userAgent = await page.evaluate(() => navigator.userAgent); + let offline = false; + registerBrowserRequest(a, userAgent, () => offline); + registerBrowserRequest(b, userAgent); + let pageResponses = 0; + page.on('response', () => pageResponses++); + const evidence: unknown[] = []; + t.after(() => { + if (process.env.STACK_BENCH_BROWSER_REQUEST_EVIDENCE) writeFileSync(process.env.STACK_BENCH_BROWSER_REQUEST_EVIDENCE, + JSON.stringify({ browser: browser.version(), evidence, requests, pageResponses }, null, 2)); + }); + await t.test('controller request avoids unreachable browser proxy and consumes the full body', async () => { + await assert.rejects(a.request.get(`${url}/body`, { timeout: 500 }), /ECONNREFUSED|proxy|timeout/i); + let used: APIRequestContext | undefined; + const result = await withBrowserRequest(a.request, async api => { + used = api; + const response = await api.get(`${url}/body`, { timeout: 1000 }); + assert.equal(response.status(), 200); + return response.json(); + }); + assert.deepEqual(result, { value: 'complete-evaluator-echo' }); + assert(used); + await assert.rejects(used.get(`${url}/body`, { timeout: 1000 }), /disposed|closed/i); + assert.equal(pageResponses, 0, 'Evaluator echoes must not enter browser privacy evidence'); + assert.equal(requests.at(-1)?.userAgent, userAgent); + evidence.push({ case: 'full-body', result, pageResponses }); + }); + await t.test('current cookies rotate and delete without losing unrelated concurrent cookies', async () => { + await a.addCookies([{ name: 'session', value: 'initial', url }, { name: 'remove', value: 'old', url }]); + const result = await withBrowserRequest(a.request, async api => { + const response = await api.get(`${url}/rotate`, { timeout: 1000 }); + const body = await response.json(); + await a.addCookies([{ name: 'concurrent', value: 'keep', url }]); + return body; + }); + assert.match(result.cookie, /(?:^|; )session=initial(?:;|$)/); + const cookies = await a.cookies(url); + assert.equal(cookies.find(c => c.name === 'session')?.value, 'rotated'); + assert.equal(cookies.find(c => c.name === 'remove'), undefined); + assert.equal(cookies.find(c => c.name === 'concurrent')?.value, 'keep'); + const later = await withBrowserRequest(a.request, async api => (await api.get(`${url}/echo`, { timeout: 1000 })).json()); + assert.match(later.cookie, /session=rotated/); + evidence.push({ case: 'cookie-updates', result, cookies, later }); + }); + await t.test('another actor stays isolated and explicit forged Cookie is preserved', async () => { + const isolated = await withBrowserRequest(b.request, async api => (await api.get(`${url}/echo`, { timeout: 1000 })).json()); + assert.equal(isolated.cookie, ''); + const forged = await withBrowserRequest(a.request, async api => (await api.get(`${url}/echo`, { + timeout: 1000, headers: { Cookie: 'session=forged' }, maxRedirects: 0, maxRetries: 0, + })).json()); + assert.equal(forged.cookie, 'session=forged'); + assert.equal((await a.cookies(url)).find(c => c.name === 'session')?.value, 'rotated'); + evidence.push({ case: 'isolation', isolated, forged }); + }); + await t.test('partitioned cookies are not flattened into an evaluator cookie jar', async () => { + await b.addCookies([{ name: 'partitioned', value: 'private', domain: '127.0.0.1', path: '/', + secure: true, sameSite: 'None', partitionKey: 'https://example.test' }]); + assert((await b.cookies()).some(cookie => cookie.partitionKey), 'The real browser must hold a partitioned cookie'); + let entered = false; + await assert.rejects(withBrowserRequest(b.request, async () => { entered = true; }), + error => unavailable(error, /partition/i)); + assert.equal(entered, false, 'Unsupported cookie identity must fail before sending a replay'); + }); + await t.test('a concurrent change to the same cookie is not overwritten', async () => { + await a.addCookies([{ name: 'session', value: 'before-race', url }]); + await assert.rejects(withBrowserRequest(a.request, async api => { + await (await api.get(`${url}/rotate`, { timeout: 1000 })).text(); + await a.addCookies([{ name: 'session', value: 'newer-browser-session', url }]); + }), error => unavailable(error, /cookie.*chang|concurrent|reconcil/i)); + assert.equal((await a.cookies(url)).find(c => c.name === 'session')?.value, 'newer-browser-session'); + }); + await t.test('unregistered contexts use their original request unchanged', async () => { + await withBrowserRequest(ordinary.request, async api => { + assert.equal(api, ordinary.request); + assert.equal((await api.get(`${url}/echo`, { timeout: 1000 })).status(), 200); + }); + }); + await t.test('an active network cut cannot be bypassed by evaluator requests', async () => { + const before = requests.length; + let entered = false; + offline = true; + try { + await assert.rejects(withBrowserRequest(a.request, async api => { + entered = true; + await api.get(`${url}/echo`, { timeout: 1000 }); + }), error => unavailable(error, /offline|network.*cut/i)); + assert.equal(entered, false); + assert.equal(requests.length, before); + } finally { offline = false; } + }); + await t.test('callback and network failures dispose their temporary request context', async () => { + for (const network of [false, true]) { + let used: APIRequestContext | undefined; + await assert.rejects(withBrowserRequest(a.request, async api => { + used = api; + if (network) await api.get(`${url}/reset`, { maxRetries: 0, timeout: 1000 }); + throw new Error('fixture callback failed'); + }), network ? /socket|reset|closed/i : /fixture callback failed/); + assert(used); + await assert.rejects(used.get(`${url}/echo`, { timeout: 1000 }), /disposed|closed/i); + } + }); + await t.test('a fulfilled response lets the browser apply relative-expiry cookies once', async () => { + const context = await browser.newContext(); + try { + registerBrowserRequest(context, userAgent); + const routed = await context.newPage(); + let routeError: unknown; + await context.route(`${url}/route-cookie`, async route => { + try { + await withBrowserRequest(context.request, async api => { + const response = await api.fetch(route.request(), { timeout: 1000 }); + const body = await response.body(); + await new Promise(resolve => setTimeout(resolve, 1200)); + await route.fulfill({ status: response.status(), headers: response.headers(), body }); + await routed.waitForLoadState('domcontentloaded'); + }, false); + } catch (error) { routeError = error; } + }); + await routed.goto(`${url}/route-cookie`, { waitUntil: 'domcontentloaded', timeout: 5000 }); + await context.unrouteAll({ behavior: 'wait' }); + assert.equal(routeError, undefined); + const cookies = await context.cookies(url); + assert.equal(cookies.find(cookie => cookie.name === 'route-session')?.value, 'accepted'); + evidence.push({ case: 'fulfilled-cookie', cookies }); + } finally { await context.close(); } + }); + await t.test('a dropped routed response does not apply evaluator response cookies', async () => { + const context = await browser.newContext(); + try { + registerBrowserRequest(context, userAgent); + const routed = await context.newPage(); + let routeError: unknown; + await context.route(`${url}/drop-cookie`, async route => { + try { + await withBrowserRequest(context.request, async api => { + await (await api.fetch(route.request(), { timeout: 1000 })).body(); + await route.abort('failed'); + }, false); + } catch (error) { routeError = error; } + }); + await assert.rejects(routed.goto(`${url}/drop-cookie`, { timeout: 5000 }), /ERR_FAILED|failed/i); + await context.unrouteAll({ behavior: 'wait' }); + assert.equal(routeError, undefined); + assert.equal((await context.cookies(url)).find(cookie => cookie.name === 'route-session'), undefined); + } finally { await context.close(); } + }); +}); diff --git a/tools/stack-bench/tests/browser-socket-delivery.test.ts b/tools/stack-bench/tests/browser-socket-delivery.test.ts new file mode 100644 index 00000000000..e56601e6976 --- /dev/null +++ b/tools/stack-bench/tests/browser-socket-delivery.test.ts @@ -0,0 +1,28 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import type { BrowserContext, WebSocketRoute } from 'playwright'; +import { connectBrowserSocket, installSocketDeliveryGate } from '../src/actions/browser-socket-delivery.js'; + +test('a socket opened before the delivery gate is installed is still gated', () => { + const sent: unknown[] = []; + const server = { send: (data: unknown) => { sent.push(data); } } as unknown as WebSocketRoute; + const client = { connectToServer: () => server } as unknown as WebSocketRoute; + const context = { once: () => {} } as unknown as BrowserContext; + const delivery = connectBrowserSocket(context, client); + // No gate yet: frames pass and nothing is recorded. + assert.equal(delivery.receive('before'), true); + const seen: string[] = []; + let created = 0; + installSocketDeliveryGate(context, () => { + created++; + return { sent: data => { seen.push(`sent ${String(data)}`); }, + received: data => { seen.push(`dropped ${String(data)}`); return false; }, + closed: () => { seen.push('closed'); } }; + }); + delivery.send('write'); + assert.equal(delivery.receive('reply'), false); + delivery.closed(); + assert.deepEqual(sent, ['write']); + assert.deepEqual(seen, ['sent write', 'dropped reply', 'closed']); + assert.equal(created, 1, 'one gate per socket'); +}); diff --git a/tools/stack-bench/tests/build-container-inspection.test.ts b/tools/stack-bench/tests/build-container-inspection.test.ts new file mode 100644 index 00000000000..a0c290ae218 --- /dev/null +++ b/tools/stack-bench/tests/build-container-inspection.test.ts @@ -0,0 +1,120 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { hasRequiredBuildContainerIsolation, inspectBuildContainer, parseCgroupResources, + parsePublishedPorts, sameHostPath, waitForBuildContainerReady } + from '../container/build-container-inspection.js'; +import type { InspectedBuildContainer } from '../container/build-container-inspection.js'; + +const dockerResult = (status: number, stdout = '', stderr = '') => ({ + status, stdout, stderr, pid: 1, output: [], signal: null, +}); + +test('build-container inspection distinguishes absence from Docker failure', () => { + assert.equal(inspectBuildContainer('missing', { + execute: () => dockerResult(1, '', 'Error: No such object: missing'), + }), null); + assert.throws(() => inspectBuildContainer('leased', { + execute: () => dockerResult(1, '', 'Cannot connect to the Docker daemon'), + }), /cannot inspect build container leased/); +}); + +test('readiness fails immediately on the exact exited container and redacts its logs without cleanup', () => { + for (const oom of [false, true]) { + const calls: string[][] = []; + assert.throws(() => waitForBuildContainerReady('owned-id', '/deps/.ready', 'SDK staging', { + execute: (_command, args) => { + calls.push([...args]); + if (args[0] === 'exec') return dockerResult(1); + if (args[0] === 'inspect') return dockerResult(0, + JSON.stringify({ Status: 'exited', ExitCode: oom ? 137 : 7, OOMKilled: oom })); + return dockerResult(0, 'startup failed\n', '--password sentinel-secret\n'); + }, + }), error => { + assert.ok(error instanceof Error); + assert.match(error.message, /owned-id exited before SDK staging/); + assert.ok(error.message.includes(`exit code ${oom ? 137 : 7}; OOMKilled=${oom}`)); + assert.match(error.message, /startup failed/); + assert.ok(!error.message.includes('sentinel-secret')); + return true; + }); + assert.deepEqual(calls, [ + ['exec', 'owned-id', 'test', '-f', '/deps/.ready'], + ['inspect', '--format', '{{json .State}}', 'owned-id'], + ['logs', '--tail', '100', 'owned-id'], + ]); + } + waitForBuildContainerReady('ready-id', '/deps/.ready', 'SDK staging', { + execute: (_command, args) => { + assert.deepEqual(args, ['exec', 'ready-id', 'test', '-f', '/deps/.ready']); + return dockerResult(0); + }, + }); +}); + +test('host path comparison preserves Linux case sensitivity', () => { + assert.equal(sameHostPath('/work/App', '/work/app', 'linux'), false); + assert.equal(sameHostPath('C:\\work\\App', 'c:\\work\\app', 'win32'), true); +}); + +test('published ports must be valid and unique', () => { + assert.deepEqual(parsePublishedPorts('3000, 5173'), ['3000', '5173']); + assert.throws(() => parsePublishedPorts('3000,3000'), /must not contain duplicates/); + assert.throws(() => parsePublishedPorts('0'), /integers from 1 through 65535/); + assert.throws(() => parsePublishedPorts('3000x'), /integers from 1 through 65535/); +}); + +test('cgroup resources retain OOM and PID-limit events even when the coding CLI succeeds', () => { + const sample = '[memory.events]\noom 2\noom_kill 1\n[memory.current]\n123\n' + + '[memory.peak]\n456\n[memory.max]\n4294967296\n[pids.current]\n7\n' + + '[pids.peak]\n512\n[pids.max]\n512\n[pids.events]\nmax 129\n'; + const expected = { + buildContainerMemory: { currentBytes: 123, peakBytes: 456, limitBytes: 4 * 1024 ** 3, + oomEvents: 2, oomKillEvents: 1 }, + buildContainerPids: { current: 7, peak: 512, limit: 512, limitEvents: 129 }, + }; + assert.deepEqual(parseCgroupResources(sample), expected); + assert.deepEqual(parseCgroupResources(sample.replaceAll('\n', '\r\n')), expected); + assert.deepEqual(parseCgroupResources('[memory.max]\nmax\n[pids.current]\n-1\n' + + '[pids.peak]\n9007199254740992\n[pids.max]\nmax\n'), { + buildContainerMemory: { currentBytes: null, peakBytes: null, limitBytes: null, + oomEvents: null, oomKillEvents: null }, + buildContainerPids: { current: null, peak: null, limit: null, limitEvents: null }, + }); +}); + +test('build-container isolation validates the effective Docker configuration', () => { + const container: InspectedBuildContainer = { + id: 'container', + image: 'image', + running: true, + networkMode: 'bridge', + readonlyRootfs: true, + tmpfs: { '/tmp': 'rw,nosuid,nodev' }, + capAdd: ['CHOWN'], + capDrop: ['ALL'], + securityOpt: ['no-new-privileges'], + pidsLimit: 32, + nanoCpus: 2_000_000_000, + memoryBytes: 1024, + memorySwapBytes: 1024, + mounts: [{ type: 'bind', source: '/workspace', name: null, + destination: '/app', readOnly: false }], + unsafeCredentialExposure: false, + }; + const requirements = { + expectedMounts: [{ kind: 'bind' as const, source: '/workspace', target: '/app', readOnly: false }], + requiredTmpfs: { '/tmp': 'rw,nosuid,nodev' }, + requiredCapabilities: ['CHOWN'], + pidsLimit: 32, + cpuCount: 2, + memoryBytes: 1024, + memorySwapBytes: 1024, + image: 'image', + }; + + assert.equal(hasRequiredBuildContainerIsolation(container, requirements), true); + assert.equal(hasRequiredBuildContainerIsolation({ ...container, readonlyRootfs: false }, requirements), false); + assert.equal(hasRequiredBuildContainerIsolation({ ...container, capDrop: [] }, requirements), false); + assert.equal(hasRequiredBuildContainerIsolation({ ...container, memoryBytes: 2048 }, requirements), false); +}); diff --git a/tools/stack-bench/tests/build-container-reconciliation.test.ts b/tools/stack-bench/tests/build-container-reconciliation.test.ts new file mode 100644 index 00000000000..9a9011d51e2 --- /dev/null +++ b/tools/stack-bench/tests/build-container-reconciliation.test.ts @@ -0,0 +1,56 @@ +import assert from 'node:assert/strict'; +import type { SpawnSyncReturns } from 'node:child_process'; +import test from 'node:test'; + +import { BUILD_CONTAINER_CREATION_LABEL, buildContainerName, removeFailedBuildContainer } + from '../container/reconcile-build-container.js'; + +const ID = 'a'.repeat(64); + +test('build container names are stable per lease and independent of app directory parents', () => { + const first = { runId: 'first-run', resources: {} }; + const second = { runId: 'second-run', resources: {} }; + assert.notEqual(buildContainerName(first), buildContainerName(second)); + assert.equal(buildContainerName(first), buildContainerName(first)); + assert.match(buildContainerName(first), /^sb-[a-f0-9]{16}-build$/); + assert.equal(buildContainerName({ ...first, resources: { + buildContainer: { name: 'existing-leased-container' }, + } }), 'existing-leased-container'); +}); +type DockerCall = [command: string, args: readonly string[]]; + +function dockerResult(stdout: string): SpawnSyncReturns { + return { pid: 0, output: [null, stdout, ''], stdout, stderr: '', status: 0, signal: null }; +} + +test('failed build-container cleanup reconciles a matching creation label', () => { + for (const createdId of [null, `${ID}\n`]) { + const calls: DockerCall[] = []; + const result = removeFailedBuildContainer({ + containerName: 'stack-bench-run', creationToken: 'token', createdId, + execute(command, args) { + calls.push([command, args]); + if (args[0] === 'inspect') { + assert.equal(args[2], `{{.Id}} {{index .Config.Labels "${BUILD_CONTAINER_CREATION_LABEL}"}}`); + return dockerResult(`${ID} token\n`); + } + return dockerResult(ID); + }, + }); + assert.deepEqual(calls.map(([, args]) => args[0]), createdId ? ['rm'] : ['inspect', 'rm']); + assert.deepEqual(calls.at(-1), ['docker', ['rm', '-f', ID]]); + assert.deepEqual(result, { removed: true, absent: false, id: ID }); + } +}); + +test('failed build-container cleanup never removes a same-name container with another label', () => { + const calls: DockerCall[] = []; + assert.throws(() => removeFailedBuildContainer({ + containerName: 'stack-bench-run', creationToken: 'token', + execute(command, args) { + calls.push([command, args]); + return dockerResult(`${ID} another-token\n`); + }, + }), /creation identity does not match/); + assert.deepEqual(calls.map(([, args]) => args[0]), ['inspect']); +}); diff --git a/tools/stack-bench/tests/build-container-recovery.integration.ts b/tools/stack-bench/tests/build-container-recovery.integration.ts new file mode 100644 index 00000000000..3152f2d0ee3 --- /dev/null +++ b/tools/stack-bench/tests/build-container-recovery.integration.ts @@ -0,0 +1,105 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import type { SpawnSyncOptionsWithStringEncoding } from 'node:child_process'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { inspectBuildContainer } from '../container/build-container-inspection.js'; +import { buildContainerName } from '../container/reconcile-build-container.js'; +import { BUILD_CONTAINER_RESOURCE_LIMITS, DEFAULT_BUILD_IMAGE } from '../src/composition/product-config.js'; +import { createBackendLease, writeBackendLease } from '../src/runtime/backend-lease.js'; +import { compiledEntrypoint } from '../src/package-root.js'; + +const required = (value: T | null | undefined, description: string): T => { + if (value === null || value === undefined) throw new Error(`${description} is required`); + return value; +}; +const record = (text: string, description: string): Record => { + const value: unknown = JSON.parse(text); + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`${description} must be an object`); + } + return value as Record; +}; +const stringField = (value: Record, field: string, description: string): string => { + const result = value[field]; + if (typeof result !== 'string') throw new Error(`${description}.${field} must be a string`); + return result; +}; + +test('Docker replaces a stopped leased build container and preserves its app mount', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-recovery-docker-')); + const app = join(root, 'app'); + const leasePath = join(root, 'lease.json'); + const runId = required(root.split(/[\\/]/).at(-1), 'temporary directory name'); + const containerName = buildContainerName({ runId, resources: {} }); + const docker = (args: readonly string[], + options: Omit = {}) => spawnSync('docker', args, + { encoding: 'utf8', timeout: 120_000, ...options }); + try { + mkdirSync(app); + const databaseContainer = docker(['inspect', '--format', '{{.Id}}', 'stack-bench-mongodb']); + assert.equal(databaseContainer.status, 0, databaseContainer.stderr); + const lease = createBackendLease({ runId, backend: 'mongodb', + track: 'ecommerce', runIndex: 91, database: 'app_ecom_run91', + container: { name: 'stack-bench-mongodb', id: required(databaseContainer.stdout, + 'database container identifier').trim() } }); + lease.state = 'active'; + writeBackendLease(leasePath, lease); + const env = { ...process.env, STACK_BENCH_LEASE: leasePath, + STACK_BENCH_LEASE_TOKEN: lease.ownershipToken }; + const build = compiledEntrypoint('container', 'run-build.js'); + const baseArgs = [build, '--prepare-only', '--app', app, '--backend', 'mongodb', '--image', + process.env.STACK_BENCH_BUILD_IMAGE ?? DEFAULT_BUILD_IMAGE]; + const runBuild = (args: readonly string[]) => spawnSync(process.execPath, args, + { encoding: 'utf8', env, timeout: 180_000 }); + + const first = runBuild(baseArgs); + assert.equal(first.status, 0, first.stderr); + const firstResult = record(required(first.stdout, 'first build result'), 'first build result'); + const firstContainerName = stringField(firstResult, 'containerName', 'first build result'); + const firstIdentity = stringField(firstResult, 'identity', 'first build result'); + const inspected = inspectBuildContainer(firstContainerName); + assert(inspected); + assert.equal(inspected.readonlyRootfs, true); + assert.deepEqual([...inspected.capAdd].sort(), + ['CHOWN', 'DAC_OVERRIDE', 'FOWNER', 'KILL', 'SETGID', 'SETUID']); + assert(inspected.capDrop.includes('ALL')); + assert(inspected.securityOpt.includes('no-new-privileges')); + assert.equal(inspected.pidsLimit, BUILD_CONTAINER_RESOURCE_LIMITS.pids); + assert.equal(inspected.tmpfs['/tmp'], 'rw,nosuid,nodev,mode=1777'); + assert.equal(inspected.tmpfs['/home/developer'], + 'rw,nosuid,nodev,uid=10001,gid=10001,mode=0700'); + assert.equal(inspected.tmpfs['/home/developer/.claude'], + 'rw,nosuid,nodev,uid=10001,gid=10001,mode=0700'); + assert.equal(inspected.tmpfs['/deps'], 'rw,exec,nosuid,nodev,mode=0755'); + assert.equal(inspected.tmpfs['/run/application'], 'rw,nosuid,nodev,mode=0700'); + assert.equal(inspected.tmpfs['/root'], undefined); + const agentCaps = docker(['exec', '--user', '10001:10001', firstContainerName, + 'sh', '-c', 'grep "^CapEff:" /proc/self/status']); + assert.equal(agentCaps.status, 0, agentCaps.stderr); + assert.match(agentCaps.stdout, /CapEff:\s+0+\s*$/); + writeFileSync(join(app, 'preserved.txt'), 'preserved\n'); + assert.equal(docker(['stop', firstContainerName]).status, 0); + + const second = runBuild([build, '--recover-stopped-container', ...baseArgs.slice(1)]); + assert.equal(second.status, 0, second.stderr); + const secondResult = record(required(second.stdout, 'second build result'), 'second build result'); + const secondContainerName = stringField(secondResult, 'containerName', 'second build result'); + const secondIdentity = stringField(secondResult, 'identity', 'second build result'); + assert.notEqual(firstIdentity, secondIdentity); + const mounted = docker(['exec', secondContainerName, 'cat', '/app/preserved.txt']); + assert.equal(mounted.status, 0, mounted.stderr); + assert.equal(mounted.stdout.trim(), 'preserved'); + const reused = runBuild(baseArgs); + assert.equal(reused.status, 0, reused.stderr); + assert.equal(stringField(record(required(reused.stdout, 'reused build result'), + 'reused build result'), 'identity', 'reused build result'), secondIdentity); + } finally { + const exact = docker(['inspect', '--format', '{{.Id}}', containerName]); + if (exact.status === 0 && exact.stdout.trim()) docker(['rm', '-f', exact.stdout.trim()]); + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/tools/stack-bench/tests/calibration-compile-cache.test.ts b/tools/stack-bench/tests/calibration-compile-cache.test.ts new file mode 100644 index 00000000000..5074c865246 --- /dev/null +++ b/tools/stack-bench/tests/calibration-compile-cache.test.ts @@ -0,0 +1,27 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { resolveCalibrationForRelease } from '../src/composition/calibration-compiler.js'; +import type { CalibrationPlan } from '../src/composition/calibration-compiler.js'; +import { requireRecipeRelease as resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { loadTrack } from '../src/composition/tracks.js'; + +test('a shared calibration compile returns the fresh result and is not recompiled', () => { + const track = loadTrack('ecommerce'); + const { release } = resolveRecipeRelease(track, 3, 'ecommerce.progression-catalog'); + const options = { trackRoot: track.dir, alias: 'L3' }; + const compiled = new Map(); + const fresh = resolveCalibrationForRelease(release, options); + const shared = resolveCalibrationForRelease(release, { ...options, compiled }); + assert(fresh && shared); + assert.deepEqual(shared, fresh); + assert(compiled.size > 0); + + // Callers own their copy; a change must not reach the next reader. + shared.qualification.checks = []; + assert.deepEqual(resolveCalibrationForRelease(release, { ...options, compiled }), fresh); + + // The saved outcome is reused, including a failure. + for (const key of compiled.keys()) compiled.set(key, new Error('saved failure')); + assert.throws(() => resolveCalibrationForRelease(release, { ...options, compiled }), /saved failure/); +}); diff --git a/tools/stack-bench/tests/calibration-compiler.contract.ts b/tools/stack-bench/tests/calibration-compiler.contract.ts new file mode 100644 index 00000000000..2d8a2162482 --- /dev/null +++ b/tools/stack-bench/tests/calibration-compiler.contract.ts @@ -0,0 +1,229 @@ +import assert from 'node:assert/strict'; +import { readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { calibrationQualificationIdentity, calibrationQualificationRelease, compileCalibrationDefinition, + compileCalibrationFile, currentLevelPoints, hasExactSelectedPackRuntime, + resolveCalibrationForRelease, validateQualificationEvidenceArtifact } from '../src/composition/calibration-compiler.js'; +import { createArtifact } from '../src/evidence/artifacts.js'; +import { buildRecipeRelease, executionPlanForRelease, + requireRecipeRelease } from '../src/composition/recipe-release.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { loadTrack } from '../src/composition/tracks.js'; +import { loadReferenceRegistry, validateReferenceRegistry } from '../src/references/reference-fixtures.js'; + +const TRACK = loadTrack('ecommerce'); +const CALIBRATIONS = join(TRACK.dir, 'composition', 'calibrations'); +const CALIBRATION = join(CALIBRATIONS, 'dependency-l3.json'); +const BACKENDS = ['mongodb', 'postgres', 'spacetime']; +const PRECONDITION_CONTROLS = [ + ['ecommerce.spec.concurrency-safety.restock-race.202-control', 'precondition'], + ['ecommerce.spec.external-data-sync.external-stock.901b', 'precondition'], +]; +const EXPECTED_CALIBRATIONS: Record = { + 'dependency-l3.json': { id: 'ecommerce.dependency-l3-calibration', recipe: 'progression-catalog', + stacks: ['convex', ...BACKENDS, 'supabase'], controls: [] }, + 'dependency-l6.json': { id: 'ecommerce.dependency-l6-calibration', recipe: 'progression-catalog', + stacks: BACKENDS, controls: [] }, + 'sequential-l1.json': { id: 'ecommerce.sequential-l1-calibration', recipe: 'sequential-l1', + stacks: BACKENDS, controls: PRECONDITION_CONTROLS }, + 'sequential-l2.json': { id: 'ecommerce.sequential-l2-calibration', recipe: 'sequential-l2', + stacks: BACKENDS, controls: PRECONDITION_CONTROLS }, +}; + +function calibrationSource(): unknown { + return JSON.parse(readFileSync(CALIBRATION, 'utf8')); +} + +function current() { + const binding = requireRecipeRelease(TRACK, 3, 'ecommerce.progression-catalog'); + const plan = compileCalibrationFile(CALIBRATION, { + trackRoot: TRACK.dir, + stackBenchRoot: STACK_BENCH_ROOT, + release: binding.release, + }); + return { ...calibrationQualificationRelease(plan, binding.release, binding.execution), plan }; +} + +test('qualification runtime must cover the exact selected pack set', () => { + const release = { checkCatalog: [{ packId: 'accounts' }, { packId: 'accounts' }, + { packId: 'orders' }] }; + assert.equal(hasExactSelectedPackRuntime({ packs: [ + { id: 'accounts', exceeded: false }, { id: 'orders', exceeded: false }, + ] }, release), true); + assert.equal(hasExactSelectedPackRuntime({ packs: [ + { id: 'accounts', exceeded: false }, { id: 'future', exceeded: false }, + ] }, release), false); +}); + +test('every current calibration binds stable authored identities', () => { + const registry = loadReferenceRegistry(); + assert.deepEqual(validateReferenceRegistry(registry).issues, []); + assert.deepEqual(readdirSync(CALIBRATIONS).filter(name => name.endsWith('.json')).sort(), + Object.keys(EXPECTED_CALIBRATIONS).sort()); + for (const [name, expected] of Object.entries(EXPECTED_CALIBRATIONS)) { + const path = join(CALIBRATIONS, name); + const { selection } = JSON.parse(readFileSync(path, 'utf8')) as { selection: { alias: string } }; + const binding = requireRecipeRelease(TRACK, Number(selection.alias.slice(1)), + `ecommerce.${expected.recipe}`); + const plan = compileCalibrationFile(path, { trackRoot: TRACK.dir, stackBenchRoot: STACK_BENCH_ROOT, + release: binding.release }); + const { release } = calibrationQualificationRelease(plan, binding.release, binding.execution); + assert.equal(plan.id, expected.id); + assert.deepEqual(plan.recipe, { + path: `composition/recipes/${expected.recipe}.json`, + id: release.id, + meaningSha256: release.meaningSha256, + executionSha256: release.executionSha256, + contentSha256: release.contentSha256, + }, name); + assert.deepEqual(plan.qualification.stacks, expected.stacks, name); + // Only the dependency L3 calibration has retained qualification evidence. Targeted + // reruns add entries, but together they cover exactly each stack's reference and + // mutation scopes plus the null control. + assert.deepEqual([...new Set(plan.qualification.evidence.map(entry => + `${entry.kind}:${entry.stack ?? ''}:${entry.repetition}`))].sort(), + name === 'dependency-l3.json' ? [...expected.stacks.flatMap(stack => + [`mutation:${stack}:1`, `reference:${stack}:1`]), 'null::1'].sort() : [], name); + assert.deepEqual(plan.references.entries.map(reference => reference.backend).sort(), expected.stacks, name); + assert.deepEqual(plan.mutations.map(mutation => mutation.backend).sort(), expected.stacks, name); + assert.match(plan.contentSha256, /^[a-f0-9]{64}$/); + assert.match(plan.qualificationSha256, /^[a-f0-9]{64}$/); + assert.deepEqual(calibrationQualificationIdentity(plan), { + id: plan.id, + contentSha256: plan.qualificationSha256, + }); + assert.deepEqual(plan.controls.map(control => [control.stableKey, control.role]), expected.controls, name); + const scored = new Set(release.checkCatalog.filter(check => check.points > 0) + .map(check => check.stableKey)); + assert(scored.size > 0, name); + for (const reference of plan.references.entries) { + assert.equal(reference.id, `ecommerce-reference-${reference.backend}`, name); + } + for (const mutation of plan.mutations) { + const covered = new Set(mutation.targets.flatMap(target => target.stableKeys)); + assert.deepEqual([...scored].filter(key => !covered.has(key)), [], `${name} ${mutation.backend}`); + const fixture = registry.fixtures.find(entry => entry.id === `ecommerce-reference-${mutation.backend}`); + assert(fixture?.recipes, `${name} ${mutation.backend}`); + assert(fixture.recipes.includes(release.id), `${name} ${mutation.backend}`); + assert.deepEqual(fixture.mutationManifests, [mutation.path], `${name} ${mutation.backend}`); + } + } +}); + +test('covered depth metadata preserves the qualification identity', () => { + const plan = compileCalibrationFile(join(TRACK.dir, 'composition/calibrations/dependency-l3.json'), + { trackRoot: TRACK.dir, stackBenchRoot: STACK_BENCH_ROOT, + release: requireRecipeRelease(TRACK, 3, 'ecommerce.progression-catalog').release }); + assert.deepEqual(plan.selection.coveredAliases, ['L1', 'L2', 'L3']); + const exactDepthOnly = { ...plan, selection: { ...plan.selection } }; + delete exactDepthOnly.selection.coveredAliases; + assert.deepEqual(calibrationQualificationIdentity(plan), + calibrationQualificationIdentity(exactDepthOnly)); +}); + +test('qualification accepts emitted artifact hashes and rejects mismatched identities', () => { + const { release, plan, execution } = current(); + const reference = plan.references.entries[0]!; + const calibration = { ...plan, qualification: { ...plan.qualification, runner: undefined } }; + const context = { calibration, qualificationIdentity: calibrationQualificationIdentity(plan), + release, references: plan.references.entries, stackBenchRoot: STACK_BENCH_ROOT, + execution, enforceQualificationScope: false }; + for (const kind of ['reference', 'mutation'] as const) { + const repetitions = kind === 'reference' ? plan.qualification.referenceRepetitions + : plan.qualification.mutationRepetitions; + const artifact = createArtifact({ id: `qualification-${kind}`, kind: 'reference_qualification', + identities: { recipe: { id: release.id, sha256: release.contentSha256 }, + calibration: { id: plan.id, sha256: plan.qualificationSha256 }, + fixture: { id: reference.id, sha256: reference.sourceSha256 }, + stackAdapter: { id: reference.backend, sha256: null } }, + payload: { fixture: reference.id, fixtureSha256: reference.sourceSha256, + requiredRepetitions: repetitions, isolation: 'docker', mutationControl: kind === 'mutation', + ok: true, stable: true, sameImage: true, sameHarness: true, harnessSha256: 'a'.repeat(64), + qualifiedCheckKeys: release.checkCatalog.map(check => check.stableKey), + runs: Array.from({ length: repetitions }, (_, index) => ({ repetition: index + 1, + ok: true, processError: null, outcome: 'passed', failures: [], + score: `${release.scoring.points}/${release.scoring.points}`, criteria: release.scoring.checks, + zeroPointCriteria: release.checkCatalog.filter(check => check.points === 0).length, + imageId: 'sha256:' + 'b'.repeat(64), + harnessSha256Before: 'a'.repeat(64), harnessSha256After: 'a'.repeat(64), + packRuntime: { packs: [...new Set(release.checkCatalog.map(check => check.packId))] + .map(id => ({ id, exceeded: false })) }, + mutations: kind === 'mutation' ? { caught: 1, total: 1 } : null })) } }); + const entry = { kind, stack: reference.backend, repetition: 1, path: 'test.json', sha256: 'c'.repeat(64) }; + assert.doesNotThrow(() => validateQualificationEvidenceArtifact(artifact, entry, context)); + const timing = createArtifact({ ...artifact, + payload: { ...artifact.payload, diagnostic: true, timingOnly: true } }); + assert.throws(() => validateQualificationEvidenceArtifact(timing, entry, context), /diagnostic evidence/); + timing.payload.diagnostic = false; + assert.throws(() => validateQualificationEvidenceArtifact(timing, entry, context), /diagnostic evidence/); + for (const key of ['recipe', 'calibration', 'fixture'] as const) { + for (const field of ['id', 'sha256'] as const) { + const changed = structuredClone(artifact); + changed.identities[key]![field] = field === 'id' ? 'wrong-id' : '0'.repeat(64); + assert.throws(() => validateQualificationEvidenceArtifact(changed, entry, context), /identity|identities/); + } + } + } +}); + +test('null qualification accepts blocked setup but rejects unmeasured or contradictory results', () => { + const { release, plan } = current(); + const scored = release.checkCatalog.filter(check => check.points > 0); + const zero = release.checkCatalog.length - scored.length; + const totals = { criteria: scored.length, points: release.scoring.points }; + const artifact = createArtifact({ id: 'null-qualification', kind: 'null_control', + identities: { recipe: { id: release.id, sha256: release.contentSha256 }, + calibration: { id: plan.id, sha256: plan.qualificationSha256 } }, + payload: { ok: true, tracks: [release.track], summary: { ...totals, expectedFailures: totals, + vacuousPasses: { criteria: 0, points: 0 }, oracleGaps: { criteria: 0, points: 0 }, + unscored: { criteria: zero, passed: 0, failed: zero, inconclusive: 0 } }, + criteria: scored.map(check => ({ scenario: check.source, feature: check.featureId, + criterion: check.criterionId, track: release.track, level: 3, points: check.points, + status: 'expected_fail', evidenceStatus: 'blocked', failureStage: 'setup' })) } }); + const entry = { kind: 'null' as const, repetition: 1, path: 'null.json', sha256: 'a'.repeat(64) }; + const context = { calibration: { ...plan, qualification: { ...plan.qualification, runner: undefined } }, + qualificationIdentity: calibrationQualificationIdentity(plan), release, + references: plan.references.entries, execution: [], stackBenchRoot: STACK_BENCH_ROOT, + enforceQualificationScope: false }; + assert.doesNotThrow(() => validateQualificationEvidenceArtifact(artifact, entry, context)); + for (const patch of [{ evidenceStatus: 'inconclusive' }, { evidenceStatus: 'harness_failure' }, + { evidenceStatus: 'passed' }, { failureStage: 'assertion' }, { failureStage: null }]) { + const changed = structuredClone(artifact); + Object.assign(changed.payload.criteria[0]!, patch); + assert.throws(() => validateQualificationEvidenceArtifact(changed, entry, context), /invalid null result/); + } +}); + +test('calibration identity changes when selected checks change', () => { + const value = compileCalibrationDefinition(calibrationSource()); + const identityInput = { ...value, mutations: value.mutations.map(mutation => ({ + ...mutation, executionSha256: 'a'.repeat(64), + })) }; + const first = calibrationQualificationIdentity(identityInput); + identityInput.qualification.checks = [...(identityInput.qualification.checks ?? []).slice(1)]; + assert.notDeepEqual(calibrationQualificationIdentity(identityInput), first); +}); + +test('calibration rejects unknown fields', () => { + const value = calibrationSource() as Record; + value.version = '1.0.0'; + assert.throws(() => compileCalibrationDefinition(value), /unknown field/); +}); + +test('current level points come from the selected recipe', () => { + const release = buildRecipeRelease(join(TRACK.dir, 'composition', 'recipes', 'sequential-l2.json')); + const base = buildRecipeRelease(join(TRACK.dir, 'composition', 'recipes', 'sequential-l1.json')); + assert.equal(currentLevelPoints(release, executionPlanForRelease( + join(TRACK.dir, 'composition', 'recipes', 'sequential-l2.json'), + { trackRoot: TRACK.dir, level: 2 }, + )), release.scoring.points - base.scoring.points); + const currentRelease = requireRecipeRelease(TRACK, 3, 'ecommerce.progression-catalog').release; + assert.equal(resolveCalibrationForRelease(currentRelease, { + trackRoot: TRACK.dir, + stackBenchRoot: STACK_BENCH_ROOT, + alias: 'L3', + })?.recipe.id, currentRelease.id); +}); diff --git a/tools/stack-bench/tests/campaign-admission.contract.ts b/tools/stack-bench/tests/campaign-admission.contract.ts new file mode 100644 index 00000000000..45a0577f7e2 --- /dev/null +++ b/tools/stack-bench/tests/campaign-admission.contract.ts @@ -0,0 +1,176 @@ +import assert from 'node:assert/strict'; +import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { runCampaignAdmission, validateCampaignAdmission } + from '../src/campaigns/campaign-admission.js'; +import { compileCampaignFile } from '../src/campaigns/campaign-compiler.js'; +import type { CampaignAdmissionPreflightRequest } + from '../src/campaigns/campaign-admission.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; + +const createdAt = '2026-08-27T12:00:00.000Z'; + +test('single-attempt admission rejects an impossible full campaign parallelism', async () => { + const root = mkdtempSync(join(tmpdir(), 'campaign-capacity-plan-')); + try { + const source = JSON.parse(readFileSync(join(STACK_BENCH_ROOT, 'tests/fixtures/campaign.deterministic.json'), 'utf8')); + source.parallelism = 2; + const path = join(root, 'plan.json'); + writeFileSync(path, JSON.stringify(source)); + const plan = compileCampaignFile(path); + await assert.rejects(runCampaignAdmission(plan, root, { + attempt: plan.attempts[0], env: { STACK_BENCH_RUNNER_CAPACITY: '1' }, + preflight: () => { throw new Error('must reject before preflight'); }, + }), /requested parallelism 2 exceeds Docker host capacity 1/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); +const passingPreflight = (request: CampaignAdmissionPreflightRequest) => ({ + schemaVersion: 1 as const, + generatedAt: createdAt, + request: { backends: request.backends, track: request.track, levels: request.levelList, + runIndex: request.runIndex, parallelism: request.parallelism, + agentAdapter: request.agentAdapter, + ...(request.providerRoute ? { providerRoute: request.providerRoute } : {}), + ...(request.maxOutputTokens ? { maxOutputTokens: request.maxOutputTokens } : {}), + packs: request.packIds, checks: request.checkKeys, image: request.image, + resultsDir: request.resultsDir, smoke: request.smoke }, + ok: true, + summary: { passed: 1, failed: 0, warnings: 0 }, + checks: [{ id: 'smoke.container', status: 'pass' as const, summary: 'passed' }], +}); + +test('campaign admission receives only the feature catalog levels in the compiled plan', { skip: process.platform !== 'linux' ? 'Kernel flock requires Linux' : false }, async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-scoped-admission-')); + try { + const value = JSON.parse(readFileSync(join(STACK_BENCH_ROOT, 'appliance', + 'campaign.ecommerce-progression-reference.json'), 'utf8')); + value.levels = [1, 2, 3]; + value.selection.levels = value.selection.levels.filter((entry: { level: number }) => + entry.level <= 3); + const campaignPath = join(root, 'campaign.json'); + writeFileSync(campaignPath, `${JSON.stringify(value, null, 2)}\n`); + const plan = compileCampaignFile(campaignPath); + const requests: CampaignAdmissionPreflightRequest[] = []; + const result = await runCampaignAdmission(plan, root, { now: createdAt, + uuid: () => 'scoped', + env: { STACK_BENCH_RUNNER_CAPACITY: '64', STACK_BENCH_RESOURCE_LOCK_DIR: join(root, 'locks') }, + preflight: request => { + requests.push(request); + return passingPreflight(request); + }, + }); + + assert.equal(result.payload.ok, true); + assert.equal(requests.length, plan.summary.parallelism); + assert(requests.every(request => request.featureCatalog!.definition.nodes + .every(node => node.level <= 3))); + assert(plan.featureCatalog); + const identity = plan.featureCatalog.identity; + assert(requests.every(request => request.featureCatalog!.identity.contentSha256 + === identity.contentSha256)); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('campaign admission selects a free run slot', { skip: process.platform !== 'linux' ? 'Kernel flock requires Linux' : false }, async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-free-slot-admission-')); + try { + const value = JSON.parse(readFileSync(join(STACK_BENCH_ROOT, 'tests', 'fixtures', + 'campaign.deterministic.json'), 'utf8')); + value.repetitions = 1; + const campaignPath = join(root, 'campaign.json'); + writeFileSync(campaignPath, `${JSON.stringify(value, null, 2)}\n`); + const plan = compileCampaignFile(campaignPath); + const requests: CampaignAdmissionPreflightRequest[] = []; + let portProbes = 0; + const result = await runCampaignAdmission(plan, root, { now: createdAt, + uuid: () => 'free-slot', + env: { STACK_BENCH_RUNNER_CAPACITY: '64', STACK_BENCH_RESOURCE_LOCK_DIR: join(root, 'locks') }, + probePort: () => ({ free: ++portProbes > 1 }), + preflight: request => { + requests.push(request); + return passingPreflight(request); + }, + }); + assert.deepEqual(result.runIndices, [1]); + assert.equal(requests.length, 1); + assert.equal(requests[0]!.runIndex, 1); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('admission requires a distinct report for each provider route and rejects substituted routes', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-route-admission-')); + try { + const plan = compileCampaignFile(join(STACK_BENCH_ROOT, 'tests', 'fixtures', + 'campaign.deterministic.json')); + const agent = plan.agents[0]!; + plan.agents = ['openai', 'azure'].map(providerRoute => ({ ...agent, + adapter: 'openrouter', providerRoute, maxOutputTokens: 8192 })); + const payload = { schemaVersion: 1, campaignId: plan.id, campaignSha256: plan.contentSha256, + createdAt, ok: true, runtime: plan.definition.runtime, conditions: plan.conditions, + agents: plan.agents.map(({ adapter, model, providerRoute, maxOutputTokens, identity }) => + ({ adapter, model, providerRoute, maxOutputTokens, identity })), + reports: plan.agents.flatMap(({ adapter, providerRoute, maxOutputTokens }) => + Array.from({ length: plan.summary.parallelism }, (_, runIndex) => ({ + schemaVersion: 1, ok: true, checks: [], summary: { passed: 0, failed: 0, warnings: 0 }, + request: { agentAdapter: adapter, providerRoute, maxOutputTokens, runIndex, + backends: plan.stacks.map(stack => stack.id), track: plan.definition.track, + levels: plan.definition.levels, parallelism: plan.summary.parallelism, + packs: plan.definition.selection.packs ?? [], checks: plan.definition.selection.checks ?? [], + smoke: false, image: plan.definition.runtime.buildImage, resultsDir: root }, + }))) }; + assert.deepEqual(validateCampaignAdmission(payload, plan, root), payload); + const missing = { ...payload, reports: payload.reports.slice(1) }; + assert.throws(() => validateCampaignAdmission(missing, plan, root), /incomplete/); + const changedLimit = structuredClone(payload); + changedLimit.reports[0]!.request.maxOutputTokens = 1; + assert.throws(() => validateCampaignAdmission(changedLimit, plan, root), /must contain one/); + const substituted = structuredClone(payload); + substituted.reports[0]!.request.providerRoute = 'another-provider'; + assert.throws(() => validateCampaignAdmission(substituted, plan, root), /must contain one/); + + } finally { rmSync(root, { recursive: true, force: true }); } +}); + + +test('occupied-port scans yield to cancellation before exhausting the TCP range', async () => { + const root = mkdtempSync(join(tmpdir(), 'campaign-scan-cancel-')); + const controller = new AbortController(); + try { + const plan = compileCampaignFile(join(STACK_BENCH_ROOT, 'tests', 'fixtures', 'campaign.deterministic.json')); + let probes = 0; + await assert.rejects(runCampaignAdmission(plan, root, { + signal: controller.signal, env: { STACK_BENCH_RUNNER_CAPACITY: '64', STACK_BENCH_RESOURCE_LOCK_DIR: join(root, 'locks') }, + probePort: () => { + if (++probes === 1) setImmediate(() => controller.abort()); + return { free: false }; + }, + preflight: () => { throw new Error('must not reach preflight'); }, + }), { name: 'AbortError' }); + assert(probes > 0 && probes <= 16, `scan did ${probes} probes without yielding`); + assert.equal(existsSync(join(root, 'admissions')), false); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('cancellation after reservation releases exact owned locks before admission returns', + { skip: process.platform !== 'linux' ? 'Kernel flock requires Linux' : false }, async () => { + const root = mkdtempSync(join(tmpdir(), 'campaign-preflight-cancel-')); + const controller = new AbortController(); + const locks = join(root, 'locks'); + try { + const plan = compileCampaignFile(join(STACK_BENCH_ROOT, 'tests', 'fixtures', 'campaign.deterministic.json')); + await assert.rejects(runCampaignAdmission(plan, root, { + signal: controller.signal, env: { STACK_BENCH_RUNNER_CAPACITY: '64', STACK_BENCH_RESOURCE_LOCK_DIR: locks }, + probePort: () => ({ free: true }), + preflight: request => { controller.abort(); return passingPreflight(request); }, + }), { name: 'AbortError' }); + assert.equal(readdirSync(locks).filter(name => name.endsWith('.lock.json')).length, 0); + assert.equal(existsSync(join(root, 'admissions')), false); + } finally { rmSync(root, { recursive: true, force: true }); } + }); diff --git a/tools/stack-bench/tests/campaign-cli.test.ts b/tools/stack-bench/tests/campaign-cli.test.ts new file mode 100644 index 00000000000..7b19f65a489 --- /dev/null +++ b/tools/stack-bench/tests/campaign-cli.test.ts @@ -0,0 +1,96 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { auditCompletedReferenceCampaign, campaignStateSummary, parseCampaignArgs, + validateResumeCampaignState } from '../commands/campaign-cli.js'; +import { SAFE_RESULT_NAME } from '../src/runtime/operational-paths.js'; + +test('publish requires a suite and exactly one destination', () => { + const args = (...flags: string[]) => parseCampaignArgs(['node', 'campaign', 'publish', 'results/finished', ...flags]); + assert.equal(args('--suite', 'ecommerce-1', '--out', 'upload.json').command, 'publish'); + assert.equal(args('--suite', 'ecommerce-1', '--to', 'http://localhost:5175').command, 'publish'); + for (const flags of [[], ['--out', 'upload.json'], ['--suite', 'x'], + ['--suite', 'x', '--to', 'https://stackbench.org', '--out', 'upload.json'], + ['--suite', 'x', '--suite', 'y', '--out', 'upload.json']]) { + assert.throws(() => args(...flags), /publish/); + } +}); + +test('campaign commands print a compact result and retain failed attempt details', () => { + const plan = { id: 'campaign', version: '1.0.0', contentSha256: 'a'.repeat(64) }; + const state = { + status: 'attention-required', + summary: { total: 3, completed: 2, invalid: 1, executions: 3 }, + attempts: [ + { plan: { id: 'passed' }, status: 'completed', executions: [ + { id: 'passed-execution1', outcome: 'passed', reason: null }, + ] }, + { plan: { id: 'application-failure' }, status: 'completed', executions: [ + { id: 'application-failure-execution1', outcome: 'app_failure', reason: null }, + ] }, + { plan: { id: 'invalid' }, status: 'invalid', executions: [ + { id: 'invalid-execution1', outcome: 'harness_failure', reason: 'grader stopped' }, + ] }, + ], + }; + + const summary = campaignStateSummary(plan, state); + assert.equal(summary.status, 'needs attention'); + assert.deepEqual(summary.summary, state.summary); + assert.deepEqual(summary.failures.map(failure => [failure.attempt, failure.outcome]), [ + ['application-failure', 'application failure'], + ['invalid', 'harness failure'], + ]); + assert.equal(JSON.stringify(summary).includes('passed-execution1'), false); +}); + +test('automatic reference audit runs only after campaign completion', () => { + let calls = 0; + const plan = { attempts: [{ mode: { id: 'dependency' }, agentAdapter: 'reference-fixture' }] }; + const audit = (directory: string) => { + calls += 1; + assert.equal(directory, 'results'); + return { ok: true }; + }; + assert.equal(auditCompletedReferenceCampaign('results', plan, { status: 'running' }, { audit }), + null); + assert.deepEqual(auditCompletedReferenceCampaign('results', plan, { status: 'completed' }, + { audit }), { ok: true }); + assert.equal(auditCompletedReferenceCampaign('results', { attempts: [] }, + { status: 'completed' }, { audit }), null); + assert.equal(calls, 1); +}); + +test('campaign CLI resumes only an existing matching dependency campaign', () => { + const sha256 = 'a'.repeat(64); + const requested = { contentSha256: sha256 }; + const existing = { + plan: { contentSha256: sha256, definition: { mode: { id: 'dependency' } } }, + state: { status: 'prepared', attempts: [{ executions: [{}] }] }, + }; + assert.equal(validateResumeCampaignState(requested, existing), existing); + assert.throws(() => validateResumeCampaignState({ contentSha256: 'b'.repeat(64) }, existing), + /exact campaign plan/); + assert.throws(() => validateResumeCampaignState(requested, { + ...existing, plan: { ...existing.plan, definition: { mode: { id: 'sequential' } } }, + }), /only for dependency/); + assert.throws(() => validateResumeCampaignState(requested, { + ...existing, state: { status: 'prepared', attempts: [{ executions: [] }] }, + }), /scheduled work/); + assert.throws(() => validateResumeCampaignState(requested, { + ...existing, state: { ...existing.state, status: 'running' }, + }), /scheduled work/); +}); + +test('new campaign folders must use a name the dashboard can open', () => { + for (const out of ['campaigns/Pilot_1', 'campaigns/l3']) { + assert.throws(() => parseCampaignArgs(['node', 'x', 'run', 'p.json', '--out', out]), /folder name/); + assert.throws(() => parseCampaignArgs(['node', 'x', 'extend', 'p.json', '--from', 'campaigns/a1', '--depth', '2', + '--out', out]), /folder name/); + assert.equal(SAFE_RESULT_NAME.test(out.slice('campaigns/'.length)), false); + } + const trial = parseCampaignArgs(['node', 'x', 'trial', 'p.json', '--out', 'campaigns/pilot-1']) as { directory: string }; + assert.match(trial.directory, /pilot-1$/); + // Resume and reconcile act on folders that already exist. + assert.doesNotThrow(() => parseCampaignArgs(['node', 'x', 'resume', 'p.json', '--out', 'campaigns/Pilot_1'])); +}); diff --git a/tools/stack-bench/tests/campaign-compiler.contract.ts b/tools/stack-bench/tests/campaign-compiler.contract.ts new file mode 100644 index 00000000000..74826d2b5bb --- /dev/null +++ b/tools/stack-bench/tests/campaign-compiler.contract.ts @@ -0,0 +1,238 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import test from 'node:test'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { compileCampaignFile, validateCampaignDefinition, + validateCompiledCampaignPlan } from '../src/campaigns/campaign-compiler.js'; +import { campaignComparisonKey } from '../src/campaigns/campaign-report.js'; +import { attemptArgv } from '../src/campaigns/campaign-runner.js'; +import { writeArtifact } from '../src/evidence/artifacts.js'; +import { parseBenchArguments } from '../commands/bench-arguments.js'; + +const APPLIANCE = resolve(STACK_BENCH_ROOT, 'appliance'); + +function manifest(name: string): Record { + return JSON.parse(readFileSync(join(APPLIANCE, name), 'utf8')) as Record; +} + +function compile(value: unknown) { + const directory = mkdtempSync(join(tmpdir(), 'stack-bench-campaign-')); + const path = join(directory, 'campaign.json'); + writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`); + try { return compileCampaignFile(path); } + finally { rmSync(directory, { recursive: true, force: true }); } +} + +test('campaign duration follows safe deadline arithmetic rather than a twelve-hour policy', () => { + const value = manifest('campaign.example.json'); + for (const minutes of [1, 721, 43_200]) { + const budgets = { ...(value.budgets as object), attemptTimeoutMinutes: minutes }; + assert.equal(validateCampaignDefinition({ ...value, budgets }).budgets.attemptTimeoutMinutes, minutes); + } + for (const minutes of [0, -1, 1.5, Number.MAX_SAFE_INTEGER]) { + assert.throws(() => validateCampaignDefinition({ ...value, + budgets: { ...(value.budgets as object), attemptTimeoutMinutes: minutes } }), /attemptTimeoutMinutes/); + } +}); + +test('campaign effort is retained per attempt, and campaign choices change campaign identity', () => { + const value = manifest('campaign.example.json'); + value.stacks = (value.stacks as Array<{ id: string }>).filter(stack => stack.id === 'postgres'); + const agents = value.agents as Array>; + const medium = compile({ ...value, agents: agents.map(agent => ({ ...agent, effort: 'medium' })) }); + const high = compile({ ...value, agents: agents.map(agent => ({ ...agent, effort: 'high' })) }); + assert(medium.attempts.every(attempt => attempt.effort === 'medium')); + assert(high.attempts.every(attempt => attempt.effort === 'high')); + assert.notEqual(medium.contentSha256, high.contentSha256); + assert.deepEqual(validateCompiledCampaignPlan(medium), medium); + assert.throws(() => compile({ ...value, + agents: agents.map(agent => ({ ...agent, effort: 'invalid' })) }), /effort/); + const reseeded = compile({ ...value, agents: agents.map(agent => ({ ...agent, effort: 'high' })), + ordering: { ...(value.ordering as object), seed: 'another-seed' } }); + assert.notEqual(reseeded.contentSha256, high.contentSha256); +}); + +test('every compiled campaign records its tool policy, and the policy changes campaign identity', () => { + const value = manifest('campaign.example.json'); + value.stacks = (value.stacks as Array<{ id: string }>).filter(stack => stack.id === 'postgres'); + const byDefault = compile(value); + assert.deepEqual(byDefault.definition.tools, { webSearch: false, subagents: false }); + assert(byDefault.attempts.every(attempt => attempt.tools?.webSearch === false && attempt.tools.subagents === false)); + assert.deepEqual(validateCompiledCampaignPlan(byDefault), byDefault); + const explicit = compile({ ...value, tools: { webSearch: false, subagents: false } }); + assert.equal(explicit.contentSha256, byDefault.contentSha256); + const searching = compile({ ...value, tools: { webSearch: true, subagents: false } }); + assert.notEqual(searching.contentSha256, byDefault.contentSha256); + // Subagents are refused for every agent until the audit is shown to see their work. + for (const tools of [{ webSearch: false, subagents: true }, { webSearch: true, subagents: true }]) { + assert.throws(() => compile({ ...value, tools }), /subagents cannot be on/); + } + assert(searching.attempts.every(attempt => attempt.tools?.webSearch === true)); + // Claude Code runs with --bare, so a campaign cannot ask it for tools it does not have. + const rates = Object.values((value.pricing as { models: Record }).models)[0]; + const claude = { ...value, agents: [{ adapter: 'claude-code', adapterVersion: '1.17.2', model: 'claude-test' }], + pricing: { ...(value.pricing as object), models: { 'claude-test': rates } } }; + assert.equal(validateCampaignDefinition({ ...claude, tools: { webSearch: false, subagents: false } }).tools?.webSearch, false); + assert.throws(() => validateCampaignDefinition({ ...claude, tools: { webSearch: true, subagents: false } }), + /claude-code webSearch cannot be on: .*--bare/); + for (const tools of [{ webSearch: 'yes', subagents: false }, { webSearch: true }, { webSearch: true, subagents: false, mcp: true }]) { + assert.throws(() => validateCampaignDefinition({ ...value, tools }), /tools/); + } +}); + +test('campaign repetitions use checked expansion rather than a hundred-run policy', () => { + const value = manifest('campaign.example.json'); + const expanded = compile({ ...value, repetitions: 101 }); + assert.equal(expanded.attempts.length, 101 * expanded.stacks.length * expanded.agents.length * expanded.conditions.length); + assert.throws(() => compile({ ...value, repetitions: Number.MAX_SAFE_INTEGER }), /array length limit/); + assert.throws(() => compile({ ...value, repetitions: 100_000_000 }), /string or available heap capacity/); + assert.throws(() => validateCampaignDefinition({ ...value, repetitions: Number.MAX_SAFE_INTEGER + 1 }), /repetitions/); +}); + +test('dependency campaigns bind a graph and feature catalog by stable ID and content hash', () => { + const plan = compile(manifest('campaign.ecommerce-progression-reference.json')); + assert(plan.featureCatalog && plan.dependencyPolicy); + assert.match(plan.featureCatalog.identity.contentSha256, /^[a-f0-9]{64}$/); + assert.match(plan.dependencyPolicy.identity.contentSha256, /^[a-f0-9]{64}$/); + assert.equal(plan.featureCatalog.identity.id, 'ecommerce.questlines'); + assert(plan.bindings.every(binding => /^[a-f0-9]{64}$/.test(binding.recipe.contentSha256))); + assert.deepEqual(plan.bindings.map(binding => binding.level), [1, 2, 3, 4, 5, 6]); + assert.deepEqual(plan.bindings.map(binding => binding.calibration?.id ?? null), + ['ecommerce.dependency-l3-calibration', 'ecommerce.dependency-l3-calibration', + 'ecommerce.dependency-l3-calibration', null, null, 'ecommerce.dependency-l6-calibration'], + 'the L3 calibration covers its lower levels, and L6 has its own'); + assert(plan.attempts.every(attempt => attempt.featureCatalog?.contentSha256 + === plan.featureCatalog?.identity.contentSha256)); + assert(plan.attempts.every(attempt => attempt.dependencyPolicy?.contentSha256 + === plan.dependencyPolicy?.identity.contentSha256)); +}); + +test('campaign definitions reject versioned authored references', () => { + const value = manifest('campaign.example.json'); + const selection = value.selection as { levels: Array<{ recipe: string }> }; + selection.levels[0]!.recipe = 'ecommerce.sequential-l1@2.5.0'; + assert.throws(() => validateCampaignDefinition(value), /recipe.*invalid/); +}); + +test('prior interface retention defaults on in new plans and freezes the JSON opt-out', () => { + const value = manifest('campaign.ecommerce-progression-reference.json'); + assert.equal(validateCampaignDefinition(value).mode.retainPriorContracts, undefined, + 'stored definitions without the option must not be reinterpreted'); + const enabled = compile(value); + assert.equal(enabled.definition.mode.retainPriorContracts, true); + assert(enabled.attempts.every(attempt => attempt.mode.retainPriorContracts === true)); + const disabled = compile({ ...value, + mode: { ...(value.mode as object), retainPriorContracts: false } }); + assert.equal(disabled.definition.mode.retainPriorContracts, false); + assert.notEqual(enabled.contentSha256, disabled.contentSha256); + assert.deepEqual(enabled.bindings, disabled.bindings, 'the authored grading definition is unchanged'); + assert.deepEqual(validateCompiledCampaignPlan(enabled), enabled); + assert.deepEqual(validateCompiledCampaignPlan(disabled), disabled); + const rewritten = structuredClone(enabled); + rewritten.definition.mode.retainPriorContracts = false; + assert.throws(() => validateCompiledCampaignPlan(rewritten), /identity|match/); + const directory = mkdtempSync(join(tmpdir(), 'stack-bench-retained-contracts-')); + try { + const path = join(directory, 'plan.json'); + for (const plan of [enabled, disabled]) { + const attempt = plan.attempts[0]!; + writeArtifact(path, { kind: 'campaign_plan', id: plan.id, payload: plan }); + const argv = attemptArgv(plan, attempt, join(directory, 'out'), 0, path); + const parsed = parseBenchArguments(['node', ...argv]); + assert.equal(parsed.retainPriorContracts, plan === enabled); + } + } finally { rmSync(directory, { recursive: true, force: true }); } +}); + +test('campaign runtime accepts immutable local IDs and registry digests, but rejects tags', () => { + const value = manifest('campaign.paid-l1.json'); + value.state = 'frozen'; + const runtime = value.runtime as Record; + const id = `sha256:${'a'.repeat(64)}`; + runtime.controllerImage = id; + runtime.buildImage = id; + for (const field of ['controllerImage', 'buildImage']) { + for (const image of [id, `stack-bench@${id}`]) { + runtime[field] = image; + assert.equal(validateCampaignDefinition(value).runtime[field as 'controllerImage' | 'buildImage'], image); + } + for (const image of ['stack-bench:local', 'stack-bench:latest', 'sha256:abc']) { + runtime[field] = image; + assert.throws(() => validateCampaignDefinition(value), /exact image digest reference/); + } + runtime[field] = id; + } +}); + +test('cost/completion thresholds are declared numeric campaign policy, not inferred from results', () => { + const value = manifest('campaign.example.json'); + const analysis = value.analysis as Record; + analysis.spendThresholdsUsd = [0, 2, 5]; + analysis.completionTargets = [0.5, 1]; + assert.deepEqual(validateCampaignDefinition(value).analysis.spendThresholdsUsd, [0, 2, 5]); + analysis.completionTargets = [1.1]; + assert.throws(() => validateCampaignDefinition(value), /completionTargets/); + analysis.completionTargets = [1]; + analysis.spendThresholdsUsd = [-1]; + assert.throws(() => validateCampaignDefinition(value), /spendThresholdsUsd/); +}); + +test('campaigns bind the repeated-finding stop independently of repair allowance', () => { + const value = manifest('campaign.ecommerce-progression-reference.json'); + const baseline = compile(value); + const plan = compile({ ...value, repair: { selection: 'feature', budget: { perFeature: 5 } }, + mode: { ...(value.mode as object), unchangedFailureLimit: 5 } }); + assert.equal(baseline.dependencyPolicy?.definition.unchangedFailureLimit, 3); + assert.equal(plan.dependencyPolicy?.definition.unchangedFailureLimit, 5); + assert.deepEqual(plan.definition.repair.budget, { perFeature: 5 }); + assert.notEqual(plan.dependencyPolicy?.identity.contentSha256, baseline.dependencyPolicy?.identity.contentSha256); + assert.deepEqual(validateCompiledCampaignPlan(plan), plan); + for (const unchangedFailureLimit of [0, -1, 1.5, '5', Number.MAX_SAFE_INTEGER + 1]) { + assert.throws(() => validateCampaignDefinition({ ...value, + mode: { ...(value.mode as object), unchangedFailureLimit } }), /unchangedFailureLimit/); + } +}); + +test('OpenRouter fixes one provider route in the campaign and attempt identity', () => { + const value = manifest('campaign.example.json'); + const rates = Object.values((value.pricing as { models: Record }).models)[0]; + value.pricing = { ...(value.pricing as object), models: { 'openai/gpt-5.3-codex': rates } }; + const selection = { adapter: 'openrouter', adapterVersion: '1.0.0', + model: 'openai/gpt-5.3-codex', providerRoute: 'openai', maxOutputTokens: 8192 }; + value.agents = [selection]; + const first = compile(value); + assert(first.attempts.every(attempt => attempt.providerRoute === 'openai' + && attempt.maxOutputTokens === 8192)); + for (const maxOutputTokens of [undefined, 0, 128001, 1.5]) { + assert.throws(() => validateCampaignDefinition({ ...value, + agents: [{ ...selection, maxOutputTokens }] }), /maxOutputTokens/); + } + assert.deepEqual(validateCompiledCampaignPlan(first), first); + const changed = compile({ ...value, agents: [{ ...selection, providerRoute: 'another-provider' }] }); + assert.notEqual(first.contentSha256, changed.contentSha256); + assert.notEqual(campaignComparisonKey(first.attempts[0]!), campaignComparisonKey(changed.attempts[0]!)); + const directory = mkdtempSync(join(tmpdir(), 'stack-bench-provider-settings-')); + try { + const path = join(directory, 'plan.json'); + writeArtifact(path, { kind: 'campaign_plan', id: first.id, payload: first }); + const argv = attemptArgv(first, first.attempts[0]!, join(directory, 'out'), 0, path); + const parsed = parseBenchArguments(['node', ...argv]); + assert.equal(parsed.providerRoute, selection.providerRoute); + assert.equal(parsed.maxOutputTokens, selection.maxOutputTokens); + assert.throws(() => parseBenchArguments(['node', ...argv, '--provider-route', 'other']), + /campaign|cannot/); + const attemptIndex = argv.indexOf('--campaign-attempt-id') + 1; + assert.throws(() => parseBenchArguments(['node', + ...argv.map((item, index) => index === attemptIndex ? 'unknown-attempt' : item)]), + /--campaign-attempt-id is not in the compiled campaign plan/); + } finally { rmSync(directory, { recursive: true, force: true }); } + assert.throws(() => validateCampaignDefinition({ ...value, + agents: [{ ...selection, providerRoute: undefined }] }), /providerRoute/); + assert.throws(() => validateCampaignDefinition({ ...value, + agents: [{ ...selection, providerRoute: 'openai,other' }] }), /providerRoute/); + assert.throws(() => validateCampaignDefinition({ ...value, + agents: [{ ...selection, adapter: 'codex' }] }), /providerRoute/); +}); diff --git a/tools/stack-bench/tests/campaign-depth-pause.test.ts b/tools/stack-bench/tests/campaign-depth-pause.test.ts new file mode 100644 index 00000000000..fe8021f2db1 --- /dev/null +++ b/tools/stack-bench/tests/campaign-depth-pause.test.ts @@ -0,0 +1,311 @@ +import assert from 'node:assert/strict'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { setTimeout as delay } from 'node:timers/promises'; +import test from 'node:test'; +import { compileCampaignFile } from '../src/campaigns/campaign-compiler.js'; +import { acquireCampaignLock, releaseCampaignLock, writeCampaignRecord } from '../src/campaigns/campaign-lock.js'; +import { campaignDepthPauseStatus, continueCampaignDepth, depthPauseDue, depthPauseDurationMs, + readDepthPause, validateDepthPauseEvidence, waitAtDepthBoundary } from '../src/campaigns/campaign-depth-pause.js'; +import { claimNextAttempt, initializeCampaignDirectory, writeCampaignState, + campaignTimeBudget, type CampaignClaim } from '../src/campaigns/campaign-scheduler.js'; +import { ARTIFACT_FILE } from '../src/evidence/artifacts.js'; +import { finalizeRunTotals, type RunTotalsInput } from '../src/evidence/benchmark-run.js'; +import { STACK_BENCH_ROOT, compiledEntrypoint } from '../src/package-root.js'; +import { dependencyRuntimeDefinition } from '../src/progression/progression-definition.js'; +import { progressionEngine } from '../src/progression/progression-engine.js'; +import { runBounded } from '../src/runtime/bounded-process.js'; +import { readCampaignTimeBudget } from '../src/campaigns/campaign-time-grant.js'; +import { campaignProviderContinuationContext } from '../src/campaigns/campaign-provider-continuation.js'; + +test('planned hold preserves the L3 action, repair history, cohort and cost accounting', async t => { + const root = mkdtempSync(join(tmpdir(), 'depth-pause-')); + let lock: ReturnType | undefined; + const abort = new AbortController(); + const waits: Promise[] = []; + try { + const manifest = JSON.parse(readFileSync(join(STACK_BENCH_ROOT, + 'appliance/campaign.ecommerce-progression-reference.json'), 'utf8')); + manifest.levels = [1, 2, 3]; + manifest.selection.levels = manifest.selection.levels.slice(0, 3); + manifest.repair = { selection: 'feature', budget: { perFeature: 5 } }; + manifest.parallelism = 3; + const path = join(root, 'manifest.json'); + writeFileSync(path, JSON.stringify(manifest)); + const uninterrupted = compileCampaignFile(path); + manifest.mode.pauseAfterDepth = 2; + writeFileSync(path, JSON.stringify(manifest)); + const plan = compileCampaignFile(path); + assert.deepEqual(plan.bindings, uninterrupted.bindings); + assert.deepEqual(plan.featureCatalog, uninterrupted.featureCatalog); + assert.deepEqual(plan.dependencyPolicy, uninterrupted.dependencyPolicy); + assert.deepEqual(plan.conditions, uninterrupted.conditions); + assert.deepEqual(plan.attempts.map(a => a.skills), uninterrupted.attempts.map(a => a.skills)); + assert.notEqual(plan.contentSha256, uninterrupted.contentSha256, 'pause is disclosed in protocol identity'); + manifest.parallelism = 2; + writeFileSync(path, JSON.stringify(manifest)); + assert.throws(() => compileCampaignFile(path), /whole cohort/); + manifest.parallelism = 3; + manifest.mode.pauseAfterDepth = 3; + writeFileSync(path, JSON.stringify(manifest)); + assert.throws(() => compileCampaignFile(path), /later depth/); + + let progression = progressionEngine.initialize(dependencyRuntimeDefinition( + plan.featureCatalog!, plan.dependencyPolicy!)); + let sequence = 0; + while (progression.phase === 'active' && progression.level <= 2) { + const selection = progressionEngine.gradingSelection(progression); + progression = progressionEngine.recordResult(progression, { + attemptId: `action-${++sequence}`, outcome: 'conclusive', + ...(progressionEngine.nextAction(progression).type === 'repair' ? { completedRepair: true } : {}), + nodes: selection.nodeIds.map(id => ({ id, checks: selection.checks + .filter(c => c.nodeId === id).map(c => ({ id: c.id, + outcome: sequence === 1 ? 'fail' : 'pass' })) })), + }); + assert(sequence < 30); + } + assert.equal(progression.level, 3); + assert(progression.attempts.some(a => a.repair), 'prefix includes repairs'); + const next = structuredClone(progressionEngine.nextAction(progression)); + const before = structuredClone(progression); + const results = join(root, 'results'); + const initialized = initializeCampaignDirectory(plan, results); + lock = acquireCampaignLock(results, plan); + let state = initialized.state; + const claims: CampaignClaim[] = []; + for (let i = 0; i < 3; i++) { + const claimed = claimNextAttempt(state, { runIndex: i, admissionId: `admission-${i}` }); + assert(claimed.claim); + state = claimed.state; + claims.push(claimed.claim); + } + writeCampaignState(initialized.paths.state, plan, state); + assert.throws(() => continueCampaignDepth(results), /every attempt/); + for (const claim of claims) { + const output = join(results, claim.output); + const app = join(output, 'app'); + mkdirSync(app, { recursive: true }); + writeFileSync(join(app, 'index.js'), 'export const accepted = true;'); + writeFileSync(join(output, ARTIFACT_FILE.progressionState), JSON.stringify(progression)); + waits.push(waitAtDepthBoundary(output, app, { directory: results, + campaignSha256: plan.contentSha256, ownershipMarkerSha256: lock.record.ownershipMarkerSha256, + attemptId: claim.attempt.id, executionId: claim.executionId, depth: 2 }, { signal: abort.signal })); + } + assert.equal(campaignDepthPauseStatus(results).attempts.filter(a => a.paused).length, 3); + await delay(25); + let now = Date.now() + 8 * 60 * 60_000; + t.mock.method(Date, 'now', () => now); + const claim = claims[0]!; + const providerEnv = { STACK_BENCH_PROVIDER_WAIT_CONTEXT: JSON.stringify({ directory: results, + campaignSha256: plan.contentSha256, ownershipMarkerSha256: lock.record.ownershipMarkerSha256, + attemptId: claim.attempt.id, executionId: claim.executionId, + root: join(results, claim.output, 'provider-waits') }) }; + const savedState = readFileSync(initialized.paths.state, 'utf8'); + assert(readCampaignTimeBudget(results, claim.attempt.id).consumedMs < 60_000); + assert(campaignProviderContinuationContext(providerEnv), 'overnight hold does not exhaust provider allowance'); + const release = continueCampaignDepth(results); + assert.deepEqual(continueCampaignDepth(results), release, 'release is idempotent'); + const durations = await Promise.all(waits); + assert(durations.every(ms => ms > 0)); + assert(campaignProviderContinuationContext(providerEnv), 'first L3 invocation uses the resumed receipt'); + assert.equal(readFileSync(initialized.paths.state, 'utf8'), savedState, 'budget reads do not rewrite history'); + now += plan.definition.budgets.attemptTimeoutMinutes * 60_000 + 1; + assert.throws(() => campaignProviderContinuationContext(providerEnv), /duration allowance is exhausted/); + t.mock.restoreAll(); + const proof = { campaignSha256: plan.contentSha256, attemptId: claims[0]!.attempt.id, + depth: 2, durationMs: durations[0]! }; + validateDepthPauseEvidence(join(results, claims[0]!.output), proof); + assert.throws(() => validateDepthPauseEvidence(join(results, claims[0]!.output), + { ...proof, durationMs: proof.durationMs + 1 }), /pause accounting/); + assert.deepEqual(progression, before); + assert.deepEqual(progressionEngine.nextAction(progression), next); + const run: RunTotalsInput = { levels: [{ level: 1, buildCostUsd: 2, repairCostUsd: 3, + repairs: 2 }, { level: 2, buildCostUsd: 4, repairCostUsd: 1, repairs: 1 }] }; + const baseline = finalizeRunTotals(run, 0, { now: 10_000 }); + run.pausedDurationMs = 5_000; + const held = finalizeRunTotals(run, 0, { now: 15_000 }); + assert.equal(held.costUsd, baseline.costUsd); + assert.equal(held.repairs, baseline.repairs); + assert.equal(held.activeDurationSec, baseline.durationSec); + assert.equal(held.durationSec, 15); + const attempt = state.attempts[0]!; + attempt.executions[0]!.pausedMs = 5_000; + const started = Date.parse(attempt.executions[0]!.startedAt); + assert.equal(campaignTimeBudget(plan, attempt, started + 15_000).consumedMs, 10_000); + // Waiting for a grading slot is held time too, on top of the planned pause. + run.gradingWaitMs = 2_000; + const queued = finalizeRunTotals(run, 0, { now: 17_000 }); + assert.equal(queued.gradingWaitSec, 2); + assert.equal(queued.activeDurationSec, baseline.durationSec); + attempt.executions[0]!.gradingWaitMs = 2_000; + assert.equal(campaignTimeBudget(plan, attempt, started + 17_000).consumedMs, 10_000); + } finally { + abort.abort(); + await Promise.allSettled(waits); + if (lock) releaseCampaignLock(lock); + rmSync(root, { recursive: true, force: true }); + } +}); + +test('hold rejects source changes, ignores a release from another controller and permits cancellation', async () => { + for (const failure of ['source', 'authority', 'cancel']) { + const root = mkdtempSync(join(tmpdir(), 'depth-pause-invalid-')); + const abort = new AbortController(); + try { + const app = join(root, 'app'); + mkdirSync(app); + writeFileSync(join(app, 'index.js'), 'accepted'); + writeFileSync(join(root, ARTIFACT_FILE.progressionState), '{}'); + const context = { directory: root, campaignSha256: 'a'.repeat(64), + ownershipMarkerSha256: 'b'.repeat(64), attemptId: 'attempt', executionId: 'execution', depth: 2 }; + const pending = waitAtDepthBoundary(root, app, context, { signal: abort.signal }); + const rejected = assert.rejects(pending, failure === 'source' ? /source or progression/ + : /abort/i); + if (failure === 'cancel') abort.abort(); + else { + if (failure === 'source') writeFileSync(join(app, 'index.js'), 'edited'); + writeCampaignRecord(join(root, 'depth-release.json'), { campaignSha256: context.campaignSha256, + ownershipMarkerSha256: (failure === 'authority' ? 'c' : 'b').repeat(64), depth: 2, + releasedAt: Date.now() }); + // Another controller's release leaves this cohort waiting for its own. + if (failure === 'authority') { await delay(600); abort.abort(); } + } + await rejected; + assert.equal(readDepthPause(root, context)?.resumedAt, null); + assert(depthPauseDurationMs(root, context) >= 0); + } finally { abort.abort(); rmSync(root, { recursive: true, force: true }); } + } +}); + +test('a planned process hold preserves remaining working time and still permits cancellation', async () => { + const start = Date.now(); + const paused = await runBounded(process.execPath, ['-e', 'setInterval(()=>{},1000)'], { + stdio: 'ignore', timeoutMs: 250, terminate: pid => process.kill(pid, 'SIGKILL'), + pauseIntervals: () => [{ startedAt: start + 100, resumedAt: Date.now() >= start + 550 ? start + 550 : null }], + }); + assert(paused.timedOut); + assert.equal(paused.error, null); + assert(Date.now() - start >= 690); + const late = await runBounded(process.execPath, ['-e', 'setInterval(()=>{},1000)'], { + stdio: 'ignore', timeoutMs: 100, terminate: pid => process.kill(pid, 'SIGKILL'), + pauseIntervals: () => [{ startedAt: Date.now(), resumedAt: null }], + }); + assert.match(late.error!.message, /invalid planned process pause/); + const abort = new AbortController(); + const heldAt = Date.now(); + const timer = setTimeout(() => abort.abort(), 350); + try { + const cancelled = await runBounded(process.execPath, ['-e', 'setInterval(()=>{},1000)'], { + stdio: 'ignore', timeoutMs: 250, signal: abort.signal, + terminate: pid => process.kill(pid, 'SIGKILL'), + pauseIntervals: () => [{ startedAt: heldAt + 100, resumedAt: null }], + }); + assert(cancelled.cancelled); + assert.equal(cancelled.timedOut, false); + } finally { clearTimeout(timer); } +}); + +test('several planned holds add up, and a recorded hold cannot change', async () => { + const start = Date.now(); + // Two 300 ms holds extend a 250 ms allowance to at least 850 ms. + const holds = () => [ + { startedAt: start + 50, resumedAt: Date.now() >= start + 350 ? start + 350 : null }, + ...(Date.now() >= start + 400 ? [{ startedAt: start + 400, resumedAt: Date.now() >= start + 700 ? start + 700 : null }] : []), + ].filter(hold => hold.startedAt <= Date.now()); + const paused = await runBounded(process.execPath, ['-e', 'setInterval(()=>{},1000)'], { + stdio: 'ignore', timeoutMs: 250, terminate: pid => process.kill(pid, 'SIGKILL'), pauseIntervals: holds, + }); + assert(paused.timedOut); + assert.equal(paused.error, null); + assert(Date.now() - start >= 840); + assert(paused.pausedMs! >= 600); + let rewritten = false; + const changed = await runBounded(process.execPath, ['-e', 'setInterval(()=>{},1000)'], { + stdio: 'ignore', timeoutMs: 2000, terminate: pid => process.kill(pid, 'SIGKILL'), + pauseIntervals: () => { + if (Date.now() < start + 1100) return []; + const recorded = { startedAt: start + 1000, resumedAt: start + 1050 }; + if (rewritten) return [{ ...recorded, resumedAt: start + 1060 }]; + rewritten = true; + return [recorded]; + }, + }); + assert.match(changed.error!.message, /invalid planned process pause/); + const overlapping = await runBounded(process.execPath, ['-e', 'setInterval(()=>{},1000)'], { + stdio: 'ignore', timeoutMs: 1000, terminate: pid => process.kill(pid, 'SIGKILL'), + pauseIntervals: () => [{ startedAt: Date.now() - 1, resumedAt: null }, { startedAt: Date.now() - 1, resumedAt: null }], + }); + assert.match(overlapping.error!.message, /invalid planned process pause/); +}); + +test('killing the pause owner preserves evidence and refuses a false live continuation', async () => { + const root = mkdtempSync(join(tmpdir(), 'depth-pause-owner-loss-')); + let child: ReturnType | undefined; + let exited: Promise | undefined; + try { + const manifest = JSON.parse(readFileSync(join(STACK_BENCH_ROOT, + 'appliance/campaign.ecommerce-progression-reference.json'), 'utf8')); + manifest.levels = [1, 2, 3]; + manifest.selection.levels = manifest.selection.levels.slice(0, 3); + manifest.parallelism = 3; + manifest.mode.pauseAfterDepth = 2; + const path = join(root, 'manifest.json'); + writeFileSync(path, JSON.stringify(manifest)); + const plan = compileCampaignFile(path); + const initialized = initializeCampaignDirectory(plan, join(root, 'results')); + const claimed = claimNextAttempt(initialized.state, { admissionId: 'owner-loss' }); + assert(claimed.claim); + writeCampaignState(initialized.paths.state, plan, claimed.state); + const directory = initialized.paths.root; + const output = join(directory, claimed.claim.output); + const app = join(output, 'app'); + mkdirSync(app, { recursive: true }); + writeFileSync(join(app, 'index.js'), 'accepted L2 source'); + writeFileSync(join(output, ARTIFACT_FILE.progressionState), '{}'); + const before = readFileSync(initialized.paths.state, 'utf8'); + const moduleUrl = (name: string) => pathToFileURL(compiledEntrypoint('src', 'campaigns', name)).href; + const script = ` + const { readCampaignState } = await import(${JSON.stringify(moduleUrl('campaign-scheduler.js'))}); + const { acquireCampaignLock } = await import(${JSON.stringify(moduleUrl('campaign-lock.js'))}); + const { waitAtDepthBoundary } = await import(${JSON.stringify(moduleUrl('campaign-depth-pause.js'))}); + const directory = ${JSON.stringify(directory)}; + const { plan } = readCampaignState(directory); + const lock = acquireCampaignLock(directory, plan); + await waitAtDepthBoundary(${JSON.stringify(output)}, ${JSON.stringify(app)}, { + directory, campaignSha256: plan.contentSha256, + ownershipMarkerSha256: lock.record.ownershipMarkerSha256, + attemptId: ${JSON.stringify(claimed.claim.attempt.id)}, + executionId: ${JSON.stringify(claimed.claim.executionId)}, depth: 2 }); + `; + child = spawn(process.execPath, ['--input-type=module', '-e', script], { stdio: 'ignore' }); + exited = once(child, 'exit'); + const receiptPath = join(output, 'depth-pause.json'); + const deadline = Date.now() + 30_000; + while (!existsSync(receiptPath) && child.exitCode === null && Date.now() < deadline) await delay(25); + assert(existsSync(receiptPath), 'child must reach the actual pause before termination'); + assert.equal(campaignDepthPauseStatus(directory).attempts.filter(a => a.paused).length, 1); + child.kill('SIGKILL'); + await exited; + assert.throws(() => continueCampaignDepth(directory), /live controller/); + assert.equal(JSON.parse(readFileSync(receiptPath, 'utf8')).resumedAt, null); + assert.equal(existsSync(join(directory, 'depth-release.json')), false); + assert.equal(readFileSync(initialized.paths.state, 'utf8'), before); + assert.equal(readFileSync(join(app, 'index.js'), 'utf8'), 'accepted L2 source'); + } finally { + if (child && child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); + await exited; + rmSync(root, { recursive: true, force: true }); + } +}); + +test('a continuation that inherits the paused depth does not pause again', () => { + const run = { levels: [{ level: 1 }, { level: 2 }] }; + assert.equal(depthPauseDue(run, 2, 3), true); + assert.equal(depthPauseDue({ ...run, pausedDurationMs: 5_000 }, 2, 3), false); + assert.equal(depthPauseDue(run, 2, 2), false); + assert.equal(depthPauseDue({ ...run, progressionResume: { inheritedLevels: [1, 2] } }, 2, 3), false); +}); diff --git a/tools/stack-bench/tests/campaign-lock-processes.integration.ts b/tools/stack-bench/tests/campaign-lock-processes.integration.ts new file mode 100644 index 00000000000..801c80cdbfc --- /dev/null +++ b/tools/stack-bench/tests/campaign-lock-processes.integration.ts @@ -0,0 +1,72 @@ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { acquireCampaignLock, readCampaignLock, releaseCampaignLock, + requestCampaignCancellation } from '../src/campaigns/campaign-lock.js'; +import { compiledEntrypoint } from '../src/package-root.js'; + +const campaign = { id: 'native-lock-proof', contentSha256: 'a'.repeat(64) }; +const linux = { skip: process.platform !== 'linux' ? 'Real campaign control requires the Linux Docker appliance and flock' : false }; + +test('native campaign lock excludes contenders and a durable Stop survives requester exit', linux, async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-native-campaign-')); + const helper = compiledEntrypoint('tests', 'fixtures', 'campaign-lock-process.js'); + const child = spawn(process.execPath, [helper, root], { stdio: ['ignore', 'pipe', 'pipe'] }); + const exited = new Promise((resolve, reject) => { + child.once('error', reject); child.once('exit', resolve); + }); + try { + await new Promise((resolve, reject) => { + let stderr = ''; + child.stderr.on('data', chunk => { stderr += chunk; }); + child.stdout.once('data', () => resolve()); + child.once('exit', code => reject(new Error(`owner exited ${code}: ${stderr}`))); + child.once('error', reject); + }); + const before = readCampaignLock(root)!; + assert.throws(() => acquireCampaignLock(root, campaign), /already controlled/); + assert.deepEqual(readCampaignLock(root), before); + assert.equal(requestCampaignCancellation(root, campaign, before.ownershipMarkerSha256), true); + assert.equal(await exited, 0); + assert.equal(readCampaignLock(root), null); + const replacement = acquireCampaignLock(root, campaign); + assert.equal(requestCampaignCancellation(root, campaign, before.ownershipMarkerSha256), false); + const bytes = readFileSync(replacement.path, 'utf8'); + assert.throws(() => releaseCampaignLock({ ...replacement, token: 'wrong' }), /token does not match/); + assert.equal(readFileSync(replacement.path, 'utf8'), bytes); + releaseCampaignLock(replacement); + } finally { + if (child.exitCode === null) { child.kill('SIGKILL'); await exited; } + rmSync(root, { recursive: true, force: true }); + } +}); + +test('concurrent native claimers cannot steal a live replacement while reclaiming a dead owner', linux, async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-native-campaign-race-')); + const helper = compiledEntrypoint('tests', 'fixtures', 'campaign-lock-process.js'); + // This pid is outside Linux pid_max on the appliance. + acquireCampaignLock(root, campaign, { ownerPid: 2147483647 }); + const children = Array.from({ length: 8 }, () => spawn(process.execPath, + [helper, root, 'race'], { stdio: ['ignore', 'pipe', 'pipe'] })); + try { + let rejected = 0; + const codes = await Promise.all(children.map(child => new Promise((resolve, reject) => { + child.once('error', reject); + child.once('exit', code => { + if (code === 2 && ++rejected === 7) { + const owner = readCampaignLock(root)!; + requestCampaignCancellation(root, campaign, owner.ownershipMarkerSha256); + } + resolve(code); + }); + }))); + assert.equal(codes.filter(code => code === 0).length, 1); + assert.equal(codes.filter(code => code === 2).length, 7); + } finally { + for (const child of children) if (child.exitCode === null) child.kill('SIGKILL'); + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/tools/stack-bench/tests/campaign-lock.test.ts b/tools/stack-bench/tests/campaign-lock.test.ts new file mode 100644 index 00000000000..28b92d4d12a --- /dev/null +++ b/tools/stack-bench/tests/campaign-lock.test.ts @@ -0,0 +1,180 @@ +import assert from 'node:assert/strict'; +import { createHash, randomUUID } from 'node:crypto'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { campaignLockIsActive, controllerInstance, campaignLockTransaction, + campaignCancellationRequested, watchCampaignCancellation, ownerAlive } from '../src/campaigns/campaign-lock.js'; +import type { CampaignIdentity, CampaignLock, CampaignLockRecord } from '../src/campaigns/campaign-lock.js'; + +const campaign = { id: 'ecommerce-l1-example', contentSha256: 'a'.repeat(64) }; + +// These tests exercise transaction invariants on every source-development platform. +// Native process exclusion is exercised separately on Linux under real flock. +function acquireCampaignLock(root: string, identity: CampaignIdentity, + { ownerPid = process.pid, ownerInstance = 'test-controller', uuid = randomUUID, + alive = () => false }: { ownerPid?: number; ownerInstance?: string; uuid?: () => string; + alive?: (record: Pick, current: string) => boolean } = {}): CampaignLock { + const token = uuid(); + const lock: CampaignLock = { path: join(root, '.campaign.lock.json'), token, + record: { version: 2, campaignId: identity.id, campaignSha256: identity.contentSha256, + ownerPid, ownerInstance, ownershipMarkerSha256: createHash('sha256').update(token).digest('hex'), + acquiredAt: new Date().toISOString() } }; + campaignLockTransaction({ operation: 'acquire', lock }, { alive }); + return lock; +} +const releaseCampaignLock = (lock: CampaignLock): boolean => + campaignLockTransaction({ operation: 'release', lock }); + +test('controller identity prefers explicit configuration over container and host identity', () => { + const id = 'ab'.repeat(32); + const containerMount = [ + '1044 997 0:81 / / rw,relatime - overlay overlay rw', + `1051 1044 0:70 /docker/containers/${id}/hostname /etc/hostname rw,relatime - tmpfs tmpfs rw`, + '1052 1044 0:70 /docker/containers/not-an-id/hostname /etc/hostname rw - tmpfs tmpfs rw', + ].join('\n'); + const noProcfs = (): string => { throw Object.assign(new Error('no procfs'), { code: 'ENOENT' }); }; + for (const [env, readMountInfo, fallback, expected] of [ + [{ STACK_BENCH_CONTROLLER_INSTANCE: ' controller-explicit ' }, null, 'unused-host', 'controller-explicit'], + [{}, () => containerMount, 'docker-desktop', id], + [{}, noProcfs, 'host-controller', 'host-controller'], + [{}, () => '/docker/containers/short-id/hostname /etc/hostname', 'host-controller', 'host-controller'], + ] as const) { + let mountRead = false; + assert.equal(controllerInstance(env, { + readMountInfo: () => { mountRead = true; return readMountInfo ? readMountInfo() : 'unused'; }, + fallbackHostname: () => fallback, + }), expected); + assert.equal(mountRead, readMountInfo !== null); + } +}); + +test('cross-controller liveness fails closed when Docker cannot answer', () => { + const container = 'ab'.repeat(32); + const record = { ownerInstance: container, ownerPid: 14 }; + assert.throws(() => ownerAlive(record, 'cd'.repeat(32), { + inspect: () => ({ error: new Error('docker unavailable'), status: null }), + }), /cannot inspect controller/); + assert.throws(() => ownerAlive(record, 'cd'.repeat(32), { + inspect: () => ({ status: 1, stdout: '', stderr: 'permission denied' }), + }), /cannot determine whether controller/); +}); + +test('cross-controller liveness reclaims only a stopped or missing exact container', () => { + const container = 'ab'.repeat(32); + const record = { ownerInstance: container, ownerPid: 14 }; + assert.equal(ownerAlive(record, 'cd'.repeat(32), { + inspect: () => ({ status: 0, stdout: 'true\n', stderr: '' }), + }), true); + assert.equal(ownerAlive(record, 'cd'.repeat(32), { + inspect: () => ({ status: 0, stdout: 'false\n', stderr: '' }), + }), false); + assert.equal(ownerAlive(record, 'cd'.repeat(32), { + inspect: () => ({ status: 1, stdout: '', stderr: `Error: No such object: ${container}` }), + }), false); +}); + +test('a campaign lock admits one exact controller and only its token can release it', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-campaign-lock-')); + try { + const lock = acquireCampaignLock(root, campaign, { ownerInstance: 'controller-a', + uuid: () => 'owner-token', alive: () => true }); + for (const ownerInstance of ['controller-a', 'controller-b']) { + assert.throws(() => acquireCampaignLock(root, campaign, { ownerInstance, + alive: record => record.ownerInstance === 'controller-a' }), + new RegExp(`already controlled by controller-a pid ${process.pid}`)); + } + assert.throws(() => releaseCampaignLock({ ...lock, token: 'wrong-token' }), /token does not match|no longer belongs/); + assert.equal(releaseCampaignLock(lock), true); + assert.equal(releaseCampaignLock(lock), false); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('dead-owner reclamation preserves campaign binding and rejects the old releaser', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-campaign-lock-stale-')); + try { + const first = acquireCampaignLock(root, campaign, { ownerPid: 1001, + ownerInstance: 'dead-controller', uuid: () => 'first-token', alive: () => false }); + const inspected: Array<{ record: { ownerInstance: string; ownerPid: number }; current: string }> = []; + const second = acquireCampaignLock(root, campaign, { ownerPid: 1002, + ownerInstance: 'new-controller', + uuid: (() => { let index = 0; return () => `second-token-${++index}`; })(), + alive: (record, current) => { inspected.push({ record, current }); return false; } }); + assert.equal(inspected[0]?.record.ownerInstance, 'dead-controller'); + assert.equal(inspected[0]?.current, 'new-controller'); + const record = JSON.parse(readFileSync(second.path, 'utf8')); + assert.equal(record.ownerPid, 1002); + assert.equal(record.campaignSha256, campaign.contentSha256); + assert.throws(() => releaseCampaignLock(first), /no longer belongs/); + assert.equal(releaseCampaignLock(second), true); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('campaign lock liveness is read without changing stale ownership evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-campaign-lock-status-')); + try { + assert.equal(campaignLockIsActive(root, campaign, { currentInstance: 'dashboard', + alive: () => true }), false); + const lock = acquireCampaignLock(root, campaign, { ownerPid: 14, + ownerInstance: 'campaign-controller', uuid: () => 'campaign-token', alive: () => false }); + assert.equal(campaignLockIsActive(root, campaign, { currentInstance: 'dashboard', + alive: record => record.ownerInstance === 'campaign-controller' }), true); + assert.equal(campaignLockIsActive(root, campaign, { currentInstance: 'dashboard', + alive: () => false }), false); + assert.throws(() => campaignLockIsActive(root, + { ...campaign, contentSha256: 'b'.repeat(64) }, { alive: () => true }), /does not belong/); + assert.equal(releaseCampaignLock(lock), true); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('malformed ownership evidence is quarantined rather than guessed or stolen', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-campaign-lock-bad-')); + try { + const path = join(root, '.campaign.lock.json'); + writeFileSync(path, '{"ownerPid":123}\n'); + assert.throws(() => acquireCampaignLock(root, campaign, { alive: () => false }), + /required|unsupported/); + assert.equal(readFileSync(path, 'utf8'), '{"ownerPid":123}\n'); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + + +test('cancellation belongs to one lock generation and a stale Stop cannot cancel its replacement', async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-campaign-cancel-')); + try { + const first = acquireCampaignLock(root, campaign); + const cancel = (lock: CampaignLock): boolean => campaignLockTransaction({ operation: 'cancel', + path: lock.path, campaign, ownershipMarkerSha256: lock.record.ownershipMarkerSha256, + requestedAt: new Date().toISOString() }); + const watcher = watchCampaignCancellation(first); + try { + assert.equal(cancel(first), true); + assert.equal(cancel(first), true); + assert.equal(campaignCancellationRequested(first), true); + await new Promise(resolve => setTimeout(resolve, 300)); + assert.equal(watcher.signal.aborted, true); + } finally { watcher.close(); } + const second = acquireCampaignLock(root, campaign); + assert.equal(cancel(first), false); + assert.equal(campaignCancellationRequested(second), false); + const before = readFileSync(second.path, 'utf8'); + assert.throws(() => releaseCampaignLock(first), /no longer belongs/); + assert.equal(readFileSync(second.path, 'utf8'), before); + assert.equal(cancel(second), true); + assert.equal(campaignCancellationRequested(second), true); + releaseCampaignLock(second); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('external cancellation reaches the owner watcher immediately', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-campaign-cancel-signal-')); + try { + const lock = acquireCampaignLock(root, campaign); + const source = new AbortController(); + const watcher = watchCampaignCancellation(lock, source.signal); + try { source.abort(); assert.equal(watcher.signal.aborted, true); } + finally { watcher.close(); releaseCampaignLock(lock); } + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/tools/stack-bench/tests/campaign-mode.test.ts b/tools/stack-bench/tests/campaign-mode.test.ts new file mode 100644 index 00000000000..af3509f886f --- /dev/null +++ b/tools/stack-bench/tests/campaign-mode.test.ts @@ -0,0 +1,27 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { CAMPAIGN_MODE_REGISTRY } + from '../src/campaigns/campaign-mode.js'; + +test('the campaign mode registry requires an exact supported mode', () => { + assert.deepEqual(CAMPAIGN_MODE_REGISTRY.validate({ id: 'sequential' }), { id: 'sequential' }); + assert.throws(() => CAMPAIGN_MODE_REGISTRY.validate({ + id: 'sequential', version: '1.0.0', + }), /version is unknown/); + assert.throws(() => CAMPAIGN_MODE_REGISTRY.validate({ + id: 'sequential', extra: true, + }), /extra is unknown/); + assert.deepEqual(CAMPAIGN_MODE_REGISTRY.validate({ + id: 'dependency', + }), { id: 'dependency', workSelection: 'progressive' }); + assert.equal(CAMPAIGN_MODE_REGISTRY.validate({ + id: 'dependency', workSelection: 'feature', + }).workSelection, 'feature'); + assert.equal(CAMPAIGN_MODE_REGISTRY.validate({ + id: 'dependency', workSelection: 'all-at-once', + }).workSelection, 'all-at-once'); + assert.throws(() => CAMPAIGN_MODE_REGISTRY.validate({ + id: 'dependency', repairSelection: 'feature', + }), /repairSelection is unknown/); +}); diff --git a/tools/stack-bench/tests/campaign-progression-grant.test.ts b/tools/stack-bench/tests/campaign-progression-grant.test.ts new file mode 100644 index 00000000000..456cd0e7b42 --- /dev/null +++ b/tools/stack-bench/tests/campaign-progression-grant.test.ts @@ -0,0 +1,186 @@ +import assert from 'node:assert/strict'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { grantCampaignDependencyRepairs, prepareGrantWorkspace } + from '../src/campaigns/campaign-progression-grant.js'; +import { compileDependencyPolicyInput, compileFeatureCatalogInput } + from '../src/progression/progression-definition.js'; + +interface TestContinuation { + grantId: string; + level: number; + nodeIds: string[]; + repairs: number; + stateSha256: string; + resumeFrom: string; + scheduledAt: string; +} + +function campaignFixture({ marker = null, campaignStatus = 'completed' }: + { marker?: TestContinuation | null; campaignStatus?: string } = {}) { + const featureCatalog = compileFeatureCatalogInput({ + schemaVersion: 1, + kind: 'feature-catalog', + id: 'test.catalog', + title: 'Test catalog', + nodes: [{ id: 'accounts', title: 'Accounts', questline: 'identity', + featureRefs: ['feature.accounts'], promptModules: [], + gradingChecks: [{ id: 'accounts.create', points: 1, role: 'feature' }], dependencies: [] }], + questlines: [{ id: 'identity', title: 'Identity', nodes: ['accounts'] }], + }); + const dependencyPolicy = compileDependencyPolicyInput({ + selection: 'feature', budget: { perFeature: 1 }, + }, featureCatalog); + const attemptPlan = { + id: 'campaign-r1-c1-a1-postgres', mode: { id: 'dependency' }, + stack: 'postgres', agentAdapter: 'claude-code', model: 'test-model', + condition: { contentSha256: 'c'.repeat(64) }, + }; + const execution = { + id: `${attemptPlan.id}-execution1`, status: 'completed', + output: `attempts/${attemptPlan.id}/execution-1`, + ...(marker === null ? {} : { continuation: marker }), + }; + const plan = { + id: 'campaign', version: '1.0.0', contentSha256: 'a'.repeat(64), + definition: { mode: { id: 'dependency' }, track: 'ecommerce' }, + featureCatalog, dependencyPolicy, + }; + const state = { status: campaignStatus, attempts: [{ plan: attemptPlan, + status: marker === null ? 'completed' : 'pending', executions: [execution] }] }; + return { plan, state, paths: { state: 'campaign-state.json' } }; +} + +const input = { + attemptId: 'campaign-r1-c1-a1-postgres', + grantId: 'operator-grant-1', + level: 1, + nodeIds: ['accounts'], + repairs: 2, +}; + +test('a grant workspace copies only resumable evidence without changing its source', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-campaign-grant-')); + try { + const execution = join(root, 'attempts', input.attemptId, 'execution-1'); + mkdirSync(join(execution, 'source'), { recursive: true }); + mkdirSync(join(execution, 'progression', 'attempt-001'), { recursive: true }); + writeFileSync(join(execution, 'run.json'), '{"status":"complete"}\n'); + writeFileSync(join(execution, 'progression-state.json'), '{}\n'); + writeFileSync(join(execution, 'source', 'app.js'), 'export const value = 1;\n'); + writeFileSync(join(execution, 'progression', 'attempt-001', 'bundle.json'), '{}\n'); + writeFileSync(join(execution, 'process.stdout.log'), 'not resumable\n'); + const prepared = prepareGrantWorkspace(root, execution, input.attemptId, input.grantId); + assert.equal(prepared.relativePath, + `continuations/${input.attemptId}/${input.grantId}`); + writeFileSync(join(prepared.directory, 'source', 'app.js'), 'export const value = 2;\n'); + assert.equal(readFileSync(join(execution, 'source', 'app.js'), 'utf8'), + 'export const value = 1;\n'); + assert.equal(existsSync(join(prepared.directory, 'process.stdout.log')), false); + assert.equal(prepareGrantWorkspace(root, execution, input.attemptId, input.grantId).created, + false); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('campaign repair grant derives exact state, owner, and continuation marker', () => { + const campaign = campaignFixture(); + const calls: { + inspect: number; + released: boolean; + written: { path: string; plan: unknown; state: unknown } | null; + } = { inspect: 0, released: false, written: null }; + const result = grantCampaignDependencyRepairs('campaign-output', input, { + inspect: () => { calls.inspect += 1; return structuredClone(campaign); }, + acquire: (_root, plan) => { assert.equal(plan.contentSha256, 'a'.repeat(64)); return 'lock'; }, + release: lock => { assert.equal(lock, 'lock'); calls.released = true; }, + prepareWorkspace: () => ({ + directory: 'campaign-output/continuations/campaign-r1-c1-a1-postgres/operator-grant-1', + relativePath: 'continuations/campaign-r1-c1-a1-postgres/operator-grant-1', + created: true, + }), + readState: (path, options) => { + assert.match(path, /operator-grant-1[\\/]progression-state\.json$/); + assert.equal(options.owner.attempt.id, input.attemptId); + assert.equal(options.owner.attempt.conditionSha256, 'c'.repeat(64)); + assert.equal(options.requireCurrentEngine, true); + return { stateSha256: 'b'.repeat(64), state: { phase: 'terminal', grants: [] } }; + }, + grantState: (_path, options) => { + assert.deepEqual(options.grant, { + grantId: input.grantId, level: 1, nodeIds: ['accounts'], repairs: 2, + }); + assert.equal(options.expectedStateSha256, 'b'.repeat(64)); + return { stateSha256: 'd'.repeat(64) }; + }, + schedule: (state, attemptId, marker, options) => { + assert.equal(state.status, 'completed'); + assert.equal(attemptId, input.attemptId); + assert.deepEqual(marker, { grantId: input.grantId, level: 1, + nodeIds: ['accounts'], repairs: 2, stateSha256: 'd'.repeat(64), + resumeFrom: 'continuations/campaign-r1-c1-a1-postgres/operator-grant-1' }); + assert.equal(options.now, '2026-08-28T12:00:00.000Z'); + return { status: 'prepared' }; + }, + writeState: (path, plan, state) => { calls.written = { path, plan, state }; }, + now: '2026-08-28T12:00:00.000Z', + }); + assert.equal(calls.inspect, 2); + assert.equal(calls.released, true); + assert.ok(calls.written); + assert.equal(calls.written.path, 'campaign-state.json'); + assert.equal(result.stateSha256, 'd'.repeat(64)); + assert.equal(result.scheduled, true); +}); + +test('campaign repair grant is idempotent only for the exact recorded marker', () => { + const marker = { grantId: input.grantId, level: 1, nodeIds: ['accounts'], repairs: 2, + stateSha256: 'd'.repeat(64), + resumeFrom: 'continuations/campaign-r1-c1-a1-postgres/operator-grant-1', + scheduledAt: '2026-08-28T12:00:00.000Z' }; + const campaign = campaignFixture({ marker, campaignStatus: 'prepared' }); + let touched = false; + const result = grantCampaignDependencyRepairs('campaign-output', input, { + inspect: () => structuredClone(campaign), + acquire: () => 'lock', release: () => {}, + readState: () => ({ stateSha256: marker.stateSha256, + state: { grants: [{ grantId: input.grantId, level: 1, + nodeIds: ['accounts'], repairs: 2 }] } }), + grantState: () => { touched = true; return { stateSha256: 'e'.repeat(64) }; }, + schedule: () => { touched = true; }, writeState: () => { touched = true; }, + }); + assert.equal(result.stateSha256, marker.stateSha256); + assert.equal(touched, false); + assert.throws(() => grantCampaignDependencyRepairs('campaign-output', { + ...input, repairs: 3, + }, { inspect: () => structuredClone(campaign), acquire: () => 'lock', release: () => {} }), + /different continuation/); +}); + +test('replaying a grant after its continuation completed schedules nothing', () => { + const marker = { grantId: input.grantId, level: 1, nodeIds: ['accounts'], repairs: 2, + stateSha256: 'd'.repeat(64), + resumeFrom: 'continuations/campaign-r1-c1-a1-postgres/operator-grant-1', + scheduledAt: '2026-08-28T12:00:00.000Z' }; + const campaign = campaignFixture({ marker }); + const attempt = campaign.state.attempts[0]!; + attempt.status = 'completed'; + attempt.executions.push({ id: `${attempt.plan.id}-execution2`, status: 'completed', + output: `attempts/${attempt.plan.id}/execution-2` }); + let touched = false; + const options = { + inspect: () => structuredClone(campaign), acquire: () => 'lock', release: () => {}, + prepareWorkspace: () => { touched = true; return { directory: 'x', relativePath: 'x', created: false }; }, + grantState: () => { touched = true; return { stateSha256: 'e'.repeat(64) }; }, + schedule: () => { touched = true; }, writeState: () => { touched = true; }, + }; + const result = grantCampaignDependencyRepairs('campaign-output', input, options); + assert.equal(touched, false); + assert.equal(result.execution, `attempts/${attempt.plan.id}/execution-1`); + assert.throws(() => grantCampaignDependencyRepairs('campaign-output', { ...input, repairs: 3 }, options), + /different continuation/); +}); diff --git a/tools/stack-bench/tests/campaign-provider-continuation.test.ts b/tools/stack-bench/tests/campaign-provider-continuation.test.ts new file mode 100644 index 00000000000..485daf39e57 --- /dev/null +++ b/tools/stack-bench/tests/campaign-provider-continuation.test.ts @@ -0,0 +1,97 @@ +import assert from 'node:assert/strict'; +import { createHash, randomUUID } from 'node:crypto'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { compileCampaignFile } from '../src/campaigns/campaign-compiler.js'; +import { campaignLockTransaction, controllerInstance } from '../src/campaigns/campaign-lock.js'; +import { claimNextAttempt, initializeCampaignDirectory, writeCampaignState } from '../src/campaigns/campaign-scheduler.js'; +import { persistCampaignProviderInvocation, readCampaignProviderContinuationStatus, + readCampaignProviderWaitHistory, requestCampaignProviderContinuation, waitForCampaignProviderContinuation } + from '../src/campaigns/campaign-provider-continuation.js'; + +test('live provider continuation binds one durable acceptance to its wait and rejects dead owners', () => { + const directory = mkdtempSync(join(tmpdir(), 'provider-continuation-')); + try { + const plan = compileCampaignFile(join(STACK_BENCH_ROOT, 'tests', 'fixtures', 'campaign.deterministic.json')); + const initialized = initializeCampaignDirectory(plan, directory); + const claimed = claimNextAttempt(initialized.state, { admissionId: 'test-admission' }); + assert(claimed.claim); + writeCampaignState(initialized.paths.state, plan, claimed.state); + const attemptId = claimed.claim.attempt.id; + const execution = claimed.state.attempts.find(a => a.plan.id === attemptId)!.executions.at(-1)!; + const marker = createHash('sha256').update('test').digest('hex'); + campaignLockTransaction({ operation: 'acquire', lock: { path: join(directory, '.campaign.lock.json'), + token: 'test', record: { version: 2, campaignId: plan.id, campaignSha256: plan.contentSha256, + ownerPid: process.pid, ownerInstance: controllerInstance(), ownershipMarkerSha256: marker, + acquiredAt: new Date().toISOString() } } }); + const root = join(directory, execution.output, 'provider-waits'); + const env = { STACK_BENCH_PROVIDER_WAIT_CONTEXT: JSON.stringify({ directory, campaignSha256: plan.contentSha256, + attemptId, executionId: execution.id, ownershipMarkerSha256: marker, root }) }; + const evidence = { actionId: 'action-1', invocation: 1, costUsd: 0.5 }; + persistCampaignProviderInvocation({ env, evidence }); + assert.throws(() => persistCampaignProviderInvocation({ env, evidence }), /EEXIST/); + const before = readFileSync(initialized.paths.state, 'utf8'); + let validations = 0; + const result = waitForCampaignProviderContinuation({ env, evidence, + validate: () => { validations++; }, sleep: () => { + assert.equal(readCampaignProviderContinuationStatus(directory, attemptId).eligible, true); + const request = requestCampaignProviderContinuation(directory, { attemptId, requestId: 'continue-1' }); + assert.deepEqual(requestCampaignProviderContinuation(directory, { attemptId, requestId: 'continue-1' }), request); + } }); + assert(result); + assert.equal(validations, 2); + assert.equal(JSON.parse(readFileSync(join(root, 'events', `${result.generation}.accepted.json`), 'utf8')).requestId, 'continue-1'); + assert.equal(JSON.parse(readFileSync(join(root, 'events', `${result.generation}.waiting.json`), 'utf8')).disposition, 'waiting'); + assert.equal(readCampaignProviderContinuationStatus(directory, attemptId).waiting, false); + assert.equal(readFileSync(initialized.paths.state, 'utf8'), before); + assert.throws(() => waitForCampaignProviderContinuation({ env, evidence, validate: () => {}, sleep: () => { + requestCampaignProviderContinuation(directory, { attemptId, requestId: 'continue-1' }); + } }), /different wait generation/); + const attempt = claimed.state.attempts.find(a => a.plan.id === attemptId)!; + const originalMinutes = plan.definition.budgets.attemptTimeoutMinutes; + execution.startedAt = new Date(Date.now() - (originalMinutes + 1) * 60_000).toISOString(); + claimed.state.createdAt = execution.startedAt; + writeCampaignState(initialized.paths.state, plan, claimed.state); + assert.match(readCampaignProviderContinuationStatus(directory, attemptId).reason ?? '', /duration/); + assert.throws(() => requestCampaignProviderContinuation(directory, { attemptId, requestId: 'after-deadline' }), /duration/); + const grantAt = new Date().toISOString(); + claimed.state.updatedAt = grantAt; + attempt.timeGrants = [{ disposition: 'accepted', acceptedAt: grantAt, + previousMinutes: originalMinutes, effectiveMinutes: originalMinutes + 60, + request: { attemptId, executionId: execution.id, campaignSha256: plan.contentSha256, + grantId: 'more-time', minutes: 60, requestedAt: grantAt } }]; + writeCampaignState(initialized.paths.state, plan, claimed.state); + assert(waitForCampaignProviderContinuation({ env, evidence, validate: () => {}, sleep: () => { + requestCampaignProviderContinuation(directory, { attemptId, requestId: 'after-grant' }); + } })); + const lockPath = join(directory, '.campaign.lock.json'); + const lock = JSON.parse(readFileSync(lockPath, 'utf8')); + writeFileSync(lockPath, JSON.stringify({ ...lock, ownerPid: 2_147_483_647 })); + assert.equal(readCampaignProviderContinuationStatus(directory, attemptId).eligible, false); + assert.throws(() => requestCampaignProviderContinuation(directory, { attemptId, requestId: 'continue-2' }), /active/); + persistCampaignProviderInvocation({ env, evidence: { ...evidence, invocation: 2 } }); + assert.equal(JSON.parse(readFileSync(join(root, 'invocations', 'action-1-2.json'), 'utf8')).evidence.costUsd, 0.5); + assert.throws(() => persistCampaignProviderInvocation({ env: { STACK_BENCH_PROVIDER_WAIT_CONTEXT: + JSON.stringify({ ...JSON.parse(env.STACK_BENCH_PROVIDER_WAIT_CONTEXT), root: join(directory, 'other') }) }, + evidence: { ...evidence, invocation: 3 } }), /does not belong/); + const killedAt = Date.now(); + const killed = { ...JSON.parse(readFileSync(join(root, 'current.json'), 'utf8')), + generation: randomUUID(), disposition: 'waiting', + createdAt: new Date(killedAt - 90_000).toISOString(), heartbeatAt: new Date(killedAt - 60_000).toISOString() }; + writeFileSync(join(root, 'events', `${killed.generation}.waiting.json`), JSON.stringify(killed)); + writeFileSync(join(root, 'current.json'), JSON.stringify(killed)); + const history = readCampaignProviderWaitHistory(directory, attemptId, execution.id); + assert(history); + assert.equal(history.waits, 4); + assert.equal(history.continued, 2); + assert.equal(history.stopped, 2); + assert.equal(history.durationKind, 'lower-bound'); + assert.equal(history.details.find(wait => wait.generation === killed.generation)?.waitedMs, 30_000); + const acceptedPath = join(root, 'events', `${result.generation}.accepted.json`); + writeFileSync(acceptedPath, JSON.stringify({ ...JSON.parse(readFileSync(acceptedPath, 'utf8')), executionId: 'foreign' })); + assert.throws(() => readCampaignProviderWaitHistory(directory, attemptId, execution.id), /acceptance binding/); + } finally { rmSync(directory, { recursive: true, force: true }); } +}); diff --git a/tools/stack-bench/tests/campaign-publish.contract.ts b/tools/stack-bench/tests/campaign-publish.contract.ts new file mode 100644 index 00000000000..4a4ecdee444 --- /dev/null +++ b/tools/stack-bench/tests/campaign-publish.contract.ts @@ -0,0 +1,388 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { createServer } from 'node:http'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import test from 'node:test'; +import { + compileCampaignFile, + campaignProgressionOwner, +} from '../src/campaigns/campaign-compiler.js'; +import { + claimNextAttempt, + finishCampaignExecution, + initializeCampaignDirectory, + writeCampaignState, +} from '../src/campaigns/campaign-scheduler.js'; +import { generateCampaignReport } from '../src/campaigns/campaign-report.js'; +import { createCampaignUpload, publishCampaign } from '../src/campaigns/campaign-publish.js'; +import { + artifactPayload, + emptyArtifactIdentities, + writeArtifact, +} from '../src/evidence/artifacts.js'; +import { createCheckEvidence } from '../src/evidence/check-evidence.js'; +import { classifyBundle } from '../src/evidence/outcomes.js'; +import { hashDirectory } from '../src/evidence/provenance.js'; +import { + compileProgressionInput, + dependencyRuntimeDefinition, +} from '../src/progression/progression-definition.js'; +import { progressionEngine } from '../src/progression/progression-engine.js'; +import { dependencyLevelRepairRecords } from '../src/progression/dependency-mode.js'; +import { liveProgressionStatus } from '../src/progression/live-progression.js'; +import { writeProgressionState } from '../src/progression/progression-state.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; + +test('retained dependency evidence exports through the CLI and sends only validated private uploads', async t => { + const root = mkdtempSync(join(tmpdir(), 'stackbench-publish-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const definition = JSON.parse( + readFileSync( + join(STACK_BENCH_ROOT, 'tests/fixtures/dependency-model-free-campaign.json'), + 'utf8' + ) + ); + definition.state = 'frozen'; + definition.budgets.maxCostUsdPerAttempt = 10; + const buildImage = `sha256:${'b'.repeat(64)}`; + definition.runtime.controllerImage = `sha256:${'c'.repeat(64)}`; + definition.runtime.buildImage = buildImage; + definition.repair.budget = { total: 0, perFeature: 0 }; + definition.featureCatalog.nodes[1].dependencies = [ + { id: 'accounts', reason: 'Catalog needs an account' }, + ]; + definition.selection.levels.push({ level: 2, recipe: 'ecommerce.sequential-l2' }); + definition.repetitions = 2; + const manifest = join(root, 'fixture.json'); + writeFileSync(manifest, JSON.stringify(definition)); + const plan = compileCampaignFile(manifest); + assert.ok(plan.featureCatalog && plan.dependencyPolicy); + const campaign = join(root, 'campaign'); + const now = '2026-10-14T12:00:00.000Z'; + let { state } = initializeCampaignDirectory(plan, campaign, { now }); + assert.throws(() => createCampaignUpload(campaign, 'fixture-suite'), /finished/); + writeArtifact(join(campaign, 'admissions/fixture.json'), { + kind: 'campaign_admission', + id: 'fixture', + identities: emptyArtifactIdentities({ + experiment: { + id: plan.id, + version: plan.version, + sha256: plan.contentSha256, + state: plan.state, + }, + }), + payload: { + schemaVersion: 1, + campaignId: plan.id, + campaignSha256: plan.contentSha256, + createdAt: now, + ok: true, + runtime: plan.definition.runtime, + conditions: plan.conditions, + agents: plan.agents.map(agent => ({ + adapter: agent.adapter, + model: agent.model, + identity: agent.identity, + })), + reports: [ + { + schemaVersion: 1, + generatedAt: now, + ok: true, + request: { + backends: plan.stacks.map(stack => stack.id), + track: plan.definition.track, + levels: plan.definition.levels, + runIndex: 0, + parallelism: plan.summary.parallelism, + agentAdapter: plan.agents[0]!.adapter, + packs: [], + checks: [], + image: buildImage, + resultsDir: campaign, + smoke: false, + }, + summary: { passed: 0, failed: 0, warnings: 0 }, + checks: [], + }, + ], + }, + }); + const claim = claimNextAttempt(state, { now, admissionId: 'fixture' }); + assert.ok(claim.claim); + const attempt = claim.claim.attempt; + const output = join(campaign, claim.claim.output); + mkdirSync(join(output, 'source'), { recursive: true }); + writeFileSync(join(output, 'source/app.js'), 'export const ready = true;\n'); + const progression = compileProgressionInput( + dependencyRuntimeDefinition(plan.featureCatalog, plan.dependencyPolicy) + ); + const initial = progressionEngine.initialize(progression.definition); + const selection = progressionEngine.gradingSelection(initial); + const selectionSha256 = 'a'.repeat(64); + const graded = progressionEngine.recordResult(initial, { + attemptId: 'fixture-grade', + outcome: 'conclusive', + selectionSha256, + nodes: selection.nodeIds.map(id => ({ + id, + checks: selection.checks + .filter(check => check.nodeId === id) + .map(check => ({ id: check.id, outcome: 'fail' as const })), + })), + }); + assert.equal(graded.phase, 'terminal'); + assert.equal(graded.nodes.catalog!.status, 'blocked'); + writeProgressionState(join(output, 'progression-state.json'), { + progression, + featureCatalogIdentity: plan.featureCatalog.identity, + dependencyPolicyIdentity: plan.dependencyPolicy.identity, + owner: campaignProgressionOwner(plan, attempt, { workspace: true }), + state: graded, + }); + const checks = selection.checks.map(check => ({ stableKey: check.id, points: check.points })); + const max = checks.reduce((sum, check) => sum + check.points, 0); + const bundle = writeArtifact(join(output, 'grading/bundle.json'), { + kind: 'grade_bundle', + id: 'fixture-grade', + payload: { + observation: 'scored', + source: { sha256: hashDirectory(join(output, 'source')).sha256 }, + suites: { + fixture: { + features: [ + { + id: 'fixture', + setupEvidence: createCheckEvidence({ + status: 'passed', + code: 'completed', + phase: 'setup', + startedAtMs: 1, + completedAtMs: 2, + }), + criteria: checks.map(check => ({ + id: check.stableKey, + stableKey: check.stableKey, + points: check.points, + evidence: createCheckEvidence({ + status: 'failed', + code: 'test_result', + phase: 'assertion', + startedAtMs: 1, + completedAtMs: 2, + }), + })), + }, + ], + }, + }, + totals: { score: 0, max }, + selection: { + sha256: selectionSha256, + checks, + attemptedChecks: checks.map(check => check.stableKey), + reportedChecks: checks.map(check => check.stableKey), + notRun: [], + }, + }, + }); + const gradeOutcome = classifyBundle(bundle.payload); + const status = liveProgressionStatus(graded); + const run = writeArtifact(join(output, 'run.json'), { + kind: 'benchmark_run', + id: 'fixture-run', + attempt: { id: 'fixture-run', parentId: attempt.id }, + identities: emptyArtifactIdentities({ + engine: plan.identities.engine, + experiment: { + id: plan.id, + version: plan.version, + sha256: plan.contentSha256, + state: plan.state, + }, + agentAdapter: plan.agents[0]!.identity, + stackAdapter: plan.stacks[0]!, + }), + payload: { + mode: attempt.mode, + track: plan.definition.track, + backend: attempt.stack, + model: attempt.model, + pricing: attempt.pricing, + guidance: attempt.guidance, + condition: attempt.condition, + selectionRequest: plan.definition.selection, + featureCatalog: attempt.featureCatalog, + dependencyPolicy: attempt.dependencyPolicy, + progressionOwner: campaignProgressionOwner(plan, attempt), + progressionStatus: status, + skills: attempt.skills, + runtime: { buildImage }, + validation: { + ladder: { + policy: 'dependency-graph', + requestedLevels: attempt.levels, + completedLevels: [1], + stoppedAfterLevel: null, + blockedLevels: [], + }, + }, + totals: { + score: 0, + max: status.score.uniqueChecks.availablePoints, + costUsd: 0, + costComplete: false, + }, + levels: [ + { + level: 1, + graded: true, + selection: { sha256: selectionSha256 }, + score: 0, + max, + repairs: 0, + repair: { + status: 'budget-exhausted', + limit: 0, + used: 0, + stopReason: null, + nodeRepairs: dependencyLevelRepairRecords(graded, 1), + }, + outcome: gradeOutcome, + }, + ], + outcome: { kind: 'app_failure', phase: 'grading', reason: null }, + }, + }); + state = finishCampaignExecution( + claim.state, + claim.claim.executionId, + { exitCode: 0, run: artifactPayload(run) }, + { now } + ); + const invalid = claimNextAttempt(state, { now, admissionId: 'fixture' }); + assert.ok(invalid.claim); + state = finishCampaignExecution( + invalid.state, + invalid.claim.executionId, + { exitCode: 1, run: null }, + { now } + ); + writeCampaignState(join(campaign, 'state.json'), plan, state); + assert.throws(() => createCampaignUpload(campaign, 'fixture-suite'), /report/); + generateCampaignReport(campaign); + const before = readFileSync(join(campaign, 'report/report.json')); + const payload = createCampaignUpload(campaign, 'fixture-suite'); + assert.equal(payload.attempts.length, 2); + assert.equal(payload.gradingStatus, 'pending'); + assert.deepEqual( + [payload.attempts[0]!.featuresPassed, payload.attempts[0]!.featuresTotal], + [0, 2] + ); + assert.equal(payload.attempts[0]!.costUsd, null); + assert.equal(payload.attempts[0]!.costStatus, 'unknown'); + assert.equal(payload.attempts[1]!.featuresPassed, null); + assert.equal(payload.attempts[1]!.excluded, true); + assert.ok(payload.attempts[1]!.excludedReason); + // An operator exclusion keeps the attempt in the upload, marked with its reason. + const replaced = payload.attempts[0]!.attemptId; + const withExclusion = createCampaignUpload(campaign, 'fixture-suite', + new Map([[replaced, 'replaced by a later run']])); + assert.equal(withExclusion.attempts.length, 2); + assert.equal(withExclusion.attempts[0]!.excluded, true); + assert.equal(withExclusion.attempts[0]!.excludedReason, 'excluded by operator: replaced by a later run'); + assert.equal(withExclusion.attempts[0]!.featuresPassed, payload.attempts[0]!.featuresPassed); + assert.throws(() => createCampaignUpload(campaign, 'fixture-suite', new Map([['no-such-attempt', 'x']])), + /unknown attempt/); + assert.throws(() => createCampaignUpload(campaign, 'fixture-suite', new Map([[replaced, ' ']])), /reason/); + assert.deepEqual(readFileSync(join(campaign, 'report/report.json')), before); + const exported = join(root, 'upload.json'); + const cli = join(dirname(fileURLToPath(import.meta.url)), '../commands/campaign-cli.js'); + const result = spawnSync( + process.execPath, + [cli, 'publish', campaign, '--suite', 'fixture-suite', '--out', exported], + { encoding: 'utf8' } + ); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(JSON.parse(readFileSync(exported, 'utf8')), payload); + await assert.rejects( + publishCampaign(campaign, { suite: 'fixture-suite', output: exported }), + /exist/i + ); + const requests: unknown[] = []; + let responseStatus = 201; + let responseVersion = 1; + const server = createServer(async (request, response) => { + assert.equal(request.url, '/api/uploads'); + assert.equal(request.headers.authorization, 'Bearer fixture-key'); + let body = ''; + for await (const chunk of request) body += String(chunk); + requests.push(JSON.parse(body)); + response.writeHead(responseStatus, { 'Content-Type': 'application/json', Connection: 'close' }); + response.end( + JSON.stringify({ + version: responseVersion, + uploadId: 'fixture-upload', + status: 'pending', + duplicate: false, + runs: 2, + ignored: 'private response metadata', + }) + ); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(() => server.close()); + const address = server.address(); + assert.ok(address && typeof address !== 'string'); + process.env.STACKBENCH_UPLOAD_KEY = 'fixture-key'; + t.after(() => { + delete process.env.STACKBENCH_UPLOAD_KEY; + }); + const published = await publishCampaign(campaign, { + suite: 'fixture-suite', + to: `http://127.0.0.1:${address.port}`, + }); + assert.deepEqual(published, { + version: 1, + uploadId: 'fixture-upload', + status: 'pending', + duplicate: false, + runs: 2, + }); + assert.deepEqual(requests, [payload]); + responseStatus = 409; + await assert.rejects( + publishCampaign(campaign, { suite: 'fixture-suite', to: `http://127.0.0.1:${address.port}` }), + /HTTP 409/ + ); + responseStatus = 201; + responseVersion = 2; + await assert.rejects( + publishCampaign(campaign, { suite: 'fixture-suite', to: `http://127.0.0.1:${address.port}` }), + /acknowledgement/ + ); + if (process.env.STACKBENCH_PUBLISH_CHECK_PAYLOAD) { + writeFileSync( + process.env.STACKBENCH_PUBLISH_CHECK_PAYLOAD, + `${JSON.stringify(payload, null, 2)}\n` + ); + } + if (process.env.STACKBENCH_PUBLISH_CHECK_CAMPAIGN) { + cpSync(campaign, process.env.STACKBENCH_PUBLISH_CHECK_CAMPAIGN, { + recursive: true, + force: false, + errorOnExist: true, + }); + } + await assert.rejects( + publishCampaign(campaign, { suite: 'fixture-suite', to: 'http://example.com' }), + /HTTPS/ + ); + state.attempts[1]!.executions[0]!.reason = 'Changed exclusion evidence'; + writeCampaignState(join(campaign, 'state.json'), plan, state); + assert.throws(() => createCampaignUpload(campaign, 'fixture-suite'), /report/); +}); diff --git a/tools/stack-bench/tests/campaign-report.contract.ts b/tools/stack-bench/tests/campaign-report.contract.ts new file mode 100644 index 00000000000..4a8545c0b40 --- /dev/null +++ b/tools/stack-bench/tests/campaign-report.contract.ts @@ -0,0 +1,954 @@ +import assert from 'node:assert/strict'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, + writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { emptyArtifactIdentities, readArtifact, writeArtifact, + writeRunJson } from '../src/evidence/artifacts.js'; +import { campaignProgressionOwner, compileCampaignFile } from '../src/campaigns/campaign-compiler.js'; +import type { CampaignAttemptPlan, CompiledCampaignPlan } + from '../src/campaigns/campaign-compiler.js'; +import { buildCampaignReport, campaignActiveDurationMs, campaignMeasuredRunWork, campaignReportCsv, exportCampaignReport, generateCampaignReport, + campaignRunMetrics, campaignRunFirstBuildObservations, progressionGradeCompletions, renderCampaignHtml, + validateCampaignReport } from '../src/campaigns/campaign-report.js'; +import type { BenchmarkRun, RunSelection } from '../src/campaigns/campaign-report.js'; +import type { RunSessionRecord } from '../src/evidence/benchmark-run.js'; +import type { RunCheckpoint } from '../src/evidence/run-checkpoints.js'; +import { canonicalDefinitionJson } from '../src/composition/definition-plan.js'; +import { createCheckEvidence } from '../src/evidence/check-evidence.js'; +import { hashDirectory, sha256 } from '../src/evidence/provenance.js'; +import { readCampaignAdmission, validateCampaignAdmission } from '../src/campaigns/campaign-admission.js'; +import { claimNextAttempt, createCampaignState, finishCampaignExecution, + initializeCampaignDirectory, writeCampaignState } from '../src/campaigns/campaign-scheduler.js'; +import { assessStoppedProviderContinuation, providerWaitSummary } from '../src/agents/provider-continuation-audit.js'; +import { progressionEngine } from '../src/progression/progression-engine.js'; +import { compileProgressionInput, dependencyRuntimeDefinition } + from '../src/progression/progression-definition.js'; +import { writeProgressionState } from '../src/progression/progression-state.js'; +import type { ProgressionState } from '../src/progression/progression-state.js'; +import { scoreDependencyState } from '../src/progression/dependency-score.js'; +import type { DependencyState } from '../src/progression/dependency-mode.js'; +import { campaignProgression } from '../dashboard/dashboard-views.js'; +import { DEPENDENCY_CAMPAIGN, writeCampaign } from './fixtures/dashboard-fixture.js'; + +const projectRoot = STACK_BENCH_ROOT; +const example = join(projectRoot, 'tests', 'fixtures', 'campaign.deterministic.json'); +const created = '2026-08-12T00:00:00.000Z'; +const compiledExample = compileCampaignFile(example); +const examplePlan = (): CompiledCampaignPlan => structuredClone(compiledExample); + +function costSession(costUsd: number, exact = true): RunSessionRecord { + return { sessionId: null, costUsd, costComplete: true, durationMs: 1, + usage: { input: 10, output: 20, cacheWrite: 0, cacheRead: 0 }, + costReceipts: [{ invocation: 1, receipt: { costUsd, exact, complete: true, + reconciled: true, error: null } }], + providerThrottle: null, resources: null, tokens: 30, outputTokens: 20, turns: 1, + promptBytes: 1, thinking: null, transcript: null, provenance: null, providerMetadata: null }; +} + +function run(id: string, attempt: { id: string; condition?: CampaignAttemptPlan['condition'] }, + { score = 8, max = 10, first = 5, cost = 2, durationSec = 30 }: + { score?: number; max?: number; first?: number; cost?: number; durationSec?: number } = {}, +): BenchmarkRun { + const passed = score === max; + const repairs = passed ? (score === first ? 0 : 1) : 3; + const status = passed ? (repairs ? 'corrected' : 'not-needed') : 'budget-exhausted'; + const outcome = { kind: passed ? 'passed' : 'app_failure' }; + // A valid run artifact carries the exact planned selection for each level. + const selection = structuredClone(attempt.condition?.requested?.levels + ?.find(item => item.level === 1)?.selection ?? null) as RunSelection | null; + return { id, parentAttemptId: attempt.id, outcome, + levels: [{ level: 1, ...(selection ? { selection } : {}), + firstBuild: { score: first, max, outcome }, score, max, + buildSessions: [costSession(repairs ? cost / 2 : cost)], + repairSessions: repairs ? [costSession(cost / 2)] : [], + repairCostUsd: repairs ? cost / 2 : 0, repairs, + repair: { status, limit: 3, used: repairs, stopReason: null }, outcome }], + totals: { score, max, costUsd: cost, costComplete: true, durationSec, repairs } }; +} + +function writeFakePackageEvidence(output: string, level: NonNullable[number]): void { + const source = join(output, 'source'); + mkdirSync(source, { recursive: true }); + writeFileSync(join(source, 'app.js'), 'export const ready = true;\n'); + const sourceHash = hashDirectory(source).sha256; + const checks = level.selection?.scoredChecks ?? []; + const checkKeys = checks.map(check => check.stableKey); + writeArtifact(join(output, 'grading', 'bundle.json'), { + kind: 'grade_bundle', id: `fake-grade-${level.level}`, payload: { + observation: 'scored', source: { sha256: sourceHash }, suites: { fake: { features: [{ + id: 'fake', setupEvidence: createCheckEvidence({ status: 'passed', code: 'completed', + phase: 'setup', startedAtMs: 1, completedAtMs: 2 }), + criteria: checks.map(check => ({ id: check.stableKey, stableKey: check.stableKey, + points: check.points, evidence: createCheckEvidence({ status: 'passed', code: 'completed', + phase: 'assertion', startedAtMs: 1, completedAtMs: 2 }) })), + }] } }, + totals: { score: level.score, max: level.max }, + selection: { sha256: level.selection?.sha256, checks, + attemptedChecks: checkKeys, reportedChecks: checkKeys, notRun: [] }, + }, + }); +} + +test('report read model keeps invalid evidence separate and computes declared dispersion', () => { + const plan = examplePlan(); + let state = createCampaignState(plan, { now: created }); + const runs = new Map(); + const admissionId = 'admission-1'; + let claimed = claimNextAttempt(state, { now: '2026-08-12T00:01:00.000Z', admissionId }); + assert.ok(claimed.claim); + state = finishCampaignExecution(claimed.state, claimed.claim.executionId, + { exitCode: 1, run: null, retryAuthority: { transient: true, recoveryClean: true, + budgetKnown: true, + cause: 'test transient provider failure' } }, { retries: 1, retryOn: ['harness_failure'], + now: '2026-08-12T00:02:00.000Z' }); + claimed = claimNextAttempt(state, { now: '2026-08-12T00:03:00.000Z', admissionId }); + assert.ok(claimed.claim); + runs.set(claimed.claim.executionId, run('run-1', claimed.claim.attempt)); + state = finishCampaignExecution(claimed.state, claimed.claim.executionId, + { exitCode: 0, run: runs.get(claimed.claim.executionId) }, + { now: '2026-08-12T00:04:00.000Z' }); + const report = buildCampaignReport(plan, state, (_attempt, execution) => runs.get(execution.id)); + assert.equal(report.summary.completedAttempts, 1); + assert.equal(report.summary.invalidAttempts, 0); + assert.equal(report.summary.invalidExecutions, 1); + const firstAttempt = report.attempts[0]; + const plannedAttempt = plan.attempts[0]; + assert.ok(firstAttempt); + assert.ok(plannedAttempt); + assert.equal(firstAttempt.executions.length, 2); + const condition = report.conditions.find(item => item.stack === plannedAttempt.stack); + assert.ok(condition); + assert.equal(condition.metrics.firstBuildScoreRate?.center, 0.5); + assert.equal(condition.metrics.finalScoreRate?.center, 0.8); + assert.equal(condition.metrics.totalCostUsd?.center, 2); + assert.equal(condition.sample.invalidExecutionRate, 0.5); + assert.deepEqual(report.scope.bindings, plan.bindings); + assert.equal(report.scope.grading.status, 'pending'); + assert(report.scope.grading.levels.every(level => level.status === 'pending' && level.reasons.length > 0)); + assert.equal(Object.hasOwn(report.scope.bindings[0]!, 'qualification'), false); + assert(report.limitations.some(item => /qualification is pending/.test(item))); + assert.deepEqual(report.scope.runtime, plan.definition.runtime); + assert.deepEqual(report.scope.pricing, plan.definition.pricing); + assert.deepEqual(report.scope.repetitionsByStack, plan.summary.repetitionsByStack); + assert.equal(report.scope.parallelism, plan.summary.parallelism); + assert.equal(firstAttempt.executions[0]?.admissionEvidence, + 'admissions/admission-1.json'); + assert.match(report.contentSha256, /^[a-f0-9]{64}$/); + assert.throws(() => validateCampaignReport({ ...report, + summary: { ...report.summary, completedAttempts: 99 } }), + /summary does not match its attempts/); + const inconsistent = structuredClone(report); + inconsistent.summary.completedAttempts = 99; + const { contentSha256: _oldIdentity, ...inconsistentBody } = inconsistent; + assert.throws(() => validateCampaignReport({ ...inconsistentBody, + contentSha256: sha256(canonicalDefinitionJson(inconsistentBody)) }), + /summary does not match its attempts/); + const unknownSummaryField = structuredClone(report) as typeof report & { + summary: typeof report.summary & { nonsense: boolean }; + }; + unknownSummaryField.summary.nonsense = true; + const { contentSha256: _oldUnknownIdentity, ...unknownSummaryBody } = unknownSummaryField; + assert.throws(() => validateCampaignReport({ ...unknownSummaryBody, + contentSha256: sha256(canonicalDefinitionJson(unknownSummaryBody)) }), + /summary\.nonsense is unknown/); +}); + +test('target delivery keeps deadline stops in the assigned population', () => { + const plan = examplePlan(); + let state = createCampaignState(plan, { now: created }); + const runs = new Map(); + // Keep one condition on one stack: one delivered app and two deadline stops. + const chosen = state.attempts[0]!.plan.stack; + state.attempts = state.attempts.filter(attempt => attempt.plan.stack === chosen); + assert.equal(state.attempts.length, 3); + for (let i = 0; i < 3; i++) { + const next = claimNextAttempt(state, { now: created, admissionId: 'delivery-admission' }); + assert(next.claim); + const result = i === 0 ? run('delivered', next.claim.attempt, { score: 10, max: 10, first: 10 }) : null; + if (result) runs.set(next.claim.executionId, result); + state = finishCampaignExecution(next.state, next.claim.executionId, + { exitCode: i === 0 ? 0 : 1, timedOut: i !== 0, run: result }, { now: created }); + } + const report = buildCampaignReport(plan, state, (_attempt, execution) => runs.get(execution.id)!); + const condition = report.conditions[0]!; + assert.equal(condition.metrics.finalScoreRate?.center, 1, 'conditional score still describes the measured app'); + assert.deepEqual(condition.targetDelivery, { passedAttempts: 1, assignedAttempts: 3, rate: 0.333333 }); + assert.equal(condition.sample.invalidAttempts, 2); + assert.match(renderCampaignHtml(report), /1\/3 assigned/); + assert.match(campaignReportCsv(report)['conditions.csv']!, /"1","3","0.333333"/); + assert.doesNotThrow(() => validateCampaignReport(report)); +}); + +test('seeded campaign reports identify parent work excluded from continuation cost', () => { + const plan = examplePlan(); + const state = createCampaignState(plan, { now: created }); + const attempt = state.attempts[0]!; + attempt.extension = { + fromDepth: 2, source: `.private/extensions/${attempt.plan.id}/source`, + sourceSha256: 'a'.repeat(64), sourceFiles: 1, + parent: { campaignId: 'parent-campaign', campaignSha256: 'b'.repeat(64), + attemptId: 'parent-attempt', executionId: 'parent-execution', + runId: 'parent-run', runSha256: 'c'.repeat(64) }, + }; + const report = buildCampaignReport(plan, state, () => { throw new Error('no execution'); }); + assert(report.limitations.some(item => item.includes('parent-campaign at L2') + && item.includes('exclude the parent build') && item.includes('source retains prior repairs'))); + assert.doesNotMatch(renderCampaignHtml(report), /Check completion: 0\//, 'a pending attempt is not a measured zero'); + delete attempt.extension; + assert(!buildCampaignReport(plan, state, () => { throw new Error('no execution'); }) + .limitations.some(item => item.startsWith('Seeded continuation from'))); +}); + +test('correction metrics separate successful cost from unresolved spend', () => { + const corrected = campaignRunMetrics({ outcome: { kind: 'passed' }, + levels: [{ firstBuild: { score: 5, max: 10 }, repairCostUsd: 1.25, + repairSessions: [costSession(1.25)] }], totals: {} }, []); + assert.equal(corrected.correctionSuccessRate, 1); + assert.equal(corrected.correctionCostUsd, 1.25); + assert.equal(corrected.correctionSpendUsd, 1.25); + + const unresolved = campaignRunMetrics({ outcome: { kind: 'app_failure' }, + levels: [{ firstBuild: { score: 5, max: 10 }, repairCostUsd: 2, + repairSessions: [costSession(2)] }], totals: {} }, []); + assert.equal(unresolved.correctionSuccessRate, 0); + assert.equal(unresolved.correctionCostUsd, null); + assert.equal(unresolved.correctionSpendUsd, 2); + + const unaided = campaignRunMetrics({ outcome: { kind: 'passed' }, + levels: [{ firstBuild: { score: 10, max: 10 }, repairCostUsd: 0 }], totals: {} }, []); + assert.equal(unaided.correctionSuccessRate, null); + assert.equal(unaided.correctionCostUsd, null); + assert.equal(unaided.correctionSpendUsd, null); +}); + +test('all-execution spend includes invalid work and never counts resumed inherited cost twice', () => { + const plan = examplePlan(); + let state = createCampaignState(plan, { now: created }); + const first = claimNextAttempt(state, { now: created, admissionId: 'cost-admission' }); + assert(first.claim); + const prior = run('prior', first.claim.attempt, { cost: 2 }); + prior.outcome = { kind: 'provider_failure' }; + state = finishCampaignExecution(first.state, first.claim.executionId, + { exitCode: 1, run: null, retryAuthority: { transient: true, recoveryClean: true, + budgetKnown: true, cause: 'test provider failure' } }, + { retries: 1, retryOn: ['harness_failure'], now: created }); + const second = claimNextAttempt(state, { now: created, admissionId: 'cost-admission' }); + assert(second.claim); + const resumed = run('resumed', second.claim.attempt, { cost: 3 }); + resumed.progressionResume = { priorRunId: 'prior', priorRunSha256: 'a'.repeat(64), + stateSha256: null, action: { type: 'build', level: 1 }, inheritedLevels: [], priorTotals: null }; + resumed.totals!.costUsd = 5; + resumed.totals!.currentExecutionCostUsd = 3; + resumed.totals!.currentExecutionCostComplete = true; + state = finishCampaignExecution(second.state, second.claim.executionId, { exitCode: 0, run: resumed }, { now: created }); + const report = buildCampaignReport(plan, state, (_attempt, execution) => + execution.id === first.claim!.executionId ? prior : resumed); + assert.equal(report.summary.spend.costUsd, 5); + const attempt = report.attempts.find(item => item.id === first.claim!.attempt.id)!; + assert.deepEqual(attempt.executions.map(execution => execution.cost.costUsd), [2, 3]); + assert.equal(attempt.metrics?.totalCostUsd, 5); + assert.equal(attempt.spend.costUsd, 5); + assert.equal(attempt.executions[1]?.usage.input, 20); + assert.equal(attempt.executions[1]?.usage.build.costUsd, 1.5); + assert.equal(attempt.executions[1]?.usage.repair.costUsd, 1.5); +}); + +test('time grants retain one efficacy result and expose the changed allowance', () => { + const plan = examplePlan(); + const claimed = claimNextAttempt(createCampaignState(plan, { now: created }), + { now: created, admissionId: 'time-admission' }); + assert(claimed.claim); + const evidence = run('extended', claimed.claim.attempt); + const state = finishCampaignExecution(claimed.state, claimed.claim.executionId, + { exitCode: 0, run: evidence }, { now: '2026-08-12T00:02:00.000Z' }); + const target = state.attempts.find(attempt => attempt.plan.id === claimed.claim!.attempt.id)!; + const original = plan.definition.budgets.attemptTimeoutMinutes; + target.timeGrants = [{ request: { campaignSha256: plan.contentSha256, + attemptId: target.plan.id, executionId: claimed.claim.executionId, + grantId: 'extra-time', minutes: 120, requestedAt: created }, disposition: 'accepted', + acceptedAt: created, previousMinutes: original, effectiveMinutes: original + 120 }]; + const report = buildCampaignReport(plan, state, () => evidence); + const result = report.attempts.find(attempt => attempt.id === target.plan.id)!; + assert.equal(result.status, 'completed'); + assert.equal(report.summary.completedAttempts, 1); + assert.equal(result.executions.length, 1); + assert.equal(result.metrics?.totalCostUsd, 2); + assert.equal(result.timeBudget?.effectiveMinutes, original + 120); + assert.equal(result.timeBudget?.consumedMs, 120_000); + assert.equal(result.timeBudget?.extensionCount, 1); + assert.equal(plan.definition.budgets.attemptTimeoutMinutes, original); +}); + +test('bounded and unknown cost remain explicit through report validation and HTML', () => { + const plan = examplePlan(); + const claimed = claimNextAttempt(createCampaignState(plan, { now: created }), { now: created, admissionId: 'cost-admission' }); + assert(claimed.claim); + const bounded = run('bounded', claimed.claim.attempt, { cost: 3 }); + bounded.levels![0]!.buildSessions = [costSession(1.5, false)]; + const state = finishCampaignExecution(claimed.state, claimed.claim.executionId, + { exitCode: 0, run: bounded }, { now: created }); + const report = buildCampaignReport(plan, state, () => bounded); + const execution = report.attempts.find(attempt => attempt.executions.length)!.executions[0]!; + assert.deepEqual(execution.cost, { status: 'upper-bound', costUsd: 3 }); + assert.equal(execution.metrics?.totalCostUsd, null); + assert.equal(execution.metrics?.totalCostUpperBoundUsd, 3); + assert.equal(report.summary.spend.status, 'upper-bound'); + assert.match(renderCampaignHtml(report), /≤ \$3/); + const malformed = structuredClone(report); + const mutable = malformed.attempts.find(attempt => attempt.executions.length)!.executions[0]!; + mutable.cost = { status: 'unknown', costUsd: 0 } as unknown as typeof mutable.cost; + assert.throws(() => validateCampaignReport(malformed)); + delete bounded.totals!.costUsd; + const missing = buildCampaignReport(plan, state, () => bounded); + assert.equal(missing.summary.spend.costUsd, null); + assert.equal(missing.summary.spend.unknownExecutions, 1); + const measured = missing.attempts.find(attempt => attempt.executions.length)!; + assert.equal(measured.status, 'completed', 'cost uncertainty alone does not erase a valid outcome'); + assert.equal(measured.metrics?.finalScoreRate, 0.8); + assert.equal(measured.metrics?.totalCostUsd, null); + assert.match(renderCampaignHtml(missing), /Unknown/); +}); + +test('historical zero-usage actions remain ineligible and keep earlier bounded costs', () => { + const plan = examplePlan(); + const claimed = claimNextAttempt(createCampaignState(plan, { now: created }), + { now: created, admissionId: 'provider-admission' }); + assert(claimed.claim); + const evidence = run('provider-stopped', claimed.claim.attempt, { cost: 3 }); + evidence.levels![0]!.buildSessions = [costSession(3, false)]; + const stopped = costSession(0); + stopped.usage = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }; + stopped.providerMetadata = { failureCode: 'provider-throttle-exhausted', invocations: 4, + providerWaits: [{ waitedMs: 1000, disposition: 'continued' }, { waitedMs: 2000, disposition: 'stopped' }] }; + stopped.costReceipts = Array.from({ length: 4 }, (_, index) => ({ invocation: index + 1, + receipt: { costUsd: 0, complete: true, reconciled: true, error: null, exact: true, + usage: { input: 0, output: 0, cacheRead: 0, cacheWrite5m: 0, cacheWrite1h: 0 } } })); + evidence.levels![0]!.repairSessions = [stopped]; + const state = finishCampaignExecution(claimed.state, claimed.claim.executionId, + { exitCode: 1, run: evidence }, { now: created }); + const report = buildCampaignReport(plan, state, () => evidence); + const execution = report.attempts.find(attempt => attempt.executions.length)!.executions[0]!; + assert.equal(execution.providerContinuation?.work, 'zero-usage-candidate'); + assert.equal(execution.providerContinuation?.eligible, false); + assert.deepEqual(execution.providerContinuation?.runCost, { status: 'upper-bound', costUsd: 3 }); + assert.deepEqual(execution.providerWaits, { waits: 2, waitedMs: 3000, continued: 1, stopped: 1 }); + assert.equal(providerWaitSummary({ ...evidence, progressionResume: { inheritedLevels: [1] } }), null); + assert.match(renderCampaignHtml(report), /Provider continuation ineligible/); + assert.match(campaignReportCsv(report)['executions.csv']!, /zero-usage-candidate/); + assert.deepEqual(validateCampaignReport(report), report); + const interrupted = buildCampaignReport(plan, state, () => { throw new Error('no final run'); }, + () => ({ waits: 1, waitedMs: 25, continued: 0, stopped: 0, waiting: 1, durationKind: 'lower-bound' })); + const interruptedExecution = interrupted.attempts.find(attempt => attempt.executions.length)!.executions[0]!; + assert.equal(interruptedExecution.providerWaits?.waiting, 1); + assert.match(renderCampaignHtml(interrupted), /provider waits 1, at least/); + assert.deepEqual(validateCampaignReport(interrupted), interrupted); + + // A zero final retry does not erase paid work in an earlier invocation. + stopped.usage.output = 1; + assert.equal(assessStoppedProviderContinuation(evidence)?.work, 'paid'); + stopped.usage.output = 0; + stopped.costReceipts.pop(); + assert.equal(assessStoppedProviderContinuation(evidence)?.work, 'unknown'); + stopped.costReceipts = []; + assert.equal(assessStoppedProviderContinuation(evidence)?.work, 'unknown'); + assert.equal(assessStoppedProviderContinuation({ levels: [] }), null); +}); + +test('campaign metrics do not treat incomplete cost as comparable evidence', () => { + const incomplete = campaignRunMetrics({ outcome: { kind: 'passed' }, levels: [], + totals: { costUsd: 4.25, costComplete: false, durationSec: 30 } }, []); + assert.equal(incomplete.totalCostUsd, null); + assert.equal(incomplete.totalDurationMs, 30_000); + + const zero = campaignRunMetrics({ outcome: { kind: 'passed' }, levels: [], + totals: { costUsd: 0, costComplete: true } }, []); + assert.equal(zero.totalCostUsd, 0); +}); + +test('dependency campaign final score is passed points over all points, with the questline average beside it', () => { + const metrics = campaignRunMetrics({ + progressionStatus: { phase: 'terminal', score: { questlineAveragePercentage: 62.5, + uniqueChecks: { gradedPoints: 16, availablePoints: 20, percentage: 70 } } }, + levels: [{ firstBuild: { score: 4, max: 10 }, score: 10, max: 10 }], + totals: { score: 30, max: 30 }, + }, []); + assert.equal(metrics.finalScoreRate, 0.7); + assert.equal(metrics.questlineAverageRate, 0.625); + assert.equal(metrics.finalCoverageRate, 0.8); + + const active = campaignRunMetrics({ + progressionStatus: { phase: 'active', score: { questlineAveragePercentage: null, + uniqueChecks: { gradedPoints: 16, availablePoints: 20, percentage: null } } }, + levels: [{ firstBuild: { score: 4, max: 10 }, score: 10, max: 10 }], + totals: { score: 30, max: 30 }, + }, []); + assert.equal(active.finalScoreRate, null); + assert.equal(active.questlineAverageRate, null); + assert.equal(active.finalCoverageRate, null); +}); + +test('completed process with inconclusive dependency grading has no final completion metric', () => { + const plan = examplePlan(); + const claimed = claimNextAttempt(createCampaignState(plan), { admissionId: 'active-grade' }); + assert.ok(claimed.claim); + const incomplete = run('active-grade', claimed.claim.attempt, { cost: 5.059995 }); + incomplete.levels!.push({ level: 2, graded: false, + firstBuild: { score: 34, max: 70 }, outcome: { kind: 'app_failure', + appFailures: ['session-reload'], inconclusive: ['no-session'] } }); + incomplete.totals!.ungraded = [2]; + incomplete.progressionStatus = { phase: 'active', score: { + completion: { selected: 107, passed: 7, failed: 2, blocked: 0, unmeasured: 98, rate: 0.065421 }, + uniqueChecks: { gradedPoints: 12, availablePoints: 173, percentage: null }, + } }; + const state = finishCampaignExecution(claimed.state, claimed.claim.executionId, + { exitCode: 0, run: incomplete }); + assert.equal(state.attempts.find(item => item.plan.id === claimed.claim!.attempt.id)!.status, 'invalid'); + // Existing states keep their recorded claim; report correction must not rewrite it. + const oldAttempt = state.attempts.find(item => item.plan.id === claimed.claim!.attempt.id)!; + oldAttempt.status = 'completed'; + oldAttempt.executions[0]!.status = 'completed'; + oldAttempt.executions[0]!.outcome = incomplete.outcome!.kind as 'passed'; + const report = buildCampaignReport(plan, state, () => incomplete); + const attempt = report.attempts.find(item => item.id === claimed.claim!.attempt.id)!; + assert.equal(attempt.status, 'invalid'); + assert.equal(attempt.executions[0]!.outcome, 'incomplete'); + assert.equal(attempt.executions[0]!.recordedStatus, 'completed'); + assert.equal(attempt.executions[0]!.recordedOutcome, incomplete.outcome!.kind); + assert.equal(oldAttempt.status, 'completed'); + assert.equal(attempt.executions[0]!.metrics?.checkCompletionRate, null); + assert.equal(attempt.metrics, null); + assert.equal(attempt.completion.passed, 7); + assert.equal(attempt.completion.unmeasured, 98); + assert.equal(attempt.spend.status, 'unknown'); + assert.equal(attempt.spend.costUsd, null); + assert.equal(attempt.spend.knownCostUsd, 5.059995); + const cohort = report.conditions.find(item => item.stack === attempt.stack)!; + assert.equal(cohort.metrics.checkCompletionRate?.n, 0); + assert.equal(cohort.metrics.checkCompletionRate?.center, null); + const csv = campaignReportCsv(report)['attempts.csv']!; + assert.match(csv, /"107","7","2","0","98","",""/); + assert.doesNotMatch(csv, /0\.065421/); + assert.match(csv, /sumOfAvailableExactCostsAndUpperBoundsUsd/); + assert.match(report.limitations.join(' '), /must not be assumed to be a lower bound/); +}); + +test('score rates keep inconclusive points separate from measurement coverage', () => { + const inconclusive = { kind: 'inconclusive', inconclusive: ['contention/203/203b'], + harnessFailures: [] }; + const selection = { checks: [{ executionId: 'contention', featureId: 203, + criterionId: '203b', points: 1 }] }; + const metrics = campaignRunMetrics({ outcome: inconclusive, levels: [{ + firstBuild: { score: 8, max: 9, outcome: inconclusive }, + score: 9, max: 9, selection, outcome: inconclusive, repairCostUsd: 1, + }], totals: { score: 9, max: 9 } }, []); + assert.equal(metrics.firstBuildScoreRate, 0.888889); + assert.equal(metrics.finalScoreRate, 1); + assert.equal(metrics.firstBuildCoverageRate, 0.9); + assert.equal(metrics.finalCoverageRate, 0.9); + + const unmapped = campaignRunMetrics({ outcome: inconclusive, levels: [{ + firstBuild: { score: 8, max: 9, outcome: inconclusive }, + score: 9, max: 9, selection: { checks: [] }, outcome: inconclusive, + }], totals: { score: 9, max: 9 } }, []); + assert.equal(unmapped.firstBuildCoverageRate, null); + assert.equal(unmapped.finalCoverageRate, null); +}); + +test('observed-only first-build behavior remains separate from scored results and repairs', () => { + const evidence = run('probe-run', { id: 'probe-attempt' }, + { score: 14, max: 14, first: 14, cost: 0 }); + const evidenceLevel = evidence.levels?.[0]; + assert.ok(evidenceLevel); + evidenceLevel.selection = { + specifications: { requested: [], expected: [], + observed: ['ecommerce.spec.state-durability'] }, + observedChecks: [ + { stableKey: 'durability/session', points: 1 }, + { stableKey: 'durability/cart', points: 2 }, + ], + }; + assert.ok(evidenceLevel.firstBuild); + evidenceLevel.firstBuild.source = { sha256: 'a'.repeat(64), files: 3 }; + evidenceLevel.firstBuild.observations = { + sourceSha256: 'a'.repeat(64), + selectionSha256: 'b'.repeat(64), + selectedChecks: ['durability/session', 'durability/cart'], + reportedChecks: ['durability/session', 'durability/cart'], + passedPoints: 1, + observedPoints: 3, + scoreContribution: false, + repairVisible: false, + artifact: 'first-build-l1-observed/bundle.json', + outcome: { kind: 'app_failure' }, + }; + + const scored = campaignRunMetrics(evidence, []); + const observed = campaignRunFirstBuildObservations(evidence); + assert.ok(observed); + assert.equal(scored.firstBuildScoreRate, 1); + assert.equal(scored.finalScoreRate, 1); + assert.equal(scored.correctionSuccessRate, null); + assert.equal(observed.selectedPoints, 3); + assert.equal(observed.observedPoints, 3); + assert.equal(observed.passedPoints, 1); + assert.equal(observed.passRate, 0.333333); + assert.equal(observed.coverageRate, 1); + assert.equal(observed.scoreContribution, false); + assert.equal(observed.repairVisible, false); + assert.deepEqual(observed.levels[0]?.specifications, + ['ecommerce.spec.state-durability']); +}); + +test('campaign HTML labels observed-only behavior as zero-score first-build observations', () => { + const plan = examplePlan(); + let state = createCampaignState(plan, { now: created }); + const claimed = claimNextAttempt(state, { now: created, admissionId: 'probe-admission' }); + assert.ok(claimed.claim); + const claim = claimed.claim; + const evidence = run('probe-run', claim.attempt, + { score: 10, max: 10, first: 10, cost: 0 }); + const evidenceLevel = evidence.levels?.[0]; + assert.ok(evidenceLevel); + evidenceLevel.selection = { specifications: { requested: [], expected: [], + observed: ['durability@1'] }, + observedChecks: [{ stableKey: 'durability/session', points: 1 }] }; + assert.ok(evidenceLevel.firstBuild); + evidenceLevel.firstBuild.observations = { sourceSha256: 'a'.repeat(64), + selectionSha256: 'b'.repeat(64), selectedChecks: ['durability/session'], + reportedChecks: ['durability/session'], passedPoints: 1, observedPoints: 1, + scoreContribution: false, repairVisible: false, + artifact: 'first-build-l1-observed/bundle.json', outcome: { kind: 'passed' } }; + state = finishCampaignExecution(claimed.state, claim.executionId, + { exitCode: 0, run: evidence }, { now: '2026-08-12T00:02:00.000Z' }); + const report = buildCampaignReport(plan, state, () => evidence); + const html = renderCampaignHtml(report); + const condition = report.conditions.find(item => item.stack === claim.attempt.stack); + assert.ok(condition?.firstBuildObservations); + assert.equal(condition.firstBuildObservations.sample.selectedAttempts, 1); + assert.equal(condition.firstBuildObservations.sample.measuredAttempts, 1); + assert.equal(condition.firstBuildObservations.metrics.passRate.center, 1); + assert.match(html, /first-build-l1-observed\/bundle\.json/); +}); + +test('campaign HTML states the build and evaluation setup in plain language', () => { + const plan = compileCampaignFile(join(projectRoot, 'appliance', + 'campaign.product-brief-reference.json')); + const state = createCampaignState(plan, { now: created }); + const report = buildCampaignReport(plan, state, () => { + throw new Error('a pending campaign must not read run evidence'); + }); + assert(report.limitations.some(item => + item.includes('Reference-fixture attempts use hand-written apps and make no model calls') + && item.includes('do not measure model implementation ability or comparative token efficiency'))); + const html = renderCampaignHtml(report); + assert.match(html, /product-brief-quality/); + assert.match(html, /ecommerce\.spec\.access-control/); + assert.match(html, /ecommerce\.spec\.transactional-integrity/); +}); + +test('report generation is byte-for-byte reproducible and links immutable raw evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-campaign-report-')); + const exported = `${root}-export`; + try { + const plan = examplePlan(); + const initialized = initializeCampaignDirectory(plan, root, { now: created }); + const admission = { id: 'report-admission' }; + writeArtifact(join(root, 'admissions', `${admission.id}.json`), { + kind: 'campaign_admission', id: admission.id, + identities: emptyArtifactIdentities({ experiment: { id: plan.id, version: plan.version, + sha256: plan.contentSha256, state: plan.state } }), + payload: { schemaVersion: 1, campaignId: plan.id, campaignSha256: plan.contentSha256, + createdAt: created, ok: true, runtime: plan.definition.runtime, conditions: plan.conditions, + agents: plan.agents.map(agent => ({ adapter: agent.adapter, model: agent.model, identity: agent.identity })), + reports: plan.agents.flatMap(agent => Array.from({ length: plan.summary.parallelism }, (_, runIndex) => ({ + schemaVersion: 1, generatedAt: created, ok: true, + request: { backends: plan.stacks.map(stack => stack.id), track: plan.definition.track, + levels: plan.definition.levels, runIndex, parallelism: plan.summary.parallelism, + agentAdapter: agent.adapter, packs: plan.definition.selection.packs ?? [], + checks: plan.definition.selection.checks ?? [], image: plan.definition.runtime.buildImage ?? 'test-image', + resultsDir: '/var/lib/docker/volumes/original/results/campaign', smoke: false }, + summary: { passed: 0, failed: 0, warnings: 0 }, checks: [], + }))), + }, + }); + assert.throws(() => readCampaignAdmission(root, admission.id, plan), /compiled scope/); + const historical = readCampaignAdmission(root, admission.id, plan, { allowRelocatedEvidence: true }); + for (const report of historical.reports) report.request.resultsDir = 'C:\\original\\campaign'; + assert.doesNotThrow(() => validateCampaignAdmission(historical, plan, root, { allowRelocatedEvidence: true })); + historical.reports[0]!.request.resultsDir = 'relative-origin'; + assert.throws(() => validateCampaignAdmission(historical, plan, root, + { allowRelocatedEvidence: true }), /compiled scope/); + const claimed = claimNextAttempt(initialized.state, + { now: '2026-08-12T00:01:00.000Z', admissionId: admission.id }); + assert.ok(claimed.claim); + const claim = claimed.claim; + const output = join(root, claim.output); + mkdirSync(output, { recursive: true }); + const evidence = run('run-1', claim.attempt, + { score: 57, max: 57, first: 57, cost: 2, durationSec: 30 }); + const timestamp = '2026-08-12T00:01:30.000Z'; + const agent = plan.agents.find(item => item.adapter === claim.attempt.agentAdapter); + const stack = plan.stacks.find(item => item.id === claim.attempt.stack); + assert.ok(agent); + assert.ok(stack); + const level = evidence.levels?.[0]; + assert.ok(level); + writeFakePackageEvidence(output, level); + writeRunJson(join(output, 'run.json'), { ...evidence, startedAt: timestamp, completedAt: timestamp, + track: plan.definition.track, backend: claim.attempt.stack, + model: claim.attempt.model, guidance: claim.attempt.guidance, + condition: claim.attempt.condition, mode: claim.attempt.mode, pricing: claim.attempt.pricing, + skills: claim.attempt.skills, selectionRequest: plan.definition.selection, + runtime: { buildImage: null }, identities: emptyArtifactIdentities({ + engine: plan.identities.engine, experiment: { id: plan.id, version: plan.version, + sha256: plan.contentSha256, state: plan.state }, + agentAdapter: agent.identity, stackAdapter: stack, + }) }); + const state = finishCampaignExecution(claimed.state, claim.executionId, + { exitCode: 0, run: evidence }, { now: '2026-08-12T00:02:00.000Z' }); + writeCampaignState(initialized.paths.state, plan, state); + mkdirSync(join(root, '.private'), { recursive: true }); + writeFileSync(join(root, '.private', 'authority.json'), '{"credential":"do-not-export"}'); + writeFileSync(join(root, 'transcript.json'), '{"authorization":"do-not-export"}'); + const first = generateCampaignReport(root); + const manifest = JSON.parse(readFileSync(first.exportManifestPath, 'utf8')); + assert.ok(manifest.files.some((file: { path: string }) => file.path === 'plan.json')); + assert.ok(manifest.files.some((file: { path: string }) => file.path === 'report/report.html')); + assert.ok(manifest.files.every((file: { path: string }) => !file.path.includes('.private') && !file.path.includes('transcript'))); + assert.equal(manifest.campaignSha256, plan.contentSha256); + assert.match(manifest.reconstruction, /Partial/); + const manifestBytes = readFileSync(first.exportManifestPath); + const json = readFileSync(first.reportPath); + const html = readFileSync(first.htmlPath); + rmSync(join(root, 'report'), { recursive: true }); + const second = generateCampaignReport(root); + assert.deepEqual(readFileSync(second.reportPath), json); + assert.deepEqual(readFileSync(second.htmlPath), html); + assert.deepEqual(readFileSync(second.exportManifestPath), manifestBytes); + const artifact = readArtifact(second.reportPath, { expectedKind: 'campaign_report' }); + assert.equal(artifact.payload.contentSha256, first.report.contentSha256); + assert.match(readFileSync(second.htmlPath, 'utf8'), /\.\.\/attempts\//); + assert.match(readFileSync(second.htmlPath, 'utf8'), /\.\.\/admissions\//); + // An interrupted regeneration leaves no manifest to offer and no partial file. + rmSync(second.htmlPath); + mkdirSync(join(second.htmlPath, 'occupied'), { recursive: true }); + assert.throws(() => generateCampaignReport(root)); + assert.equal(existsSync(second.exportManifestPath), false); + assert.deepEqual(readdirSync(join(root, 'report')).filter(name => name.endsWith('.tmp')), []); + rmSync(second.htmlPath, { recursive: true }); + assert.deepEqual(readFileSync(generateCampaignReport(root).exportManifestPath), manifestBytes); + const invalid = finishCampaignExecution(claimed.state, claim.executionId, + { exitCode: 1, run: null }, { now: '2026-08-12T00:02:00.000Z' }); + writeCampaignState(initialized.paths.state, plan, invalid); + const invalidReport = generateCampaignReport(root).report; + assert.equal(invalidReport.summary.invalidExecutions, 1); + assert.equal(invalidReport.summary.spend.knownCostUsd, 2, 'valid receipts survive an invalid outcome in the persisted artifact'); + assert.equal(invalidReport.summary.spend.unknownExecutions, 0); + assert.throws(() => exportCampaignReport(root, join(root, 'nested-export')), /outside/); + assert.equal(exportCampaignReport(root, exported), exported); + const portable = JSON.parse(readFileSync(join(exported, 'export-manifest.json'), 'utf8')); + for (const file of portable.files) { + const bytes = readFileSync(join(exported, file.path)); + assert.equal(bytes.length, file.bytes); + assert.equal(sha256(bytes), file.sha256); + } + assert.ok(existsSync(join(exported, 'report/report.html'))); + assert.ok(existsSync(join(exported, 'plan.json'))); + assert.ok(!existsSync(join(exported, '.private'))); + assert.ok(!existsSync(join(exported, claim.output, 'source'))); + assert.ok(!existsSync(join(exported, 'transcript.json'))); + assert.deepEqual(readFileSync(join(exported, 'report/report.html')), readFileSync(join(root, 'report/report.html'))); + const csv = campaignReportCsv(invalidReport); + assert.equal(readFileSync(join(exported, 'executions.csv'), 'utf8'), csv['executions.csv']); + assert.match(csv['executions.csv']!, /"invalid","harness_failure","invalid","harness_failure","exact","2"/); + assert.match(readFileSync(join(exported, 'README.txt'), 'utf8'), /Partial research export/); + assert.throws(() => exportCampaignReport(root, exported), /must not exist/); + } finally { + rmSync(root, { recursive: true, force: true }); + rmSync(exported, { recursive: true, force: true }); + } +}); + +test('dependency HTML distinguishes accepted completion from raw grade outcomes', () => { + const plan = examplePlan(); + const state = createCampaignState(plan); + for (const attempt of state.attempts) attempt.plan.mode = { ...attempt.plan.mode, id: 'dependency' }; + const built = buildCampaignReport(plan, state, () => { throw new Error('pending'); }); + const { contentSha256: _built, ...body } = built; + body.attempts[0]!.completion = { selected: 9, passed: 2, failed: 3, blocked: 1, unmeasured: 3, rate: 0.222222 }; + const report = { ...body, contentSha256: sha256(canonicalDefinitionJson(body)) }; + const completion = structuredClone(report.attempts[0]!.completion); + const html = renderCampaignHtml(report); + assert.ok(html.includes(`2/${completion.selected}`)); + assert.ok(html.includes(`${completion.unmeasured} without an accepted outcome`)); + assert.match(html, /does not separate prerequisite deferral from missing conclusive evidence/); + assert.deepEqual(report.attempts[0]!.completion, completion); +}); + +test('CSV preserves blank unknowns, zero costs, denominators and inert spreadsheet labels', () => { + const plan = examplePlan(); + const report = buildCampaignReport(plan, createCampaignState(plan), () => { throw new Error('pending'); }); + const attempt = report.attempts[0]!; + attempt.id = '=SUM(1,2)'; + attempt.completion = { selected: 9, passed: 2, failed: 3, blocked: 1, unmeasured: 3, rate: 2 / 9 }; + attempt.spend = { status: 'unknown', costUsd: null, knownCostUsd: 0, + unknownExecutions: 1, boundedExecutions: 0 }; + const csv = campaignReportCsv(report)['attempts.csv']!; + assert.match(csv, /"'=SUM\(1,2\)"/); + assert.match(csv, /"9","2","3","1","3"/); + assert.match(csv, /"unknown","","0","1","0"/); + assert.match(csv, /"diagnosticPassedChecks"/); + assert.match(csv, /"9","2","3","1","3","",""/, + 'pending attempts have no outcome rate despite retained diagnostic counts'); + attempt.status = 'invalid'; + assert.match(campaignReportCsv(report)['attempts.csv']!, /"9","2","3","1","3","",""/, + 'invalid retained checkpoint counts do not become eligible outcome metrics'); + attempt.status = 'completed'; + attempt.metrics = { checkCompletionRate: 2 / 9 }; + assert.match(campaignReportCsv(report)['attempts.csv']!, /"9","2","3","1","3","0\.2222222222222222"/); + attempt.spend = { status: 'upper-bound', costUsd: 4, knownCostUsd: 4, + unknownExecutions: 0, boundedExecutions: 1 }; + assert.match(campaignReportCsv(report)['attempts.csv']!, /"upper-bound","4","4","0","1"/); +}); + +test('HTML escapes caller-controlled labels and reports exact scope', () => { + const plan = examplePlan(); + const state = createCampaignState(plan, { now: created }); + const report = buildCampaignReport({ ...plan, title: '`); + } + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const sockets = new wsServer({ server }); + sockets.on('connection', (socket: { on(event: string, fn: (data: Buffer) => void): void; send(data: string): void; close(): void }) => { + socket.on('message', data => { + const body = JSON.parse(String(data)); + if (mode === 'disconnect') { socket.close(); return; } + socket.send(JSON.stringify({ type: 'MutationResponse', requestId: body.requestId + 50, success: true })); + if (mode === 'wrong-id' || mode === 'query') return; + socket.send(JSON.stringify({ type: 'MutationResponse', requestId: body.requestId, success: mode !== 'rejected' })); + }); + }); + const browser = await chromium.launch({ headless: true }); + const observations: unknown[] = []; + let result = 'failed'; + try { + for (const action of ['replayAs', 'repeatFormWrite']) for (mode of ['accepted', 'other-module', 'bearer', 'foreign-origin', 'encoded-value', 'rejected', 'wrong-id', 'ambiguous', 'duplicate-value', 'query', 'disconnect', 'signed-out', 'replay-refused', 'lost-reply']) { + rows.length = 0; + const context = await browser.newContext(); + try { + const page = await context.newPage(), actor = new Actor('admin', page, context); + await actor.ready; await page.goto(url); + await page.evaluate(async ({ url, mode, action }) => { + Object.assign(window, { getSessionToken: () => mode === 'signed-out' ? null : 'caller-session' }); + const socket = new WebSocket(url.replace('http:', 'ws:').replace('127.0.0.1', mode === 'foreign-origin' ? 'localhost' : '127.0.0.1') + '/api/1.0.0/sync'); + Object.assign(window, { replayTestSocket: socket }); + await new Promise(resolve => socket.addEventListener('open', resolve, { once: true })); + if (mode === 'bearer') socket.send(JSON.stringify({ type: 'Authenticate', tokenType: 'User', value: 'caller-session' })); + const args = { name: 'Original product', category: mode === 'duplicate-value' ? 'Original product' : 'Volume', + price: mode === 'encoded-value' ? { $integer: 'AQAAAAAAAAA=' } : 1.25, variants: ['Standard'], + ...(mode === 'bearer' ? {} : { token: 'caller-session' }) }; + for (let n = 0; n < (mode === 'ambiguous' ? 2 : 1); n++) socket.send(JSON.stringify({ + type: mode === 'query' ? 'ModifyQuerySet' : 'Mutation', requestId: n, + udfPath: mode === 'other-module' ? 'catalog/products:create' : 'api:create_product', args: [args], + })); + if (action === 'repeatFormWrite') socket.send(JSON.stringify({ + type: mode === 'query' ? 'ModifyQuerySet' : 'Mutation', requestId: 10, + udfPath: mode === 'other-module' ? 'catalog/products:create' : 'api:create_product', + args: [{ ...args, name: 'Control product' }], + })); + }, { url, mode, action }); + // The fixture intentionally emits unmatched responses and disconnects. + await page.waitForTimeout(100); + const capabilities = { actors: { get: () => actor }, 'browser-interaction': { + defaultWithin: 1000, expand: (s: string) => s, sleep: async () => {}, + }, 'transport-observation': { + defaultWithin: 1000, expand: (s: string) => s, sleep: async () => {}, + verification: { verified: () => {}, unverified: () => {} }, + }, 'named-actions': { + request: (action: { reducer: string }, input: { values: Record }) => ({ + ...convexFunctionRequest({ deploymentUrl: url, kind: 'mutation', path: action.reducer.includes(':') ? action.reducer : `api:${action.reducer}`, args: input.values }), + responseContract: 'convex-mutation', + }), + fetch, + } }; + const replay = await executeAction(ACTION_REGISTRY, action, action === 'repeatFormWrite' ? { + do: action, actor: 'admin', match: 'Original product', replacement: 'New "product" $1', + control: 'Control product', + fields: [{ testid: 'name', text: 'New "product" $1' }], submit: 'save', + } : { + do: 'replayAs', actor: 'admin', from: 'admin', match: 'Original product', + swap: { find: 'Original product', with: 'New "product" $1' }, settleMs: 0, + }, { capabilities }); + const accepted = ['accepted', 'other-module', 'bearer'].includes(mode); + const supported = accepted || ['replay-refused', 'lost-reply'].includes(mode); + const fallback = action === 'repeatFormWrite' && !supported; + const expected = action === 'repeatFormWrite' + ? mode === 'lost-reply' ? 'inconclusive' : mode === 'replay-refused' ? 'failed' : 'passed' + : supported ? 'passed' : 'inconclusive'; + assert.equal(replay.status, expected, `${action} ${mode}: ${JSON.stringify(replay)}`); + assert.equal(await page.evaluate(() => (window as unknown as { formSubmits: number }).formSubmits), fallback ? 1 : 0); + if (supported && action === 'replayAs') { + const outcome = await executeAction(ACTION_REGISTRY, 'expectReplayCompleted', { + do: 'expectReplayCompleted', actor: 'admin', requireAccepted: true, + }, { capabilities }); + assert.equal(outcome.status, accepted ? 'passed' : mode === 'lost-reply' ? 'inconclusive' : 'failed'); + } + await page.waitForTimeout(30); + assert.deepEqual(rows, accepted || fallback || mode === 'lost-reply' ? [{ name: 'New "product" $1', category: 'Volume', + price: 1.25, variants: ['Standard'], ...(mode === 'bearer' ? {} : { token: 'caller-session' }) }] : []); + observations.push({ action, mode, status: replay.status, writes: rows.length }); + } finally { await context.close(); } + } + result = 'passed'; + } finally { + await browser.close(); sockets.close(); server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + const out = process.env.STACK_BENCH_REPLAY_EVIDENCE; + if (out) { + mkdirSync(out, { recursive: true }); + writeFileSync(`${out}/receipt.json`, JSON.stringify({ result, observations, + rerun: 'STACK_BENCH_REPLAY_EVIDENCE= node --test dist/tests/convex-captured-replay.integration.js' }, null, 2)); + } + } +}); diff --git a/tools/stack-bench/tests/convex-coding-deploy.integration.ts b/tools/stack-bench/tests/convex-coding-deploy.integration.ts new file mode 100644 index 00000000000..69d991d6b97 --- /dev/null +++ b/tools/stack-bench/tests/convex-coding-deploy.integration.ts @@ -0,0 +1,131 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import test from 'node:test'; +import { workRoot } from '../src/composition/tracks.js'; +import { backendResourceLockKeys, claimBackendResources, createBackendLease, publicBackendLease, + readBackendLease, resourceLockScope } from '../src/runtime/backend-lease.js'; +import { codingContainerAgentCommand, codingContainerAgentExecOptions } from '../src/runtime/coding-container-policy.js'; +import { attemptDocker, requireAttemptNetwork } from '../src/runtime/docker-network.js'; +import { activateConvex } from '../src/stacks/backends/convex-lifecycle.js'; +import { releaseBackendLease } from '../src/runtime/backend-teardown.js'; + +test('private Convex fixture deploys as the normal coding user with owned credentials', { + skip: process.env.STACK_BENCH_CONVEX_OWNED_TEST !== '1', timeout: 180_000, +}, async () => { + assert.equal(process.platform, 'linux'); + assert(process.env.STACK_BENCH_WORK_DIR, 'Require the normal daemon-visible work directory'); + const fixture = process.env.STACK_BENCH_CONVEX_FIXTURE!; + const evidenceDirectory = process.env.STACK_BENCH_CONVEX_EVIDENCE_DIR!; + const image = process.env.STACK_BENCH_BUILD_IMAGE!; + assert(fixture && evidenceDirectory && /^sha256:[a-f0-9]{64}$/.test(image)); + mkdirSync(workRoot(), { recursive: true }); + const root = mkdtempSync(join(workRoot(), 'convex-coding-deploy-')); + const app = join(root, 'app'); + const leasePath = join(root, 'lease.json'); + const ports = { vite: 14315, express: 14317, dbPort: null }; + const lease = createBackendLease({ runId: root.split('/').at(-1)!, backend: 'convex', + track: 'ecommerce', runIndex: 2, serverUri: 'http://127.0.0.1:14316' }); + const evidence: Record = { result: 'running', image, + note: 'Pinned app source deployed by coding UID 10001; no model call; registered adapter and normal build owner.' }; + const save = () => writeFileSync(join(evidenceDirectory, 'coding-deploy.json'), JSON.stringify(evidence, null, 2)); + const buildPath = fileURLToPath(new URL('../container/run-build.js', import.meta.url)); + let active; + let volumes: string[] = []; + let failure: unknown; + save(); + try { + claimBackendResources(leasePath, lease, { ...resourceLockScope(), keys: backendResourceLockKeys(lease, ports) }); + activateConvex({ leasePath, leaseToken: lease.ownershipToken, ports }); + active = readBackendLease(leasePath, { active: true }); + const backend = active.resources.container!; + volumes = JSON.parse(attemptDocker(['inspect', '--format', '{{json .Mounts}}', backend.id])) + .filter((mount: { Type: string }) => mount.Type === 'volume').map((mount: { Name: string }) => mount.Name); + mkdirSync(app); + // Only application source and its setup enter the coding workspace. No probes or readers. + for (const name of ['package.json', 'package-lock.json', 'convex.json', 'prepare-auth.mjs', 'convex']) { + cpSync(join(fixture, name), join(app, name), { recursive: true }); + } + const sourceFiles = ['package.json', 'package-lock.json', 'convex.json', 'prepare-auth.mjs', + ...readdirSync(join(app, 'convex')).filter(name => name.endsWith('.js')).map(name => `convex/${name}`)].sort(); + const sourceHashes = () => Object.fromEntries(sourceFiles.map(name => + [name, createHash('sha256').update(readFileSync(join(app, name))).digest('hex')])); + evidence.submittedSource = sourceHashes(); + const prepare = () => execFileSync(process.execPath, [buildPath, + '--prepare-only', '--backend', 'convex', '--app', app, '--image', image], { + env: { ...process.env, STACK_BENCH_LEASE: leasePath, STACK_BENCH_LEASE_TOKEN: lease.ownershipToken }, + encoding: 'utf8', stdio: 'pipe', timeout: 90_000, + }); + evidence.prepared = JSON.parse(prepare().trim()); + active = readBackendLease(leasePath, { active: true }); + const coding = active.resources.buildContainer!; + const inspected = JSON.parse(attemptDocker(['inspect', coding.id]))[0]; + assert.equal(inspected.Image, image); + assert.equal(inspected.HostConfig.NetworkMode, requireAttemptNetwork(active)); + assert.equal(inspected.HostConfig.ReadonlyRootfs, true); + assert.deepEqual(inspected.Mounts.filter((mount: { Type: string }) => mount.Type === 'bind') + .map((mount: { Source: string; Destination: string }) => [mount.Source, mount.Destination]), [[app, '/app']]); + assert(!inspected.Config.Env.some((value: string) => /^(?:STACK_BENCH_LEASE|CONVEX_SELF_HOSTED_ADMIN_KEY|OPENAI_API_KEY|ANTHROPIC_API_KEY)=/.test(value))); + const settings = { CONVEX_SELF_HOSTED_URL: active.resources.serverUri!, + CONVEX_SELF_HOSTED_ADMIN_KEY: attemptDocker(['exec', backend.id, 'bash', './generate_admin_key.sh']) }; + const developer = (command: string, args: string[]) => execFileSync('docker', ['exec', + ...codingContainerAgentExecOptions(), '-w', '/app', + ...Object.keys(settings).flatMap(name => ['-e', name]), coding.id, ...codingContainerAgentCommand(command, args)], { + env: { ...process.env, ...settings }, encoding: 'utf8', stdio: 'pipe', timeout: 90_000, + }); + assert.equal(developer('id', ['-u']).trim(), '10001'); + developer('sh', ['-ec', 'test ! -e /var/run/docker.sock; test ! -e /opt/stack-bench; test ! -e /fixture; ' + + 'test ! -e /app/probe-owned.mjs; test ! -e /app/snapshot.mjs; ' + + 'test -z "$STACK_BENCH_LEASE_TOKEN$OPENAI_API_KEY$ANTHROPIC_API_KEY"; ' + + 'test ! -w /usr/bin/node; test ! -r /run/application']); + developer('npm', ['ci', '--ignore-scripts', '--no-audit', '--no-fund']); + developer('node', ['prepare-auth.mjs']); + developer('node', ['node_modules/convex/bin/main.js', 'dev', '--once', '--typecheck', 'disable']); + evidence.deployedSource = sourceHashes(); + assert.deepEqual(evidence.deployedSource, evidence.submittedSource, 'Native deployment must retain submitted application source'); + const generatedAuth = readFileSync(join(app, 'convex/auth.config.js'), 'utf8'); + assert.equal(generatedAuth, 'export default { providers: [{ domain: process.env.CONVEX_SITE_URL, applicationID: "convex" }] };\n'); + evidence.generatedAuthConfiguration = { source: generatedAuth, sha256: createHash('sha256').update(generatedAuth).digest('hex') }; + evidence.deployment = 'npm install, local auth setup and native Convex CLI deploy ran as UID 10001 in the owned coding namespace'; + const browser = active.resources.browserContainer!.id; + attemptDocker(['exec', browser, 'mkdir', '-p', '/tmp/convex-owned']); + const archive = execFileSync('tar', ['-C', app, '-cf', '-', 'node_modules', 'package.json'], { maxBuffer: 128 * 1024 * 1024 }); + execFileSync('docker', ['exec', '-i', browser, 'tar', '--no-same-owner', '-C', '/tmp/convex-owned', '-xf', '-'], { input: archive }); + for (const name of ['probe-owned.mjs', 'snapshot.mjs']) { + attemptDocker(['exec', '-i', browser, 'sh', '-c', `cat > /tmp/convex-owned/${name}`], readFileSync(join(fixture, name), 'utf8')); + } + attemptDocker(['exec', '-i', browser, 'sh', '-c', 'umask 077; cat > /tmp/convex-owned/owned-private.json'], JSON.stringify(settings)); + try { attemptDocker(['exec', '-w', '/tmp/convex-owned', browser, 'node', 'probe-owned.mjs']); } + finally { + evidence.native = JSON.parse(attemptDocker(['exec', browser, 'cat', '/tmp/convex-owned/owned-probe.json'])); + } + const reused = JSON.parse(prepare().trim()); + assert.equal(reused.identity, (evidence.prepared as { identity: string }).identity); + evidence.reusedSameCodingContainer = true; + evidence.activeLease = publicBackendLease(readBackendLease(leasePath)); + evidence.result = 'passed'; + } catch (error) { + evidence.result = 'failed'; evidence.error = error instanceof Error ? error.message : String(error); failure = error; + } finally { + try { + if (existsSync(leasePath)) { + assert.equal(releaseBackendLease(leasePath, lease.ownershipToken), true); + const final = readBackendLease(leasePath); + evidence.finalLease = publicBackendLease(final); + assert.equal(final.state, 'released'); + if (final.resources.buildContainer) assert(final.resources.buildContainer.workspaceHandedBackAt); + assert(final.resources.locks.every(lock => lock.releasedAt && !existsSync(lock.path))); + for (const volume of volumes) assert.throws(() => attemptDocker(['volume', 'inspect', volume]), /no such volume/i); + evidence.anonymousVolumeCleanup = volumes.length; + rmSync(root, { recursive: true }); + evidence.workspaceRemoved = !existsSync(root); + } + } catch (error) { + evidence.result = 'failed'; evidence.cleanupError = error instanceof Error ? error.message : String(error); failure ??= error; + } finally { save(); } + } + if (failure) throw failure; +}); diff --git a/tools/stack-bench/tests/convex-observer.integration.ts b/tools/stack-bench/tests/convex-observer.integration.ts new file mode 100644 index 00000000000..16222437198 --- /dev/null +++ b/tools/stack-bench/tests/convex-observer.integration.ts @@ -0,0 +1,78 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import test from 'node:test'; +import { createBackendLease, claimBackendResources, backendResourceLockKeys, resourceLockScope, readBackendLease } + from '../src/runtime/backend-lease.js'; +import type { TextCommandExecutor } from '../src/runtime/command-executor.js'; +import { activateConvex, controlConvex } from '../src/stacks/backends/convex-lifecycle.js'; +import { CONVEX_BACKEND_IMAGE } from '../src/stacks/backends/convex-identity.js'; +import { releaseBackendLease } from '../src/runtime/backend-teardown.js'; +import { readConvexTables } from '../src/stacks/backends/convex-operations.js'; + +// Failure cases defined before the transport change: stale ownership must still +// refuse access; missing tables must still fail; both routes must authenticate +// after restart/reset. Compare the same reader over both real routes. +test('Convex native observer preserves ownership and lifecycle behavior over its published port', async () => { + assert.equal(process.platform, 'linux', 'Run inside the Docker controller'); + const out = process.env.STACK_BENCH_OBSERVER_EVIDENCE; + assert(out, 'Set STACK_BENCH_OBSERVER_EVIDENCE to a fresh evidence directory'); + mkdirSync(out, { recursive: false }); + const leasePath = join(out, 'lease.json'); + const lease = createBackendLease({ runId: `observer-transport-${Date.now()}`, backend: 'convex', + track: 'ecommerce', runIndex: 0, serverUri: 'http://127.0.0.1:14310' }); + const ports = { vite: 14309, express: 14311, dbPort: null }; + const samples: { phase: string; route: string; ms: number }[] = []; + const checks: string[] = []; + const receipt = { result: 'failed', cleanup: 'pending', samples, checks, + note: 'Empty native snapshots measure fixed transport overhead. Not a whole-catalog timing or qualification receipt.', + image: CONVEX_BACKEND_IMAGE, + sourceSha256: createHash('sha256').update(readFileSync(new URL('../src/stacks/backends/convex-operations.js', import.meta.url))).digest('hex'), + rerun: 'STACK_BENCH_OBSERVER_EVIDENCE= node --test dist/tests/convex-observer.integration.js' }; + try { + claimBackendResources(leasePath, lease, { ...resourceLockScope(), keys: backendResourceLockKeys(lease, ports) }); + activateConvex({ leasePath, leaseToken: lease.ownershipToken, ports }); + for (const phase of ['initial', 'restart', 'reset'] as const) { + if (phase !== 'initial') await controlConvex({ leasePath, leaseToken: lease.ownershipToken, ports, mode: phase }); + const active = readBackendLease(leasePath, { token: lease.ownershipToken, active: true }); + for (let n = 0; n < 12; n++) { + const route = n % 2 ? 'current' : 'docker-exec-baseline'; + const exec: TextCommandExecutor = (command, args, options) => { + const configured = { ...options, env: { ...process.env, http_proxy: 'http://127.0.0.1:9', + HTTP_PROXY: 'http://127.0.0.1:9', ALL_PROXY: 'http://127.0.0.1:9', no_proxy: '', NO_PROXY: '' } }; + return route === 'docker-exec-baseline' && command === 'curl' + ? execFileSync('docker', ['exec', '-i', active.resources.container!.id, 'curl', ...args], configured) + : execFileSync(command, args, configured); + }; + const started = performance.now(); + assert.deepEqual(readConvexTables([], { lease: active, exec }), {}); + samples.push({ phase, route, ms: performance.now() - started }); + } + assert.throws(() => readConvexTables(['missing_observer_table'], { lease: active }), /required Convex data table is missing/); + const stale = structuredClone(active); + stale.resources.network!.namespaceStartedAt = 'not-the-current-start'; + assert.throws(() => readConvexTables([], { lease: stale }), /namespace anchor/); + const replaced = structuredClone(active); + replaced.resources.container!.id = 'not-the-owned-container'; + assert.throws(() => readConvexTables([], { lease: replaced }), /changed after lease creation/); + checks.push(`${phase}: consistent snapshots; missing table refused; stale namespace refused; replaced container refused`); + } + receipt.result = 'passed'; + } catch (error) { + Object.assign(receipt, { error: String(error) }); + throw error; + } finally { + try { + releaseBackendLease(leasePath, lease.ownershipToken); + receipt.cleanup = readBackendLease(leasePath).state; + assert.equal(receipt.cleanup, 'released'); + } catch (error) { + Object.assign(receipt, { result: 'failed', cleanup: String(error) }); + // A cleanup failure must fail the test even after an earlier error. + // eslint-disable-next-line no-unsafe-finally + throw error; + } finally { writeFileSync(join(out, 'receipt.json'), JSON.stringify(receipt, null, 2)); } + } +}); diff --git a/tools/stack-bench/tests/convex-operations.test.ts b/tools/stack-bench/tests/convex-operations.test.ts new file mode 100644 index 00000000000..3f244cf1be0 --- /dev/null +++ b/tools/stack-bench/tests/convex-operations.test.ts @@ -0,0 +1,99 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import type { BackendLease } from '../src/runtime/backend-lease.js'; +import type { TextCommandExecutor } from '../src/runtime/command-executor.js'; +import { getConvexStock, setConvexStock, getConvexCheckoutState, readConvexTables } from '../src/stacks/backends/convex-operations.js'; +import { classifyResponseContract } from '../src/actions/named-action-runtime.js'; + +const lease = { backend: 'convex', resources: { serverUri: 'http://127.0.0.1:13210', + container: { id: 'owned', name: 'owned-name' }, network: { namespaceContainerId: 'owned', + namespaceStartedAt: 'start', firewallSha256: 'firewall', firewallInstalledAt: 'installed' } } } as BackendLease; +const rows = () => ({ + item: [{ _id: 'item-native', name: 'Keyboard', price: 19.99 }], + warehouse: [{ _id: 'warehouse-native', name: 'Main' }], + stock: [{ _id: 'stock-native', item_id: 'item-native', warehouse_id: 'warehouse-native', quantity: 3 }], + order_account: [{ _id: 'account-native', username: 'buyer' }], + order_header: [], order_line: [], order_cart: [], order_reservation: [], order_allocation: [], +}); +function native(data: Record[]>, options: { stall?: boolean; namespaceStarted?: string; omitTable?: string; noTruncate?: boolean; noTimestamp?: boolean } = {}) { + const calls: Record[] = []; + const exec: TextCommandExecutor = (command, args, commandOptions) => { + assert(!args.some(arg => arg.includes('private-admin')), 'admin key must never enter argv'); + if (args[0] === 'inspect') return JSON.stringify({ Id: 'owned', + State: { Running: true, StartedAt: options.namespaceStarted ?? 'start' } }); + if (args.includes('./generate_admin_key.sh')) return 'private-admin'; + assert(command === 'curl' || args.includes('curl')); + const config = commandOptions.input!; + assert(config.includes('Authorization: Convex private-admin')); + const body = JSON.parse(JSON.parse(config.split('\n').find(line => line.startsWith('data = '))!.slice(7))); + calls.push(body); + if (body.path === '_system/cli/tables') return JSON.stringify({ status: 'success', value: { + page: Object.keys(data).map(name => ({ name })), isDone: true } }); + if (body.path === '_system/frontend/patchDocumentsFields') { + assert.deepEqual(body.args, { table: 'stock', ids: ['stock-native'], fields: { quantity: 17 }, componentId: null }); + data.stock![0]!.quantity = 17; + return JSON.stringify({ status: 'success', value: { success: true } }); + } + const names = Object.keys(body.selection['']).filter(name => name !== '_other' && name !== options.omitTable); + return JSON.stringify({ status: options.stall ? { type: 'snapshotting' } : { type: 'upToDate', ...(options.noTimestamp ? {} : { snapshotTs: 123 }) }, + truncates: options.noTruncate ? [] : names.map(table => ({ component: '', table })), + values: names.flatMap(table => data[table]!.map(value => ({ component: '', table, deleted: false, value }))), + pagination: { nextCursor: options.stall ? 'same-cursor' : null } }); + }; + return { exec, calls }; +} + +test('native stock observer reads stored rows and patches only the selected native document', () => { + const data = rows(), client = native(data); + assert.deepEqual(getConvexStock({ item: 'Keyboard', warehouse: 'Main', lease, exec: client.exec }), + { backend: 'convex', item: 'Keyboard', warehouse: 'Main', quantity: 3 }); + setConvexStock({ item: 'Keyboard', warehouse: 'Main', quantity: 17, lease, exec: client.exec }); + assert.equal(data.stock[0]!.quantity, 17); + assert(!client.calls.some(call => String(call.path).startsWith('api:')), 'never use an app observer'); +}); + +test('native order snapshots normalize only IDs and use the shared order oracle', () => { + const data = rows(), client = native(data); + const result = getConvexCheckoutState({ account: 'buyer', item: 'Keyboard', lease, exec: client.exec, + storage: { kind: 'order-data', cart: true, warehouses: true } }); + assert.equal(result.state.accountId, 'buyer'); + assert.equal(result.state.priceMinor, 1999); + assert.equal(result.state.stock[0]!.quantity, 3); + // A diagnostic read without a storage selection is the caller's error, not an invalid app interface. + assert.throws(() => getConvexCheckoutState({ account: 'buyer', item: 'Keyboard', lease, exec: client.exec }), + (error: unknown) => error instanceof Error && /declared order data/.test(error.message) + && !('orderDataInterface' in error)); +}); + +test('native observer rejects missing tables, duplicate links, invalid quantities and stalled snapshots', () => { + for (const defect of ['missing', 'duplicate', 'quantity', 'link']) { + const data: Record[]> = rows(); + if (defect === 'missing') delete data.stock; + if (defect === 'duplicate') data.stock!.push({ ...data.stock![0], _id: 'another' }); + if (defect === 'quantity') data.stock![0]!.quantity = 0.5; + if (defect === 'link') data.stock![0]!.warehouse_id = 'orphan'; + assert.throws(() => getConvexStock({ item: 'Keyboard', lease, exec: native(data).exec }), + (error: unknown) => (error as { stockInterface?: boolean }).stockInterface === true, defect); + } + const nestedTimestamp = rows(); Object.assign(nestedTimestamp.item[0]!, { snapshotTs: 456 }); + assert.throws(() => readConvexTables(['item'], { lease, exec: native(nestedTimestamp, { noTimestamp: true }).exec }), /consistency timestamp/); + assert.throws(() => readConvexTables(['order_cart'], { lease, exec: native(rows(), { omitTable: 'order_cart' }).exec }), /omitted a selected table/); + assert.throws(() => readConvexTables(['item'], { lease, exec: native(rows(), { noTruncate: true }).exec }), /invalid document/); + assert.throws(() => readConvexTables(['item'], { lease, exec: native(rows(), { stall: true }).exec }), /cursor did not advance/); + assert.throws(() => getConvexStock({ item: 'Keyboard', lease, exec: native(rows(), { namespaceStarted: 'changed' }).exec }), /namespace anchor/); +}); + +test('native envelopes preserve HTTP status semantics without turning errors into acceptance or refusals', () => { + const request = { responseContract: 'convex-mutation' as const }; + for (const status of [200, 560]) { + const rejected = classifyResponseContract(request, { status, text: JSON.stringify({ status: 'error', errorMessage: 'no', errorData: null }) }); + assert.equal(rejected.ok, false); assert.equal(rejected.applicationRejected, true); + } + for (const [status, text] of [[400, 'bad args'], [200, '{"status":"error","errorMessage":"missing function"}']] as const) { + const result = classifyResponseContract(request, { status, text }); + assert.equal(result.ok, false); assert.equal(result.refusalKind, null); + } + assert.equal(classifyResponseContract(request, { status: 200, text: '{}' }).complete, true); + assert.equal(classifyResponseContract(request, { status: 200, text: '{}' }).ok, false); + assert.equal(classifyResponseContract(request, { status: 200, text: '{"status":"success","value":null}' }).ok, true); +}); diff --git a/tools/stack-bench/tests/convex-owned-lifecycle.integration.ts b/tools/stack-bench/tests/convex-owned-lifecycle.integration.ts new file mode 100644 index 00000000000..9005e0f7d74 --- /dev/null +++ b/tools/stack-bench/tests/convex-owned-lifecycle.integration.ts @@ -0,0 +1,285 @@ +import assert from 'node:assert/strict'; +import { execFileSync, spawn } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { once } from 'node:events'; +import { cpSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, writeFileSync, existsSync } from 'node:fs'; +import { createServer } from 'node:net'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { setTimeout as delay } from 'node:timers/promises'; +import { backendResourceLockKeys, claimBackendResources, createBackendLease, publicBackendLease, + readBackendLease, resourceLockScope, verifyResourceLocks } from '../src/runtime/backend-lease.js'; +import type { ReservationLease } from '../src/runtime/backend-lease.js'; +import { borrowCampaignReservation, delegateCampaignReservation, closeCampaignDelegation, + releaseCampaignReservation } from '../src/campaigns/campaign-admission.js'; +import { attemptDocker, requireAttemptNetwork } from '../src/runtime/docker-network.js'; +import { activateConvex, controlConvex, recoverConvex, CONVEX_PROCESS_RECORD } from '../src/stacks/backends/convex-lifecycle.js'; +import { releaseBackendLease } from '../src/runtime/backend-teardown.js'; +import { prepareProcessCrash } from '../src/stacks/process-crash.js'; + +// Run inside the Linux controller with its normal lock directory and Docker socket. +// The fixture and evidence directory are explicit mounts; no paid calls or registry entry. +test('Convex campaign-delegated launch, warm restart, full reset, and exact cleanup', { + skip: process.env.STACK_BENCH_CONVEX_OWNED_TEST !== '1', timeout: 240_000, +}, async () => { + assert.equal(process.platform, 'linux'); + const fixture = process.env.STACK_BENCH_CONVEX_FIXTURE!; + const evidenceDirectory = process.env.STACK_BENCH_CONVEX_EVIDENCE_DIR!; + assert(fixture && evidenceDirectory); + const root = mkdtempSync(join(tmpdir(), 'stack-bench-convex-owned-')); + mkdirSync(join(root, '.private')); + const path = join(root, 'lease.json'); + const ports = { vite: 14309, express: 14311, dbPort: null }; + const lease = createBackendLease({ runId: root.split('/').at(-1)!, backend: 'convex', + track: 'ecommerce', runIndex: 0, serverUri: 'http://127.0.0.1:14310' }); + const parentPath = join(root, 'reservation.json'); + const parent: ReservationLease = { ...createBackendLease({ runId: `${lease.runId}-reservation`, + backend: 'stub', track: lease.track, runIndex: 0 }), + campaign: { sha256: createHash('sha256').update(lease.runId).digest('hex'), + admissionId: lease.runId, runIndices: [0] } }; + const reservation = { path: parentPath, token: parent.ownershipToken }; + const evidence: Record = { result: 'running', ports, + note: 'Campaign reservation delegation into the real Convex lifecycle; direct-owned siblings exercise isolation and failure cleanup.' }; + const save = () => writeFileSync(join(evidenceDirectory, 'owned-lifecycle.json'), JSON.stringify(evidence, null, 2)); + save(); + let active; + let volumes: string[] = []; + let failure: unknown; + try { + const keys = backendResourceLockKeys(lease, ports); + claimBackendResources(parentPath, parent, { ...resourceLockScope(), keys }); + const input = { campaignSha256: parent.campaign.sha256, admissionId: lease.runId, + executionId: lease.runId, output: join(root, 'output'), backend: lease.backend, runIndex: 0 }; + const env = delegateCampaignReservation(reservation, root, input); + borrowCampaignReservation({ ...input, env, leasePath: path, lease, keys }); + assert.deepEqual(readBackendLease(path).resources.locks, []); + activateConvex({ leasePath: path, leaseToken: lease.ownershipToken, ports }); + active = readBackendLease(path, { token: lease.ownershipToken, active: true }); + evidence.activeLease = publicBackendLease(active); + requireAttemptNetwork(active); + const backend = active.resources.container!; + const bindings = attemptDocker(['inspect', '--format', '{{json .HostConfig.PortBindings}}', backend.id]); + volumes = JSON.parse(attemptDocker(['inspect', '--format', '{{json .Mounts}}', backend.id])) + .filter((mount: { Type: string }) => mount.Type === 'volume').map((mount: { Name: string }) => mount.Name); + assert.equal(volumes.length, 1, 'The backend data is one container-owned anonymous volume'); + assert.equal((await fetch(`${active.resources.serverUri}/version`)).status, 200); + const key = attemptDocker(['exec', backend.id, 'bash', './generate_admin_key.sh']); + assert(key.includes('|'), 'Native admin key format'); + const environment = { CONVEX_SELF_HOSTED_URL: active.resources.serverUri!, CONVEX_SELF_HOSTED_ADMIN_KEY: key }; + // Deploy trusted fixture code from the controller. The browser's /tmp is + // intentionally noexec and must not be weakened to run the Convex compiler. + const workspace = join(root, 'fixture'); + cpSync(fixture, workspace, { recursive: true }); + const sourceHashes = () => Object.fromEntries(readdirSync(join(workspace, 'convex')).filter(name => name.endsWith('.js')) + .sort().map(name => [name, createHash('sha256').update(readFileSync(join(workspace, 'convex', name))).digest('hex')])); + const run = (command: string, args: string[], settings = environment) => execFileSync(command, args, { + cwd: workspace, env: { ...process.env, ...settings }, encoding: 'utf8', stdio: 'pipe', timeout: 120_000 }); + const cache = active.resources.network!.services[0]!; + run('npm', ['ci', '--ignore-scripts', '--no-audit', '--no-fund', '--registry', `http://${cache.address}:${cache.port}`]); + run(process.execPath, ['prepare-auth.mjs']); + run(process.execPath, ['node_modules/convex/bin/main.js', 'dev', '--once', '--typecheck', 'disable']); + const deployedSource = sourceHashes(); + const browser = active.resources.browserContainer!.id; + attemptDocker(['exec', browser, 'mkdir', '-p', '/tmp/convex-owned']); + const archive = execFileSync('tar', ['-C', workspace, '-cf', '-', '.'], { maxBuffer: 128 * 1024 * 1024 }); + execFileSync('docker', ['exec', '-i', browser, 'tar', '-C', '/tmp/convex-owned', '-xf', '-'], { input: archive }); + attemptDocker(['exec', '-i', browser, 'sh', '-c', 'umask 077; cat > /tmp/convex-owned/owned-private.json'], JSON.stringify(environment)); + const probeEvidence = (phase: string, target = browser) => JSON.parse(attemptDocker(['exec', target, 'cat', + `/tmp/convex-owned/owned-probe${phase === 'initial' ? '' : `-${phase}`}.json`])); + const probe = (phase: string, target = browser) => { + execFileSync('docker', ['exec', '-w', '/tmp/convex-owned', target, 'node', 'probe-owned.mjs', phase], + { encoding: 'utf8', stdio: 'pipe', timeout: 60_000 }); + return probeEvidence(phase, target); + }; + evidence.native = probe('initial'); + evidence.nativeValidation = probe('validation'); + const siblingPorts = { vite: 14312, express: 14314, dbPort: null }; + // Exercise one other attempt at a time. It overlaps the live primary attempt. + for (const mode of ['overlap', 'failed-start', 'cancel-start'] as const) { + const siblingPath = join(root, `${mode}.json`); + const sibling = createBackendLease({ runId: `${lease.runId}-${mode}`, backend: 'convex', + track: 'ecommerce', runIndex: 1, serverUri: 'http://127.0.0.1:14313' }); + const record: Record = { result: 'running' }; + evidence[mode] = record; + let siblingVolumes: string[] = []; + try { + claimBackendResources(siblingPath, sibling, { ...resourceLockScope(), keys: backendResourceLockKeys(sibling, siblingPorts) }); + const activate = () => activateConvex({ leasePath: siblingPath, leaseToken: sibling.ownershipToken, ports: siblingPorts }); + if (mode === 'failed-start') { + const listener = createServer(); + listener.listen(siblingPorts.vite, '127.0.0.1'); + await once(listener, 'listening'); + try { + assert.throws(activate, error => { + assert(error instanceof Error); assert.match(error.message, /Docker could not start owned backend container/); + record.startFailure = error.message; return true; + }); + } + finally { await new Promise((resolve, reject) => listener.close(error => error ? reject(error) : resolve())); } + assert.equal(readBackendLease(siblingPath).state, 'starting'); + record.failedStartLease = publicBackendLease(readBackendLease(siblingPath)); + } else if (mode === 'cancel-start') { + const child = spawn(process.execPath, ['--input-type=module', '-e', + `import {activateConvex} from ${JSON.stringify(new URL('../src/stacks/backends/convex-lifecycle.js', import.meta.url).href)}; ` + + 'activateConvex(JSON.parse(process.env.CONVEX_START_INPUT));'], { + detached: true, stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, + CONVEX_START_INPUT: JSON.stringify({ leasePath: siblingPath, leaseToken: sibling.ownershipToken, ports: siblingPorts }) }, + }); + const closed = once(child, 'close'); + let output = ''; + child.stdout.on('data', chunk => { output += chunk; }); + child.stderr.on('data', chunk => { output += chunk; }); + try { + const deadline = Date.now() + 20_000; + while (!readBackendLease(siblingPath).resources.container) { + assert(child.exitCode === null && child.signalCode === null && Date.now() < deadline, 'Activation must record a resource before cancellation'); + await delay(25); + } + assert.equal(readBackendLease(siblingPath).state, 'starting'); + process.kill(-child.pid!, 'SIGTERM'); + const [code, signal] = await closed; + assert.equal(code, null); assert.equal(signal, 'SIGTERM'); + record.interrupted = publicBackendLease(readBackendLease(siblingPath)); + } finally { + if (child.exitCode === null && child.signalCode === null) { process.kill(-child.pid!, 'SIGKILL'); await closed; } + record.controllerOutput = output; + } + } else { + activate(); + const other = readBackendLease(siblingPath, { active: true }); + assert.notEqual(other.resources.network!.id, active.resources.network!.id); + const otherKey = attemptDocker(['exec', other.resources.container!.id, 'bash', './generate_admin_key.sh']); + const settings = { CONVEX_SELF_HOSTED_URL: other.resources.serverUri!, CONVEX_SELF_HOSTED_ADMIN_KEY: otherKey }; + run(process.execPath, ['prepare-auth.mjs'], settings); + run(process.execPath, ['node_modules/convex/bin/main.js', 'dev', '--once', '--typecheck', 'disable'], settings); + const otherBrowser = other.resources.browserContainer!.id; + attemptDocker(['exec', otherBrowser, 'mkdir', '-p', '/tmp/convex-owned']); + execFileSync('docker', ['exec', '-i', otherBrowser, 'tar', '-C', '/tmp/convex-owned', '-xf', '-'], { input: archive }); + attemptDocker(['exec', '-i', otherBrowser, 'sh', '-c', 'umask 077; cat > /tmp/convex-owned/owned-private.json'], JSON.stringify(settings)); + record.native = probe('initial', otherBrowser); + for (const [target, source] of [[browser, otherBrowser], [otherBrowser, browser]]) { + const foreign = attemptDocker(['exec', source!, 'cat', '/tmp/convex-owned/lifecycle-private.json']); + attemptDocker(['exec', '-i', target!, 'sh', '-c', 'umask 077; cat > /tmp/convex-owned/foreign-private.json'], foreign); + record[target === browser ? 'primaryIsolation' : 'siblingIsolation'] = probe('isolation', target!); + } + for (const [target, ownUrl, blocked] of [ + [browser, environment.CONVEX_SELF_HOSTED_URL, [`http://${other.resources.network!.ownAddresses![0]}:14313/version`, 'http://host.docker.internal:14313/version']], + [otherBrowser, settings.CONVEX_SELF_HOSTED_URL, [`http://${active.resources.network!.ownAddresses![0]}:14310/version`, 'http://host.docker.internal:14310/version']], + ] as const) { + attemptDocker(['exec', target, 'node', '--input-type=module', '-e', + `import assert from 'node:assert/strict'; assert.equal((await fetch(${JSON.stringify(`${ownUrl}/version`)})).status,200); ` + + `for(const url of ${JSON.stringify(blocked)}) await assert.rejects(fetch(url,{signal:AbortSignal.timeout(1500)}));`]); + } + record.networkIsolation = 'each browser reaches itself but cannot reach the other attempt through bridge or host ports'; + } + record.result = 'passed'; + } catch (error) { + record.result = 'failed'; record.error = error instanceof Error ? error.message : String(error); + const failedBrowser = readBackendLease(siblingPath).resources.browserContainer?.id; + for (const [name, target, phase] of [['primaryIsolation', browser, 'isolation'], + ['native', failedBrowser, 'initial'], ['siblingIsolation', failedBrowser, 'isolation']]) { + if (target) try { record[name!] = probeEvidence(phase!, target); } catch { /* Probe did not start. */ } + } + throw error; + } finally { + const partial = readBackendLease(siblingPath); + if (partial.resources.container) { + siblingVolumes = JSON.parse(attemptDocker(['inspect', '--format', '{{json .Mounts}}', partial.resources.container.id])) + .filter((mount: { Type: string }) => mount.Type === 'volume').map((mount: { Name: string }) => mount.Name); + } + assert.equal(releaseBackendLease(siblingPath, sibling.ownershipToken), true); + const released = readBackendLease(siblingPath); + record.finalLease = publicBackendLease(released); + assert.equal(released.state, 'released'); + assert(released.resources.locks.every(lock => lock.releasedAt && !existsSync(lock.path))); + for (const volume of siblingVolumes) assert.throws(() => attemptDocker(['volume', 'inspect', volume]), /no such volume/i); + record.anonymousVolumeCleanup = siblingVolumes.length; + assert.equal((await fetch(`${active.resources.serverUri}/version`)).status, 200); + // Recheck the actual stored primary data and original session after each cleanup. + record.primaryAfterCleanup = probe('retained'); + save(); + } + } + const jwks = await fetch(`http://127.0.0.1:${ports.express}/.well-known/jwks.json`); + assert.equal(jwks.status, 200); + assert.equal((await jwks.json() as { keys: unknown[] }).keys.length, 1); + assert.throws(() => releaseBackendLease(path, 'wrong-token'), /ownership token does not match/); + assert.equal(attemptDocker(['inspect', '--format', '{{.State.Running}}', backend.id]), 'true'); + const processRecord = attemptDocker(['exec', backend.id, 'cat', CONVEX_PROCESS_RECORD]); + const [pid, started] = processRecord.split(' '); + const writeRecord = (value: string) => attemptDocker(['exec', '-i', backend.id, 'sh', '-c', + `cat > ${CONVEX_PROCESS_RECORD}`], value); + try { + for (const record of ['', `${pid} ${BigInt(started!) + 1n}\n`]) { + writeRecord(record); + await assert.rejects(controlConvex({ leasePath: path, leaseToken: lease.ownershipToken, ports, mode: 'reset' }), + /Command failed|Convex process identity changed/); + assert.equal((await fetch(`${active.resources.serverUri}/version`)).status, 200); + } + } finally { writeRecord(`${processRecord}\n`); } + evidence.resetGuard = 'empty and stale process records refused without stopping the backend'; + await assert.rejects(prepareProcessCrash(active, 'application'), /share one boundary/); + assert.throws(() => recoverConvex({ leasePath: path, leaseToken: lease.ownershipToken, ports }), + /Command failed/, 'Crash recovery must refuse a still-live backend'); + const crash = await prepareProcessCrash(active, 'database'); + try { + evidence.crash = await crash.crash(); + assert.equal(attemptDocker(['inspect', '--format', '{{.State.Running}}', backend.id]), 'true'); + await assert.rejects(fetch(`${active.resources.serverUri}/version`, { signal: AbortSignal.timeout(2000) })); + recoverConvex({ leasePath: path, leaseToken: lease.ownershipToken, ports }); + evidence.crashRetained = probe('retained'); + } finally { await crash.close(); } + await controlConvex({ leasePath: path, leaseToken: lease.ownershipToken, ports, mode: 'restart' }); + evidence.warm = probe('warm'); + evidence.pending = probe('prepare-reset'); + assert.throws(() => probe('reset')); + const retainedStateControl = probeEvidence('reset'); + evidence.retainedStateControl = retainedStateControl; + assert.equal(retainedStateControl.result, 'failed'); + assert.match(retainedStateControl.error, /Reset requires empty stored state/); + await controlConvex({ leasePath: path, leaseToken: lease.ownershipToken, ports, mode: 'reset' }); + environment.CONVEX_SELF_HOSTED_ADMIN_KEY = attemptDocker(['exec', backend.id, 'bash', './generate_admin_key.sh']); + run(process.execPath, ['prepare-auth.mjs']); + run(process.execPath, ['node_modules/convex/bin/main.js', 'dev', '--once', '--typecheck', 'disable']); + assert.deepEqual(sourceHashes(), deployedSource, 'Reset must redeploy identical application source'); + evidence.deployedSource = deployedSource; + attemptDocker(['exec', '-i', browser, 'sh', '-c', 'umask 077; cat > /tmp/convex-owned/owned-private.json'], JSON.stringify(environment)); + evidence.reset = probe('reset'); + assert.equal(attemptDocker(['inspect', '--format', '{{json .HostConfig.PortBindings}}', backend.id]), bindings); + assert.deepEqual(readBackendLease(path).resources.network, active.resources.network); + requireAttemptNetwork(readBackendLease(path)); + evidence.namespaceAndPortsPreserved = true; + evidence.result = 'passed'; + } catch (error) { + evidence.result = 'failed'; + evidence.error = error instanceof Error ? error.message : String(error); + if (active?.resources.browserContainer) { + for (const [key, suffix] of [['native', ''], ['warm', '-warm'], ['pending', '-prepare-reset'], ['reset', '-reset']]) { + try { evidence[key!] = JSON.parse(attemptDocker(['exec', active.resources.browserContainer.id, 'cat', + `/tmp/convex-owned/owned-probe${suffix}.json`])); } catch { /* Before this probe starts. */ } + } + } + failure = error; + } finally { + try { + if (existsSync(path)) { + assert.equal(releaseBackendLease(path, lease.ownershipToken), true, 'Exact owned cleanup must succeed'); + const final = readBackendLease(path); + evidence.finalLease = publicBackendLease(final); + assert.equal(final.state, 'released'); + for (const volume of volumes) assert.throws(() => attemptDocker(['volume', 'inspect', volume]), /no such volume/i); + evidence.anonymousVolumeCleanup = volumes.length; + verifyResourceLocks(parent); + closeCampaignDelegation(root, lease.runId); + evidence.parentLocksSurvivedChildCleanup = true; + } + if (existsSync(parentPath)) releaseCampaignReservation(reservation); + } catch (error) { + evidence.result = 'failed'; + evidence.cleanupError = error instanceof Error ? error.message : String(error); + failure ??= error; + } finally { save(); } + } + if (failure) throw failure; +}); diff --git a/tools/stack-bench/tests/convex-protocol.test.ts b/tools/stack-bench/tests/convex-protocol.test.ts new file mode 100644 index 00000000000..5a3aa1ef05e --- /dev/null +++ b/tools/stack-bench/tests/convex-protocol.test.ts @@ -0,0 +1,72 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { classifyConvexFunctionResponse, convexFunctionRequest } from '../src/stacks/backends/convex-protocol.js'; + +test('Convex business requests use native functions and only end-user bearer auth', () => { + const request = convexFunctionRequest({ deploymentUrl: 'http://localhost:3210/', kind: 'mutation', + path: 'shop/orders:buy', args: { item: 'native-id', quantity: 1 }, token: 'user-token' }); + assert.equal(request.url, 'http://localhost:3210/api/mutation'); + assert.equal(request.method, 'POST'); + assert.deepEqual(request.headers, { 'Content-Type': 'application/json', Authorization: 'Bearer user-token' }); + assert.deepEqual(JSON.parse(request.body), { + path: 'shop/orders:buy', args: { item: 'native-id', quantity: 1 }, format: 'json', + }); + const anonymous = convexFunctionRequest({ deploymentUrl: 'http://localhost:3210', kind: 'query', + path: 'shop/orders', args: {} }); + assert.equal(anonymous.headers.Authorization, undefined); + assert.equal(JSON.parse(anonymous.body).path, 'shop/orders'); +}); + +test('Convex request serialization must not silently erase malformed test inputs', () => { + for (const bad of [undefined, NaN, Infinity, () => 1, Symbol('x'), 1n]) { + assert.throws(() => convexFunctionRequest({ deploymentUrl: 'http://localhost:3210', kind: 'mutation', + path: 'shop:buy', args: { quantity: bad } }), TypeError); + } + assert.throws(() => convexFunctionRequest({ deploymentUrl: 'http://admin:secret@localhost:3210', + kind: 'query', path: 'shop:list', args: {} }), TypeError); +}); + +test('HTTP success requires a valid native success envelope, including null returns', () => { + assert.deepEqual(classifyConvexFunctionResponse(200, '{"status":"success","value":null}'), + { kind: 'accepted', httpStatus: 200, value: null }); + for (const body of ['{}', '[]', 'null', 'not JSON', '{"status":"success"}', + '{"status":"success","value":1,"errorData":"failure"}', + '{"status":"success","value":1,"logLines":[1]}']) { + assert.equal(classifyConvexFunctionResponse(200, body).kind, 'invalid-response', body); + } + assert.equal(classifyConvexFunctionResponse(560, '{"status":"success","value":1}').kind, 'invalid-response'); +}); + +test('ConvexError payloads identify application errors; HTTP 200 does not accept them', () => { + for (const httpStatus of [200, 560]) { + for (const errorData of [null, false, 0, '', { code: 'NO_STOCK' }]) { + assert.deepEqual(classifyConvexFunctionResponse(httpStatus, + JSON.stringify({ status: 'error', errorMessage: 'refused', errorData })), + { kind: 'application-error', httpStatus, message: 'refused', errorData }); + } + } +}); + +test('Unhandled and missing-function errors are not deliberate application refusals', () => { + for (const message of ['Uncaught Error: failed', 'Could not find public function for shop:missing']) { + assert.deepEqual(classifyConvexFunctionResponse(200, JSON.stringify({ status: 'error', errorMessage: message })), + { kind: 'function-error', httpStatus: 200, message }); + } + const nativeError = '{"code":"BadRequest","message":"missing function"}'; + assert.deepEqual(classifyConvexFunctionResponse(400, nativeError), + { kind: 'http-error', httpStatus: 400, text: nativeError }); + assert.equal(classifyConvexFunctionResponse(503, '{"status":"success","value":1}').kind, 'http-error'); + assert.equal(classifyConvexFunctionResponse(200, '{"status":"error","errorData":{}}').kind, 'invalid-response'); +}); + + +test('pinned native argument validation is distinct from missing exports and user exceptions', () => { + const message = '[Request ID: c3c0e4b69f8972e5] Server Error\nArgumentValidationError: Object contains extra field `username` that is not in the validator.\n'; + assert.equal(classifyConvexFunctionResponse(200, JSON.stringify({ status: 'error', errorMessage: message })).kind, 'validation-error'); + for (const errorMessage of [ + '[Request ID: c3c0e4b69f8972e5] Server Error\nCould not find public function for api:missing.', + '[Request ID: c3c0e4b69f8972e5] Server Error\nUncaught Error: ArgumentValidationError: made up', + 'ArgumentValidationError: unqualified bare text', + ]) assert.equal(classifyConvexFunctionResponse(200, JSON.stringify({ status: 'error', errorMessage })).kind, 'function-error'); + assert.equal(classifyConvexFunctionResponse(400, JSON.stringify({ status: 'error', errorMessage: message })).kind, 'http-error'); +}); diff --git a/tools/stack-bench/tests/convex-reader.integration.ts b/tools/stack-bench/tests/convex-reader.integration.ts new file mode 100644 index 00000000000..223b48dea00 --- /dev/null +++ b/tools/stack-bench/tests/convex-reader.integration.ts @@ -0,0 +1,120 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import test from 'node:test'; +import { createBackendLease, claimBackendResources, backendResourceLockKeys, resourceLockScope, readBackendLease, writeBackendLease } + from '../src/runtime/backend-lease.js'; +import { activateConvex, controlConvex } from '../src/stacks/backends/convex-lifecycle.js'; +import { CONVEX_BACKEND_IMAGE } from '../src/stacks/backends/convex-identity.js'; +import { releaseBackendLease } from '../src/runtime/backend-teardown.js'; +import { createConvexOrderDataReader, getConvexCheckoutState, convexAdminKey } from '../src/stacks/backends/convex-operations.js'; +import { orderDataColumns } from '../src/stacks/order-data.js'; + +// Failure cases, before implementation: cached rows can hide new writes; cached +// clients can retain expired deadlines; cached credentials must not cross lease, +// error, close, restart or reset boundaries. Compare full fresh snapshots, and +// require a real ownership failure before any HTTP request on a warm reader. +test('scoped Convex reader retains fresh state and refuses stale ownership', async () => { + assert.equal(process.platform, 'linux'); + const out = process.env.STACK_BENCH_OBSERVER_EVIDENCE; + assert(out, 'Set STACK_BENCH_OBSERVER_EVIDENCE to a fresh directory'); + mkdirSync(out, { recursive: false }); + const leasePath = `${out}/lease.json`; + const lease = createBackendLease({ runId: `convex-reader-${Date.now()}`, backend: 'convex', track: 'ecommerce', + runIndex: 0, serverUri: 'http://127.0.0.1:14310' }); + const ports = { vite: 14309, express: 14311, dbPort: null }; + const storage = { kind: 'order-data', cart: true, warehouses: true } as const; + const selection = { storage, account: 'reader-observer', item: 'Product 0' }; + const checks: string[] = []; + const samples: { phase: string; route: string; population: number; ms: number }[] = []; + const receipt = { result: 'failed', cleanup: 'pending', checks, samples, image: CONVEX_BACKEND_IMAGE, + controller: process.env.STACK_BENCH_CONTROLLER_IMAGE_ID, + sourceSha256: createHash('sha256').update(readFileSync(new URL('../src/stacks/backends/convex-operations.js', import.meta.url))).digest('hex'), + rerun: 'STACK_BENCH_OBSERVER_EVIDENCE= node --test dist/tests/convex-reader.integration.js' }; + try { + claimBackendResources(leasePath, lease, { ...resourceLockScope(), keys: backendResourceLockKeys(lease, ports) }); + activateConvex({ leasePath, leaseToken: lease.ownershipToken, ports }); + let population = 0; + for (const phase of ['initial', 'restart', 'reset'] as const) { + if (phase !== 'initial') await controlConvex({ leasePath, leaseToken: lease.ownershipToken, ports, mode: phase }); + const active = readBackendLease(leasePath, { token: lease.ownershipToken, active: true }); + const key = convexAdminKey(active); + const mutation = (path: string, args: unknown) => { + const config = [`url = ${JSON.stringify(active.resources.serverUri + '/api/mutation')}`, + `header = ${JSON.stringify('Authorization: Convex ' + key)}`, 'header = "Content-Type: application/json"', + `data = ${JSON.stringify(JSON.stringify({ path, args, format: 'json' }))}`].join('\n'); + const result = JSON.parse(execFileSync('curl', ['--disable', '--noproxy', '*', '--silent', '--show-error', '--fail', + '--max-time', '30', '--config', '-'], { encoding: 'utf8', input: config, timeout: 30_000 })); + assert.equal(result.status, 'success'); + }; + if (phase !== 'restart') { + population = 0; + for (const table of Object.keys(orderDataColumns(storage))) mutation('_system/frontend/createTable', { table, componentId: null }); + } + let keys = 0; + let requests = 0; + let stale = false; + const reader = createConvexOrderDataReader({ path: leasePath, lease: active, exec(command, args, options) { + if (args.includes('./generate_admin_key.sh')) keys++; + if (command === 'curl') requests++; + const result = execFileSync(command, args, options); + // A real Docker inspection, with the namespace start replaced to emulate + // an external restart without destroying the owned fixture mid-test. + if (stale && args[0] === 'inspect') return result.replaceAll(active.resources.network!.namespaceStartedAt!, 'stale-start'); + return result; + } }); + try { + for (const count of [3, 247, 750]) { + mutation('_system/frontend/addDocument', { table: 'item', componentId: null, + documents: Array.from({ length: count }, (_, i) => ({ name: `Product ${population + i}`, price: 11.25, + description: 'Catalog product', category: 'Performance', variants: [] })) }); + population += count; + for (let n = 0; n < 6; n++) { + const route = n % 2 ? 'scoped' : 'stateless'; + const start = performance.now(); + const snapshot = route === 'scoped' ? reader.read(selection) : getConvexCheckoutState({ ...selection, lease: active }); + samples.push({ phase, route, population, ms: performance.now() - start }); + assert.equal(snapshot.catalog.length, population); + assert.equal(new Set(snapshot.catalog.map(row => row.name)).size, population); + assert(snapshot.catalog.some(row => row.name === `Product ${population - 1}`)); + assert.deepEqual(snapshot, getConvexCheckoutState({ ...selection, lease: active })); + } + } + assert.equal(keys, 1, 'one native credential per feature'); + stale = true; + const before = requests; + assert.throws(() => reader.read(selection), /namespace anchor/); + assert.equal(requests, before, 'ownership refusal precedes HTTP'); + stale = false; + assert.deepEqual(reader.read(selection), getConvexCheckoutState({ ...selection, lease: active })); + assert.equal(keys, 2, 'read failure discards credential'); + const beforeRelease = requests; + try { + writeBackendLease(leasePath, { ...active, state: 'released' }); + assert.throws(() => reader.read(selection), /active/); + assert.equal(requests, beforeRelease, 'released lease refuses access while container still runs'); + } finally { writeBackendLease(leasePath, active); } + assert.deepEqual(reader.read(selection), getConvexCheckoutState({ ...selection, lease: active })); + assert.equal(keys, 3, 'lease failure discards credential'); + checks.push(`${phase}: full snapshots match; new writes visible; stale ownership refused; error discards credential`); + checks.push(`${phase}: released lease refused before HTTP while its container remains alive`); + } finally { reader.close(); } + assert.throws(() => reader.read(selection), /closed/); + checks.push(`${phase}: closed reader refuses reuse before the next lifecycle operation`); + } + receipt.result = 'passed'; + } catch (error) { Object.assign(receipt, { error: String(error) }); throw error; } + finally { + try { + releaseBackendLease(leasePath, lease.ownershipToken); + receipt.cleanup = readBackendLease(leasePath).state; + assert.equal(receipt.cleanup, 'released'); + } catch (error) { + Object.assign(receipt, { result: 'failed', cleanup: String(error) }); + // A cleanup failure must fail the test even after an earlier error. + // eslint-disable-next-line no-unsafe-finally + throw error; + } finally { writeFileSync(`${out}/receipt.json`, JSON.stringify(receipt, null, 2)); } + } +}); diff --git a/tools/stack-bench/tests/cost-ledger.test.ts b/tools/stack-bench/tests/cost-ledger.test.ts new file mode 100644 index 00000000000..8397b987d10 --- /dev/null +++ b/tools/stack-bench/tests/cost-ledger.test.ts @@ -0,0 +1,217 @@ +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createBackendLease, publicBackendLease, writeBackendLease } from '../src/runtime/backend-lease.js'; +import { retainedRunCost } from '../src/evidence/retained-run-cost.js'; +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { durableCostLedger, runCostEvidence, sumCostEvidence } from '../src/evidence/cost-proof.js'; +import { recordedExecutionSpend } from '../src/evidence/run-checkpoints.js'; +import { executionSpend, formatCostEvidence } from '../src/campaigns/campaign-report.js'; +import { validateAgentCostReceipt } from '../src/agents/agent-result-contract.js'; +import { noUnpriced } from '../container/credential-broker-accounting.js'; +import { spend } from '../dashboard/public/format.js'; +import type { CostRun } from '../src/evidence/cost-proof.js'; + +const receipt = (costUsd: number) => ({ invocation: 1, receipt: { + complete: true, reconciled: true, error: null, costUsd, exact: true, +} }); + +test('cost ledger uses stored receipts and does not reprice usage', () => { + const run: CostRun = { id: 'run-1', totals: { costUsd: 3.5, costComplete: true }, + pricing: { id: 'recorded-pricing' }, levels: [{ level: 1, buildSessions: [{ + costUsd: 2, costComplete: true, costReceipts: [receipt(2)], + usage: { input: 999_999_999, output: 999_999_999 }, + }], repairSessions: [{ costUsd: 1.5, costComplete: true, + costReceipts: [receipt(1.5)] }] }] }; + const ledger = durableCostLedger(run); + assert.equal(ledger.receiptCostUsd, 3.5); + assert.equal(ledger.differenceUsd, 0); + assert.equal(ledger.complete, true); + assert.equal(ledger.exact, true); +}); + +test('cost ledger stays complete and marks itself inexact when a receipt charged a ceiling', () => { + const estimated = { invocation: 1, receipt: { + complete: true, reconciled: true, error: null, costUsd: 2, exact: false } }; + const ledger = durableCostLedger({ id: 'run-estimated', totals: { costUsd: 2, costComplete: true }, + levels: [{ level: 1, buildSessions: [{ costUsd: 2, costComplete: true, + costReceipts: [estimated] }] }] }); + assert.equal(ledger.complete, true); + assert.equal(ledger.exact, false); + assert.deepEqual(ledger.rows.map(row => row.exact), [false]); +}); + +const brokerReceipt = (schemaVersion: 3 | 4, costUsd: number, { estimated = 0, unpriced = 0 } = {}) => ({ + schemaVersion, source: 'credential-broker', model: 'claude-sonnet-5', maxBudgetUsd: 50, costUsd, + cliCostUsd: costUsd, calculatedCostUsd: costUsd, usage: { input: 1, output: 1, cacheRead: 0, cacheWrite5m: 0, cacheWrite1h: 0 }, + pricingRates: { input: 2, output: 10, cacheWrite5m: 2.5, cacheWrite1h: 4, cacheRead: 0.2 }, + exact: estimated === 0 && unpriced === 0, estimatedRequests: estimated, + estimatedByReason: { 'no-usage': estimated, 'response-aborted': 0, 'upstream-error': 0 }, + ...(schemaVersion === 4 ? { unpricedRequests: unpriced, + unpricedByReason: { ...noUnpriced(), 'server-tool': unpriced } } : {}), + complete: true, reconciled: true, error: null }); +const brokerRun = (...receipts: Array>): CostRun => { + const total = Number(receipts.reduce((sum, item) => sum + item.costUsd, 0).toFixed(6)); + return { totals: { costUsd: total, costComplete: true }, levels: [{ level: 1, buildSessions: receipts.map(item => ({ + costUsd: item.costUsd, costComplete: true, costReceipts: [{ invocation: 1, receipt: item }] })) }] }; +}; + +test('schema 3 receipts keep their exact and upper-bound meaning', () => { + for (const receipt of [brokerReceipt(3, 2), brokerReceipt(3, 2, { estimated: 1 })]) { + validateAgentCostReceipt(receipt, 'claude-sonnet-5', 'receipt'); + const ledger = durableCostLedger(brokerRun(receipt)); + assert.equal(ledger.complete, true); + assert.equal(ledger.priced, true); + } + assert.deepEqual(runCostEvidence(brokerRun(brokerReceipt(3, 2))), { status: 'exact', costUsd: 2 }); + assert.deepEqual(runCostEvidence(brokerRun(brokerReceipt(3, 2, { estimated: 1 }))), { status: 'upper-bound', costUsd: 2 }); + assert.throws(() => validateAgentCostReceipt({ ...brokerReceipt(3, 2), unpricedRequests: 0 }, 'claude-sonnet-5', 'receipt')); +}); + +test('unpriced receipts make spend unknown with the priced part as a lower bound', () => { + const unpriced = brokerReceipt(4, 2, { unpriced: 1 }); + validateAgentCostReceipt(unpriced, 'claude-sonnet-5', 'receipt'); + assert.throws(() => validateAgentCostReceipt({ ...unpriced, exact: true }, 'claude-sonnet-5', 'receipt'), /unpriced/); + assert.throws(() => validateAgentCostReceipt({ ...unpriced, unpricedRequests: 2 }, 'claude-sonnet-5', 'receipt'), /unpriced/); + const ledger = durableCostLedger(brokerRun(unpriced, brokerReceipt(4, 1))); + assert.equal(ledger.complete, true, 'unpriced spend is still reconciled accounting'); + assert.equal(ledger.priced, false); + assert.equal(ledger.exact, false); + assert.deepEqual(ledger.rows.map(row => row.priced), [false, true]); + const cost = runCostEvidence(brokerRun(unpriced, brokerReceipt(4, 1))); + assert.deepEqual(cost, { status: 'unknown', costUsd: null, lowerBoundUsd: 3 }); + assert.equal(formatCostEvidence(cost), '≥ $3'); + assert.match(spend({ ...cost, knownCostUsd: 3 }), /≥\$3\.00/); + // A ceiling mixed into the priced figure makes it neither bound. + assert.deepEqual(runCostEvidence(brokerRun(unpriced, brokerReceipt(4, 1, { estimated: 1 }))), + { status: 'unknown', costUsd: null }); + assert.deepEqual(runCostEvidence(brokerRun(brokerReceipt(4, 2, { estimated: 1, unpriced: 1 }))), + { status: 'unknown', costUsd: null }); + assert.deepEqual(sumCostEvidence([cost, { status: 'exact', costUsd: 1 }]), + { status: 'unknown', costUsd: null, lowerBoundUsd: 4 }); + assert.deepEqual(sumCostEvidence([cost, { status: 'upper-bound', costUsd: 1 }]), { status: 'unknown', costUsd: null }); + const total = executionSpend([{ cost }, { cost: { status: 'exact', costUsd: 1 } }]); + assert.equal(total.status, 'unknown'); + assert.equal(total.lowerBoundUsd, 4); + assert.equal(total.knownCostUsd, 4); + assert.equal(total.unknownExecutions, 1); +}); + +test('cost ledger rejects incomplete receipt proof', () => { + const run: CostRun = { id: 'run-2', totals: { costUsd: 2, costComplete: false }, + levels: [{ level: 1, buildSessions: [{ + costUsd: 2, costComplete: false, costReceipts: [{ invocation: 1, receipt: { + complete: false, reconciled: false, error: 'provider result was incomplete', costUsd: 2, + } }], + }] }] }; + const ledger = durableCostLedger(run); + assert.equal(ledger.receiptCostUsd, 2); + assert.equal(ledger.complete, false); +}); + +test('cost ledger accepts a complete non-billable session without receipts', () => { + const run: CostRun = { id: 'run-3', totals: { costUsd: 0, costComplete: true }, + levels: [{ level: 1, buildSessions: [{ + costUsd: 0, costComplete: true, costReceipts: [], + }] }] }; + const ledger = durableCostLedger(run); + assert.equal(ledger.receiptCostUsd, 0); + assert.equal(ledger.complete, true); +}); + +test('cost ledger preserves every same-depth feature build session', () => { + const run: CostRun = { id: 'run-feature', totals: { costUsd: 3, costComplete: true }, + levels: [{ level: 1, buildSessions: [ + { costUsd: 1, costComplete: true, costReceipts: [receipt(1)] }, + { costUsd: 2, costComplete: true, costReceipts: [receipt(2)] }, + ] }] }; + const ledger = durableCostLedger(run); + assert.equal(ledger.rows.length, 2); + assert.equal(ledger.receiptCostUsd, 3); + assert.equal(ledger.complete, true); +}); + +test('cost evidence rejects negative and malformed money values', () => { + assert.throws(() => durableCostLedger({ levels: [{ level: 1, buildSessions: [{ + costUsd: -1, + }] }] }), /costUsd/); + assert.throws(() => durableCostLedger({ totals: { costUsd: -1 } }), /totals\.costUsd/); + assert.throws(() => durableCostLedger({ levels: [{ level: 1, buildSessions: [{ + costUsd: 1, costReceipts: [{ receipt: { complete: true, reconciled: true, error: null } }], + }] }] }), /receipt\[0\]\.costUsd/); +}); + +test('missing totals never become an exact zero, even with complete set', () => { + for (const totals of [undefined, {}, { costComplete: true }, { costComplete: true, costUsd: null }]) { + assert.deepEqual(runCostEvidence({ totals }), { status: 'unknown', costUsd: null }); + } + assert.deepEqual(runCostEvidence({ totals: { costComplete: true, costUsd: 0 } }), + { status: 'exact', costUsd: 0 }); +}); + +test('resumed execution spend excludes inherited costs and retains current bounds', () => { + const resumed = { progressionResume: { inheritedLevels: [1] }, + totals: { costUsd: 5, currentExecutionCostUsd: 3, currentExecutionCostComplete: true, costComplete: true }, levels: [ + { level: 1, buildSessions: [{ costUsd: 2, costComplete: true, costReceipts: [receipt(2)] }] }, + { level: 2, buildSessions: [{ costUsd: 3, costComplete: true, + costReceipts: [{ receipt: { ...receipt(3).receipt, exact: false } }] }] }, + ] }; + assert.deepEqual(runCostEvidence(resumed, 'execution'), { status: 'upper-bound', costUsd: 3 }); + // A session killed mid-flight leaves its spend out of the rows that still reconcile. + resumed.totals.currentExecutionCostComplete = false; + assert.equal(durableCostLedger(resumed, 'execution').complete, false); + assert.deepEqual(runCostEvidence(resumed, 'execution'), { status: 'unknown', costUsd: null }); + resumed.totals.currentExecutionCostComplete = true; + delete (resumed.totals as Partial).currentExecutionCostUsd; + assert.deepEqual(runCostEvidence(resumed, 'execution'), { status: 'unknown', costUsd: null }); +}); + + +test('interrupted totals cannot hide a later checkpoint or claim complete spend', () => { + const run: CostRun = { totals: { costUsd: 2, costComplete: true }, + levels: [{ level: 2, buildSessions: [{ costUsd: 2, costComplete: true, costReceipts: [receipt(2)] }] }], + checkpoints: [{ executionCost: { status: 'exact', costUsd: 12 } }] }; + const cost = runCostEvidence(run, 'execution'); + assert.equal(cost.status, 'unknown'); + const recorded = recordedExecutionSpend(run); + assert.deepEqual(recorded, { status: 'exact', costUsd: 12 }); + const total = executionSpend([{ cost, recorded }]); + assert.equal(total.status, 'unknown'); + assert.equal(total.knownCostUsd, 12); + assert.match(spend(total), /\$12.00 recorded/); + run.checkpoints = []; + run.progressionStatus = { phase: 'active' }; + assert.equal(runCostEvidence(run).status, 'unknown'); +}); + + +test('retained broker accounting is bound to the run lease and keeps interrupted requests incomplete', () => { + const root = mkdtempSync(join(tmpdir(), 'retained-cost-')); + const lease = createBackendLease({ runId: 'retained', backend: 'spacetime', track: 'ecommerce', + runIndex: 0, module: 'app', serverUri: 'http://127.0.0.1:3000', dataDir: join(root, 'data') }); + const directory = join(root, lease.runId); + mkdirSync(join(directory, 'stack-bench-credential-broker-one'), { recursive: true }); + writeBackendLease(join(directory, 'backend-lease.json'), lease); + const ledger = { schemaVersion: 4, model: 'claude-sonnet-5', maxBudgetUsd: 50, + acceptedRequests: 2, billableRequests: 2, completedBillableRequests: 2, + estimatedBillableRequests: 0, estimatedByReason: { 'no-usage': 0, 'response-aborted': 0, 'upstream-error': 0 }, + spentUsd: 12, reservedUsd: 0, usage: { input: 0, output: 0, cacheRead: 0, cacheWrite5m: 0, cacheWrite1h: 0 }, + complete: true, updatedAt: new Date().toISOString() }; + const file = join(directory, 'stack-bench-credential-broker-one', 'spend-ledger.json'); + const run = { id: lease.runId, backend: 'spacetime', model: ledger.model, backendLease: publicBackendLease(lease), levels: [] }; + try { + writeFileSync(file, JSON.stringify(ledger)); + assert.deepEqual(retainedRunCost(run, root)?.cost, { status: 'exact', costUsd: 12 }); + writeFileSync(file, JSON.stringify({ ...ledger, complete: false, reservedUsd: 3, completedBillableRequests: 1 })); + const partial = retainedRunCost(run, root); + assert.equal(partial?.cost.status, 'unknown'); + assert.equal(partial?.recorded.costUsd, 12); + assert.equal(retainedRunCost({ ...run, id: '../retained' }, root), null); + writeFileSync(file, JSON.stringify({ ...ledger, schemaVersion: 5, unpricedBillableRequests: 1, + unpricedByReason: { ...noUnpriced(), 'server-tool': 1 } })); + assert.deepEqual(retainedRunCost(run, root)?.cost, { status: 'unknown', costUsd: null, lowerBoundUsd: 12 }); + assert.equal(retainedRunCost({ ...run, backendLease: { ownership: { markerSha256: 'wrong' } } }, root), null); + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/tools/stack-bench/tests/crash-diagnostic.test.ts b/tools/stack-bench/tests/crash-diagnostic.test.ts new file mode 100644 index 00000000000..893ba689de0 --- /dev/null +++ b/tools/stack-bench/tests/crash-diagnostic.test.ts @@ -0,0 +1,603 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { openCrashReducerConnection } from '../src/stacks/spacetime-crash-transport.js'; +import { executeAction } from '../src/actions/action-contract.js'; +import { ACTION_REGISTRY } from '../src/actions/action-catalog.js'; +import { checkoutStateSchema, type CheckoutState } from '../src/stacks/checkout-state.js'; +import { createCheckEvidence } from '../src/evidence/check-evidence.js'; + +test('baseline checkout retains completion evidence and cannot confirm queued or lost responses', async () => { + type Receipt = { outcome: string; protocol: string; actor: string; action: string; status: number | null }; + const confirm = async (request: { url: string; method: string; responseContract?: string }, + fetch: () => Promise, writeUrl = request.url) => { + let unsettled = false; + const result = await executeAction(ACTION_REGISTRY, 'confirmCheckout', { + do: 'confirmCheckout', actor: 'buyer', + }, { capabilities: { + actors: { get: () => ({ name: 'buyer', page: { evaluate: async () => [] }, context: { cookies: async () => [] }, + writes: [{ url: writeUrl, headers: { authorization: 'Bearer private-token' } }] }) }, + 'database-read': { markCheckoutUnsettled: () => { unsettled = true; } }, + 'named-actions': { now: Date.now, resolve: () => ({ id: 'checkout' }), request: () => request, fetch }, + } }); + assert(!JSON.stringify(result).includes('private')); + return { result, unsettled, receipt: result.observation as Receipt }; + }; + for (const status of [200, 201, 204, 202, 409, 500, null]) { + const { result, unsettled, receipt } = await confirm({ url: 'http://app/checkout', method: 'POST' }, async () => { + if (status === null) throw new Error('response lost'); + return { ok: status < 300, status, text: async () => 'private response body' }; + }, 'http://app/session'); + const confirmed = status !== null && [200, 201, 204].includes(status); + assert.equal(result.status, confirmed ? 'passed' : status === null || status === 202 ? 'inconclusive' : 'failed'); + assert.equal(unsettled, !confirmed); + assert.equal(receipt.outcome, confirmed ? 'committed' : 'not-confirmed'); + assert.equal(receipt.protocol, 'http'); + assert.equal(receipt.actor, 'buyer'); + assert.equal(receipt.action, 'checkout'); + } + // A native mutation is confirmed by its envelope, and the receipt keeps the actual HTTP status. + for (const mode of ['committed', 'refused', 'function-error', 'malformed', 'lost-body']) { + const { result, unsettled, receipt } = await confirm( + { url: 'http://app/api/mutation', method: 'POST', responseContract: 'convex-mutation' }, + async () => ({ ok: true, status: 200, text: async () => { + if (mode === 'lost-body') throw new Error('private body unavailable'); + return mode === 'committed' ? '{"status":"success","value":null}' + : mode === 'refused' ? '{"status":"error","errorMessage":"private refusal","errorData":null}' + : mode === 'function-error' ? '{"status":"error","errorMessage":"private failure"}' : '{}'; + } })); + assert.equal(result.status, mode === 'committed' ? 'passed' + : mode === 'lost-body' ? 'inconclusive' : 'failed', mode); + assert.equal(receipt.status, mode === 'lost-body' ? null : 200); + assert.equal(receipt.protocol, 'convex-mutation'); + assert.equal(receipt.outcome, mode === 'committed' ? 'committed' : 'not-confirmed'); + assert.equal(unsettled, mode !== 'committed'); + } +}); + +class Socket extends EventTarget { + protocol = 'v1.json.spacetimedb'; + readyState: number = WebSocket.OPEN; + sent: Array<{ CallReducer: { reducer: string; request_id: number; args: string; flags: number } }> = []; + send(value: string) { this.sent.push(JSON.parse(value)); } + close() { this.readyState = WebSocket.CLOSED; } + message(value: unknown) { + const data = JSON.stringify(value).replace(/"(22451491060779870000000000000000000000[12])"/g, '$1'); + this.dispatchEvent(Object.assign(new Event('message'), { data })); + } + reply(requestId: number, status: object, caller = `0x${'1'.repeat(64)}`, + connection = '224514910607798700000000000000000000001') { + this.message({ TransactionUpdate: { caller_identity: { __identity__: caller }, + caller_connection_id: { __connection_id__: connection }, timing: 1.5, + reducer_call: { request_id: requestId, reducer_name: 'checkout' }, status } }); + } +} + +test('baseline SpacetimeDB checkout records native completion or refusal and closes its connection', async t => { + const websocket = globalThis.WebSocket; + t.after(() => { globalThis.WebSocket = websocket; }); + for (const committed of [true, false]) { + const sockets: Socket[] = []; + globalThis.WebSocket = class extends Socket { + static OPEN = websocket.OPEN; + static CLOSED = websocket.CLOSED; + constructor(address: string, protocol: string) { + super(); sockets.push(this); + assert.equal(new URL(address).searchParams.get('confirmed'), 'true'); + assert.equal(protocol, this.protocol); + queueMicrotask(() => this.message({ IdentityToken: { identity: { __identity__: `0x${'1'.repeat(64)}` }, + connection_id: { __connection_id__: '224514910607798700000000000000000000001' } } })); + } + override send(value: string) { + super.send(value); + queueMicrotask(() => this.reply(1, committed ? { Committed: {} } : { Failed: 'refused' })); + } + } as unknown as typeof WebSocket; + const result = await executeAction(ACTION_REGISTRY, 'confirmCheckout', { + do: 'confirmCheckout', actor: 'buyer', + }, { capabilities: { + actors: { get: () => ({ name: 'buyer', page: { evaluate: async () => [] }, context: { cookies: async () => [] }, writes: [{ url: 'http://app/session', headers: { authorization: 'Bearer private-token' } }] }) }, + 'database-read': { markCheckoutUnsettled: () => assert.fail('native completion or refusal must settle') }, + 'named-actions': { spacetime: { uri: 'http://127.0.0.1:3000', mod: 'shop' }, now: Date.now, + resolve: () => ({ id: 'checkout', reducer: 'checkout' }), + request: () => ({ url: 'http://app/call/checkout', body: '[]' }), + fetch: async () => assert.fail('baseline must not use the unconfirmed HTTP reducer path') }, + } }); + assert.equal(result.status, committed ? 'passed' : 'failed'); + assert.equal((result.observation as { protocol: string }).protocol, 'websocket-v1-confirmed'); + assert.equal(sockets.length, 1); + assert.equal(sockets[0]!.sent.length, 1); + assert.equal(sockets[0]!.readyState, websocket.CLOSED); + } +}); + +test('native crash transport requires a correlated confirmed result and drains unknowns without replay', async () => { + const socket = new Socket(); + const controller = new AbortController(); + const connection = await openCrashReducerConnection({ uri: 'http://127.0.0.1:3000', mod: 'shop' }, + 'private-token', controller.signal, (address, protocol) => { + const url = new URL(address); + assert.equal(url.protocol, 'ws:'); + assert.equal(url.searchParams.get('confirmed'), 'true'); + assert.equal(url.searchParams.get('token'), 'private-token'); + assert.equal(protocol, socket.protocol); + queueMicrotask(() => socket.message({ IdentityToken: { identity: { __identity__: `0x${'1'.repeat(64)}` }, + connection_id: { __connection_id__: '224514910607798700000000000000000000001' } } })); + return socket as unknown as WebSocket; + }); + const first = connection.call('checkout', '[]', controller.signal); + const second = connection.call('checkout', '[]', controller.signal); + assert.deepEqual(socket.sent[0], { CallReducer: { reducer: 'checkout', args: '[]', request_id: 1, flags: 0 } }); + socket.reply(1, { Committed: {} }, 'another-user'); + socket.reply(1, { Committed: {} }, `0x${'1'.repeat(64)}`, '224514910607798700000000000000000000002'); + socket.reply(2, { Committed: {} }); + controller.abort(); + assert.deepEqual(await first, { outcome: 'unknown' }); + assert.deepEqual(await second, { outcome: 'committed' }); + assert.equal(socket.sent.length, 2); + assert.equal(socket.readyState, WebSocket.CLOSED); +}); + +test('crash setup rejects missing cart scope, changed scope and old native reservations before issuing writes', async () => { + for (const mode of ['missing-cart', 'changed-scope', 'old-reservation']) { + const before: CheckoutState = { accountId: 'a', itemId: 'i', priceMinor: 100, + stock: [{ warehouseId: 'w', quantity: 10 }], cart: [], reservations: [], orders: [], payments: [], + orphanOrderLines: 0, orphanAllocations: 0 }; + const prepared = structuredClone(before); + prepared.cart.push({ itemId: 'i', quantity: 1 }); + if (mode === 'old-reservation') { + prepared.stock[0]!.quantity--; + prepared.reservations.push({ itemId: 'i', warehouseId: 'w', quantity: 1 }); + } + const wrap = (state: CheckoutState, ready: boolean) => ({ state, schemaSha256: { schema: 'same' }, + account: 'a', item: 'i', scope: 'orders' as const, recordedAtMs: Date.now() - 31_000, + storage: { kind: 'order-data' as const, cart: mode !== 'missing-cart', warehouses: !(ready && mode === 'changed-scope') } }); + let closed = false; + const result = await executeAction(ACTION_REGISTRY, 'crashCheckout', { + do: 'crashCheckout', actor: 'buyer', before: 'before', prepared: 'prepared', quantity: 1, + requests: 1, offsetMs: 0, target: 'database', + }, { capabilities: { + actors: { get: () => ({ name: 'buyer', page: { evaluate: async () => [] }, context: { cookies: async () => [] }, writes: [{ url: 'http://app/session', headers: { authorization: 'Bearer private-token' } }] }) }, + 'database-read': { checkoutSnapshots: new Map([['before', wrap(before, false)], ['prepared', wrap(prepared, true)]]) }, + 'named-actions': { resolve: () => ({ id: 'checkout' }), request: () => ({ url: 'http://app/checkout' }), + fetch: async () => assert.fail('invalid crash setup must not submit checkout') }, + 'browser-observation': { recorded: new Map() }, + 'process-crash': { prepare: async () => { + assert.equal(mode, 'old-reservation', 'invalid scopes must fail before process preparation'); + return { spacetime: null, close: async () => { closed = true; }, + crash: async () => assert.fail('expired preparation must not inject a fault') }; + } }, + } }); + assert.equal(result.status, mode === 'old-reservation' ? 'inconclusive' : 'harness_failure', mode); + assert.equal(closed, mode === 'old-reservation'); + } +}); + +test('crash action retains partial fault evidence and distinguishes recovered state from acknowledged loss', async () => { + for (const record of [false, true]) for (const mode of ['absent', 'committed', 'orders-only', 'orders-scoped', 'orders-scoped-wrong-total', 'orders-scoped-changed-after', 'orders-scoped-lost-acknowledged', 'empty-stock', 'lost-acknowledged', 'partial', 'queued', 'queued-committed', 'queued-drained', 'queued-recovery-error', 'fault-error', 'cancelled-recovery', 'cancelled-read', 'recovery-error', 'disconnected-database', 'disconnected-application', 'drained-application', 'undrained-application', 'drained-recovery-error', 'disconnected-recovery-error', 'cancelled-disconnected-recovery-error']) { + const recorded = new Map(record + ? [['captured-crash', { verdicts: { atomicity: [], durability: [] } }]] : []); + const cancellation = new AbortController(); + const timers: number[] = []; + let recoveryStopped = false; + let unsettled = false; + const before: CheckoutState = { accountId: 'a', itemId: 'i', priceMinor: 100, + stock: [{ warehouseId: 'w', quantity: 10 }], cart: [], reservations: [], orders: [], payments: [], orphanOrderLines: 0 }; + const prepared = structuredClone(before); + prepared.cart.push({ itemId: 'i', quantity: 1 }); + const after = structuredClone(prepared); + if (mode === 'committed' || mode === 'partial' || mode === 'orders-only' || mode === 'orders-scoped' || mode === 'orders-scoped-wrong-total' || mode === 'orders-scoped-changed-after' || mode === 'empty-stock' || mode === 'queued-committed') { + after.cart = []; after.stock[0]!.quantity--; + after.orders.push({ id: 'o', accountId: 'a', status: 'pending', totalMinor: 100, + lines: [{ itemId: 'i', quantity: 1, priceMinor: 100, allocations: [{ warehouseId: 'w', quantity: 1 }] }] }); + if (mode === 'committed' || mode === 'empty-stock' || mode === 'queued-committed') after.payments.push({ id: 'p', orderId: 'o', amountMinor: 100, status: 'paid' }); + } + if (mode === 'empty-stock') after.stock = []; + if (mode.startsWith('orders-')) for (const state of [before, prepared, after]) { + state.orphanAllocations = 0; + if (mode.startsWith('orders-scoped')) { + state.stock = []; + for (const order of state.orders) for (const line of order.lines) line.allocations = []; + } + for (const order of state.orders) order.refundedMinor = 0; + } + if (mode === 'orders-scoped-wrong-total') after.orders[0]!.totalMinor++; + const wrap = (state: CheckoutState) => ({ state: checkoutStateSchema.parse(state), schemaSha256: { schema: 'same' }, account: 'a', item: 'i', + ...(mode.startsWith('orders-') ? { scope: 'orders' as const } : {}), + ...(mode.startsWith('orders-scoped') ? { storage: { kind: 'order-data' as const, cart: true, warehouses: mode === 'orders-scoped-changed-after' && state === after } } : {}), + recordedAtMs: Date.now() - (mode === 'orders-only' ? 90_000 : 0) }); + let release!: () => void; + const waiting = new Promise(resolve => { release = resolve; }); + const result = await executeAction(ACTION_REGISTRY, 'crashCheckout', { + do: 'crashCheckout', actor: 'buyer', before: 'before', prepared: 'prepared', quantity: 1, + ...(record ? { as: 'captured-crash', + ...(mode !== 'disconnected-database' ? { reuseCombinedFrom: 'database' } : {}) } : {}), + requests: 1, offsetMs: 0, target: mode === 'disconnected-database' ? 'database' : 'application', + }, { signal: cancellation.signal, onAbort: async () => {}, capabilities: { + actors: { get: () => ({ name: 'buyer', page: { evaluate: async () => [] }, context: { cookies: async () => [] }, writes: [{ url: 'http://app/session', headers: { authorization: 'Bearer private-token' } }] }) }, + 'database-read': { checkoutSnapshots: new Map([['before', wrap(before)], ['prepared', wrap(prepared)]]), + markCheckoutUnsettled: () => { unsettled = true; }, + getCheckoutState: () => { if (mode === 'cancelled-read') throw new Error('reader is not ready'); return wrap(after); } }, + 'named-actions': { now: Date.now, sleep: async () => { + if (mode === 'cancelled-read') { cancellation.abort('cancelled during read retry'); throw cancellation.signal.reason; } + }, resolve: () => ({ id: 'checkout' }), + request: () => ({ url: 'http://app/checkout' }), fetch: async () => { + await waiting; + if (mode.includes('disconnected-') || mode.endsWith('drained-application')) throw new Error('socket closed'); + if (mode === 'absent' || mode === 'partial') return { ok: false, status: 409, text: async () => '' }; + if (mode.startsWith('queued')) return { ok: true, status: 202, text: async () => '' }; + return { ok: true, status: 200, text: async () => '' }; + } }, + 'browser-observation': { recorded }, + 'process-crash': { combinedBoundary: false, prepare: async () => ({ spacetime: null, + close: async () => {}, + crash: async () => { + const now = Date.now(); + const receipt = { requestedAtMs: now, completedAtMs: now, clockOffsetBeforeMs: 0, clockOffsetAfterMs: 0, + signal: 'SIGKILL', processEvidence: `KILLED 42 123 ${now}\nQUIET\n` }; + if (mode === 'fault-error') throw Object.assign(new Error('partial injection'), { receipt }); + return receipt; + }, recover: async (signal: AbortSignal) => { + if (mode === 'cancelled-recovery' || mode === 'cancelled-disconnected-recovery-error') { + cancellation.abort('cancelled during recovery'); + assert(signal.aborted, 'recovery must receive action cancellation'); + // Model a worker waiting for an in-flight native call to finish. + await new Promise(resolve => setTimeout(resolve, 10)); + } + recoveryStopped = true; + release(); + if (mode.endsWith('recovery-error')) throw Object.assign(new Error('application did not recover'), { + code: 'generated_app_not_restartable', + ...(mode === 'drained-recovery-error' ? { databaseDrain: { settled: true, samples: [{ pending: 0 }] } } : {}), + }); + return mode.endsWith('drained-application') || mode === 'queued-drained' ? { settled: mode !== 'undrained-application', + startedAtMs: Date.now(), completedAtMs: Date.now(), backend: 'postgres', database: 'app', + samples: [{ atMs: Date.now(), pending: mode === 'undrained-application' ? 1 : 0 }] } : null; + } }) }, + } }, { setTimer: (callback, ms) => { + timers.push(ms); + return setTimeout(callback, ms === 5000 ? 1 : ms === 150_000 ? 100 : ms); + } }); + assert(recoveryStopped, 'outer action must not return before recovery has stopped'); + if (mode.startsWith('cancelled-')) { + assert(timers.includes(150_000)); + assert.equal(result.code, 'cancelled'); + } + assert.equal(unsettled, mode.startsWith('queued') || mode.includes('disconnected-') || mode === 'undrained-application'); + const expected = (mode.startsWith('queued') && mode !== 'queued-recovery-error') || mode.startsWith('cancelled-') || ['disconnected-database', 'disconnected-application', 'undrained-application'].includes(mode) ? 'inconclusive' : ['fault-error', 'orders-scoped-changed-after'].includes(mode) ? 'harness_failure' + : ['partial', 'lost-acknowledged', 'empty-stock', 'orders-scoped-wrong-total', 'orders-scoped-lost-acknowledged'].includes(mode) || mode.endsWith('recovery-error') ? 'failed' : 'passed'; + const capturedFailure = record && ['partial', 'lost-acknowledged', 'empty-stock', 'orders-scoped-wrong-total', 'orders-scoped-lost-acknowledged'].includes(mode); + assert.equal(result.status, capturedFailure ? 'passed' : expected, mode); + // An unfinished checkout is how the app is built; running the suite again would only repeat it. + if (result.status === 'inconclusive' && !mode.startsWith('cancelled-')) assert.equal(result.retryable, false, mode); + if (record && result.status === 'passed') { + const results = []; + for (const verdict of ['atomicity', 'durability']) { + results.push(await executeAction(ACTION_REGISTRY, 'expectCrashCheckout', { + do: 'expectCrashCheckout', from: 'captured-crash', verdict, + }, { capabilities: { 'browser-observation': { recorded } } })); + } + assert.equal(results.some(row => row.status === 'failed'), expected === 'failed', mode); + assert(results.every(row => ['passed', 'failed'].includes(row.status)), mode); + assert(results.every(row => row.observation === recorded.get('captured-crash'))); + } else assert.equal(recorded.get('captured-crash'), undefined, 'failed capture must clear the previous verdict'); + if (mode === 'empty-stock') { + assert.equal(result.status, record ? 'passed' : 'failed'); + assert.deepEqual((result.observation as { after: { state: CheckoutState } }).after.state.stock, []); + } + if (mode.startsWith('queued')) { + const evidence = result.observation as { confirmed: boolean; outcomes: Array<{ outcome: string; status: number }> }; + assert.equal(evidence.confirmed, false); + assert.deepEqual(evidence.outcomes.map(row => [row.outcome, row.status]), [['not-confirmed', 202]]); + } + if (mode === 'disconnected-recovery-error' || mode === 'queued-recovery-error') assert.equal(result.code, 'application_failure'); + if (mode === 'drained-recovery-error') assert.equal((result.observation as { databaseDrain: { settled: boolean } }).databaseDrain.settled, true); + assert(result.observation, mode); + assert(!JSON.stringify(result.observation).includes('private-token')); + assert.doesNotThrow(() => createCheckEvidence({ status: result.status, code: result.code, phase: 'assertion', + observation: result.observation, actions: [{ actor: 'buyer', evidence: result }], + startedAtMs: result.timing.startedAtMs, completedAtMs: result.timing.completedAtMs })); + } +}); + + +test('combined native crash reconciles lost responses with stored state and rejects a non-native checkout', async () => { + for (const multi of [false, true]) for (const mode of ['lost-unchanged', 'lost-partial', 'lost-complete', 'committed', 'acknowledged-lost', 'wrong-route']) { + const before: CheckoutState = { accountId: 'a', itemId: 'i', priceMinor: 100, + stock: [{ warehouseId: 'w', quantity: 10 }], cart: [], reservations: [], orders: [], payments: [], orphanOrderLines: 0 }; + if (multi) { before.stock = []; before.orphanAllocations = 0; } + const prepared = structuredClone(before); + prepared.cart.push({ itemId: 'i', quantity: 1 }); + if (multi) prepared.cart.push({ itemId: 'j', quantity: 1 }); + const after = structuredClone(prepared); + if (mode === 'lost-complete' || mode === 'committed' || multi && mode === 'lost-partial') { + after.cart = []; + after.orders.push({ id: 'order', accountId: 'a', status: 'pending', totalMinor: multi ? 350 : 100, + refundedMinor: 0, lines: [{ itemId: 'i', priceMinor: 100, quantity: 1, allocations: multi ? [] : [{ warehouseId: 'w', quantity: 1 }] }] }); + if (multi && mode !== 'lost-partial') after.orders[0]!.lines.push({ itemId: 'j', priceMinor: 250, quantity: 1, allocations: [] }); + if (!multi) { after.stock[0]!.quantity--; after.payments.push({ id: 'p', orderId: 'order', amountMinor: 100, status: 'paid' }); } + } + if (!multi && mode === 'lost-partial') after.stock[0]!.quantity--; + const wrap = (state: CheckoutState) => ({ state, schemaSha256: { schema: 'same' }, + ...(multi ? { scope: 'orders' as const, storage: { kind: 'order-data' as const, cart: true, warehouses: false }, + catalog: [{ name: 'First', itemId: 'i', priceMinor: 100 }, { name: 'Second', itemId: 'j', priceMinor: 250 }] } : {}), + account: 'a', item: 'i', recordedAtMs: Date.now() }); + let release!: () => void; + const stopped = new Promise(resolve => { release = resolve; }); + let closed = false, calls = 0, reads = 0; + const result = await executeAction(ACTION_REGISTRY, 'crashCheckout', { + do: 'crashCheckout', actor: 'buyer', before: 'before', prepared: 'prepared', + quantity: multi ? [{ item: 'First', quantity: 1 }, { item: 'Second', quantity: 1 }] : 1, + requests: 1, offsetMs: 0, target: 'database', + }, { capabilities: { + actors: { get: () => ({ name: 'buyer', page: { evaluate: async () => [] }, context: { cookies: async () => [] }, + writes: [{ url: 'http://app/api/mutation', headers: { authorization: 'Bearer private-token' } }] }) }, + 'database-read': { checkoutSnapshots: new Map([['before', wrap(before)], ['prepared', wrap(prepared)]]), + markCheckoutUnsettled: () => assert.fail('verified combined process kill leaves no original database writer'), + getCheckoutState: () => { reads++; return wrap(after); } }, + 'named-actions': { now: Date.now, resolve: () => ({ id: 'checkout' }), + request: () => ({ url: 'http://app/api/mutation', method: 'POST', + responseContract: mode === 'wrong-route' ? 'convex-action' : 'convex-mutation' }), + fetch: async () => { + calls++; await stopped; + if (mode.startsWith('lost-')) throw new Error('socket closed'); + return { ok: true, status: 200, text: async () => '{"status":"success","value":null}' }; + }, + }, + 'browser-observation': { recorded: new Map() }, + 'process-crash': { combinedBoundary: true, prepare: async () => ({ spacetime: null, combinedBoundary: true, + close: async () => { closed = true; }, + crash: async () => { + const now = Date.now(); + return { backend: 'convex', target: 'database', requestedAtMs: now, completedAtMs: now, + clockOffsetBeforeMs: 0, clockOffsetAfterMs: 0, signal: 'SIGKILL', processEvidence: `KILLED 42 123 ${now}\nQUIET\n` }; + }, + recover: async () => { release(); return null; }, + }) }, + } }); + assert.equal(result.status, mode === 'wrong-route' ? 'inconclusive' : ['lost-unchanged', 'lost-complete', 'committed'].includes(mode) ? 'passed' : 'failed', `${multi}:${mode}`); + if (multi && mode === 'lost-partial') { + const verdicts = (result.observation as { verdicts: { atomicity: { control: string }[]; durability: unknown[] } }).verdicts; + assert(verdicts.atomicity.some(row => row.control === 'checkout quantity for item j')); + assert.deepEqual(verdicts.durability, [], 'a missing sibling on an unacknowledged new order is an atomicity failure'); + } + assert.equal(calls, mode === 'wrong-route' ? 0 : 1); + assert.equal(reads, mode === 'wrong-route' ? 0 : 1); + assert(closed, 'prepared crash must always close'); + assert(!JSON.stringify(result).includes('private-token')); + } +}); + +test('crash verdict without a measured observation remains inconclusive', async () => { + for (const verdict of ['atomicity', 'durability']) { + const result = await executeAction(ACTION_REGISTRY, 'expectCrashCheckout', { + do: 'expectCrashCheckout', from: 'missing', verdict, + }, { capabilities: { 'browser-observation': { recorded: new Map() } } }); + assert.equal(result.status, 'inconclusive'); + } +}); + +test('a bounded checkout burst requests the fault before fast calls drain', async () => { + const before: CheckoutState = { accountId: 'a', itemId: 'i', priceMinor: 100, cart: [], + stock: [{ warehouseId: 'w', quantity: 10 }], reservations: [], orders: [], payments: [], orphanOrderLines: 0 }; + const prepared = structuredClone(before); + prepared.cart.push({ itemId: 'i', quantity: 1 }); + const wrap = (state: CheckoutState) => ({ state: checkoutStateSchema.parse(state), schemaSha256: { schema: 'same' }, + account: 'a', item: 'i', recordedAtMs: Date.now() }); + const base = Date.now(); + let clock = base, calls = 0, callsAtFault = 0; + const result = await executeAction(ACTION_REGISTRY, 'crashCheckout', { + do: 'crashCheckout', actor: 'buyer', before: 'before', prepared: 'prepared', quantity: 1, + requests: 16, offsetMs: 0, target: 'database', + }, { capabilities: { + actors: { get: () => ({ name: 'buyer', page: { evaluate: async () => [] }, + context: { cookies: async () => [] }, writes: [{ url: 'http://app/session', headers: { authorization: 'Bearer private-token' } }] }) }, + 'named-actions': { now: () => clock, resolve: () => ({ id: 'checkout' }), + request: () => ({ url: 'http://app/checkout', responseContract: 'http' }), + fetch: async () => { + const index = ++calls; + return { status: 409, text: async () => { clock = base + (index === 16 ? 20 : index); return 'refused'; } }; + } }, + 'database-read': { checkoutSnapshots: new Map([['before', wrap(before)], ['prepared', wrap(prepared)]]), + markCheckoutUnsettled: () => {}, getCheckoutState: async () => wrap(prepared) }, + 'browser-observation': { recorded: new Map() }, + 'process-crash': { prepare: async () => ({ spacetime: null, close: async () => {}, + crash: async () => { + callsAtFault = calls; + const killedAt = base + (calls === 1 ? 2 : 40); + return { backend: 'postgres', target: 'database', requestedAtMs: killedAt, completedAtMs: killedAt, + clockOffsetBeforeMs: 0, clockOffsetAfterMs: 0, signal: 'SIGKILL', + processEvidence: `KILLED 42 123 ${killedAt}\nQUIET\n` }; + }, recover: async () => null }) }, + } }); + assert.equal(calls, 16); + assert.equal(result.status, 'passed', `${result.code} ${JSON.stringify(result.finding)}`); + const evidence = result.observation as { outstandingAtFault: number; outcomes: unknown[] }; + assert.equal(evidence.outcomes.length, 16); + assert(evidence.outstandingAtFault > 0); + assert.equal(callsAtFault, 1); +}); + +test('a recovered replacement cannot stand in for a checkout acknowledged before the cut', async () => { + const prior = { id: 'prior', accountId: 'a', status: 'pending', totalMinor: 100, + lines: [{ itemId: 'i', quantity: 1, priceMinor: 100, allocations: [{ warehouseId: 'w', quantity: 1 }] }] }; + const payment = { id: 'prior-payment', orderId: 'prior', amountMinor: 100, status: 'paid' }; + for (const mode of ['masked', 'idempotent', 'clock-drift', 'no-precut-ack', 'lost-prior', 'partial', 'direct', 'application-ack-after-service-cut'] as const) { + const before: CheckoutState = { accountId: 'a', itemId: 'i', priceMinor: 100, cart: [], + stock: [{ warehouseId: 'w', quantity: 10 }], reservations: [], orders: [prior], payments: [payment], + orphanOrderLines: 0 }; + const prepared = structuredClone(before); + prepared.cart.push({ itemId: 'i', quantity: 1 }); + const after = structuredClone(before); + after.stock[0]!.quantity = mode === 'partial' ? 8 : 9; + if (mode === 'lost-prior') { after.orders = []; after.payments = []; } + after.orders.push({ ...prior, id: 'replacement' }); + after.payments.push({ ...payment, id: 'replacement-payment', orderId: 'replacement' }); + const wrap = (state: CheckoutState) => ({ state: checkoutStateSchema.parse(state), + schemaSha256: { schema: 'same' }, account: 'a', item: 'i', recordedAtMs: Date.now() }); + const base = Date.now(); + let clock = base, calls = 0; + let firstDone!: () => void, release!: () => void; + const first = new Promise(resolve => { firstDone = resolve; }); + const resumed = new Promise(resolve => { release = resolve; }); + const recorded = new Map(); + const result = await executeAction(ACTION_REGISTRY, 'crashCheckout', { + do: 'crashCheckout', actor: 'buyer', before: 'before', prepared: 'prepared', quantity: 1, + requests: 16, offsetMs: 0, target: mode === 'application-ack-after-service-cut' ? 'application' : 'database', ...(mode === 'direct' ? {} : { as: 'captured' }), + }, { capabilities: { + actors: { get: () => ({ name: 'buyer', page: { evaluate: async () => [] }, + context: { cookies: async () => [] }, writes: [{ url: 'http://app/session', headers: { authorization: 'Bearer private-token' } }] }) }, + 'named-actions': { now: () => clock, resolve: () => ({ id: 'checkout' }), + request: () => ({ url: 'http://app/checkout', responseContract: 'http' }), + fetch: async () => { + const index = ++calls; + if (index !== 1 || mode === 'no-precut-ack') await resumed; + return { status: index <= (['masked', 'idempotent', 'clock-drift', 'direct', 'application-ack-after-service-cut'].includes(mode) ? 2 : 1) ? 200 : 409, + text: async () => { + if (index === 1 && mode !== 'no-precut-ack') { clock = base + (['clock-drift', 'application-ack-after-service-cut'].includes(mode) ? 8 : 1); firstDone(); } + if (index !== 1 || mode === 'no-precut-ack') clock = base + 20; + return index <= (['masked', 'idempotent', 'clock-drift', 'direct', 'application-ack-after-service-cut'].includes(mode) ? 2 : 1) ? '{}' : 'refused'; + } }; + } }, + 'database-read': { checkoutSnapshots: new Map([['before', wrap(before)], ['prepared', wrap(prepared)]]), + markCheckoutUnsettled: () => {}, getCheckoutState: async () => wrap(after) }, + 'browser-observation': { recorded }, + 'process-crash': { combinedBoundary: false, prepare: async () => ({ spacetime: null, + close: async () => {}, crash: async () => { + if (mode !== 'no-precut-ack') await first; + clock = base + 10; + return { backend: 'postgres', target: mode === 'application-ack-after-service-cut' ? 'application' : 'database', + requestedAtMs: base, completedAtMs: clock, + clockOffsetBeforeMs: mode === 'clock-drift' ? 4 : 0, clockOffsetAfterMs: 0, signal: 'SIGKILL', + // Freeze ends at +5, acknowledgement at +8, final kill at +10. + // Narrowing the overlap interval must not bless a replacement order. + ...(mode === 'application-ack-after-service-cut' ? { + applicationFreeze: { startedAtMs: base + 4, completedAtMs: base + 5, + processes: [{ pid: 41, startTicks: 122, threads: [41] }, { pid: 42, startTicks: 123, threads: [42] }] }, + } : {}), + processEvidence: `${mode === 'application-ack-after-service-cut' ? `KILLED 41 122 ${base + 5}\n` : ''}KILLED 42 123 ${clock}\nQUIET\n` }; + }, recover: async () => { release(); return null; } }) }, + } }); + assert.equal(calls, 16, mode); + const expected = mode === 'direct' ? 'inconclusive' : 'passed'; + assert.equal(result.status, expected, `${mode}: ${result.code} ${JSON.stringify(result.finding)}`); + if (mode === 'direct') continue; + const verdict = async (name: 'atomicity' | 'durability') => executeAction(ACTION_REGISTRY, 'expectCrashCheckout', { + do: 'expectCrashCheckout', from: 'captured', verdict: name, + }, { capabilities: { 'browser-observation': { recorded } } }); + const atomicity = await verdict('atomicity'), durability = await verdict('durability'); + assert.equal(atomicity.status, mode === 'partial' ? 'failed' : 'passed', mode); + assert.equal(durability.status, ['masked', 'idempotent', 'clock-drift', 'partial', 'application-ack-after-service-cut'].includes(mode) ? 'inconclusive' + : mode === 'lost-prior' ? 'failed' : 'passed', mode); + // An acknowledgement that cannot be linked is a timing outcome, so its suite may run once more. + if (durability.status === 'inconclusive') assert.equal(durability.retryable, true, mode); + } +}); + +test('combined crash reuse keeps an inconclusive durability verdict', async () => { + const prior = { receipt: { backend: 'convex', target: 'database' }, + verdicts: { atomicity: [], durability: [] }, + unmeasuredVerdicts: { durability: 'acknowledged checkout cannot be linked to recovered order' } }; + const recorded = new Map([['database', prior]]); + const result = await executeAction(ACTION_REGISTRY, 'crashCheckout', { + do: 'crashCheckout', actor: 'buyer', before: 'before', prepared: 'prepared', quantity: 1, + requests: 16, offsetMs: 0, target: 'application', as: 'application', reuseCombinedFrom: 'database', + }, { capabilities: { actors: {}, 'named-actions': {}, 'database-read': {}, 'browser-observation': { recorded }, + 'process-crash': { combinedBoundary: true, prepare: () => assert.fail('must reuse the measured database boundary') } } }); + assert.equal(result.status, 'passed'); + assert.equal(recorded.get('application'), prior); + const verdict = await executeAction(ACTION_REGISTRY, 'expectCrashCheckout', { + do: 'expectCrashCheckout', from: 'application', verdict: 'durability', + }, { capabilities: { 'browser-observation': { recorded } } }); + assert.equal(verdict.status, 'inconclusive'); + assert.equal(verdict.retryable, true); +}); + +test('a correlated native refusal after the cut cannot replace a pre-cut acknowledgement', async t => { + const websocket = globalThis.WebSocket; + t.after(() => { globalThis.WebSocket = websocket; }); + for (const secondCommitted of [false, true]) { + const base = Date.now(); + let clock = base; + let firstDone!: () => void, secondDone!: () => void, releaseReplies!: () => void; + const first = new Promise(resolve => { firstDone = resolve; }); + const second = new Promise(resolve => { secondDone = resolve; }); + const resumed = new Promise(resolve => { releaseReplies = resolve; }); + globalThis.WebSocket = class extends Socket { + static OPEN = websocket.OPEN; + static CLOSED = websocket.CLOSED; + constructor() { + super(); + queueMicrotask(() => this.message({ IdentityToken: { identity: { __identity__: `0x${'1'.repeat(64)}` }, + connection_id: { __connection_id__: '224514910607798700000000000000000000001' } } })); + } + override send(value: string) { + super.send(value); + const id = this.sent.at(-1)!.CallReducer.request_id; + if (id === 1) queueMicrotask(() => { clock = base + 1; this.reply(id, { Committed: {} }); firstDone(); }); + else if (id === 2 && secondCommitted) queueMicrotask(() => { + clock = base + 2; this.reply(id, { Committed: {} }); secondDone(); + }); + else void resumed.then(() => { clock = base + 20; this.reply(id, { Failed: 'empty cart' }); }); + } + } as unknown as typeof WebSocket; + const before: CheckoutState = { accountId: 'a', itemId: 'i', priceMinor: 100, cart: [], + stock: [{ warehouseId: 'w', quantity: 10 }], reservations: [], orders: [], payments: [], orphanOrderLines: 0 }; + const prepared = structuredClone(before); prepared.cart.push({ itemId: 'i', quantity: 1 }); + const after = structuredClone(before); after.stock[0]!.quantity--; + after.orders.push({ id: 'order', accountId: 'a', status: 'pending', totalMinor: 100, + lines: [{ itemId: 'i', priceMinor: 100, quantity: 1, allocations: [{ warehouseId: 'w', quantity: 1 }] }] }); + after.payments.push({ id: 'payment', orderId: 'order', amountMinor: 100, status: 'paid' }); + const wrap = (state: CheckoutState) => ({ state, schemaSha256: { schema: 'same' }, + account: 'a', item: 'i', recordedAtMs: Date.now() }); + const recorded = new Map(); + const result = await executeAction(ACTION_REGISTRY, 'crashCheckout', { + do: 'crashCheckout', actor: 'buyer', before: 'before', prepared: 'prepared', quantity: 1, + requests: 16, offsetMs: 0, target: 'database', as: 'native', + }, { capabilities: { + actors: { get: () => ({ name: 'buyer', page: { evaluate: async () => [] }, + context: { cookies: async () => [] }, writes: [{ url: 'http://app/session', headers: { authorization: 'Bearer private-token' } }] }) }, + 'named-actions': { now: () => clock, spacetime: { uri: 'http://127.0.0.1:3000', mod: 'shop' }, + resolve: () => ({ id: 'checkout', reducer: 'checkout' }), + request: () => ({ url: 'http://app/call/checkout', body: '[]' }), + fetch: async () => assert.fail('native checkout must use the confirmed socket') }, + 'database-read': { checkoutSnapshots: new Map([['before', wrap(before)], ['prepared', wrap(prepared)]]), + markCheckoutUnsettled: () => {}, getCheckoutState: async () => wrap(after) }, + 'browser-observation': { recorded }, + 'process-crash': { combinedBoundary: true, prepare: async () => ({ combinedBoundary: true, + spacetime: { uri: 'http://127.0.0.1:3000', mod: 'shop' }, close: async () => {}, + crash: async () => { await (secondCommitted ? second : first); clock = base + 10; return { backend: 'spacetime', target: 'database', + requestedAtMs: clock, completedAtMs: clock, clockOffsetBeforeMs: 0, clockOffsetAfterMs: 0, + signal: 'SIGKILL', processEvidence: `KILLED 42 123 ${clock}\nQUIET\n` }; }, + recover: async () => { releaseReplies(); return null; } }) }, + } }); + assert.equal(result.status, 'passed'); + const verdict = await executeAction(ACTION_REGISTRY, 'expectCrashCheckout', { + do: 'expectCrashCheckout', from: 'native', verdict: 'durability', + }, { capabilities: { 'browser-observation': { recorded } } }); + assert.equal(verdict.status, secondCommitted ? 'inconclusive' : 'passed'); + } +}); + +test('combined application boundary reuses only a measured database crash and retains its failures', async () => { + for (const prior of [undefined, { receipt: { backend: 'postgres', target: 'database' }, verdicts: { atomicity: [], durability: [] } }, + ...['spacetime', 'convex'].map(backend => ({ receipt: { backend, target: 'database' }, verdicts: { + atomicity: [{ control: 'partial order', observed: 0, expected: 1 }], durability: [] } }))]) { + const recorded = new Map([['database', prior]]); + const result = await executeAction(ACTION_REGISTRY, 'crashCheckout', { + do: 'crashCheckout', actor: 'buyer', before: 'before', prepared: 'prepared', quantity: 1, + requests: 16, offsetMs: 0, target: 'application', as: 'application', reuseCombinedFrom: 'database', + }, { capabilities: { actors: {}, 'named-actions': {}, 'database-read': {}, 'browser-observation': { recorded }, + 'process-crash': { combinedBoundary: true, prepare: () => assert.fail('must not crash the same process twice') } } }); + if (prior?.receipt.backend === 'spacetime' || prior?.receipt.backend === 'convex') { + assert.equal(result.status, 'passed'); + assert.equal(recorded.get('application'), prior); + const check = await executeAction(ACTION_REGISTRY, 'expectCrashCheckout', { + do: 'expectCrashCheckout', from: 'application', verdict: 'atomicity', + }, { capabilities: { 'browser-observation': { recorded } } }); + assert.equal(check.status, 'failed'); + } else { + assert.equal(result.status, 'inconclusive'); + assert.equal(recorded.get('application'), undefined); + } + } +}); diff --git a/tools/stack-bench/tests/credential-aliases.test.ts b/tools/stack-bench/tests/credential-aliases.test.ts new file mode 100644 index 00000000000..b5d042ff5ab --- /dev/null +++ b/tools/stack-bench/tests/credential-aliases.test.ts @@ -0,0 +1,109 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { gradeArgv } from '../commands/bench.js'; +import { contractLintArgv } from '../commands/run-suite.js'; +import { agentRequestArgv } from '../src/agents/agent-adapter-contract.js'; +import { AGENT_ADAPTER_REGISTRY } from '../src/agents/agent-adapters.js'; +import type { AgentRequest } from '../src/agents/agent-adapter-contract.js'; +import { resolveGuidanceProfile } from '../src/campaigns/condition-compiler.js'; +import { agentVisibleContractText } from '../src/composition/agent-visible-contract.js'; +import { applyCredentialAliases, materializeScenarioCredentials, validateCredentialAliases } + from '../src/composition/credential-aliases.js'; +import { loadTrack } from '../src/composition/tracks.js'; +import { seededAdminPassword } from '../tracks/ecommerce/walk.js'; + +const aliases = { + 'stackbench-admin-2026': 'store-admin-2026', + 'stackbench-customer-2026': 'store-customer-2026', + 'stackbench-staff-2026': 'store-staff-2026', +}; + +test('neutral guidance gives the agent and grader the same credential aliases', () => { + const guidance = resolveGuidanceProfile('neutral', ['mongodb']); + assert.deepEqual(guidance.credentialAliases, aliases); + + const source = 'Admin: `stackbench-admin-2026`; staff: `stackbench-staff-2026`; ' + + 'customer: `stackbench-customer-2026`.'; + const visible = agentVisibleContractText(source, guidance.credentialAliases); + assert.equal(visible, + 'Admin: `store-admin-2026`; staff: `store-staff-2026`; customer: `store-customer-2026`.'); + + const scenario = { features: [{ setup: [ + { do: 'signIn', password: 'stackbench-admin-2026' }, + { do: 'fill', text: 'stackbench-staff-2026' }, + { do: 'fill', text: 'ordinary text' }, + ] }] }; + const materialized = materializeScenarioCredentials(scenario, guidance.credentialAliases); + const feature = materialized.features[0]; + const originalFeature = scenario.features[0]; + assert(feature); + assert(originalFeature); + const [signIn, staff, ordinary] = feature.setup; + const [originalSignIn] = originalFeature.setup; + assert(signIn); + assert(staff); + assert(ordinary); + assert(originalSignIn); + assert.equal(signIn.password, 'store-admin-2026'); + assert.equal(staff.text, 'store-staff-2026'); + assert.equal(ordinary.text, 'ordinary text'); + assert.equal(originalSignIn.password, 'stackbench-admin-2026'); + assert.equal(seededAdminPassword(guidance.credentialAliases), 'store-admin-2026'); +}); + +test('campaign agent and grader argv carry aliases only when the condition declares them', () => { + const adapter = AGENT_ADAPTER_REGISTRY.get('deterministic'); + const request: AgentRequest = { mode: 'build', backend: 'mongodb', level: 1, app: '/app', track: 'ecommerce', + runIndex: 0, model: 'fake', guidance: 'neutral', credentialAliases: aliases }; + const agentArgs = agentRequestArgv(adapter, request); + assert.equal(agentArgs[agentArgs.indexOf('--credential-aliases-json') + 1], JSON.stringify(aliases)); + + const args = { backend: 'mongodb', track: 'ecommerce', runIndex: 0, media: false, + condition: { guidance: { credentialAliases: aliases } } }; + const track = loadTrack('ecommerce'); + const graderArgs = gradeArgv(args, '/app', 'http://app', 'attempt', 1, + track, 'attempt'); + assert.equal(graderArgs[graderArgs.indexOf('--credential-aliases-json') + 1], + JSON.stringify(aliases)); + const lintArgs = { url: 'http://app', level: '1', track: 'ecommerce', label: 'attempt', + out: '/results', bundleArtifactId: 'attempt', credentialAliases: aliases }; + const lintArgv = contractLintArgv(lintArgs); + assert.equal(lintArgv[lintArgv.indexOf('--credential-aliases-json') + 1], JSON.stringify(aliases)); + + const fixtureArgs = { ...args, gradingCredentialAliases: {} }; + const fixtureGraderArgs = gradeArgv(fixtureArgs, '/app', 'http://app', 'attempt', 1, track, 'attempt'); + assert.equal(fixtureGraderArgs[fixtureGraderArgs.indexOf('--credential-aliases-json') + 1], '{}'); + assert.deepEqual(fixtureArgs.condition.guidance.credentialAliases, aliases); + + const unaliasedRequest: AgentRequest = { ...request, credentialAliases: null }; + const unaliasedArgs = { ...args, condition: undefined }; + assert.equal(agentRequestArgv(adapter, unaliasedRequest).includes('--credential-aliases-json'), false); + assert.equal(gradeArgv(unaliasedArgs, '/app', 'http://app', 'attempt', 1, + track, 'attempt').includes('--credential-aliases-json'), false); + assert.equal(contractLintArgv({ ...lintArgs, credentialAliases: undefined }) + .includes('--credential-aliases-json'), false); +}); + +test('model-free scenarios keep their original credentials without aliases', () => { + const scenario = { features: [{ criteria: [{ actions: [ + { do: 'signIn', password: 'stackbench-admin-2026' }, + { do: 'fill', text: 'stackbench-customer-2026' }, + ] }] }] }; + assert.strictEqual(materializeScenarioCredentials(scenario), scenario); + assert.throws(() => agentVisibleContractText('stackbench-staff-2026'), + /contains internal language/); + assert.equal(seededAdminPassword({}), 'stackbench-admin-2026'); +}); + +test('credential aliases reject ambiguous or ineffective maps', () => { + assert.throws(() => validateCredentialAliases({ same: 'same' }), /must change/); + assert.throws(() => validateCredentialAliases({ first: 'shared', second: 'shared' }), + /duplicated/); + assert.throws(() => validateCredentialAliases([]), /must be an object/); +}); + +test('credential aliases apply once without cascading', () => { + assert.equal(applyCredentialAliases('first second', { first: 'second', second: 'third' }), + 'second third'); +}); diff --git a/tools/stack-bench/tests/credential-broker.test.ts b/tools/stack-bench/tests/credential-broker.test.ts new file mode 100644 index 00000000000..debcc471d02 --- /dev/null +++ b/tools/stack-bench/tests/credential-broker.test.ts @@ -0,0 +1,1775 @@ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { Agent as HttpAgent, request as httpRequest, createServer } from 'node:http'; +import type { IncomingHttpHeaders, OutgoingHttpHeaders, Server } from 'node:http'; +import { connect, createServer as createNetServer } from 'node:net'; +import type { Socket } from 'node:net'; +import { tmpdir } from 'node:os'; +import { connect as tlsConnect } from 'node:tls'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { zstdCompressSync } from 'node:zlib'; +import test from 'node:test'; +import { gzipSync } from 'node:zlib'; + +import { brokerHostname, brokerProtocol, imageTokenAdjustment } from '../container/broker-protocols.js'; +import { createCredentialBroker, heartbeatStale } from '../container/credential-broker.js'; +import { noUnpriced, readCredentialBrokerLedger, reconcileCredentialBrokerReceipt, writeCredentialBrokerLedger } + from '../container/credential-broker-accounting.js'; +import type { BrokerConfig, BrokerLedger, BrokerMode, UnpricedReason } + from '../container/credential-broker-accounting.js'; +import { credentialBrokerDiagnostics, lookupCertificate, startCredentialBroker, stopCredentialBroker } + from '../container/credential-broker-process.js'; +import type { CredentialBrokerHandle } from '../container/credential-broker-process.js'; +import type { BrokerStats } from '../container/credential-broker.js'; +import { compiledEntrypoint } from '../src/package-root.js'; +import { validateAgentCostReceipt } from '../src/agents/agent-result-contract.js'; +import { durableCostLedger, runCostEvidence } from '../src/evidence/cost-proof.js'; + +const NO_ESTIMATES = { 'no-usage': 0, 'response-aborted': 0, 'upstream-error': 0 }; +const NO_UNPRICED = noUnpriced(); +const PRICING_RATES = { + input: 3, output: 15, cacheWrite5m: 3.75, cacheWrite1h: 6, cacheRead: 0.3, +}; +const ONE_REQUEST_USAGE = { input_tokens: 100, output_tokens: 100, + cache_read_input_tokens: 0, cache_creation_input_tokens: 0 }; +const ZERO_USAGE = { input: 0, output: 0, cacheRead: 0, cacheWrite5m: 0, cacheWrite1h: 0 }; +const ZERO_RAW_USAGE = { input_tokens: 0, output_tokens: 0, cache_read_input_tokens: 0, + cache_creation_input_tokens: 0 }; + +interface SendOptions { + method?: string; + path?: string; + headers?: OutgoingHttpHeaders; + body?: string | Buffer; +} + +interface SendResult { + status: number | undefined; + body: string; +} + +interface SeenRequest { + method: string | undefined; + url: string | undefined; + headers: IncomingHttpHeaders; + body: string; +} + +interface BrokerTestContext { + brokerPort: number; + sessionToken: string; + credential: string; + seen: SeenRequest[]; + stats: () => BrokerStats; +} + +interface BrokerReady { + pid: number; + baseUrl: string; + sessionToken: string; + root: string; + ledgerPath: string; +} + +type BrokerTestConfig = Partial> & { + upstreamBody?: string | Buffer; + upstreamStatus?: number; + upstreamHeaders?: OutgoingHttpHeaders; + }; + +function record(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function brokerReady(value: unknown): BrokerReady { + if (!record(value) || typeof value.pid !== 'number' || typeof value.baseUrl !== 'string' + || typeof value.sessionToken !== 'string' || typeof value.root !== 'string' + || typeof value.ledgerPath !== 'string') { + throw new Error('credential broker subprocess returned invalid readiness data'); + } + return { pid: value.pid, baseUrl: value.baseUrl, sessionToken: value.sessionToken, + root: value.root, ledgerPath: value.ledgerPath }; +} + +const listen = (server: Server): Promise => new Promise((resolveListen, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', () => { + const address = server.address(); + assert.ok(address && typeof address !== 'string'); + resolveListen(address.port); + }); +}); + +const close = (server: Server): Promise => new Promise((resolveClose, reject) => { + server.close(error => error ? reject(error) : resolveClose()); +}); + +async function waitFor(predicate: () => boolean, timeoutMs = 2_000): Promise { + const deadline = Date.now() + timeoutMs; + while (!predicate()) { + if (Date.now() >= deadline) throw new Error('timed out waiting for broker state'); + await new Promise(resolveWait => setTimeout(resolveWait, 10)); + } +} + +function send(port: number, { method = 'POST', path = '/v1/messages', headers = {}, body = '{}' }: + SendOptions = {}): Promise { + return new Promise((resolveSend, reject) => { + const request = httpRequest({ hostname: '127.0.0.1', port, method, path, headers }, response => { + const chunks: Buffer[] = []; + response.on('data', (chunk: Buffer) => chunks.push(chunk)); + response.on('end', () => resolveSend({ status: response.statusCode, + body: Buffer.concat(chunks).toString('utf8') })); + }); + request.once('error', reject); + request.end(body); + }); +} + +async function withBroker(mode: BrokerMode, body: (context: BrokerTestContext) => Promise, + config: BrokerTestConfig = {}): Promise { + const { upstreamBody = '{"ok":true}', upstreamStatus = 200, upstreamHeaders = {}, ...brokerConfig } = config; + const seen: SeenRequest[] = []; + const upstreamServer = createServer((request, response) => { + const chunks: Buffer[] = []; + request.on('data', (chunk: Buffer) => chunks.push(chunk)); + request.on('end', () => { + seen.push({ method: request.method, url: request.url, headers: request.headers, + body: Buffer.concat(chunks).toString('utf8') }); + response.writeHead(upstreamStatus, { 'content-type': 'application/json', ...upstreamHeaders }); + response.end(upstreamBody); + }); + }); + const upstreamPort = await listen(upstreamServer); + const sessionToken = 'session-token-value-1234567890'; + const credential = 'provider-secret-value-1234567890'; + const { server, stats } = createCredentialBroker({ mode, credential, sessionToken, + model: 'test-model', maxOutputTokens: 4096, ...brokerConfig }, { + requestUpstream: httpRequest, + upstream: { protocol: 'http:', hostname: '127.0.0.1', port: upstreamPort }, + }); + const brokerPort = await listen(server); + try { await body({ brokerPort, sessionToken, credential, seen, stats }); } + finally { await close(server); await close(upstreamServer); } +} + +test('credential broker rejects unauthorized and unsupported requests before upstream', async () => { + await withBroker('api-key', async ({ brokerPort, sessionToken, seen, stats }) => { + assert.equal((await send(brokerPort)).status, 401); + assert.equal((await send(brokerPort, { method: 'GET', + headers: { authorization: `Bearer ${sessionToken}` }, body: '' })).status, 404); + assert.equal((await send(brokerPort, { path: '/v1/complete', + headers: { authorization: `Bearer ${sessionToken}` } })).status, 404); + assert.equal(seen.length, 0); + assert.equal(stats().acceptedRequests, 0); + }); +}); + +// Seen live: agy lost a request with a broken pipe after a tool ran for six seconds. Node's +// default closes an idle connection after five seconds plus a one-second buffer. +test('the broker keeps an idle connection open for the next request', async () => { + await withBroker('api-key', async ({ brokerPort, sessionToken }) => { + const agent = new HttpAgent({ keepAlive: true, maxSockets: 1 }); + const localPort = () => new Promise((resolvePort, reject) => { + const request = httpRequest({ hostname: '127.0.0.1', port: brokerPort, method: 'POST', path: '/v1/messages', agent, + headers: { authorization: `Bearer ${sessionToken}` } }, response => { + const port = response.socket.localPort!; + response.resume().on('end', () => resolvePort(port)); + }); + request.once('error', reject).end(JSON.stringify({ model: 'test-model', max_tokens: 1 })); + }); + try { + const first = await localPort(); + await new Promise(resolveWait => setTimeout(resolveWait, 7_000)); + assert.equal(await localPort(), first, 'the second request reuses the first connection'); + } finally { agent.destroy(); } + }); +}); + +// A spend cap does not change the capacity tier requested by the agent. +test('Anthropic requests preserve the selected capacity tier under a cap', async () => { + for (const maxBudgetUsd of [1, null]) { + for (const serviceTier of [undefined, 'auto', 'standard_only']) { + await withBroker('api-key', async ({ brokerPort, sessionToken, seen }) => { + const request = { model: 'test-model', max_tokens: 1, + ...(serviceTier === undefined ? {} : { service_tier: serviceTier }) }; + assert.equal((await send(brokerPort, { headers: { authorization: `Bearer ${sessionToken}` }, + body: JSON.stringify(request) })).status, 200); + assert.equal(seen.length, 1); + assert.deepEqual(JSON.parse(seen[0]!.body), request); + }, { maxBudgetUsd, pricingRates: PRICING_RATES, + upstreamBody: JSON.stringify({ usage: { ...ONE_REQUEST_USAGE, service_tier: 'standard' } }) }); + } + } +}); + +// A maker's own Anthropic endpoint gets the same request under its path, keyed as its Claude Code guide sets it. +test('Anthropic-compatible upstreams receive the request under their path with a bearer key', async () => { + for (const [upstream, prefix, hostname] of [['zai', '/api/anthropic', 'api.z.ai'], + ['deepseek', '/anthropic', 'api.deepseek.com'], ['moonshot', '/anthropic', 'api.moonshot.ai']] as const) { + await withBroker('api-key', async ({ brokerPort, sessionToken, credential, seen }) => { + assert.equal((await send(brokerPort, { headers: { authorization: `Bearer ${sessionToken}` }, + body: JSON.stringify({ model: 'test-model', max_tokens: 1 }) })).status, 200); + assert.equal(seen[0]!.url, `${prefix}/v1/messages`); + assert.equal(seen[0]!.headers.authorization, `Bearer ${credential}`); + assert.equal(seen[0]!.headers['x-api-key'], undefined); + }, { upstream }); + assert.equal(brokerHostname({ mode: 'api-key', upstream }), hostname); + } +}); + +// The CLI names the model in the path and sends the broker's token as its Gemini key. +test('Google requests reach the selected model with the real key and only file, shell and task tools', async () => { + const path = '/v1beta/models/test-model:streamGenerateContent?alt=sse'; + const tools = [{ functionDeclarations: ['view_file', 'run_command', 'search_web', 'invoke_subagent', 'ask_question'] + .map(name => ({ name })) }]; + await withBroker('api-key', async ({ brokerPort, sessionToken, credential, seen }) => { + const send_ = (to: string, body: Record) => send(brokerPort, { path: to, + headers: { 'x-goog-api-key': sessionToken, 'content-type': 'application/json' }, body: JSON.stringify(body) }); + assert.equal((await send_(path, { contents: [], tools })).status, 200); + assert.equal(seen[0]!.url, path); + assert.equal(seen[0]!.headers['x-goog-api-key'], credential); + const forwarded = JSON.parse(seen[0]!.body); + assert.deepEqual(forwarded.tools[0].functionDeclarations.map((tool: { name: string }) => tool.name), + ['view_file', 'run_command']); + assert.equal(forwarded.generationConfig.maxOutputTokens, 4096); + // Another model, or a "title" call that is not one, never reaches Google. + assert.equal((await send_('/v1beta/models/other-model:streamGenerateContent?alt=sse', { contents: [] })).status, 404); + const title = '/v1beta/models/gemini-3.1-flash-lite-preview:streamGenerateContent?alt=sse'; + assert.equal((await send_(title, { contents: [], systemInstruction: { parts: [{ text: 'Write code.' }] } })).status, 400); + assert.equal((await send_(title, { contents: [], + systemInstruction: { parts: [{ text: 'You are a conversation title generator. Respond with a title.' }] } })).status, 200); + assert.equal(seen.length, 2); + }, { provider: 'google', upstreamBody: `data: ${JSON.stringify({ candidates: [], + usageMetadata: { promptTokenCount: 10, candidatesTokenCount: 1, totalTokenCount: 11 } })}\r\n\r\n` }); + const protocol = brokerProtocol({ provider: 'google', mode: 'api-key', credential: 'x'.repeat(16), + sessionToken: 'y'.repeat(16), model: 'test-model', maxOutputTokens: 4096 }); + const events = [{ candidates: [{ content: { parts: [{ text: 'a' }] } }] }, + { usageMetadata: { promptTokenCount: 1000, cachedContentTokenCount: 400, candidatesTokenCount: 50, + thoughtsTokenCount: 30, totalTokenCount: 1080 } }].map(event => `data: ${JSON.stringify(event)}\r\n\r\n`).join(''); + assert.deepEqual(protocol.responseUsage(Buffer.from(events)), { input_tokens: 600, output_tokens: 80, + cache_read_input_tokens: 400, cache_creation_input_tokens: 0, web_search_calls: 0, unpriced_reason: null }); + assert.equal(protocol.requestPricing({ tools: [{ googleSearch: {} }] }).requiresUsage, 'hosted-tool'); + assert.equal(protocol.requestPricing({ tools: [{ urlContext: {} }] }).unpriced, 'hosted-tool'); + assert.equal(brokerHostname({ provider: 'google', mode: 'api-key' }), 'generativelanguage.googleapis.com'); +}); + +// With a Google account sign-in the CLI calls Code Assist: the model is named in a wrapper, +// account reads are POSTs forwarded unbilled, and each request carries the latest renewed token. +test('Google account requests reach Code Assist with the renewed token and only file, shell and task tools', async () => { + const root = mkdtempSync(join(tmpdir(), 'agy-account-')); + const credentialPath = join(root, 'credential'); + writeFileSync(credentialPath, 'renewed-token-value-1234567890'); + const call = '/v1internal:streamGenerateContent?alt=sse'; + const tools = [{ functionDeclarations: ['view_file', 'run_command', 'search_web', 'invoke_subagent'] + .map(name => ({ name })) }]; + const usageEvent = (usageMetadata: Record) => + `data: ${JSON.stringify({ response: { candidates: [], usageMetadata }, traceId: 't' })}\r\n\r\n`; + try { + await withBroker('subscription-token', async ({ brokerPort, sessionToken, seen, stats }) => { + const post = (to: string, body: Record) => send(brokerPort, { path: to, + headers: { authorization: `Bearer ${sessionToken}`, 'content-type': 'application/json' }, body: JSON.stringify(body) }); + assert.equal((await post('/v1internal:loadCodeAssist', { metadata: { ideType: 'ANTIGRAVITY' } })).status, 200); + assert.deepEqual(JSON.parse(seen[0]!.body), { metadata: { ideType: 'ANTIGRAVITY' } }); + assert.equal((await post(call, { project: 'p', model: 'gemini-pro-agent', request: { contents: [], tools } })).status, 200); + assert.equal(seen[1]!.url, call); + const forwarded = JSON.parse(seen[1]!.body); + assert.deepEqual(forwarded.request.tools[0].functionDeclarations.map((tool: { name: string }) => tool.name), + ['view_file', 'run_command']); + assert.equal(forwarded.request.generationConfig.maxOutputTokens, 4096); + assert.equal(stats().billableRequests, 1); + writeFileSync(credentialPath, 'later-token-value-1234567890'); + assert.equal((await post(call, { model: 'gemini-3.5-flash-lite', + request: { contents: [], systemInstruction: { parts: [{ text: 'You are a conversation title generator.' }] } } })).status, 200); + assert.deepEqual(seen.map(request => request.headers.authorization), ['Bearer renewed-token-value-1234567890', + 'Bearer renewed-token-value-1234567890', 'Bearer later-token-value-1234567890']); + // Another model, and any other account call, never reaches Google. + assert.equal((await post(call, { model: 'gemini-3.8-flash-high', request: { contents: [] } })).status, 400); + assert.equal((await post('/v1internal:writeTrajectoryAcls', {})).status, 404); + assert.equal(seen.length, 3); + }, { provider: 'google', model: 'gemini-3.1-pro-preview-customtools', credentialPath, maxBudgetUsd: 10, + pricingRates: PRICING_RATES, upstreamBody: usageEvent({ promptTokenCount: 10, candidatesTokenCount: 1 }) }); + } finally { rmSync(root, { recursive: true, force: true }); } + const config = { provider: 'google', mode: 'subscription-token', credential: 'x'.repeat(16), + sessionToken: 'y'.repeat(16), model: 'gemini-3.1-pro-preview-customtools', maxOutputTokens: 4096 } as const; + assert.deepEqual(brokerProtocol(config).responseUsage(Buffer.from(usageEvent({ promptTokenCount: 1000, + cachedContentTokenCount: 400, candidatesTokenCount: 50, thoughtsTokenCount: 30 }))), { input_tokens: 600, + output_tokens: 80, cache_read_input_tokens: 400, cache_creation_input_tokens: 0, web_search_calls: 0, + unpriced_reason: null }); + assert.throws(() => brokerProtocol({ ...config, model: 'test-model' }), /no Antigravity account model/); + // The account names a model once per effort; another model's name is refused. + const flash = brokerProtocol({ ...config, model: 'gemini-3.8-flash' }); + const generate = (model: string) => flash.parseRequest( + Buffer.from(JSON.stringify({ model, request: { contents: [] } })), '/v1internal:streamGenerateContent'); + assert.equal(generate('gemini-3.8-flash-medium').model, 'gemini-3.8-flash-medium'); + assert.throws(() => generate('gemini-pro-agent'), /does not match/); + const argon = brokerProtocol({ ...config, model: 'gemini-4-argon' }).parseRequest( + Buffer.from(JSON.stringify({ model: 'gemini-4-argon-medium', request: { contents: [] } })), '/v1internal:streamGenerateContent'); + assert.equal(argon.model, 'gemini-4-argon-medium'); + // Seen live: the CLI asks a small model whether a running command waits for input. The broker + // refused it 556 times in one study; it is forwarded as the title call is, by its own prompt. + const helper = (text: string) => flash.parseRequest(Buffer.from(JSON.stringify({ model: 'gemini-2.5-flash-lite', + request: { contents: [], systemInstruction: { parts: [{ text }] } } })), '/v1internal:streamGenerateContent'); + const waiting = helper('You are a classifier that determines if a terminal command is actively waiting for the user to type.'); + assert.equal(flash.sideCall!('/v1internal:streamGenerateContent', waiting), true); + assert.throws(() => helper('Write the code for me.'), /does not match/); + assert.equal(brokerHostname({ provider: 'google', mode: 'subscription-token' }), 'daily-cloudcode-pa.googleapis.com'); +}); + +// The CLI checks its account against Google's user info service through its HTTPS proxy. +test('the broker answers the account check over its own certificate and tunnels every other host', async () => { + const root = mkdtempSync(join(tmpdir(), 'agy-lookup-')); + const seen: IncomingHttpHeaders[] = []; + const google = createServer((request, response) => { + seen.push(request.headers); + response.writeHead(200, { 'content-type': 'application/json' }); + response.end(JSON.stringify({ id: '1234', email: 'person@example.com', verified_email: true, name: 'A Person', picture: 'p' })); + }); + const echo = createNetServer(socket => socket.pipe(socket)); + try { + const certificate = lookupCertificate(root, 'www.googleapis.com'); + const googlePort = await listen(google); + const echoPort = await new Promise(resolveListen => echo.listen(0, '127.0.0.1', + () => resolveListen((echo.address() as { port: number }).port))); + const sessionToken = 'session-token-value-1234567890'; + const { server } = createCredentialBroker({ provider: 'google', mode: 'subscription-token', + credential: 'provider-secret-value-1234567890', sessionToken, model: 'gemini-3.1-pro-preview-customtools', + maxOutputTokens: 4096, tlsKeyPath: certificate.keyPath, tlsCertPath: certificate.certPath }, + { requestUpstream: httpRequest, upstream: { protocol: 'http:', hostname: '127.0.0.1', port: googlePort } }); + const brokerPort = await listen(server); + const tunnel = (to: string) => new Promise((resolveTunnel, reject) => httpRequest({ hostname: '127.0.0.1', + port: brokerPort, method: 'CONNECT', path: to }).on('connect', (_response, socket) => resolveTunnel(socket)) + .on('error', reject).end()); + const lookup = async (token: string) => { + const secure = tlsConnect({ socket: await tunnel('www.googleapis.com:443'), servername: 'www.googleapis.com', + ca: certificate.authority }); + secure.write(`GET /oauth2/v2/userinfo HTTP/1.1\r\nHost: www.googleapis.com\r\nAuthorization: Bearer ${token}\r\nConnection: close\r\n\r\n`); + const chunks: Buffer[] = []; + for await (const chunk of secure) chunks.push(chunk as Buffer); + return Buffer.concat(chunks).toString('utf8'); + }; + const answer = await lookup(sessionToken); + assert.match(answer, /^HTTP\/1\.1 200/); + assert.deepEqual(JSON.parse(answer.slice(answer.indexOf('\r\n\r\n') + 4)), + { id: '1234', email: 'agent@stack-bench.invalid', verified_email: true }); + assert.equal(seen[0]!.authorization, 'Bearer provider-secret-value-1234567890'); + assert.match(await lookup('wrong-token'), /^HTTP\/1\.1 404/); + assert.equal(seen.length, 1); + const other = await tunnel(`127.0.0.1:${echoPort}`); + other.write('ping'); + assert.equal(String((await once(other, 'data'))[0]), 'ping'); + other.destroy(); + await close(server); + } finally { + await close(google); + echo.close(); + rmSync(root, { recursive: true, force: true }); + } +}); + +// Remote and file-backed content remains available; its extra input is unpriced. +test('Anthropic forwards nested remote content and retains unpriced spend', async t => { + for (const source of [{ type: 'url', url: 'https://example.test/content' }, + { type: 'file', file_id: 'file-content' }]) { + for (const type of ['image', 'document']) { + for (const capped of [true, false]) { + await t.test(`${type}-${source.type}-${capped ? 'capped' : 'uncapped'}`, async () => { + const root = mkdtempSync(join(tmpdir(), 'anthropic-remote-budget-')); + const ledgerPath = join(root, 'ledger.json'); + try { + await withBroker('api-key', async ({ brokerPort, sessionToken, seen }) => { + const request = { model: 'test-model', max_tokens: 1, + messages: [{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'tool1', + content: [{ type, source }] }] }] }; + const response = await send(brokerPort, { headers: { authorization: `Bearer ${sessionToken}` }, + body: JSON.stringify(request) }); + assert.equal(response.status, 200); + assert.equal(seen.length, 1); + assert.deepEqual(JSON.parse(seen[0]!.body), request); + const ledger = readCredentialBrokerLedger(ledgerPath); + if (capped) { + assert.equal(ledger.billableRequests, 1); + assert.equal(ledger.unpricedBillableRequests, 1); + assert.equal(ledger.unpricedByReason['unpriced-input'], 1); + assert.equal(ledger.providerFailure, null); + } + }, { ledgerPath, maxBudgetUsd: capped ? 1 : null, pricingRates: PRICING_RATES, + upstreamBody: JSON.stringify({ usage: ONE_REQUEST_USAGE }) }); + } finally { rmSync(root, { recursive: true, force: true }); } + }); + } + } + } +}); + +test('Anthropic broker forwards server-side features and marks spend it cannot price', async () => { + const request = { model: 'test-model', max_tokens: 1, messages: [{ role: 'user', content: 'hello' }] }; + const webSearch = { tools: [{ type: 'web_search_20250305', name: 'web_search' }] }; + const usage = (extra: Record = {}) => JSON.stringify({ usage: { ...ONE_REQUEST_USAGE, ...extra } }); + const cases: Array<{ override: Record; upstreamBody: string; reason: UnpricedReason | null; + estimated?: boolean }> = [ + { override: webSearch, upstreamBody: usage({ server_tool_use: { web_search_requests: 2 } }), reason: 'server-tool' }, + { override: webSearch, upstreamBody: usage({ server_tool_use: { web_search_requests: 0 } }), reason: null }, + { override: webSearch, upstreamBody: usage(), reason: null }, + { override: webSearch, upstreamBody: JSON.stringify({ content: [{ type: 'server_tool_use', name: 'web_search' }], + usage: ONE_REQUEST_USAGE }), reason: 'server-tool' }, + { override: { tools: [{ type: 'web_fetch_20250910', name: 'web_fetch' }] }, + upstreamBody: usage({ server_tool_use: { web_fetch_requests: 1 } }), reason: 'server-tool' }, + { override: webSearch, upstreamBody: '{"ok":true}', reason: 'server-tool', estimated: true }, + { override: { mcp_servers: [{ type: 'url', url: 'https://mcp.example', name: 'remote' }] }, + upstreamBody: usage(), reason: 'mcp' }, + { override: { container: 'container-id' }, upstreamBody: usage(), reason: 'container' }, + { override: { speed: 'fast' }, upstreamBody: usage(), reason: 'speed' }, + { override: { service_tier: 'priority' }, upstreamBody: usage({ service_tier: 'priority' }), reason: 'service-tier' }, + { override: { service_tier: 'auto' }, upstreamBody: usage({ service_tier: 'standard' }), reason: null }, + { override: { tools: [{ name: 'Bash', description: 'Run a command', input_schema: { type: 'object' } }, + { type: 'custom', name: 'Edit', input_schema: { type: 'object' } }] }, upstreamBody: usage(), reason: null }, + ]; + for (const { override, upstreamBody, reason, estimated = false } of cases) { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-broker-unpriced-')); + const ledgerPath = join(root, 'ledger.json'); + try { + await withBroker('api-key', async ({ brokerPort, sessionToken, seen }) => { + const body = JSON.stringify({ ...request, ...override }); + assert.equal((await send(brokerPort, { headers: { authorization: `Bearer ${sessionToken}`, + 'content-type': 'application/json' }, body })).status, 200, body); + assert.equal(seen.length, 1); + assert.deepEqual(JSON.parse(seen[0]!.body), { ...request, ...override }); + const ledger = readCredentialBrokerLedger(ledgerPath, { model: 'test-model', maxBudgetUsd: 1 }); + assert.equal(ledger.complete, true); + assert.equal(ledger.unpricedBillableRequests, reason ? 1 : 0, body); + assert.deepEqual(ledger.unpricedByReason, reason ? { ...NO_UNPRICED, [reason]: 1 } : NO_UNPRICED); + assert.equal(ledger.estimatedBillableRequests, estimated ? 1 : 0); + // The CLI can report server-tool fees above the broker's priced figure. + const reconciled = reconcileCredentialBrokerReceipt({ ledger, + cliResult: { type: 'result', is_error: false, total_cost_usd: 0.0218, + usage: estimated ? ZERO_RAW_USAGE : ONE_REQUEST_USAGE }, + model: 'test-model', maxBudgetUsd: 1, pricingRates: PRICING_RATES }); + assert.equal(reconciled.ok, true, reconciled.receipt.error ?? undefined); + const { receipt } = reconciled; + assert.equal(receipt.reconciled, true); + assert.equal(receipt.schemaVersion, 4); + assert.equal(receipt.exact, reason === null && !estimated); + assert.equal(receipt.unpricedRequests, reason ? 1 : 0); + assert.deepEqual(receipt.unpricedByReason, ledger.unpricedByReason); + assert.equal(receipt.costUsd, ledger.spentUsd); + assert.equal(receipt.cliCostUsd, 0.0218); + assert.equal(reconciled.result.terminal_reason, undefined); + validateAgentCostReceipt(receipt, 'test-model', 'receipt'); + const run = { totals: { costUsd: receipt.costUsd, costComplete: true }, levels: [{ level: 1, + buildSessions: [{ costUsd: receipt.costUsd, costComplete: true, costReceipts: [{ invocation: 1, receipt }] }] }] }; + const cost = durableCostLedger(run); + assert.equal(cost.complete, true); + assert.equal(cost.priced, reason === null); + assert.deepEqual(runCostEvidence(run), reason === null + ? { status: estimated ? 'upper-bound' : 'exact', costUsd: receipt.costUsd } + : estimated ? { status: 'unknown', costUsd: null } + : { status: 'unknown', costUsd: null, lowerBoundUsd: receipt.costUsd }); + }, { ledgerPath, maxBudgetUsd: 1, pricingRates: PRICING_RATES, upstreamBody }); + } finally { rmSync(root, { recursive: true, force: true }); } + } +}); + +test('credential broker rejects an unauthorized request before its body completes', async () => { + await withBroker('api-key', async ({ brokerPort }) => { + const result = await new Promise((resolveResult, reject) => { + const request = httpRequest({ hostname: '127.0.0.1', port: brokerPort, + method: 'POST', path: '/v1/messages', headers: { 'content-length': '1000000' } }, response => { + const chunks: Buffer[] = []; + response.on('data', (chunk: Buffer) => chunks.push(chunk)); + response.on('end', () => resolveResult({ status: response.statusCode, + body: Buffer.concat(chunks).toString('utf8') })); + }); + request.once('error', reject); + request.write('{'); + }); + assert.deepEqual(result, { status: 401, body: 'unauthorized' }); + }); +}); + +test('credential broker survives a downstream disconnect and settles exact upstream usage', async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-broker-disconnect-')); + const ledgerPath = join(root, 'ledger.json'); + const upstreamServer = createServer((request, response) => { + request.resume(); + request.on('end', () => { + response.writeHead(200, { 'content-type': 'application/json' }); + response.write('{"usage":'); + setTimeout(() => response.end(`${JSON.stringify(ONE_REQUEST_USAGE)}}`), 25); + }); + }); + const upstreamPort = await listen(upstreamServer); + const sessionToken = 'session-token-value-1234567890'; + const { server, stats } = createCredentialBroker({ mode: 'api-key', + credential: 'provider-secret-value-1234567890', sessionToken, + ledgerPath, model: 'test-model', maxOutputTokens: 4096, + maxBudgetUsd: 1, pricingRates: PRICING_RATES }, { + requestUpstream: httpRequest, + upstream: { protocol: 'http:', hostname: '127.0.0.1', port: upstreamPort }, + }); + const brokerPort = await listen(server); + const requestOptions = { hostname: '127.0.0.1', port: brokerPort, method: 'POST', + path: '/v1/messages', headers: { authorization: `Bearer ${sessionToken}`, + 'content-type': 'application/json' } }; + try { + await new Promise((resolveRequest, reject) => { + const request = httpRequest(requestOptions, response => { + response.once('data', () => { response.destroy(); resolveRequest(); }); + }); + request.once('error', reject); + request.end('{"model":"test-model","max_tokens":1000}'); + }); + await waitFor(() => stats().completedBillableRequests === 1); + const second = await send(brokerPort, { + headers: requestOptions.headers, + body: '{"model":"test-model","max_tokens":1000}', + }); + assert.equal(second.status, 200); + assert.deepEqual(stats(), { acceptedRequests: 2, billableRequests: 2, + completedBillableRequests: 2, estimatedBillableRequests: 0, estimatedByReason: NO_ESTIMATES, + spentUsd: 0.0036, reservedUsd: 0 }); + assert.equal(readCredentialBrokerLedger(ledgerPath, + { model: 'test-model', maxBudgetUsd: 1 }).complete, true); + } finally { + await close(server); + await close(upstreamServer); + rmSync(root, { recursive: true, force: true }); + } +}); + +test('credential broker survives malformed client socket input', async () => { + await withBroker('api-key', async ({ brokerPort, sessionToken }) => { + await new Promise((resolveSocket, reject) => { + const socket = connect({ host: '127.0.0.1', port: brokerPort }); + socket.once('connect', () => socket.write('not-http\r\n\r\n')); + socket.once('data', () => socket.destroy()); + socket.once('close', () => resolveSocket()); + socket.once('error', reject); + }); + const result = await send(brokerPort, { + headers: { authorization: `Bearer ${sessionToken}` }, + body: '{"model":"test-model","max_tokens":1}', + }); + assert.equal(result.status, 200); + }); +}); + +for (const mode of ['api-key', 'subscription-token'] satisfies BrokerMode[]) { + test(`credential broker replaces the session credential for ${mode}`, async () => { + await withBroker(mode, async ({ brokerPort, sessionToken, credential, seen, stats }) => { + const result = await send(brokerPort, { + path: '/v1/messages?beta=true', + headers: { authorization: `Bearer ${sessionToken}`, 'content-type': 'application/json' }, + body: '{"model":"test-model","max_tokens":1024}', + }); + assert.deepEqual(result, { status: 200, body: '{"ok":true}' }); + assert.equal(seen.length, 1); + const forwarded = seen[0]; + assert.ok(forwarded); + assert.equal(forwarded.url, '/v1/messages?beta=true'); + assert.equal(forwarded.body, '{"model":"test-model","max_tokens":1024}'); + if (mode === 'api-key') { + assert.equal(forwarded.headers['x-api-key'], credential); + assert.equal(forwarded.headers.authorization, undefined); + } else { + assert.equal(forwarded.headers.authorization, `Bearer ${credential}`); + assert.equal(forwarded.headers['x-api-key'], undefined); + } + assert.doesNotMatch(JSON.stringify(forwarded.headers), new RegExp(sessionToken)); + assert.equal(stats().acceptedRequests, 1); + }); + }); +} + +test('credential broker limits body size without truncating authorized sessions at 512 requests', async () => { + const root = mkdtempSync(join(tmpdir(), 'broker-local-limit-')); + const ledgerPath = join(root, 'ledger.json'); + const seen: string[] = []; + const upstreamServer = createServer((request, response) => { + request.resume(); + request.on('end', () => { + assert.ok(request.url); + seen.push(request.url); + response.end('ok'); + }); + }); + const upstreamPort = await listen(upstreamServer); + const sessionToken = 'session-token-value-1234567890'; + const { server } = createCredentialBroker({ mode: 'api-key', ledgerPath, + credential: 'provider-secret-value-1234567890', sessionToken, + model: 'test-model', maxOutputTokens: 4096 }, { + requestUpstream: httpRequest, + upstream: { protocol: 'http:', hostname: '127.0.0.1', port: upstreamPort }, + maxRequestBytes: 64, + }); + const brokerPort = await listen(server); + const headers = { authorization: `Bearer ${sessionToken}` }; + try { + assert.equal((await send(brokerPort, { headers, body: 'x'.repeat(65) })).status, 413); + for (let index = 0; index < 513; index++) { + assert.equal((await send(brokerPort, { headers, + body: '{"model":"test-model","max_tokens":1}' })).status, 200); + } + assert.equal(seen.length, 513); + assert.equal(readCredentialBrokerLedger(ledgerPath).providerFailure, null); + } finally { + await close(server); + await close(upstreamServer); + rmSync(root, { recursive: true, force: true }); + } +}); + +test('credential broker enforces model, output token, and session cost limits', async () => { + const rates = { input: 3, output: 15, cacheWrite5m: 3.75, cacheWrite1h: 6, cacheRead: 0.3 }; + await withBroker('api-key', async ({ brokerPort, sessionToken, seen, stats }) => { + const headers = { authorization: `Bearer ${sessionToken}`, + 'content-type': 'application/json' }; + assert.deepEqual(await send(brokerPort, { headers, + body: '{"model":"other-model","max_tokens":1}' }), + { status: 400, body: 'invalid provider request' }); + assert.equal((await send(brokerPort, { headers, + body: '{"model":"test-model","max_tokens":4097}' })).status, 400); + assert.equal((await send(brokerPort, { headers, + body: '{"model":"test-model","max_tokens":1000}' })).status, 402); + assert.equal(seen.length, 0); + assert.deepEqual(stats(), { acceptedRequests: 3, billableRequests: 0, + completedBillableRequests: 0, estimatedBillableRequests: 0, estimatedByReason: NO_ESTIMATES, + spentUsd: 0, reservedUsd: 0 }); + }, { maxBudgetUsd: 0.01, pricingRates: rates }); +}); + +test('credential broker charges reported usage and reserves enough for the next request', async t => { + const root = mkdtempSync(join(tmpdir(), 'broker-budget-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const ledgerPath = join(root, 'ledger.json'); + const rates = { input: 3, output: 15, cacheWrite5m: 3.75, cacheWrite1h: 6, cacheRead: 0.3 }; + const upstreamBody = JSON.stringify({ usage: { input_tokens: 100, output_tokens: 100, + cache_read_input_tokens: 0, cache_creation_input_tokens: 0 } }); + await withBroker('api-key', async ({ brokerPort, sessionToken, credential, seen, stats }) => { + const request = { headers: { authorization: `Bearer ${sessionToken}`, + 'content-type': 'application/json' }, + body: '{"model":"test-model","max_tokens":1000}' }; + assert.equal((await send(brokerPort, request)).status, 200); + assert.equal((await send(brokerPort, request)).status, 200); + assert.equal((await send(brokerPort, request)).status, 200); + assert.equal((await send(brokerPort, request)).status, 402); + assert.equal(seen.length, 3); + assert.deepEqual(stats(), { acceptedRequests: 4, billableRequests: 3, + completedBillableRequests: 3, estimatedBillableRequests: 0, estimatedByReason: NO_ESTIMATES, + spentUsd: 0.0054, reservedUsd: 0 }); + const failure = readCredentialBrokerLedger(ledgerPath).providerFailure!; + assert.equal(failure.category, 'broker-budget'); + assert.equal(failure.code, 'reservation-exceeds-budget'); + assert.equal(failure.budget?.spentUsd, 0.0054); + assert.equal(failure.budget?.estimatedSpendUsd, 0); + assert.equal(failure.budget?.reservedUsd, 0); + assert.equal(failure.budget?.maxBudgetUsd, 0.02); + assert.ok(failure.budget!.spentUsd + failure.budget!.requestCeilingUsd > 0.02); + const ledger = readCredentialBrokerLedger(ledgerPath, { model: 'test-model', maxBudgetUsd: 0.02 }); + assert.equal(ledger.complete, true); + assert.equal(ledger.billableRequests, 3); + assert.equal(ledger.completedBillableRequests, 3); + assert.equal(ledger.estimatedBillableRequests, 0); + assert.equal(ledger.spentUsd, 0.0054); + assert.deepEqual(ledger.usage, + { input: 300, output: 300, cacheRead: 0, cacheWrite5m: 0, cacheWrite1h: 0 }); + const text = readFileSync(ledgerPath, 'utf8'); + assert.doesNotMatch(text, new RegExp(sessionToken)); + assert.doesNotMatch(text, new RegExp(credential)); + assert.deepEqual(readdirSync(root), ['ledger.json']); + }, { ledgerPath, maxBudgetUsd: 0.02, pricingRates: rates, upstreamBody }); +}); + +test('credential broker clears different concurrent reservations before the next request', async () => { + const rates = { input: 2, output: 10, cacheWrite5m: 2.5, cacheWrite1h: 4, cacheRead: 0.2 }; + const upstreamBody = JSON.stringify({ usage: ONE_REQUEST_USAGE }); + const bodyWithBytes = (bytes: number): string => { + const prefix = '{"model":"test-model","max_tokens":65536,"padding":"'; + const suffix = '"}'; + return `${prefix}${'x'.repeat(bytes - prefix.length - suffix.length)}${suffix}`; + }; + await withBroker('api-key', async ({ brokerPort, sessionToken, stats }) => { + const headers = { authorization: `Bearer ${sessionToken}`, + 'content-type': 'application/json' }; + const first = send(brokerPort, { headers, body: bodyWithBytes(4_049) }); + const second = send(brokerPort, { headers, body: bodyWithBytes(6_840) }); + assert.deepEqual((await Promise.all([first, second])).map(result => result.status), [200, 200]); + assert.equal(stats().reservedUsd, 0); + assert.equal((await send(brokerPort, { headers, body: bodyWithBytes(4_049) })).status, 200); + assert.deepEqual(stats(), { acceptedRequests: 3, billableRequests: 3, + completedBillableRequests: 3, estimatedBillableRequests: 0, estimatedByReason: NO_ESTIMATES, + spentUsd: 0.0036, reservedUsd: 0 }); + }, { maxBudgetUsd: 100, maxOutputTokens: 65_536, pricingRates: rates, upstreamBody }); +}); + +test('credential broker prices repeated compressed streams instead of exhausting the request cap', async () => { + const rates = { input: 3, output: 15, cacheWrite5m: 3.75, cacheWrite1h: 6, cacheRead: 0.3 }; + const stream = [ + 'data: {"type":"message_start","message":{"usage":{"input_tokens":100,"output_tokens":0,"cache_read_input_tokens":0,"cache_creation_input_tokens":0}}}', + '', + 'data: {"type":"message_delta","usage":{"output_tokens":100}}', + '', + 'data: {"type":"message_stop"}', + '', + ].join('\n'); + await withBroker('api-key', async ({ brokerPort, sessionToken, seen, stats }) => { + const request = { headers: { authorization: `Bearer ${sessionToken}`, + 'content-type': 'application/json', 'accept-encoding': 'gzip' }, + body: '{"model":"test-model","max_tokens":128000}' }; + for (let turn = 0; turn < 60; turn += 1) { + assert.equal((await send(brokerPort, request)).status, 200); + } + const forwarded = seen[0]; + assert.ok(forwarded); + assert.equal(forwarded.headers['accept-encoding'], undefined); + assert.deepEqual(stats(), { acceptedRequests: 60, billableRequests: 60, + completedBillableRequests: 60, estimatedBillableRequests: 0, estimatedByReason: NO_ESTIMATES, + spentUsd: 0.108, reservedUsd: 0 }); + }, { maxBudgetUsd: 100, maxOutputTokens: 128_000, pricingRates: rates, + upstreamHeaders: { 'content-type': 'text/event-stream', 'content-encoding': 'gzip' }, + upstreamBody: gzipSync(stream) }); +}); + +test('a provider refusal inside a successful stream is a free rejection, and a capacity refusal can be waited out', async () => { + const rates = { input: 2, output: 10, cacheRead: 0.1, cacheWrite5m: 0, cacheWrite1h: 0 }; + const failed = (error: Record) => + `event: response.failed +data: ${JSON.stringify({ type: 'response.failed', response: { status: 'failed', error } })} + +`; + const started = `event: response.output_item.added +data: ${JSON.stringify({ type: 'response.output_item.added' })} + +`; + for (const { upstreamBody, failure, estimated } of [ + { upstreamBody: failed({ code: 'server_is_overloaded', message: 'Selected model is at capacity. Please try a different model.' }), + failure: { category: 'rate-limit', status: 200, code: 'server_is_overloaded' }, estimated: 0 }, + { upstreamBody: failed({ message: 'Selected model is at capacity.' }), + failure: { category: 'rate-limit', status: 200, code: null }, estimated: 0 }, + { upstreamBody: failed({ code: 'invalid_prompt', message: 'private detail' }), + failure: { category: 'request', status: 200, code: 'invalid_prompt' }, estimated: 0 }, + // A failure after output began may have been billed, so it stays an estimated transport failure. + { upstreamBody: started + failed({ code: 'server_error', message: 'stream broke' }), + failure: { category: 'transport', status: 200, code: 'incomplete-response' }, estimated: 1 }, + ]) { + const root = mkdtempSync(join(tmpdir(), 'openai-broker-rejection-')); + const ledgerPath = join(root, 'ledger.json'); + try { + await withBroker('api-key', async ({ brokerPort, sessionToken }) => { + assert.equal((await send(brokerPort, { path: '/v1/responses', + headers: { authorization: `Bearer ${sessionToken}` }, + body: JSON.stringify({ model: 'test-model', input: 'hello' }) })).status, 200); + const ledger = readCredentialBrokerLedger(ledgerPath); + assert.deepEqual(ledger.providerFailure, failure); + assert.equal(ledger.estimatedBillableRequests, estimated); + assert.equal(ledger.spentUsd > 0, estimated === 1); + assert.doesNotMatch(JSON.stringify(ledger), /private detail|at capacity/); + }, { provider: 'openai', ledgerPath, pricingRates: rates, maxBudgetUsd: 10, + upstreamHeaders: { 'content-type': 'text/event-stream' }, upstreamBody }); + } finally { rmSync(root, { recursive: true, force: true }); } + } +}); + +// agy gives up on a rate limit after one attempt, then titles the conversation. The +// title call succeeding must leave the refused call on record so the session is resumed. +test('a side call that succeeds after a rate-limited session call leaves the rate limit on record', async () => { + const root = mkdtempSync(join(tmpdir(), 'broker-side-call-')); + const upstream = createServer((request, response) => { + request.resume().on('end', () => { + if (request.url!.includes('gemini-3.1-flash-lite-preview')) { + response.writeHead(200, { 'content-type': 'text/event-stream' }); + response.end(`data: ${JSON.stringify({ usageMetadata: { promptTokenCount: 10, candidatesTokenCount: 1 } })}\r\n\r\n`); + } else { + response.writeHead(429, { 'content-type': 'application/json' }); + response.end('{"error":{"code":429,"status":"RESOURCE_EXHAUSTED"}}'); + } + }); + }); + const upstreamPort = await listen(upstream); + const ledgerPath = join(root, 'ledger.json'); + const sessionToken = 'session-token-value-1234567890'; + const { server } = createCredentialBroker({ provider: 'google', mode: 'api-key', credential: 'provider-secret-value-1234567890', + sessionToken, model: 'test-model', maxOutputTokens: 4096, ledgerPath, maxBudgetUsd: 10, + pricingRates: { input: 2, output: 10, cacheRead: 0.1, cacheWrite5m: 2, cacheWrite1h: 2 } }, { + requestUpstream: httpRequest, upstream: { protocol: 'http:', hostname: '127.0.0.1', port: upstreamPort } }); + const brokerPort = await listen(server); + try { + const headers = { 'x-goog-api-key': sessionToken, 'content-type': 'application/json' }; + assert.equal((await send(brokerPort, { path: '/v1beta/models/test-model:streamGenerateContent?alt=sse', headers, + body: '{"contents":[]}' })).status, 429); + assert.equal((await send(brokerPort, { path: '/v1beta/models/gemini-3.1-flash-lite-preview:streamGenerateContent?alt=sse', + headers, body: JSON.stringify({ contents: [], systemInstruction: { parts: [{ text: 'You are a conversation title generator.' }] } }) })).status, 200); + assert.equal(readCredentialBrokerLedger(ledgerPath).providerFailure?.category, 'rate-limit'); + } finally { await close(server); await close(upstream); rmSync(root, { recursive: true, force: true }); } +}); + +test('a provider the broker never reached costs nothing and can be waited out; a dropped connection is estimated', async () => { + const root = mkdtempSync(join(tmpdir(), 'broker-unreachable-')); + // A listener that accepts and immediately drops the connection, and a port with no listener. + const dropping = createServer(); + dropping.on('connection', socket => socket.destroy()); + const droppingPort = await listen(dropping); + const closed = createServer(); + const closedPort = await listen(closed); + await close(closed); + try { + for (const { port, failure, estimated } of [ + { port: closedPort, failure: { category: 'unreachable', status: null, code: null }, estimated: 0 }, + { port: droppingPort, failure: { category: 'transport', status: null, code: null }, estimated: 1 }, + ]) { + const ledgerPath = join(root, `ledger-${port}.json`); + const sessionToken = 'session-token-value-1234567890'; + const { server } = createCredentialBroker({ mode: 'api-key', credential: 'provider-secret-value-1234567890', + sessionToken, model: 'test-model', maxOutputTokens: 4096, ledgerPath, maxBudgetUsd: 10, + pricingRates: { input: 2, output: 10, cacheRead: 0.1, cacheWrite5m: 0, cacheWrite1h: 0 } }, { + requestUpstream: httpRequest, upstream: { protocol: 'http:', hostname: '127.0.0.1', port } }); + const brokerPort = await listen(server); + try { + assert.equal((await send(brokerPort, { headers: { authorization: `Bearer ${sessionToken}` }, + body: '{"model":"test-model","max_tokens":1000}' })).status, 502); + const ledger = readCredentialBrokerLedger(ledgerPath); + assert.deepEqual(ledger.providerFailure, failure); + assert.equal(ledger.estimatedBillableRequests, estimated); + assert.equal(ledger.spentUsd > 0, estimated === 1); + } finally { await close(server); } + } + } finally { await close(dropping); rmSync(root, { recursive: true, force: true }); } +}); + +test('credential broker estimates a settled request without exact provider usage', async t => { + const errorStream = [ + 'data: {"type":"message_start","message":{"usage":{"input_tokens":100,"output_tokens":0}}}', + '', + 'event: error', + 'data: {"type":"error","error":{"type":"api_error","message":"connection lost"}}', + '', + ].join('\n'); + const incompleteStreams = [ + { start: { output_tokens: 0 }, final: { output_tokens: 100 } }, + { start: { input_tokens: 100, output_tokens: 0 }, final: {} }, + { start: { input_tokens: 100, output_tokens: 0 }, final: { output_tokens: '100' } }, + ].map(({ start, final }) => [ + `data: ${JSON.stringify({ type: 'message_start', message: { usage: start } })}`, + '', + `data: ${JSON.stringify({ type: 'message_delta', usage: final })}`, + '', + 'data: {"type":"message_stop"}', + '', + ].join('\n')); + for (const { name, upstreamBody, upstreamHeaders, isError } of [ + { name: 'absent usage', upstreamBody: '{"ok":true}', upstreamHeaders: {}, isError: false }, + { name: 'error stream', upstreamBody: errorStream, upstreamHeaders: { 'content-type': 'text/event-stream' }, isError: true }, + ...[ + {}, + { input_tokens: 100 }, + { output_tokens: 100 }, + ...[-1, '100', null, 0.5, Number.MAX_SAFE_INTEGER + 1].flatMap(value => [ + { input_tokens: value, output_tokens: 100 }, + { input_tokens: 100, output_tokens: value }, + ]), + { input_tokens: 100, output_tokens: 100, cache_read_input_tokens: '50' }, + { input_tokens: 100, output_tokens: 100, cache_creation_input_tokens: -1 }, + { input_tokens: 100, output_tokens: 100, cache_creation: {} }, + ].map(usage => ({ name: `invalid usage ${JSON.stringify(usage)}`, + upstreamBody: JSON.stringify({ usage }), upstreamHeaders: {}, isError: false })), + ...incompleteStreams.map((upstreamBody, index) => ({ name: `incomplete stream ${index + 1}`, + upstreamBody, upstreamHeaders: { 'content-type': 'text/event-stream' }, isError: false })), + ]) { + await t.test(name, async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-broker-estimated-ledger-test-')); + const ledgerPath = join(root, 'ledger.json'); + try { + await withBroker('api-key', async ({ brokerPort, sessionToken }) => { + const response = await send(brokerPort, { + headers: { authorization: `Bearer ${sessionToken}`, 'content-type': 'application/json' }, + body: '{"model":"test-model","max_tokens":1000}', + }); + assert.equal(response.status, 200); + const ledger = readCredentialBrokerLedger(ledgerPath, + { model: 'test-model', maxBudgetUsd: 1 }); + assert.equal(ledger.complete, true); + assert.equal(ledger.estimatedBillableRequests, 1); + assert.deepEqual(ledger.estimatedByReason, { ...NO_ESTIMATES, 'no-usage': 1 }); + // The request was charged its ceiling, so the receipt is an upper bound + // and says so; the session it belongs to stays valid. + const reconciled = reconcileCredentialBrokerReceipt({ ledger, + cliResult: { type: 'result', is_error: isError, total_cost_usd: 0, usage: ZERO_RAW_USAGE }, + model: 'test-model', maxBudgetUsd: 1, pricingRates: PRICING_RATES }); + assert.equal(reconciled.receipt.error, null); + assert.equal(reconciled.ok, true, reconciled.receipt.error ?? undefined); + assert.equal(reconciled.receipt.reconciled, true); + assert.equal(reconciled.receipt.exact, false); + assert.equal(reconciled.receipt.estimatedRequests, 1); + assert.deepEqual(reconciled.receipt.estimatedByReason, { ...NO_ESTIMATES, 'no-usage': 1 }); + assert.equal(reconciled.receipt.costUsd, ledger.spentUsd); + assert.ok(reconciled.receipt.costUsd > 0); + if (!isError) assert.equal(reconciled.receipt.calculatedCostUsd, 0); + }, { ledgerPath, maxBudgetUsd: 1, pricingRates: PRICING_RATES, upstreamHeaders, upstreamBody }); + } finally { rmSync(root, { recursive: true, force: true }); } + }); + } +}); + +test('streamed provider usage produces a reconciled campaign-priced receipt', async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-broker-cache-ledger-test-')); + const ledgerPath = join(root, 'ledger.json'); + const stream = [ + 'data: {"type":"message_start","message":{"usage":{"input_tokens":100,"output_tokens":0,"cache_read_input_tokens":50,"cache_creation_input_tokens":30,"cache_creation":{"ephemeral_5m_input_tokens":20,"ephemeral_1h_input_tokens":10}}}}', + '', + 'data: {"type":"message_delta",', + 'data: "usage":{"input_tokens":null,"output_tokens":100,"cache_read_input_tokens":null,"cache_creation_input_tokens":null}}', + '', + 'data: {"type":"message_stop"}', + '', + ].join('\n'); + try { + await withBroker('api-key', async ({ brokerPort, sessionToken }) => { + const response = await send(brokerPort, { + headers: { authorization: `Bearer ${sessionToken}`, 'content-type': 'application/json' }, + body: '{"model":"test-model","max_tokens":1000}', + }); + assert.equal(response.status, 200); + const ledger = readCredentialBrokerLedger(ledgerPath, + { model: 'test-model', maxBudgetUsd: 1 }); + assert.deepEqual(ledger.usage, + { input: 100, output: 100, cacheRead: 50, cacheWrite5m: 20, cacheWrite1h: 10 }); + assert.equal(ledger.spentUsd, 0.00195); + const reconciled = reconcileCredentialBrokerReceipt({ ledger, + cliResult: { type: 'result', is_error: false, total_cost_usd: 0.002925, + usage: { input_tokens: 100, output_tokens: 100, cache_read_input_tokens: 50, + cache_creation_input_tokens: 30 } }, + model: 'test-model', maxBudgetUsd: 1, pricingRates: PRICING_RATES }); + assert.equal(reconciled.ok, true, reconciled.receipt.error ?? undefined); + assert.equal(reconciled.receipt.exact, true); + assert.equal(reconciled.receipt.estimatedRequests, 0); + assert.equal(reconciled.receipt.costUsd, 0.00195); + assert.equal(reconciled.receipt.calculatedCostUsd, 0.00195); + assert.equal(reconciled.receipt.cliCostUsd, 0.002925); + }, { ledgerPath, maxBudgetUsd: 1, pricingRates: PRICING_RATES, + upstreamHeaders: { 'content-type': 'text/event-stream' }, upstreamBody: stream }); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('campaign pricing is authoritative when CLI pricing differs', () => { + const ledger = { schemaVersion: 5, model: 'test-model', maxBudgetUsd: 1, + acceptedRequests: 2, billableRequests: 2, completedBillableRequests: 2, + estimatedBillableRequests: 0, estimatedByReason: NO_ESTIMATES, + unpricedBillableRequests: 0, unpricedByReason: NO_UNPRICED, + spentUsd: 0.0036, reservedUsd: 0, + usage: { input: 200, output: 200, cacheRead: 0, cacheWrite5m: 0, cacheWrite1h: 0 }, + complete: true, + updatedAt: new Date().toISOString() }; + const reconciled = reconcileCredentialBrokerReceipt({ ledger, + cliResult: { type: 'result', is_error: false, total_cost_usd: 0.0054, + usage: { ...ONE_REQUEST_USAGE, input_tokens: 200, output_tokens: 200 } }, + model: 'test-model', maxBudgetUsd: 1, pricingRates: PRICING_RATES }); + assert.equal(reconciled.ok, true, reconciled.receipt.error ?? undefined); + assert.equal(reconciled.result.is_error, false); + assert.equal(reconciled.result.total_cost_usd, 0.0036); + assert.equal(reconciled.receipt.source, 'credential-broker'); + assert.equal(reconciled.receipt.cliCostUsd, 0.0054); + assert.equal(reconciled.receipt.error, null); +}); + +test('spend ledgers count unpriced requests consistently and still read version 4', t => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-ledger-versions-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const path = join(root, 'ledger.json'); + const ledger = { schemaVersion: 5, model: 'test-model', maxBudgetUsd: 1, + acceptedRequests: 1, billableRequests: 1, completedBillableRequests: 1, + estimatedBillableRequests: 0, estimatedByReason: NO_ESTIMATES, + unpricedBillableRequests: 1, unpricedByReason: { ...NO_UNPRICED, mcp: 1 }, + spentUsd: 0.0018, reservedUsd: 0, usage: { ...ZERO_USAGE, input: 100, output: 100 }, + complete: true, updatedAt: new Date().toISOString() }; + writeCredentialBrokerLedger(path, ledger); + assert.deepEqual(readCredentialBrokerLedger(path), ledger); + assert.throws(() => writeCredentialBrokerLedger(path, { ...ledger, unpricedByReason: NO_UNPRICED }), /do not add up/); + assert.throws(() => writeCredentialBrokerLedger(path, { ...ledger, unpricedBillableRequests: 2, + unpricedByReason: { ...NO_UNPRICED, mcp: 2 } }), /unpriced request count/); + const { unpricedBillableRequests: _count, unpricedByReason: _reasons, ...fields } = ledger; + assert.throws(() => writeCredentialBrokerLedger(path, fields), /unpriced request counts/); + const previous = { ...fields, schemaVersion: 4 }; + assert.throws(() => writeCredentialBrokerLedger(path, previous), /not current/); + writeFileSync(path, JSON.stringify(previous)); + const read = readCredentialBrokerLedger(path); + assert.equal(read.unpricedBillableRequests, 0); + assert.deepEqual(read.unpricedByReason, NO_UNPRICED); + assert.throws(() => readCredentialBrokerLedger(path.replace('ledger', 'missing'))); + writeFileSync(path, JSON.stringify(ledger).replace('"schemaVersion":5', '"schemaVersion":4')); + assert.throws(() => readCredentialBrokerLedger(path), /unpriced request counts/); +}); + +test('broker retains cache write classes that the CLI summary combines', () => { + const ledger = { schemaVersion: 5, model: 'claude-sonnet-5', maxBudgetUsd: 50, + acceptedRequests: 66, billableRequests: 66, completedBillableRequests: 66, + estimatedBillableRequests: 0, estimatedByReason: NO_ESTIMATES, + unpricedBillableRequests: 0, unpricedByReason: NO_UNPRICED, + spentUsd: 1.686071, reservedUsd: 0, + usage: { input: 124, output: 48559, cacheRead: 5100226, + cacheWrite5m: 66627, cacheWrite1h: 3405 }, + complete: true, updatedAt: new Date().toISOString() }; + const reconciled = reconcileCredentialBrokerReceipt({ ledger, + cliResult: { type: 'result', is_error: false, total_cost_usd: 2.529107, + usage: { input_tokens: 124, output_tokens: 48559, + cache_read_input_tokens: 5100226, cache_creation_input_tokens: 70032 } }, + model: 'claude-sonnet-5', maxBudgetUsd: 50, + pricingRates: { input: 2, output: 10, cacheWrite5m: 2.5, + cacheWrite1h: 4, cacheRead: 0.2 } }); + assert.equal(reconciled.ok, true, reconciled.receipt.error ?? undefined); + assert.equal(reconciled.receipt.costUsd, 1.686071); + assert.equal(reconciled.receipt.calculatedCostUsd, 1.686071); + assert.equal(reconciled.receipt.cliCostUsd, 2.529107); + assert.ok(reconciled.receipt.usage); + assert.equal(reconciled.receipt.usage.cacheWrite1h, 3405); +}); + +test('missing or incomplete broker ledgers fail closed with a conservative receipt', () => { + const missing = reconcileCredentialBrokerReceipt({ ledger: null, + cliResult: { type: 'result', is_error: false, total_cost_usd: 0.25 }, + model: 'test-model', maxBudgetUsd: 1, pricingRates: PRICING_RATES }); + assert.equal(missing.ok, false); + assert.equal(missing.result.total_cost_usd, 1); + assert.equal(missing.result.stack_bench_cost_receipt.complete, false); + + const incomplete = reconcileCredentialBrokerReceipt({ ledger: { + schemaVersion: 5, model: 'test-model', maxBudgetUsd: 1, + acceptedRequests: 1, billableRequests: 1, completedBillableRequests: 0, + estimatedBillableRequests: 0, estimatedByReason: NO_ESTIMATES, + unpricedBillableRequests: 0, unpricedByReason: NO_UNPRICED, + spentUsd: 0.1, reservedUsd: 0.4, usage: ZERO_USAGE, complete: false, + updatedAt: new Date().toISOString(), + }, cliResult: { type: 'result', is_error: false, total_cost_usd: 0.1 }, + model: 'test-model', maxBudgetUsd: 1, pricingRates: PRICING_RATES }); + assert.equal(incomplete.ok, false); + assert.equal(incomplete.result.total_cost_usd, 0.5); + assert.match(incomplete.receipt.error ?? '', /incomplete/); +}); + +test('receipt reconciliation accepts provider usage that includes CLI-omitted calls', () => { + const cases = [{ + ledger: { schemaVersion: 5, model: 'test-model', maxBudgetUsd: 1, + acceptedRequests: 2, billableRequests: 2, completedBillableRequests: 2, + estimatedBillableRequests: 0, estimatedByReason: NO_ESTIMATES, + unpricedBillableRequests: 0, unpricedByReason: NO_UNPRICED, + spentUsd: 0.00201, reservedUsd: 0, + usage: { input: 120, output: 110, cacheRead: 0, cacheWrite5m: 0, cacheWrite1h: 0 }, + complete: true, + updatedAt: new Date().toISOString() }, + cliResult: { type: 'result', is_error: false, total_cost_usd: 0.00351, + usage: ONE_REQUEST_USAGE }, + model: 'test-model', maxBudgetUsd: 1, pricingRates: PRICING_RATES, + usage: { + input_tokens: 120, output_tokens: 110, cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + cache_creation: { ephemeral_5m_input_tokens: 0, ephemeral_1h_input_tokens: 0 }, + }, + }, { + // Real paid-session totals. + ledger: { schemaVersion: 5, model: 'claude-sonnet-5', maxBudgetUsd: 50, + acceptedRequests: 97, billableRequests: 97, completedBillableRequests: 97, + estimatedBillableRequests: 0, estimatedByReason: NO_ESTIMATES, + unpricedBillableRequests: 0, unpricedByReason: NO_UNPRICED, + spentUsd: 2.665346, reservedUsd: 0, + usage: { input: 2647, output: 77041, cacheRead: 8026721, + cacheWrite5m: 113719, cacheWrite1h: 0 }, + complete: true, updatedAt: new Date().toISOString() }, + cliResult: { type: 'result', is_error: false, total_cost_usd: 3.998019, + usage: { input_tokens: 156, output_tokens: 77016, + cache_read_input_tokens: 8026721, cache_creation_input_tokens: 113719 } }, + model: 'claude-sonnet-5', maxBudgetUsd: 50, + pricingRates: { input: 2, output: 10, cacheWrite5m: 2.5, cacheWrite1h: 4, cacheRead: 0.2 }, + usage: { + input_tokens: 2647, output_tokens: 77041, cache_read_input_tokens: 8026721, + cache_creation_input_tokens: 113719, + cache_creation: { ephemeral_5m_input_tokens: 113719, ephemeral_1h_input_tokens: 0 }, + }, + }]; + for (const { usage, ...input } of cases) { + const reconciled = reconcileCredentialBrokerReceipt(input); + assert.equal(reconciled.ok, true, reconciled.receipt.error ?? undefined); + assert.equal(reconciled.receipt.costUsd, input.ledger.spentUsd); + assert.deepEqual(reconciled.result.usage, usage); + } +}); + +test('receipt reconciliation rejects provider usage below the CLI lower bound', () => { + const ledger = { schemaVersion: 5, model: 'test-model', maxBudgetUsd: 1, + acceptedRequests: 1, billableRequests: 1, completedBillableRequests: 1, + estimatedBillableRequests: 0, estimatedByReason: NO_ESTIMATES, + unpricedBillableRequests: 0, unpricedByReason: NO_UNPRICED, + spentUsd: 0.0018, reservedUsd: 0, + usage: { input: 100, output: 100, cacheRead: 0, cacheWrite5m: 0, cacheWrite1h: 0 }, + complete: true, updatedAt: new Date().toISOString() }; + const reconciled = reconcileCredentialBrokerReceipt({ ledger, + cliResult: { type: 'result', is_error: false, total_cost_usd: 0.0018, + usage: { ...ONE_REQUEST_USAGE, output_tokens: 101 } }, + model: 'test-model', maxBudgetUsd: 1, pricingRates: PRICING_RATES }); + assert.equal(reconciled.ok, false); + assert.match(reconciled.receipt.error ?? '', /usage is lower than CLI usage/); +}); + +test('estimated receipts cover both recorders without making missing usage exact', () => { + const ledger = { schemaVersion: 5, model: 'test-model', maxBudgetUsd: 1, + acceptedRequests: 2, billableRequests: 2, completedBillableRequests: 2, + estimatedBillableRequests: 1, estimatedByReason: { ...NO_ESTIMATES, 'no-usage': 1 }, + unpricedBillableRequests: 0, unpricedByReason: NO_UNPRICED, + spentUsd: 0.01, reservedUsd: 0, + usage: { input: 100, output: 100, cacheRead: 0, cacheWrite5m: 0, cacheWrite1h: 0 }, + complete: true, updatedAt: new Date().toISOString() }; + const reconcile = (candidate: unknown, output: number, input = 100) => reconcileCredentialBrokerReceipt({ + ledger: candidate, cliResult: { type: 'result', is_error: false, total_cost_usd: 0.01, + usage: { ...ONE_REQUEST_USAGE, input_tokens: input, output_tokens: output } }, + model: 'test-model', maxBudgetUsd: 1, pricingRates: PRICING_RATES, + }); + const covered = reconcile(ledger, 200); + assert.equal(covered.ok, true); + assert.equal(covered.receipt.exact, false); + assert.equal(covered.receipt.costUsd, 0.01); + assert.equal(covered.receipt.usage?.output, 100); + assert.equal(covered.receipt.estimatedRequests, 1); + const exceeded = reconcile(ledger, 10000); + assert.equal(exceeded.ok, false); + assert.match(exceeded.receipt.error ?? '', /exceeds credential broker spend ceiling/); + // Each recorder alone fits, but their combined observed components do not. + assert.equal(reconcile({ ...ledger, spentUsd: 0.003 }, 200, 0).ok, false); +}); + +test('receipt reconciliation rejects spend that cannot be reproduced from broker usage', () => { + const ledger = { schemaVersion: 5, model: 'test-model', maxBudgetUsd: 1, + acceptedRequests: 1, billableRequests: 1, completedBillableRequests: 1, + estimatedBillableRequests: 0, estimatedByReason: NO_ESTIMATES, + unpricedBillableRequests: 0, unpricedByReason: NO_UNPRICED, + spentUsd: 0.002, reservedUsd: 0, + usage: { input: 100, output: 100, cacheRead: 0, cacheWrite5m: 0, cacheWrite1h: 0 }, + complete: true, updatedAt: new Date().toISOString() }; + const reconciled = reconcileCredentialBrokerReceipt({ ledger, + cliResult: { type: 'result', is_error: false, total_cost_usd: 0.002, + usage: ONE_REQUEST_USAGE }, + model: 'test-model', maxBudgetUsd: 1, pricingRates: PRICING_RATES }); + assert.equal(reconciled.ok, false); + assert.match(reconciled.receipt.error ?? '', /usage-priced spend/); +}); + +test('credential broker lifecycle removes credentials and retains its spend ledger', async t => { + const providerCredential = 'provider-secret-value-1234567890'; + const rates = { input: 3, output: 15, cacheWrite5m: 3.75, cacheWrite1h: 6, cacheRead: 0.3 }; + const broker = await startCredentialBroker({ mode: 'api-key', credential: providerCredential }, + { networkMode: 'host', deadlineMs: 10_000, model: 'test-model', + maxBudgetUsd: 1, pricingRates: rates }); + t.after(() => rmSync(broker.root, { recursive: true, force: true })); + let ledger: BrokerLedger | null = null; + try { + assert.equal(existsSync(broker.root), true); + assert.equal(broker.listenHost, '127.0.0.1'); + assert.doesNotMatch(broker.baseUrl, new RegExp(providerCredential)); + assert.match(broker.sessionToken, /^[a-f0-9]{64}$/); + assert.equal((await send(Number(new URL(broker.baseUrl).port))).status, 401); + } finally { + ledger = await stopCredentialBroker(broker); + } + assert.deepEqual(readdirSync(broker.root), ['spend-ledger.json']); + assert.deepEqual(readCredentialBrokerLedger(broker.ledgerPath), ledger); + assert.doesNotMatch(readFileSync(broker.ledgerPath, 'utf8'), + new RegExp(`${providerCredential}|${broker.sessionToken}`)); + assert.ok(ledger); + assert.equal(ledger.complete, true); + assert.equal(ledger.spentUsd, 0); + const diagnostics = credentialBrokerDiagnostics(broker); + assert.ok(diagnostics?.drain && diagnostics.ledger); + assert.equal(diagnostics.endpointKind, 'local-credential-broker'); + assert.equal(diagnostics.drain.timedOut, false); + assert.equal(diagnostics.ledger.complete, true); + assert.equal(diagnostics.ledger.spentUsd, 0); +}); + +test('credential broker diagnostics retain child exit, stderr, and the final ledger', async t => { + const providerCredential = 'provider-secret-value-1234567890'; + const broker = await startCredentialBroker({ mode: 'api-key', credential: providerCredential }, + { networkMode: 'host', deadlineMs: 10_000, model: 'test-model' }); + t.after(() => rmSync(broker.root, { recursive: true, force: true })); + const providerCut = 11; + const sessionCut = 19; + const stderr = broker.child.stderr; + assert.ok(stderr); + stderr.emit('data', Buffer.from( + `${'x'.repeat(20_000)} broker failure ${providerCredential.slice(0, providerCut)}`)); + stderr.emit('data', Buffer.from( + `${providerCredential.slice(providerCut)} ${broker.sessionToken.slice(0, sessionCut)}`)); + stderr.emit('data', Buffer.from( + `${broker.sessionToken.slice(sessionCut)}\n`)); + broker.child.kill(); + await once(broker.child, 'exit'); + const ledger = await stopCredentialBroker(broker, { drainTimeoutMs: 0 }); + const diagnostics = credentialBrokerDiagnostics(broker); + assert.ok(ledger); + assert.ok(diagnostics?.drain && diagnostics.termination && diagnostics.ledger); + assert.equal(ledger.complete, true); + assert.ok(diagnostics.child.exitCode !== null || diagnostics.child.signal !== null); + assert.match(diagnostics.child.stderrTail ?? '', /broker failure \[REDACTED\] \[REDACTED\]/); + assert.equal(diagnostics.child.stderrTruncated, true); + assert.doesNotMatch(JSON.stringify(diagnostics), new RegExp(providerCredential)); + assert.doesNotMatch(JSON.stringify(diagnostics), new RegExp(broker.sessionToken)); + assert.doesNotMatch(diagnostics.child.stderrTail ?? '', + new RegExp(providerCredential.slice(0, providerCut))); + assert.doesNotMatch(diagnostics.child.stderrTail ?? '', + new RegExp(broker.sessionToken.slice(0, sessionCut))); + assert.equal(diagnostics.drain.terminationRequested, false); + assert.equal(diagnostics.termination.exited, true); + assert.equal(diagnostics.ledger.complete, true); +}); + +test('credential broker closes an active request after its parent exits', + { timeout: 10_000 }, async () => { + const moduleUrl = pathToFileURL(compiledEntrypoint('container', 'credential-broker-process.js')).href; + const script = ` + import { startCredentialBroker } from ${JSON.stringify(moduleUrl)}; + const broker = await startCredentialBroker({ mode: 'api-key', + credential: 'provider-secret-value-1234567890' }, + { networkMode: 'host', deadlineMs: 10000, model: 'test-model' }); + process.stdout.write(JSON.stringify({ pid: broker.child.pid, baseUrl: broker.baseUrl, + sessionToken: broker.sessionToken, root: broker.root, ledgerPath: broker.ledgerPath }) + '\\n'); + process.stdin.once('data', () => process.exit(0)); + process.stdin.resume(); + `; + const parent = spawn(process.execPath, ['--input-type=module', '--eval', script], { + stdio: ['pipe', 'pipe', 'pipe'], windowsHide: true, + }); + let active: Socket | null = null; + let ready: BrokerReady | null = null; + const alive = (pid: number): boolean => { + try { process.kill(pid, 0); return true; } + catch (error) { return !record(error) || error.code !== 'ESRCH'; } + }; + try { + ready = await new Promise((resolveReady, reject) => { + let output = ''; + parent.stdout.on('data', (chunk: Buffer) => { + output += chunk.toString('utf8'); + const newline = output.indexOf('\n'); + if (newline === -1) return; + try { resolveReady(brokerReady(JSON.parse(output.slice(0, newline)))); } + catch (error) { reject(error); } + }); + parent.once('error', reject); + parent.once('exit', code => reject(new Error(`broker parent exited before ready (${code})`))); + }); + const brokerInfo = ready; + const port = Number(new URL(brokerInfo.baseUrl).port); + active = connect({ host: '127.0.0.1', port }); + active.on('error', () => {}); + await once(active, 'connect'); + const activeClosed = new Promise(resolveClosed => + active?.once('close', () => resolveClosed())); + active.write('POST /v1/messages HTTP/1.1\r\n' + + `Host: 127.0.0.1:${port}\r\n` + + `Authorization: Bearer ${brokerInfo.sessionToken}\r\n` + + 'Content-Type: application/json\r\nContent-Length: 1000\r\n\r\n' + + '{"model":"test-model"'); + await waitFor(() => readCredentialBrokerLedger(brokerInfo.ledgerPath, + { model: 'test-model', maxBudgetUsd: null }).acceptedRequests === 1); + parent.stdin.end('exit'); + await once(parent, 'exit'); + const started = Date.now(); + await waitFor(() => !alive(brokerInfo.pid), 4_000); + assert.ok(Date.now() - started < 4_000); + await Promise.race([activeClosed, new Promise((_, reject) => setTimeout( + () => reject(new Error('active broker socket stayed open')), 1_000))]); + await new Promise((resolveProbe, reject) => { + const probe = connect({ host: '127.0.0.1', port }); + const timeout = setTimeout(() => { + probe.destroy(); + reject(new Error('broker port did not close')); + }, 1_000); + probe.once('error', () => { clearTimeout(timeout); resolveProbe(); }); + probe.once('connect', () => { + clearTimeout(timeout); + probe.destroy(); + reject(new Error('broker remained reachable after parent exit')); + }); + }); + } finally { + active?.destroy(); + if (parent.exitCode === null) parent.kill(); + if (ready && alive(ready.pid)) { + try { process.kill(ready.pid, 'SIGKILL'); } catch { /* already stopped */ } + } + if (ready?.root) rmSync(ready.root, { recursive: true, force: true }); + } + }); + +test('credential broker shutdown retains settled spend even if its caller later loses the receipt', async t => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-broker-drain-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + writeFileSync(join(root, 'config.json'), 'private-credential'); + writeFileSync(join(root, 'ready.json'), 'private-session'); + const incomplete = { schemaVersion: 5, model: 'test-model', maxBudgetUsd: 1, + acceptedRequests: 1, billableRequests: 1, completedBillableRequests: 0, + estimatedBillableRequests: 0, estimatedByReason: NO_ESTIMATES, + unpricedBillableRequests: 0, unpricedByReason: NO_UNPRICED, + spentUsd: 0, reservedUsd: 0.4, usage: ZERO_USAGE, complete: false, + updatedAt: new Date().toISOString() }; + const complete = { ...incomplete, completedBillableRequests: 1, + spentUsd: 0.0018, reservedUsd: 0, complete: true }; + let reads = 0; + let slept = 0; + let graceful = null; + let alive = true; + let clock = 0; + const broker: CredentialBrokerHandle = { root, ledgerPath: join(root, 'spend-ledger.json'), + model: 'test-model', maxBudgetUsd: 1, child: { pid: 42 } }; + const ledger = await stopCredentialBroker(broker, { + drainTimeoutMs: 1_000, + pollMs: 100, + readLedger: path => { + writeCredentialBrokerLedger(path, ++reads < 3 ? incomplete : complete); + return readCredentialBrokerLedger(path); + }, + requestStop: child => { graceful = child.pid; alive = false; }, + alive: () => alive, + sleep: async ms => { slept += ms; clock += ms; }, + now: () => clock, + }); + assert.ok(ledger); + assert.equal(ledger.complete, true); + assert.equal(reads, 4, 'the final read must happen after termination'); + assert.equal(slept, 200); + assert.equal(graceful, 42); + assert.equal(broker.finalDiagnostics?.termination?.forceRequested, false); + // No run artifact or in-memory receipt is needed to recover settled spend. + broker.finalLedger = null; + broker.finalDiagnostics = null; + assert.deepEqual(readdirSync(root), ['spend-ledger.json']); + assert.deepEqual(readCredentialBrokerLedger(broker.ledgerPath, + { model: 'test-model', maxBudgetUsd: 1 }), complete); +}); + +test('credential broker forces termination and records typed shutdown errors', async t => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-broker-force-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + let alive = true; + let forced = null; + let clock = 0; + const broker: CredentialBrokerHandle = { root, ledgerPath: join(root, 'missing-ledger.json'), + model: 'test-model', maxBudgetUsd: 1, child: { pid: 43 }, processState: {} }; + const ledger = await stopCredentialBroker(broker, { + drainTimeoutMs: 0, gracefulTimeoutMs: 100, forceTimeoutMs: 100, pollMs: 25, + readLedger: () => { throw new Error('ledger unavailable'); }, + requestStop: () => {}, + terminate: pid => { forced = pid; alive = false; }, + alive: () => alive, + sleep: async ms => { clock += ms; }, + now: () => clock, + }); + assert.equal(ledger, null); + assert.equal(forced, 43); + assert.deepEqual(broker.finalDiagnostics?.termination, { + gracefulRequested: true, forceRequested: true, exited: true, + gracefulTimeoutMs: 100, forceTimeoutMs: 100, + }); + assert.deepEqual(broker.finalDiagnostics?.errors, [ + { type: 'ledger-read-error', phase: 'drain', message: 'ledger unavailable' }, + { type: 'ledger-read-error', phase: 'final', message: 'ledger unavailable' }, + ]); + assert.deepEqual(readdirSync(root), []); +}); + + +test('OpenAI broker isolates credentials, bounds requests and reconciles cached usage without CLI dollars', async () => { + for (const model of ['gpt-5.6-sol', 'gpt-6-sol', 'gpt-6-astra']) { + assert.doesNotThrow(() => createCredentialBroker({ provider: 'openai', mode: 'subscription-token', + accountId: 'account', model, maxOutputTokens: 4096, + credential: 'provider-secret-value-1234567890', sessionToken: 'session-token-value-1234567890' })); + } + assert.throws(() => createCredentialBroker({ provider: 'openai', mode: 'subscription-token', + accountId: 'account', model: 'constructor', maxOutputTokens: 4096, + credential: 'provider-secret-value-1234567890', sessionToken: 'session-token-value-1234567890' }), + /no verified output-token bound/); + assert.doesNotThrow(() => createCredentialBroker({ provider: 'openai', mode: 'subscription-token', + accountId: 'account', model: 'new-account-model', maxOutputTokens: 4096, + explicitOutputLimit: true, credential: 'provider-secret-value-1234567890', + sessionToken: 'session-token-value-1234567890' })); + const root = mkdtempSync(join(tmpdir(), 'openai-broker-')); + const ledgerPath = join(root, 'ledger.json'); + const rates = { input: 1.75, output: 14, cacheRead: 0.175, cacheWrite5m: 0, cacheWrite1h: 0 }; + const usage = { input_tokens: 100, output_tokens: 10, input_tokens_details: { cached_tokens: 80 } }; + try { + await withBroker('api-key', async ({ brokerPort, sessionToken, credential, seen }) => { + const request = { path: '/v1/responses', headers: { authorization: `Bearer ${sessionToken}`, + 'chatgpt-account-id': 'spoofed', 'openai-project': 'spoofed' }, + body: JSON.stringify({ model: 'test-model', input: 'hello' }) }; + assert.equal((await send(brokerPort, request)).status, 200); + assert.equal(seen[0]!.headers.authorization, `Bearer ${credential}`); + assert.equal(seen[0]!.headers['x-api-key'], undefined); + assert.equal(seen[0]!.headers['chatgpt-account-id'], undefined); + assert.equal(seen[0]!.headers['openai-project'], undefined); + assert.equal(JSON.parse(seen[0]!.body).max_output_tokens, 4096); + assert.equal(JSON.parse(seen[0]!.body).service_tier, 'default'); + const ledger = readCredentialBrokerLedger(ledgerPath); + assert.deepEqual(ledger.usage, { input: 20, output: 10, cacheRead: 80, cacheWrite5m: 0, cacheWrite1h: 0 }); + const reconciled = reconcileCredentialBrokerReceipt({ ledger, provider: 'openai', + cliResult: { usage: { input_tokens: 20, output_tokens: 10, cache_read_input_tokens: 80, cache_creation_input_tokens: 0 } }, + model: 'test-model', maxBudgetUsd: 10, pricingRates: rates }); + assert.equal(reconciled.ok, true, reconciled.receipt.error ?? undefined); + assert.equal(reconciled.receipt.cliCostUsd, null); + assert.equal(reconciled.receipt.costUsd, 0.000189); + assert.equal(reconciled.receipt.exact, true); + // Features whose charges or input the token rates do not cover are forwarded and marked. + const unpriced: Array<[Record, UnpricedReason]> = [ + [{ tools: [{ type: 'file_search' }] }, 'hosted-tool'], + [{ input: [{ type: 'item_reference', id: 'hidden-input' }] }, 'unpriced-input'], + [{ input: [{ type: 'input_file', file_id: 'file-1' }] }, 'unpriced-input'], + [{ input: [{ type: 'input_image', image_url: 'https://example.com/image.png' }] }, 'unpriced-input'], + [{ audio: { voice: 'alloy' } }, 'unpriced-output'], + [{ prompt: { id: 'stored-prompt' } }, 'stored-context'], + [{ previous_response_id: 'resp_1' }, 'stored-context'], + [{ conversation: 'conv_1' }, 'stored-context'], + [{ service_tier: 'priority' }, 'service-tier'], + ]; + for (const [override] of unpriced) { + assert.equal((await send(brokerPort, { ...request, + body: JSON.stringify({ model: 'test-model', ...override }) })).status, 200, JSON.stringify(override)); + } + for (const override of [{ truncation: 'auto' }, { background: true }]) { + assert.equal((await send(brokerPort, { ...request, + body: JSON.stringify({ model: 'test-model', input: 'hello', ...override }) })).status, 200); + } + assert.equal(seen.length, unpriced.length + 3); + assert.equal(JSON.parse(seen.at(-3)!.body).service_tier, 'priority'); + const marked = readCredentialBrokerLedger(ledgerPath); + const expected = noUnpriced(); + for (const [, reason] of unpriced) expected[reason] += 1; + assert.equal(marked.unpricedBillableRequests, unpriced.length); + assert.deepEqual(marked.unpricedByReason, expected); + const receipt = reconcileCredentialBrokerReceipt({ ledger: marked, provider: 'openai', + cliResult: { usage: { input_tokens: 20, output_tokens: 10, cache_read_input_tokens: 80, cache_creation_input_tokens: 0 } }, + model: 'test-model', maxBudgetUsd: 10, pricingRates: rates }); + assert.equal(receipt.ok, true, receipt.receipt.error ?? undefined); + assert.equal(receipt.receipt.exact, false); + assert.equal(receipt.receipt.unpricedRequests, unpriced.length); + assert.equal(receipt.receipt.costUsd, 0.002268); + }, { provider: 'openai', ledgerPath, pricingRates: rates, maxBudgetUsd: 10, + upstreamBody: `event: response.completed\ndata: ${JSON.stringify({ type: 'response.completed', response: { usage } })}\n\n` }); + await withBroker('subscription-token', async ({ brokerPort, sessionToken, seen, stats }) => { + assert.equal((await send(brokerPort, { path: '/v1/responses', + headers: { authorization: `Bearer ${sessionToken}`, 'chatgpt-account-id': 'spoofed' }, + body: JSON.stringify({ model: 'gpt-5.3-codex', input: 'hello' }) })).status, 200); + assert.equal(seen[0]!.url, '/backend-api/codex/responses'); + assert.equal(seen[0]!.headers['chatgpt-account-id'], 'trusted-account'); + assert.equal(JSON.parse(seen[0]!.body).max_output_tokens, undefined); + assert.equal(stats().estimatedBillableRequests, 1); + assert.equal(readCredentialBrokerLedger(ledgerPath).providerFailure?.category, 'transport'); + assert.ok(stats().spentUsd >= 128_000 * rates.output / 1e6); + }, { provider: 'openai', ledgerPath, accountId: 'trusted-account', model: 'gpt-5.3-codex', + pricingRates: rates, maxBudgetUsd: 10, + upstreamBody: 'event: response.created\ndata: {"type":"response.created"}\n\n' }); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + + +test('broker persists structured quota rejection without recording secret error text', async () => { + const root = mkdtempSync(join(tmpdir(), 'provider-quota-')); + const ledgerPath = join(root, 'ledger.json'); + try { + await withBroker('api-key', async ({ brokerPort, sessionToken }) => { + assert.equal((await send(brokerPort, { headers: { authorization: `Bearer ${sessionToken}` }, + body: JSON.stringify({ model: 'test-model', max_tokens: 10 }) })).status, 429); + const ledger = readCredentialBrokerLedger(ledgerPath); + assert.deepEqual(ledger.providerFailure, { category: 'quota', status: 429, code: 'insufficient_quota' }); + assert.equal(ledger.spentUsd, 0); + assert.equal(ledger.complete, true); + assert.ok(!readFileSync(ledgerPath, 'utf8').includes('do-not-record')); + }, { ledgerPath, maxBudgetUsd: 1, pricingRates: PRICING_RATES, upstreamStatus: 429, + upstreamBody: JSON.stringify({ error: { code: 'insufficient_quota', message: 'do-not-record' } }) }); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + + +test('OpenRouter broker freezes routing and records reported cost, not reservation pricing', async () => { + const root = mkdtempSync(join(tmpdir(), 'openrouter-broker-')); + const ledgerPath = join(root, 'ledger.json'); + const model = 'openai/test-model'; + const rates = { input: 3, output: 15, cacheRead: 0.3, cacheWrite5m: 6, cacheWrite1h: 6 }; + for (const invalid of [{ providerRoute: undefined }, { mode: 'subscription-token' }, + { maxBudgetUsd: undefined }, { model: 'openrouter/auto' }, { model: 'openai/test:nitro' }]) { + assert.throws(() => createCredentialBroker({ provider: 'openrouter', mode: 'api-key', providerRoute: 'openai', + credential: 'provider-secret-value-1234567890', sessionToken: 'session-token-value-1234567890', + model, maxOutputTokens: 4096, maxBudgetUsd: 10, pricingRates: rates, ...invalid }), /OpenRouter/); + } + const response = { object: 'response', status: 'completed', model, + usage: { input_tokens: 100, output_tokens: 10, input_tokens_details: { cached_tokens: 80 }, cost: 0.001 }, + openrouter_metadata: { requested: model, strategy: 'direct', attempt: 1, is_byok: false, + endpoints: { available: [{ model, provider: 'OpenAI', selected: true }] } } }; + try { + await withBroker('api-key', async ({ brokerPort, sessionToken, credential, seen }) => { + // Request fields observed from the pinned Codex runner, without its prompt. + const input = [{ type: 'message', role: 'user', content: [{ type: 'input_text', text: 'hello' }] }]; + const tools = [{ type: 'function', name: 'exec_command', parameters: { type: 'object', properties: {} } }]; + const request = { path: '/v1/responses', headers: { authorization: `Bearer ${sessionToken}`, + 'x-openrouter-provider': 'spoofed', 'x-api-key': 'spoofed', 'x-openrouter-metadata': 'disabled' }, + body: JSON.stringify({ model, instructions: 'Use the workspace.', input, tools, + tool_choice: 'auto', parallel_tool_calls: true, reasoning: { effort: 'low', summary: 'auto' }, + store: false, stream: true, include: ['reasoning.encrypted_content'], + prompt_cache_key: 'native-session', client_metadata: {} }) }; + for (const override of [{ provider: { allow_fallbacks: true } }, { models: ['other/model'] }, + { plugins: [{ id: 'web' }] }, { transforms: ['middle-out'] }, { preset: 'spoofed' }]) { + assert.equal((await send(brokerPort, { ...request, body: JSON.stringify({ model, ...override }) })).status, 400); + } + assert.equal((await send(brokerPort, request)).status, 200); + assert.equal(seen.length, 1); + assert.equal(seen[0]!.url, '/api/v1/responses'); + assert.equal(seen[0]!.headers.authorization, `Bearer ${credential}`); + assert.equal(seen[0]!.headers['x-api-key'], undefined); + assert.equal(seen[0]!.headers['x-openrouter-provider'], undefined); + assert.equal(seen[0]!.headers['x-openrouter-metadata'], 'enabled'); + const forwarded = JSON.parse(seen[0]!.body); + assert.deepEqual(forwarded.input, input); + assert.deepEqual(forwarded.tools, tools); + assert.deepEqual(forwarded.provider, { only: ['openai'], order: ['openai'], allow_fallbacks: false, + require_parameters: true, max_price: { prompt: 3, completion: 15, request: 0 } }); + assert.equal(forwarded.max_output_tokens, 4096); + assert.deepEqual(forwarded.transforms, []); + assert.deepEqual(forwarded.plugins, []); + const ledger = readCredentialBrokerLedger(ledgerPath); + assert.equal(ledger.spentUsd, 0.001); + assert.equal(ledger.providerReportedCostUsd, 0.001); + assert.deepEqual(ledger.upstreamProviders, ['OpenAI']); + const reconciled = reconcileCredentialBrokerReceipt({ ledger, provider: 'openrouter', + cliResult: { usage: ZERO_RAW_USAGE }, model, maxBudgetUsd: 10, pricingRates: rates }); + assert.equal(reconciled.ok, true, reconciled.receipt.error ?? undefined); + assert.equal(reconciled.receipt.costSource, 'provider-reported'); + assert.equal(reconciled.receipt.calculatedCostUsd, null); + assert.equal(reconciled.receipt.costUsd, 0.001); + assert.equal(reconciled.receipt.exact, true); + // Reported cost includes hosted features, so they stay priced. + for (const override of [{ tools: [{ type: 'web_search' }] }, { previous_response_id: 'resp_1' }, + { modalities: ['image'] }, { truncation: 'auto' }]) { + assert.equal((await send(brokerPort, { ...request, body: JSON.stringify({ model, input, ...override }) })).status, 200); + } + const hosted = readCredentialBrokerLedger(ledgerPath); + assert.equal(hosted.unpricedBillableRequests, 0); + assert.equal(hosted.spentUsd, 0.005); + const hostedReceipt = reconcileCredentialBrokerReceipt({ ledger: hosted, provider: 'openrouter', + cliResult: { usage: ZERO_RAW_USAGE }, model, maxBudgetUsd: 10, pricingRates: rates }); + assert.equal(hostedReceipt.ok, true, hostedReceipt.receipt.error ?? undefined); + assert.equal(hostedReceipt.receipt.exact, true); + }, { provider: 'openrouter', providerRoute: 'openai', model, ledgerPath, pricingRates: rates, maxBudgetUsd: 10, + upstreamBody: `data: ${JSON.stringify({ type: 'response.completed', response })}\n\n` }); + // A hosted tool's reported fee can exceed a token reservation without failing routing integrity. + await withBroker('api-key', async ({ brokerPort, sessionToken, seen }) => { + const request = { path: '/v1/responses', headers: { authorization: `Bearer ${sessionToken}` }, + body: JSON.stringify({ model, input: 'hello', tools: [{ type: 'web_search' }] }) }; + assert.equal((await send(brokerPort, request)).status, 200); + assert.equal((await send(brokerPort, request)).status, 200); + assert.equal(seen.length, 2); + assert.equal(readCredentialBrokerLedger(ledgerPath).providerIntegrityError, undefined); + }, { provider: 'openrouter', providerRoute: 'openai', model, ledgerPath, pricingRates: rates, maxBudgetUsd: 50, + upstreamBody: JSON.stringify({ ...response, usage: { ...response.usage, cost: 11 } }) }); + for (const changed of [ { usage: { ...response.usage, cost: undefined } }, + { openrouter_metadata: { ...response.openrouter_metadata, is_byok: true } }, + { openrouter_metadata: { ...response.openrouter_metadata, attempt: 2 } }, + { model: 'other/model' }, { usage: { ...response.usage, cost: 11 } } ]) { + await withBroker('api-key', async ({ brokerPort, sessionToken, seen }) => { + const request = { path: '/v1/responses', headers: { authorization: `Bearer ${sessionToken}` }, + body: JSON.stringify({ model, input: 'hello' }) }; + assert.equal((await send(brokerPort, request)).status, 200); + assert.equal((await send(brokerPort, request)).status, 502); + assert.equal(seen.length, 1); + const ledger = readCredentialBrokerLedger(ledgerPath); + const receipt = reconcileCredentialBrokerReceipt({ ledger, provider: 'openrouter', + cliResult: { usage: ZERO_RAW_USAGE }, model, maxBudgetUsd: 10, pricingRates: rates }); + assert.equal(receipt.ok, false); + if (changed.usage?.cost === 11) assert.equal(receipt.receipt.costUsd, 11, 'never clip actual spend'); + }, { provider: 'openrouter', providerRoute: 'openai', model, ledgerPath, pricingRates: rates, maxBudgetUsd: 10, + upstreamBody: JSON.stringify({ ...response, ...changed }) }); + } + } finally { rmSync(root, { recursive: true, force: true }); } +}); + + +test('OpenAI inline screenshots retain input and reserve bounded vision tokens', () => { + const image = { type: 'input_image', image_url: 'data:image/png;base64,aGVsbG8=', detail: 'original' }; + for (const model of ['gpt-6-astra', 'gpt-5.6-sol']) { + const protocol = brokerProtocol({ provider: 'openai', mode: 'subscription-token', + model, accountId: 'test' } as BrokerConfig); + const input = [{ type: 'custom_tool_call_output', output: [image] }]; + const request = protocol.parseRequest(Buffer.from(JSON.stringify({ model, input })), '/v1/responses'); + assert.deepEqual(request.input, input); + assert.equal(protocol.inputTokenAdjustment!(request), 36_001 - image.image_url.length); + assert.equal(imageTokenAdjustment([image, image], model), 2 * (36_001 - image.image_url.length)); + const large = { ...image, image_url: 'data:image/png;base64,' + 'A'.repeat(1_000_000) }; + const body = JSON.stringify({ model, input: [large] }); + assert.ok(Buffer.byteLength(body) + imageTokenAdjustment([large], model) < 37_000, + 'a large screenshot must not reserve its base64 bytes as text tokens'); + assert.deepEqual(protocol.requestPricing(request), { unpriced: null, requiresUsage: null, bounded: true }); + const remote = protocol.parseRequest(Buffer.from(JSON.stringify({ model, + input: [{ ...image, image_url: 'https://example.com/image.png' }] })), '/v1/responses'); + assert.equal(protocol.inputTokenAdjustment!(remote), 0); + assert.deepEqual(protocol.requestPricing(remote), { unpriced: 'unpriced-input', requiresUsage: 'unpriced-input', bounded: false }); + } + const unverified = brokerProtocol({ provider: 'openai', mode: 'subscription-token', + model: 'gpt-5.3-codex', accountId: 'test' } as BrokerConfig); + const inline = unverified.parseRequest(Buffer.from(JSON.stringify({ model: 'gpt-5.3-codex', input: [image] })), '/v1/responses'); + assert.deepEqual(unverified.requestPricing(inline), { unpriced: null, requiresUsage: 'unpriced-input', bounded: false }); +}); + +test('Sol screenshot receipts require usage, not an image reservation bound', async t => { + const root = mkdtempSync(join(tmpdir(), 'sol-priced-image-')); + const ledgerPath = join(root, 'ledger.json'); + const rates = { input: 2, output: 14, cacheRead: 0.2, cacheWrite5m: 0, cacheWrite1h: 0 }; + const usage = { input_tokens: 1000, output_tokens: 100, input_tokens_details: { cached_tokens: 500 } }; + const cases = [ + { name: 'complete usage', body: { object: 'response', status: 'completed', usage }, unknown: false }, + { name: 'missing usage', body: { object: 'response', status: 'completed' }, unknown: true }, + { name: 'invalid usage', body: { object: 'response', status: 'completed', + usage: { ...usage, input_tokens: -1 } }, unknown: true }, + { name: 'HTTP refusal has no usage charge', body: { error: { message: 'unavailable' } }, status: 503, unknown: false }, + { name: 'hosted tool', body: { object: 'response', status: 'completed', usage }, + tools: [{ type: 'file_search' }], unknown: true }, + // A web search is priced from tokens until the response shows one ran. + { name: 'unused web search', body: { object: 'response', status: 'completed', usage, output: [] }, + tools: [{ type: 'web_search' }], unknown: false }, + { name: 'web search', body: { object: 'response', status: 'completed', usage, + output: [{ id: 'ws_1', type: 'web_search_call', status: 'completed' }] }, + tools: [{ type: 'web_search' }], unknown: true, searches: 1 }, + ]; + const evidence: unknown[] = []; + try { for (const row of cases) await t.test(row.name, async () => { + await withBroker('subscription-token', async ({ brokerPort, sessionToken, seen }) => { + const input = [{ type: 'custom_tool_call_output', output: [{ type: 'input_image', + image_url: 'data:image/png;base64,aGVsbG8=', detail: 'original' }] }]; + const response = await send(brokerPort, { path: '/v1/responses', + headers: { authorization: `Bearer ${sessionToken}` }, body: JSON.stringify({ model: 'gpt-6-sol', input, + ...(row.tools ? { tools: row.tools } : {}) }) }); + assert.equal(response.status, row.status ?? 200); + assert.deepEqual(JSON.parse(seen[0]!.body).input, input); + const ledger = readCredentialBrokerLedger(ledgerPath); + const reconciled = reconcileCredentialBrokerReceipt({ ledger, provider: 'openai', + cliResult: { usage: ZERO_RAW_USAGE }, model: 'gpt-6-sol', maxBudgetUsd: 2, pricingRates: rates }); + assert.equal(reconciled.ok, true, reconciled.receipt.error ?? undefined); + evidence.push({ name: row.name, ledger, receipt: reconciled.receipt, expectedUnknown: row.unknown }); + assert.equal(reconciled.receipt.exact, !row.unknown); + if (!row.unknown) assert.equal(reconciled.receipt.costUsd, row.status ? 0 : 0.0025); + assert.equal(reconciled.receipt.unpricedRequests, row.unknown ? 1 : 0); + assert.equal(reconciled.receipt.webSearchCalls, row.searches ?? 0); + }, { provider: 'openai', model: 'gpt-6-sol', accountId: 'test', ledgerPath, + pricingRates: rates, maxBudgetUsd: 2, + upstreamBody: JSON.stringify(row.body), upstreamStatus: row.status ?? 200 }); + }); } finally { + if (process.env.STACK_BENCH_IMAGE_COST_EVIDENCE) writeFileSync(process.env.STACK_BENCH_IMAGE_COST_EVIDENCE, + JSON.stringify({ rerun: 'node --test --test-name-pattern="Sol screenshot receipts" dist/tests/credential-broker.test.js', + modelCalls: 0, rates, cases: evidence }, null, 2)); + rmSync(root, { recursive: true, force: true }); + } +}); + +test('output reservations honor API caps without assuming account endpoints enforce them', () => { + for (const model of ['gpt-6-astra', 'gpt-6-sol']) { + for (const mode of ['api-key', 'subscription-token'] as const) { + const protocol = brokerProtocol({ provider: 'openai', mode, model, + accountId: 'test', maxOutputTokens: 4096 } as BrokerConfig); + const payload = protocol.parseRequest(Buffer.from(JSON.stringify({ model, + input: 'hello', max_output_tokens: 1024, reasoning: { effort: 'medium' } })), '/v1/responses'); + assert.equal(protocol.outputLimit(payload), mode === 'api-key' ? 1024 : 128_000); + assert.deepEqual(payload.reasoning, { effort: 'medium' }); + assert.equal(payload.model, model); + } + } +}); + +test('a Docker broker treats a missing or stale controller heartbeat as controller loss', () => { + const root = mkdtempSync(join(tmpdir(), 'broker-heartbeat-')); + try { + const heartbeat = join(root, 'heartbeat'); + assert.equal(heartbeatStale(heartbeat, Date.now()), true, 'missing heartbeat'); + writeFileSync(heartbeat, ''); + assert.equal(heartbeatStale(heartbeat, Date.now()), false, 'fresh heartbeat'); + assert.equal(heartbeatStale(heartbeat, Date.now() + 61_000), true, 'stale heartbeat'); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('a SuperGrok broker signs chat proxy calls itself and forwards account reads unbilled', async () => { + const completed = `event: response.completed\ndata: ${JSON.stringify({ type: 'response.completed', response: { + object: 'response', status: 'completed', model: 'grok-4.6', + usage: { input_tokens: 100, input_tokens_details: { cached_tokens: 40 }, output_tokens: 10 } } })}\n\n`; + const call = (sessionToken: string, payload: Record) => ({ path: '/v1/responses', + headers: { authorization: `Bearer ${sessionToken}`, 'x-xai-token-auth': 'claimed', 'content-type': 'application/json' }, + body: JSON.stringify(payload) }); + await withBroker('subscription-token', async ({ brokerPort, sessionToken, credential, seen, stats }) => { + const reads = { authorization: `Bearer ${sessionToken}` }; + assert.equal((await send(brokerPort, { method: 'GET', path: '/v1/user', headers: reads, body: '' })).status, 200); + assert.equal((await send(brokerPort, { method: 'GET', path: '/v1/conversations', headers: reads, body: '' })).status, 404); + assert.equal((await send(brokerPort, { method: 'GET', path: '/v1/user', body: '' })).status, 401); + assert.equal((await send(brokerPort, call(sessionToken, { model: 'grok-4.7', max_output_tokens: 128_001, input: [] }))).status, 400); + assert.equal((await send(brokerPort, call(sessionToken, { model: 'grok-4.7', input: [] }))).status, 200); + assert.equal(JSON.parse(seen[1]!.body).max_output_tokens, undefined, 'an account call goes as sent'); + // The CLI compresses large bodies; the broker forwards them as plain JSON. + const compressed = call(sessionToken, { model: 'grok-4.7', input: [] }); + assert.equal((await send(brokerPort, { ...compressed, headers: { ...compressed.headers, 'content-encoding': 'zstd' }, + body: zstdCompressSync(Buffer.from(compressed.body)) })).status, 200); + assert.equal(JSON.parse(seen[2]!.body).model, 'grok-4.7'); + assert.equal(seen[2]!.headers['content-encoding'], undefined); + assert.equal((await send(brokerPort, { ...compressed, headers: { ...compressed.headers, 'content-encoding': 'br' } })).status, 400); + // A grok-4.7 session still titles itself on grok-4.6; only that capped call is accepted. + assert.equal((await send(brokerPort, call(sessionToken, { model: 'grok-4.6', max_output_tokens: 100, input: [] }))).status, 200); + assert.equal((await send(brokerPort, call(sessionToken, { model: 'grok-4.6', max_output_tokens: 101, input: [] }))).status, 400); + assert.equal((await send(brokerPort, call(sessionToken, { model: 'grok-4.5', input: [] }))).status, 400); + assert.deepEqual(seen.map(request => [request.method, request.url]), + [['GET', '/v1/user'], ['POST', '/v1/responses'], ['POST', '/v1/responses'], ['POST', '/v1/responses']]); + for (const request of seen) { + assert.equal(request.headers.authorization, `Bearer ${credential}`); + assert.equal(request.headers['x-xai-token-auth'], 'xai-grok-cli'); + } + await waitFor(() => stats().completedBillableRequests === 3); + assert.equal(stats().billableRequests, 3, 'account reads are not provider spend'); + }, { provider: 'xai', model: 'grok-4.7', maxOutputTokens: 128_000, upstreamBody: completed, + upstreamHeaders: { 'content-type': 'text/event-stream' }, maxBudgetUsd: 10, + pricingRates: { input: 2, output: 6, cacheRead: 0.5, cacheWrite5m: 2, cacheWrite1h: 2 } }); + await withBroker('api-key', async ({ brokerPort, sessionToken, credential, seen }) => { + assert.equal((await send(brokerPort, { method: 'GET', path: '/v1/user', + headers: { authorization: `Bearer ${sessionToken}` }, body: '' })).status, 404, 'an API key has no session service'); + assert.equal((await send(brokerPort, call(sessionToken, { model: 'grok-4.6', input: [] }))).status, 200); + assert.equal(JSON.parse(seen[0]!.body).max_output_tokens, 128_000, 'an API call gets the plan cap'); + assert.equal(seen[0]!.headers.authorization, `Bearer ${credential}`); + assert.equal(seen[0]!.headers['x-xai-token-auth'], undefined); + assert.equal(seen[0]!.headers['x-api-key'], undefined); + }, { provider: 'xai', model: 'grok-4.6', maxOutputTokens: 128_000 }); +}); + +test('a call still open when the broker stops is settled at its reserved ceiling, not left unpriced', async () => { + // The upstream sends headers, then never finishes the response. + const upstreamServer = createServer((request, response) => { + request.resume(); + request.on('end', () => { response.writeHead(200, { 'content-type': 'text/event-stream' }); response.write(': open\n\n'); }); + }); + const upstreamPort = await listen(upstreamServer); + const sessionToken = 'session-token-value-1234567890'; + const { server, stats, abandonInFlight } = createCredentialBroker({ provider: 'xai', mode: 'api-key', + credential: 'provider-secret-value-1234567890', sessionToken, model: 'grok-4.6', maxOutputTokens: 1000, maxBudgetUsd: 10, + pricingRates: { input: 2, output: 6, cacheRead: 0.5, cacheWrite5m: 2, cacheWrite1h: 2 } }, { + requestUpstream: httpRequest, upstream: { protocol: 'http:', hostname: '127.0.0.1', port: upstreamPort } }); + const brokerPort = await listen(server); + try { + const client = httpRequest({ hostname: '127.0.0.1', port: brokerPort, method: 'POST', path: '/v1/responses', + headers: { authorization: `Bearer ${sessionToken}`, 'content-type': 'application/json' } }); + client.on('error', () => {}); + client.end(JSON.stringify({ model: 'grok-4.6', input: [] })); + await waitFor(() => stats().billableRequests === 1 && stats().reservedUsd > 0); + const reserved = stats().reservedUsd; + abandonInFlight(); + const { acceptedRequests, billableRequests, completedBillableRequests, estimatedBillableRequests, spentUsd, reservedUsd } = stats(); + assert.deepEqual({ acceptedRequests, billableRequests, completedBillableRequests, estimatedBillableRequests, spentUsd, reservedUsd }, + { acceptedRequests: 1, billableRequests: 1, completedBillableRequests: 1, estimatedBillableRequests: 1, spentUsd: reserved, + reservedUsd: 0 }); + client.destroy(); + } finally { + server.closeAllConnections(); upstreamServer.closeAllConnections(); + await close(server); await close(upstreamServer); + } +}); diff --git a/tools/stack-bench/tests/credential-profiles.test.ts b/tools/stack-bench/tests/credential-profiles.test.ts new file mode 100644 index 00000000000..e54a61f0ecf --- /dev/null +++ b/tools/stack-bench/tests/credential-profiles.test.ts @@ -0,0 +1,171 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { resolveContainerAuth } from '../container/container-auth.js'; +import { readPinnedExecutionCredential, resolveExecutionCredentials, executionCredentialsSchema, validateExecutionCredentialTargets } + from '../src/agents/credential-profiles.js'; + +test('named execution credentials select explicitly, preserve other providers, and fail closed on drift', () => { + const root = mkdtempSync(join(tmpdir(), 'credential-profiles-')); + try { + const secretFile = join(root, 'secret'); + const registry = join(root, 'profiles.json'); + writeFileSync(secretFile, 'SYNTHETIC_SECRET_ONE'); + const profiles = { + first: { provider: 'anthropic', mode: 'subscription-token', secretFile, version: 'v1' }, + second: { provider: 'anthropic', mode: 'api-key', secretFile, version: 'v2' }, + openai: { provider: 'openai', mode: 'subscription-token', secretFile, version: 'v1' }, + router: { provider: 'openrouter', mode: 'api-key', secretFile, version: 'v1' }, + }; + writeFileSync(registry, JSON.stringify(profiles)); + const source = { STACK_BENCH_CREDENTIAL_PROFILES_FILE: registry, + ANTHROPIC_API_KEY: 'ambient', OPENAI_API_KEY_FILE: '/other-provider', + STACK_BENCH_API_KEY_FILE: '/generic', STACK_BENCH_AGENT_API_KEY: 'generic', STACK_BENCH_AGENT_API_KEY_FILE: '/stale-agent-key', PATH: '/bin' }; + assert.deepEqual(resolveExecutionCredentials('claude-code', 'a', undefined, source), { env: source, assignment: null }); + const selected = resolveExecutionCredentials('claude-code', 'a', { default: 'first', + adapters: { 'claude-code': 'second' }, attempts: { a: 'first' } }, source); + assert.deepEqual(selected.assignment, { id: 'first', version: 'v1', provider: 'anthropic', mode: 'subscription-token' }); + assert.equal(selected.env.CLAUDE_CODE_OAUTH_TOKEN_FILE, secretFile); + assert.equal(selected.env.ANTHROPIC_API_KEY, undefined); + assert.equal(selected.env.STACK_BENCH_API_KEY_FILE, undefined); + assert.equal(selected.env.STACK_BENCH_AGENT_API_KEY, undefined); + assert.equal(selected.env.STACK_BENCH_AGENT_API_KEY_FILE, undefined); + assert.equal(selected.env.OPENAI_API_KEY_FILE, '/other-provider'); + assert.equal(source.ANTHROPIC_API_KEY, 'ambient'); + assert.doesNotMatch(JSON.stringify(selected.assignment), /SYNTHETIC|secretFile/); + assert.doesNotThrow(() => readPinnedExecutionCredential(selected.env)); + const auth = resolveContainerAuth({ provider: 'anthropic', env: selected.env, + read: () => { throw new Error('must not reopen the secret file after checking its pin'); } }); + assert.equal(auth.credential, 'SYNTHETIC_SECRET_ONE'); + const raced = resolveContainerAuth({ provider: 'anthropic', env: selected.env, + exists: () => { writeFileSync(secretFile, 'SYNTHETIC_REPLACEMENT'); return true; } }); + assert.equal(raced.credential, 'SYNTHETIC_SECRET_ONE', 'broker uses the verified snapshot, not a replacement file'); + assert.throws(() => resolveContainerAuth({ provider: 'anthropic', env: selected.env }), /changed after admission/); + writeFileSync(secretFile, 'SYNTHETIC_SECRET_ONE'); + + const api = resolveExecutionCredentials('claude-code', 'b', { default: 'second' }, source); + assert.equal(resolveContainerAuth({ provider: 'anthropic', env: api.env, + apiKey: 'stale-caller-key' }).credential, 'SYNTHETIC_SECRET_ONE'); + assert.throws(() => resolveContainerAuth({ provider: 'openai', env: api.env }), /provider does not match/); + assert.doesNotThrow(() => validateExecutionCredentialTargets({ attempts: { a: 'first' } }, ['claude-code'], ['a'])); + assert.throws(() => validateExecutionCredentialTargets({ attempts: { typo: 'first' } }, ['claude-code'], ['a']), /unknown attempt/); + assert.throws(() => validateExecutionCredentialTargets({ adapters: { typo: 'first' } }, ['claude-code'], ['a']), /unknown adapter/); + + assert.equal(resolveExecutionCredentials('claude-code', 'b', { default: 'first', + adapters: { 'claude-code': 'second' } }, source).assignment?.id, 'second'); + assert.equal(resolveExecutionCredentials('codex', 'a', { default: 'openai' }, source).env.CODEX_AUTH_FILE, secretFile); + assert.equal(resolveExecutionCredentials('openrouter', 'a', { default: 'router' }, source).env.OPENROUTER_API_KEY_FILE, secretFile); + assert.throws(() => resolveExecutionCredentials('codex', 'a', { default: 'first' }, source), /does not match/); + assert.throws(() => resolveExecutionCredentials('claude-code', 'a', { default: 'absent' }, source), /not registered/); + assert.throws(() => executionCredentialsSchema.parse({ default: 'first', secret: 'unwanted' })); + writeFileSync(secretFile, 'SYNTHETIC_SECRET_TWO'); + assert.throws(() => readPinnedExecutionCredential(selected.env), /changed after admission/); + assert.throws(() => resolveContainerAuth({ provider: 'anthropic', env: selected.env }), /changed after admission/); + writeFileSync(secretFile, 'SYNTHETIC_SECRET_ONE'); + profiles.first.version = 'v3'; + writeFileSync(registry, JSON.stringify(profiles)); + assert.throws(() => readPinnedExecutionCredential(selected.env), /changed after admission/); + writeFileSync(registry, '{not-json SYNTHETIC_SECRET_TWO'); + assert.throws(() => resolveExecutionCredentials('claude-code', 'a', { default: 'first' }, source), + error => error instanceof Error && !error.message.includes('SYNTHETIC')); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('simultaneous account assignments keep independent secrets and drift checks', () => { + const root = mkdtempSync(join(tmpdir(), 'credential-isolation-')); + try { + const registry = join(root, 'profiles.json'); + const firstFile = join(root, 'first'); + const secondFile = join(root, 'second'); + writeFileSync(firstFile, 'SYNTHETIC_FIRST_ACCOUNT'); + writeFileSync(secondFile, 'SYNTHETIC_SECOND_ACCOUNT'); + writeFileSync(registry, JSON.stringify({ + first: { provider: 'anthropic', mode: 'api-key', secretFile: firstFile, version: 'v1' }, + second: { provider: 'anthropic', mode: 'api-key', secretFile: secondFile, version: 'v1' }, + })); + const source = { STACK_BENCH_CREDENTIAL_PROFILES_FILE: registry, ANTHROPIC_API_KEY: 'ambient' }; + const first = resolveExecutionCredentials('claude-code', 'a', { default: 'first' }, source); + const second = resolveExecutionCredentials('claude-code', 'a', { default: 'second' }, source); + assert.equal(readPinnedExecutionCredential(first.env)?.secret, 'SYNTHETIC_FIRST_ACCOUNT'); + assert.equal(readPinnedExecutionCredential(second.env)?.secret, 'SYNTHETIC_SECOND_ACCOUNT'); + writeFileSync(firstFile, 'SYNTHETIC_CHANGED_ACCOUNT'); + assert.throws(() => readPinnedExecutionCredential(first.env), /changed after admission/); + assert.equal(readPinnedExecutionCredential(second.env)?.secret, 'SYNTHETIC_SECOND_ACCOUNT'); + assert.deepEqual(source, { STACK_BENCH_CREDENTIAL_PROFILES_FILE: registry, ANTHROPIC_API_KEY: 'ambient' }); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('a Grok sign-in profile stays pinned to its account while its tokens rotate', () => { + const root = mkdtempSync(join(tmpdir(), 'credential-profiles-grok-')); + try { + const secretFile = join(root, 'grok_auth'); + const registry = join(root, 'profiles.json'); + const login = (key: string, principal = 'user-1') => JSON.stringify({ 'https://auth.x.ai::client': { key, + refresh_token: `refresh-${key}`, expires_at: '2026-10-06T23:00:00Z', principal_id: principal, + oidc_issuer: 'https://auth.x.ai', oidc_client_id: 'client' } }); + writeFileSync(secretFile, login('first')); + writeFileSync(registry, JSON.stringify({ grok: { provider: 'xai', mode: 'subscription-token', secretFile, version: 'v1' } })); + const selected = resolveExecutionCredentials('grok-build', 'a', { default: 'grok' }, + { STACK_BENCH_CREDENTIAL_PROFILES_FILE: registry, XAI_API_KEY: 'ambient' }); + assert.equal(selected.env.GROK_AUTH_FILE, secretFile); + assert.equal(selected.env.XAI_API_KEY, undefined); + writeFileSync(secretFile, login('refreshed')); + assert.equal(readPinnedExecutionCredential(selected.env)!.secret, login('refreshed')); + writeFileSync(secretFile, login('other', 'user-2')); + assert.throws(() => readPinnedExecutionCredential(selected.env), /changed after admission/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('an upstream profile is an Anthropic API key, pinned with its endpoint', () => { + const root = mkdtempSync(join(tmpdir(), 'credential-profiles-')); + try { + const secretFile = join(root, 'secret'); + const registry = join(root, 'profiles.json'); + writeFileSync(secretFile, 'SYNTHETIC_SECRET_ZAI'); + const profiles: Record = { + zai: { provider: 'anthropic', mode: 'api-key', upstream: 'zai', secretFile, version: 'v1' }, + bad: { provider: 'anthropic', mode: 'subscription-token', upstream: 'zai', secretFile, version: 'v1' }, + }; + writeFileSync(registry, JSON.stringify(profiles)); + const source = { STACK_BENCH_CREDENTIAL_PROFILES_FILE: registry }; + const selected = resolveExecutionCredentials('claude-code', 'a', { default: 'zai' }, source); + assert.deepEqual(selected.assignment, { id: 'zai', version: 'v1', provider: 'anthropic', mode: 'api-key', upstream: 'zai' }); + assert.deepEqual(resolveContainerAuth({ provider: 'anthropic', env: selected.env }), + { mode: 'api-key', credential: 'SYNTHETIC_SECRET_ZAI', upstream: 'zai' }); + assert.throws(() => resolveExecutionCredentials('claude-code', 'a', { default: 'bad' }, source), /invalid/); + profiles.zai = { ...profiles.zai as object, upstream: 'deepseek' }; + writeFileSync(registry, JSON.stringify(profiles)); + assert.throws(() => readPinnedExecutionCredential(selected.env), /changed after admission/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('a Google profile is a Gemini API key or an agy sign-in pinned by its account', () => { + const root = mkdtempSync(join(tmpdir(), 'credential-profiles-')); + try { + const secretFile = join(root, 'secret'); + const loginFile = join(root, 'agy_auth'); + const registry = join(root, 'profiles.json'); + const claims = Buffer.from(JSON.stringify({ iss: 'https://accounts.google.com', sub: 'user-1' })).toString('base64url'); + const login = (access: string) => JSON.stringify({ token: { access_token: access, refresh_token: '1//r', + expiry: '2026-10-09T13:00:00Z' }, auth_method: 'consumer', id_token: `h.${claims}.s` }); + writeFileSync(secretFile, 'SYNTHETIC_SECRET_GEMINI'); + writeFileSync(loginFile, login('ya29.one')); + writeFileSync(registry, JSON.stringify({ + gemini: { provider: 'google', mode: 'api-key', secretFile, version: 'v1' }, + account: { provider: 'google', mode: 'subscription-token', secretFile: loginFile, version: 'v1' } })); + const source = { STACK_BENCH_CREDENTIAL_PROFILES_FILE: registry }; + const selected = resolveExecutionCredentials('antigravity', 'a', { default: 'gemini' }, source); + assert.equal(selected.env.GEMINI_API_KEY_FILE, secretFile); + assert.deepEqual(resolveContainerAuth({ provider: 'google', env: selected.env }), + { provider: 'google', mode: 'api-key', credential: 'SYNTHETIC_SECRET_GEMINI' }); + const account = resolveExecutionCredentials('antigravity', 'a', { default: 'account' }, source); + assert.equal(account.env.AGY_AUTH_FILE, loginFile); + assert.deepEqual(resolveContainerAuth({ provider: 'google', env: account.env }), + { provider: 'google', mode: 'subscription-token', credential: 'ya29.one' }); + // A renewed token is the same account; the pin holds. + writeFileSync(loginFile, login('ya29.two')); + assert.equal(resolveContainerAuth({ provider: 'google', env: account.env }).credential, 'ya29.two'); + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/tools/stack-bench/tests/cross-origin-request.test.ts b/tools/stack-bench/tests/cross-origin-request.test.ts new file mode 100644 index 00000000000..c482a9b9163 --- /dev/null +++ b/tools/stack-bench/tests/cross-origin-request.test.ts @@ -0,0 +1,65 @@ +import assert from 'node:assert/strict'; +import { createServer } from 'node:http'; +import { once } from 'node:events'; +import test from 'node:test'; +import { chromium } from 'playwright'; +import { crossOriginPost } from '../src/actions/cross-origin-request.js'; +import { ActionInconclusive } from '../src/actions/action-contract.js'; +import { isFinding } from '../src/actions/action-findings.js'; +import { harnessBrowserFailure } from '../src/evidence/harness-errors.js'; + +test('real browser origin probes expose cookie writes despite opaque responses and keep missing responses unmeasured', async () => { + let protectOrigin = true, writes = 0; + const observed: { origin?: string; cookie?: string; authorization?: string }[] = []; + const server = createServer((req, res) => { + observed.push({ origin: req.headers.origin, cookie: req.headers.cookie, + authorization: req.headers.authorization }); + if (req.url === '/lost') { req.socket.destroy(); return; } + if (req.method !== 'POST' || req.headers.cookie !== 'sid=private-session' + || (protectOrigin && req.headers.origin !== `http://${req.headers.host}`)) { + res.writeHead(403, { 'Content-Type': 'application/json' }).end('{"error":"refused"}'); return; + } + // An application that blocks cross-origin reads with CORP still performs the write. + const policy = req.url === '/corp' ? { 'Cross-Origin-Resource-Policy': 'same-origin' } : {}; + writes++; res.writeHead(200, { 'Content-Type': 'application/json', ...policy }).end('{"ok":true}'); + }).listen(0, '0.0.0.0'); + await once(server, 'listening'); + const port = (server.address() as { port: number }).port; + const url = `http://127.0.0.1:${port}/buy`; + const browser = await chromium.launch({ headless: true }); + try { + const context = await browser.newContext(); + await context.addCookies([{ name: 'sid', value: 'private-session', url, + httpOnly: true, sameSite: 'Lax' }]); + const signal = AbortSignal.timeout(15_000); + const safe = await crossOriginPost(context, { url }, 'same-site', signal); + assert.equal(safe.cookieSent, true); assert.equal(safe.responseStatus, 403); assert.equal(writes, 0); + protectOrigin = false; + const unsafe = await crossOriginPost(context, { url }, 'same-site', signal); + assert.equal(unsafe.cookieSent, true); assert.equal(unsafe.responseStatus, 200); + assert.equal(unsafe.browserResponseType, 'opaque'); assert.equal(writes, 1); + const policyBlocked = await crossOriginPost(context, { url: url.replace('/buy', '/corp') }, 'same-site', signal); + assert.equal(policyBlocked.browserResponseType, 'blocked-by-policy'); + assert.equal(policyBlocked.responseStatus, 200); assert.equal(writes, 2); + const crossSite = await crossOriginPost(context, { url }, 'cross-site', signal); + assert.equal(crossSite.cookieSent, false); assert.equal(writes, 2); + await assert.rejects(crossOriginPost(context, { url: url.replace('/buy', '/lost') }, 'same-site', signal), + (error: unknown) => error instanceof ActionInconclusive + && isFinding(error.details.finding) && error.details.finding.kind === 'replay-unavailable' + && /complete browser request and response/.test(String(error.details.finding.fields.detail))); + assert.equal(context.pages().length, 0); + assert(observed.every(r => r.authorization === undefined)); + assert(!JSON.stringify({ safe, unsafe, policyBlocked, crossSite }).includes('private-session')); + await context.close(); + } finally { + await browser.close(); server.closeAllConnections(); await new Promise(resolve => server.close(() => resolve())); + } +}); + +test('a timeout loading the harness-served origin page is a harness failure', async () => { + const timeout = Object.assign(new Error('page.goto: Timeout 10000ms exceeded.'), { name: 'TimeoutError' }); + const page = { on() {}, close: async () => {}, route: async () => {}, goto: async () => { throw timeout; } }; + const context = { newPage: async () => page, grantPermissions: async () => {} } as never; + await assert.rejects(crossOriginPost(context, { url: 'http://127.0.0.1:18081/api' }, 'same-site', + new AbortController().signal), error => harnessBrowserFailure(error) !== null); +}); diff --git a/tools/stack-bench/tests/dashboard-check-guide.integration.ts b/tools/stack-bench/tests/dashboard-check-guide.integration.ts new file mode 100644 index 00000000000..e3e9820c24a --- /dev/null +++ b/tools/stack-bench/tests/dashboard-check-guide.integration.ts @@ -0,0 +1,82 @@ +/// +import assert from 'node:assert/strict'; +import { mkdirSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import test from 'node:test'; +import { chromium } from 'playwright'; +import { createDashboardServer } from '../dashboard/dashboard-server.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; + +// Failure cases: dead navigation; wrong default scope; missing repeated work; +// unreadable source; filters lost on reload; blank results; mobile overflow; +// script injection through search; accidental campaign/model work from this page. +test('check guide exposes current procedures through the real dashboard', async () => { + const evidence = join(STACK_BENCH_ROOT, 'local-notes', 'dashboard-check-guide-evidence', + new Date().toISOString().replace(/[:.]/g, '-')); + mkdirSync(evidence, { recursive: true }); + const { server } = createDashboardServer({ resultsRoot: join(evidence, 'results'), + plansRoot: join(evidence, 'plans'), allowLaunch: false, token: 'test' }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + assert(address && typeof address !== 'string'); + const browser = await chromium.launch(); + let result = 'failed'; + const errors: string[] = [], api: string[] = []; + let count = ''; + let definition = ''; + try { + const page = await browser.newPage({ viewport: { width: 1280, height: 950 } }); + page.on('pageerror', error => errors.push(String(error))); + page.on('request', request => { if (new URL(request.url()).pathname.startsWith('/api/')) api.push(request.url()); }); + await page.goto(`http://127.0.0.1:${address.port}/checks`); + await page.getByRole('heading', { name: 'Checks', exact: true }).waitFor(); + count = await page.locator('[data-guide-count]').innerText(); + definition = await page.locator('.guide-help .guide-key').textContent() ?? ''; + assert.equal(await page.getByText(/See the exact input below for its fields/).count(), 0, + 'all current actions have a readable description'); + assert((await page.locator('.guide-check:visible').count()) > 0); + assert.equal(await page.locator('.guide-check[data-active="false"]:visible').count(), 0); + await page.getByLabel('Search checks').fill('catalog-volume'); + assert.equal(await page.locator('.guide-check:visible').count(), 1); + await page.getByRole('button', { name: 'Expand visible' }).click(); + const check = page.locator('.guide-check:visible'); + assert.match(await check.innerText(), /Repeat 1000 times/); + await check.getByText('Technical details', { exact: true }).click(); + await page.evaluate(() => window.scrollTo(0, 0)); + const raw = JSON.parse(await check.locator('pre').innerText()) as { + criterion: { steps: { forEach?: unknown[] }[] }; + }; + assert.equal(raw.criterion.steps.find(step => step.forEach)?.forEach?.length, 1000); + assert.match(await check.innerText(), /dbExpectCatalogItem|committed-write barrier/); + await check.getByText('Technical details', { exact: true }).click(); + await page.screenshot({ path: join(evidence, 'desktop.png') }); + await page.reload(); + assert.equal(await page.getByLabel('Search checks').inputValue(), 'catalog-volume'); + await page.getByLabel('Search checks').fill(''); + await page.getByLabel('Include other selections').check(); + assert((await page.locator('.guide-check[data-active="false"]:visible').count()) > 0); + await page.getByLabel('Search checks').fill(''); + await page.getByText('No checks match this search.', { exact: true }).waitFor(); + assert.equal(await page.locator('.guide img').count(), 0); + await page.getByLabel('Search checks').fill('catalog-volume'); + await page.getByRole('button', { name: 'Expand visible' }).click(); + await page.setViewportSize({ width: 390, height: 844 }); + assert(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)); + await page.screenshot({ path: join(evidence, 'mobile.png') }); + assert.deepEqual(api, [], 'guide needs no campaign reads or model calls'); + await page.getByRole('link', { name: 'Campaigns', exact: true }).click(); + await page.getByRole('link', { name: 'Checks', exact: true }).click(); + await page.getByRole('heading', { name: 'Checks', exact: true }).waitFor(); + assert.deepEqual(errors, []); + result = 'passed'; + } finally { + await browser.close(); + await new Promise(resolve => server.close(() => resolve())); + writeFileSync(join(evidence, 'receipt.json'), JSON.stringify({ result, count, definition, errors, + runtime: { node: process.version, chromium: browser.version() }, + command: 'node --test dist/tests/dashboard-check-guide.integration.js', + inputs: 'Current local definitions; catalog-volume search; 1280px and 390px viewports', + browserClosed: true, serverClosed: true, + }, null, 2)); + } +}); diff --git a/tools/stack-bench/tests/dashboard-refresh.integration.ts b/tools/stack-bench/tests/dashboard-refresh.integration.ts new file mode 100644 index 00000000000..215c7d471ba --- /dev/null +++ b/tools/stack-bench/tests/dashboard-refresh.integration.ts @@ -0,0 +1,134 @@ +/// +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { chromium } from 'playwright'; +import { createDashboardServer } from '../dashboard/dashboard-server.js'; +import { campaignSheet } from '../dashboard/dashboard-views.js'; +import { EXAMPLE_CAMPAIGN, writeCampaign, writeRunEvidence } from './fixtures/dashboard-fixture.js'; +import { compileCampaignFile } from '../src/campaigns/campaign-compiler.js'; +import { claimNextAttempt, createCampaignState, finishCampaignExecution } from '../src/campaigns/campaign-scheduler.js'; + +test('dashboard refresh follows child events, coalesces bursts, and refreshes cached tabs', async t => { + const root = mkdtempSync(join(tmpdir(), 'dashboard-refresh-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const key = 'fixture-run-0'; + const plan = compileCampaignFile(EXAMPLE_CAMPAIGN); + const claimed = claimNextAttempt(createCampaignState(plan), { admissionId: 'test' }); + assert.ok(claimed.claim); + const state = finishCampaignExecution(claimed.state, claimed.claim.executionId, + { exitCode: 0, run: { outcome: { kind: 'passed' } } }); + const campaign = join(root, 'campaigns', key); + writeCampaign(campaign, plan, state); + writeRunEvidence(join(campaign, claimed.claim.output), plan, claimed.claim.attempt, 0); + const attemptId = campaignSheet(root, key).stacks[0]!.attempts[0]!.id; + const { server } = createDashboardServer({ resultsRoot: root, plansRoot: join(root, 'plans'), + allowLaunch: false, token: 'test' }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(() => server.close()); + const address = server.address(); + assert.ok(address && typeof address !== 'string'); + const browser = await chromium.launch({ headless: true }); + t.after(() => browser.close()); + const page = await browser.newPage(); + await page.addInitScript(() => { + const target = window as unknown as { events: EventTarget; hidden: boolean; EventSource: unknown; streams: number }; + target.hidden = false; + target.streams = 0; + Object.defineProperty(document, 'hidden', { get: () => target.hidden }); + target.EventSource = class extends EventTarget { + constructor() { super(); target.events = this; target.streams++; } + close() { target.streams--; } + }; + }); + const counts = new Map(); + let hold: Promise | null = null; + let transcriptHold: Promise | null = null; + let release = (): void => {}; + await page.route('**/api/**', async request => { + const path = new URL(request.request().url()).pathname; + counts.set(path, (counts.get(path) ?? 0) + 1); + if (path === `/api/campaigns/${key}` && hold) await hold; + if (path.endsWith('/transcript')) { + if (transcriptHold) await transcriptHold; + const session = new URL(request.request().url()).searchParams.get('session') || 'first'; + await request.fulfill({ json: { sessions: [{ id: 'first', label: 'First' }, { id: 'second', label: 'Second' }], + session, before: null, skipped: 0, messages: [{ id: session, role: 'assistant', text: `${session} message`, tool: false }] } }); + return; + } + await request.continue(); + }); + const idle = () => page.locator('main[aria-busy="false"]').waitFor(); + const change = (type = 'campaign', id: string | null = attemptId) => page.evaluate(({ type, key, id }) => { + (window as unknown as { events: EventTarget }).events.dispatchEvent( + new MessageEvent(type, { data: JSON.stringify({ key, ...(id ? { attemptId: id } : {}) }) })); + }, { type, key, id }); + await page.goto(`http://127.0.0.1:${address.port}/c/${key}`); + await idle(); + const campaignPath = `/api/campaigns/${key}`; + const before = counts.get(campaignPath)!; + hold = new Promise(resolve => { release = resolve; }); + const started = page.waitForRequest(request => new URL(request.url()).pathname === campaignPath); + await change(); + await started; + await Promise.all([change(), change(), change()]); + assert.equal(counts.get(campaignPath), before + 1); + hold = null; + release(); + await page.waitForResponse(response => new URL(response.url()).pathname === campaignPath); + await idle(); + assert.equal(counts.get(campaignPath), before + 2); + + const fullReads = counts.get(campaignPath); + const liveRead = page.waitForResponse(response => new URL(response.url()).pathname === `${campaignPath}/live`); + await change('log'); + await liveRead; + await idle(); + assert.equal(counts.get(campaignPath), fullReads, 'log changes fetch live fields, not the full evidence'); + + for (const [tab, endpoint] of [['checks', 'checks'], ['files', 'package']] as const) { + await page.goto(`http://127.0.0.1:${address.port}/c/${key}/a/${attemptId}?tab=${tab}`); + await idle(); + const path = `${campaignPath}/attempts/${attemptId}/${endpoint}`; + const previous = counts.get(path)!; + const updated = page.waitForResponse(response => new URL(response.url()).pathname === path); + await change('campaign', null); + await updated; + await idle(); + assert.equal(counts.get(path), previous + 1); + } + const previous = counts.get(campaignPath)!; + await page.evaluate(() => { + (window as unknown as { hidden: boolean }).hidden = true; + document.dispatchEvent(new Event('visibilitychange')); + }); + assert.equal(await page.evaluate(() => (window as unknown as { streams: number }).streams), 0); + await change(); + assert.equal(counts.get(campaignPath), previous); + const visible = page.waitForResponse(response => new URL(response.url()).pathname === campaignPath); + await page.evaluate(() => { + (window as unknown as { hidden: boolean }).hidden = false; + document.dispatchEvent(new Event('visibilitychange')); + }); + await visible; + await idle(); + assert.equal(counts.get(campaignPath), previous + 1); + await page.evaluate(() => document.dispatchEvent(new Event('visibilitychange'))); + assert.equal(await page.evaluate(() => (window as unknown as { streams: number }).streams), 1); + await idle(); + + await page.goto(`http://127.0.0.1:${address.port}/c/${key}/a/${attemptId}?tab=transcript`); + await idle(); + transcriptHold = new Promise(resolve => { release = resolve; }); + const reading = page.waitForRequest(request => new URL(request.url()).pathname.endsWith('/transcript')); + await change(); + await reading; + await page.locator('[data-transcript-session]').selectOption('second'); + const selected = page.waitForResponse(response => response.url().includes('session=second')); + transcriptHold = null; + release(); + await selected; + await page.getByText('second message', { exact: true }).waitFor(); +}); diff --git a/tools/stack-bench/tests/dashboard/arbitrary-stacks.test.ts b/tools/stack-bench/tests/dashboard/arbitrary-stacks.test.ts new file mode 100644 index 00000000000..3ee46d340bc --- /dev/null +++ b/tools/stack-bench/tests/dashboard/arbitrary-stacks.test.ts @@ -0,0 +1,26 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import type { OverviewCampaign } from '../../dashboard/dashboard-views.js'; +import { campaignsPage } from '../../dashboard/public/views/campaigns.js'; +import { compareCampaign } from '../../dashboard/public/metrics.js'; + +test('campaign history derives all stack columns from data, including new stacks', () => { + const stacks = ['custom-a', 'postgres', 'custom-b', 'spacetime', 'mongodb']; + const campaign: OverviewCampaign = { + key: 'five-stacks', id: 'five-stacks', title: 'Five stacks', status: 'completed', + mode: 'dependency', levels: [1], repetitions: 1, updatedAt: null, + scores: Object.fromEntries(stacks.map((stack, index) => [stack, 90 + index])), + attempts: { total: 5, running: 0, completed: 5 }, + }; + const input = { campaigns: [campaign], sheets: [] }; + const html = campaignsPage({ ...input, filter: 'all' }); + assert.equal((html.match(//g) ?? []).length, stacks.length); + assert.match(html, /custom-a<\/th>/); + assert.match(html, /custom-b<\/th>/); + assert.match(html, /94%<\/td>/); + + const comparison = compareCampaign({ attempts: stacks.map(stack => ({ + id: stack, stack, status: 'pending', execution: null, result: null, dependency: null, + })) }); + assert.deepEqual(comparison.rows.map(row => row.stack), stacks); +}); diff --git a/tools/stack-bench/tests/dashboard/campaign-cache.test.ts b/tools/stack-bench/tests/dashboard/campaign-cache.test.ts new file mode 100644 index 00000000000..c2ca987915a --- /dev/null +++ b/tools/stack-bench/tests/dashboard/campaign-cache.test.ts @@ -0,0 +1,70 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { syncBuiltinESMExports } from 'node:module'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { campaignSheet, campaignProgression } from '../../dashboard/dashboard-views.js'; +import { compileCampaignFile } from '../../src/campaigns/campaign-compiler.js'; +import { createCampaignState, claimNextAttempt } from '../../src/campaigns/campaign-scheduler.js'; +import { discoverPlans } from '../../dashboard/dashboard-model.js'; +import { EXAMPLE_CAMPAIGN, writeCampaign, writePlanFixtures } from '../fixtures/dashboard-fixture.js'; + +test('dashboard shares validated plan/state until either file changes, but checks liveness each read', t => { + const root = fs.mkdtempSync(join(tmpdir(), 'dashboard-campaign-cache-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const directory = join(root, 'campaigns', 'sample'); + const plan = compileCampaignFile(EXAMPLE_CAMPAIGN); + const { state } = claimNextAttempt(createCampaignState(plan), { admissionId: 'test' }); + writeCampaign(directory, plan, state); + const planPath = join(directory, 'plan.json'); + let planReads = 0; + const originalRead = fs.readFileSync; + t.mock.method(fs, 'readFileSync', (...args: Parameters) => { + if (String(args[0]) === planPath) planReads++; + return originalRead(...args); + }); + syncBuiltinESMExports(); + t.after(() => { t.mock.restoreAll(); syncBuiltinESMExports(); }); + + assert.equal(campaignSheet(root, 'sample', { controllerActive: () => true }).status, 'running'); + campaignProgression(root, 'sample'); + assert.equal(campaignSheet(root, 'sample', { controllerActive: () => false }).status, 'attention-required'); + assert.equal(planReads, 1); + + const later = new Date(Date.now() + 5000); + fs.utimesSync(join(directory, 'state.json'), later, later); + campaignSheet(root, 'sample', { controllerActive: () => true }); + assert.equal(planReads, 2); + fs.writeFileSync(planPath, '{}'); + assert.throws(() => campaignProgression(root, 'sample')); + assert.equal(planReads, 3); +}); + +test('saved plans compile once until a plan file changes', t => { + const root = fs.mkdtempSync(join(tmpdir(), 'dashboard-plan-cache-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + writePlanFixtures(root); + const names = fs.readdirSync(root).filter(name => name.endsWith('.json')); + const changed = names[0]!; + let reads = 0; + const originalRead = fs.readFileSync; + t.mock.method(fs, 'readFileSync', (...args: Parameters) => { + if (names.some(name => String(args[0]).endsWith(name))) reads++; + return originalRead(...args); + }); + syncBuiltinESMExports(); + t.after(() => { t.mock.restoreAll(); syncBuiltinESMExports(); }); + + const first = discoverPlans(root); + assert.ok(reads >= names.length); + reads = 0; + assert.deepEqual(discoverPlans(root), first); + assert.equal(reads, 0, 'an unchanged plan file is not read again'); + const later = new Date(Date.now() + 5000); + fs.utimesSync(join(root, changed), later, later); + discoverPlans(root); + assert.ok(reads > 0 && reads <= 2, 'only the changed plan is compiled again'); + fs.rmSync(join(root, changed)); + assert.equal(discoverPlans(root).length, first.length - 1); +}); diff --git a/tools/stack-bench/tests/dashboard/charts.test.ts b/tools/stack-bench/tests/dashboard/charts.test.ts new file mode 100644 index 00000000000..b7f3edb05bf --- /dev/null +++ b/tools/stack-bench/tests/dashboard/charts.test.ts @@ -0,0 +1,219 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { bigClimb } from '../../dashboard/public/climb.js'; +import { graph } from '../../dashboard/public/graph.js'; + +test('score charts explain missing grades', () => { + assert.match(bigClimb([], String), /Awaiting first grade/); + const series = [{ score: 1, max: 2, level: 1, unaided: true }]; + const html = bigClimb(series, String); + assert.match(html, /1 \/ 2 points/); + assert.match(html, /First build at this level; earlier fixes and feedback retained/); + assert.doesNotMatch(html, /NaN|Infinity/); + assert.match(html, /aria-label="Score history" tabindex="0"/); +}); + +test('an empty feature graph states that no graph is available', () => { + assert.match(graph({ key: 'empty', depths: [], questlines: [], nodes: [], stacks: [] }, []), + /No feature graph is available/); +}); + +for (const count of [1, 4, 6]) { + test(`feature graph fits all ${count} stack markers without overlapping labels or columns`, () => { + const html = graph({ + key: 'dynamic-stacks', depths: [1, 2], stacks: [], + questlines: [{ id: 'shop', title: 'Shop', nodes: ['catalog', 'cart'] }], + nodes: [ + { id: 'catalog', title: 'Product catalog', questline: 'shop', depth: 1, dependencies: [] }, + { id: 'cart', title: 'Shopping cart', questline: 'shop', depth: 2, dependencies: ['catalog'] }, + ], + }, Array.from({ length: count }, (_, index) => ({ stack: `stack-${index}`, statuses: ['passed', 'active'] }))); + const groups = [...html.matchAll(/([\s\S]*?)<\/g>/g)]; + assert.equal(groups.length, 2); + assert.equal((html.match(/class="passed-check"/g) ?? []).length, count); + let previousRight = 0; + for (const group of groups) { + const rect = / previousRight, 'depth columns must not overlap'); + const markers = [...group[1]!.matchAll(/]*cx="([\d.]+)"[^>]*r="([\d.]+)"/g)]; + assert.equal(markers.length, count); + let previousMarkerRight = left + 170; // Space reserved for the truncated feature label. + for (const marker of markers) { + const center = Number(marker[1]); + const radius = Number(marker[2]); + assert.ok(center - radius > previousMarkerRight, 'markers must clear labels and preceding markers'); + assert.ok(center + radius < right, 'markers must stay inside the feature node'); + previousMarkerRight = center + radius; + } + previousRight = right; + } + const canvas = /viewBox="0 0 ([\d.]+)/.exec(html); + assert.ok(canvas && Number(canvas[1]) > previousRight, 'canvas must contain the final node'); + for (let index = 0; index < count; index += 1) assert.match(html, new RegExp(`stack-${index}`)); + }); +} + +import { progressChart } from '../../dashboard/public/progress-chart.js'; +import type { CampaignSheet, CampaignProgression } from '../../dashboard/dashboard-views.js'; + +// A run's plotted [x, y] points, so assertions survive layout and margin changes. +const line = (html: string, run: string): number[][] => + [...new RegExp(`data-chart-series="${run}"[^>]*> [Number(match[1]), Number(match[2])]); + +test('time chart uses measured elapsed time, preserves regressions, and labels excluded runs', () => { + const sheet = { key: 'test', repetitions: 3, stacks: [{ stack: 'postgres', attempts: [{ id: 'a', repetition: 1, + executionStartedAt: '2026-09-08T00:00:00Z', excluded: 'Provider failure' }] }] } as CampaignSheet; + const progression = { stacks: [{ stack: 'postgres', attemptId: 'a', steps: [ + { completedAt: null, completion: 1 }, + { completedAt: '2026-09-08T00:01:00Z', completion: 0.75 }, + { completedAt: '2026-09-08T00:02:00Z', completion: 0.5 }, + ] }] } as CampaignProgression; + const html = progressChart(sheet, progression, 'completion', 'grid', new Set(), 'checks'); + const [start, best, regressed] = line(html, 'a'); + assert.ok(start![0]! < best![0]! && best![0]! < regressed![0]!, 'points follow elapsed time'); + assert.ok(best![1]! < regressed![1]! && regressed![1]! < start![1]!, 'the regression plots below the best grade'); + assert.match(html, /Run start; no checks graded/); + assert.match(html, /Rep 1 · 50% · Excluded/); + assert.doesNotMatch(html, /NaN|Infinity/); + assert.match(progressChart(sheet, null), /Awaiting first timed grade/); +}); + + +test('cost chart uses cumulative checkpoint costs, labels bounds, and omits unknowns', () => { + const sheet = { key: 'test', repetitions: 3, stacks: [{ stack: 'postgres', attempts: [{ id: 'a', repetition: 1, + executionStartedAt: '2026-09-08T00:00:00Z', excluded: null }] }] } as CampaignSheet; + const progression = { key: 'test', depths: [], questlines: [], nodes: [], + stacks: [{ stack: 'postgres', attemptId: 'a', updatedAt: '', steps: [], costs: [ + { completedAt: '2026-09-08T00:01:00Z', cost: { status: 'exact', costUsd: 2 } }, + { completedAt: '2026-09-08T00:01:30Z', cost: { status: 'unknown', costUsd: null } }, + { completedAt: '2026-09-08T00:02:00Z', cost: { status: 'upper-bound', costUsd: 4 } }, + ] }] } as CampaignProgression; + const html = progressChart(sheet, progression, 'cost', 'graph'); + const cost = line(html, 'a'); + assert.equal(cost.length, 3, 'the unknown receipt is omitted, not plotted as zero'); + assert.ok(cost[0]![1]! > cost[1]![1]! && cost[1]![1]! > cost[2]![1]!, 'cumulative cost rises'); + assert.match(html, /Rep 1 · ≤\$4.00/); + assert.match(html, /Run start; no recorded cost/); + assert.match(html, /questlines=graph&chart=completion/); + assert.doesNotMatch(html, /NaN|Infinity/); + assert.match(progressChart(sheet, null, 'cost'), /Awaiting first timed cost receipt/); +}); + +test('chart filters individual runs without changing the scale or hiding pending controls', () => { + const sheet = { key: 'test', repetitions: 3, stacks: [{ stack: 'custom-stack', attempts: [1, 2, 3].map(repetition => ({ + id: `run-${repetition}`, repetition, executionStartedAt: '2026-09-08T00:00:00Z', excluded: null, + })) }] } as CampaignSheet; + const progression = { stacks: [1, 2].map(repetition => ({ stack: 'custom-stack', attemptId: `run-${repetition}`, + steps: [{ completedAt: `2026-09-08T00:0${repetition}:00Z`, completion: repetition / 4 }], + })) } as CampaignProgression; + const html = progressChart(sheet, progression, 'completion', 'grid', new Set(['run-2']), 'checks'); + assert.match(html, /data-chart-stack="custom-stack" aria-pressed="mixed"/); + assert.match(html, /class="progress-series" data-chart-series="run-1"/); + assert.doesNotMatch(html, /class="progress-series" data-chart-series="run-2"/); + assert.deepEqual(line(html, 'run-1'), + line(progressChart(sheet, progression, 'completion', 'grid', new Set(), 'checks'), 'run-1'), + 'hiding a run keeps the scale'); + assert.match(html, /Rep 3<\/button>/); + const empty = progressChart(sheet, progression, 'completion', 'grid', new Set(['run-1', 'run-2', 'run-3']), 'checks'); + assert.match(empty, /Select a run/); + assert.match(empty, /data-chart-run="run-1"/); // Controls remain available to restore runs. +}); + +test('distribution shows completed run percentages across providers and preserves filters', () => { + const sheet = { repetitions: 3, stacks: ['spacetime', 'mongodb', 'postgres'].map(stack => ({ stack, + attempts: [1, 2].map(repetition => ({ id: `${stack}-${repetition}`, repetition, + status: repetition === 1 ? 'completed' : 'running', completion: { rate: 0.75 } })) })) } as CampaignSheet; + const html = progressChart(sheet, null, 'distribution', 'graph', new Set(['mongodb-1']), 'checks'); + assert.match(html, /Completion distribution by provider/); + assert.match(html, /class="progress-series" data-chart-series="spacetime-1"[^>]*tabindex="0"/); + assert.match(html, /class="distribution-run-label"[^>]*>[^<]*Rep 1<\/text>/); + assert.equal((html.match(/>75%<\/text><\/g>/g) ?? []).length, 2); + assert.match(html, /Rep 2<\/button>/); + assert.match(html, /questlines=graph&chart=distribution/); + assert.doesNotMatch(html, /NaN|Infinity|Elapsed run time/); + for (const stack of sheet.stacks) for (const attempt of stack.attempts) attempt.status = 'running'; + const pending = progressChart(sheet, null, 'distribution'); + assert.doesNotMatch(pending, /class="progress-series"|Awaiting first completed run/); +}); + +test('completion units use distinct saved metrics in both chart views', () => { + const sheet = { repetitions: 3, stacks: [{ stack: 'spacetime', attempts: [{ id: 'a', repetition: 1, + status: 'completed', executionStartedAt: '2026-09-08T00:00:00Z', + completion: { rate: 0.75 }, featureCompletion: { rate: 0.5 } }] }] } as CampaignSheet; + const progression = { stacks: [{ stack: 'spacetime', attemptId: 'a', steps: [ + { completedAt: '2026-09-08T00:01:00Z', completion: 0.75, featureCompletion: 0.5 }, + ] }] } as CampaignProgression; + for (const metric of ['completion', 'distribution'] as const) { + const features = progressChart(sheet, progression, metric, 'graph', new Set(), 'features'); + const checks = progressChart(sheet, progression, metric, 'graph', new Set(), 'checks'); + assert.match(features, /Rep 1[^<]*50%/); + assert.match(checks, /Rep 1[^<]*75%/); + assert.match(features, /aria-label="Completion unit"/); + assert.match(features, new RegExp(`questlines=graph&chart=${metric}&unit=checks`)); + assert.match(features, /chart=distribution&unit=features/); + } + assert.match(progressChart(sheet, progression), /Rep 1[^<]*50%/); + sheet.stacks[0]!.attempts[0]!.featureCompletion = null; + progression.stacks[0]!.steps[0]!.featureCompletion = null; + for (const metric of ['completion', 'distribution'] as const) { + const html = progressChart(sheet, progression, metric, 'grid', new Set(), 'features'); + assert.doesNotMatch(html, /progress-series|>75%<\/text><\/g>/); + assert.match(html, /data-chart-run="a"/); + } + const cost = progressChart(sheet, progression, 'cost'); + const units = cost.split('')[0]!; + assert.equal((units.match(/aria-disabled="true"/g) ?? []).length, 2); + assert.match(units, />FeaturesChecks { + const sheet = { key: 'test', repetitions: 3, stacks: [{ stack: 'postgres', attempts: [{ + id: 'failed', repetition: 1, model: 'gpt-6-astra', effort: 'medium', + status: 'invalid', excluded: 'Request failure', + executionStartedAt: '2026-09-11T00:00:00Z', + executionCompletedAt: '2026-09-11T00:02:00Z', + executionCost: { status: 'exact', costUsd: 3.25 }, + spend: { status: 'exact', costUsd: 9.25 }, + }] }] } as CampaignSheet; + const html = progressChart(sheet, null, 'cost'); + assert.match(html, /Astra.*medium.*Rep 1/); + assert.match(html, /\$3.25/); + assert.doesNotMatch(html, /\$9.25/); + assert.match(html, /Current-execution cost/); + assert.match(html, /data-chart-series="failed"/); + assert.doesNotMatch(html, /Awaiting first timed cost receipt/); + sheet.repetitions = 1; + const compact = progressChart(sheet, null, 'cost').match(/]*data-chart-run="failed"[^>]*>(.*?)<\/button>/)![1]!; + assert.match(compact, /Astra \(medium\) · \$3.25/); + assert.match(compact, /· Excluded/); + assert.doesNotMatch(compact, /Rep 1|Pending/); +}); + +test('distribution omits excluded measurements but identifies both exclusion statuses visibly', () => { + const sheet = { repetitions: 3, stacks: [{ stack: 'postgres', attempts: [ + { id: 'valid', repetition: 1, status: 'completed', completion: { rate: 0.75 } }, + { id: 'invalid', repetition: 2, status: 'invalid', excluded: 'Missing evidence', completion: { rate: 0.1 } }, + { id: 'failed', repetition: 3, status: 'completed', excluded: 'harness_failure', completion: { rate: 0.2 } }, + ] }] } as CampaignSheet; + const html = progressChart(sheet, null, 'distribution', 'grid', new Set(), 'checks'); + assert.match(html, /class="progress-series" data-chart-series="valid"/); + assert.doesNotMatch(html, /class="progress-series" data-chart-series="(?:invalid|failed)"/); + assert.match(html, /Rep 2 · Excluded<\/button>/); + assert.match(html, /Rep 3 · Excluded<\/button>/); +}); + +test('Supabase has its own label and a series color distinct from SpacetimeDB', () => { + const sheet = { key: 'test', repetitions: 1, stacks: ['spacetime', 'supabase'].map(stack => ({ stack, + attempts: [{ id: stack, repetition: 1, executionStartedAt: '2026-09-08T00:00:00Z', excluded: null }] })) } as CampaignSheet; + const html = progressChart(sheet, null, 'completion', 'grid', new Set(), 'checks'); + assert.match(html, /data-chart-stack="supabase"[^>]*>.*?stroke="#1f9d6a"[^>]*\/><\/svg>Supabase<\/button>/); + assert.match(html, /data-chart-stack="spacetime"[^>]*>.*?stroke="#4cf490"[^>]*\/><\/svg>SpacetimeDB<\/button>/); +}); diff --git a/tools/stack-bench/tests/dashboard/check-evidence.test.ts b/tools/stack-bench/tests/dashboard/check-evidence.test.ts new file mode 100644 index 00000000000..8fad85ee7fd --- /dev/null +++ b/tools/stack-bench/tests/dashboard/check-evidence.test.ts @@ -0,0 +1,82 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { attemptChecks } from '../../dashboard/dashboard-views.js'; +import { compileCampaignFile } from '../../src/campaigns/campaign-compiler.js'; +import { claimNextAttempt, createCampaignState } from '../../src/campaigns/campaign-scheduler.js'; +import { writeArtifact } from '../../src/evidence/artifacts.js'; +import { createCheckEvidence } from '../../src/evidence/check-evidence.js'; +import { EXAMPLE_CAMPAIGN, writeCampaign, writeRunEvidence } from '../fixtures/dashboard-fixture.js'; + +test('check details retain measured and unmeasured evidence without leaking credentials', t => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-check-evidence-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const plan = compileCampaignFile(EXAMPLE_CAMPAIGN); + const claimed = claimNextAttempt(createCampaignState(plan), { admissionId: 'evidence-test' }); + assert.ok(claimed.claim); + const directory = join(root, 'campaigns', 'evidence'); + writeCampaign(directory, plan, claimed.state); + const output = join(directory, claimed.claim.output); + writeRunEvidence(output, plan, claimed.claim.attempt, 0); + const read = () => attemptChecks(root, 'evidence', claimed.claim!.attempt.id); + assert.equal(read().checks[0]?.observations[0]?.status, 'PASS'); + const bundlePath = join(output, 'grading', 'bundle.json'); + const bundle = JSON.parse(readFileSync(bundlePath, 'utf8')); + const criterion = bundle.payload.suites.fixture.features[0].criteria[0]; + for (const status of ['failed', 'blocked', 'inconclusive', 'harness_failure'] as const) { + criterion.evidence = createCheckEvidence({ status, code: 'test.observation', + phase: status === 'blocked' ? 'setup' : 'assertion', startedAtMs: 1, completedAtMs: 2, + summary: 'authorization: Bearer secret-summary', expected: { orders: 1 }, + observation: { orders: 2, detail: 'authorization: Bearer secret-observation' } }); + writeArtifact(bundlePath, bundle); + const observed = read().checks[0]?.observations[0]; + assert.equal(observed?.status, status === 'failed' ? 'FAIL' : status.replaceAll('_', ' ').toUpperCase()); + assert.match(observed?.expected ?? '', /"orders": 1/); + assert.match(observed?.actual ?? '', /"orders": 2/); + assert.doesNotMatch(JSON.stringify(observed), /secret-summary|secret-observation/); + } + criterion.evidence.sensitivity = ['credentials']; + criterion.evidence.summary = 'unlabelled-sensitive-value'; + writeArtifact(bundlePath, bundle); + assert.deepEqual(read().checks[0]?.observations[0], { + status: 'HARNESS FAILURE', summary: 'Sensitive evidence omitted.', expected: null, actual: null, + }); + criterion.evidence.sensitivity = []; + criterion.evidence.observation = 'x'.repeat(20_000); + writeArtifact(bundlePath, bundle); + const large = read().checks[0]?.observations[0]?.actual; + assert.ok(large && large.length < 13_000); + assert.match(large, /Truncated. Full evidence is in Files/); + // An interface prerequisite is a measured omission, not evidence that the + // later integrity assertion failed. Do not infer that for a generic mismatch. + criterion.evidence = createCheckEvidence({ status: 'failed', code: 'test.interface', + phase: 'assertion', startedAtMs: 1, completedAtMs: 2, + finding: { kind: 'interface-missing', fields: { action: 'transfer', control: 'admin-item-row', attribute: 'data-transfer-input' } }, + actions: [{ actor: 'admin', evidence: { schemaVersion: 2, action: { id: 'callAction', version: '1' }, + status: 'failed', type: 'action', code: 'test.interface', phase: 'execution', summary: null, finding: null, + observation: null, expected: null, retryable: false, attachments: [], sensitivity: [], + timing: { startedAtMs: 1, completedAtMs: 2, durationMs: 1, deadlineMs: 100 } } }] }); + writeArtifact(bundlePath, bundle); + const missing = read().checks[0]!.observations[0]!; + assert.match(missing.context ?? '', /Interface failure.*remaining steps were not reached/); + assert.match(missing.context ?? '', /callAction/); + assert.match(missing.context ?? '', /data-transfer-input/); + assert.equal(missing.status, 'FAIL', 'reporting must not change the recorded outcome'); + criterion.evidence.finding = { kind: 'value-mismatch', fields: { control: 'order-status', observed: 'cancelled', expected: 'pending' } }; + writeArtifact(bundlePath, bundle); + assert.doesNotMatch(read().checks[0]!.observations[0]!.context ?? '', /Interface failure|not reached/); + criterion.evidence.actions[0].evidence.sensitivity = ['credentials']; + writeArtifact(bundlePath, bundle); + assert.doesNotMatch(JSON.stringify(read().checks[0]!.observations[0]), /callAction/); + // A missing later grade must not erase earlier evidence or become a pass. + writeArtifact(join(output, 'l1-fix1-grading', 'bundle.json'), bundle); + rmSync(join(output, 'l1-fix1-grading', 'bundle.json')); + const incomplete = read(); + // Oldest first: the final grade comes after every repair grade. + assert.deepEqual(incomplete.grades.map(grade => grade.id), ['l1-fix1-grading', 'grading']); + assert.equal(incomplete.grades[0]?.error, 'grade bundle is missing'); + assert.equal(incomplete.checks[0]?.observations.length, 2); + assert.equal(incomplete.checks[0]?.observations[0], null); +}); diff --git a/tools/stack-bench/tests/dashboard/dashboard.test.ts b/tools/stack-bench/tests/dashboard/dashboard.test.ts new file mode 100644 index 00000000000..5831983968d --- /dev/null +++ b/tools/stack-bench/tests/dashboard/dashboard.test.ts @@ -0,0 +1,1282 @@ +import assert from 'node:assert/strict'; +import { appendFileSync, mkdtempSync, mkdirSync, readFileSync, readdirSync, rmSync, utimesSync, + watch, writeFileSync } from 'node:fs'; +import { request as httpRequest } from 'node:http'; +import type { Server } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { EventEmitter } from 'node:events'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test, { after, before } from 'node:test'; + +import { ARTIFACT_FILE, emptyArtifactIdentities, writeArtifact } from '../../src/evidence/artifacts.js'; +import { compileCampaignFile } from '../../src/campaigns/campaign-compiler.js'; +import type { CampaignAttemptPlan, CompiledCampaignPlan } from '../../src/campaigns/campaign-compiler.js'; +import { campaignComparisonKey } from '../../src/campaigns/campaign-report.js'; +import { claimNextAttempt, createCampaignState, finishCampaignExecution } + from '../../src/campaigns/campaign-scheduler.js'; +import { canonicalDefinitionJson } from '../../src/composition/definition-plan.js'; +import { attemptChecks, attemptPackage, campaignLiveUpdate, campaignProgression, campaignSheet, + overviewSummary } from '../../dashboard/dashboard-views.js'; +import { parseRunProgress, attemptPause, + discoverCampaigns, discoverPlans, readCampaignArtifactBody, readJsonLines, + resolveCampaignArtifact, summarizeCampaign, +} from '../../dashboard/dashboard-model.js'; +import { checkpointRegressions, firstGradeAbort, inspectCampaignSummary } + from '../../src/campaigns/campaign-inspection.js'; +import { attemptExcluded, compareCampaign } from '../../dashboard/public/metrics.js'; +import { createDashboardServer, parseDashboardArgs } from '../../dashboard/dashboard-server.js'; +import type { DashboardOperation, LaunchInput } from '../../dashboard/dashboard-server.js'; +import { sha256 } from '../../src/evidence/provenance.js'; +import { acquireCampaignLock, campaignCancellationRequested, releaseCampaignLock } + from '../../src/campaigns/campaign-lock.js'; +import { progressionEngine } from '../../src/progression/progression-engine.js'; +import { compileProgressionInput, dependencyRuntimeDefinition } + from '../../src/progression/progression-definition.js'; +import { writeProgressionState } from '../../src/progression/progression-state.js'; +import { attemptPage } from '../../dashboard/public/views/attempt.js'; +import { campaignPage } from '../../dashboard/public/views/campaign.js'; +import { graph } from '../../dashboard/public/graph.js'; +import { campaignsPage } from '../../dashboard/public/views/campaigns.js'; +import { plansPage, topbar } from '../../dashboard/public/views/plans.js'; + +import { DEPENDENCY_CAMPAIGN, EXAMPLE_CAMPAIGN, + dependencyProgressionEvidence, writeCampaign, writeDependencyResults, writeFixtureResults, + writePlanFixtures, writeRunEvidence } from '../fixtures/dashboard-fixture.js'; + +// The two campaign plans compile in seconds; every test works on a clone of +// one compile. Fixture results are written once: a completed root, which +// also holds a dependency campaign and the plan files, and a running root. +// Tests read them; the one test that touches a timestamp leaves the evidence +// readable for the tests that follow. +let examplePlanCache: CompiledCampaignPlan | null = null; +const examplePlan = (): CompiledCampaignPlan => + structuredClone(examplePlanCache ??= compileCampaignFile(EXAMPLE_CAMPAIGN)); +let dependencyPlanCache: CompiledCampaignPlan | null = null; +const dependencyPlan = (): CompiledCampaignPlan => + structuredClone(dependencyPlanCache ??= compileCampaignFile(DEPENDENCY_CAMPAIGN)); +const shared = { completed: '', running: '' }; + +before(() => { + shared.completed = mkdtempSync(join(tmpdir(), 'stack-bench-dashboard-completed-')); + writeFixtureResults(shared.completed); + writeDependencyResults(shared.completed, 'progression-run'); + writePlanFixtures(join(shared.completed, 'plans')); + shared.running = mkdtempSync(join(tmpdir(), 'stack-bench-dashboard-running-')); + writeFixtureResults(shared.running, { running: true }); +}); + +after(() => { + for (const root of Object.values(shared)) rmSync(root, { recursive: true, force: true }); +}); + +async function listenOrigin(server: Server): Promise { + await new Promise((resolveListen, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolveListen); + }); + const address = server.address(); + assert.ok(address && typeof address !== 'string'); + return `http://127.0.0.1:${(address as AddressInfo).port}`; +} + +test('dashboard run progress reports only completed grades while the next repair is underway', () => { + const progress = parseRunProgress(` +=== postgres-l1-first (postgres) === + TOTAL ... 31/58 +--- repair 1/3 --- +=== postgres-l1-fix1 (postgres) === + TOTAL ... 41/58 +--- repair 2/3 --- +=== postgres-l1-fix2 (postgres) === +`, { repairs: 3 }); + assert.deepEqual(progress.firstScore, { score: 31, max: 58 }); + assert.deepEqual(progress.latestScore, { score: 41, max: 58 }); + assert.equal(progress.completedGrades, 2); + assert.equal(progress.level, 1); + assert.deepEqual(progress.repair, { round: 2, budget: 3 }); + assert.equal(progress.phase, 'Grading L1 after repair 2 of 3'); +}); + +test('dashboard compares the declared stack treatments but separates changed models, guidance, and scope', () => { + const inspected = inspectCampaignSummary(join(shared.completed, 'campaigns', 'fixture-run-0')); + const attempts = inspected.attempts.map(attempt => ({ ...attempt, + spend: { status: 'exact' as const, costUsd: 1 }, + measuredCost: { status: 'exact' as const, costUsd: 1 } })); + const comparison = compareCampaign({ attempts }); + assert.equal(comparison.rows.length, 3); + assert.equal(comparison.mixedScope, false); + assert.equal(comparison.comparable, true); + assert.equal(campaignSheet(shared.completed, 'fixture-run-0').mixedScope, false); + const source = examplePlan().attempts.find(attempt => attempt.id === attempts[0]!.id)!; + for (const change of [ + (attempt: CampaignAttemptPlan) => { attempt.model = 'another-model'; }, + (attempt: CampaignAttemptPlan) => { attempt.guidance = 'another-guidance'; }, + (attempt: CampaignAttemptPlan) => { attempt.condition.contentSha256 = 'a'.repeat(64); }, + (attempt: CampaignAttemptPlan) => { attempt.levels = [1, 2]; }, + ]) { + const changed = structuredClone(source); + change(changed); + const result = compareCampaign({ attempts: attempts.map((attempt, index) => index === 0 + ? { ...attempt, comparisonKey: campaignComparisonKey(changed) } : attempt) }); + assert.equal(result.mixedScope, true); + assert.equal(result.comparable, false); + assert.equal(result.rows.find(row => row.stack === changed.stack)?.final, null); + } +}); + +test('dashboard operation feed ignores only a truncated final write', t => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-dashboard-feed-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const path = join(root, 'operations.jsonl'); + writeFileSync(path, '{"id":"complete"}\n{"id":"partial"'); + assert.deepEqual(readJsonLines(path), [{ id: 'complete' }]); + writeFileSync(path, '{broken}\n{"id":"complete"}\n'); + assert.throws(() => readJsonLines(path), /line 1/); +}); + +test('dashboard follows the actual level when a completed L1 advances to the first L2 grade', () => { + const progress = parseRunProgress(` +=== spacetime-l1 (spacetime) === + TOTAL ... 52/58 +--- repair 1/10 --- +=== spacetime-l1-fix1 (spacetime) === + TOTAL ... 58/58 +=== spacetime-l2 (spacetime) === + TOTAL ... 52/59 +--- repair 1/10 --- +`, { repairs: 10 }); + assert.equal(progress.level, 2); + assert.equal(progress.phase, 'Repairing L2 · round 1 of 10'); + assert.deepEqual(progress.firstScore, { score: 52, max: 58 }); + assert.deepEqual(progress.latestScore, { score: 52, max: 59 }); +}); + +test('dashboard reports a dependency repair by feature instead of the level session count', () => { + const progress = parseRunProgress(` +=== mongodb-l3-first (mongodb) === + TOTAL ... 26/43 +--- feature repair 1: Customer recommendations --- +=== mongodb-l3-fix3 (mongodb) === +`, { repairs: 3 }); + assert.equal(progress.phase, + 'Grading L3 after repair 1 for Customer recommendations'); + assert.deepEqual(progress.repair, { round: 1, budget: null }); + const repairing = parseRunProgress(` +=== mongodb-l3-first (mongodb) === + TOTAL ... 26/43 +--- feature repair 2: Customer recommendations --- +`, { repairs: 3, running: true }); + assert.equal(repairing.phase, 'Repairing Customer recommendations · 2'); + assert.deepEqual(repairing.repair, { round: 2, budget: null }); + assert.equal(parseRunProgress(` +=== mongodb-l2-first (mongodb) === +`, { dependency: true }).phase, 'Grading the first depth 2 build'); +}); + +test('a regression is a kept pass that later fails, not an unmeasured or never-passed check', () => { + const point = (accepted: boolean, checks: Record) => + ({ accepted, checks: Object.entries(checks).map(([id, status]) => ({ id, status })) }) as Parameters[0][number]; + assert.equal(checkpointRegressions([ + point(true, { kept: 'passed', never: 'failed', aborted: 'passed', rolledBack: 'passed' }), + point(false, { kept: 'passed', never: 'failed', aborted: 'passed', rolledBack: 'failed' }), + point(true, { kept: 'failed', never: 'failed', aborted: 'unmeasured', rolledBack: 'passed' }), + ]), 1); +}); + +test('an aborted first grade is classified, not treated as a scored zero', () => { + // A readiness failure stops the grade before any suite runs, so 0/58 must + // read as an abort with its reason. + assert.deepEqual(firstGradeAbort({ score: 0, max: 58, outcome: { kind: 'app_failure', + phase: 'application-readiness', reason: 'application did not respond' } }), + { phase: 'application-readiness', reason: 'application did not respond' }); + // A graded first build — even a zero — is a score, not an abort. + assert.equal(firstGradeAbort({ score: 0, max: 58, + outcome: { kind: 'app_failure', phase: 'grading', reason: null } }), null); + assert.equal(firstGradeAbort({ score: 31, max: 58, + outcome: { kind: 'passed', phase: 'grading' } }), null); + assert.equal(firstGradeAbort(null), null); + assert.equal(firstGradeAbort({ score: 31, max: 58 }), null); +}); + +test('dashboard CLI is deliberately loopback-only', () => { + const parsed = parseDashboardArgs(['node', 'dashboard', '--port', '7444', '--host', 'localhost'], {}); + assert.equal(parsed.port, 7444); + assert.equal(parsed.host, 'localhost'); + const container = parseDashboardArgs(['node', 'dashboard', '--host', '0.0.0.0', + '--allow-container-bind'], { STACK_BENCH_APPLIANCE: '1' }); + assert.equal(container.host, '0.0.0.0'); + assert.equal(container.allowContainerBind, true); + assert.throws(() => parseDashboardArgs(['node', 'dashboard', '--host', '0.0.0.0'], {}), + /loopback/); + assert.throws(() => parseDashboardArgs(['node', 'dashboard', '--host', '0.0.0.0', + '--allow-container-bind'], {}), /loopback/); + assert.throws(() => parseDashboardArgs(['node', 'dashboard', '--port', '70000'], {}), + /port/); +}); + +test('dashboard marks a current-schema campaign as interrupted when its controller stops', t => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-dashboard-interrupted-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const plan = examplePlan(); + const now = '2026-08-18T12:00:00.000Z'; + const claimed = claimNextAttempt(createCampaignState(plan, { now }), + { now, admissionId: 'admission-1' }); + assert.ok(claimed.claim); + writeCampaign(root, plan, claimed.state); + const output = join(root, claimed.claim.output); + mkdirSync(output, { recursive: true }); + writeFileSync(join(output, 'process.stdout.log'), + '=== postgres-l1-first (postgres) ===\n TOTAL ... 31/58\n--- repair 1/3 ---\n'); + + const summary = summarizeCampaign(root); + assert.equal(summary.status, 'running'); + const activeAttempt = summary.attempts[0]; + assert.ok(activeAttempt); + assert.equal(activeAttempt.progress.phase, 'Repairing L1 · round 1 of 3'); + assert.deepEqual(activeAttempt.progress.latestScore, { score: 31, max: 58 }); + + const interrupted = summarizeCampaign(root, { controllerActive: () => false }); + assert.equal(interrupted.status, 'attention-required'); + assert.equal(interrupted.interrupted, true); + assert.equal(interrupted.summary.running, 0); + assert.equal(interrupted.summary.interrupted, 1); + const interruptedAttempt = interrupted.attempts[0]; + assert.ok(interruptedAttempt?.execution); + assert.equal(interruptedAttempt.status, 'interrupted'); + assert.equal(interruptedAttempt.execution.status, 'interrupted'); + assert.equal(interruptedAttempt.progress.phase, 'Controller stopped before completion'); +}); + +test('dashboard keeps frozen campaign evidence readable after the controller is upgraded', t => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-dashboard-frozen-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const currentPlan = examplePlan(); + const state = createCampaignState(currentPlan, { now: '2026-08-18T12:00:00.000Z' }); + const frozenPlan = structuredClone(currentPlan); + frozenPlan.identities.engine.sha256 = 'f'.repeat(64); + frozenPlan.contentSha256 = sha256(canonicalDefinitionJson({ + campaignSchemaVersion: frozenPlan.campaignSchemaVersion, + definition: frozenPlan.definition, + engine: frozenPlan.identities.engine, + bindings: frozenPlan.bindings, + stacks: frozenPlan.stacks, + agents: frozenPlan.agents, + conditions: frozenPlan.conditions, + })); + state.campaignSha256 = frozenPlan.contentSha256; + writeArtifact(join(root, 'plan.json'), { kind: 'campaign_plan', id: `${frozenPlan.id}-plan`, + identities: emptyArtifactIdentities(), payload: frozenPlan }); + writeArtifact(join(root, 'state.json'), { kind: 'campaign_state', id: `${frozenPlan.id}-state`, + identities: emptyArtifactIdentities(), payload: state }); + + const summary = summarizeCampaign(root); + assert.equal(summary.id, frozenPlan.id); + assert.equal(summary.status, 'prepared'); + assert.equal(summary.attempts.length, frozenPlan.attempts.length); +}); + +test('dashboard rejects a run artifact that does not belong to its frozen attempt', t => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-dashboard-run-identity-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const plan = examplePlan(); + const now = '2026-08-18T12:00:00.000Z'; + const claimed = claimNextAttempt(createCampaignState(plan, { now }), + { now, admissionId: 'admission-1' }); + assert.ok(claimed.claim); + writeCampaign(root, plan, claimed.state); + const output = join(root, claimed.claim.output); + mkdirSync(output, { recursive: true }); + const attemptPlan = claimed.claim.attempt; + writeArtifact(join(output, 'run.json'), { kind: 'benchmark_run', id: 'other-run', + attempt: { id: 'other-run', parentId: 'other-attempt' }, + identities: emptyArtifactIdentities(), payload: { + mode: attemptPlan.mode, track: plan.definition.track, backend: attemptPlan.stack, + model: attemptPlan.model, pricing: attemptPlan.pricing, guidance: attemptPlan.guidance, + condition: attemptPlan.condition, selectionRequest: plan.definition.selection, + featureCatalog: attemptPlan.featureCatalog ?? null, + dependencyPolicy: attemptPlan.dependencyPolicy ?? null, + progressionOwner: null, skills: attemptPlan.skills, levels: [], + runtime: { buildImage: plan.definition.runtime.buildImage }, + totals: { costUsd: 0, costComplete: true }, outcome: { kind: 'passed' }, + } }); + + const attempt = summarizeCampaign(root).attempts[0]; + assert.match(attempt?.result?.unreadable ?? '', /attempt\.parentId/); + const inspection = inspectCampaignSummary(root); + const inspected = inspection.attempts[0]; + assert.equal(inspection.schemaVersion, 1); + assert.match(inspected?.result?.unreadable ?? '', /attempt\.parentId/); + assert.equal(inspected?.artifacts?.run, `${claimed.claim.output}/run.json`); +}); + +test('inspection corrects historical incomplete counts while retaining recorded classification', t => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-inspection-incomplete-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const plan = examplePlan(); + const claimed = claimNextAttempt(createCampaignState(plan), { admissionId: 'incomplete' }); + assert.ok(claimed.claim); + const state = finishCampaignExecution(claimed.state, claimed.claim.executionId, + { exitCode: 0, run: { outcome: { kind: 'passed' } } }); + writeCampaign(root, plan, state); + const output = join(root, claimed.claim.output); + writeRunEvidence(output, plan, claimed.claim.attempt, 0); + const path = join(output, 'run.json'); + const saved = JSON.parse(readFileSync(path, 'utf8')); + saved.payload.outcome = { kind: 'passed', inconclusive: ['unmeasured-check'] }; + writeArtifact(path, { kind: 'benchmark_run', id: saved.id, attempt: saved.attempt, + identities: saved.identities, payload: saved.payload }); + const inspected = inspectCampaignSummary(root); + const attempt = inspected.attempts.find(item => item.id === claimed.claim!.attempt.id)!; + assert.equal(attempt.status, 'invalid'); + assert.equal(attempt.execution?.outcome, 'inconclusive'); + assert.equal(attempt.execution?.recordedOutcome, 'passed'); + assert.equal(inspected.summary.completed, 0); + assert.equal(inspected.summary.invalid, 1); + assert.equal(inspected.recordedSummary?.completed, 1); + assert.equal(state.summary.completed, 1); +}); + +test('dashboard overview defers historical evidence validation until the sheet is opened', t => { + const resultsRoot = mkdtempSync(join(tmpdir(), 'stack-bench-dashboard-history-')); + t.after(() => rmSync(resultsRoot, { recursive: true, force: true })); + const key = 'historical-run'; + const campaign = join(resultsRoot, 'campaigns', key); + const plan = examplePlan(); + let state = createCampaignState(plan, { now: '2026-08-18T12:00:00.000Z' }); + let firstClaim = null; + for (;;) { + const claimed = claimNextAttempt(state, + { now: '2026-08-18T12:00:00.000Z', admissionId: 'admission-1' }); + if (!claimed.claim) break; + firstClaim ??= claimed.claim; + state = finishCampaignExecution(claimed.state, claimed.claim.executionId, + { exitCode: 0, run: { outcome: { kind: 'passed' } } }, + { now: '2026-08-18T12:00:01.000Z' }); + } + assert.ok(firstClaim); + writeCampaign(campaign, plan, state); + const output = join(campaign, firstClaim.output); + mkdirSync(output, { recursive: true }); + writeFileSync(join(output, 'run.json'), '{"not":"a benchmark run"}\n'); + + const overview = discoverCampaigns(join(resultsRoot, 'campaigns'))[0]; + assert.ok(overview && overview.status !== 'unreadable'); + assert.equal(overview.status, 'completed'); + assert.equal(overview.attempts.length, 0); + + const sheet = campaignSheet(resultsRoot, key); + const excluded = sheet.stacks.flatMap(stack => stack.attempts) + .map(attempt => attempt.excluded); + assert.ok(excluded.some(reason => reason?.startsWith('Result validation failed:'))); +}); + +test('dashboard keeps the recorded invalidation reason when the result is unreadable', () => { + assert.equal(attemptExcluded({ + id: 'failed-attempt', stack: 'mongodb', status: 'invalid', + execution: { outcome: 'harness_failure', reason: 'disk filled while saving the run' }, + result: { unreadable: 'partial run is invalid' }, dependency: null, + }), 'disk filled while saving the run'); +}); + +test('the attempt package exposes the evidence but not arbitrary campaign files', async t => { + const resultsRoot = mkdtempSync(join(tmpdir(), 'stack-bench-dashboard-package-')); + t.after(() => rmSync(resultsRoot, { recursive: true, force: true })); + const campaign = join(resultsRoot, 'campaigns', 'evidence-run'); + const plan = examplePlan(); + const now = '2026-08-18T12:00:00.000Z'; + const claimed = claimNextAttempt(createCampaignState(plan, { now }), + { now, admissionId: 'admission-1' }); + assert.ok(claimed.claim); + const state = finishCampaignExecution(claimed.state, claimed.claim.executionId, + { exitCode: 0, run: { outcome: { kind: 'passed' } } }, + { now: '2026-08-18T12:00:01.000Z' }); + const outputRelative = claimed.claim.output; + writeCampaign(campaign, plan, state); + const output = join(campaign, outputRelative); + const media = join(output, 'grading', 'failure-media'); + mkdirSync(media, { recursive: true }); + writeFileSync(join(output, 'process.stdout.log'), 'authorization: Bearer secret-token-value\n' + + 'ANTHROPIC_API_KEY=provider-secret\nCLAUDE_CODE_OAUTH_TOKEN=oauth-secret\n' + + '{"apiKey":"json-secret"}\nresult ok\n'); + writeFileSync(join(output, 'run.json'), '{"result":"ok"}\n'); + writeArtifact(join(output, 'process.json'), { kind: 'campaign_process', id: 'execution-process', + payload: { schemaVersion: 1, executionId: claimed.claim.executionId, runIndex: 0, + exitCode: 1, signal: null, timedOut: false, streams: null } }); + writeFileSync(join(media, 'failed-check.png'), Buffer.from('89504e470d0a1a0a', 'hex')); + mkdirSync(join(campaign, 'admissions'), { recursive: true }); + writeFileSync(join(campaign, 'admissions', 'private.json'), '{"token":"do-not-serve"}\n'); + + const attemptId = claimed.claim.attempt.id; + const evidence = attemptPackage(resultsRoot, 'evidence-run', attemptId); + const packageExecution = evidence.executions[0]; + assert.ok(packageExecution); + assert.equal(evidence.executions.length, 1); + assert.equal(packageExecution.truncated, false); + const initialVisual = packageExecution.visuals[0]; + assert.ok(initialVisual); + assert.equal(initialVisual.path, + `${outputRelative}/grading/failure-media/failed-check.png`); + const log = packageExecution.artifacts.find(item => item.path.endsWith('process.stdout.log')); + assert.ok(log); + const process = packageExecution.artifacts.find(item => item.path.endsWith('process.json')); + assert.ok(process); + const servedLog = readCampaignArtifactBody( + resolveCampaignArtifact(resultsRoot, 'evidence-run', log.id)).toString(); + assert.match(servedLog, /\[redacted credential\]/); + assert.doesNotMatch(servedLog, /secret-token-value|provider-secret|oauth-secret|json-secret/); + assert.throws(() => resolveCampaignArtifact(resultsRoot, 'evidence-run', + Buffer.from('admissions/private.json').toString('base64url')), /not available/); + assert.throws(() => resolveCampaignArtifact(resultsRoot, 'evidence-run', + Buffer.from('../outside.json').toString('base64url')), /not available|outside/); + + const feed = { append() {}, list() { return []; } }; + const { server } = createDashboardServer({ resultsRoot, plansRoot: join(resultsRoot, 'plans'), + allowLaunch: false, token: 'package-token', feed, plans: () => [] }); + const origin = await listenOrigin(server); + t.after(() => server.close()); + const visual = packageExecution.visuals[0]; + assert.ok(visual); + const visualResponse = await fetch(`${origin}/api/campaigns/evidence-run/artifacts/${visual.id}`); + assert.equal(visualResponse.status, 200); + assert.equal(visualResponse.headers.get('content-type'), 'image/png'); + assert.deepEqual(Buffer.from(await visualResponse.arrayBuffer()), Buffer.from('89504e470d0a1a0a', 'hex')); + const logResponse = await fetch(`${origin}/api/campaigns/evidence-run/artifacts/${log.id}`); + assert.equal(logResponse.status, 200); + assert.match(await logResponse.text(), /\[redacted credential\]/); + const processResponse = await fetch(`${origin}/api/campaigns/evidence-run/artifacts/${process.id}`); + assert.equal(processResponse.status, 200); + assert.match(await processResponse.text(), /"timedOut": false/); + const forbidden = Buffer.from('admissions/private.json').toString('base64url'); + assert.equal((await fetch(`${origin}/api/campaigns/evidence-run/artifacts/${forbidden}`)).status, 404); +}); + +test('dashboard serves real state and protects campaign launch with same-origin browser tokens', async t => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-dashboard-')); + const plansRoot = join(root, 'plans'); + mkdirSync(plansRoot, { recursive: true }); + const events: DashboardOperation[] = []; + const launches: LaunchInput[] = []; + const feed = { + append(event: DashboardOperation) { events.push(structuredClone(event)); }, + list() { + const latest = new Map(); + for (const event of events) latest.set(event.id, { ...(latest.get(event.id) ?? {}), ...event }); + return [...latest.values()]; + }, + }; + const storedPlan = dependencyPlan(); + const frozenPlan = { id: storedPlan.id, version: storedPlan.version, title: storedPlan.title, + state: 'frozen', mode: 'dependency', track: storedPlan.definition.track, + levels: storedPlan.definition.levels, stacks: storedPlan.stacks.map(stack => stack.id), + attempts: storedPlan.summary.attempts, parallelism: storedPlan.summary.parallelism, + budgets: storedPlan.definition.budgets, sha256: storedPlan.contentSha256, + file: 'ecommerce-progression-reference.json' }; + const campaignDirectory = join(root, 'campaigns', 'prepared-run'); + const now = '2026-08-25T12:00:00.000Z'; + const claimed = claimNextAttempt(createCampaignState(storedPlan, { now }), + { now, admissionId: 'old' }); + assert.ok(claimed.claim); + const storedState = finishCampaignExecution(claimed.state, claimed.claim.executionId, { + exitCode: 1, + run: { outcome: { kind: 'harness_failure', reason: 'controller stopped' } }, + retryAuthority: { transient: true, recoveryClean: true, budgetKnown: true, + cause: 'controller stopped' }, + }, { now: '2026-08-25T12:00:01.000Z', retries: 1, retryOn: ['harness_failure'] }); + writeCampaign(campaignDirectory, storedPlan, storedState); + const { server } = createDashboardServer({ resultsRoot: root, plansRoot, allowLaunch: true, + token: 'test-session-token', + feed, plans: () => [frozenPlan], + launch(input) { launches.push(input); const child = Object.assign(new EventEmitter(), { pid: 1234 }); + return child; } }); + const origin = await listenOrigin(server); + t.after(() => { server.close(); rmSync(root, { recursive: true, force: true }); }); + + const page = await fetch(origin); + assert.equal(page.status, 200); + const pageHtml = await page.text(); + assert.match(pageHtml, /Stack Bench<\/title>/); + assert.match(pageHtml, /src="\/app\.js"/); + const sessionResponse = await fetch(`${origin}/api/session`); + assert.equal(sessionResponse.headers.get('cache-control'), 'no-store'); + assert.deepEqual(await sessionResponse.json(), { canStart: true, csrfToken: 'test-session-token' }); + const overview = await (await fetch(`${origin}/api/overview`)).json() as { + canStart: boolean; + csrfToken: string; + campaigns: Array<{ key: string }>; + }; + assert.equal(overview.canStart, true); + assert.equal(overview.csrfToken, 'test-session-token'); + assert.doesNotMatch(JSON.stringify(overview), /test-control-secret-value/); + assert.deepEqual(overview.campaigns.map(campaign => campaign.key), ['prepared-run']); + const served = await (await fetch(`${origin}/api/plans`)).json() as Array<{ id: string }>; + assert.deepEqual(served.map(plan => plan.id), [frozenPlan.id]); + + const reboundStatus = await new Promise((resolveStatus, reject) => { + const rebound = httpRequest(`${origin}/api/overview`, { headers: { host: 'attacker.invalid' } }, + response => { response.resume(); response.once('end', () => resolveStatus(response.statusCode)); }); + rebound.once('error', reject); + rebound.end(); + }); + assert.equal(reboundStatus, 421); + + const rejected = await fetch(`${origin}/api/runs`, { method: 'POST', + headers: { origin, 'content-type': 'application/json', + 'x-stack-bench-token': 'wrong-token' }, + body: JSON.stringify({ planId: frozenPlan.id, outputName: 'meeting-run-1' }) }); + assert.equal(rejected.status, 403); + assert.equal(launches.length, 0); + + const wrongOrigin = await fetch(`${origin}/api/runs`, { method: 'POST', + headers: { origin: 'https://attacker.invalid', 'content-type': 'application/json', + 'x-stack-bench-token': 'test-session-token' }, + body: JSON.stringify({ planId: frozenPlan.id, outputName: 'meeting-run-1' }) }); + assert.equal(wrongOrigin.status, 403); + assert.equal(launches.length, 0); + + const resumed = await fetch(`${origin}/api/campaigns/prepared-run/resume`, { method: 'POST', + headers: { origin, 'content-type': 'application/json', + 'x-stack-bench-token': 'test-session-token' }, body: '{}' }); + assert.equal(resumed.status, 202); + assert.equal(launches.length, 1); + assert.equal(launches[0]?.output, campaignDirectory); + assert.equal(launches[0]?.command, 'resume'); + assert.equal(((await resumed.json()) as { type?: string }).type, 'campaign.resume'); + const duplicate = await fetch(`${origin}/api/campaigns/prepared-run/resume`, { method: 'POST', + headers: { origin, 'content-type': 'application/json', + 'x-stack-bench-token': 'test-session-token' }, body: '{}' }); + assert.equal(duplicate.status, 409); + assert.equal(launches.length, 1); +}); + +test('host development mode is read-only even with a valid browser request', async t => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-dashboard-readonly-')); + mkdirSync(join(root, 'plans'), { recursive: true }); + const feed = { append() {}, list() { return []; } }; + const { server } = createDashboardServer({ resultsRoot: root, plansRoot: join(root, 'plans'), + allowLaunch: false, token: 'readonly-token', feed, plans: () => [] }); + const origin = await listenOrigin(server); + t.after(() => { server.close(); rmSync(root, { recursive: true, force: true }); }); + const response = await fetch(`${origin}/api/runs`, { method: 'POST', + headers: { origin, 'content-type': 'application/json', 'x-stack-bench-token': 'readonly-token' }, + body: JSON.stringify({ planId: 'anything', outputName: 'meeting-run-2' }) }); + assert.equal(response.status, 503); +}); + +test('the overview stays a summary and the sheet stays one campaign at appliance scale', () => { + const resultsRoot = shared.completed; + const campaignsRoot = join(resultsRoot, 'campaigns'); + const controllerActive = (): boolean => true; + + const overview = overviewSummary(campaignsRoot, { controllerActive }); + const first = overview.find(campaign => campaign.key === 'fixture-run-0'); + assert.ok(first && 'scores' in first); + assert.equal(Object.keys(first.scores).length, 3); + assert.equal(first.attempts.total, 9); + assert.ok(Object.values(first.scores).every(score => score !== null)); + assert.equal(JSON.stringify(overview).includes('process.stdout.log'), false); + const overviewBytes = Buffer.byteLength(JSON.stringify(overview)); + assert.ok(overviewBytes < 12 * 1024, `overview payload is ${overviewBytes} bytes`); + const startedAt = performance.now(); + overviewSummary(campaignsRoot, { controllerActive }); + const warmMs = performance.now() - startedAt; + assert.ok(warmMs < 150, `warm overview took ${warmMs.toFixed(1)}ms`); + + const sheet = campaignSheet(resultsRoot, 'fixture-run-0', { controllerActive }); + assert.equal(sheet.mode, 'sequential'); + assert.equal(sheet.facts.timeLimitMinutes, 240); + assert.equal(sheet.stacks.length, 3); + const stack = sheet.stacks[0]; + assert.ok(stack); + assert.equal(stack.n, 3); + assert.ok(stack.score !== null && stack.unaided !== null); + assert.ok(stack.unaided < stack.score); + assert.deepEqual(stack.levels?.map(level => level.level), [1]); + assert.equal(stack.questlines, null); + assert.equal(stack.attempts.length, 3); + assert.ok(stack.attempts.some(attempt => attempt.climb.length >= 2)); + const sheetBytes = Buffer.byteLength(JSON.stringify(sheet)); + assert.ok(sheetBytes < 60 * 1024, `campaign sheet is ${sheetBytes} bytes`); +}); + +test('a liveness probe that cannot answer still reads the campaign', () => { + const resultsRoot = shared.running; + const controllerActive = (): boolean => { + throw new Error('failed to connect to the docker API at unix:///var/run/docker.sock'); + }; + + const campaign = overviewSummary(join(resultsRoot, 'campaigns'), { controllerActive }) + .find(entry => entry.key === 'fixture-run-0'); + assert.ok(campaign && 'scores' in campaign); + assert.equal(campaign.status, 'running'); + assert.equal(campaignSheet(resultsRoot, 'fixture-run-0', { controllerActive }).status, 'running'); +}); + +test('the overview caches a running campaign until its evidence changes', () => { + const resultsRoot = shared.running; + const campaignsRoot = join(resultsRoot, 'campaigns'); + const controllerActive = (): boolean => true; + + const first = overviewSummary(campaignsRoot, { controllerActive }) + .find(campaign => campaign.key === 'fixture-run-0'); + assert.ok(first && 'scores' in first && first.status === 'running'); + const cached = overviewSummary(campaignsRoot, { controllerActive }) + .find(campaign => campaign.key === 'fixture-run-0'); + assert.equal(cached, first); + + const attempts = join(campaignsRoot, 'fixture-run-0', 'attempts'); + const attempt = readdirSync(attempts)[0]; + assert.ok(attempt); + const run = join(attempts, attempt, 'execution-1', 'run.json'); + const later = new Date(Date.now() + 5000); + utimesSync(run, later, later); + const refreshed = overviewSummary(campaignsRoot, { controllerActive }) + .find(campaign => campaign.key === 'fixture-run-0'); + assert.ok(refreshed); + assert.notEqual(refreshed, first); + assert.deepEqual(refreshed, first); +}); + +test('dashboard pause state validates the receipt and observes resume changes', t => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-dashboard-pause-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const plan = dependencyPlan(); + const claimed = claimNextAttempt(createCampaignState(plan), { admissionId: 'pause-test' }); + assert.ok(claimed.claim); + plan.definition.mode.pauseAfterDepth = 2; + const attempt = claimed.state.attempts.find(item => item.plan.id === claimed.claim!.attempt.id)!; + const marker = 'c'.repeat(64); + writeFileSync(join(root, '.campaign.lock.json'), JSON.stringify({ version: 2, + campaignId: plan.id, campaignSha256: plan.contentSha256, ownerPid: process.pid, + ownerInstance: 'dashboard-test', ownershipMarkerSha256: marker, + acquiredAt: new Date().toISOString() })); + const output = join(root, claimed.claim.output); + mkdirSync(output, { recursive: true }); + assert.equal(attemptPause(plan, attempt, root), null); + const receipt = { campaignSha256: plan.contentSha256, + ownershipMarkerSha256: marker, depth: 2, + attemptId: attempt.plan.id, executionId: claimed.claim.executionId, + startedAt: 1000, resumedAt: null as number | null, + sourceSha256: 'a'.repeat(64), progressionSha256: 'b'.repeat(64) }; + const path = join(output, 'depth-pause.json'); + writeFileSync(path, JSON.stringify(receipt)); + assert.equal(attemptPause(plan, attempt, root)?.resumedAt, null); + receipt.resumedAt = 2000; + writeFileSync(path, JSON.stringify(receipt)); + assert.equal(attemptPause(plan, attempt, root)?.resumedAt, 2000); + receipt.attemptId = 'wrong-attempt'; + writeFileSync(path, JSON.stringify(receipt)); + assert.throws(() => attemptPause(plan, attempt, root), /attemptId changed/); +}); + +test('the sheet reports dependency submodes and questlines in definition order', t => { + const resultsRoot = mkdtempSync(join(tmpdir(), 'stack-bench-dashboard-sheet-dependency-')); + t.after(() => rmSync(resultsRoot, { recursive: true, force: true })); + const plan = dependencyPlan(); + const now = '2026-08-25T12:00:00.000Z'; + const claimed = claimNextAttempt(createCampaignState(plan, { now }), + { now, admissionId: 'admission-1' }); + assert.ok(claimed.claim); + const directory = join(resultsRoot, 'campaigns', 'dependency-run'); + const output = join(directory, claimed.claim.output); + mkdirSync(join(output, 'source'), { recursive: true }); + writeCampaign(directory, plan, claimed.state); + assert(plan.featureCatalog && plan.dependencyPolicy); + const attemptPlan = claimed.claim.attempt; + writeProgressionState(join(output, 'progression-state.json'), { + progression: compileProgressionInput(dependencyRuntimeDefinition( + plan.featureCatalog, plan.dependencyPolicy)), + featureCatalogIdentity: plan.featureCatalog.identity, + dependencyPolicyIdentity: plan.dependencyPolicy.identity, + owner: { schemaVersion: 1, + campaign: { id: plan.id, version: plan.version, sha256: plan.contentSha256 }, + attempt: { id: attemptPlan.id, track: plan.definition.track, stack: attemptPlan.stack, + agentAdapter: attemptPlan.agentAdapter, model: attemptPlan.model, + conditionSha256: attemptPlan.condition.contentSha256 }, + workspace: { appDirectory: 'source' } }, + state: progressionEngine.initialize(compileProgressionInput(dependencyRuntimeDefinition( + plan.featureCatalog, plan.dependencyPolicy)).definition) }); + + const sheet = campaignSheet(resultsRoot, 'dependency-run', { controllerActive: () => true }); + assert.equal(sheet.mode, 'dependency'); + assert.equal(sheet.facts.workSelection, 'progressive'); + assert.equal(sheet.facts.repairSelection, 'feature'); + assert.equal(sheet.facts.guidance, attemptPlan.guidance); + const stack = sheet.stacks.find(item => item.questlines !== null); + assert.ok(stack?.questlines); + assert.equal(stack.levels, null); + assert.ok(stack.questlines.length > 0); + const questline = stack.questlines[0]; + assert.ok(questline); + assert.ok(questline.nodes.length > 0); + assert.ok(questline.nodes.every(node => typeof node.status === 'string')); + assert.equal(stack.attempts.find(attempt => attempt.id === stack.selectedAttemptId)?.repairs.budget, 0); + assert.equal(stack.n, 0); + assert.equal(stack.completionRate, null, 'running progress is not a completed comparison'); + assert.equal(stack.regressions, null, 'no eligible runs means no regression statistic'); + const bytes = Buffer.byteLength(JSON.stringify(sheet)); + assert.ok(bytes < 60 * 1024, `dependency sheet is ${bytes} bytes`); + + const summary = summarizeCampaign(directory, { includePackage: true }); + assert.equal(summary.mode, 'dependency'); + const firstAttempt = summary.attempts[0]; + assert.ok(firstAttempt?.dependency); + assert.deepEqual(firstAttempt.dependency.activeDepths, [1]); + assert.deepEqual(firstAttempt.dependency.history, + { firstTryPercentage: 0, repairAttempts: 0 }); + assert(firstAttempt.dependency.work.current.some(node => node.id === 'accounts')); + assert.deepEqual(firstAttempt.dependency.attempts, + { total: 0, maxRemaining: 0, features: [] }); + assert.ok(summary.package); + assert(summary.package.executions[0]?.artifacts + .some(item => item.path.endsWith('/progression-state.json'))); +}); + +test('log-only refresh keeps the evidence sheet cached and returns live fields only', t => { + const resultsRoot = mkdtempSync(join(tmpdir(), 'stack-bench-live-fields-')); + t.after(() => rmSync(resultsRoot, { recursive: true, force: true })); + const key = 'live-fields'; + const plan = examplePlan(); + const claimed = claimNextAttempt(createCampaignState(plan), { admissionId: 'test' }); + assert.ok(claimed.claim); + const directory = join(resultsRoot, 'campaigns', key); + writeCampaign(directory, plan, claimed.state); + const output = join(directory, claimed.claim.output); + writeRunEvidence(output, plan, claimed.claim.attempt, 0); + const options = { controllerActive: () => true }; + const sheet = campaignSheet(resultsRoot, key, options); + appendFileSync(join(output, 'process.stdout.log'), '\n--- repair 2/3 ---\n'); + const later = new Date(Date.now() + 1000); + utimesSync(join(output, 'process.stdout.log'), later, later); + const update = campaignLiveUpdate(resultsRoot, key, options); + assert.equal(campaignSheet(resultsRoot, key, options), sheet, 'log writes must not replay the evidence'); + const attempt = update.stacks.flatMap(stack => stack.attempts).find(entry => entry.id === claimed.claim!.attempt.id)!; + assert.equal(attempt.logUpdatedAt, later.toISOString()); + assert.match(attempt.phase, /repair/i); + assert.ok(!('completion' in attempt) && !('checks' in attempt) && !('nodes' in update)); + assert.ok(Buffer.byteLength(JSON.stringify(update)) < 4096); + writeCampaign(directory, plan, finishCampaignExecution(claimed.state, claimed.claim.executionId, + { exitCode: 0, run: { outcome: { kind: 'passed' } } })); + const finished = campaignSheet(resultsRoot, key, options); + appendFileSync(join(output, 'process.stdout.log'), ' TOTAL ... 1/1\n'); + utimesSync(join(output, 'process.stdout.log'), new Date(later.getTime() + 1000), new Date(later.getTime() + 1000)); + const finalLog = campaignLiveUpdate(resultsRoot, key, options).stacks.flatMap(stack => stack.attempts)[0]!; + assert.equal(finalLog.climb.length, attempt.climb.length + 1, 'a late final log write must remain visible'); + assert.equal(campaignSheet(resultsRoot, key, options), finished); +}); + +function writeSingleAttemptCampaign(resultsRoot: string, key: string): string { + const plan = examplePlan(); + const now = '2026-08-18T12:00:00.000Z'; + const claimed = claimNextAttempt(createCampaignState(plan, { now }), + { now, admissionId: 'admission-1' }); + assert.ok(claimed.claim); + const state = finishCampaignExecution(claimed.state, claimed.claim.executionId, + { exitCode: 0, run: { outcome: { kind: 'passed' } } }, { now }); + const directory = join(resultsRoot, 'campaigns', key); + writeCampaign(directory, plan, state); + const output = join(directory, claimed.claim.output); + mkdirSync(output, { recursive: true }); + writeRunEvidence(output, plan, claimed.claim.attempt, 1); + writeFileSync(join(output, 'process.stdout.log'), + `authorization: Bearer secret-token-value\n${'log line\n'.repeat(8)}`); + return claimed.claim.attempt.id; +} + +test('the sheet says a dependency campaign that stopped between executions can resume', t => { + const resultsRoot = mkdtempSync(join(tmpdir(), 'stack-bench-resume-')); + t.after(() => rmSync(resultsRoot, { recursive: true, force: true })); + const plan = dependencyPlan(); + const now = '2026-08-25T12:00:00.000Z'; + const claimed = claimNextAttempt(createCampaignState(plan, { now }), + { now, admissionId: 'admission-1' }); + assert.ok(claimed.claim); + const stopped = finishCampaignExecution(claimed.state, claimed.claim.executionId, { + exitCode: 1, + run: { outcome: { kind: 'harness_failure', reason: 'controller stopped' } }, + retryAuthority: { transient: true, recoveryClean: true, budgetKnown: true, + cause: 'controller stopped' }, + }, { now, retries: 1, retryOn: ['harness_failure'] }); + writeCampaign(join(resultsRoot, 'campaigns', 'prepared-run'), plan, stopped); + const sheet = campaignSheet(resultsRoot, 'prepared-run', { controllerActive: () => false }); + assert.equal(sheet.status, 'prepared'); + assert.equal(sheet.executions, 1); + assert.equal(sheet.resumable, true); + // A campaign that has run nothing, and a sequential one, have nothing to resume. + writeCampaign(join(resultsRoot, 'campaigns', 'fresh-run'), plan, createCampaignState(plan, { now })); + assert.equal(campaignSheet(resultsRoot, 'fresh-run', { controllerActive: () => false }) + .resumable, false); + assert.equal(campaignSheet(shared.completed, 'fixture-run-0', { controllerActive: () => false }) + .resumable, false); +}); + +test('every view has a route, and a name that is not a campaign never reaches the evidence', + async t => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-dashboard-routes-')); + const attemptId = writeSingleAttemptCampaign(root, 'route-run'); + const feed = { append() {}, list() { return []; } }; + const { server } = createDashboardServer({ resultsRoot: root, plansRoot: join(root, 'plans'), + allowLaunch: false, token: 'route-token', feed, plans: () => [] }); + const origin = await listenOrigin(server); + t.after(() => { server.close(); rmSync(root, { recursive: true, force: true }); }); + + const overview = await fetch(`${origin}/api/overview`); + assert.equal(overview.headers.get('cache-control'), 'no-store'); + const summary = await overview.json() as { campaigns: Array<{ key: string }>; + canStart: boolean; csrfToken: string }; + assert.deepEqual(summary.campaigns.map(campaign => campaign.key), ['route-run']); + assert.equal(summary.canStart, false); + assert.equal(summary.csrfToken, 'route-token'); + assert.equal('plans' in summary, false); + assert.deepEqual(await (await fetch(`${origin}/api/plans`)).json(), []); + + const sheet = await (await fetch(`${origin}/api/campaigns/route-run`)).json() as { + key: string; stacks: Array<{ stack: string }> }; + assert.equal(sheet.key, 'route-run'); + assert.equal(sheet.stacks.length, 3); + + const checks = await (await fetch( + `${origin}/api/campaigns/route-run/attempts/${attemptId}/checks`)).json() as { + checks: Array<{ outcome: string; history: unknown[]; regressed: boolean }> }; + assert.ok(checks.checks.length > 0); + assert.ok(checks.checks.every(check => check.outcome === 'pass')); + assert.ok(checks.checks.every(check => check.history.length === 1)); + assert.equal(checks.checks.some(check => check.regressed), false); + const evidence = await (await fetch( + `${origin}/api/campaigns/route-run/attempts/${attemptId}/package`)).json() as { + executions: Array<{ artifacts: Array<{ path: string }> }> }; + assert.equal(evidence.executions.length, 1); + assert.equal(evidence.executions[0]!.artifacts.some(item => item.path.includes('/source/')), false); + + const log = await fetch(`${origin}/api/campaigns/route-run/attempts/${attemptId}/log?from=0`); + assert.equal(log.headers.get('content-type'), 'text/plain; charset=utf-8'); + const offset = Number(log.headers.get('x-stack-bench-log-offset')); + const text = await log.text(); + assert.match(text, /\[redacted credential\]/); + assert.doesNotMatch(text, /secret-token-value/); + assert.ok(offset > 0); + const rest = await fetch( + `${origin}/api/campaigns/route-run/attempts/${attemptId}/log?from=${offset}`); + assert.equal(await rest.text(), ''); + assert.equal(Number(rest.headers.get('x-stack-bench-log-offset')), offset); + + assert.equal((await fetch(`${origin}/api/campaigns/route-run/progression`)).status, 404); + assert.equal((await fetch(`${origin}/api/campaigns/missing-run`)).status, 404); + assert.equal((await fetch(`${origin}/api/campaigns/Route-Run`)).status, 400); + assert.equal((await fetch( + `${origin}/api/campaigns/route-run/attempts/missing-attempt/checks`)).status, 404); + assert.equal((await fetch( + `${origin}/api/campaigns/route-run/attempts/${attemptId}/log?from=-1`)).status, 400); + assert.equal((await fetch( + `${origin}/api/campaigns/route-run/attempts/${attemptId}/log?from=eight`)).status, 400); + const missing = await (await fetch(`${origin}/api/campaigns/missing-run`)).json() as { + error: string }; + assert.equal(missing.error, 'Not found'); + writeFileSync(join(root, 'campaigns', 'route-run', 'state.json'), '{broken'); + const damaged = await fetch(`${origin}/api/campaigns/route-run`); + assert.equal(damaged.status, 422); + assert.match(String((await damaged.json() as { error: string }).error), /Cannot read campaign evidence/); + + for (const path of ['/', '/plans', '/c/route-run', `/c/route-run/a/${attemptId}`]) { + const page = await fetch(`${origin}${path}`); + assert.equal(page.status, 200, path); + assert.equal(page.headers.get('cache-control'), 'no-store'); + assert.match(await page.text(), /<title>Stack Bench/); + } + }); + +async function nextServerEvents(origin: string, signal: AbortSignal, count: number, + deadlineMs: number): Promise<string[]> { + const response = await fetch(`${origin}/api/events`, { signal }); + assert.equal(response.headers.get('content-type'), 'text/event-stream; charset=utf-8'); + assert.ok(response.body); + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + const events: string[] = []; + const deadline = setTimeout(() => { void reader.cancel(); }, deadlineMs); + let buffered = ''; + try { + while (events.length < count) { + const chunk = await reader.read(); + if (chunk.done) break; + buffered += decoder.decode(chunk.value, { stream: true }); + const frames = buffered.split('\n\n'); + buffered = frames.pop() ?? ''; + for (const frame of frames) { + const match = frame.match(/^event: (\w+)\ndata: (.+)$/); + if (match) events.push(`${match[1]} ${match[2]}`); + } + } + } finally { + clearTimeout(deadline); + } + return events; +} + +test('the event stream reports a campaign write and the log bytes that follow it', async t => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-dashboard-events-')); + const attemptId = writeSingleAttemptCampaign(root, 'event-run'); + const feed = { append() {}, list() { return []; } }; + const { server } = createDashboardServer({ resultsRoot: root, plansRoot: join(root, 'plans'), + allowLaunch: false, token: 'event-token', feed, plans: () => [] }); + const origin = await listenOrigin(server); + const abort = new AbortController(); + t.after(() => { abort.abort(); server.close(); rmSync(root, { recursive: true, force: true }); }); + + let recursive = true; + try { watch(root, { recursive: true, persistent: false }).close(); } + catch { recursive = false; } + // Where the platform has no recursive watch the watcher polls every five + // seconds instead; the same events arrive. + const deadlineMs = recursive ? 1000 : 8000; + + const execution = join(root, 'campaigns', 'event-run', 'attempts', attemptId, 'execution-1'); + const events = nextServerEvents(origin, abort.signal, 2, deadlineMs); + await new Promise(done => setTimeout(done, 50)); + const run = join(execution, 'run.json'); + writeFileSync(run, readFileSync(run, 'utf8')); + appendFileSync(join(execution, 'process.stdout.log'), 'another line\n'); + assert.deepEqual(await events, ['campaign {"key":"event-run"}', + `log {"key":"event-run","attemptId":"${attemptId}"}`]); +}); + +test('the progression view replays the graph once per stack within its budget', t => { + const resultsRoot = mkdtempSync(join(tmpdir(), 'stack-bench-dashboard-progression-')); + t.after(() => rmSync(resultsRoot, { recursive: true, force: true })); + const plan = dependencyPlan(); + assert.ok(plan.featureCatalog && plan.dependencyPolicy); + const now = '2026-08-25T12:00:00.000Z'; + const directory = join(resultsRoot, 'campaigns', 'progression-run'); + let state = createCampaignState(plan, { now }); + const claims = []; + for (;;) { + const claimed = claimNextAttempt(state, { now, admissionId: 'admission-1' }); + if (!claimed.claim) break; + claims.push(claimed.claim); + state = claimed.state; + } + writeCampaign(directory, plan, state); + const progression = compileProgressionInput(dependencyRuntimeDefinition( + plan.featureCatalog, plan.dependencyPolicy)); + const finalScores = new Map<string, number | null>(); + for (const claim of claims) { + const output = join(directory, claim.output); + mkdirSync(join(output, 'source'), { recursive: true }); + let timedState = progressionEngine.initialize(progression.definition); + for (const [index, event] of dependencyProgressionEvidence(plan, claim.attempt).events.entries()) { + if (!event.result) continue; + const completedAt = new Date(Date.parse(now) + (index + 1) * 60_000).toISOString(); + // Repair observations can share an ID; the evidence digest identifies the version. + const artifact = writeArtifact(join(output, 'progression', + `attempt-${String(index + 1).padStart(3, '0')}`, ARTIFACT_FILE.gradeBundle), { + kind: 'grade_bundle', id: 'shared-grade-id', + timestamps: { startedAt: now, completedAt }, payload: { suites: {} }, + }); + timedState = progressionEngine.recordResult(timedState, { ...event.result, + runId: claim.attempt.id, attemptId: `${claim.attempt.id}-progression-${index + 1}`, + evidence: { kind: 'grade_bundle', id: artifact.id, + sha256: sha256(canonicalDefinitionJson(artifact)) } }); + } + writeProgressionState(join(output, 'progression-state.json'), { + progression, + featureCatalogIdentity: plan.featureCatalog.identity, + dependencyPolicyIdentity: plan.dependencyPolicy.identity, + owner: { schemaVersion: 1, + campaign: { id: plan.id, version: plan.version, sha256: plan.contentSha256 }, + attempt: { id: claim.attempt.id, track: plan.definition.track, + stack: claim.attempt.stack, agentAdapter: claim.attempt.agentAdapter, + model: claim.attempt.model, conditionSha256: claim.attempt.condition.contentSha256 }, + workspace: { appDirectory: 'source' } }, + state: timedState }); + finalScores.set(claim.attempt.id, progressionEngine.score(timedState).uniqueChecks.percentage); + } + + const view = campaignProgression(resultsRoot, 'progression-run'); + assert.ok(view); + assert.deepEqual(view.depths, plan.definition.levels); + assert.ok(view.nodes.length > 0); + assert.ok(view.nodes.every(node => view.depths.includes(node.depth))); + assert.equal(view.stacks.length, claims.length); + const track = view.stacks[0]; + assert.ok(track); + assert.ok(track.steps.length > 1); + // The replay score is passed points over selected points, as the weighted score is. + const finalScore = finalScores.get(track.attemptId); + assert.equal(typeof finalScore, 'number'); + assert.equal(track.steps.at(-1)?.score, Math.round(finalScore! * 10) / 10); + assert.deepEqual(track.steps.map(step => step.completedAt), track.steps.map((_, index) => + new Date(Date.parse(now) + (index + 1) * 60_000).toISOString())); + assert.ok(track.steps.every(step => step.statuses.length === view.nodes.length)); + assert.ok(track.steps.every(step => step.featureCompletion === + step.statuses.filter(status => status === 'passed').length / view.nodes.length)); + const sheet = campaignSheet(resultsRoot, 'progression-run'); + const attemptView = sheet.stacks.flatMap(stack => stack.attempts).find(attempt => attempt.id === track.attemptId)!; + assert.equal(attemptView.featureCompletion?.rate, track.steps.at(-1)?.featureCompletion); + assert.equal(Object.values(attemptView.checkCategories ?? {}).reduce((total, category) => total + category.selected, 0), attemptView.completion?.selected); + assert.ok(track.steps.some(step => step.action === 'build')); + assert.equal(track.steps.some(step => step.action === 'repair'), false); + assert.ok(track.steps.at(-1)?.statuses.includes('passed')); + const graphHtml = graph(view, [{ stack: track.stack, statuses: view.nodes.map(() => 'working') }]); + assert.match(graphHtml, /feature checks passed; guarantees incomplete/); + assert.doesNotMatch(graphHtml, / working/); + assert.equal(campaignProgression(resultsRoot, 'progression-run'), view); + const bytes = Buffer.byteLength(JSON.stringify(view)); + assert.ok(bytes < 80 * 1024, `progression payload is ${bytes} bytes`); +}); + +// The client is a set of pure functions, so the rules the pages are held to — +// no prose, no badge, one value per cell, stacks in fixed order — are asserted +// here on rendered HTML rather than in a browser. + +test('cost and completion keep unknown spend and the full selected scope visible', () => { + const sheet = structuredClone(campaignSheet(shared.completed, 'fixture-run-0', { controllerActive: () => true })); + const stack = sheet.stacks[0]!; + const attempt = stack.attempts[0]!; + attempt.status = 'running'; + attempt.model = '<script>bad()</script>'; + attempt.completion = { selected: 107, passed: 1, failed: 0, blocked: 0, + unmeasured: 106, rate: 1 / 107 }; + stack.completionRate = 1 / 107; + attempt.spend = stack.spend = { status: 'upper-bound', costUsd: 5 }; + const campaign = campaignPage({ sheet, progression: null, view: 'grid', step: 0 }); + assert.match(campaign, /≤\$5\.00/); + assert.match(campaign, /1 \/ 107/); + assert.match(campaign, /<script>bad\(\)<\/script>/); + attempt.spend = stack.spend = { status: 'unknown', costUsd: null }; + assert.match(campaignPage({ sheet, progression: null, view: 'grid', step: 0 }), /Unknown/); + sheet.mode = 'dependency'; + attempt.status = 'completed'; + stack.questlines = [{ id: 'catalog', title: 'Catalog', score: 0, + nodes: [{ id: 'items', status: 'working' }] }]; + assert.match(campaignPage({ sheet, progression: null, view: 'grid', step: 0 }), + /items: feature checks passed; guarantees incomplete/); + const input = { sheet, attemptId: attempt.id, tab: 'checks' as const, + checks: null, evidence: null, log: '' }; + const detail = attemptPage(input); + assert.match(detail, /Feature dependencies/); + assert.match(detail, /1 \/ 107/); + assert.equal(attempt.completion.unmeasured, 106); + attempt.completion = null; + assert.doesNotMatch(attemptPage(input), /count without an accepted outcome/); +}); + +test('time controls require authorization and validate minutes before requesting a grant', async t => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-dashboard-time-')); + const { server } = createDashboardServer({ resultsRoot: root, plansRoot: join(root, 'plans'), + allowLaunch: true, token: 'time-token' }); + const origin = await listenOrigin(server); + t.after(() => { server.close(); rmSync(root, { recursive: true, force: true }); }); + const send = (token: string, minutes: number) => fetch(`${origin}/api/campaigns/time-run/attempts/attempt-1/time`, { + method: 'POST', headers: { origin, 'content-type': 'application/json', + 'x-stack-bench-token': token }, + body: JSON.stringify({ grantId: 'time-1', minutes }), + }); + assert.equal((await send('wrong', 120)).status, 403); + for (const minutes of [0, -1, 1.5, Number.MAX_SAFE_INTEGER]) { + assert.equal((await send('time-token', minutes)).status, 400); + } +}); + +test('Stop requires the browser token and cannot cancel a successor controller', + { skip: process.platform !== 'linux' ? 'native campaign control is tested in the Linux appliance' : false }, async t => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-dashboard-stop-')); + const directory = join(root, 'campaigns', 'stop-run'); + const plan = examplePlan(); + writeCampaign(directory, plan, createCampaignState(plan)); + let lock = acquireCampaignLock(directory, plan); + const { server } = createDashboardServer({ resultsRoot: root, plansRoot: join(root, 'plans'), + allowLaunch: true, token: 'stop-token' }); + const origin = await listenOrigin(server); + t.after(() => { server.close(); releaseCampaignLock(lock); rmSync(root, { recursive: true, force: true }); }); + const send = (owner: string, token: string) => fetch(`${origin}/api/campaigns/stop-run/stop`, { + method: 'POST', headers: { origin, 'content-type': 'application/json', + 'x-stack-bench-token': token }, + body: JSON.stringify({ owner }), + }); + const prior = lock.record.ownershipMarkerSha256; + assert.equal((await send(prior, 'wrong')).status, 403); + assert.equal(campaignCancellationRequested(lock), false); + assert.equal((await send(prior, 'stop-token')).status, 202); + assert.equal(campaignCancellationRequested(lock), true); + releaseCampaignLock(lock); + lock = acquireCampaignLock(directory, plan); + assert.equal((await send(prior, 'stop-token')).status, 409); + assert.equal(campaignCancellationRequested(lock), false); + }); + +test('the client renders controls, evidence links, and every supported page', () => { + const resultsRoot = shared.completed; + const controllerActive = (): boolean => true; + const overview = overviewSummary(join(resultsRoot, 'campaigns'), { controllerActive }); + const sequential = campaignSheet(resultsRoot, 'fixture-run-0', { controllerActive }); + const dependency = campaignSheet(resultsRoot, 'progression-run', { controllerActive }); + const progression = campaignProgression(resultsRoot, 'progression-run'); + assert.ok(progression); + const attempt = sequential.stacks[0]?.attempts[0]; + assert.ok(attempt); + const screenshotEvidence = attemptPackage(resultsRoot, 'fixture-run-0', attempt.id); + screenshotEvidence.executions[0]?.visuals.push({ id: 'screenshot-id', + path: 'grading/failure-media/example.png', name: 'Example failure', kind: 'visual', + contentType: 'image/png', size: 8 }); + const plans = discoverPlans(join(resultsRoot, 'plans')); + const pages: Array<[string, string]> = [ + ['campaigns', campaignsPage({ campaigns: overview, filter: 'all' })], + ['campaigns filtered', + campaignsPage({ campaigns: overview, filter: 'completed' })], + ['campaign sequential', + campaignPage({ sheet: sequential, progression: null, view: 'grid', step: 0 })], + ['campaign grid', campaignPage({ sheet: dependency, progression, view: 'grid', step: 0 })], + ['campaign graph', campaignPage({ sheet: dependency, progression, view: 'graph', step: 0 })], + ['campaign replay', campaignPage({ sheet: dependency, progression, view: 'replay', step: 3 })], + ['plans', plansPage({ plans })], + ['topbar plans', topbar({ page: 'plans', key: '', canStart: true, resumable: false, error: '' })], + ['topbar resume', topbar({ page: 'campaign', key: 'progression-run', canStart: true, + resumable: true, error: 'Only an interrupted campaign that is ready can resume.' })], + ['attempt', attemptPage({ sheet: sequential, attemptId: attempt.id, tab: 'checks', + checks: attemptChecks(resultsRoot, 'fixture-run-0', attempt.id), + evidence: attemptPackage(resultsRoot, 'fixture-run-0', attempt.id), log: '' })], + ['attempt screenshots', attemptPage({ sheet: sequential, attemptId: attempt.id, + tab: 'screenshots', checks: null, evidence: screenshotEvidence, log: '' })], + ]; + + for (const [name, html] of pages) assert.ok(html.length > 100, name); + const comparison = campaignPage({ sheet: { ...dependency, + facts: { ...dependency.facts, repairLimits: { perFeature: 5 } } }, + progression, view: 'grid', step: 0 }); + assert.match(comparison, /5 per feature/); + assert.match(comparison, /aria-label="About Total spend"/); + assert.match(comparison, /aria-label="Stack comparison" tabindex="0"/); + const screenshots = pages.find(([name]) => name === 'attempt screenshots')?.[1] ?? ''; + assert.match(screenshots, /<button type="button" data-shot="[^"]+"/); + assert.match(screenshots, /<dialog class="lightbox">/); + // Saved plans have no launch form; new runs use the setup page. + const readOnly = plansPage({ plans }); + assert.equal(/<form/.test(readOnly), false); + // An unreadable plan is a row whose state is invalid and whose error is a hover. + const invalid = /<span class="state \w+" title="([^"]+)">invalid</.exec(readOnly); + assert.ok(invalid?.[1] && invalid[1] !== 'invalid'); + // New run stays reachable read-only; its page explains that starting is unavailable. + assert.match(topbar({ page: 'campaigns', key: '', canStart: false, resumable: false, error: '' }), + /href="\/new">New run</); +}); + + +test('job endpoints authenticate writes, paginate reads, and queue without launching', async t => { + const root = mkdtempSync(join(tmpdir(), 'dashboard-jobs-')); + const plansRoot = join(root, 'plans'); + mkdirSync(plansRoot); + writeFileSync(join(plansRoot, 'test.json'), readFileSync(EXAMPLE_CAMPAIGN)); + let launches = 0; + const { server } = createDashboardServer({ resultsRoot: root, plansRoot, allowLaunch: true, + token: 'job-token', + launch() { launches++; throw new Error('queue submission must not launch'); } }); + const origin = await listenOrigin(server); + t.after(() => { server.close(); rmSync(root, { recursive: true, force: true }); }); + const headers = { origin, 'content-type': 'application/json', 'x-stack-bench-token': 'job-token' }; + const input = { key: 'dashboard-job', planFile: 'test.json' }; + assert.equal((await fetch(`${origin}/api/jobs`, { method: 'POST', body: JSON.stringify(input) })).status, 403); + assert.equal((await fetch(`${origin}/api/jobs`, { method: 'POST', headers, + body: JSON.stringify({ ...input, planFile: '../outside.json' }) })).status, 400); + const submitted = await fetch(`${origin}/api/jobs`, { method: 'POST', headers, body: JSON.stringify(input) }); + assert.equal(submitted.status, 202); + const job = await submitted.json() as { job: { id: string }; status: string }; + assert.equal(job.status, 'queued'); + assert.equal((await fetch(`${origin}/api/jobs/${job.job.id}`)).status, 200); + const page = await (await fetch(`${origin}/api/jobs?limit=1`)).json() as { jobs: unknown[]; next: unknown }; + assert.equal(page.jobs.length, 1); + const after = await (await fetch(`${origin}/api/jobs?after=${job.job.id}&limit=1`)).json() as { jobs: unknown[] }; + assert.deepEqual(after.jobs, []); + assert.equal((await fetch(`${origin}/api/jobs?limit=0`)).status, 400); + assert.equal((await fetch(`${origin}/api/jobs/${job.job.id}/cancel`, { method: 'POST' })).status, 403); + const cancelled = await fetch(`${origin}/api/jobs/${job.job.id}/cancel`, { method: 'POST', headers }); + assert.equal(cancelled.status, 202); + assert.equal((await cancelled.json() as { status: string }).status, 'cancelled'); + assert.equal(launches, 0); +}); + +function writeStoppedDependencyCampaign(directory: string, plan: CompiledCampaignPlan): void { + const now = '2026-08-25T12:00:00.000Z'; + const claimed = claimNextAttempt(createCampaignState(plan, { now }), { now, admissionId: 'old' }); + assert.ok(claimed.claim); + writeCampaign(directory, plan, finishCampaignExecution(claimed.state, claimed.claim.executionId, { + exitCode: 1, + run: { outcome: { kind: 'harness_failure', reason: 'controller stopped' } }, + retryAuthority: { transient: true, recoveryClean: true, budgetKnown: true, cause: 'controller stopped' }, + }, { now: '2026-08-25T12:00:01.000Z', retries: 1, retryOn: ['harness_failure'] })); +} + +test('Resume keeps a job campaign on its job and accepts a model-free draft', async t => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-dashboard-job-resume-')); + const plan = dependencyPlan(); + assert.equal(plan.state, 'draft'); + const draftPlan = { id: plan.id, title: plan.title, state: plan.state, mode: 'dependency', + sha256: plan.contentSha256, file: 'reference.json' }; + // The route needs only the job record; the job path itself is covered by the job CLI tests. + const jobId = sha256('resume-job'); + mkdirSync(join(root, 'jobs', jobId), { recursive: true }); + writeFileSync(join(root, 'jobs', jobId, 'job.json'), '{}'); + writeStoppedDependencyCampaign(join(root, 'campaigns', `job-${jobId}`), plan); + writeStoppedDependencyCampaign(join(root, 'campaigns', 'draft-run'), plan); + const launches: LaunchInput[] = []; + const feed = { append() {}, list() { return []; } }; + const { server } = createDashboardServer({ resultsRoot: root, plansRoot: join(root, 'plans'), + allowLaunch: true, token: 'resume-token', feed, plans: () => [draftPlan], + launch(input) { launches.push(input); return Object.assign(new EventEmitter(), { pid: 1 }); } }); + const origin = await listenOrigin(server); + t.after(() => { server.close(); rmSync(root, { recursive: true, force: true }); }); + const resume = (key: string) => fetch(`${origin}/api/campaigns/${key}/resume`, { method: 'POST', + headers: { origin, 'content-type': 'application/json', 'x-stack-bench-token': 'resume-token' }, body: '{}' }); + + assert.equal((await resume(`job-${jobId}`)).status, 202); + assert.equal(launches[0]?.command, 'resume'); + assert.equal(launches[0]?.jobId, jobId); + assert.equal((await resume('draft-run')).status, 202); + assert.equal(launches[1]?.jobId, undefined); + assert.equal(launches[1]?.plan.state, 'draft'); +}); + +test('bad client input on campaign controls is a client error', async t => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-dashboard-client-errors-')); + const attemptId = writeSingleAttemptCampaign(root, 'known-run'); + const feed = { append() {}, list() { return []; } }; + const { server } = createDashboardServer({ resultsRoot: root, plansRoot: join(root, 'plans'), + allowLaunch: true, token: 'input-token', feed, plans: () => [] }); + const origin = await listenOrigin(server); + t.after(() => { server.close(); rmSync(root, { recursive: true, force: true }); }); + const post = (path: string, body: string) => fetch(`${origin}/api/campaigns/${path}`, { method: 'POST', + headers: { origin, 'content-type': 'application/json', 'x-stack-bench-token': 'input-token' }, body }); + + assert.equal((await fetch(`${origin}/api/campaigns/known-run/attempts/${attemptId}/time`)).status, 200); + assert.equal((await fetch(`${origin}/api/campaigns/known-run/attempts/unknown-attempt/time`)).status, 404); + assert.equal((await fetch(`${origin}/api/campaigns/missing-run/attempts/${attemptId}/time`)).status, 404); + assert.equal((await fetch(`${origin}/api/campaigns/known-run/attempts/%E0%A4%A/time`)).status, 400); + assert.equal((await post(`known-run/attempts/${attemptId}/time`, '{')).status, 400); + assert.equal((await post(`known-run/attempts/${attemptId}/time`, 'x'.repeat(17 * 1024))).status, 400); + assert.equal((await post(`missing-run/attempts/${attemptId}/time`, + JSON.stringify({ minutes: 5, grantId: 'grant-1' }))).status, 404); + assert.equal((await post('known-run/stop', '{')).status, 400); + assert.equal((await post('missing-run/stop', JSON.stringify({ owner: 'a'.repeat(64) }))).status, 404); + assert.equal((await post('missing-run/resume', '{}')).status, 404); +}); + +test('the sheet links a report only while it describes the current state', t => { + const resultsRoot = mkdtempSync(join(tmpdir(), 'stack-bench-dashboard-report-')); + t.after(() => rmSync(resultsRoot, { recursive: true, force: true })); + const plan = examplePlan(); + const state = createCampaignState(plan, { now: '2026-08-18T12:00:00.000Z' }); + const directory = join(resultsRoot, 'campaigns', 'report-run'); + writeCampaign(directory, plan, state); + const options = { controllerActive: () => false }; + mkdirSync(join(directory, 'report')); + writeFileSync(join(directory, 'report', 'report.html'), '<p>report</p>'); + assert.deepEqual(campaignSheet(resultsRoot, 'report-run', options).reportFiles, []); + writeArtifact(join(directory, 'report', 'report.json'), { kind: 'campaign_report', id: 'report-run-report', + timestamps: { startedAt: state.createdAt, completedAt: state.updatedAt }, + identities: emptyArtifactIdentities({ experiment: { id: plan.id, version: plan.version, + sha256: plan.contentSha256, state: plan.state } }), payload: { reportSchemaVersion: 1 } }); + assert.deepEqual(campaignSheet(resultsRoot, 'report-run', options).reportFiles, ['report/report.html']); + writeCampaign(directory, plan, { ...state, updatedAt: '2026-08-18T13:00:00.000Z' }); + const later = new Date(Date.now() + 5000); + utimesSync(join(directory, 'state.json'), later, later); + assert.deepEqual(campaignSheet(resultsRoot, 'report-run', options).reportFiles, []); +}); diff --git a/tools/stack-bench/tests/dashboard/form-navigation.browser.ts b/tools/stack-bench/tests/dashboard/form-navigation.browser.ts new file mode 100644 index 00000000000..12ab3bd0982 --- /dev/null +++ b/tools/stack-bench/tests/dashboard/form-navigation.browser.ts @@ -0,0 +1,40 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { chromium } from 'playwright'; + +// Run against an isolated appliance with its setup presets installed: +// STACK_BENCH_BROWSER_TEST_URL=http://127.0.0.1:7331 node --test dist/tests/dashboard/form-navigation.browser.js +// All review submissions are intercepted. This test cannot start a model run. +test('form errors stay on their page, including responses received after navigation', async t => { + const origin = process.env.STACK_BENCH_BROWSER_TEST_URL; + assert.ok(origin, 'set STACK_BENCH_BROWSER_TEST_URL to the isolated test dashboard'); + const browser = await chromium.launch({ headless: true }); + t.after(() => browser.close()); + const page = await browser.newPage(); + let release: (() => void) | undefined; + let delayed = false; + await page.route('**/api/runs/prepare', async route => { + if (delayed) await new Promise<void>(resolve => { release = resolve; }); + await route.fulfill({ status: 400, contentType: 'application/json', + body: JSON.stringify({ error: 'Navigation test: no account' }) }); + }); + await page.goto(`${origin}/new`); + await page.getByRole('button', { name: 'Review run', exact: true }).click(); + await page.getByRole('alert').filter({ hasText: 'Navigation test: no account' }).waitFor(); + await page.getByRole('link', { name: 'Campaigns', exact: true }).first().click(); + await page.getByRole('link', { name: 'New run', exact: true }).click(); + assert.equal(await page.getByText('Navigation test: no account', { exact: true }).count(), 0); + + delayed = true; + const submitted = page.waitForRequest('**/api/runs/prepare'); + await page.getByRole('button', { name: 'Review run', exact: true }).click(); + await submitted; + await page.getByRole('link', { name: 'Campaigns', exact: true }).first().click(); + const received = page.waitForResponse('**/api/runs/prepare'); + assert.ok(release, 'the review request reached the interceptor'); + release(); + await received; + await page.getByRole('link', { name: 'New run', exact: true }).click(); + await page.getByRole('button', { name: 'Review run', exact: true }).waitFor(); + assert.equal(await page.getByText('Navigation test: no account', { exact: true }).count(), 0); +}); diff --git a/tools/stack-bench/tests/dashboard/live-cost.test.ts b/tools/stack-bench/tests/dashboard/live-cost.test.ts new file mode 100644 index 00000000000..4379a8777f0 --- /dev/null +++ b/tools/stack-bench/tests/dashboard/live-cost.test.ts @@ -0,0 +1,93 @@ +import assert from 'node:assert/strict'; +import childProcess from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { syncBuiltinESMExports } from 'node:module'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { responseCosts, codexResponseCosts, cumulativeResponseCosts, liveCostTotal, + liveTranscriptCost } from '../../dashboard/dashboard-live-cost.js'; +import { emptyArtifactIdentities, writeArtifact } from '../../src/evidence/artifacts.js'; + +test('live cost uses pinned response usage once, excludes inherited history, and rejects conflicting or unpriced usage', () => { + const rates = { input: 2, output: 10, cacheRead: 0.2, cacheWrite5m: 2.5, cacheWrite1h: 4 }; + const start = '2026-09-10T10:00:00Z'; + const row = (id: string, timestamp: string, output = 100, model = 'claude-sonnet-5') => JSON.stringify({ + type: 'assistant', requestId: id, timestamp, message: { model, stop_reason: 'tool_use', usage: { + input_tokens: 1000, output_tokens: output, cache_read_input_tokens: 0, cache_creation_input_tokens: 0, + } }, + }); + const response = row('one', '2026-09-10T10:01:00Z'); + const points = responseCosts([row('old', '2026-09-10T09:00:00Z'), response, response, + row('two', '2026-09-10T10:02:00Z')].join('\n'), rates, 'claude-sonnet-5', start); + assert.deepEqual(cumulativeResponseCosts(points).map(point => point.costUsd), [0.003, 0.006]); + assert.throws(() => cumulativeResponseCosts([...points, + ...responseCosts(row('one', '2026-09-10T10:01:00Z', 200), rates, 'claude-sonnet-5', start)]), /Conflicting/); + assert.throws(() => responseCosts(row('other', start, 100, 'other-model'), rates, 'claude-sonnet-5', start), /pinned price/); + assert.throws(() => responseCosts('{broken', rates, 'claude-sonnet-5', start), /Incomplete/); + assert.equal(liveCostTotal('running', 12, 10), 12); + assert.equal(liveCostTotal('running', 8, 10), undefined); + assert.equal(liveCostTotal('completed', 12, 11), undefined, 'final receipt replaces estimate, never adds it'); + assert.equal(liveCostTotal('invalid', 12, 11), undefined); +}); + +test('Codex live usage counts cumulative deltas across sessions without counting cached or reasoning tokens twice', () => { + const rates = { input: 2, output: 10, cacheRead: 0.2, cacheWrite5m: 0, cacheWrite1h: 0 }; + const start = '2026-09-12T01:00:00Z'; + const header = (id: string, model = 'gpt-6-astra') => [ + JSON.stringify({ type: 'session_meta', payload: { id } }), + JSON.stringify({ type: 'turn_context', payload: { model } }), + ].join('\n') + '\n'; + const row = (input: number, cached: number, output: number, timestamp = start) => JSON.stringify({ + type: 'event_msg', timestamp, payload: { type: 'token_count', info: { total_token_usage: { + input_tokens: input, cached_input_tokens: cached, output_tokens: output, reasoning_output_tokens: output, + } } }, + }); + const parse = (text: string) => codexResponseCosts(text, rates, 'gpt-6-astra', start); + const prefix = header('one') + row(1000, 800, 100, '2026-09-12T00:59:00Z') + '\n'; + assert.deepEqual(parse(prefix), []); + const first = parse(prefix + row(2000, 1600, 200)); + const next = parse(prefix + [row(2000, 1600, 200), row(2000, 1600, 200), row(3000, 2400, 300)].join('\n')); + const other = codexResponseCosts(header('two') + row(1000, 800, 100), rates, 'gpt-6-astra', start); + assert.deepEqual(cumulativeResponseCosts([...first, ...first, ...next, ...other]).map(p => p.costUsd), + [0.00156, 0.00312, 0.00468]); + assert.throws(() => parse(prefix + row(999, 800, 100)), /decreased/); + assert.throws(() => codexResponseCosts(header('x', 'other') + row(1000, 0, 1), rates, 'gpt-6-astra', start), /pinned price/); + assert.throws(() => codexResponseCosts(header('x') + row(1, 2, 1), rates, 'gpt-6-astra', start), /Invalid Codex/); +}); + +test('Codex live cost reads only bytes appended since the last refresh', async t => { + const directory = mkdtempSync(join(tmpdir(), 'dashboard-codex-live-')); + t.after(() => rmSync(directory, { recursive: true, force: true })); + writeArtifact(join(directory, 'backend-lease.json'), { kind: 'backend_lease_evidence', id: 'lease', + identities: emptyArtifactIdentities(), payload: { track: 'ecommerce', backend: 'spacetime', + runIndex: 1, runId: 'codex-live-test', state: 'active', + resources: { buildContainer: { owned: true, name: 'owned-build', id: 'exact-id' } } } }); + const rates = { input: 2, output: 10, cacheRead: 0.2, cacheWrite5m: 0, cacheWrite1h: 0 }; + const start = '2026-09-12T01:00:00Z'; + const row = (input: number, cached: number, output: number) => JSON.stringify({ + type: 'event_msg', timestamp: start, payload: { type: 'token_count', info: { total_token_usage: { + input_tokens: input, cached_input_tokens: cached, output_tokens: output } } } }) + '\n'; + let text = JSON.stringify({ type: 'session_meta', payload: { id: 'one' } }) + '\n' + + JSON.stringify({ type: 'turn_context', payload: { model: 'gpt-6-astra' } }) + '\n' + row(1000, 800, 100); + let modified = 1000; + const reads: number[] = []; + t.mock.method(childProcess, 'execFile', (...args: unknown[]) => { + const command = args[1] as string[]; + const done = args[3] as (error: null, stdout: Buffer) => void; + const [name, from, count] = command.slice(-3).map(String) as [string, string, string]; + if (command[0] === 'inspect') return done(null, Buffer.from('exact-id')); + if (name === '') return done(null, Buffer.from(JSON.stringify([['s.jsonl', Buffer.byteLength(text), modified]]))); + reads.push(Number(from)); + done(null, Buffer.from(text).subarray(Number(from), Number(from) + Number(count))); + }); + syncBuiltinESMExports(); + const read = () => liveTranscriptCost(directory, 'codex', rates, 'gpt-6-astra', start); + assert.deepEqual((await read()).costs.map(point => point.costUsd), [0.00156]); + const offset = Buffer.byteLength(text); + text += row(2000, 1600, 200); modified = 2000; + const grown = await read(); + assert.deepEqual(reads, [0, offset], 'the second refresh starts after the first complete line set'); + assert.deepEqual(grown.costs.map(point => point.costUsd), + cumulativeResponseCosts(codexResponseCosts(text, rates, 'gpt-6-astra', start)).map(point => point.costUsd)); +}); diff --git a/tools/stack-bench/tests/dashboard/live-metrics.test.ts b/tools/stack-bench/tests/dashboard/live-metrics.test.ts new file mode 100644 index 00000000000..f24334427cc --- /dev/null +++ b/tools/stack-bench/tests/dashboard/live-metrics.test.ts @@ -0,0 +1,143 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { parseRunProgress } from '../../dashboard/dashboard-model.js'; +import { elapsed, executionClock } from '../../dashboard/public/format.js'; +import { attemptMetrics, compareCampaign, outputSilentMinutes, type MetricAttempt } from '../../dashboard/public/metrics.js'; +import { recordedExecutionSpend } from '../../src/evidence/run-checkpoints.js'; +import { runCostEvidence } from '../../src/evidence/cost-proof.js'; +import { campaignActiveDurationMs, campaignFirstBuildRate, campaignMeasuredRunCost, + campaignMeasuredRunWork, campaignRunMetrics, executionSpend } from '../../src/campaigns/campaign-report.js'; + +test('dashboard and export share retry, prior-repair, wait and pause definitions', () => { + const session = (costUsd: number) => ({ costUsd, costComplete: true, + costReceipts: [{ receipt: { costUsd, exact: true, complete: true, reconciled: true, error: null } }] }); + const failed = { id: 'failed', totals: { costUsd: 2, costComplete: true }, + levels: [{ level: 1, buildSessions: [session(2)] }] }; + const measured = { id: 'measured', totals: { costUsd: 3, costComplete: true, tokens: 2_400_000, + durationSec: 100, pausedDurationSec: 10 }, + levels: [ + { level: 1, firstBuild: { score: 5, max: 10 }, buildSessions: [session(1)], repairSessions: [session(1)], + sessionTotals: { providerThrottle: { waitedMs: 20_000 } } }, + { level: 2, firstBuild: { score: 10, max: 10 }, buildSessions: [session(1)] }, + ] }; + const cost = campaignMeasuredRunCost(measured, [failed, measured]); + const report = campaignRunMetrics(measured as Parameters<typeof campaignRunMetrics>[0], []); + const first = campaignFirstBuildRate(measured, []), active = campaignActiveDurationMs(measured); + assert.equal(first, 0.75); assert.equal(active, 70_000); + const spend = executionSpend([failed, measured].map(run => ({ cost: runCostEvidence(run, 'execution') }))); + const attempt: MetricAttempt = { id: 'measured', stack: 'example', status: 'completed', execution: null, + dependency: null, measuredCost: cost, measuredDurationSec: active! / 1000, spend, result: { + firstBuildRate: first, durationSec: 100, + levels: [{ level: 2, firstScore: { score: 10, max: 10 }, firstAbort: null, + finalScore: { score: 10, max: 10 }, used: 1, repairStatus: null, outcome: null, + durationSec: 100, costUsd: 3, cost, failures: [], repairs: null, continued: false }], + } }; + const row = compareCampaign({ attempts: [attempt] }).rows[0]!; + assert.equal(row.first, report.firstBuildScoreRate); + assert.equal(row.duration, report.totalDurationMs! / 1000); + assert.equal(report.totalTokens, 2_400_000); + assert.equal(row.costPerValidRun, report.totalCostUsd); + assert.equal(row.costPerValidRun, 3); assert.equal(row.spendSoFar, 5); + assert.equal(attemptMetrics({ ...attempt, result: { ...attempt.result!, firstBuildRate: null } })!.raw.first, null); + assert.equal(attemptMetrics({ ...attempt, result: { ...attempt.result!, unreachedPoints: 10 } })!.final, 0.5); + const resumed = { id: 'resumed', progressionResume: { priorRunId: 'measured', inheritedLevels: [1, 2] }, + totals: { currentExecutionCostUsd: 1, currentExecutionCostComplete: true }, levels: [{ level: 3, buildSessions: [session(1)] }] }; + assert.deepEqual(campaignMeasuredRunCost(resumed, [failed, measured, resumed]), { status: 'exact', costUsd: 4 }); + assert.equal(campaignMeasuredRunCost(resumed, [resumed]).status, 'unknown'); + // A continued attempt's dashboard time is the export's chain time, not its latest execution. + const continued = { ...resumed, totals: { ...resumed.totals, durationSec: 50 } }; + const chainMs = campaignMeasuredRunWork(continued, [measured, continued]).durationMs!; + assert.equal(chainMs, 120_000); + const chainRow = compareCampaign({ attempts: [{ ...attempt, id: 'continued', + measuredDurationSec: chainMs / 1000 }] }).rows[0]!; + assert.equal(chainRow.duration, 120); +}); + +test('live spend validates recorded sessions without treating them as final execution cost', () => { + const session = (costUsd: number) => ({ costUsd, costComplete: true, + costReceipts: [{ receipt: { costUsd, exact: true, complete: true, reconciled: true, error: null } }] }); + const run = { progressionResume: { inheritedLevels: [1] }, levels: [ + { level: 1, buildSessions: [session(100)] }, + { level: 2, buildSessions: [session(2)], repairSessions: [session(3)] }, + ] }; + assert.deepEqual(recordedExecutionSpend(run), { status: 'exact', costUsd: 5 }); + assert.equal(runCostEvidence(run, 'execution').status, 'unknown'); + run.levels[1]!.repairSessions!.push(session(4)); + assert.equal(recordedExecutionSpend(run).costUsd, 9); + run.levels[1]!.repairSessions![0]!.costComplete = false; + assert.equal(recordedExecutionSpend(run).status, 'unknown'); + assert.equal(recordedExecutionSpend({}).status, 'unknown'); + assert.deepEqual(recordedExecutionSpend({ ...run, checkpoints: [ + { executionCost: { status: 'exact', costUsd: 12 } }, + ] }), { status: 'exact', costUsd: 12 }, 'active-level checkpoints include repairs missing from completed levels'); + assert.deepEqual(recordedExecutionSpend({ ...run, checkpoints: [ + { executionCost: { status: 'upper-bound', costUsd: 15 } }, + ] }), { status: 'upper-bound', costUsd: 15 }); + assert.equal(recordedExecutionSpend({ ...run, checkpoints: [ + { executionCost: { status: 'unknown', costUsd: null } }, + ] }).status, 'unknown'); +}); + +test('cost per valid run uses only completed comparable runs with complete exact costs', () => { + const run: MetricAttempt = { + id: 'valid', stack: 'example', status: 'completed', execution: null, dependency: null, + spend: { status: 'exact', costUsd: 2 }, measuredCost: { status: 'exact', costUsd: 2 }, result: { levels: [{ + level: 1, firstScore: null, firstAbort: null, finalScore: { score: 5, max: 10 }, + used: 0, repairStatus: null, outcome: null, durationSec: null, costUsd: 2, + cost: { status: 'exact', costUsd: 2 }, failures: [], repairs: null, continued: false, + }] }, + }; + const second = { ...run, id: 'second', spend: { status: 'exact' as const, costUsd: 10 }, measuredCost: { status: 'exact' as const, costUsd: 10 } }; + const attempts = [run, second, + { ...run, id: 'invalid', status: 'invalid', spend: { status: 'exact' as const, costUsd: 100 } }, + { ...run, id: 'running', status: 'running', spend: { status: 'exact' as const, costUsd: 50 } }]; + const row = compareCampaign({ attempts }).rows[0]!; + assert.equal(row.n, 2); + assert.equal(row.costPerValidRun, 6); + assert.equal(row.spendSoFar, 162); + for (const spend of [{ status: 'unknown' as const, costUsd: null }, + { status: 'upper-bound' as const, costUsd: 10 }]) { + assert.equal(compareCampaign({ attempts: [run, { ...second, measuredCost: spend }] }).rows[0]!.costPerValidRun, null); + } + assert.equal(compareCampaign({ attempts: [run, { ...second, comparisonKey: 'different' }] }).rows[0]!.costPerValidRun, null); + assert.equal(compareCampaign({ attempts: [{ ...run, status: 'running' }] }).rows[0]!.costPerValidRun, null); +}); + +test('execution elapsed time uses the execution clock, and stops at completion', () => { + const start = '2026-09-07T12:00:00Z'; + const end = '2026-09-07T12:03:00Z'; + const now = Date.parse('2026-09-07T12:05:00Z'); + assert.equal(elapsed(start, null, now), '5m 0s'); + assert.equal(elapsed(start, null, now + 1000), '5m 1s'); + assert.equal(elapsed(start, end, now), '3m 0s'); + assert.match(executionClock(start, null), /data-started-at=/); + assert.doesNotMatch(executionClock(start, end), /data-started-at=/); + assert.equal(elapsed(null, null, now), '—'); + assert.equal(elapsed('invalid', null, now), '—'); + assert.equal(elapsed(start, null, Date.parse(start) - 1), '0m 0s'); +}); + +test('stopped attempts do not claim a previous grading or repair phase is live', () => { + for (const log of ['=== postgres-l1-first (postgres) ===', + '--- feature repair 2: Catalog ---']) { + assert.equal(parseRunProgress(log, { running: false, status: 'invalid' }).phase, + 'Stopped without a valid result'); + assert.equal(parseRunProgress(log, { running: false, status: 'completed' }).phase, 'Finished'); + assert.equal(parseRunProgress(log, { running: false, status: 'pending' }).phase, 'Waiting to start'); + assert.notEqual(parseRunProgress(log, { running: true, status: 'running' }).phase, 'Finished'); + } + const prepared = parseRunProgress('', { repairs: 10, running: false, status: 'pending' }); + assert.equal(prepared.phase, 'Waiting to start'); + assert.equal(prepared.completedGrades, 0); + assert.equal(prepared.latestScore, null); +}); + +test('activity warnings require agent evidence and exclude planned pauses', () => { + const now = Date.parse('2026-09-10T12:00:00Z'); + const attempt = { status: 'running', activityUpdatedAt: '2026-09-10T11:40:00Z' }; + assert.equal(outputSilentMinutes(attempt, now), 20); + assert.equal(outputSilentMinutes({ ...attempt, paused: true }, now), 0); + assert.equal(outputSilentMinutes({ ...attempt, status: 'completed' }, now), 0); + assert.equal(outputSilentMinutes({ status: 'running' }, now), 0); + assert.equal(outputSilentMinutes({ ...attempt, activityUpdatedAt: 'invalid' }, now), 0); +}); diff --git a/tools/stack-bench/tests/dashboard/paging.test.ts b/tools/stack-bench/tests/dashboard/paging.test.ts new file mode 100644 index 00000000000..6bec533bc7c --- /dev/null +++ b/tools/stack-bench/tests/dashboard/paging.test.ts @@ -0,0 +1,87 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { syncBuiltinESMExports } from 'node:module'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import type { AddressInfo } from 'node:net'; +import test from 'node:test'; +import { overviewPage } from '../../dashboard/dashboard-views.js'; +import { createDashboardServer } from '../../dashboard/dashboard-server.js'; +import { campaignsPage } from '../../dashboard/public/views/campaigns.js'; +import { compileCampaignFile } from '../../src/campaigns/campaign-compiler.js'; +import { createCampaignState, claimNextAttempt, finishCampaignExecution } from '../../src/campaigns/campaign-scheduler.js'; +import { EXAMPLE_CAMPAIGN, writeCampaign } from '../fixtures/dashboard-fixture.js'; + +test('campaign paging filters the full population but inspects only visible evidence; reads do not block HTTP', async t => { + const root = fs.mkdtempSync(join(tmpdir(), 'dashboard-paging-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const plan = compileCampaignFile(EXAMPLE_CAMPAIGN); + const { state } = claimNextAttempt(createCampaignState(plan), { admissionId: 'test' }); + const execution = state.attempts[0]!.executions[0]!; + for (let i = 0; i < 21; i++) { + const directory = join(root, 'campaigns', `sample-${String(i).padStart(2, '0')}`); + writeCampaign(directory, plan, state); + fs.mkdirSync(join(directory, execution.output), { recursive: true }); + fs.writeFileSync(join(directory, execution.output, 'run.json'), '{}'); + } + writeCampaign(join(root, 'campaigns', 'ready'), plan, createCampaignState(plan)); + writeCampaign(join(root, 'campaigns', 'broken'), plan, createCampaignState(plan)); + fs.writeFileSync(join(root, 'campaigns', 'broken', 'plan.json'), '{}'); + const reads: string[] = []; + const originalRead = fs.readFileSync; + t.mock.method(fs, 'readFileSync', (...args: Parameters<typeof fs.readFileSync>) => { + if (String(args[0]).endsWith('run.json')) reads.push(String(args[0])); + return originalRead(...args); + }); + syncBuiltinESMExports(); + t.after(() => { t.mock.restoreAll(); syncBuiltinESMExports(); }); + const options = { controllerActive: () => true }; + const first = overviewPage(join(root, 'campaigns'), 1, 'all', options); + assert.equal(first.campaigns.length, 20); + assert.equal(first.total, 23); + assert.equal(first.pages, 2); + const second = overviewPage(join(root, 'campaigns'), 2, 'all', options); + assert.equal(second.campaigns.length, 3); + assert.equal(new Set([...first.campaigns, ...second.campaigns].map(item => item.key)).size, 23); + assert.equal(second.campaigns.at(-1)!.status, 'unreadable'); + assert.equal(overviewPage(join(root, 'campaigns'), 99, 'all', options).page, 2); + const ready = overviewPage(join(root, 'campaigns'), 2, 'ready', options); + assert.equal(ready.total, 1); + assert.equal(ready.page, 1); + assert.equal(ready.counts.all, 23); + assert.equal(ready.counts.attention, 1); + const html = campaignsPage({ campaigns: first.campaigns, filter: 'all', pagination: first }); + assert.match(html, /Page 1 of 2 · 23 campaigns/); + assert.match(html, /href="\/\?filter=all&page=2">Next/); + assert.equal(reads.length, 0); // Running attempts have no comparable final score. + const completed = finishCampaignExecution(state, execution.id, { exitCode: 0, run: { outcome: { kind: 'passed' } } }); + writeCampaign(join(root, 'campaigns', 'sample-00'), plan, completed); + const updated = overviewPage(join(root, 'campaigns'), 1, 'all', options); + assert.ok(reads.length > 0); // Completed scores still require valid evidence. + const invalid = updated.campaigns.find(item => item.key === 'sample-00'); + assert.ok(invalid && 'scores' in invalid); + assert.ok(Object.values(invalid.scores).every(score => score === null)); + fs.writeFileSync(join(root, 'campaigns', 'sample-00', execution.output, 'run.json'), '{"changed":true}'); + reads.length = 0; + overviewPage(join(root, 'campaigns'), 1, 'attention', options); + assert.equal(reads.length, 0); // Changed evidence outside this page is not inspected. + t.mock.restoreAll(); syncBuiltinESMExports(); + + const { server } = createDashboardServer({ resultsRoot: root, plansRoot: join(root, 'plans') }); + await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(() => new Promise<void>(resolve => server.close(() => resolve()))); + const origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + // The cold worker is still reading when the main HTTP thread answers health. + let finished = false; + const overview = fetch(`${origin}/api/overview`).then(async response => { + const value = await response.json(); finished = true; return value; + }); + assert.equal((await fetch(`${origin}/api/health`)).status, 200); + assert.equal(finished, false); + assert.equal((await overview).campaigns.length, 20); + assert.equal((await fetch(`${origin}/api/overview?page=0`)).status, 400); + assert.equal((await fetch(`${origin}/api/overview?filter=unknown`)).status, 400); + const empty = await (await fetch(`${origin}/api/overview?filter=completed`)).json(); + assert.equal(empty.total, 0); + assert.deepEqual(empty.campaigns, []); +}); diff --git a/tools/stack-bench/tests/dashboard/reference-runs.test.ts b/tools/stack-bench/tests/dashboard/reference-runs.test.ts new file mode 100644 index 00000000000..fbd92b60320 --- /dev/null +++ b/tools/stack-bench/tests/dashboard/reference-runs.test.ts @@ -0,0 +1,52 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { referenceRuns, referenceLogPoints } from '../../dashboard/dashboard-reference-runs.js'; +import { campaignsPage } from '../../dashboard/public/views/campaigns.js'; + +test('standalone reference runs use real controller state, keep the denominator and survive controller removal', async () => { + const root = mkdtempSync(join(tmpdir(), 'reference-dashboard-')); + try { + const directory = join(root, 'reference-live'); mkdirSync(directory); + const id = 'a'.repeat(64), output = join(directory, 'convex.json'); + let running = true, present = true; + const log = 'qualifying convex: clean run 1/1\n scope: 117 check(s), 186 point(s)\n' + + ' selected-source-001 ... 1/1\n selected-source-002 ... 0/2\n selected-source-003 ... not selected\n'; + const docker = async (args: string[]) => { + if (args[0] === 'ps') return present ? JSON.stringify({ ID: id, Command: 'node /opt/stack-bench/dist/src/references/reference-live.js' }) : ''; + if (args[0] === 'inspect') return JSON.stringify([ + { Id: id, Args: ['--out', output, '--backend', 'convex', '--level', '3'], + State: { Running: running, StartedAt: '2026-09-21', FinishedAt: '2026-09-22' } }, + { Id: 'b'.repeat(64), Args: ['--out', join(root, 'outside.json')], State: { Running: true } }, + ]); + assert.equal(args.at(-1), id); return log; + }; + const first = await referenceRuns(root, docker); + assert.equal(first.runs.length, 1); + assert.equal(first.runs[0]!.status, 'running'); + assert.deepEqual(first.runs[0]!.points, { passed: 1, measured: 3, planned: 186 }); + assert.match(campaignsPage({ campaigns: [], filter: 'all', references: first }), /<strong>1\/186<\/strong> points passed<\/span>/); + assert.match(campaignsPage({ campaigns: [], filter: 'all', references: { runs: [ + { ...first.runs[0]!, points: { passed: 1, measured: 3, planned: null } }], error: null } }), /<strong>1<\/strong> points passed/); + running = false; + assert.equal((await referenceRuns(root, docker)).runs[0]!.status, 'incomplete'); + writeFileSync(output, JSON.stringify({ kind: 'reference_qualification', timestamps: { completedAt: '2026-09-22' }, + payload: { fixture: 'convex', ok: false, runs: [{ score: '184/186', failures: ['<failure>'] }] } })); + assert.equal((await referenceRuns(root, docker)).runs[0]!.status, 'failed'); + present = false; + const final = await referenceRuns(root, docker); + assert.equal(final.runs[0]!.status, 'failed'); + assert.equal(final.runs[0]!.updatedAt, '2026-09-22'); + assert.deepEqual(final.runs[0]!.points, { passed: 184, measured: 186, planned: 186 }); + // Only a run in progress is shown above the campaign history. + assert.doesNotMatch(campaignsPage({ campaigns: [], filter: 'all', references: final }), /reference-run/); + assert((await referenceRuns(root, async () => { throw new Error('offline'); })).error); + writeFileSync(output, JSON.stringify({ kind: 'reference_qualification', timestamps: { completedAt: '2026-09-22' }, + payload: { fixture: 'convex', ok: true, runs: [{ score: '186/186', failures: [] }] } })); + assert.equal((await referenceRuns(root, docker)).runs[0]!.status, 'passed'); + assert.deepEqual(referenceLogPoints(log + 'qualifying convex: clean run 2/2\n selected-source-001 ... 1/1\n'), + { passed: 1, measured: 1, planned: null }); + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/tools/stack-bench/tests/dashboard/run-setup.test.ts b/tools/stack-bench/tests/dashboard/run-setup.test.ts new file mode 100644 index 00000000000..004ef7858f4 --- /dev/null +++ b/tools/stack-bench/tests/dashboard/run-setup.test.ts @@ -0,0 +1,217 @@ +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import { mkdtempSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync, existsSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import type { AddressInfo } from 'node:net'; +import test from 'node:test'; +import { prepareRun, runSetupCatalog, submitPreparedRun } from '../../src/campaigns/run-setup.js'; +import { createDashboardServer, type LaunchInput } from '../../dashboard/dashboard-server.js'; +import { initialRun, runSetupPage, selectGuidance } from '../../dashboard/public/views/run-setup.js'; +import { writePlanFixtures } from '../fixtures/dashboard-fixture.js'; +import { STACK_BENCH_ROOT } from '../../src/package-root.js'; + +// One invariant: only the exact reviewed configuration can dispatch, and retries +// dispatch it once. No fixture invokes a controller, database, or model. +test('setup reviews exact dimensions, rejects changes, and dispatches one durable job', async t => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-setup-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + writePlanFixtures(join(root, 'plans')); + mkdirSync(join(root, 'run-presets')); + const presetPath = join(root, 'run-presets', 'ecommerce.json'); + const source = readFileSync(join(root, 'plans', 'ecommerce-progression-reference.json'), 'utf8'); + writeFileSync(presetPath, source); + const catalog = runSetupCatalog(root, {}); + const choices = structuredClone(catalog); + choices.workloads[0]!.conditions = [ + { id: 'neutral-dev-no-sdk', guidance: 'neutral-dev-no-sdk', sdkSkills: false, devWorkflow: true }, + { id: 'neutral', guidance: 'neutral', sdkSkills: true, devWorkflow: false }, + { id: 'neutral-no-sdk', guidance: 'neutral-no-sdk', sdkSkills: false, devWorkflow: false }, + { id: 'neutral-dev', guidance: 'neutral-dev', sdkSkills: true, devWorkflow: true }, + ]; + const initial = initialRun(choices)!; + assert.equal(initial.productionQuality, true); + assert.deepEqual(initial.conditions, ['neutral-dev']); + const single = { ...choices.workloads[0]!, id: 'single-build', workSelection: 'all-at-once' as const }; + const reordered = { ...choices, workloads: [single, ...choices.workloads] }; + assert.equal(initialRun(reordered)!.workload, choices.workloads[0]!.id); + assert.equal(initialRun(reordered, single.id)!.workload, single.id); + const conditions = choices.workloads[0]!.conditions; + for (const c of conditions) assert.deepEqual(selectGuidance(conditions, c.sdkSkills ? 'on' : 'off', + c.devWorkflow ? 'on' : 'off'), [c.id]); + const request = { ...initialRun(catalog)!, key: 'setup-smoke', level: 1, + repetitions: 2, parallelism: 6, maxCostUsd: 12, repairs: 0, pauseAfterDepth: null }; + const review = prepareRun(root, request, {}); + assert.equal(review.attempts, 6); + assert.equal(review.parallelism, 6); + assert.equal(review.maxCostUsd, 72); + assert.equal(review.mode.workSelection, 'progressive'); + assert.equal(review.request.productionQuality, true); + const prototypeReview = prepareRun(root, { ...request, productionQuality: false }, {}); + assert.notEqual(prototypeReview.planSha256, review.planSha256); + assert.match(runSetupPage(catalog, request, review, '', true), /Production-quality app<\/dt><dd>Requested/); + assert.match(runSetupPage(catalog, prototypeReview.request, prototypeReview, '', true), + /Production-quality app<\/dt><dd>Not requested/); + assert.equal(prepareRun(root, request, {}).reviewId, review.reviewId); + assert.equal(existsSync(join(root, 'jobs')), false); + assert.throws(() => prepareRun(root, { ...request, stacks: [...request.stacks, request.stacks[0]] }, {}), /duplicates/); + assert.throws(() => prepareRun(root, { ...request, level: 99 }, {}), /Target level/); + assert.throws(() => prepareRun(root, { ...request, parallelism: 0 }, {})); + assert.throws(() => prepareRun(root, { ...request, workload: '../escape' }, {})); + assert.throws(() => submitPreparedRun(root, { request: { ...request, maxCostUsd: 24 }, reviewId: review.reviewId }, {}), /Setup changed/); + assert.throws(() => submitPreparedRun(root, + { request: { ...request, productionQuality: false }, reviewId: review.reviewId }, {}), /Setup changed/); + const changed = JSON.parse(source); changed.pricing.models['reference-fixture'].input = 1; + writeFileSync(presetPath, JSON.stringify(changed)); + assert.throws(() => submitPreparedRun(root, review, {}), /Workload changed/); + writeFileSync(presetPath, source); + assert.throws(() => prepareRun(root, request, { STACK_BENCH_CONTROLLER_IMAGE: 'different' }), /runtime/); + + assert.doesNotMatch(runSetupPage(catalog, request, review, '', true), /Operator secret|name="secret"/); + assert.doesNotMatch(runSetupPage(catalog, request, null, '', true), /name="secret"/); + const launches: LaunchInput[] = []; + const { server } = createDashboardServer({ resultsRoot: root, plansRoot: join(root, 'plans'), + allowLaunch: true, token: 'test-token', + launch(input) { launches.push(input); return Object.assign(new EventEmitter(), { pid: 1 }); } }); + await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(() => server.close()); + const origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + const send = (path: string, value: unknown, token = 'test-token') => + fetch(origin + path, { method: 'POST', headers: { origin, 'content-type': 'application/json', + 'x-stack-bench-token': token }, body: JSON.stringify(value) }); + assert.equal((await send('/api/runs', review, 'wrong')).status, 403); + const preparation = await send('/api/runs/prepare', request); + assert.equal(preparation.status, 200, await preparation.clone().text()); + const acceptedReview = await preparation.json(); + assert.equal(launches.length, 0); + const first = await send('/api/runs', acceptedReview); + assert.equal(first.status, 202, await first.clone().text()); + const result = await first.json() as { campaignKey: string; job: { id: string } }; + assert.equal((await send('/api/runs', acceptedReview)).status, 202); + assert.equal((await send(`/api/jobs/${result.job.id}/start`, {})).status, 202); + assert.equal(launches.length, 1); + assert.equal(launches[0]!.command, 'work'); + assert.equal(launches[0]!.jobId, result.job.id); + const pending = await (await fetch(`${origin}/api/campaigns/${result.campaignKey}`)).json() as { pendingJob: { status: string } }; + assert.equal(pending.pendingJob.status, 'queued'); + assert.equal((await send(`/api/jobs/${result.job.id}/cancel`, {})).status, 202); + assert.equal((await send('/api/runs', acceptedReview)).status, 202); + assert.equal(launches.length, 1, 'cancelled work must not restart'); +}); + +test('the four-stack L1–L3 preset exposes Convex and prepares only supported levels', t => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-convex-setup-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + writePlanFixtures(join(root, 'plans')); + mkdirSync(join(root, 'run-presets')); + const preset = JSON.parse(readFileSync(join(root, 'plans', 'ecommerce-progression-reference.json'), 'utf8')); + preset.id = 'ecommerce-progressive-l3'; + preset.levels = [1, 2, 3]; + preset.selection.levels = preset.selection.levels.filter((selection: { level: number }) => selection.level <= 3); + preset.stacks.push({ id: 'convex', adapterVersion: '1.0.0' }); + writeFileSync(join(root, 'run-presets', `${preset.id}.json`), JSON.stringify(preset)); + const catalog = runSetupCatalog(root, {}); + const request = { ...initialRun(catalog)!, key: 'convex-setup', stacks: ['convex'], maxCostUsd: 12 }; + assert.equal(request.level, 3); + assert.match(runSetupPage(catalog, request, null, '', true), /value="convex" checked>Convex/); + const review = prepareRun(root, request, {}); + assert.deepEqual(review.request.stacks, ['convex']); + assert.equal(review.attempts, preset.repetitions); + const saved = JSON.parse(readFileSync(review.planFile, 'utf8')); + assert.deepEqual(saved.stacks.map((stack: { id: string }) => stack.id), ['convex']); + assert.deepEqual(saved.levels, [1, 2, 3]); + assert.throws(() => prepareRun(root, { ...request, level: 4 }, {}), /Target level is not in this workload/); + assert.equal(existsSync(join(root, 'jobs')), false, 'preparation must not launch work'); +}); + +test('automatic accounts are explicit in the review and ambiguous accounts require selection', t => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-accounts-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + writePlanFixtures(join(root, 'plans')); + mkdirSync(join(root, 'run-presets')); + const preset = JSON.parse(readFileSync(join(root, 'plans', 'ecommerce-progression-reference.json'), 'utf8')); + preset.agents = [{ adapter: 'codex', adapterVersion: '1.0.0', model: 'gpt-6-astra', effort: 'medium' }, + { adapter: 'claude-code', adapterVersion: '1.17.2', model: 'claude-sonnet-5', effort: 'medium' }]; + for (const a of preset.agents) preset.pricing.models[a.model] = preset.pricing.models['reference-fixture']; + writeFileSync(join(root, 'run-presets', 'accounts.json'), JSON.stringify(preset)); + const secretFile = join(root, 'synthetic-secret'); + writeFileSync(secretFile, 'SYNTHETIC_ONLY'); + const profiles = { openai: { provider: 'openai', mode: 'api-key', version: '1', secretFile }, + claude: { provider: 'anthropic', mode: 'api-key', version: '1', secretFile } }; + const registry = join(root, 'profiles.json'); + writeFileSync(registry, JSON.stringify(profiles)); + const env = { STACK_BENCH_CREDENTIAL_PROFILES_FILE: registry }; + const catalog = runSetupCatalog(root, env); + const request = { ...initialRun(catalog)!, level: 1, maxCostUsd: 12, + agents: catalog.workloads[0]!.agents.map((_, index) => ({ index, effort: 'medium' as const })) }; + const review = prepareRun(root, request, env); + assert.deepEqual(review.request.credentials.adapters, { 'claude-code': 'claude', codex: 'openai' }); + assert.equal(prepareRun(root, review.request, env).reviewId, review.reviewId); + assert.deepEqual(request.credentials, {}); + assert.doesNotMatch(JSON.stringify(review), /SYNTHETIC_ONLY/); + writeFileSync(registry, JSON.stringify({ ...profiles, other: profiles.openai })); + const saved = readdirSync(join(root, 'plans')); + assert.throws(() => prepareRun(root, request, env), /Choose an account for codex/); + assert.throws(() => prepareRun(root, { ...request, key: 'rejected-review' }, env), /Choose an account for codex/); + assert.deepEqual(readdirSync(join(root, 'plans')), saved, 'a rejected review must not save a plan'); + assert.equal(prepareRun(root, review.request, env).reviewId, review.reviewId); + assert.equal(existsSync(join(root, 'jobs')), false); +}); + +test('a new provider model freezes its declared rates and output bound before dispatch', t => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-new-model-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + mkdirSync(join(root, 'run-presets')); + const preset = JSON.parse(readFileSync(join(STACK_BENCH_ROOT, 'tests', 'fixtures', 'campaign.deterministic.json'), 'utf8')); + preset.state = 'frozen'; + preset.budgets.maxCostUsdPerAttempt = 12; + preset.runtime.controllerImage = `sha256:${'a'.repeat(64)}`; + preset.runtime.buildImage = `sha256:${'b'.repeat(64)}`; + preset.agents = [{ adapter: 'codex', adapterVersion: '1.0.0', model: 'gpt-6-astra', effort: 'medium' }, + { adapter: 'claude-code', adapterVersion: '1.17.2', model: 'claude-sonnet-5', effort: 'medium' }, + { adapter: 'openrouter', adapterVersion: '1.0.0', model: 'openai/gpt-5.3-codex', + effort: 'medium', providerRoute: 'openai', maxOutputTokens: 8192 }]; + preset.pricing.models['gpt-6-astra'] = preset.pricing.models.deterministic; + preset.pricing.models['claude-sonnet-5'] = preset.pricing.models.deterministic; + preset.pricing.models['openai/gpt-5.3-codex'] = preset.pricing.models.deterministic; + writeFileSync(join(root, 'run-presets', 'models.json'), JSON.stringify(preset)); + const secretFile = join(root, 'synthetic-secret'); + writeFileSync(secretFile, 'SYNTHETIC_ONLY'); + const registry = join(root, 'profiles.json'); + writeFileSync(registry, JSON.stringify({ openai: { provider: 'openai', mode: 'api-key', version: '1', secretFile }, + anthropic: { provider: 'anthropic', mode: 'api-key', version: '1', secretFile }, + openrouter: { provider: 'openrouter', mode: 'api-key', version: '1', secretFile } })); + const env = { STACK_BENCH_CREDENTIAL_PROFILES_FILE: registry }; + const catalog = runSetupCatalog(root, env); + const modelIndex = (adapter: string) => catalog.workloads[0]!.agents.findIndex(a => a.adapter === adapter); + const request = { ...initialRun(catalog)!, key: 'new-model', level: 1, + stacks: ['postgres'], maxCostUsd: 12, agents: [{ index: modelIndex('codex'), effort: 'medium' as const, + model: 'gpt-6-sol', maxOutputTokens: 128_000, + outputLimitSource: 'https://example.test/model-limit', + pricing: { input: 1, output: 5, cacheWrite5m: 0, cacheWrite1h: 0, cacheRead: 0 }, + pricingSource: 'https://example.test/pricing', pricingCapturedAt: '2026-09-25T00:00:00.000Z' }] }; + const review = prepareRun(root, request, env); + const saved = JSON.parse(readFileSync(review.planFile, 'utf8')); + assert.equal(saved.agents[0].model, 'gpt-6-sol'); + assert.equal(saved.agents[0].maxOutputTokens, 128_000); + assert.equal(saved.pricing.models['gpt-6-sol'].output, 5); + assert.match(saved.pricing.source, /https:\/\/example.test\/pricing/); + for (const [adapter, model, extra] of [ + ['claude-code', 'claude-opus-5-5', {}], + ['openrouter', 'openai/gpt-5.4', { providerRoute: 'openai', maxOutputTokens: 8192, + outputLimitSource: 'https://example.test/router-limit' }], + ] as const) { + const other = prepareRun(root, { ...request, key: `new-model-${adapter}`, agents: [{ index: modelIndex(adapter), effort: 'medium', + model, pricing: { input: 1, output: 5, cacheWrite5m: 0, cacheWrite1h: 0, cacheRead: 0 }, + pricingSource: 'https://example.test/pricing', pricingCapturedAt: '2026-09-25T00:00:00.000Z', + ...extra }] }, env); + const definition = JSON.parse(readFileSync(other.planFile, 'utf8')); + assert.equal(definition.agents[0].model, model); + } + assert.throws(() => prepareRun(root, { ...request, key: 'missing-price', + agents: [{ index: 0, effort: 'medium', model: 'gpt-6-sol' }] }, env), /pricing/i); + assert.throws(() => submitPreparedRun(root, { request: { ...request, + agents: [{ ...request.agents[0]!, pricing: { ...request.agents[0]!.pricing, output: 6 } }] }, + reviewId: review.reviewId }, env), /Setup changed/); + assert.equal(existsSync(join(root, 'jobs')), false); +}); diff --git a/tools/stack-bench/tests/dashboard/transcript.test.ts b/tools/stack-bench/tests/dashboard/transcript.test.ts new file mode 100644 index 00000000000..9f04f6431f8 --- /dev/null +++ b/tools/stack-bench/tests/dashboard/transcript.test.ts @@ -0,0 +1,103 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import childProcess from 'node:child_process'; +import { syncBuiltinESMExports } from 'node:module'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { setImmediate as nextTurn } from 'node:timers/promises'; +import { attemptTranscriptFiles, readAttemptTranscript, transcriptMessages, transcriptLease } from '../../dashboard/dashboard-transcript.js'; +import { emptyArtifactIdentities, writeArtifact } from '../../src/evidence/artifacts.js'; +import { createBackendLease, publicBackendLease, writeBackendLease } from '../../src/runtime/backend-lease.js'; + +test('transcript normalizes Claude and Codex text/tools, redacts credentials, and ignores metadata', () => { + const rows = [ + { type: 'assistant', message: { role: 'assistant', content: [{ type: 'text', text: '<script>hello</script>' }, + { type: 'tool_use', name: 'Bash', input: { command: 'echo hello' } }] } }, + { type: 'user', message: { role: 'user', content: [{ type: 'tool_result', content: 'Authorization: Bearer secret-example-token' }] } }, + { type: 'response_item', payload: { role: 'assistant', content: [{ type: 'output_text', text: 'Codex reply' }] } }, + { type: 'response_item', payload: { type: 'function_call', name: 'exec_command', arguments: '{"cmd":"ls"}' } }, + { type: 'item.completed', item: { type: 'command_execution', command: 'ls', aggregated_output: 'file.ts' } }, + { type: 'system', accessToken: 'never-show-metadata' }, + ]; + const result = transcriptMessages(rows.map(row => JSON.stringify(row)).join('\n') + '\n{broken'); + assert.equal(result.messages.length, 6); + assert.equal(result.messages.filter(message => message.tool).length, 4); + assert.equal(result.skipped, 1); + assert.doesNotMatch(JSON.stringify(result), /secret-example-token|never-show-metadata/); + assert.equal(result.messages[0]?.text, '<script>hello</script>'); // Escaped by the view, never interpreted as markup. +}); + +test('live transcript reads yield, share concurrent reads, and retain lease and range guards', async t => { + const directory = mkdtempSync(join(tmpdir(), 'dashboard-transcript-')); + const executions = [{ directory, label: 'Execution 1' }]; + writeArtifact(join(directory, 'backend-lease.json'), { kind: 'backend_lease_evidence', id: 'lease', + identities: emptyArtifactIdentities(), payload: { track: 'ecommerce', backend: 'spacetime', + runIndex: 1, runId: 'transcript-test', state: 'active', + resources: { buildContainer: { owned: true, name: 'owned-build', id: 'exact-id' } } } }); + const text = JSON.stringify({ type: 'assistant', message: { role: 'assistant', content: 'hello' } }) + '\n'; + let actualId = 'exact-id', calls = 0; + const pending: Array<() => void> = []; + t.mock.method(childProcess, 'execFile', (...args: unknown[]) => { + calls++; + assert.equal(args[0], 'docker'); + assert.deepEqual(args[2], { timeout: 5000, maxBuffer: 2 * 1024 * 1024, encoding: 'buffer' }); + const command = args[1] as string[]; + const reply = command[0] === 'inspect' ? actualId + : command.at(-3) === '' ? JSON.stringify([['session.jsonl', Buffer.byteLength(text), 1000]]) : text; + const done = args[3] as (error: null, stdout: Buffer) => void; + pending.push(() => done(null, Buffer.from(reply))); + }); + syncBuiltinESMExports(); + try { + const first = readAttemptTranscript(executions, 'claude-code'); + const second = readAttemptTranscript(executions, 'claude-code'); + await nextTurn(); // Other HTTP work can proceed while Docker has not answered. + assert.equal(calls, 1); + for (let phase = 0; phase < 3; phase++) { + assert.equal(pending.length, 1); + pending.shift()!(); + await nextTurn(); + } + assert.deepEqual(await first, await second); + assert.equal((await first).messages[0]?.text, 'hello'); + assert.equal(calls, 3); // One inspect, listing, and page read for both viewers. + const files = attemptTranscriptFiles(executions, 'claude-code'); + for (let phase = 0; phase < 2; phase++) { + pending.shift()!(); + await nextTurn(); + } + await assert.rejects((await files)[0]!.read(0, 256 * 1024 + 1), /Invalid transcript range/); + actualId = 'replacement-id'; + const rejected = assert.rejects(readAttemptTranscript(executions, 'claude-code'), /changed after lease creation/); + pending.shift()!(); + await rejected; + assert.equal(pending.length, 0); // A changed container never gets a transcript command. + } finally { + t.mock.restoreAll(); + syncBuiltinESMExports(); + rmSync(directory, { recursive: true, force: true }); + } +}); + +test('active transcript resolves the current runtime lease only for the saved owner', t => { + const directory = mkdtempSync(join(tmpdir(), 'dashboard-active-transcript-')); + t.after(() => rmSync(directory, { recursive: true, force: true })); + const runtimeRoot = join(directory, 'runtime'); + const lease = createBackendLease({ runId: 'live-run', backend: 'postgres', track: 'ecommerce', + runIndex: 0, database: 'app_ecom_run0', container: { name: 'database', id: 'a'.repeat(64) } }); + lease.state = 'active'; + writeArtifact(join(directory, 'run.json'), { kind: 'benchmark_run', id: lease.runId, + identities: emptyArtifactIdentities(), payload: { backendLease: publicBackendLease(lease) } }); + assert.equal(transcriptLease(directory, runtimeRoot), null); + lease.resources.buildContainer = { name: 'current-build', id: 'b'.repeat(64), image: 'build-image', owned: true, + resourceLimits: { cpuCount: 1, memoryBytes: 1024, memorySwapBytes: 1024, pids: 100 } }; + const runtimePath = join(runtimeRoot, lease.runId, 'backend-lease.json'); + writeBackendLease(runtimePath, lease); + const observed = transcriptLease(directory, runtimeRoot); + assert.equal(observed?.resources.buildContainer?.id, lease.resources.buildContainer.id); + assert.doesNotMatch(JSON.stringify(observed), /ownershipToken/); + lease.ownershipToken = 'different-owner'; + writeBackendLease(runtimePath, lease); + assert.throws(() => transcriptLease(directory, runtimeRoot), /ownership changed/); +}); diff --git a/tools/stack-bench/tests/dashboard/view-clarity.test.ts b/tools/stack-bench/tests/dashboard/view-clarity.test.ts new file mode 100644 index 00000000000..63349218be3 --- /dev/null +++ b/tools/stack-bench/tests/dashboard/view-clarity.test.ts @@ -0,0 +1,184 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { completionLabel, spend } from '../../dashboard/public/format.js'; +import type { CampaignProgression, CampaignSheet, SheetAttempt } from '../../dashboard/dashboard-views.js'; +import { campaignPage, replayTimeline, selectedProgression } from '../../dashboard/public/views/campaign.js'; +import { attemptPage } from '../../dashboard/public/views/attempt.js'; +import { topbar } from '../../dashboard/public/views/plans.js'; + +test('report links appear only for files present in the campaign', () => { + const input = { page: 'campaign' as const, key: 'example', canStart: false, resumable: false, error: '' }; + assert.doesNotMatch(topbar(input), />report<|>export manifest</); + const reportFiles = ['report/report.html']; + assert.match(topbar({ ...input, reportFiles }), />report</); + assert.doesNotMatch(topbar({ ...input, reportFiles }), />export manifest</); + assert.match(topbar({ ...input, reportFiles: [...reportFiles, 'report/export-manifest.json'] }), />export manifest</); +}); + +test('invalid completion is not displayed as a low application score', () => { + const attempt = { status: 'invalid', excluded: 'unmeasured checks', + completion: { passed: 9, selected: 109, failed: 0, blocked: 0, unmeasured: 100, rate: 9 / 109 } }; + assert.equal(completionLabel(attempt), 'Incomplete'); + assert.equal(completionLabel({ ...attempt, status: 'running' }), '9 / 109'); + assert.equal(completionLabel({ ...attempt, status: 'completed' }), 'Excluded'); + assert.equal(completionLabel({ ...attempt, status: 'completed', excluded: null }), '9 / 109'); + const features = { passed: 2, selected: 27 }; + assert.equal(completionLabel(attempt, features), 'Incomplete'); + assert.equal(completionLabel({ ...attempt, status: 'running' }, features), '2 / 27'); + assert.equal(completionLabel({ ...attempt, status: 'completed' }, features), 'Excluded'); +}); + +// Distinct aggregate and selected values catch accidental cross-repetition labels. +test('campaign separates aggregate scores from selected evidence and explains pending tabs', () => { + const attempt: SheetAttempt = { + id: 'selected', repetition: 2, status: 'running', phase: 'building', stalling: false, + excluded: null, continued: false, logUpdatedAt: null, score: null, unaided: null, + repairs: { used: 2, budget: 10 }, timeSec: null, executionStartedAt: null, + executionCompletedAt: null, spendPending: true, spend: { status: 'unknown', costUsd: null }, + completion: null, variant: 'neutral', climb: [], + }; + const sheet: CampaignSheet = { + key: 'example', id: 'example', title: 'Example', status: 'running', mode: 'dependency', + levels: [1], repetitions: 2, provisional: true, mixedScope: false, executions: 1, + resumable: false, createdAt: '', updatedAt: '', + facts: { mode: 'dependency', workSelection: 'progressive', repairSelection: 'feature', + repairLimits: { perFeature: 5 }, agent: null, model: null, + guidance: 'neutral', productionQuality: false, recipes: [], timeLimitMinutes: 240, spendLimitUsd: 50, + controllerImage: null, buildImage: null, planSha256: 'example', grading: 'pending', gradingReasons: [] }, + stacks: [{ stack: 'spacetime', costPerValidRun: 6, selectedAttemptId: attempt.id, score: 82, points: null, + unaided: null, continued: false, regressions: 0, + timeSec: null, spend: attempt.spend, spendPending: true, completionRate: null, n: 1, + attempts: [attempt], levels: null, + questlines: [{ id: 'catalog', title: 'Catalog', score: 20, nodes: [] }] }], + }; + const page = campaignPage({ sheet, progression: null, view: 'grid', step: 0 }); + const selected = page.slice(page.indexOf('feature-progress'), page.indexOf('<h3>Results</h3>')); + assert.match(page, /82%/); + assert.match(selected, /20%/); + assert.match(selected, /2 \/ 10/); + assert.doesNotMatch(selected, /82%|9 \/ 10|Questline average/); + assert.match(page, /Valid runs/); + const metricsTable = page.split('<h3>Results</h3>')[1]!.split('</table>')[0]!; + assert.match(metricsTable, /\$6\.00/); + const noRepairSheet = { ...sheet, repetitions: 1, stacks: sheet.stacks.map(stack => ({ + ...stack, attempts: [{ ...attempt, model: 'gpt-6-astra', effort: 'medium', repairs: { used: 0, budget: 0 }, + featureCompletion: { passed: 20, selected: 27, rate: 20 / 27 }, + completion: { passed: 98, selected: 111, failed: 13, blocked: 0, unmeasured: 0, rate: 98 / 111 } }], + })) }; + const noRepairPage = campaignPage({ sheet: noRepairSheet, progression: null, view: 'grid', step: 0 }); + assert.doesNotMatch(noRepairPage, /<th>Repairs<|Before repairs|<small>Rep 1/); + assert.match(noRepairPage, /Astra[^(]*\(medium\)/); + assert.match(noRepairPage, /Features passed/); + assert.match(noRepairPage, /Checks passed/); + assert.match(noRepairPage, /<td>20 \/ 27<\/td><td>98 \/ 111<\/td>/); + for (const tab of ['checks', 'screenshots', 'files', 'log'] as const) { + const detail = attemptPage({ sheet, attemptId: attempt.id, tab, checks: null, evidence: null, log: '' }); + assert.match(detail, /Earlier fixes and feedback are retained/); + assert.match(detail, /No (check results|screenshots|files|log output)/); + assert.match(detail, /aria-current="page"/); + assert.match(detail, /popovertarget="help-checks-passed"/); + } + attempt.checkCategories = { + production: { selected: 2, passed: 1, failed: 1, blocked: 0, unmeasured: 0, rate: 0.5 }, + feature: { selected: 0, passed: 0, failed: 0, blocked: 0, unmeasured: 0, rate: null }, + interface: { selected: 0, passed: 0, failed: 0, blocked: 0, unmeasured: 0, rate: null }, + unknown: { selected: 0, passed: 0, failed: 0, blocked: 0, unmeasured: 0, rate: null }, + }; + const categorized = attemptPage({ sheet, attemptId: attempt.id, tab: 'checks', evidence: null, log: '', + checks: { attemptId: attempt.id, stack: 'spacetime', grades: [], checks: [{ id: 'p', key: 'p', description: 'Durable', feature: 'Orders', points: 1, + category: 'production', outcome: 'pass', regressed: false, history: ['fail', 'not-run', 'pass'], + observations: [ + { status: 'FAIL', summary: '<script>Two orders for one cart</script>', expected: '1 order', actual: '2 orders', context: 'Stopped at <callAction>.' }, + { status: 'INCONCLUSIVE', summary: 'Reader timed out', expected: null, actual: null }, + { status: 'PASS', summary: null, expected: null, actual: null }, + ] }] } }); + assert.match(categorized, /<td>Production<\/td>/); + assert.match(categorized, /1 \/ 2/); + assert.match(categorized, /Expected<\/div><pre>1 order/); + assert.match(categorized, /Observed<\/div><pre>2 orders/); + assert.match(categorized, /Grade 2 · INCONCLUSIVE/); + assert.match(categorized, /Grade 3 · PASS/); + assert.match(categorized, /No observation details were recorded/); + assert.match(categorized, /<script>Two orders/); + assert.match(categorized, /Stopped at <callAction>/); + assert.doesNotMatch(categorized, /<script>|check-evidence[^>]* open/); + const missing = attemptPage({ sheet, attemptId: attempt.id, tab: 'checks', evidence: null, log: '', + checks: { attemptId: attempt.id, stack: 'spacetime', checks: [], + grades: [{ id: 'grading', level: null, round: 0, score: null, error: 'grade bundle is missing' }] } }); + assert.match(missing, /grade bundle is missing/); + delete attempt.checkCategories; + const grantInput = { sheet, attemptId: attempt.id, tab: 'checks' as const, + checks: null, evidence: null, log: '', canControl: true }; + const request = { campaignSha256: 'a'.repeat(64), attemptId: attempt.id, + executionId: 'execution-1', grantId: 'extension-1', minutes: 120, requestedAt: '2026-09-07T00:00:00.000Z' }; + const budget = { originalMinutes: 240, effectiveMinutes: 240, consumedMs: 0, liveGrantSupported: true, + extensionCount: 0, grants: [{ request, disposition: 'pending' as const }] }; + const pending = attemptPage({ ...grantInput, timeBudget: budget }); + assert.match(pending, /disabled>Awaiting controller/); + assert.match(pending, /limit has not changed yet/); + const accepted = attemptPage({ ...grantInput, timeBudget: { ...budget, + effectiveMinutes: 360, extensionCount: 1, + grants: [{ request, disposition: 'accepted', effectiveMinutes: 360 }] } }); + assert.match(accepted, /Time added. Limit: 6h 0m/); + assert.doesNotMatch(attemptPage({ ...grantInput, canControl: false }), /data-run="grant-time"/); + attempt.status = 'invalid'; + assert.doesNotMatch(attemptPage(grantInput), /data-run="grant-time"/); + assert.match(attemptPage({ ...grantInput, timeBudget: { ...budget, grants: [], + continuation: { eligible: true } } }), /Add time and resume/); + assert.match(attemptPage({ ...grantInput, timeBudget: { ...budget, grants: [], + continuation: { eligible: false, reason: 'Coding session was interrupted.' } } }), + /Cannot resume: Coding session was interrupted/); + const transcriptHtml = attemptPage({ ...grantInput, tab: 'transcript', transcript: { + sessions: [{ id: 'one', label: 'Session 1' }], session: 'one', before: 100, skipped: 0, + messages: [{ id: 'm1', role: 'assistant', text: '<script>alert(1)</script>', tool: false }, + { id: 'm2', role: 'Bash', text: 'echo hello', tool: true }], + } }); + assert.match(transcriptHtml, /<script>/); + assert.doesNotMatch(transcriptHtml, /<script>|<details[^>]*open/); + assert.match(transcriptHtml, /data-transcript-before="100"/); + attempt.status = 'running'; + const progression: CampaignProgression = { + key: sheet.key, depths: [1], questlines: [{ id: 'catalog', title: 'Catalog', nodes: ['item'] }], + nodes: [{ id: 'item', title: 'Item', questline: 'catalog', depth: 1, dependencies: [] }], + stacks: [], + }; + sheet.stacks = ['spacetime', 'postgres', 'mongodb', 'custom-sql', 'custom-kv'].map(stack => ({ + ...sheet.stacks[0]!, stack, selectedAttemptId: `${stack}-2`, + })); + progression.stacks = sheet.stacks.flatMap(stack => [1, 2, 3].map(repetition => ({ + stack: stack.stack, attemptId: `${stack.stack}-${repetition}`, updatedAt: '', + steps: [{ sequence: 1, action: repetition === 2 ? 'build' as const : 'repair' as const, + targets: ['item'], statuses: [repetition === 2 ? 'passed' : 'failed'], + score: repetition === 2 ? 100 : 99, repairs: repetition === 2 ? 0 : 77 }], + }))); + const detailSheet = { ...sheet, stacks: [{ ...sheet.stacks[0]!, + attempts: [{ ...attempt, id: 'spacetime-1' }] }] }; + const detail = attemptPage({ sheet: detailSheet, progression, attemptId: 'spacetime-1', + tab: 'checks', checks: null, evidence: null, log: '' }); + assert.equal((detail.match(/class="d f"/g) ?? []).length, 1); + assert.doesNotMatch(detail, /class="d p"/); + assert.match(detail, /Feature dependency graph/); + const selectedTracks = selectedProgression(progression, sheet); + const gridPage = campaignPage({ sheet, progression, view: 'grid', step: 0 }); + const comparison = gridPage.split('<table class="sheet">')[1]!.split('</table>')[0]!; + assert.equal((comparison.split('</thead>')[0]!.match(/scope="col"/g) ?? []).length, 6); + assert.match(comparison, /custom-sql/); + assert.match(comparison, /custom-kv/); + assert.equal(selectedTracks.stacks.length, 5); + assert.ok(selectedTracks.stacks.every(track => track.attemptId.endsWith('-2'))); + assert.equal(replayTimeline(selectedTracks).length, 5); + for (const view of ['graph', 'replay'] as const) { + const html = campaignPage({ sheet, progression, view, step: 99 }); + assert.doesNotMatch(html, /class="d f"|99%|>77</); + assert.equal((html.match(/class="d p"/g) ?? []).length, 5); + if (view === 'replay') assert.match(html, /5 \/ 5/); + } + +}); + +test('pending cost has an accessible activity dot without changing the amount', () => { + const unknown = { status: 'unknown' as const, costUsd: null }; + assert.equal(spend(unknown), 'Unknown'); + assert.match(spend(unknown, true), /Unknown.*aria-label="Cost still updating"/); + assert.doesNotMatch(spend(unknown, true), /so far|partial/); +}); diff --git a/tools/stack-bench/tests/database-containers.test.ts b/tools/stack-bench/tests/database-containers.test.ts new file mode 100644 index 00000000000..f23e99bacf8 --- /dev/null +++ b/tools/stack-bench/tests/database-containers.test.ts @@ -0,0 +1,15 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { databaseContainerName } from '../src/stacks/database-containers.js'; + +test('database container defaults isolate development from appliance services', () => { + assert.equal(databaseContainerName('postgres', {}), 'stack-bench-dev-postgres'); + assert.equal(databaseContainerName('mongodb', {}), 'stack-bench-dev-mongodb'); + assert.equal(databaseContainerName('postgres', { STACK_BENCH_APPLIANCE: '1' }), + 'stack-bench-postgres'); + assert.equal(databaseContainerName('mongodb', { STACK_BENCH_APPLIANCE: '1' }), + 'stack-bench-mongodb'); + assert.equal(databaseContainerName('postgres', { POSTGRES_CONTAINER: 'custom-pg' }), 'custom-pg'); + assert.equal(databaseContainerName('mongodb', { MONGO_CONTAINER: 'custom-mongo' }), 'custom-mongo'); +}); diff --git a/tools/stack-bench/tests/database-provenance.integration.ts b/tools/stack-bench/tests/database-provenance.integration.ts new file mode 100644 index 00000000000..b95b5607c68 --- /dev/null +++ b/tools/stack-bench/tests/database-provenance.integration.ts @@ -0,0 +1,65 @@ +import assert from 'node:assert/strict'; +import { once } from 'node:events'; +import { createServer } from 'node:http'; +import test from 'node:test'; +import { chromium, type Browser } from 'playwright'; +import { loadTrack } from '../src/composition/tracks.js'; +import { verifyApplicationDatabaseMarker, writeApplicationDatabaseMarker } from '../commands/run-suite.js'; + +test('browser signup provenance requires independent storage and releases its browser context', async () => { + const stored = new Set<string>(); + let persist = false; + let signedUp: string | null = null; + const app = createServer(async (request, response) => { + if (request.url === '/signup') { + let body = ''; + for await (const chunk of request) body += String(chunk); + const { username } = JSON.parse(body) as { username: string }; + signedUp = username; + if (persist) stored.add(username); + response.writeHead(200, { 'Content-Type': 'application/json' }); + response.end('{}'); + return; + } + response.writeHead(200, { 'Content-Type': 'text/html' }); + response.end(`<input id="signup-username"><input id="signup-password" type="password"> + <button id="signup-submit">Create account</button><script> + document.querySelector('#signup-submit').onclick = async () => { + const username = document.querySelector('#signup-username').value; + await fetch('/signup', {method:'POST', body:JSON.stringify({username})}); + const current = document.createElement('strong'); current.id = 'current-user'; + current.textContent = username; document.body.append(current); + }; + </script>`); + }); + app.listen(0, '127.0.0.1'); + await once(app, 'listening'); + const address = app.address(); + assert(address && typeof address !== 'string'); + let browser: Browser | undefined; + try { + browser = await chromium.launch({ headless: true }); + const track = loadTrack('ecommerce'); + assert.deepEqual(track.databaseProvenance, { browserAction: 'signUp' }); + for (persist of [false, true]) { + signedUp = null; + let observed: string | null = null; + const result = await verifyApplicationDatabaseMarker( + { backend: 'postgres', url: `http://127.0.0.1:${address.port}` }, track.databaseProvenance, { + write: (args, definition) => writeApplicationDatabaseMarker(args, definition, { browser }), + read: (_args, marker) => { + observed = marker ?? null; + return { ok: !!marker && stored.has(marker), verified: true, reason: 'independent store' }; + }, + }); + assert.equal(result.write.ok, true, 'the same visible signup succeeds in both controls'); + assert(signedUp); + assert.equal(observed, signedUp, 'the independent read looks for the marker the browser submitted'); + assert.equal(result.runtime?.ok, persist, 'a success page alone cannot prove persistence'); + assert.equal(browser.contexts().length, 0); + } + } finally { + await browser?.close(); + await new Promise<void>((resolve, reject) => app.close(error => error ? reject(error) : resolve())); + } +}); diff --git a/tools/stack-bench/tests/definition-compiler.test.ts b/tools/stack-bench/tests/definition-compiler.test.ts new file mode 100644 index 00000000000..cada5d1c073 --- /dev/null +++ b/tools/stack-bench/tests/definition-compiler.test.ts @@ -0,0 +1,376 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { + ACTION_IDS, + DEFINITION_SCHEMA_VERSION, + compileScenarioDefinition, + compileTrackManifest, + type CompiledStep, +} from '../src/composition/definition-compiler.js'; +import { ACTION_IMPLEMENTATIONS, ACTION_REGISTRY } from '../src/actions/action-catalog.js'; +import { TRACKS_DIR } from '../src/composition/tracks.js'; + +interface CurrentDefinition { + kind: 'scenario' | 'track'; + source: string; + value: unknown; +} + +function currentDefinitions(): CurrentDefinition[] { + const definitions: CurrentDefinition[] = []; + for (const track of readdirSync(TRACKS_DIR)) { + const root = join(TRACKS_DIR, track); + let manifest: unknown; + try { manifest = JSON.parse(readFileSync(join(root, 'track.json'), 'utf8')); } + catch { continue; } + definitions.push({ kind: 'track', source: join(root, 'track.json'), value: manifest }); + const scenarios = join(root, 'scenarios'); + for (const file of readdirSync(scenarios).filter(name => name.endsWith('.json'))) { + definitions.push({ kind: 'scenario', source: join(scenarios, file), + value: JSON.parse(readFileSync(join(scenarios, file), 'utf8')) }); + } + } + return definitions; +} + +test('all current definitions compile deterministically without source mutation', () => { + const definitions = currentDefinitions(); + assert(definitions.length > 10); + for (const definition of definitions) { + const original = structuredClone(definition.value); + const first = definition.kind === 'track' + ? compileTrackManifest(definition.value, { source: definition.source }) + : compileScenarioDefinition(definition.value, { source: definition.source }); + const second = definition.kind === 'track' + ? compileTrackManifest(definition.value, { source: definition.source }) + : compileScenarioDefinition(definition.value, { source: definition.source }); + assert.equal(first.schemaVersion, DEFINITION_SCHEMA_VERSION); + assert.deepEqual(first, second); + assert.deepEqual(definition.value, original, 'compiler must not mutate source'); + } +}); + +test('bounded repeats expand to independent ordinary actions and reject nesting or invalid input', () => { + const wrap = (step: unknown) => ({ schemaVersion: 1, level: 1, features: [{ id: 1, name: 'Repeated writes', setup: [], + criteria: [{ id: 'a', desc: 'writes retain evidence', points: 1, steps: [step] }] }] }); + const input = wrap({ repeat: 2, steps: [{ do: 'fill', actor: 'a', testid: 'name', text: 'value' }, + { do: 'click', actor: 'a', testid: 'save' }] }); + const original = structuredClone(input), output = compileScenarioDefinition(input); + const steps = output.features[0]!.criteria[0]!.steps; + assert.deepEqual(steps.map(step => step.do), ['fill', 'click', 'fill', 'click']); + assert.equal(output.features[0]!.criteria[0]!.points, 1); + assert.deepEqual(compileScenarioDefinition(output), output); + steps[0]!.text = 'changed'; assert.equal(steps[2]!.text, 'value'); + assert.deepEqual(input, original); + const eachInput = wrap({ forEach: ['Beta', 'Alpha'], steps: [ + { do: 'fill', actor: 'a', testid: 'name', text: '{value}' }, + { do: 'expectSequence', actor: 'a', testid: 'name', equals: ['{value}'] }, + ] }); + const eachOriginal = structuredClone(eachInput), eachOutput = compileScenarioDefinition(eachInput); + assert.deepEqual(eachOutput.features[0]!.criteria[0]!.steps.map(step => step.text ?? step.equals), + ['Beta', ['Beta'], 'Alpha', ['Alpha']]); + assert.deepEqual(compileScenarioDefinition(eachOutput), eachOutput); + assert.deepEqual(eachInput, eachOriginal); + for (const step of [ + ...[0, -1, 1.5, 1001, Infinity, '2'].map(repeat => ({ repeat, steps: [{ do: 'click', actor: 'a', testid: 'save' }] })), + { repeat: 2, steps: [] }, { repeat: 2, steps: [{ repeat: 2, steps: [{ do: 'click', actor: 'a', testid: 'save' }] }] }, + { repeat: 2, steps: [{ do: 'noSuchAction' }] }, { repeat: 2, steps: [{ do: 'fill' }] }, + { repeat: 2, steps: [{ do: 'click', actor: 'a', testid: 'save' }], extra: true }, + ...[[], [''], [1], 'value', Array(1001).fill('x')].map(forEach => ({ forEach, steps: [{ do: 'click', actor: 'a', testid: 'save' }] })), + { repeat: 2, forEach: ['a'], steps: [{ do: 'click', actor: 'a', testid: 'save' }] }, + { forEach: ['a'], steps: [] }, + { forEach: ['a'], steps: [{ forEach: ['b'], steps: [{ do: 'click', actor: 'a', testid: 'save' }] }] }, + { forEach: ['a'], steps: [{ repeat: 2, steps: [{ do: 'click', actor: 'a', testid: 'save' }] }] }, + ]) assert.throws(() => compileScenarioDefinition(wrap(step))); + assert.throws(() => compileScenarioDefinition(wrap({ repeat: 1000, + steps: Array.from({ length: 11 }, () => ({ do: 'click', actor: 'a', testid: 'save' })) })), /expanded steps exceed 10000/); +}); + +test('authored waits and observation windows fit inside their action deadlines', () => { + const visit = (step: CompiledStep, source: string): void => { + const action = ACTION_REGISTRY.get(step.do); + assert(action, `${source}: ${step.do} must be registered`); + const deadline = action.timeoutMs; + if (step.do === 'wait') { + assert(typeof step.ms === 'number', `${source}: wait must have a duration`); + assert(deadline > step.ms, `${source}: ${step.do} ${step.ms}ms exceeds its deadline`); + } + if (step.within !== undefined) { + assert(typeof step.within === 'number', `${source}: within must be a number`); + assert(deadline > step.within, `${source}: ${step.do} ${step.within}ms exceeds its deadline`); + } + if (step.branches) { + assert(Array.isArray(step.branches), `${source}: race must have branches`); + for (const branch of step.branches) { + assert(Array.isArray(branch), `${source}: each race branch must be an array`); + for (const child of branch) { + assert(isRecord(child) && typeof child.do === 'string', `${source}: race step must be valid`); + visit(child, source); + } + } + } + }; + for (const definition of currentDefinitions().filter(item => item.kind === 'scenario')) { + const compiled = compileScenarioDefinition(definition.value, { source: definition.source }); + for (const feature of compiled.features) { + [...feature.setup, ...feature.criteria.flatMap(criterion => criterion.steps)] + .forEach(step => visit(step, definition.source)); + } + } +}); + +test('definitions require schema v1 and explicit suite inheritance', () => { + const manifest = { title: 'Example', validatedThrough: 1, plannedThrough: 1, + portOffset: 500, restartProbe: '/ready', suites: { 1: [ + { id: 'features', spec: 'scenarios/01-features.json' }, + ] } }; + assert.throws(() => compileTrackManifest(manifest), /schemaVersion: unsupported version undefined/); + assert.throws(() => compileTrackManifest({ ...manifest, schemaVersion: 1 }), + /inherit: must be none or all-higher-levels/); + const { schemaVersion: _, ...missingScenarioVersion } = scenario() as Record<string, unknown>; + assert.throws(() => compileScenarioDefinition(missingScenarioVersion, { source: 'scenario.json' }), + /schemaVersion: unsupported version undefined/); +}); + +test('the action implementation registry covers every action exactly once', () => { + assert.deepEqual(Object.keys(ACTION_IMPLEMENTATIONS).sort(), ACTION_IDS); + assert.doesNotThrow(() => compileScenarioDefinition(scenario({ do: 'openItem', actor: 'a', item: 'Desk Lamp' }))); + assert.throws(() => compileScenarioDefinition(scenario({ do: 'openItem', actor: 'a' })), /item/); +}); + +function scenario(step: unknown = { do: 'wait', actor: 'a', ms: 1 }): unknown { + return { schemaVersion: 1, level: 1, features: [{ id: 1, name: 'feature', actors: ['a'], setup: [], + criteria: [{ id: '1a', desc: 'criterion', steps: [step] }] }] }; +} + +test('unknown actions and action fields fail before runtime', () => { + const staleScenario = scenario() as Record<string, unknown>; + assert.throws(() => compileScenarioDefinition({ ...staleScenario, status: 'draft' }), + /status: unknown field/); + assert.throws(() => compileScenarioDefinition(scenario({ do: 'notReal', actor: 'a' })), + /unknown action "notReal"/); + assert.throws(() => compileScenarioDefinition(scenario( + { do: 'wait', actor: 'a', ms: 1, miliseconds: 1 })), /miliseconds: unknown field/); +}); + +test('required action fields and nested race steps are validated', () => { + assert.throws(() => compileScenarioDefinition(scenario({ do: 'wait', actor: 'a' })), + /\.ms: is required/); + assert.throws(() => compileScenarioDefinition(scenario({ do: 'race', settleMs: 1, + branches: [[{ do: 'wait', actor: 'a', ms: 1 }], [{ do: 'wat', actor: 'a', ms: 1 }]] })), + /branches\[1\]\[0\]\.do: unknown action "wat"/); +}); + +test('extracted action inputs expose their runtime options without allowing script escapes', () => { + const compiled = compileScenarioDefinition(scenario({ do: 'forgeWrite', actor: 'a', + fromActor: 'victim', settleMs: 1, field: 'room', value: 7, text: 'probe' })); + const feature = compiled.features[0]; + assert(feature, 'the fixture must compile one feature'); + const criterion = feature.criteria[0]; + assert(criterion, 'the fixture must compile one criterion'); + const step = criterion.steps[0]; + assert(step, 'the fixture must compile one step'); + assert.equal(step.value, 7); + assert.equal(criterion.points, 1, + 'the compiler must materialize the default before scoring'); + assert.doesNotThrow(() => compileScenarioDefinition(scenario({ do: 'callConcurrently', + actors: ['a', 'b'], action: 'checkout', settleMs: 1, args: [], body: {} }))); + for (const requests of [1, 4, 16, 64]) assert.doesNotThrow(() => + compileScenarioDefinition(scenario({ do: 'callConcurrently', actors: ['a', 'b'], + action: 'checkout', settleMs: 0, requests, requestTimeoutMs: 30000 }))); + for (const requests of [0, 65, 1.5]) assert.throws(() => + compileScenarioDefinition(scenario({ do: 'callConcurrently', actors: ['a', 'b'], + action: 'checkout', settleMs: 0, requests })), /requests/); + assert.throws(() => compileScenarioDefinition(scenario({ do: 'callConcurrently', + actors: ['a', 'b'], action: 'checkout', settleMs: 0, requestTimeoutMs: 60001 })), /requestTimeoutMs/); + assert.doesNotThrow(() => compileScenarioDefinition(scenario({ do: 'dbRecordStock', item: 'Keyboard', as: 'before' }))); + assert.doesNotThrow(() => compileScenarioDefinition(scenario({ do: 'dbExpectStock', item: 'Keyboard', relativeTo: 'before', plus: -1 }))); + assert.throws(() => compileScenarioDefinition(scenario({ do: 'dbExpectStock', item: 'Keyboard' })), /requires equals/); + assert.doesNotThrow(() => compileScenarioDefinition(scenario({ do: 'dbExpectStock', item: 'Keyboard', atLeast: 74, atMost: 75 }))); + assert.throws(() => compileScenarioDefinition(scenario({ do: 'dbExpectStock', item: 'Keyboard', atLeast: 75, atMost: 74 })), /lower bound/); + assert.throws(() => compileScenarioDefinition(scenario({ do: 'dbExpectStock', item: 'Keyboard', equals: 0, plus: 1 })), /plus requires/); + assert.throws(() => compileScenarioDefinition(scenario({ do: 'callConcurrently', + actors: ['a', 'a'], action: 'checkout', settleMs: 1 })), /at least two distinct actors/); + assert.throws(() => compileScenarioDefinition(scenario({ do: 'clickConcurrently', + actors: ['a', 'a'], testid: 'checkout', settleMs: 1 })), /at least two distinct actors/); + assert.throws(() => compileScenarioDefinition(scenario({ do: 'replayConcurrently', + actors: ['a', 'a'], settleMs: 1 })), /at least two distinct actors/); + assert.throws(() => compileScenarioDefinition(scenario({ do: 'sendConcurrently', + senders: [{ actor: 'a', prefix: 'one', count: 1 }, + { actor: 'a', prefix: 'two', count: 1 }], delayMs: 1 })), + /at least two distinct actors/); + assert.throws(() => compileScenarioDefinition(scenario({ do: 'expectAgreement', + actors: ['a'], testid: 'total' })), /at least two distinct actors/); + assert.throws(() => compileScenarioDefinition(scenario({ do: 'wait', actor: 'a', ms: -1 })), + /wrong type or value/); + assert.doesNotThrow(() => compileScenarioDefinition(scenario({ do: 'expectNotReceived', + actor: 'a', contains: 'secret', within: 1000 }))); + assert.doesNotThrow(() => compileScenarioDefinition(scenario({ do: 'signUp', actor: 'a', + name: 'seeded', exact: true }))); + assert.doesNotThrow(() => compileScenarioDefinition(scenario({ do: 'callAction', actor: 'a', + action: 'buy', input: { testid: 'item-card', contains: 'Desk Lamp', + attribute: 'data-buy-input' }, authentication: 'none' }))); + const checkout = { do: 'callAction', actor: 'a', action: 'checkout', + namedAction: { id: 'checkout', path: '/api/checkout', reducer: 'checkout', args: [] } }; + assert.doesNotThrow(() => compileScenarioDefinition(scenario(checkout))); + assert.doesNotThrow(() => compileScenarioDefinition(scenario({ ...checkout, browserOrigin: 'same-site' }))); + assert.throws(() => compileScenarioDefinition(scenario({ ...checkout, + browserOrigin: 'same-site', authentication: 'none' })), /cannot override browser authentication/); + assert.doesNotThrow(() => compileScenarioDefinition(scenario({ ...checkout, + namedAction: { ...checkout.namedAction, args: [1], params: [{ name: 'price', in: 'body' }] } }))); + assert.throws(() => compileScenarioDefinition(scenario({ ...checkout, + namedAction: { ...checkout.namedAction, args: [1] } })), /input: is required/); + assert.throws(() => compileScenarioDefinition(scenario({ do: 'callAction', actor: 'a', + action: 'checkout' })), /inline action/); + assert.doesNotThrow(() => compileScenarioDefinition(scenario({ do: 'expectActionOutcome', + actor: 'a', outcome: 'refused', routeProvenBy: 'owner' }))); + assert.throws(() => compileScenarioDefinition(scenario({ do: 'callAction', actor: 'a', + action: 'buy', input: { testid: 'item-card', attribute: 'data-buy-input' }, + authentication: 'guest' })), /authentication: must be "actor", "optional", "none", "session-control", or "tampered-session"/); + const namedReplay = { do: 'replayAs', actor: 'a', from: 'staff', match: 'ship', + namedAction: { id: 'ship', path: '/api/fulfilment/ship', reducer: 'ship_order', args: [0], + params: [{ name: 'orderId', in: 'body', wireType: 'u64' }] }, + namedTarget: { testid: 'order-item', contains: 'Webcam', + attribute: 'data-entity-id', valueType: 'number' } }; + assert.doesNotThrow(() => compileScenarioDefinition(scenario(namedReplay))); + const { namedTarget: _target, ...literalNamedReplay } = namedReplay; + assert.doesNotThrow(() => compileScenarioDefinition(scenario(literalNamedReplay))); + assert.doesNotThrow(() => compileScenarioDefinition(scenario({ ...namedReplay, + namedAction: { ...namedReplay.namedAction, method: 'PATCH' } }))); + assert.throws(() => compileScenarioDefinition(scenario({ ...namedReplay, + namedAction: { ...namedReplay.namedAction, method: 'GET' } })), + /method: must be "DELETE", "PATCH", "POST", or "PUT"/); + const { namedAction: _omitted, ...missingNamedAction } = namedReplay; + assert.throws(() => compileScenarioDefinition(scenario(missingNamedAction)), + /namedTarget requires namedAction/); + assert.throws(() => compileScenarioDefinition(scenario({ ...namedReplay, + namedTarget: { ...namedReplay.namedTarget, valueType: 'bigint' } })), + /valueType: must be "number" or "string"/); + assert.throws(() => compileScenarioDefinition(scenario({ ...namedReplay, + namedAction: { ...namedReplay.namedAction, + params: [{ name: 'orderId', in: 'body', wireType: 'f64' }] } })), + /wireType: must be "u64"/); + assert.throws(() => compileScenarioDefinition(scenario({ do: 'runScript', + script: '../outside.mjs', args: [] })), /\.script: has the wrong type or value/); + assert.throws(() => compileScenarioDefinition(scenario({ do: 'runScript', + script: 'C:\\outside.mjs', args: [] })), /\.script: has the wrong type or value/); + assert.throws(() => compileScenarioDefinition(scenario({ do: 'runScript', + script: 'inside.mjs', args: [], timeoutMs: 90_000 })), + /\.timeoutMs: has the wrong type or value/); +}); + +test('declared scenario level is authoritative and can be checked explicitly', () => { + assert.throws(() => compileScenarioDefinition(scenario(), { expectedLevel: 2 }), + /declares L1, expected L2/); +}); + +test('only explicitly withheld zero-point criteria may omit executable steps', () => { + const withheld = { schemaVersion: 1, level: 1, + features: [{ id: 1, name: 'feature', actors: ['a'], setup: [], + criteria: [{ id: '1a', desc: 'future oracle', points: 0, + withheld: 'awaiting calibration', steps: [] }] }] }; + assert.doesNotThrow(() => compileScenarioDefinition(withheld)); + const missingReason = { schemaVersion: 1, level: 1, + features: [{ id: 1, name: 'feature', actors: ['a'], setup: [], + criteria: [{ id: '1a', desc: 'future oracle', points: 0, steps: [] }] }] }; + assert.throws(() => compileScenarioDefinition(missingReason), /may be empty only/); +}); + +test('track manifests reject unknown fields and malformed named actions', () => { + const base = { schemaVersion: 1, title: 'Example', slug: 'example', validatedThrough: 1, plannedThrough: 1, + portOffset: 500, restartProbe: '/ready', suites: { 1: [{ id: 'features', + inherit: 'none', spec: 'scenarios/01.json' }] } }; + assert.throws(() => compileTrackManifest({ ...base, unknowable: true }), + /unknowable: unknown field/); + assert.throws(() => compileTrackManifest({ ...base, actions: [{ id: 'buy', path: 'api/buy', + reducer: 'buy', args: [] }] }), /path: must be an absolute HTTP path/); + assert.throws(() => compileTrackManifest({ ...base, actions: [{ id: 'buy', path: '/api/buy', + reducer: 'buy', args: [0], params: [{ name: 'itemId', in: 'path', placeholder: ':id' }] }] }), + /placeholder: does not appear in path/); + for (const browserAction of [undefined, 'signIn', 'missing']) { + assert.throws(() => compileTrackManifest({ ...base, databaseProvenance: { browserAction } }), + /browserAction: must be signUp/); + } + assert.throws(() => compileTrackManifest({ ...base, + databaseProvenance: { browserAction: 'signUp', action: 'signUp' } }), /action: unknown field/); + assert.doesNotThrow(() => compileTrackManifest({ ...base, + databaseProvenance: { browserAction: 'signUp' } })); + assert.doesNotThrow(() => compileTrackManifest({ ...base, reseedOnReset: true })); +}); + +test('the live grader rejects malformed definitions before launching a browser', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-definition-preflight-')); + const spec = join(root, 'invalid.json'); + try { + writeFileSync(spec, JSON.stringify(scenario( + { do: 'wait', actor: 'a', ms: 1, miliseconds: 1 }))); + assert.throws(() => execFileSync(process.execPath, + [join(TRACKS_DIR, '..', 'dist', 'grader', 'grade.js'), '--url', 'http://127.0.0.1:1', + '--level', '1', '--spec', spec], { encoding: 'utf8', stdio: 'pipe', timeout: 10_000 }), + error => { + assert(isRecord(error), 'the grader must return a process error'); + assert.match(`${String(error.stdout ?? '')}${String(error.stderr ?? '')}`, + /miliseconds: unknown field/); + return true; + }); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +function isRecord(value: unknown): value is Record<string, unknown> { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +test('named-action overrides accept only declared locator fields', () => { + const input = { testid: 'support-link-order', attribute: 'data-action-input', + overrides: { caseId: { actor: 'b', testid: 'support-ticket', attribute: 'data-entity-id' } } }; + const call = { do: 'callAction', actor: 'a', action: 'link', input }; + assert.doesNotThrow(() => compileScenarioDefinition(scenario(call))); + for (const override of [{ ...input.overrides.caseId, actor: '' }, + { ...input.overrides.caseId, value: 'untrusted literal' }]) { + assert.throws(() => compileScenarioDefinition(scenario({ ...call, + input: { ...input, overrides: { caseId: override } } }))); + } +}); + +test('a named-action input can name the container that holds its control', () => { + const input = { testid: 'support-refund', attribute: 'data-action-input', + in: { testid: 'support-ticket', contains: 'Witness refund case' } }; + const call = { do: 'callAction', actor: 'a', action: 'refund', input }; + assert.doesNotThrow(() => compileScenarioDefinition(scenario(call))); + for (const scope of [{ testid: '' }, { testid: 'support-ticket', attribute: 'data-entity-id' }]) { + assert.throws(() => compileScenarioDefinition(scenario({ ...call, input: { ...input, in: scope } }))); + } +}); + +test('named-action input fields name distinct declared parameters', () => { + const namedAction = { id: 'cart', path: '/api/cart/:itemId', method: 'PATCH', reducer: 'update_cart_quantity', + args: [0, 2], params: [{ name: 'itemId', in: 'path', placeholder: ':itemId' }, { name: 'quantity', in: 'body' }] }; + const call = (fields: unknown) => scenario({ do: 'callAction', actor: 'a', action: 'cart', namedAction, + input: { testid: 'cart-item', attribute: 'data-cart-input', fields } }); + assert.doesNotThrow(() => compileScenarioDefinition(call(['itemId']))); + for (const fields of [[], ['itemId', 'itemId'], [''], ['price'], 'itemId']) { + assert.throws(() => compileScenarioDefinition(call(fields)), /fields/); + } +}); + +test('relative number comparisons require a baseline and one unambiguous bound', () => { + const step = { do: 'expectNumber', actor: 'a', testid: 'timer', relativeTo: 'initial', plus: -1 }; + for (const comparison of ['atMost', 'atLeast']) { + assert.doesNotThrow(() => compileScenarioDefinition(scenario({ ...step, comparison }))); + assert.throws(() => compileScenarioDefinition(scenario({ do: 'expectNumber', actor: 'a', testid: 'timer', equals: 2, comparison })), /requires relativeTo/); + assert.throws(() => compileScenarioDefinition(scenario({ ...step, comparison, [comparison]: 2 })), /same bound/); + } + assert.throws(() => compileScenarioDefinition(scenario({ ...step, comparison: 'less' })), /comparison/); +}); + + +test('catalog creation expectations require exact nonnegative minor units', () => { + const wrap = (priceMinor: unknown) => ({ schemaVersion: 1, level: 1, features: [{ id: 1, name: 'Catalog', setup: [], + criteria: [{ id: 'a', desc: 'stored price', points: 1, steps: [{ do: 'dbExpectCatalogItem', before: 'catalog', name: 'Product', priceMinor }] }] }] }); + for (const price of [0, 125, Number.MAX_SAFE_INTEGER]) assert.doesNotThrow(() => compileScenarioDefinition(wrap(price))); + for (const price of [-1, 1.25, Number.MAX_SAFE_INTEGER + 1, Infinity, '125']) assert.throws(() => compileScenarioDefinition(wrap(price))); +}); diff --git a/tools/stack-bench/tests/definition-snapshots.contract.ts b/tools/stack-bench/tests/definition-snapshots.contract.ts new file mode 100644 index 00000000000..ae021c49e9a --- /dev/null +++ b/tools/stack-bench/tests/definition-snapshots.contract.ts @@ -0,0 +1,33 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { checkDefinitionSnapshots } from '../commands/definition-snapshots.js'; +import { ACTION_IDS, compileScenarioDefinition } from '../src/composition/definition-compiler.js'; +import { canonicalDefinitionJson } from '../src/composition/definition-plan.js'; +import { TRACKS_DIR } from '../src/composition/tracks.js'; + +test('committed normalized definition plans have no semantic drift', () => { + const result = checkDefinitionSnapshots(); + assert.deepEqual(result.changed, []); + assert(result.checked > 0); +}); + +test('the action fixture represents every registered action', () => { + const path = join(TRACKS_DIR, '..', 'tests', 'fixtures', 'definitions', 'all-actions.json'); + const compiled = compileScenarioDefinition(JSON.parse(readFileSync(path, 'utf8')), { source: path }); + const represented = compiled.features[0]?.criteria[0]?.steps.map(step => step.do).sort(); + assert.deepEqual(represented, ACTION_IDS); +}); + +test('canonical definition JSON ignores object-key order but preserves execution order', () => { + assert.equal( + canonicalDefinitionJson({ b: 2, a: { d: 4, c: 3 } }), + canonicalDefinitionJson({ a: { c: 3, d: 4 }, b: 2 }), + ); + assert.notEqual( + canonicalDefinitionJson({ steps: ['first', 'second'] }), + canonicalDefinitionJson({ steps: ['second', 'first'] }), + ); +}); diff --git a/tools/stack-bench/tests/demo.test.ts b/tools/stack-bench/tests/demo.test.ts new file mode 100644 index 00000000000..2d7ab904d50 --- /dev/null +++ b/tools/stack-bench/tests/demo.test.ts @@ -0,0 +1,22 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { demoConfiguration } from '../appliance/demo.js'; + +test('demo uses three workers, isolated release dependencies and only a model-free trial', () => { + const digest = 'a'.repeat(64); + const config = demoConfiguration(`STACK_BENCH_CONTROLLER_IMAGE=sha256:${digest}\n` + + 'STACK_BENCH_STATE_ROOT=/var/lib/docker/volumes/stack-bench-state/_data\n' + + 'STACK_BENCH_RELEASE_MANIFEST=\n' + + 'STACK_BENCH_BUILD_IMAGE=example=value\n', { UNRELATED_SECRET: 'not-saved' }); + assert.equal(config.env.STACK_BENCH_BUILD_IMAGE, 'example=value'); + assert.equal(config.env.STACK_BENCH_RELEASE_DEPS_VOLUME, 'stack-bench-release-deps-aaaaaaaaaaaa'); + assert.doesNotMatch(config.savedEnvironment, /not-saved|UNRELATED_SECRET/); + const other = demoConfiguration(`STACK_BENCH_CONTROLLER_IMAGE=sha256:${'b'.repeat(64)}\n`, {}); + assert.notEqual(other.output, config.output); + assert.notEqual(other.env.STACK_BENCH_RELEASE_DEPS_VOLUME, config.env.STACK_BENCH_RELEASE_DEPS_VOLUME); +}); + +test('demo refuses an unresolved image or malformed setup environment', () => { + assert.throws(() => demoConfiguration('STACK_BENCH_CONTROLLER_IMAGE=local:tag\n', {}), /digest/); + assert.throws(() => demoConfiguration('invalid\n', {}), /environment/); +}); diff --git a/tools/stack-bench/tests/dependency-campaign-live.integration.ts b/tools/stack-bench/tests/dependency-campaign-live.integration.ts new file mode 100644 index 00000000000..06fff8a591b --- /dev/null +++ b/tools/stack-bench/tests/dependency-campaign-live.integration.ts @@ -0,0 +1,417 @@ +import assert from 'node:assert/strict'; +import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { execFile, spawnSync } from 'node:child_process'; +import { createServer } from 'node:http'; +import { promisify } from 'node:util'; +import test from 'node:test'; + +import { generateCampaignReport } from '../src/campaigns/campaign-report.js'; +import { validateCampaignRun } from '../src/campaigns/campaign-run-validation.js'; +import { sha256 } from '../src/evidence/provenance.js'; +import { compileCampaignFile } from '../src/campaigns/campaign-compiler.js'; +import { campaignUsesNoExternalResources, runCampaignAdmission } + from '../src/campaigns/campaign-admission.js'; +import { loadTrack } from '../src/composition/tracks.js'; +import { requireRecipeRelease as resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { readCampaignState } from '../src/campaigns/campaign-scheduler.js'; +import { artifactPayload, readArtifact } from '../src/evidence/artifacts.js'; +import type { BenchmarkRunRecord, GradeBundlePayload } + from '../src/evidence/benchmark-run.js'; +import { validateProgressionCampaignLevelScope } + from '../src/progression/progression-recipe-selection.js'; +import { replayDependencyMode } from '../src/progression/dependency-mode.js'; +import { dependencyRuntimeDefinition } from '../src/progression/progression-definition.js'; +import type { ProgressionStatePayload } from '../src/progression/progression-state.js'; +import { STACK_BENCH_ROOT, compiledEntrypoint } from '../src/package-root.js'; + +const CAMPAIGN = join(STACK_BENCH_ROOT, 'tests', 'fixtures', + 'dependency-model-free-campaign.json'); +const CLI = compiledEntrypoint('commands', 'campaign-cli.js'); + +test('dependency campaign scope must match the graph-derived scope', () => { + const plan = compileCampaignFile(CAMPAIGN); + const declared = plan.conditions[0].requested.levels[0]; + assert(declared); + assert(plan.featureCatalog); + const featureCatalog = plan.featureCatalog; + const binding = resolveRecipeRelease(loadTrack(plan.definition.track), 1, declared.recipe.id); + assert.doesNotThrow(() => validateProgressionCampaignLevelScope(binding, + featureCatalog, declared, 1)); + const tampered = structuredClone(declared); + tampered.selection.sha256 = '0'.repeat(64); + assert.throws(() => validateProgressionCampaignLevelScope(binding, + featureCatalog, tampered, 1), /graph-derived scope changed/); +}); + +test('real stack campaigns retain full preflight admission', async () => { + const output = mkdtempSync(join(tmpdir(), 'stack-bench-standard-admission-')); + try { + const plan = compileCampaignFile(join(STACK_BENCH_ROOT, 'tests', 'fixtures', + 'campaign.deterministic.json')); + assert.equal(campaignUsesNoExternalResources(plan), false); + let calls = 0; + const admission = await runCampaignAdmission(plan, output, { + preflight: request => { + calls += 1; + return { + schemaVersion: 1, + generatedAt: new Date(0).toISOString(), + request: { + backends: request.backends, + track: request.track, + levels: request.levelList, + runIndex: request.runIndex, + parallelism: request.parallelism, + agentAdapter: request.agentAdapter, + guidance: request.guidance, + packs: request.packIds, + checks: request.checkKeys, + recipe: request.recipe ?? null, + requestedScopeCount: request.requestedScopes?.length ?? 0, + image: request.image, + resultsDir: request.resultsDir, + agentSkills: request.agentSkills ?? null, + smoke: request.smoke, + }, + ok: true, + summary: { passed: 1, failed: 0, warnings: 0 }, + checks: [{ id: 'full-preflight', status: 'pass', summary: 'Full preflight ran' }], + }; + }, + }); + assert.equal(calls, plan.summary.parallelism); + assert.equal(admission.payload.ok, true); + // Application smoke runs in each real attempt after its resources exist. + assert(admission.payload.reports.every(report => report.request.smoke === false)); + assert(admission.payload.reports.every(report => + report.request.backends.some(backend => backend !== 'stub'))); + } finally { + rmSync(output, { recursive: true, force: true }); + } +}); + +test('a real model-free campaign persists dependency repairs and evidence', { timeout: 300_000 }, () => { + const output = mkdtempSync(join(tmpdir(), 'stack-bench-dependency-live-')); + try { + const result = spawnSync(process.execPath, [CLI, 'trial', CAMPAIGN, '--out', output], { + cwd: STACK_BENCH_ROOT, + encoding: 'utf8', + timeout: 240_000, + maxBuffer: 32 * 1024 * 1024, + }); + assert.equal(result.error, undefined, result.error?.message); + assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`); + + const { plan, state } = readCampaignState(output); + assert.equal(state.status, 'completed'); + assert.equal(state.summary.completed, 1); + assert.equal(state.summary.invalid, 0); + assert.equal(state.attempts.length, 1); + const campaignAttempt = state.attempts[0]; + assert(campaignAttempt); + assert.equal(campaignAttempt.plan.mode.id, 'dependency'); + assert.equal(campaignAttempt.plan.stack, 'stub'); + assert.equal(campaignAttempt.plan.agentAdapter, 'deterministic'); + assert.equal(campaignAttempt.plan.model, 'deterministic-stall'); + assert.equal(campaignAttempt.executions.length, 1); + const execution = campaignAttempt.executions[0]; + assert(execution); + assert.equal(execution.outcome, 'app_failure'); + + const executionDirectory = join(output, execution.output); + const run = readArtifact<BenchmarkRunRecord>(join(executionDirectory, 'run.json'), { + expectedKind: 'benchmark_run', + }); + assert.equal(run.attempt.parentId, campaignAttempt.plan.id); + assert(plan.featureCatalog); + assert(plan.dependencyPolicy); + assert.deepEqual(run.payload.featureCatalog, plan.featureCatalog.identity); + assert.deepEqual(run.payload.dependencyPolicy, plan.dependencyPolicy.identity); + assert(run.payload.progressionStatus); + assert.equal(run.payload.progressionStatus.phase, 'terminal'); + assert.equal(run.payload.progressionStatus.attempts, 3); + assert.equal(run.payload.levels.length, 1); + const level = run.payload.levels[0]; + assert(level); + assert.equal(level.repairs, 2); + assert(run.payload.outcome); + assert.equal(run.payload.outcome.kind, 'app_failure'); + + const progression = readArtifact<ProgressionStatePayload>( + join(executionDirectory, 'progression-state.json'), { + expectedKind: 'progression_state', + }); + assert.equal(progression.attempt.id, campaignAttempt.plan.id); + assert(progression.identities.experiment); + assert.equal(progression.identities.experiment.sha256, plan.contentSha256); + assert.equal(progression.payload.events.length, 3); + assert.deepEqual(progression.payload.events.map(event => event.type), [ + 'attempt-recorded', + 'attempt-recorded', + 'attempt-recorded', + ]); + const runtimeDefinition = dependencyRuntimeDefinition( + plan.featureCatalog, plan.dependencyPolicy); + const replayed = replayDependencyMode(runtimeDefinition, + progression.payload.events); + assert.equal(replayed.phase, 'terminal'); + assert.equal(replayed.attempts.length, 3); + const accounts = replayed.nodes.accounts; + assert(accounts); + assert.equal(accounts.repairs.used, 1); + assert.equal(accounts.exhaustionReason, 'feature-repairs-exhausted'); + const catalog = replayed.nodes.catalog; + assert(catalog); + assert.equal(catalog.repairs.used, 1); + assert.equal(catalog.exhaustionReason, 'feature-repairs-exhausted'); + assert.deepEqual(replayed.attempts.map(attempt => attempt.attemptId), + Array.from({ length: 3 }, (_, index) => `${run.id}-progression-${index + 1}`)); + assert(replayed.attempts.every(attempt => attempt.evidence?.kind === 'grade_bundle')); + assert(replayed.attempts.every(attempt => attempt.evidence !== undefined + && /^[a-f0-9]{64}$/.test(attempt.evidence.sha256))); + const duplicate = structuredClone(progression.payload.events); + const firstEvent = duplicate[0]; + const secondEvent = duplicate[1]; + assert(firstEvent?.result); + assert(secondEvent?.result); + secondEvent.result.attemptId = firstEvent.result.attemptId; + assert.throws(() => replayDependencyMode(runtimeDefinition, duplicate), + /owned progression sequence|duplicate attempt id/); + + const expectedChecks = [ + [ + 'ecommerce.feature.accounts.accounts.1a', + 'ecommerce.feature.catalog.catalog-values.2a', + ], + ['ecommerce.feature.accounts.accounts.1a'], + ['ecommerce.feature.catalog.catalog-values.2a'], + ]; + for (let sequence = 1; sequence <= 3; sequence += 1) { + const attempt = String(sequence).padStart(3, '0'); + const bundlePath = join(executionDirectory, 'progression', `attempt-${attempt}`, 'bundle.json'); + assert(existsSync(bundlePath), `missing progression evidence ${bundlePath}`); + const bundle = readArtifact<GradeBundlePayload>(bundlePath, { + expectedKind: 'grade_bundle', + }); + assert.equal(bundle.attempt.parentId, run.id); + assert(bundle.payload.selection); + assert(bundle.payload.selection.checks); + assert.deepEqual(bundle.payload.selection.checks.map(check => check.stableKey), + expectedChecks[sequence - 1]); + } + assert(existsSync(join(executionDirectory, 'level-l1-checkpoint.json'))); + } finally { + rmSync(output, { recursive: true, force: true }); + } +}); + +test('a shuffled repair order runs in the frozen order the plan drew', { timeout: 300_000 }, () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-shuffled-live-')); + try { + const campaign = JSON.parse(readFileSync(CAMPAIGN, 'utf8')) as { + repair: { order?: string }; + ordering: { seed: string }; + }; + campaign.repair.order = 'shuffled'; + // This seed draws the catalog roots in reverse, so the run proves the + // permutation reaches the runtime rather than repeating declared order. + campaign.ordering.seed = 'internal-shuffled-proof-3'; + const campaignPath = join(root, 'campaign.json'); + const output = join(root, 'results'); + writeFileSync(campaignPath, `${JSON.stringify(campaign, null, 2)}\n`); + const result = spawnSync(process.execPath, [CLI, 'trial', campaignPath, '--out', output], { + cwd: STACK_BENCH_ROOT, + encoding: 'utf8', + timeout: 240_000, + maxBuffer: 32 * 1024 * 1024, + }); + assert.equal(result.error, undefined, result.error?.message); + assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`); + const { plan, state } = readCampaignState(output); + assert(plan.featureCatalog && plan.dependencyPolicy); + assert.equal(plan.dependencyPolicy.definition.repair.order, 'shuffled'); + assert.deepEqual(plan.dependencyPolicy.definition.nodeOrder, ['catalog', 'accounts']); + const runtimeDefinition = dependencyRuntimeDefinition(plan.featureCatalog, plan.dependencyPolicy); + assert.deepEqual(runtimeDefinition.nodes.map(node => node.id), ['catalog', 'accounts']); + const execution = state.attempts[0]?.executions[0]; + assert(execution); + assert.equal(execution.outcome, 'app_failure'); + const progression = readArtifact<ProgressionStatePayload>( + join(output, execution.output, 'progression-state.json'), { expectedKind: 'progression_state' }); + const replayed = replayDependencyMode(runtimeDefinition, progression.payload.events); + assert.equal(replayed.phase, 'terminal'); + // Catalog fails first, so it is the first feature repaired. + assert.deepEqual(replayed.attempts.filter(attempt => attempt.repair) + .map(attempt => attempt.repair?.nodeIds), [['catalog'], ['accounts']]); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('feature work runs same-depth features and writes one depth record', { timeout: 300_000 }, () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-feature-live-')); + try { + const campaign = JSON.parse(readFileSync(CAMPAIGN, 'utf8')) as { + mode: { workSelection: string }; + }; + campaign.mode.workSelection = 'feature'; + const campaignPath = join(root, 'campaign.json'); + const output = join(root, 'results'); + writeFileSync(campaignPath, `${JSON.stringify(campaign, null, 2)}\n`); + const result = spawnSync(process.execPath, [CLI, 'trial', campaignPath, '--out', output], { + cwd: STACK_BENCH_ROOT, + encoding: 'utf8', + timeout: 240_000, + maxBuffer: 32 * 1024 * 1024, + }); + assert.equal(result.error, undefined, result.error?.message); + assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`); + const { state } = readCampaignState(output); + const execution = state.attempts[0]?.executions[0]; + assert(execution); + const run = readArtifact<BenchmarkRunRecord>(join(output, execution.output, 'run.json'), { + expectedKind: 'benchmark_run', + }); + assert.equal(run.payload.levels.length, 1); + assert.equal(run.payload.levels[0]?.firstBuild, undefined); + assert.equal(run.payload.levels[0]?.buildSessions?.length, 2); + assert.equal(run.payload.levels[0]?.repairSessions?.length, 2); + assert.equal(run.payload.levels[0]?.repairs, 2); + assert.equal(run.payload.progressionStatus?.attempts, 4); + assert(existsSync(join(output, execution.output, + 'candidate-grades', 'l1-action001'))); + assert(existsSync(join(output, execution.output, + 'candidate-grades', 'l1-action003'))); + assert(existsSync(join(output, execution.output, 'level-l1-checkpoint.json'))); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + + +test('repairing an earlier feature keeps the current execution grade, checkpoint, and costs together', { timeout: 300_000 }, async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-cross-depth-repair-')); + const work = join(root, 'work'); + // The stub deliberately has no application lifecycle. Serve its changing + // source here while the real CLI executes and grades both levels. + mkdirSync(work); + const server = createServer((_request, response) => { + const app = readdirSync(work)[0]; + const index = app && join(work, app, 'app', 'index.html'); + response.setHeader('Content-Type', 'text/html'); + response.end(index && existsSync(index) ? readFileSync(index) : 'Not built'); + }); + await new Promise<void>((resolve, reject) => server.once('error', reject).listen(7300, resolve)); + try { + const campaign = JSON.parse(readFileSync(CAMPAIGN, 'utf8')); + campaign.agents[0].model = 'deterministic-ecommerce'; + campaign.pricing.models['deterministic-ecommerce'] = campaign.pricing.models['deterministic-stall']; + delete campaign.pricing.models['deterministic-stall']; + campaign.repair.budget = { total: 3, perFeature: 2 }; + campaign.featureCatalog.nodes[1].dependencies = [{ id: 'accounts', reason: 'Catalog requires an account' }]; + campaign.selection.levels.push({ level: 2, recipe: 'ecommerce.sequential-l2' }); + const campaignPath = join(root, 'campaign.json'); + const output = join(root, 'results'); + writeFileSync(campaignPath, `${JSON.stringify(campaign, null, 2)}\n`); + const result = await promisify(execFile)(process.execPath, [CLI, 'trial', campaignPath, '--out', output], { + cwd: STACK_BENCH_ROOT, encoding: 'utf8', timeout: 240_000, + maxBuffer: 32 * 1024 * 1024, + env: { ...process.env, STACK_BENCH_WORK_DIR: work }, + }).catch(error => { throw new Error(`${error.stdout?.slice(-7000) ?? ''}\n${error.stderr ?? error.message}`); }); + const { plan, state } = readCampaignState(output); + const execution = state.attempts[0]?.executions[0]; + assert(execution); + assert.equal(execution.status, 'completed', execution.reason ?? result.stdout); + const directory = join(output, execution.output); + const recordedRun = readArtifact<BenchmarkRunRecord>(join(directory, 'run.json'), { expectedKind: 'benchmark_run' }); + const run = recordedRun.payload; + assert.deepEqual(run.levels.map(level => ({ level: level.level, score: level.score, + max: level.max, repairs: level.repairs, sessions: level.repairSessions?.length, + cost: level.repairCostUsd, outcome: level.outcome.kind })), [ + { level: 1, score: 1, max: 1, repairs: 1, sessions: 1, cost: 0.05, outcome: 'passed' }, + { level: 2, score: 0, max: 0, repairs: 1, sessions: 1, cost: 0.05, outcome: 'passed' }, + ]); + assert.equal(run.levels[1]?.regression?.score, 2); + assert.equal(run.levels[1]?.regression?.max, 2); + const altered = structuredClone(recordedRun); + altered.payload.levels[1]!.regression = { score: 0, max: 0 }; + assert.throws(() => validateCampaignRun(plan, plan.attempts[0]!, artifactPayload(altered), + { resultDir: directory }), /score|regression/); + assert.equal(run.outcome?.kind, 'passed'); + assert.equal(run.totals?.costUsd, 1.1); + assert(existsSync(join(directory, 'level-l2-checkpoint.json'))); + const saved = readArtifact<ProgressionStatePayload>(join(directory, 'progression-state.json')); + const progression = replayDependencyMode(dependencyRuntimeDefinition(plan.featureCatalog!, plan.dependencyPolicy!), saved.payload.events); + assert.equal(progression.attempts.at(-1)?.level, 2); + const repair = progression.attempts.findLast(attempt => attempt.repair); + assert.equal(repair?.level, 2); + assert.equal(repair?.repair?.depth, 1); + assert.deepEqual(repair?.repair?.nodeIds, ['accounts']); + const checkpoint = run.checkpoints?.at(-1); + assert.equal(checkpoint?.phase, 'final'); + assert.equal(checkpoint?.completion.passed, 2); + assert.deepEqual(checkpoint?.checks.map(check => [check.id, check.status]).sort(), [ + ['ecommerce.feature.accounts.accounts.1a', 'passed'], + ['ecommerce.feature.catalog.catalog-values.2a', 'passed'], + ]); + } finally { + await new Promise<void>((resolve, reject) => server.close(error => error ? reject(error) : resolve())); + rmSync(root, { recursive: true, force: true }); + } +}); + +test('bench checkpoints retain actual grade bytes, full check scope, and unknown unreceipted cost', { timeout: 300_000 }, () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-checkpoint-live-')); + try { + const campaign = JSON.parse(readFileSync(CAMPAIGN, 'utf8')); + campaign.agents[0].model = 'deterministic'; + campaign.pricing.models.deterministic = campaign.pricing.models['deterministic-stall']; + delete campaign.pricing.models['deterministic-stall']; + campaign.repair.budget.total = 1; + const campaignPath = join(root, 'campaign.json'); + const output = join(root, 'results'); + writeFileSync(campaignPath, `${JSON.stringify(campaign, null, 2)}\n`); + const result = spawnSync(process.execPath, [CLI, 'trial', campaignPath, '--out', output], { + cwd: STACK_BENCH_ROOT, encoding: 'utf8', timeout: 240_000, + maxBuffer: 32 * 1024 * 1024, + }); + assert.equal(result.error, undefined, result.error?.message); + assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`); + const { plan, state } = readCampaignState(output); + const execution = state.attempts[0]?.executions[0]; + assert(execution); + assert.equal(execution.status, 'completed', `${execution.reason}\n${result.stdout}\n${result.stderr}`); + const directory = join(output, execution.output); + const run = readArtifact<BenchmarkRunRecord>(join(directory, 'run.json'), { expectedKind: 'benchmark_run' }); + const checkpoints = run.payload.checkpoints; + assert(checkpoints); + assert.deepEqual(checkpoints.map(point => point.phase), ['first-build', 'repair']); + const selected = new Set(plan.conditions[0].requested.levels.flatMap(level => + (level.selection.scoredChecks ?? []).filter(check => check.points > 0).map(check => check.stableKey))); + assert.equal(selected.size, 2, 'the fixture selects two independent product checks'); + for (const [index, point] of checkpoints.entries()) { + assert.equal(point.sequence, index + 1); + assert.equal(point.completion.selected, selected.size, 'a feature repair cannot shrink the denominator'); + assert.deepEqual(new Set(point.checks.map(check => check.id)), selected); + assert.deepEqual(point.cost, { status: 'unknown', costUsd: null }); + assert.deepEqual(point.executionCost, { status: 'unknown', costUsd: null }); + const path = join(directory, point.evidence.path); + const bundle = readArtifact<GradeBundlePayload>(path, { expectedKind: 'grade_bundle' }); + assert.equal(point.evidence.sha256, sha256(readFileSync(path)), 'hash the exact retained grade bytes'); + assert.equal(bundle.payload.source?.sha256, point.sourceSha256); + assert.equal(bundle.payload.selection?.sha256, point.selectionSha256); + } + assert.notEqual(checkpoints[0]?.sourceSha256, checkpoints[1]?.sourceSha256, + 'the immutable first-build grade survives the repaired source'); + assert.equal(run.payload.levels[0]?.buildSessions?.length, 1); + assert.equal(run.payload.levels[0]?.repairSessions?.length, 1); + assert.equal(run.payload.totals?.costUsd, 0.55, 'the existing fixture emits synthetic costs, without receipts'); + const report = generateCampaignReport(output).report; + assert.equal(report.attempts[0]?.curve.checkpoints.length, 2); + assert.equal(report.attempts[0]?.completion.selected, selected.size); + assert.equal(report.summary.spend.status, 'unknown', 'synthetic totals do not invent receipt proof'); + assert.equal(report.attempts[0]?.metrics?.totalCostUsd, null); + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/tools/stack-bench/tests/dependency-mode.test.ts b/tools/stack-bench/tests/dependency-mode.test.ts new file mode 100644 index 00000000000..fe240ed9146 --- /dev/null +++ b/tools/stack-bench/tests/dependency-mode.test.ts @@ -0,0 +1,872 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { dependencyRepairStopReason, type ConclusiveResult, type DependencyGradingSelection, type DependencyState, + type DependencyPromptSelection } from '../src/progression/dependency-mode.js'; +import { compileDependencyMode } from '../src/progression/dependency-definition.js'; +import { dependencyCompletionBreakdown, scoreDependencyState, type DependencyScore } from '../src/progression/dependency-score.js'; +import { progressionGradeCompletions } from '../src/campaigns/campaign-report.js'; +import { compileDependencyPolicyInput, compileFeatureCatalogInput } + from '../src/progression/progression-definition.js'; +import { progressionEngine, type ProgressionWorkAction } + from '../src/progression/progression-engine.js'; +import type { CheckCategory } from '../src/composition/definition-compiler.js'; +import type { ProgressionState } from '../src/progression/progression-state.js'; +import { resolveFeatureCatalog } from '../src/progression/feature-catalog-selection.js'; +import { loadTrack } from '../src/composition/tracks.js'; +import { dependencyRuntimeDefinition } from '../src/progression/progression-definition.js'; +import type { RepairPlan } from '../src/progression/repair-plan.js'; + +type RepairPlanInput = Pick<RepairPlan, 'selection' | 'budget'> & { order?: RepairPlan['order'] }; +type Outcome = 'pass' | 'fail' | 'blocked' | 'not-run'; +type Outcomes = Record<string, Outcome | Record<string, Outcome>>; + +test('an ecommerce account feature must pass its production checks before dependent work', () => { + // A production guarantee gates like a feature check: a password-security + // failure and broken sign-in both stop account-dependent work. + const catalog = resolveFeatureCatalog('progression/ecommerce.json', loadTrack('ecommerce')); + const policy = compileDependencyPolicyInput({ selection: 'feature', budget: { total: 0 } }, catalog, + { selectedLevels: [1, 2, 3] }); + const definition = dependencyRuntimeDefinition(catalog, policy); + for (const failed of ['1c', '1d']) { + let state = progressionEngine.initialize(definition); + const accounts = definition.nodes.find(node => node.id === 'accounts')!; + const check = accounts.gradingChecks.find(check => check.id.endsWith(`.${failed}`)); + assert(check, `${failed} must remain scored`); + state = progressionEngine.recordResult(state, grade(state, failed, { accounts: { [check.id]: 'fail' } })); + const dependent = definition.nodes.find(node => node.dependencies.length === 1 && node.dependencies[0] === 'accounts'); + assert(dependent); + assert.equal(state.nodes.accounts!.status, 'failed'); + assert.equal(prompt(state).nodeIds.includes(dependent.id), false); + const score = progressionEngine.score(state) as DependencyScore; + assert(score.completion.failed > 0, 'a security failure must not earn full completion'); + } +}); + +test('a failed live-state check stops work that depends on its feature', () => { + const catalog = resolveFeatureCatalog('progression/ecommerce.json', loadTrack('ecommerce')); + const policy = compileDependencyPolicyInput({ selection: 'feature', budget: { total: 0 } }, catalog, + { selectedLevels: [1, 2, 3] }); + const definition = dependencyRuntimeDefinition(catalog, policy); + const owner = definition.nodes.find(node => node.gradingChecks.some(check => check.id.endsWith('.3b')))!; + const check = owner.gradingChecks.find(check => check.id.endsWith('.3b'))!; + assert.equal(check.role, 'guarantee'); + let state = progressionEngine.initialize(definition); + for (let attempt = 0; prompt(state).nodeIds.length > 0; attempt++) { + state = progressionEngine.recordResult(state, grade(state, `pass-${attempt}`, { [owner.id]: { [check.id]: 'fail' } })); + } + assert.equal(state.nodes[owner.id]!.status, 'failed'); + const dependents = definition.nodes.filter(node => node.dependencies.includes(owner.id)); + assert(dependents.length > 0); + for (const dependent of dependents) assert.equal(state.nodes[dependent.id]!.status, 'blocked', dependent.id); +}); + +interface FixtureNode { + id: string; + title: string; + questline: string; + dependencies: Array<{ id: string; reason: string }>; + featureRefs: string[]; + promptModules: string[]; + gradingChecks: Array<{ id: string; points: number; role: 'feature' | 'guarantee'; category?: CheckCategory; + requiresFeatures?: string[] }>; +} + +interface FixtureDefinition { + schemaVersion: number; + kind: string; + id: string; + title: string; + policy: string; + repair: RepairPlanInput; + unchangedFailureLimit: number; + workSelection: 'feature' | 'progressive' | 'all-at-once'; + nodes: FixtureNode[]; + questlines: Array<{ id: string; title: string }>; +} + +const node = (id: string, dependencies: string[], questline: string, + points: number[] = [1], requires: string[] = []): FixtureNode => ({ + id, title: id, questline, + dependencies: dependencies.map(parent => ({ id: parent, reason: `${id} requires ${parent}` })), + featureRefs: [`features.${id}`], + promptModules: [`prompt.${id}`], + gradingChecks: points.map((value, index) => ({ + id: `check.${id}.${index + 1}`, points: value, role: 'feature', category: 'feature', + ...(requires.length ? { requiresFeatures: requires.map(feature => `features.${feature}`) } : {}), + })), +}); + +const fixture = (): FixtureDefinition => ({ + schemaVersion: 7, + kind: 'progression-mode', + id: 'storefront-paths', + title: 'Storefront paths', + policy: 'dependency-graph', + repair: { selection: 'feature', budget: { total: 4, perFeature: 1 } }, + unchangedFailureLimit: 3, + workSelection: 'progressive', + nodes: [ + node('accounts', [], 'identity', [2, 1]), + node('catalog', [], 'discovery'), + node('ownership', ['accounts'], 'identity'), + node('search', ['catalog'], 'discovery'), + node('recovery', ['ownership'], 'identity'), + node('recommendations', ['search'], 'discovery'), + ], + questlines: [ + { id: 'identity', title: 'Identity' }, + { id: 'discovery', title: 'Discovery' }, + ], +}); + +const prompt = (state: ProgressionState): DependencyPromptSelection => + progressionEngine.promptSelection(state) as DependencyPromptSelection; +const grading = (state: ProgressionState): DependencyGradingSelection => + progressionEngine.gradingSelection(state) as DependencyGradingSelection; +const action = (state: ProgressionState): ProgressionWorkAction => { + const next = progressionEngine.nextAction(state); + if (next.type === 'terminal') throw new Error('expected active work'); + return next; +}; + +function grade(state: ProgressionState, attemptId: string, + outcomes: Outcomes): ConclusiveResult { + const selection = grading(state); + return { + attemptId, + outcome: 'conclusive', + nodes: selection.nodeIds.map(nodeId => ({ + id: nodeId, + checks: selection.checks.filter(check => check.nodeId === nodeId).map(check => ({ + id: check.id, + outcome: typeof outcomes[nodeId] === 'string' + ? outcomes[nodeId] : outcomes[nodeId]?.[check.id] ?? 'pass', + })), + })), + }; +} + +function repairedGrade(state: ProgressionState, attemptId: string, + outcomes: Outcomes): ConclusiveResult { + const current = action(state); + if (current.type !== 'repair') throw new Error('expected repair work'); + return { ...grade(state, attemptId, outcomes), completedRepair: true }; +} + +test('cross-depth repair records execution depth without changing the feature budget', () => { + let state = progressionEngine.initialize(fixture()); + state = progressionEngine.recordResult(state, grade(state, 'l1', {})); + state = progressionEngine.recordResult(state, grade(state, 'l2', {})); + state = progressionEngine.recordResult(state, grade(state, 'l3', { ownership: 'fail' })); + assert.equal(action(state).level, 2); + const completed = progressionEngine.recordResult(state, { + ...repairedGrade(state, 'repair', {}), executionLevel: 3, + }); + assert.equal(completed.attempts.at(-1)?.level, 3); + assert.equal(completed.attempts.at(-1)?.repair?.depth, 2); + assert.equal(completed.nodes.ownership?.repairs.used, 1); + assert.deepEqual(progressionEngine.replay(completed.definition, completed.events), completed); + const interrupted = progressionEngine.recordResult(state, { + attemptId: 'interrupted', executionLevel: 3, outcome: 'inconclusive', + category: 'harness_failure', reason: 'grading failed', completedRepair: true, + }); + assert.equal(interrupted.attempts.at(-1)?.level, 3); + assert.equal(interrupted.attempts.at(-1)?.repair?.depth, 2); + assert.equal(interrupted.nodes.ownership?.repairs.used, 1); + assert.deepEqual(progressionEngine.replay(interrupted.definition, interrupted.events), interrupted); + assert.throws(() => progressionEngine.recordResult(state, { + ...repairedGrade(state, 'invalid', {}), executionLevel: 99, + }), /executionLevel/); +}); + +function guaranteeFixture(): FixtureDefinition { + const definition = fixture(); + definition.workSelection = 'feature'; + definition.nodes = [ + node('owner', [], 'identity'), + node('other', [], 'discovery'), + node('descendant', ['owner'], 'identity'), + ]; + definition.questlines = [ + { id: 'identity', title: 'Identity' }, + { id: 'discovery', title: 'Discovery' }, + ]; + definition.nodes[0]!.gradingChecks.push({ + id: 'check.owner.guarantee', points: 1, role: 'guarantee', category: 'production', + requiresFeatures: ['features.other'], + }); + return definition; +} + +test('blocked prerequisites retain zero credit, repair eligibility and separate completion counts', () => { + let state = progressionEngine.initialize(fixture()); + state = progressionEngine.recordResult(state, grade(state, 'blocked-setup', { + accounts: 'blocked', catalog: 'pass', + })); + const score = progressionEngine.score(state) as DependencyScore; + assert.equal(score.completion.failed, 0); + assert(score.completion.blocked >= 2); + assert.equal(score.nodes.find(n => n.id === 'accounts')!.blockedPoints, 3); + assert.equal(action(state).type, 'repair'); + state = progressionEngine.recordResult(state, repairedGrade(state, 'fixed-setup', { + accounts: 'pass', catalog: 'pass', + })); + assert.equal((progressionEngine.score(state) as DependencyScore).nodes.find(n => n.id === 'accounts')!.passedPoints, 3); +}); + +test('a deferred guarantee runs and must be repaired before descendants unlock', () => { + let state = progressionEngine.initialize(guaranteeFixture()); + state = progressionEngine.recordResult(state, grade(state, 'owner', { owner: 'pass' })); + assert.equal(state.nodes.owner!.status, 'working'); + assert.deepEqual(prompt(state).nodeIds, ['other']); + const initial = dependencyCompletionBreakdown(state); + assert.deepEqual(initial.featureCompletion, { selected: 3, passed: 0, rate: 0 }); + assert.equal(initial.checkCategories.feature.passed, 1); + assert.equal(initial.checkCategories.production.unmeasured, 1); + + state = progressionEngine.recordResult(state, grade(state, 'other', { + owner: { 'check.owner.1': 'pass', 'check.owner.guarantee': 'fail' }, + other: 'pass', + })); + assert.equal(state.nodes.owner!.status, 'working'); + assert.equal(state.nodes.descendant!.status, 'locked'); + assert.deepEqual(action(state).repair.nodeIds, ['owner']); + + state = progressionEngine.recordResult(state, repairedGrade(state, 'guarantee-repair', { + owner: { 'check.owner.1': 'pass', 'check.owner.guarantee': 'pass' }, + other: 'pass', + })); + assert.equal(state.nodes.owner!.status, 'passed'); + assert.deepEqual(prompt(state).nodeIds, ['descendant']); +}); + +test('a guarantee that cannot run keeps its node from unlocking descendants', () => { + const definition = guaranteeFixture(); + definition.repair.budget = { total: 0 }; + let state = progressionEngine.initialize(definition); + state = progressionEngine.recordResult(state, grade(state, 'owner', { owner: 'pass' })); + state = progressionEngine.recordResult(state, grade(state, 'other-fails', { + owner: { 'check.owner.1': 'pass', 'check.owner.guarantee': 'fail' }, + other: 'fail', + })); + assert.equal(state.nodes.owner!.checks['check.owner.guarantee'], null); + assert.equal(state.nodes.owner!.status, 'working'); + assert.equal(state.nodes.descendant!.status, 'locked'); + assert.deepEqual(prompt(state).nodeIds, []); + const score = progressionEngine.score(state) as DependencyScore; + const owner = score.nodes.find(node => node.id === 'owner')!; + assert.equal(owner.completion.blocked, 1); + assert.equal(owner.completion.unmeasured, 0); + assert.equal(owner.blockedPoints, 1); + assert.equal(owner.ungradedPoints, 0); + assert.equal(dependencyCompletionBreakdown(state).checkCategories.production.blocked, 1); +}); + +test('a regressed prerequisite clears a guarantee result from an earlier pass', () => { + let state = progressionEngine.initialize(guaranteeFixture()); + state = progressionEngine.recordResult(state, grade(state, 'owner', { owner: 'pass' })); + state = progressionEngine.recordResult(state, grade(state, 'other', { + owner: { 'check.owner.1': 'pass', 'check.owner.guarantee': 'pass' }, other: 'pass', + })); + state = progressionEngine.recordResult(state, grade(state, 'other-regresses', { + owner: { 'check.owner.1': 'pass', 'check.owner.guarantee': 'fail' }, + other: 'fail', descendant: 'pass', + })); + assert.equal(state.nodes.owner!.checks['check.owner.guarantee'], null); + assert.equal(dependencyCompletionBreakdown(state).checkCategories.production.blocked, 1); +}); + +test('the graph compiles by depth, then declared order, and the order is identity', () => { + const declared = compileDependencyMode(fixture()); + assert.deepEqual(declared.nodes.map(item => item.id), [ + 'accounts', 'catalog', 'ownership', 'search', 'recovery', 'recommendations', + ]); + const input = fixture(); + input.nodes.reverse(); + input.questlines.reverse(); + const compiled = compileDependencyMode(input); + assert.deepEqual(compiled.nodes.map(item => item.id), [ + 'catalog', 'accounts', 'search', 'ownership', 'recommendations', 'recovery', + ]); + assert.deepEqual(compiled.questlines.map(item => item.id), ['discovery', 'identity']); + assert.deepEqual(compiled.repair, + { selection: 'feature', budget: { total: 4, perFeature: 1 }, order: 'declared' }); + const { policy: _policy, repair, unchangedFailureLimit: _limit, + workSelection: _workSelection, ...catalogDefinition } = compiled; + const catalog = compileFeatureCatalogInput({ + ...catalogDefinition, schemaVersion: 1, kind: 'feature-catalog', + }); + const first = compileDependencyPolicyInput(repair, catalog, { unchangedFailureLimit: 2 }); + const second = compileDependencyPolicyInput(repair, catalog, { unchangedFailureLimit: 3 }); + assert.equal(first.definition.id, 'dependency-graph'); + assert.notEqual(first.identity.contentSha256, second.identity.contentSha256); + // Declared order is repair priority, so it is part of the catalog identity + // that every campaign binds; reordering the catalog is a new identity. + const { policy: _p, repair: _r, unchangedFailureLimit: _l, + workSelection: _w, ...declaredCatalog } = declared; + const declaredIdentity = compileFeatureCatalogInput({ + ...declaredCatalog, schemaVersion: 1, kind: 'feature-catalog', + }).identity.contentSha256; + assert.notEqual(declaredIdentity, catalog.identity.contentSha256); +}); + +test('invalid graphs and repair plans fail before execution', async t => { + const cases: Array<[string, (value: FixtureDefinition) => void, RegExp]> = [ + ['duplicate node', value => value.nodes.push(structuredClone(value.nodes[0]!)), /duplicates/], + ['missing parent', value => { value.nodes[2]!.dependencies[0]!.id = 'missing'; }, /unknown parent/], + ['dependency cycle', value => { + value.nodes[0]!.dependencies = [{ id: 'ownership', reason: 'cycle' }]; + }, /dependency cycle/], + ['empty budget', value => { value.repair.budget = {}; }, /contain a limit/], + ['undefined budget', value => { + value.repair.budget = { total: undefined } as unknown as FixtureDefinition['repair']['budget']; + }, /contain a limit/], + ['negative budget', value => { value.repair.budget.total = -1; }, /non-negative safe integer/], + ['bad selection', value => { value.repair.selection = 'other' as 'feature'; }, /feature.*batch/], + ['bad order', value => { value.repair.order = 'random' as 'declared'; }, /declared.*shuffled/], + ['bad work selection', value => { + value.workSelection = 'other' as 'feature'; + }, /feature.*progressive.*all-at-once/], + ['unknown questline', value => { value.nodes[0]!.questline = 'missing'; }, /unknown questline/], + ['empty questline', value => { value.questlines.push({ id: 'empty', title: 'Empty' }); }, + /owns no nodes/], + ['no feature check', value => { + value.nodes[1]!.gradingChecks[0]!.role = 'guarantee'; + }, /at least one feature check/], + ['check owned twice', value => { + value.nodes[1]!.gradingChecks[0]!.id = value.nodes[0]!.gradingChecks[0]!.id; + }, /already owned by another node/], + ['missing dependency reason', value => { + value.nodes[2]!.dependencies[0] = { id: 'accounts' } as { id: string; reason: string }; + }, /reason/], + ['cross-branch prerequisite', value => { + value.nodes.push(node('dashboard', ['search'], 'discovery', [1], ['ownership'])); + }, /feature check check\.dashboard\.1 on dashboard cannot require features\.ownership owned by ownership/], + ]; + for (const [name, mutate, expected] of cases) { + await t.test(name, () => { + const value = fixture(); + mutate(value); + assert.throws(() => compileDependencyMode(value), expected); + }); + } + await t.test('unknown policy', () => { + assert.throws(() => progressionEngine.initialize({ ...fixture(), policy: 'missing' }), /policy/); + }); +}); + +test('an initial failure costs no repair and passed branches continue', () => { + let state = progressionEngine.initialize(fixture()); + state = progressionEngine.recordResult(state, grade(state, 'initial', { + accounts: 'fail', catalog: 'pass', + })); + assert.equal(state.nodes.accounts!.repairs.used, 0); + assert.deepEqual(action(state).repair, { nodeIds: ['accounts'], remaining: 1 }); + state = progressionEngine.recordResult(state, repairedGrade(state, 'repair', { + accounts: 'fail', catalog: 'pass', search: 'pass', + })); + assert.equal(state.nodes.accounts!.repairs.used, 1); + assert.equal(state.nodes.accounts!.status, 'failed'); + assert.equal(state.nodes.accounts!.exhaustionReason, 'feature-repairs-exhausted'); + assert.equal(state.nodes.ownership!.status, 'blocked'); + assert.equal(state.nodes.search!.status, 'active'); + assert.deepEqual(prompt(state).nodeIds, ['search']); +}); + +test('zero repair budget stops only the failed branch', () => { + const definition = fixture(); + definition.repair.budget = { total: 0 }; + let state = progressionEngine.initialize(definition); + state = progressionEngine.recordResult(state, grade(state, 'initial', { + accounts: 'fail', catalog: 'pass', + })); + assert.equal(state.nodes.accounts!.status, 'failed'); + assert.equal(state.nodes.accounts!.exhaustionReason, 'total-repairs-exhausted'); + assert.equal(state.nodes.accounts!.repairs.used, 0); + assert.deepEqual(prompt(state).nodeIds, ['search']); +}); + +test('a run-wide total never allows a fourth repair anywhere in the run', () => { + const definition = fixture(); + definition.repair.budget = { total: 3, perFeature: 5 }; + definition.unchangedFailureLimit = 10; + let state = progressionEngine.initialize(definition); + state = progressionEngine.recordResult(state, grade(state, 'initial', { + accounts: 'fail', catalog: 'pass', + })); + for (const attemptId of ['repair-1', 'repair-2', 'repair-3']) { + assert.equal(action(state).type, 'repair'); + state = progressionEngine.recordResult(state, repairedGrade(state, attemptId, { + accounts: 'fail', catalog: 'pass', + })); + } + assert.equal(state.attempts.filter(attempt => attempt.repair).length, 3); + assert.equal(state.nodes.accounts!.repairs.used, 3); + assert.equal(state.nodes.accounts!.status, 'failed'); + assert.equal(state.nodes.accounts!.exhaustionReason, 'total-repairs-exhausted'); + // Exhausting the run-wide total ends repairs, not the passed branch's new work. + assert.equal(action(state).type, 'build'); + assert.deepEqual(prompt(state).nodeIds, ['search']); + state = progressionEngine.recordResult(state, grade(state, 'search', { + accounts: 'fail', catalog: 'pass', search: 'fail', + })); + assert.equal(state.nodes.search!.status, 'failed'); + assert.equal(state.nodes.search!.repairs.used, 0); + assert.equal(state.nodes.search!.exhaustionReason, 'total-repairs-exhausted'); + assert.equal(state.phase, 'terminal'); +}); + +test('one batch repair charges the session and each selected feature once', () => { + const definition = fixture(); + definition.repair = { selection: 'batch', budget: { total: 1, perFeature: 1 } }; + let state = progressionEngine.initialize(definition); + state = progressionEngine.recordResult(state, grade(state, 'initial', { + accounts: 'fail', catalog: 'fail', + })); + assert.deepEqual(action(state).repair.nodeIds, ['accounts', 'catalog']); + state = progressionEngine.recordResult(state, repairedGrade(state, 'batch-repair', { + accounts: 'fail', catalog: 'fail', + })); + assert.equal(state.attempts.filter(attempt => attempt.repair).length, 1); + assert.equal(state.nodes.accounts!.repairs.used, 1); + assert.equal(state.nodes.catalog!.repairs.used, 1); +}); + +test('provider failure costs zero and a completed repair with failed grading awaits its grade', () => { + let state = progressionEngine.initialize(fixture()); + state = progressionEngine.recordResult(state, grade(state, 'initial', { + accounts: 'fail', catalog: 'pass', + })); + state = progressionEngine.recordResult(state, { + attemptId: 'provider-failure', outcome: 'inconclusive', + category: 'provider_failure', reason: 'provider stopped', + }); + assert.equal(state.nodes.accounts!.repairs.used, 0); + assert.equal(action(state).repair.awaitingGrade, undefined); + state = progressionEngine.recordResult(state, { + attemptId: 'grade-failure', outcome: 'inconclusive', + category: 'harness_failure', reason: 'grader stopped', + completedRepair: true, + }); + // Charged, but nothing about the feature changes until the repaired source + // is graded: no finalization, no children opened, no fourth state. + assert.equal(state.nodes.accounts!.repairs.used, 1); + assert.equal(state.nodes.accounts!.status, 'active'); + assert.equal(state.nodes.ownership!.status, 'locked'); + assert.equal(state.phase, 'active'); + assert.deepEqual(action(state).repair, { nodeIds: ['accounts'], remaining: 0, awaitingGrade: true }); + // A second grader failure on the regrade neither charges nor changes anything. + state = progressionEngine.recordResult(state, { + attemptId: 'regrade-failure', outcome: 'inconclusive', + category: 'harness_failure', reason: 'grader stopped again', + }); + assert.equal(state.nodes.accounts!.repairs.used, 1); + assert.deepEqual(action(state).repair, { nodeIds: ['accounts'], remaining: 0, awaitingGrade: true }); + // The conclusive regrade settles the charged repair without charging again. + state = progressionEngine.recordResult(state, grade(state, 'regrade', { + accounts: 'fail', catalog: 'pass', + })); + assert.equal(state.nodes.accounts!.repairs.used, 1); + assert.equal(state.nodes.accounts!.status, 'failed'); + assert.equal(state.nodes.accounts!.exhaustionReason, 'feature-repairs-exhausted'); + assert.equal(state.nodes.ownership!.status, 'blocked'); + assert.deepEqual(prompt(state).nodeIds, ['search']); + assert.equal(JSON.stringify(progressionEngine.replay(fixture(), state.events)), + JSON.stringify(state)); +}); + +test('a completed repair with an inconclusive grade spends shared budget but settles only when graded', () => { + const definition = fixture(); + definition.repair.budget = { total: 1 }; + let state = progressionEngine.initialize(definition); + state = progressionEngine.recordResult(state, grade(state, 'initial', { + accounts: 'fail', catalog: 'fail', + })); + state = progressionEngine.recordResult(state, { + attemptId: 'inconclusive-repair', outcome: 'inconclusive', + category: 'harness_failure', reason: 'grader stopped', + completedRepair: true, + }); + assert.equal(state.attempts.filter(attempt => attempt.repair).length, 1); + assert.equal(state.nodes.accounts!.status, 'active'); + assert.equal(state.nodes.catalog!.status, 'active'); + assert.equal(state.phase, 'active'); + state = progressionEngine.recordResult(state, grade(state, 'regrade', { + accounts: 'pass', catalog: 'fail', + })); + assert.equal(state.nodes.accounts!.status, 'passed'); + assert.equal(state.nodes.catalog!.status, 'failed'); + assert.equal(state.nodes.catalog!.exhaustionReason, 'total-repairs-exhausted'); + assert.deepEqual(prompt(state).nodeIds, ['ownership']); +}); + +test('a rolled-back repair with regression feedback is charged exactly once', () => { + let state = progressionEngine.initialize(fixture()); + state = progressionEngine.recordResult(state, grade(state, 'roots', { + accounts: 'pass', catalog: 'fail', + })); + state = progressionEngine.recordResult(state, { + ...repairedGrade(state, 'rolled-back', { accounts: 'pass', catalog: 'fail', ownership: 'pass' }), + repairRegression: { ownerNodeIds: ['catalog'], report: 'The account list no longer loads.' }, + }); + assert.equal(state.attempts.filter(attempt => attempt.repair).length, 1); + assert.equal(state.nodes.catalog!.repairs.used, 1); + assert.equal(state.attempts.at(-1)?.repairRegression?.ownerNodeIds[0], 'catalog'); + assert.equal(state.nodes.catalog!.status, 'failed'); + assert.equal(state.nodes.catalog!.exhaustionReason, 'feature-repairs-exhausted'); +}); + +test('unused depth repairs carry forward when configured', () => { + for (const carry of [true, false]) { + const definition = fixture(); + definition.repair.budget = { perDepth: { count: 1, carry } }; + let state = progressionEngine.initialize(definition); + state = progressionEngine.recordResult(state, grade(state, 'roots', { + accounts: 'pass', catalog: 'pass', + })); + state = progressionEngine.recordResult(state, grade(state, 'depth-two', { + accounts: 'pass', catalog: 'pass', ownership: 'fail', search: 'fail', + })); + assert.deepEqual(action(state).repair, { nodeIds: ['ownership'], remaining: carry ? 2 : 1 }); + state = progressionEngine.recordResult(state, repairedGrade(state, 'ownership-repair', { + accounts: 'pass', ownership: 'fail', + })); + if (carry) { + assert.deepEqual(action(state).repair, { nodeIds: ['ownership'], remaining: 1 }); + continue; + } + assert.equal(state.nodes.ownership!.exhaustionReason, 'depth-repairs-exhausted'); + assert.equal(state.nodes.search!.exhaustionReason, 'depth-repairs-exhausted'); + assert.equal(state.nodes.search!.repairs.used, 0); + } +}); + +test('repeated failures stop their branch while unmeasured work remains build work', () => { + const definition = fixture(); + definition.unchangedFailureLimit = 2; + definition.repair.budget = { perFeature: 5 }; + let state = progressionEngine.initialize(definition); + state = progressionEngine.recordResult(state, grade(state, 'initial', { + accounts: 'not-run', catalog: 'fail', + })); + assert.equal(state.nodes.catalog!.unchangedFailure.count, 1); + state = progressionEngine.recordResult(state, repairedGrade(state, 'unchanged', { + catalog: 'fail', + })); + assert(Object.values(state.nodes.accounts!.checks).every(outcome => outcome === null)); + assert.equal(state.nodes.accounts!.repairs.used, 0); + assert.equal(state.nodes.catalog!.unchangedFailure.count, 2); + assert.equal(state.nodes.catalog!.exhaustionReason, 'repeated-findings'); + assert.equal(action(state).type, 'build'); + assert.deepEqual(prompt(state).nodeIds, ['accounts']); +}); + +test('only a completed repair can count as a repeated finding', () => { + const definition = fixture(); + definition.unchangedFailureLimit = 2; + definition.repair.budget = { perFeature: 5 }; + let state = progressionEngine.initialize(definition); + state = progressionEngine.recordResult(state, grade(state, 'initial', { + accounts: 'fail', catalog: 'pass', + })); + // A regrade of unchanged source observes the same findings without a repair. + state = progressionEngine.recordResult(state, grade(state, 'regrade', { + accounts: 'fail', catalog: 'pass', search: 'pass', + })); + assert.equal(state.nodes.accounts!.unchangedFailure.count, 1); + assert.equal(state.nodes.accounts!.status, 'active'); + // A repair that changes the findings starts the count over. + state = progressionEngine.recordResult(state, repairedGrade(state, 'changed', { + accounts: { 'check.accounts.1': 'pass', 'check.accounts.2': 'fail' }, catalog: 'pass', + })); + assert.equal(state.nodes.accounts!.unchangedFailure.count, 1); + assert.equal(state.nodes.accounts!.repairs.used, 1); + state = progressionEngine.recordResult(state, repairedGrade(state, 'unchanged', { + accounts: { 'check.accounts.1': 'pass', 'check.accounts.2': 'fail' }, catalog: 'pass', + })); + assert.equal(state.nodes.accounts!.unchangedFailure.count, 2); + assert.equal(state.nodes.accounts!.exhaustionReason, 'repeated-findings'); +}); + +test('setup recovery is progress once; cycling through blocked and failed findings stops', () => { + const definition = fixture(); + definition.unchangedFailureLimit = 3; + definition.repair.budget = { perFeature: 10 }; + let state = progressionEngine.initialize(definition); + state = progressionEngine.recordResult(state, grade(state, 'initial-blocked', { + accounts: 'blocked', catalog: 'pass', + })); + state = progressionEngine.recordResult(state, repairedGrade(state, 'setup-recovered', { + accounts: 'fail', catalog: 'pass', + })); + assert.equal(state.nodes.accounts!.unchangedFailure.count, 1); + state = progressionEngine.recordResult(state, repairedGrade(state, 'setup-broken-again', { + accounts: 'blocked', catalog: 'pass', + })); + assert.equal(state.nodes.accounts!.unchangedFailure.count, 2); + state = progressionEngine.recordResult(state, { + attemptId: 'inconclusive', outcome: 'inconclusive', category: 'harness_failure', reason: 'browser crashed', + }); + assert.equal(state.nodes.accounts!.unchangedFailure.count, 2); + state = progressionEngine.recordResult(state, repairedGrade(state, 'setup-recovered-again', { + accounts: 'fail', catalog: 'pass', + })); + assert.equal(state.nodes.accounts!.exhaustionReason, 'repeated-findings'); + assert.deepEqual(progressionEngine.replay(definition, state.events), state); +}); + +test('repair order follows dependency depth, then the declared catalog order', () => { + // The queue repairs the first failed feature by depth, then declared order, + // until it passes or stops, then moves to the next. Alphabetical id order + // is not a factor, and a replay of the same events reproduces the sequence. + const definition = fixture(); + definition.repair.budget = { total: 4, perFeature: 5 }; + definition.unchangedFailureLimit = 3; + definition.nodes = [ + node('zeta', [], 'identity'), + node('alpha', [], 'discovery'), + node('mid', [], 'identity'), + ]; + const compiled = compileDependencyMode(definition); + assert.deepEqual(compiled.nodes.map(item => item.id), ['zeta', 'alpha', 'mid']); + let state = progressionEngine.initialize(definition); + state = progressionEngine.recordResult(state, grade(state, 'initial', { + zeta: 'fail', alpha: 'fail', mid: 'fail', + })); + const sequence: string[][] = []; + while (progressionEngine.nextAction(state).type === 'repair') { + sequence.push([...action(state).repair.nodeIds]); + state = progressionEngine.recordResult(state, repairedGrade(state, `repair-${sequence.length}`, { + zeta: 'fail', alpha: 'fail', mid: 'fail', + })); + } + assert.deepEqual(sequence, [['zeta'], ['zeta'], ['alpha'], ['alpha']]); + assert.equal(state.nodes.zeta!.exhaustionReason, 'repeated-findings'); + // alpha's second repair both reached the stall limit and spent the last + // run-wide repair; the budget check runs first and names the reason. + assert.equal(state.nodes.alpha!.exhaustionReason, 'total-repairs-exhausted'); + assert.equal(state.nodes.mid!.exhaustionReason, 'total-repairs-exhausted'); + assert.equal(state.nodes.mid!.repairs.used, 0); + const replayed = progressionEngine.replay(definition, state.events); + assert.equal(JSON.stringify(replayed), JSON.stringify(state)); + assert.deepEqual(state.attempts.filter(attempt => attempt.repair) + .map(attempt => attempt.repair!.nodeIds), sequence); +}); + +test('application startup failures target current work without consuming a repair', () => { + let state = progressionEngine.initialize(fixture()); + state = progressionEngine.recordResult(state, { + ...grade(state, 'startup-failure', { accounts: 'fail', catalog: 'fail' }), + applicationFailure: { phase: 'startup', reason: 'application did not start' }, + }); + assert.equal(state.nodes.accounts!.repairs.used, 0); + assert.equal(state.nodes.catalog!.repairs.used, 0); + assert.deepEqual(action(state).repair, { nodeIds: ['accounts'], remaining: 1 }); + const missing = grade(state, 'missing-check', { accounts: 'fail' }); + missing.nodes[0]!.checks.pop(); + assert.throws(() => progressionEngine.recordResult(state, missing), /missing checks/); +}); + +test('a result must cover the selected graph exactly', () => { + let state = progressionEngine.initialize(fixture()); + state = progressionEngine.recordResult(state, grade(state, 'initial', { + accounts: 'fail', catalog: 'pass', + })); + const missingNode = grade(state, 'missing-node', { accounts: 'fail' }); + missingNode.nodes.pop(); + assert.throws(() => progressionEngine.recordResult(state, missingNode), /missing nodes/); + const repeated = grade(state, 'repeated-check', { accounts: 'fail' }); + repeated.nodes[0]!.checks.push({ ...repeated.nodes[0]!.checks[0]! }); + assert.throws(() => progressionEngine.recordResult(state, repeated), /repeats check/); + const unselected = grade(state, 'unselected-node', { accounts: 'fail' }); + unselected.nodes.push({ id: 'recovery', checks: [] }); + assert.throws(() => progressionEngine.recordResult(state, unselected), /unselected node/); + const foreignRegression = { + ...repairedGrade(state, 'foreign-regression', { accounts: 'fail', catalog: 'pass' }), + repairRegression: { ownerNodeIds: ['catalog'], report: 'Catalog broke.' }, + }; + assert.throws(() => progressionEngine.recordResult(state, foreignRegression), + /does not belong to the current repair/); + const staleRepair = { ...grade(state, 'stale-repair', { accounts: 'fail' }), completedRepair: true }; + const passed = progressionEngine.recordResult(state, grade(state, 'passed', { + accounts: 'pass', catalog: 'pass', + })); + assert.throws(() => progressionEngine.recordResult(passed, staleRepair), + /only a repair action can record a completed repair/); +}); + +test('feature work selection requests one ready feature at a time', () => { + const definition = fixture(); + definition.workSelection = 'feature'; + definition.repair.budget = { total: 0 }; + let state = progressionEngine.initialize(definition); + assert.deepEqual(prompt(state).nodeIds, ['accounts']); + state = progressionEngine.recordResult(state, grade(state, 'accounts', { accounts: 'pass' })); + assert.deepEqual(prompt(state).nodeIds, ['catalog']); + state = progressionEngine.recordResult(state, grade(state, 'catalog', { + accounts: 'pass', catalog: 'pass', + })); + assert.deepEqual(prompt(state).nodeIds, ['ownership']); +}); + +test('all-at-once repairs one failure and regrades the complete graph', () => { + const definition = fixture(); + definition.workSelection = 'all-at-once'; + let state = progressionEngine.initialize(definition); + assert.deepEqual(prompt(state).nodeIds, [ + 'accounts', 'catalog', 'ownership', 'search', 'recovery', 'recommendations', + ]); + assert.equal(grading(state).checks.length, 7); + state = progressionEngine.recordResult(state, grade(state, 'initial', { + accounts: 'fail', + })); + assert.deepEqual(prompt(state).nodeIds, ['accounts']); + assert.equal(grading(state).nodeIds.length, 6); + state = progressionEngine.recordResult(state, repairedGrade(state, 'repair', { + accounts: 'pass', + })); + assert.equal(state.nodes.accounts!.repairs.used, 1); + assert.equal(state.phase, 'terminal'); + assert.equal(state.terminalOutcome?.kind, 'passed'); +}); + +test('multi-parent work waits for both parents and an ancestor regression closes its path', () => { + const definition = fixture(); + definition.nodes.find(item => item.id === 'ownership')!.dependencies = [ + { id: 'accounts', reason: 'ownership requires accounts' }, + { id: 'catalog', reason: 'ownership requires catalog' }, + ]; + + let waiting = progressionEngine.initialize(definition); + waiting = progressionEngine.recordResult(waiting, grade(waiting, 'one-parent', { + accounts: 'pass', catalog: 'fail', + })); + assert.equal(waiting.nodes.ownership!.status, 'locked'); + + let regressed = progressionEngine.initialize(definition); + regressed = progressionEngine.recordResult(regressed, grade(regressed, 'roots', { + accounts: 'pass', catalog: 'pass', + })); + regressed = progressionEngine.recordResult(regressed, grade(regressed, 'children', { + ownership: 'pass', search: 'pass', + })); + regressed = progressionEngine.recordResult(regressed, grade(regressed, 'ancestor-regression', { + accounts: 'fail', recommendations: 'pass', recovery: 'pass', + })); + assert.deepEqual(prompt(regressed).nodeIds, ['accounts']); + assert.equal(regressed.nodes.ownership!.status, 'locked'); + assert.equal(regressed.nodes.recovery!.status, 'locked'); + assert.equal(regressed.nodes.recommendations!.status, 'passed'); +}); + +test('a failed feature stays failed and replayable when its prerequisite later regresses', () => { + const definition = fixture(); + definition.repair = { selection: 'feature', budget: { total: 0, perFeature: 0 } }; + let state = progressionEngine.initialize(definition); + state = progressionEngine.recordResult(state, grade(state, 'roots', {})); + state = progressionEngine.recordResult(state, grade(state, 'children', { ownership: 'fail' })); + assert.equal(state.nodes.ownership!.status, 'failed'); + state = progressionEngine.recordResult(state, grade(state, 'regression', { accounts: 'fail' })); + assert.equal(state.nodes.ownership!.status, 'failed'); + assert.equal(state.nodes.ownership!.exhaustedAtLevel, 2); + assert.deepEqual(progressionEngine.replay(definition, state.events), state); +}); + +test('a repair grant reopens the exact exhausted branch', () => { + const definition = fixture(); + definition.repair.budget = { total: 0 }; + let state = progressionEngine.initialize(definition); + state = progressionEngine.recordResult(state, grade(state, 'initial', { + accounts: 'fail', catalog: 'fail', + })); + assert.equal(state.phase, 'terminal'); + state = progressionEngine.grantRepairs(state, { + grantId: 'grant-1', level: 1, nodeIds: ['accounts'], repairs: 1, + }); + assert.deepEqual(action(state).repair, { + nodeIds: ['accounts'], remaining: 1, grantId: 'grant-1', + }); + state = progressionEngine.recordResult(state, repairedGrade(state, 'granted-repair', { + accounts: 'pass', + })); + assert.equal(state.nodes.accounts!.status, 'passed'); + assert.equal(state.nodes.accounts!.repairs.used, 1); + assert.deepEqual(prompt(state).nodeIds, ['ownership']); +}); + +test('score keeps blocked points and averages questlines equally', () => { + const definition = fixture(); + definition.repair.budget = { total: 0 }; + let state = progressionEngine.initialize(definition); + state = progressionEngine.recordResult(state, grade(state, 'initial', { + accounts: 'pass', catalog: 'fail', + })); + state = progressionEngine.recordResult(state, grade(state, 'ownership', { + accounts: 'pass', ownership: 'pass', + })); + state = progressionEngine.recordResult(state, grade(state, 'recovery', { + accounts: 'pass', ownership: 'pass', recovery: 'pass', + })); + const score = progressionEngine.score(state) as DependencyScore; + assert.equal(score.questlines.find(item => item.id === 'identity')!.passedPoints, 5); + assert.equal(score.questlines.find(item => item.id === 'discovery')!.blockedPoints, 2); + assert.equal(score.questlineAveragePercentage, 50); + const breakdown = dependencyCompletionBreakdown(state); + assert.deepEqual(breakdown.featureCompletion, { selected: 6, passed: 3, rate: 0.5 }); + assert.equal(breakdown.checkCategories.feature.blocked, 2); + assert.equal(breakdown.checkCategories.feature.selected, 7); + assert.equal(breakdown.checkCategories.production.rate, null); +}); + +test('repair stop summaries distinguish unchanged findings from budget limits', () => { + assert.equal(dependencyRepairStopReason([]), null); + assert.equal(dependencyRepairStopReason([{ exhaustionReason: null }]), null); + assert.equal(dependencyRepairStopReason([{ exhaustionReason: 'repeated-findings' }]), 'repeated-findings'); + assert.equal(dependencyRepairStopReason([{ exhaustionReason: 'repeated-findings' }, + { exhaustionReason: 'feature-repairs-exhausted' }]), 'feature-repairs-exhausted'); +}); + +// Classification is reporting metadata. It must not change dependency gates or scores. +test('check categories preserve scoring and leave historical definitions unclassified', () => { + const definition = fixture(); + const classified = progressionEngine.initialize(definition); + const historicalDefinition = structuredClone(definition); + for (const node of historicalDefinition.nodes) { + for (const check of node.gradingChecks) delete check.category; + } + const historical = progressionEngine.initialize(historicalDefinition); + assert.deepEqual(progressionEngine.score(classified), progressionEngine.score(historical)); + assert.deepEqual(prompt(classified), prompt(historical)); + const unknown = dependencyCompletionBreakdown(historical).checkCategories; + assert.equal(unknown.unknown.selected, 7); + assert.equal(unknown.feature.selected, 0); + assert.equal(unknown.production.selected, 0); + assert.throws(() => compileDependencyMode({ ...definition, nodes: definition.nodes.map(node => ({ + ...node, gradingChecks: node.gradingChecks.map(check => ({ ...check, category: 'browser' })), + })) }), /category/); +}); + +test('an application failure keeps earlier failures measured before the abort', () => { + let state = progressionEngine.initialize(fixture()); + state = progressionEngine.recordResult(state, grade(state, 'initial', {})); + const aborted = { ...grade(state, 'abort', { accounts: 'fail', catalog: 'not-run', ownership: 'fail', search: 'fail' }), + applicationFailure: { phase: 'application-readiness', reason: 'application stopped answering' } }; + const recorded = progressionEngine.recordResult(state, aborted); + assert.deepEqual(Object.values(recorded.nodes.accounts!.checks), ['fail', 'fail']); + assert.deepEqual(Object.values(recorded.nodes.catalog!.checks), ['pass']); + assert.throws(() => progressionEngine.recordResult(state, { ...aborted, + ...grade(state, 'unrun', { ownership: 'not-run' }) }), /measured or fail/); +}); + +test('each recorded grade has the completion the dependency score gave at that moment', () => { + let state = progressionEngine.initialize(fixture()); + const expected = new Map<string, unknown>(); + for (const [index, outcomes] of [{ accounts: 'fail' }, {}].entries()) { + const evidence = { kind: 'grade_bundle' as const, id: `grade-${index}`, sha256: String(index).repeat(64) }; + state = progressionEngine.recordResult(state, { ...grade(state, `run-1-progression-${index + 1}`, outcomes as Outcomes), + runId: 'run-1', evidence }); + expected.set(`${evidence.id}:${evidence.sha256}`, scoreDependencyState(state as DependencyState).completion); + } + assert.deepEqual(progressionGradeCompletions(state as DependencyState), expected); +}); diff --git a/tools/stack-bench/tests/dependency-neutral-prompt.contract.ts b/tools/stack-bench/tests/dependency-neutral-prompt.contract.ts new file mode 100644 index 00000000000..b81551bbefc --- /dev/null +++ b/tools/stack-bench/tests/dependency-neutral-prompt.contract.ts @@ -0,0 +1,319 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { resolve } from 'node:path'; +import test from 'node:test'; + +import { resolveGuidanceProfile } from '../src/campaigns/condition-compiler.js'; +import type { ResolvedGuidanceProfile } from '../src/campaigns/condition-compiler.js'; +import { requireRecipeRelease as resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { loadTrack, portsFor } from '../src/composition/tracks.js'; +import { resolveBoundRecipeTaskRequest } from '../src/composition/recipe-selection.js'; +import { buildPrompt, hostServiceAddress, parseAgentArgs } from '../commands/agent.js'; +import { resolveFeatureCatalog } from '../src/progression/feature-catalog-selection.js'; +import { resolveProgressionRecipeLevelSelection } + from '../src/progression/progression-recipe-selection.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { readAgentSkillDocuments } from '../src/agents/agent-materials.js'; + +test('production framing is one sentence and never changes restoration prompts', () => { + const sentence = 'Build a production-quality application suitable for real users, not a prototype or demo.'; + const track = loadTrack('ecommerce'); + for (const backend of ['mongodb', 'postgres', 'spacetime', 'convex', 'supabase']) for (const level of [1, 2, 3, 4, 5, 6]) + for (const mode of ['build', 'upgrade', 'fix', 'resume']) { + const argv = ['node', 'agent', '--mode', mode, '--backend', backend, '--level', String(level), '--app', '/app', '--guidance', 'neutral']; + const enabled = parseAgentArgs(argv), disabled = parseAgentArgs([...argv, '--no-production-quality']); + const materials = { skillsText: '', requirementText: 'Build the requested store.', contractText: '' }; + const render = (args: typeof enabled) => buildPrompt(args, portsFor(track, backend, 0), track, materials); + const on = render(enabled), off = render(disabled); + assert.equal(on.includes(sentence), mode !== 'resume'); + assert.equal(on.replace(sentence + '\n\n', ''), off); + } +}); + +const AGENT = resolve(STACK_BENCH_ROOT, 'dist', 'commands', 'agent.js'); +const STACKS = ['mongodb', 'postgres', 'spacetime', 'convex', 'supabase'] as const; +const EVALUATION_LANGUAGE = + /\b(?:benchmark|harness|grader|graded|grading|scored|scoring|tests?|testing|evaluation|criterion|testids?)\b|stackbench|Stack Bench|external client|run configuration/i; +const UNSTATED_QUALITY_LANGUAGE = [ + /\b(?:reload|reconnect|live)\b/i, + /\b(?:another customer|another account)\b/i, + /\b(?:changes nothing|without (?:a )?reload)\b/i, + /\benforce (?:this|the) rule on the server\b/i, + /\binvalid quantity\b|`-3`/i, +]; +type Stack = typeof STACKS[number]; +type Level = 1 | 2 | 3 | 4 | 5 | 6; + +test('SDK skills and dev guidance vary independently without changing product requests', () => { + const profiles = ['neutral', 'neutral-no-sdk', 'neutral-dev', 'neutral-dev-no-sdk'] + .map(id => resolveGuidanceProfile(id, STACKS)); + const standard = profiles[0]!; + for (const profile of profiles) { + for (const key of ['mode', 'material', 'documents', 'credentialAliases'] as const) + assert.deepEqual(profile[key], standard[key]); + for (const stack of ['mongodb', 'postgres', 'convex', 'supabase']) assert.deepEqual(profile.skills[stack], standard.skills[stack]); + } + assert.deepEqual(profiles.map(p => p.skills.spacetime!.ids), [ + ['typescript-server', 'typescript-client', 'cli'], [], + ['typescript-server', 'typescript-client', 'cli', 'spacetime-dev'], ['spacetime-dev'], + ]); + const track = loadTrack('ecommerce'); + const catalog = resolveFeatureCatalog('progression/ecommerce.json', track); + for (const level of [1, 2, 3, 4, 5, 6] as const) { + const binding = resolveRecipeRelease(track, level, 'ecommerce.progression-catalog'); + const task = resolveProgressionRecipeLevelSelection(binding, catalog, level, { cumulative: true }).agent.request; + for (const stack of STACKS) for (const repair of [false, true]) { + const prompts = profiles.map(guidance => { + const skills = readAgentSkillDocuments(STACK_BENCH_ROOT, guidance.skills[stack]!.ids); + const prompt = renderPrompt({ level, stack, task, guidance, repair }); + assert(prompt.includes(skills)); + return skills ? prompt.replace('\n\n## Selected API reference\n\n' + skills, '') : prompt; + }); + for (const prompt of prompts) { + assert.equal(prompt, prompts[0], `${stack} L${level}${repair ? ' repair' : ''} differs outside the supplied skills`); + } + } + } +}); + +function renderPrompt({ level, stack, task, guidance, repair = false, cli = false }: { + level: Level; + stack: Stack; + task: unknown; + guidance: ResolvedGuidanceProfile; + repair?: boolean; + cli?: boolean; +}): string { + const document = guidance.documents[stack]; + const skills = guidance.skills[stack]; + assert.notEqual(document, undefined); + assert.ok(skills); + const argv = [AGENT, + '--mode', repair ? 'fix' : level === 1 ? 'build' : 'upgrade', + '--backend', stack, + '--track', 'ecommerce', + '--level', String(level), + '--run-index', '0', + '--app', '/prompt-review/app', + '--guidance', 'neutral', + '--guidance-document-json', JSON.stringify(document), + '--credential-aliases-json', JSON.stringify(guidance.credentialAliases), + '--skill-identity-json', JSON.stringify(skills), + '--recipe-task-json', JSON.stringify(task), + '--print-prompt', + ]; + if (!cli) { + const args = parseAgentArgs([process.execPath, ...argv]); + const track = loadTrack('ecommerce'); + const binding = resolveRecipeRelease(track, level, 'ecommerce.progression-catalog'); + const selected = resolveBoundRecipeTaskRequest(binding, args.recipeTask!); + return buildPrompt(args, portsFor(track, stack, 0), track, { + skillsText: readAgentSkillDocuments(STACK_BENCH_ROOT, skills.ids), + requirementText: selected.task.requirementText, + contractText: selected.task.contractText, + startingCatalog: JSON.stringify({ + warehouses: binding.plan.fixture.warehouses, items: binding.plan.fixture.items, + }, null, 2), + }); + } + return execFileSync(process.execPath, argv, { + encoding: 'utf8', + stdio: 'pipe', + maxBuffer: 64 * 1024 * 1024, + env: { ...process.env, STACK_BENCH_APPLIANCE: '1', + STACK_BENCH_HOST_ALIAS: hostServiceAddress(), + STACK_BENCH_IMAGE: 'prompt-review-does-not-use-docker' }, + }); +} + +test('neutral dependency prompts include only selected product and stack contracts', () => { + const track = loadTrack('ecommerce'); + const catalog = resolveFeatureCatalog('progression/ecommerce.json', track); + const guidance = resolveGuidanceProfile('neutral', STACKS); + const spacetimeReference = readAgentSkillDocuments( + STACK_BENCH_ROOT, guidance.skills.spacetime?.ids ?? []); + assert.deepEqual(guidance.skills.spacetime?.ids, ['typescript-server', 'typescript-client', 'cli']); + assert.match(spacetimeReference, /spacetime publish/); + assert.match(spacetimeReference, /withToken/); + assert.match(spacetimeReference, /ctx\.sender/); + const disclosedGuarantees = /change together|applied only once|does not create a second payment|without a reload|cannot attach or inspect another|another customer's purchase history|cancelled restock never changes stock|restores the stock to its original warehouse|same authorization and price rules|same administrator, stock, and warehouse rules|invalid quantity|`-3`/i; + const browserClients = new Set<string>(); + for (const level of [1, 2, 3, 4, 5, 6] as const) { + const binding = resolveRecipeRelease(track, level, 'ecommerce.progression-catalog'); + const selected = resolveProgressionRecipeLevelSelection(binding, catalog, level, + { cumulative: true }); + assert.deepEqual(selected.agent.request.selection.requested.specifications, { + requested: [], + expected: [], + observed: [], + }); + assert(selected.grader.request.selection.requested.specifications.expected.length > 0); + if (level === 2) { + assert(selected.grader.selection.scoredChecks.some(check => + check.stableKey.endsWith('.620a') && check.treatment === 'expected')); + } + const moduleTypes = new Map(binding.release.components.packs + .map(pack => [pack.id, pack.moduleType])); + for (const check of selected.grader.selection.scoredChecks) { + assert.equal(check.treatment, + moduleTypes.get(check.packId ?? '') === 'specification' ? 'expected' : 'requested'); + } + for (const stack of STACKS) { + const prompt = renderPrompt({ level, stack, task: selected.agent.request, guidance }); + const repair = renderPrompt({ level, stack, task: selected.agent.request, guidance, repair: true }); + for (const request of [prompt, repair]) { + const browserClient = request.split('\n').find(line => line.startsWith('Chromium is installed')); + assert.ok(browserClient); + browserClients.add(browserClient); + if (level === 3) { + assert.match(request, /`delaySeconds`/); + if (stack === 'spacetime') { + assert.match(request, /`item: string`, `warehouse: string`/); + assert.match(request, /`quantity: u32`, `delaySeconds: u32`/); + } + } + assert.doesNotMatch(request, /an interrupted checkout leaves|earlier orders remain recorded correctly/); + assert.doesNotMatch(request, /__stackBenchScriptCanary|Stored review marker|stored-review-script/); + assert.match(request, /Startup must work with an empty database by creating the supplied starting data and accounts/); + assert.match(request, /On an existing database, preserve current quantities, prices, and user data/); + assert.match(request, /This applies after upgrades and repairs too/); + // Direct level requests disclose it with purchasing/checkout. Campaigns retain prior contracts separately. + assert.equal(request.split('# Order data interface').length - 1, level === 2 || level === 3 ? 1 : 0, + `${stack} L${level} order data contract`); + if (level === 2 || level === 3) { + assert.match(request, /database-native views over the\s+application's current records are allowed/); + assert.match(request, /When carts are available/); + assert.match(request, /order_reservation\(account_id, item_id, warehouse_id, quantity\)/); + assert.match(request, /This does not require adding stock reservations to the app/); + assert.match(request, /When warehouse stock is available/); + } + } + if (level === 2) { + assert.match(prompt, /Use `profile-address-summary` to display\s+the saved address in the profile view/); + assert.doesNotMatch(prompt, /in the same session or a new one/); + } + // Retain process-boundary coverage for all stacks and all three modes. + if ((level === 1 && (stack === 'mongodb' || stack === 'convex')) || (level === 2 && (stack === 'postgres' || stack === 'supabase')) + || (level === 3 && stack === 'spacetime')) { + assert.equal(renderPrompt({ level, stack, task: selected.agent.request, guidance, + repair: level === 3, cli: true }), level === 3 ? repair : prompt); + } + assert.doesNotMatch(repair, /Gaming Mouse row uses|`1\.00` as the price/); + const repairInterface = repair.slice(repair.lastIndexOf('## Application interface')); + assert.doesNotMatch(repairInterface, EVALUATION_LANGUAGE); + assert.doesNotMatch(repairInterface, stack === 'spacetime' + ? /\b(?:GET|POST|PATCH|DELETE|PUT) \// : /\breducer(?:s)?\b/i); + if (stack === 'convex' || stack === 'supabase') { + assert.doesNotMatch(repairInterface, /\b(?:GET|POST|PATCH|DELETE|PUT) \//); + assert.doesNotMatch(prompt, /<CONVEX|<EXPRESS_PORT>|<VITE_PORT>/); + } + assert.doesNotMatch(prompt, + new RegExp(`${EVALUATION_LANGUAGE.source}|Branding & Styling|App title:|<!-- /?interface`, 'i')); + assert.doesNotMatch(prompt, /\blevel\s+\d+\b/i); + assert.doesNotMatch(prompt, + /After the client|client must listen|client architecture|application behavior/i); + const marker = level === 1 ? '## New application' : '## Existing application'; + const markerIndex = prompt.indexOf(marker); + assert.notEqual(markerIndex, -1); + const applicationRequest = prompt.slice(markerIndex); + assert.doesNotMatch(applicationRequest, /Gaming Mouse row uses|`1\.00` as the price/); + assert.doesNotMatch(applicationRequest, disclosedGuarantees, `${stack} depth ${level}`); + assert.match(applicationRequest, /## Application interface/); + // Later product features explicitly request live or account-specific behavior. + if (level <= 3) for (const language of UNSTATED_QUALITY_LANGUAGE) { + assert.doesNotMatch(applicationRequest, language); + } + assert.doesNotMatch(applicationRequest, /## External data synchronization/); + const startingCatalog = JSON.stringify({ + warehouses: binding.plan.fixture.warehouses, + items: binding.plan.fixture.items, + }, null, 2); + for (const request of [applicationRequest, repair]) { + assert.ok(request.includes(startingCatalog)); + assert.equal(request.split('## Starting catalog').length, 2); + } + assert.match(repair, /Do not reset current quantities, prices, or user data/); + if (level > 1) assert.match(applicationRequest, /original catalog baseline/); + if (stack === 'spacetime') { + assert.match(prompt, /file:\/deps\/spacetimedb\.tgz/); + assert.doesNotMatch(prompt, /file:\/deps\/bindings-typescript/); + assert.doesNotMatch(applicationRequest, /\b(?:GET|POST|PATCH|DELETE) \//); + if (level === 1) { + assert.match(applicationRequest, /application's real authentication path/); + assert.doesNotMatch(applicationRequest, /`signUp` and `signIn` reducers/); + } + } else { + // Account-only L1 permits hosted login; later product operations have their stack's native interface. + if (level >= 2 && level <= 3) assert.match(applicationRequest, stack === 'convex' + ? /api:[a-z_]+/ : stack === 'supabase' ? /`[a-z_]+` PostgreSQL function/ + : /\b(?:GET|POST|PATCH|DELETE) \//); + assert.doesNotMatch(applicationRequest, /\breducer(?:s)?\b/i); + if (level === 1) { + assert.match(applicationRequest, /application's real authentication path/); + assert.doesNotMatch(applicationRequest, /POST \/api\/auth\/(?:signup|signin)/); + } + } + } + } + assert.equal(browserClients.size, 1); + assert.doesNotMatch([...browserClients][0]!, EVALUATION_LANGUAGE); +}); + +test('direct neutral guidance uses the current stack access documents', () => { + const cases = [...STACKS.map(stack => ['neutral', stack] as const), ['prescribed', 'spacetime'] as const]; + for (const [guidance, stack] of cases) { + const prompt = execFileSync(process.execPath, [AGENT, + '--mode', 'build', + '--backend', stack, + '--track', 'ecommerce', + '--level', '1', + '--run-index', '0', + '--app', '/prompt-review/app', + '--guidance', guidance, + '--print-prompt', + ], { + encoding: 'utf8', + stdio: 'pipe', + maxBuffer: 64 * 1024 * 1024, + env: { ...process.env, STACK_BENCH_APPLIANCE: '1', + STACK_BENCH_IMAGE: 'prompt-review-does-not-use-docker' }, + }); + assert.doesNotMatch(prompt, /Branding & Styling|App title:/i); + assert.doesNotMatch(prompt, EVALUATION_LANGUAGE); + assert.match(prompt, /store-admin-2026/); + if (stack !== 'spacetime') { + assert.match(prompt, /service is already running/); + assert.match(prompt, /Do not\s+start another .* server/); + assert.match(prompt, /Serve the complete application on `\d+`/); + assert.doesNotMatch(prompt, /Application service port/); + } else { + assert.match(prompt, /withToken/); + if (guidance === 'prescribed') assert.match(prompt, /localStorage/); + else { + assert.match(prompt, /spacetime publish/); + assert.match(prompt, /ctx\.sender/); + } + } + } +}); + +test('campaign skill material cannot change after compilation', () => { + const identity = resolveGuidanceProfile('neutral', ['spacetime']).skills.spacetime; + assert(identity); + assert.throws(() => execFileSync(process.execPath, [AGENT, + '--mode', 'build', + '--backend', 'spacetime', + '--track', 'ecommerce', + '--level', '1', + '--run-index', '0', + '--app', '/prompt-review/app', + '--guidance', 'neutral', + '--skill-identity-json', JSON.stringify({ ...identity, bytes: identity.bytes + 1 }), + '--print-prompt', + ], { + encoding: 'utf8', + stdio: 'pipe', + env: { ...process.env, STACK_BENCH_APPLIANCE: '1', + STACK_BENCH_IMAGE: 'prompt-review-does-not-use-docker' }, + }), /campaign skill material changed after compilation/); +}); diff --git a/tools/stack-bench/tests/dependency-volume.test.ts b/tools/stack-bench/tests/dependency-volume.test.ts new file mode 100644 index 00000000000..f966e11b13f --- /dev/null +++ b/tools/stack-bench/tests/dependency-volume.test.ts @@ -0,0 +1,52 @@ +import assert from 'node:assert/strict'; +import { chmodSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test, { type TestContext } from 'node:test'; + +import { createDependencyManifest, initializeDependencyVolume, manifestSha256, + verifyDependencyTree } from '../appliance/dependency-volume.js'; + +function fixture(t: TestContext): { source: string; target: string } { + const root = join(tmpdir(), `stack-bench-deps-${process.pid}-${Math.random().toString(16).slice(2)}`); + const source = join(root, 'source'); + const target = join(root, 'target'); + mkdirSync(join(source, 'bindings-typescript'), { recursive: true }); + writeFileSync(join(source, 'bindings-typescript', 'package.json'), '{"name":"sdk"}\n'); + writeFileSync(join(source, 'spacetimedb-cli'), 'cli'); + chmodSync(join(source, 'spacetimedb-cli'), 0o755); + t.after(() => rmSync(root, { recursive: true, force: true })); + return { source, target }; +} + +test('dependency volume initialization is exact and idempotent for one release', t => { + const { source, target } = fixture(t); + const manifest = createDependencyManifest(source); + const first = initializeDependencyVolume({ source, target, manifest }); + assert.equal(first.initialized, true); + assert.equal(first.manifestSha256, manifestSha256(manifest)); + assert.deepEqual(verifyDependencyTree(target, manifest, { allowMarker: true }), { + manifestSha256: manifestSha256(manifest), files: 2, + }); + assert.equal(initializeDependencyVolume({ source, target, manifest }).initialized, false); +}); + +test('dependency volume refuses unmarked, stale, changed and wrong-release content', t => { + const { source, target } = fixture(t); + const manifest = createDependencyManifest(source); + mkdirSync(target); + writeFileSync(join(target, 'foreign'), 'x'); + assert.throws(() => initializeDependencyVolume({ source, target, manifest }), /no release marker/); + + const clean = join(target, '..', 'clean'); + initializeDependencyVolume({ source, target: clean, manifest }); + writeFileSync(join(clean, 'spacetimedb-cli'), 'changed'); + assert.throws(() => verifyDependencyTree(clean, manifest, { allowMarker: true }), /does not match/); + assert.throws(() => initializeDependencyVolume({ source, target: clean, manifest }), /does not match/); + + const marker = JSON.parse(readFileSync(join(clean, '.stack-bench-release-deps.json'), 'utf8')); + marker.manifestSha256 = '0'.repeat(64); + chmodSync(join(clean, '.stack-bench-release-deps.json'), 0o644); + writeFileSync(join(clean, '.stack-bench-release-deps.json'), JSON.stringify(marker)); + assert.throws(() => initializeDependencyVolume({ source, target: clean, manifest }), /different release/); +}); diff --git a/tools/stack-bench/tests/diagnostic-sanitizer.test.ts b/tools/stack-bench/tests/diagnostic-sanitizer.test.ts new file mode 100644 index 00000000000..34322608452 --- /dev/null +++ b/tools/stack-bench/tests/diagnostic-sanitizer.test.ts @@ -0,0 +1,64 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + redactCredentials, + sanitiseConsoleError, + sanitiseDiagnostic, +} from '../src/evidence/diagnostic-sanitizer.js'; + +const forbidden = /data-(?:role|testid)|getBy(?:TestId|Role|Text)|waitForSelector|locator\(|selectOption|control,#|data-state|localhost|127\.0\.0\.1|host\.docker\.internal|https?:\/\/|[A-Za-z]:[\\/]|\/tools\/stack-bench|\b\d+(?:ms|s)\b|runExpect/i; + +test('sanitiser removes selector, timing, endpoint and harness-path mechanics', () => { + const input = `Timeout 5000ms exceeded at C:\\repo\\tools\\stack-bench\\dist\\grader\\grade.js\n` + + `waiting for locator('[data-testid="admin-panel"]') at http://127.0.0.1:6573/private\n` + + `Call log:\n - waiting for getByTestId('admin-panel')\n - element is not enabled after 250ms\n` + + ` - /tools/stack-bench/dist/grader/grade.js:99\n` + + ` at runExpect (/tools/stack-bench/dist/grader/grade.js:99:2)\n` + + ` waiting for getByRole('button', { name: 'secret grader label' }) for 2s`; + const result = sanitiseDiagnostic(input); + assert.doesNotMatch(result, forbidden); + assert.match(result, /element is not enabled/i); +}); + +test('sanitiser handles single-quoted and unquoted test selectors', () => { + const result = sanitiseDiagnostic("[data-role='one'],#one [data-test=two] [data-cy=three]"); + assert.equal(result, 'the control the control the control'); +}); + +test('credentials are redacted from diagnostics and console output', () => { + const result = sanitiseConsoleError('Authorization: Bearer secret.payload api_key=sk-abcdefghijklmnop `oops`'); + assert.doesNotMatch(result, /secret\.payload|sk-abcdefghijklmnop|`/); + assert.match(result, /redacted/i); +}); + +test('credential redaction covers provider environment and JSON spellings', () => { + const input = 'ANTHROPIC_API_KEY=provider-value ' + + 'CLAUDE_CODE_OAUTH_TOKEN=subscription-value ' + + '{"apiKey":"json-value","oauth_token":"oauth-value"} ' + + 'mongodb://user:database-password@database:27017/app'; + const result = redactCredentials(input); + assert.doesNotMatch(result, /provider-value|subscription-value|json-value|oauth-value|database-password/); + assert.equal(result.match(/\[redacted credential\]/g)?.length, 5); +}); + +test('Docker errors redact password arguments while retaining command and failure details', () => { + const input = 'Command failed: docker exec backend mongosh --password "space secret" --eval ping\n' + + "mongosh --password='quoted secret' --username appuser Authentication failed\n" + + 'mongosh --password=plain-secret --quiet\n' + + 'docker create --publish 127.0.0.1:5432:5432 -e PGPASSWORD=env-secret'; + const output = redactCredentials(input); + assert.doesNotMatch(output, /space secret|quoted secret|plain-secret|env-secret/); + assert.match(output, /Authentication failed/); + assert.match(output, /--publish 127\.0\.0\.1:5432:5432/); + assert.match(output, /--username appuser/); +}); + +test('SQL password redaction covers doubled quotes and escaped string literals', () => { + const input = "CREATE USER appuser PASSWORD 'plain-secret'; ALTER ROLE appuser PASSWORD 'dou''bled-secret'; " + + String.raw`ALTER ROLE appuser PASSWORD E'escaped\'secret'; syntax error near PASSWORD`; + for (const output of [redactCredentials(input), sanitiseConsoleError(input)]) { + assert.doesNotMatch(output, /plain-secret|dou|bled-secret|escaped|secret'/); + assert.match(output, /CREATE USER appuser PASSWORD \[redacted credential\]/); + assert.match(output, /syntax error near PASSWORD/); + } +}); diff --git a/tools/stack-bench/tests/disclosure.contract.ts b/tools/stack-bench/tests/disclosure.contract.ts new file mode 100644 index 00000000000..1b8ae5ea7b6 --- /dev/null +++ b/tools/stack-bench/tests/disclosure.contract.ts @@ -0,0 +1,187 @@ +import assert from 'node:assert/strict'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { compileCampaignFile } from '../src/campaigns/campaign-compiler.js'; +import { resolveGuidanceProfile } from '../src/campaigns/condition-compiler.js'; +import { agentVisibleContractText, contractInterfaceNames } + from '../src/composition/agent-visible-contract.js'; +import { loadTrack } from '../src/composition/tracks.js'; +import type { NamedAction, Track } from '../src/composition/tracks.js'; +import { requireRecipeRelease } from '../src/composition/recipe-release.js'; +import { resolveBoundRecipeTaskRequest } from '../src/composition/recipe-selection.js'; +import type { CompiledStep } from '../src/composition/definition-compiler.js'; +import { compileDependencyPolicyInput, dependencyRuntimeDefinition } + from '../src/progression/progression-definition.js'; +import { progressionEngine } from '../src/progression/progression-engine.js'; +import { priorProgressionContractIds, resolveProgressionRecipeAction } + from '../src/progression/progression-recipe-selection.js'; + +const reference = join(STACK_BENCH_ROOT, 'appliance', 'campaign.ecommerce-progression-reference.json'); +const placeholder = (route: string): string => route.replace(/\{[a-zA-Z]+\}|:[a-zA-Z]+/g, '{}'); + +function* eachStep(steps: readonly CompiledStep[]): Generator<CompiledStep> { + for (const step of steps) { + yield step; + for (const branch of step.branches ?? []) yield* eachStep(branch); + } +} + +// Presence is a disclosure guardrail, not semantic proof. Values used for assertions, +// adversarial inputs, wire types, status semantics and parameter order need review. +// Track actions describe executor requirements; their existence is NOT disclosure. +function missingNames(steps: readonly CompiledStep[], delivered: string, + applicationInterface: string, track: Track): string[] { + const missing = new Set<string>(); + const roleIds = new Set<string>(); + if (delivered.includes('`staff-role-account-`') && delivered.includes('`encodeURIComponent(username)`')) { + // Seeded accounts can be write targets without signing in during this check. + for (const match of delivered.matchAll(/account named `([^`]+)`/g)) { + roleIds.add(`staff-role-account-${encodeURIComponent(match[1]!)}`); + } + for (const step of eachStep(steps)) { + if (['signIn', 'ensureSignedIn'].includes(step.do) && step.exact === true && typeof step.name === 'string') { + roleIds.add(`staff-role-account-${encodeURIComponent(step.name)}`); + } + } + } + const needs = (kind: string, name: string | undefined): void => { + if (name && !delivered.includes(name) && !(kind === 'control' && roleIds.has(name))) { + missing.add(`${kind} ${name}`); + } + }; + for (const step of eachStep(steps)) { + for (const control of [step.testid, + ...(Array.isArray(step.unlessVisible) ? step.unlessVisible as string[] : [step.unlessVisible as string | undefined]), + step.in?.testid, + (step.namedTarget as { testid?: string } | undefined)?.testid, + (step.input as { testid?: string } | undefined)?.testid]) needs('control', control); + for (const attribute of [(step.input as { attribute?: string } | undefined)?.attribute, + (step.namedTarget as { attribute?: string } | undefined)?.attribute]) needs('attribute', attribute); + const named = step.namedAction as NamedAction | undefined; + const action = named ?? ((step.do === 'callAction' || step.do === 'callConcurrently') + ? track.actions.find(action => action.id === step.action) : undefined); + if (action) { + if (applicationInterface === 'reducer') needs('reducer', action.reducer); + else if (applicationInterface === 'convex') needs('mutation', action.reducer ? `api:${action.reducer}` : undefined); + else if (applicationInterface === 'supabase') needs('function', action.reducer ? `\`${action.reducer}\`` : undefined); + else if (action.path && !placeholder(delivered).includes(placeholder(action.path))) { + missing.add(`route ${action.path}`); + } + for (const parameter of action.params ?? []) { + // HTTP path parameters are disclosed by their route placeholder. Their + // internal mapping name need not be part of the public application API. + if (applicationInterface !== 'http' || parameter.in !== 'path') needs('parameter', parameter.name); + } + } else if (step.do === 'callAction' || step.do === 'callConcurrently') { + missing.add(`action ${step.action}`); + } + if (step.do === 'dbSetStock' && !['item', 'warehouse', 'stock'] + .every(table => delivered.includes(`\`${table}(`))) missing.add('stock data interface'); + } + return [...missing].sort(); +} + +test('scored interface names are disclosed by their issued dependency step on each stack', () => { + const track = loadTrack('ecommerce'); + const campaign = compileCampaignFile(reference); + const catalog = campaign.featureCatalog; + assert(catalog); + const bindings = new Map(campaign.definition.levels.map(level => + [level, requireRecipeRelease(track, level, 'ecommerce.progression-catalog')])); + const binding = bindings.values().next().value!; + const steps = new Map<string, CompiledStep[]>(); + for (const execution of binding.plan.execution) for (const group of execution.checkGroups) { + for (const criterion of group.feature.criteria) { + steps.set(`${group.packId}|${group.checkGroupId}|${criterion.id}`, + [...group.feature.setup, ...criterion.steps]); + } + } + const byKey = new Map(binding.plan.checks.map(check => + [check.stableKey, steps.get(`${check.packId}|${check.checkGroupId}|${check.criterionId}`)])); + const stacks = [...new Set([...campaign.definition.stacks.map(stack => stack.id), 'convex', 'supabase'])]; + const guidance = resolveGuidanceProfile('neutral', stacks); + const missing: string[] = []; + const checked = new Set<string>(); + // Use the production scheduler and compiler, including simultaneous work. No + // hand-written dependency traversal or facts from future requests are allowed. + for (const workSelection of ['feature', 'progressive', 'all-at-once'] as const) { + const policy = compileDependencyPolicyInput(campaign.definition.repair, catalog, + { workSelection }); + let state = progressionEngine.initialize(dependencyRuntimeDefinition(catalog, policy)); + for (;;) { + const current = bindings.get(state.level)!; + const prior = priorProgressionContractIds(state, bindings, current); + const selected = resolveProgressionRecipeAction(current, state, prior); + if (!('agent' in selected)) break; + const task = resolveBoundRecipeTaskRequest(current, selected.agent.request).task; + for (const stack of stacks) { + const applicationInterface = guidance.documents[stack]!.applicationInterface; + const delivered = agentVisibleContractText( + `${task.requirementText}\n${task.contractText}`, guidance.credentialAliases, applicationInterface); + for (const key of selected.grader.checkKeys) { + const list = byKey.get(key); + assert(list, `${key} has scenario steps`); + checked.add(key); + const needs = missingNames(list, delivered, applicationInterface, track); + if (needs.length) missing.push(`${workSelection} ` + + `${stack} step=${state.attempts.length + 1} ${key}: ${needs.join(', ')}`); + } + } + const grading = progressionEngine.gradingSelection(state); + state = progressionEngine.recordResult(state, { + attemptId: `disclosure-${state.attempts.length + 1}`, outcome: 'conclusive', + nodes: grading.nodeIds.map(id => ({ id, checks: grading.checks + .filter(check => check.nodeId === id).map(check => ({ id: check.id, outcome: 'pass' })) })), + }); + } + } + assert.deepEqual([...checked].sort(), [...new Set(catalog.definition.nodes + .flatMap(node => node.gradingChecks.map(check => check.id)))].sort()); + assert.deepEqual(missing, []); +}); + +test('later disclosure cannot satisfy an earlier check; only issued contracts can be retained', () => { + const track = loadTrack('ecommerce'); + const campaign = compileCampaignFile(reference); + assert(campaign.featureCatalog && campaign.dependencyPolicy); + const guidance = resolveGuidanceProfile('neutral', campaign.definition.stacks.map(stack => stack.id)); + let state = progressionEngine.initialize(dependencyRuntimeDefinition( + campaign.featureCatalog, campaign.dependencyPolicy)); + const bindings = new Map(campaign.definition.levels.map(level => + [level, requireRecipeRelease(track, level, 'ecommerce.progression-catalog')])); + const first = resolveProgressionRecipeAction(bindings.get(state.level)!, state); + assert('agent' in first); + const grading = progressionEngine.gradingSelection(state); + state = progressionEngine.recordResult(state, { + attemptId: 'first-request', outcome: 'conclusive', nodes: grading.nodeIds.map(id => ({ + id, checks: grading.checks.filter(check => check.nodeId === id) + .map(check => ({ id: check.id, outcome: 'pass' })), + })), + }); + const binding = bindings.get(state.level)!; + const second = resolveProgressionRecipeAction(binding, state); + const retained = resolveProgressionRecipeAction(binding, state, + priorProgressionContractIds(state, bindings, binding)); + assert('agent' in second && 'agent' in retained); + for (const applicationInterface of ['http', 'reducer', 'supabase'] as const) { + const text = (source: string): string => agentVisibleContractText(source, + guidance.credentialAliases, applicationInterface); + const firstText = text(first.agent.task.contractText); + const secondText = text(second.agent.task.contractText); + const retainedText = text(retained.agent.task.contractText); + const laterControl = contractInterfaceNames(secondText).find(name => !firstText.includes(name)); + const earlierControl = contractInterfaceNames(firstText).find(name => !secondText.includes(name)); + assert(laterControl && earlierControl, 'successive requests have distinct interface controls'); + const earlyCheck: CompiledStep[] = [{ do: 'click', testid: laterControl }]; + const earlierCheck: CompiledStep[] = [{ do: 'click', testid: earlierControl }]; + assert.deepEqual(missingNames(earlyCheck, firstText, applicationInterface, track), [`control ${laterControl}`]); + assert.deepEqual(missingNames(earlyCheck, secondText, applicationInterface, track), []); + assert.deepEqual(missingNames(earlierCheck, secondText, applicationInterface, track), [`control ${earlierControl}`]); + assert.deepEqual(missingNames(earlierCheck, retainedText, applicationInterface, track), []); + // Opt-out removes the earlier contract from the current request, but does + // not erase its historical disclosure. Later material cannot travel back. + assert.deepEqual(missingNames(earlierCheck, `${firstText}\n${secondText}`, applicationInterface, track), []); + } +}); diff --git a/tools/stack-bench/tests/docker-isolation.integration.ts b/tools/stack-bench/tests/docker-isolation.integration.ts new file mode 100644 index 00000000000..c898ea973ff --- /dev/null +++ b/tools/stack-bench/tests/docker-isolation.integration.ts @@ -0,0 +1,199 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { randomBytes } from 'node:crypto'; +import test from 'node:test'; +import { fileURLToPath } from 'node:url'; + +import { attemptNetworkRules } from '../src/runtime/docker-network.js'; + +// Explicit opt-in: this creates short-lived model-free Docker resources. +test('attempt namespace permits owned services and app replies while denying host and other attempts', { + skip: process.env.STACK_BENCH_DOCKER_ISOLATION_TEST !== '1', timeout: 90_000, +}, async t => { + const helperImage = process.env.STACK_BENCH_NETWORK_HELPER_IMAGE; + const appImage = process.env.STACK_BENCH_NETWORK_PROBE_IMAGE; + const controllerImage = process.env.STACK_BENCH_NETWORK_CONTROLLER_IMAGE; + assert.ok(helperImage && appImage && controllerImage, + 'supply existing nftables helper, Node probe, and controller image IDs'); + const prefix = `stack-bench-isolation-probe-${randomBytes(6).toString('hex')}`; + const containers: string[] = []; + const networks: string[] = []; + const volumes: string[] = []; + function docker(args: string[], input?: string): string { + const result = spawnSync('docker', args, { encoding: 'utf8', input, timeout: 15_000 }); + assert.equal(result.status, 0, `${args.slice(0, 3).join(' ')}: ${result.stdout}\n${result.stderr || result.error}`); + return result.stdout.trim(); + } + const run = (name: string, args: string[], script: string): string => { + const id = docker(['create', '--name', `${prefix}-${name}`, '--cap-drop', 'ALL', + '--security-opt', 'no-new-privileges:true', ...args, '--entrypoint', 'node', appImage!, '-e', script]); + containers.push(id); + docker(['start', id]); + return id; + }; + const serve = (ports: number[]) => `const h=require('node:http'); for(const p of ${JSON.stringify(ports)})` + + `h.createServer((q,s)=>s.end('ok')).listen(p,'::',function(){console.log(this.address().port)});`; + const endpoint = (id: string, network: string): { IPAddress: string; Gateway: string; GlobalIPv6Address: string } => + JSON.parse(docker(['inspect', '--format', '{{json .NetworkSettings.Networks}}', id]))[network]; + const probe = (id: string, urls: string[]): boolean[] => JSON.parse(docker(['exec', id, 'node', '-e', + `Promise.all(${JSON.stringify(urls)}.map(async u=>{try{await fetch(u,{signal:AbortSignal.timeout(1500)});` + + `return true}catch{return false}})).then(r=>console.log(JSON.stringify(r)))`])); + try { + const trackedBeforeFailure = containers.length; + assert.throws(() => run('failed-start', ['--network', `container:${prefix}-absent`], serve([18083])), + /No such container/); + assert.equal(containers.length, trackedBeforeFailure + 1, 'failed starts must retain the created ID for cleanup'); + const first = `${prefix}-a`; + const second = `${prefix}-b`; + for (const name of [first, second]) networks.push(docker(['network', 'create', '--ipv6', name])); + const app = run('app', ['--network', first, '-p', '127.0.0.1::18083'], serve([18083])); + const own = run('own', ['--network', first], serve([18080, 18081])); + const other = run('other', ['--network', second], serve([18080])); + const host = run('host', ['--network', 'host'], serve([0])); + const broker = run('broker', ['--network', `container:${app}`], + "require('node:http').createServer((q,s)=>s.end('ok')).listen(18084,'127.0.0.1')"); + assert.ok(broker); + const ownAddress = endpoint(own, first).IPAddress; + const otherAddress = endpoint(other, second).IPAddress; + const hostAddress = endpoint(app, first).Gateway; + const hostPort = Number(docker(['logs', host])); + assert.ok(hostPort > 0, 'host listener became ready'); + const ownUrl = `http://${ownAddress}:18080`; + const ipv6Url = `http://[${endpoint(own, first).GlobalIPv6Address}]:18080`; + const deniedPortUrl = `http://${ownAddress}:18081`; + const hostUrl = `http://${hostAddress}:${hostPort}`; + const otherUrl = `http://${otherAddress}:18080`; + assert.deepEqual(probe(app, [ownUrl, deniedPortUrl, hostUrl, ipv6Url, 'http://127.0.0.1:18084']), + [true, true, true, true, true], 'positive controls before firewall'); + assert.deepEqual(probe(other, ['http://127.0.0.1:18080']), [true], 'other attempt is live'); + const rules = attemptNetworkRules({ services: [{ address: ownAddress, port: 18080 }], + hostAddresses: [hostAddress] }); + docker(['run', '--rm', '-i', '--network', `container:${app}`, '--cap-drop', 'ALL', + '--cap-add', 'NET_ADMIN', '--security-opt', 'no-new-privileges:true', '--read-only', + '--entrypoint', 'nft', helperImage, '-f', '-'], rules); + assert.deepEqual(probe(app, [ownUrl, deniedPortUrl, hostUrl, otherUrl, ipv6Url, 'http://127.0.0.1:18084']), + [true, false, false, false, false, true], 'owned backend/broker allowed, live forbidden endpoints denied'); + assert.deepEqual(probe(app, ['https://api.anthropic.com', 'https://registry.npmjs.org']), + [true, true], 'public provider and registry HTTPS routes work without model calls'); + const binding = docker(['port', app, '18083/tcp']); + const response = await fetch(`http://${binding}`, { signal: AbortSignal.timeout(1500) }); + assert.equal(await response.text(), 'ok', 'published app replies reach host grader'); + const volume = docker(['volume', 'create', `${prefix}-private`]); + volumes.push(volume); + const privateDirectory = docker(['volume', 'inspect', '--format', '{{.Mountpoint}}', volume]); + const brokerProcess = fileURLToPath(new URL('../container/credential-broker-process.js', import.meta.url)); + const brokerProof = ` + const { startCredentialBroker, stopCredentialBroker, credentialBrokerDiagnostics } = + await import('/opt/stack-bench/dist/container/credential-broker-process.js'); + const { spawnSync } = await import('node:child_process'); + const { existsSync, writeFileSync } = await import('node:fs'); + const assert = (await import('node:assert/strict')).default; + const privateDirectory = ${JSON.stringify(privateDirectory)}; + const imageId = ${JSON.stringify(controllerImage)}; + const networkContainerId = ${JSON.stringify(app)}; + let recorded; + const credential = 'model-free-probe-fake-credential'; + const options = { networkMode: 'bridge', deadlineMs: 10_000, model: 'test-model', + docker: { imageId, networkContainerId, privateDirectory, + name: ${JSON.stringify(`${prefix}-real-broker`)}, creationToken: '${randomBytes(16).toString('hex')}', + onCreated(container) { + recorded = container; + assert.equal(spawnSync('docker', ['inspect', '--format', '{{.State.Running}}', container.id], + { encoding: 'utf8' }).stdout.trim(), 'false'); + writeFileSync(privateDirectory + '/broker-authority.json', JSON.stringify(container), { mode: 0o600 }); + } + } + }; + const broker = await startCredentialBroker({ mode: 'api-key', credential }, options); + try { + assert.equal(recorded.id, broker.container.id); + const inspection = JSON.parse(spawnSync('docker', ['inspect', broker.container.id], + { encoding: 'utf8' }).stdout)[0]; + assert.equal(JSON.stringify(inspection.Config).includes(credential), false); + assert.equal(inspection.HostConfig.NetworkMode, 'container:' + networkContainerId); + assert.equal(inspection.Mounts.length, 1); + assert.equal((await fetch(broker.baseUrl)).status, 401); + assert.equal((await fetch(broker.baseUrl, { headers: { authorization: 'Bearer ' + broker.sessionToken } })).status, 404); + const appRead = spawnSync('docker', ['exec', networkContainerId, 'test', '-e', broker.root]); + assert.notEqual(appRead.status, 0, 'coding container cannot read provider files'); + } finally { + const ledger = await stopCredentialBroker(broker); + assert.equal(ledger.complete, true); + assert.equal(ledger.spentUsd, 0); + assert.equal(credentialBrokerDiagnostics(broker).termination.exited, true); + assert.equal(existsSync(broker.ledgerPath), true); + assert.equal(existsSync(broker.root + '/config.json'), false); + assert.equal(existsSync(broker.root + '/ready.json'), false); + } + assert.notEqual(spawnSync('docker', ['inspect', recorded.id]).status, 0); + console.log('sidecar identity, loopback authentication, private mount, receipt drain, and removal passed'); + `; + const controller = docker(['create', '--name', `${prefix}-controller`, '--network', `container:${app}`, + '--mount', 'type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock', + '--mount', `type=volume,src=${volume},dst=${privateDirectory}`, + '--mount', `type=bind,src=${brokerProcess},dst=/opt/stack-bench/dist/container/credential-broker-process.js,readonly`, + '--entrypoint', 'node', controllerImage, '--input-type=module', '-e', brokerProof]); + containers.push(controller); + t.diagnostic(docker(['start', '--attach', controller])); + assert.equal(docker(['inspect', '--format', '{{.State.ExitCode}}', controller]), '0'); + const browserName = `${prefix}-browser`; + const browser = docker(['create', '--name', browserName, '--network', `container:${app}`, + '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges:true', '--read-only', + '--env', 'HOME=/tmp', '--env', 'XDG_CONFIG_HOME=/tmp/.config', '--env', 'XDG_CACHE_HOME=/tmp/.cache', + '--tmpfs', '/tmp:size=256m,mode=1777', '--shm-size', '256m', '--pids-limit', '128', + '--memory', '1g', '--entrypoint', 'sleep', controllerImage, 'infinity']); + containers.push(browser); + docker(['start', browser]); + const browserProof = ` + const assert = (await import('node:assert/strict')).default; + const { chromium } = await import('playwright'); + const { createBackendLease, writeBackendLease } = await import('./dist/src/runtime/backend-lease.js'); + const { attemptBrowserLaunchOptions } = await import('./dist/container/browser-pipe.js'); + const lease = createBackendLease({runId:'browser-probe', backend:'postgres', track:'ecommerce', + runIndex:0, database:'probe', container:{name:'probe-anchor',id:${JSON.stringify(app)}}}); + lease.resources.browserContainer = {name:${JSON.stringify(browserName)},id:${JSON.stringify(browser)}, + image:${JSON.stringify(controllerImage)},owned:true,networkMode:'container:'+${JSON.stringify(app)}}; + process.env.STACK_BENCH_LEASE='/tmp/browser-lease.json'; + process.env.STACK_BENCH_LEASE_TOKEN=lease.ownershipToken; + writeBackendLease(process.env.STACK_BENCH_LEASE,lease); + for (const shared of [false,true]) { + const server = shared ? await chromium.launchServer({...attemptBrowserLaunchOptions(),timeout:5000}) : null; + const browser = server ? await chromium.connect(server.wsEndpoint()) + : await chromium.launch({...attemptBrowserLaunchOptions(),timeout:5000}); + try { + const page = await browser.newPage(); + await page.goto('http://127.0.0.1:18083',{timeout:5000}); + assert.equal(await page.textContent('body'),'ok'); + assert.deepEqual(await page.evaluate(async urls => Promise.all(urls.map(async url => { + try { await fetch(url,{signal:AbortSignal.timeout(1000)});return true; } catch { return false; } + })), ${JSON.stringify([hostUrl, otherUrl, ipv6Url])}), [false,false,false]); + assert.ok((await page.screenshot()).length > 100); + } finally { await browser.close(); await server?.close(); } + } + console.log('browser pipe launch, shared driver, screenshots, and page-origin denial passed'); + `; + const browserController = docker(['create', '--name', `${prefix}-browser-controller`, '--network', 'none', + '--mount', 'type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock', + '--entrypoint', 'node', controllerImage, '--input-type=module', '-e', browserProof]); + containers.push(browserController); + t.diagnostic(docker(['start', '--attach', browserController])); + assert.equal(docker(['inspect', '--format', '{{.State.ExitCode}}', browserController]), '0'); + const table = docker(['run', '--rm', '--network', `container:${app}`, '--cap-drop', 'ALL', + '--cap-add', 'NET_ADMIN', '--entrypoint', 'nft', helperImage, 'list', 'table', 'inet', 'stack_bench']); + t.diagnostic(JSON.stringify({ docker: docker(['version', '--format', '{{.Server.Version}}']), + helperImage, appImage, firewall: 'nftables in attempt namespace', publicEgress: 'IPv4 TCP 80/443', table })); + } finally { + // IDs came from successful creations in this test. Never discover/delete by prefix. + const errors: unknown[] = []; + for (const id of containers.reverse()) { + try { docker(['rm', '-f', id]); } catch (error) { errors.push(error); } + } + for (const id of networks.reverse()) { + try { docker(['network', 'rm', id]); } catch (error) { errors.push(error); } + } + for (const name of volumes.reverse()) { + try { docker(['volume', 'rm', name]); } catch (error) { errors.push(error); } + } + assert.deepEqual(errors, [], 'all owned Docker resources were removed'); + } +}); diff --git a/tools/stack-bench/tests/docker-network.test.ts b/tools/stack-bench/tests/docker-network.test.ts new file mode 100644 index 00000000000..5ea29954c26 --- /dev/null +++ b/tools/stack-bench/tests/docker-network.test.ts @@ -0,0 +1,315 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { tmpdir } from 'node:os'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { createBackendLease, publicBackendLease, readBackendLease, validateBackendLease, writeBackendLease } from '../src/runtime/backend-lease.js'; +import { attemptBrowserLaunchOptions } from '../container/browser-pipe.js'; + +import { attemptNetworkRules, recordAttemptCreation, dockerHostGatewayArguments, dockerHostServiceAddress, + installAttemptFirewall, createAttemptNetwork, createAttemptContainer, createAttemptService, + startAttemptContainer } from '../src/runtime/docker-network.js'; + +test('only the authenticated owned browser uses its private shared memory', () => { + const root = mkdtempSync(join(tmpdir(), 'browser-shm-')); + const saved = { lease: process.env.STACK_BENCH_LEASE, + token: process.env.STACK_BENCH_LEASE_TOKEN, appliance: process.env.STACK_BENCH_APPLIANCE }; + try { + delete process.env.STACK_BENCH_LEASE; + delete process.env.STACK_BENCH_APPLIANCE; + assert.deepEqual(attemptBrowserLaunchOptions(), {}); + const lease = createBackendLease({ backend: 'postgres', track: 'ecommerce', runIndex: 0, + runId: 'browser-shm', database: 'browser_shm' }); + const anchor = 'b'.repeat(64); + lease.resources.container = { name: 'anchor', id: anchor, owned: true }; + lease.resources.network = { name: 'attempt', id: 'c'.repeat(64), namespaceContainerId: anchor, + hostAddresses: ['172.20.0.1'], services: [], firewallSha256: null, firewallInstalledAt: null }; + lease.resources.browserContainer = { name: 'browser', id: 'a'.repeat(64), owned: true, + image: `sha256:${'d'.repeat(64)}`, networkMode: `container:${anchor}` }; + process.env.STACK_BENCH_LEASE = join(root, 'lease.json'); + process.env.STACK_BENCH_LEASE_TOKEN = lease.ownershipToken; + writeBackendLease(process.env.STACK_BENCH_LEASE, lease); + const options = attemptBrowserLaunchOptions(); + assert.match(options.executablePath ?? '', /browser-pipe\.js$/); + assert.deepEqual(options.ignoreDefaultArgs, ['--disable-dev-shm-usage']); + process.env.STACK_BENCH_LEASE_TOKEN = 'wrong-token'; + assert.throws(() => attemptBrowserLaunchOptions(), /token/); + } finally { + for (const [key, value] of Object.entries({ STACK_BENCH_LEASE: saved.lease, + STACK_BENCH_LEASE_TOKEN: saved.token, STACK_BENCH_APPLIANCE: saved.appliance })) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + rmSync(root, { recursive: true, force: true }); + } +}); + +test('attempt cache attachment preserves the shared gateway and binds only its exact service address', () => { + const root = mkdtempSync(join(tmpdir(), 'cache-attachment-')); + const image = process.env.STACK_BENCH_CONTROLLER_IMAGE_ID; + process.env.STACK_BENCH_CONTROLLER_IMAGE_ID = `sha256:${'f'.repeat(64)}`; + try { + const lease = createBackendLease({ backend: 'postgres', track: 'chat', runIndex: 0, + runId: 'cache-attachment', database: 'cache_attachment' }); + const network = 'a'.repeat(64), cache = 'b'.repeat(64), anchor = 'c'.repeat(64); + lease.resources.container = { name: 'anchor', id: anchor, owned: true }; + lease.resources.network = { name: 'attempt', id: network, namespaceContainerId: anchor, + hostAddresses: ['172.20.0.1'], services: [], firewallSha256: null, firewallInstalledAt: null }; + const path = join(root, 'lease.json'); + writeBackendLease(path, lease); + const commands: string[][] = []; + const active = installAttemptFirewall(path, lease, (args, input) => { + commands.push(args); + if (args[0] === 'inspect') return args[2] === '{{.Id}}' ? cache : JSON.stringify({ + original: { NetworkID: 'd'.repeat(64), IPAddress: '172.31.0.4', GwPriority: 0 }, + attempt: { NetworkID: network, IPAddress: '172.20.0.3', GwPriority: -1 }, + }); + if (args[0] === 'run') assert.match(input ?? '', /ip daddr 172\.20\.0\.3 tcp dport 4873 accept/); + return ''; + }); + assert.deepEqual(commands.filter(args => args[0] === 'network'), + [['network', 'connect', '--gw-priority', '-1', network, cache]]); + assert.equal(active.resources.network?.cacheContainerId, cache); + assert.deepEqual(active.resources.network?.services, [{ address: '172.20.0.3', port: 4873 }]); + } finally { + if (image === undefined) delete process.env.STACK_BENCH_CONTROLLER_IMAGE_ID; + else process.env.STACK_BENCH_CONTROLLER_IMAGE_ID = image; + rmSync(root, { recursive: true, force: true }); + } +}); + +test('lease binds sidecars to the exact backend namespace and excludes creation authority from evidence', () => { + const lease = createBackendLease({ backend: 'spacetime', track: 'chat', runIndex: 0, runId: 'network-proof', + serverUri: 'http://127.0.0.1:3210', module: 'proof', dataDir: tmpdir() }); + const anchor = 'a'.repeat(64); + lease.resources.container = { name: 'anchor', id: anchor, owned: true }; + lease.resources.network = { name: 'attempt', id: 'b'.repeat(64), namespaceContainerId: anchor, + hostAddresses: ['172.20.0.1'], services: [], firewallSha256: null, firewallInstalledAt: null }; + lease.resources.browserContainer = { name: 'browser', id: 'c'.repeat(64), image: `sha256:${'d'.repeat(64)}`, + owned: true, networkMode: `container:${anchor}` }; + lease.resources.creationIntents = { browser: { name: 'browser', creationToken: 'e'.repeat(32) } }; + assert.equal(validateBackendLease(lease), lease); + assert.equal('creationIntents' in publicBackendLease(lease).resources, false); + assert.equal(lease.resources.creationIntents.browser?.creationToken, 'e'.repeat(32)); + lease.resources.browserContainer.networkMode = `container:${'f'.repeat(64)}`; + assert.throws(() => validateBackendLease(lease), /outside the leased network namespace/); +}); + +test('attempt firewall limits private services to their port and rejects unsafe rule inputs', () => { + attemptNetworkRules({ services: [{ address: '172.20.0.3', port: 5432 }], + hostAddresses: ['172.20.0.1', '203.1.2.3'] }); + assert.throws(() => attemptNetworkRules({ services: [], hostAddresses: [] }), /host addresses/); + assert.throws(() => attemptNetworkRules({ services: [{ address: '172.20.0.1', port: 5432 }], + hostAddresses: ['172.20.0.1'] }), /cannot be a host/); + for (const address of ['backend', '127.0.0.1; accept', '::1']) { + assert.throws(() => attemptNetworkRules({ services: [], hostAddresses: [address] }), /literal IPv4/); + } + assert.throws(() => attemptNetworkRules({ services: [{ address: '172.20.0.3', port: 0 }], + hostAddresses: ['172.20.0.1'] }), /port is invalid/); +}); + +test('host services use the address reachable from the selected network namespace', () => { + for (const [mode, address, gateway] of [ + ['bridge', 'host.docker.internal', ['--add-host', 'host.docker.internal:host-gateway']], + ['host', '127.0.0.1', []], + [`container:${'a'.repeat(64)}`, '127.0.0.1', []], + ] as const) { + assert.equal(dockerHostServiceAddress(mode), address); + assert.deepEqual(dockerHostGatewayArguments(mode), gateway); + } + assert.throws(() => dockerHostServiceAddress('ambient'), /unsupported Docker network mode/); + assert.throws(() => dockerHostGatewayArguments('ambient'), /unsupported Docker network mode/); +}); + +test('attempt subnets avoid host routes and Docker IPAM, and retry only overlap races', () => { + const header = 'Iface Destination Gateway Flags RefCnt Use Metric Mask MTU Window IRTT'; + for (const mode of ['race', 'host-route', 'exhausted', 'daemon-error', 'removed-network', 'inspect-error', 'inspect-timeout'] as const) { + const root = mkdtempSync(join(tmpdir(), 'attempt-subnet-')); + try { + const lease = createBackendLease({ backend: 'postgres', track: 'chat', runIndex: 0, + runId: `subnet-${mode}`, database: 'subnet_test' }); + const path = join(root, 'lease.json'); + writeBackendLease(path, lease); + const attempts: string[][] = []; + const route = mode === 'host-route' ? 'eth0 0000000A 00000000 0001 0 0 0 00FFFFFF 0 0 0' + : 'eth0 00000000 0100000A 0003 0 0 0 00000000 0 0 0'; + const reserved = mode === 'exhausted' ? ['10.0.0.0/8', '172.16.0.0/12', '192.168.0.0/16'] + : [mode === 'host-route' ? '10.0.1.0/29' : '10.0.0.0/29', 'fd00::/64']; + const id = 'b'.repeat(64); + const docker = (args: string[]) => { + if (args[1] === 'ls') return 'existing removed'; + if (args[1] === 'inspect') { + const output = JSON.stringify([{ IPAM: { Config: + args[2] === id ? [{ Subnet: attempts.at(-1)![5], Gateway: '10.0.0.17' }] + : reserved.map(Subnet => ({ Subnet })) } }]); + if (args[2] !== id && ['removed-network', 'inspect-error', 'inspect-timeout'].includes(mode)) { + throw Object.assign(new Error('inspect failed'), { stdout: output, + status: mode === 'inspect-timeout' ? null : 1, signal: mode === 'inspect-timeout' ? 'SIGTERM' : null, stderr: + `Error response from daemon: network removed not found\n${mode === 'inspect-error' ? 'permission denied\n' : ''}` }); + } + return output; + } + assert.equal(args[1], 'create'); + attempts.push(args); + if (mode === 'daemon-error') throw Object.assign(new Error('permission denied'), { stderr: 'permission denied' }); + if (mode === 'race' && attempts.length === 1) { + throw Object.assign(new Error('overlap'), { stderr: 'Error response from daemon: Pool overlaps with other one on this address space' }); + } + return id; + }; + const create = () => createAttemptNetwork(path, lease, docker, `${header}\n${route}\n`); + if (mode === 'exhausted') { + assert.throws(create, /No unused private IPv4/); + assert.equal(attempts.length, 0); + } else if (mode === 'inspect-error' || mode === 'inspect-timeout') { + assert.throws(create, /inspect failed/); + assert.equal(attempts.length, 0, 'do not allocate after an unknown inspection error'); + } else if (mode === 'daemon-error') { + assert.throws(create, /permission denied/); + assert.equal(attempts.length, 1); + } else { + const updated = create(); + assert.equal(updated.resources.network?.id, id); + assert.deepEqual(attempts.map(args => args[5]), mode === 'race' + ? ['10.0.0.8/29', '10.0.0.16/29'] : mode === 'removed-network' ? ['10.0.0.8/29'] : ['10.0.1.8/29']); + assert.ok(attempts.every(args => args[7]!.includes(updated.resources.creationIntents!.network!.creationToken))); + } + } finally { rmSync(root, { recursive: true, force: true }); } + } +}); + +for (const kind of ['backend', 'browser'] as const) { + test(`${kind} start failure retains sanitized Docker diagnostics before cleanup`, () => { + const root = mkdtempSync(join(tmpdir(), 'container-start-failure-')); + try { + const lease = createBackendLease({ backend: 'postgres', track: 'chat', runIndex: 0, + runId: 'container-start-failure', database: 'container_start_failure' }); + const id = 'c'.repeat(64); + const anchor = 'a'.repeat(64); + lease.resources.container = { name: 'anchor', id: anchor, owned: true }; + lease.resources.network = { name: 'attempt', id: 'b'.repeat(64), namespaceContainerId: anchor, + hostAddresses: [], services: [], firewallSha256: null, firewallInstalledAt: null }; + const path = join(root, 'lease.json'); + writeBackendLease(path, lease); + const commands: string[][] = []; + assert.throws(() => createAttemptContainer(path, lease, kind, + `sha256:${'d'.repeat(64)}`, kind === 'backend' ? 'attempt' : `container:${anchor}`, [], args => { + commands.push(args); + if (args[0] === 'create') return id; + if (args[0] === 'start') throw Object.assign(new Error('Command failed: docker start'), + { status: 1, signal: null, stderr: Buffer.from('password="private value"'), stdout: '' }); + assert.deepEqual(args, ['inspect', '--format', '{{json .State}}', id]); + return JSON.stringify({ Status: 'created', Error: 'OCI start failed: api_key="private key"', + ExitCode: 128, OOMKilled: false }); + }), error => { + const message = String(error); + assert.match(message, /OCI start failed/); + assert.match(message, /"status":1/); + assert.match(message, /"exitCode":128/); + assert.match(message, /"oomKilled":false/); + assert.doesNotMatch(message, /private|value|key/); + return true; + }); + assert.equal(commands.filter(args => args[0] === 'start').length, 1, 'do not retry activation'); + const retained = readBackendLease(path); + assert.equal((kind === 'backend' ? retained.resources.container : retained.resources.browserContainer)?.id, id); + } finally { rmSync(root, { recursive: true, force: true }); } + }); + + test(`${kind} creation enforces its own memory cap without swap`, () => { + const root = mkdtempSync(join(tmpdir(), 'container-limits-')); + try { + const lease = createBackendLease({ backend: 'postgres', track: 'chat', runIndex: 0, + runId: 'container-limits', database: 'container_limits' }); + const anchor = 'a'.repeat(64); + lease.resources.container = { name: 'anchor', id: anchor, owned: true }; + lease.resources.network = { name: 'attempt', id: 'b'.repeat(64), namespaceContainerId: anchor, + hostAddresses: ['172.20.0.1'], services: [], firewallSha256: null, firewallInstalledAt: null }; + const path = join(root, 'lease.json'); + writeBackendLease(path, lease); + const commands: string[][] = []; + createAttemptContainer(path, lease, kind, `sha256:${'d'.repeat(64)}`, `container:${anchor}`, [], args => { + commands.push(args); + if (args[0] === 'create') return 'c'.repeat(64); + if (args[0] === 'inspect') return args[2] === '{{json .NetworkSettings.Networks}}' ? '{}' : '2026-09-10T00:00:00Z'; + return ''; + }); + const command = commands.find(args => args[0] === 'create')!; + const memory = String((kind === 'browser' ? 4 : 2) * 1024 ** 3); + assert.equal(command[command.indexOf('--memory') + 1], memory); + assert.equal(command[command.indexOf('--memory-swap') + 1], memory); + assert.equal(command[command.indexOf('--cpus') + 1], '1'); + assert.equal(command[command.indexOf('--pids-limit') + 1], kind === 'browser' ? '512' : '256'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +} + +test('a released lease cannot authorize new attempt resources', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-released-lease-')); + try { + const path = join(root, 'lease.json'); + const lease = createBackendLease({ runId: 'released-lease-test', backend: 'mongodb', + track: 'ecommerce', runIndex: 0, database: 'app_ecom_run0', + container: { name: 'stack-bench-mongodb', id: 'database-id' } }); + lease.state = 'released'; + writeBackendLease(path, lease); + assert.throws(() => recordAttemptCreation(path, lease, 'broker'), /released/); + assert.equal(readBackendLease(path, { token: lease.ownershipToken }).resources.creationIntents?.broker, undefined); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('a platform service joins the exact anchor namespace, capped, with its secrets outside the command line', () => { + const root = mkdtempSync(join(tmpdir(), 'attempt-service-')); + try { + const lease = createBackendLease({ backend: 'supabase', track: 'ecommerce', runIndex: 0, + runId: 'attempt-service', serverUri: 'http://127.0.0.1:13410', database: 'postgres' }); + const anchor = 'a'.repeat(64), id = 'c'.repeat(64); + const path = join(root, 'lease.json'); + writeBackendLease(path, lease); + assert.throws(() => createAttemptService(path, lease, 'auth', `sha256:${'d'.repeat(64)}`, + { args: [], command: [], limits: { cpuCount: 1, memoryBytes: 1, pids: 1 } }), /no backend namespace/); + lease.resources.container = { name: 'anchor', id: anchor, owned: true }; + lease.resources.network = { name: 'attempt', id: 'b'.repeat(64), namespaceContainerId: anchor, + hostAddresses: ['172.20.0.1'], services: [], firewallSha256: null, firewallInstalledAt: null }; + writeBackendLease(path, lease); + const calls: { args: string[]; environment?: Record<string, string> }[] = []; + const docker = (args: string[], _input?: string, environment?: Record<string, string>) => { + calls.push({ args, ...(environment ? { environment } : {}) }); + if (args[0] === 'create') return id; + if (args[0] === 'start') throw Object.assign(new Error('Command failed: docker start'), { status: 1, signal: null, + stderr: 'password="private value"', stdout: '' }); + return JSON.stringify({ Status: 'created', Error: 'OCI start failed', ExitCode: 128, OOMKilled: false }); + }; + const container = createAttemptService(path, lease, 'auth', `sha256:${'d'.repeat(64)}`, { + args: ['-e', 'GOTRUE_API_PORT=9999', '-e', 'GOTRUE_JWT_SECRET'], command: ['auth'], + limits: { cpuCount: 1, memoryBytes: 128 * 1024 ** 2, pids: 256 }, + environment: { GOTRUE_JWT_SECRET: 'private-jwt-secret' }, docker, + }); + const create = calls[0]!; + assert.equal(create.args[create.args.indexOf('--network') + 1], `container:${anchor}`); + for (const flag of ['--init', 'no-new-privileges:true']) assert(create.args.includes(flag)); + assert.equal(create.args[create.args.indexOf('--cap-drop') + 1], 'ALL'); + assert.equal(create.args[create.args.indexOf('--memory') + 1], String(128 * 1024 ** 2)); + assert.equal(create.args[create.args.indexOf('--memory-swap') + 1], String(128 * 1024 ** 2)); + assert.equal(create.args[create.args.indexOf('--pids-limit') + 1], '256'); + assert.deepEqual(create.args.slice(-2), [`sha256:${'d'.repeat(64)}`, 'auth']); + assert(!create.args.join(' ').includes('private-jwt-secret')); + assert.deepEqual(create.environment, { GOTRUE_JWT_SECRET: 'private-jwt-secret' }); + const recorded = readBackendLease(path); + assert.deepEqual(recorded.resources.serviceContainers?.auth, container); + assert.equal(container.networkMode, `container:${anchor}`); + assert.equal(create.args[create.args.indexOf('--name') + 1], recorded.resources.creationIntents?.['service-auth']?.name); + assert.throws(() => startAttemptContainer(container.id, 'auth service', docker), error => { + assert.match(String(error), /Docker could not start owned auth service container/); + assert.match(String(error), /OCI start failed/); + assert.doesNotMatch(String(error), /private value/); + return true; + }); + assert.throws(() => createAttemptService(path, recorded, 'auth', `sha256:${'d'.repeat(64)}`, + { args: [], command: [], limits: { cpuCount: 1, memoryBytes: 1, pids: 1 }, docker }), /already has creation authority/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/tools/stack-bench/tests/ecommerce-observation.contract.ts b/tools/stack-bench/tests/ecommerce-observation.contract.ts new file mode 100644 index 00000000000..691b19278b8 --- /dev/null +++ b/tools/stack-bench/tests/ecommerce-observation.contract.ts @@ -0,0 +1,310 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import test from 'node:test'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; +import { agentVisibleContractText } from '../src/composition/agent-visible-contract.js'; + +const read = (name: string) => compileScenarioDefinition(JSON.parse(readFileSync( + join(STACK_BENCH_ROOT, 'tracks/ecommerce/scenarios', name), 'utf8'))); + +test('selected privacy checks prove a working positive path without sibling criteria', () => { + const alerts = read('progression-stock-alerts.json').features[0]!.criteria.find(c => c.id === '631b')!.steps; + const delivered = alerts.findIndex(s => s.testid === 'stock-alert-delivery'); + const absent = alerts.findIndex(s => s.absent === true); + assert(delivered >= 0 && absent > delivered); + assert.equal(alerts[delivered]!.actor, 'subscriber-fresh'); + const promotions = read('progression-promotion-rules.json').features[0]!.criteria.find(c => c.id === '620b')!.steps; + const promotionReload = promotions.findIndex(s => s.do === 'reload'); + const created = promotions.findIndex((s, index) => index > promotionReload && s.do === 'expect' && s.testid === 'promotion-item' && s.contains === 'ACCESS10'); + assert(created > promotions.findIndex(s => s.testid === 'promotion-submit')); + assert(created > promotions.findIndex(s => s.do === 'reload')); + assert(created < promotions.findIndex(s => s.do === 'replayAs')); + const roles = read('progression-staff-roles.json').features[0]!.criteria.find(c => c.id === '621b')!.steps; + const negative = roles.findIndex(s => s.do === 'replayAs' && s.actor === 'staff'); + assert(negative >= 0); + for (const account of ['staff', 'staff2']) { + const target = `staff-role-account-${account}`; + const reset = roles.findIndex(s => s.do === 'replayAs' && s.actor === 'replayAdmin' + && (s.namedTarget as { testid: string }).testid === target + && Array.isArray(s.namedAction?.args) && s.namedAction.args[1] === 'staff'); + const reload = roles.findIndex((s, index) => index > reset && s.do === 'reload' && s.actor === 'replayAdmin'); + const saved = roles.findIndex((s, index) => index > reload && s.do === 'expect' + && s.testid === 'staff-role-select' && s.in?.testid === target && s.value === 'staff'); + assert(reset >= 0 && reload > reset && saved > reload && saved < negative, + `${account} must have its own reset and fresh stored-state proof before the unauthorized write`); + assert(roles.slice(reset, reload).some(s => s.do === 'expectReplayCompleted' && s.requireAccepted === true)); + } +}); + +test('order ownership switches the same browser and reconciles the next write under the new account', () => { + const criterion = read('01-order-ownership.json').features[0]!.criteria[0]!; + const steps = criterion.steps; + assert.equal(criterion.points, 1); + assert(!steps.some(step => ['freshClient', 'reload', 'openClient'].includes(step.do)), + 'a grader-created page reset would hide stale account caches'); + const logout = steps.findIndex(step => step.do === 'click' && step.testid === 'signout'); + const login = steps.findIndex(step => step.do === 'signIn'); + assert(logout > 0 && login > logout); + assert.equal(steps[logout]!.actor, 'two'); + assert.equal(steps[login]!.actor, 'two'); + assert.equal(steps[login]!.name, 'quin'); + const views = steps.filter(step => step.do === 'expect' && step.testid === 'order-item'); + assert.deepEqual(views.map(step => [step.actor, step.contains, step.count ?? step.absent]), [ + ['two', 'Desk Lamp', 1], ['two', 'Coffee Grinder', true], + ['two', 'Coffee Grinder', 1], ['two', 'Desk Lamp', true], + ]); + const snapshot = steps.find(step => step.do === 'dbRecordCheckout')!; + assert.equal(snapshot.account, '{user:quin}'); + assert.deepEqual(snapshot.storage, { kind: 'order-data', cart: false, warehouses: false }); + assert.deepEqual(steps.slice(-3).map(step => [step.do, step.actor, step.outcome]), [ + ['callAction', 'two', undefined], ['expectActionOutcome', 'two', 'accepted'], ['dbExpectPurchase', 'two', undefined], + ]); +}); + +test('scheduled-restock identifiers match each native interface and stay opaque in replay', () => { + const text = readFileSync(join(STACK_BENCH_ROOT, + 'tracks/ecommerce/contracts/scheduled-restocks.md'), 'utf8'); + const convex = agentVisibleContractText(text, {}, 'convex'); + assert.match(convex, /native `_id` string for `data-entity-id` and `restockId`/); + assert.doesNotMatch(convex, /decimal|u64/); + const supabase = agentVisibleContractText(text, {}, 'supabase'); + assert.match(supabase, /identifier accepted by `restockId` for the restock's `data-entity-id`/); + assert.doesNotMatch(supabase, /decimal|u64|native `_id`|api:/); + for (const kind of ['http', 'reducer'] as const) { + const rendered = agentVisibleContractText(text, {}, kind); + assert.match(rendered, /`data-entity-id` as a decimal number/); + assert.doesNotMatch(rendered, /native `_id`/); + } + const replay = read('03-deferred-access.json').features[0]!.criteria[0]!.steps + .find(step => step.do === 'replayAs')!; + assert.deepEqual(replay.namedTarget, { testid: 'pending-restock-item', + attribute: 'data-entity-id', valueType: 'string' }); +}); + +test('restock observations use one pending row and a ledger delta without item-name labels', () => { + const source = read('03-scheduled-restock-apply.json'); + const due = source.features.find(feature => feature.id === 305)!.criteria[0]!.steps; + const record = due.findIndex(step => step.do === 'recordNumber' && step.count === true); + const submit = due.findIndex(step => step.testid === 'schedule-restock-submit'); + assert(record >= 0 && record < submit); + assert(due.some(step => step.do === 'expectElementCount' + && step.relativeTo === 'ledger-before' && step.plus === 1)); + const pack = JSON.parse(readFileSync(join(STACK_BENCH_ROOT, + 'tracks/ecommerce/composition/packs/l3-scheduled-restocks-features.json'), 'utf8')); + const sources = pack.checks.map((check: { source: string }) => check.source); + assert.equal(new Set(sources).size, 3, 'each restock check needs its own database reset boundary'); + for (const [index, name] of ['03-scheduled-restocks.json', '03-scheduled-restock-apply.json', + '03-scheduled-restock-cancel.json'].entries()) { + assert.equal(sources[index], `scenarios/${name}`); + assert.deepEqual(read(name).features.flatMap(feature => feature.criteria).map(check => check.id), + [['302a'], ['305a'], ['306a']][index]); + } + for (const name of ['03-scheduled-restocks.json', '03-scheduled-restock-apply.json', + '03-scheduled-restock-cancel.json', '03-deferred-access.json', + '03-deferred-durability.json', '03-deferred-integrity.json', '03-server-time.json']) { + const scenario = JSON.stringify(read(name)); + assert(!/"testid":"pending-restock-item","contains"/.test(scenario)); + } + // An overdraw failure cannot change the next transfer or authorization probe state. + const isolated = read('02-transfer-overdraw.json'); + const shared = read('02-strengthened.json'); + assert.deepEqual(isolated.features.flatMap(feature => feature.criteria).map(check => check.id), ['2c']); + assert(!shared.features.flatMap(feature => feature.criteria).some(check => check.id === '2c')); + assert.deepEqual(isolated.features[0]!.setup, shared.features.find(feature => feature.id === 2)!.setup); + const integrity = JSON.parse(readFileSync(join(STACK_BENCH_ROOT, + 'tracks/ecommerce/composition/packs/spec-transactional-integrity.json'), 'utf8')); + assert.equal(integrity.checks.find((check: { id: string }) => check.id === 'stock-transfer-overdraw').source, + 'scenarios/02-transfer-overdraw.json'); +}); + +test('support logout checks fresh traffic with post-logout storage and a usable public page', () => { + const steps = read('progression-support-history.json').features[0]!.criteria.find(c => c.id === '612d')!.steps; + const received = steps.findIndex(s => s.do === 'expectReceived'); + const logout = steps.findIndex(s => s.testid === 'signout'); + const confirmed = steps.findIndex(s => s.do === 'waitUntilAbsent' && s.testid === 'current-user'); + const fresh = steps.findIndex(s => s.do === 'freshClient'); + const ready = steps.findIndex(s => s.actor === 'owner-fresh' && s.testid === 'support-email'); + const negative = steps.findIndex(s => s.do === 'expectNotReceived'); + assert(received >= 0 && logout > received && confirmed > logout && fresh > confirmed); + assert.equal(steps[fresh]!.preserveStorage, true); + assert(ready > fresh && negative > ready); + assert.equal(steps[negative]!.contains, steps[received]!.contains); + assert.equal(steps[negative]!.actor, 'owner-fresh'); + const graph = JSON.parse(readFileSync(join(STACK_BENCH_ROOT, 'tracks/ecommerce/progression/ecommerce.json'), 'utf8')); + assert(graph.nodes.find((node: { id: string }) => node.id === 'support-history').gradingGroups + .includes('ecommerce.spec.access-control#support-history-logout')); +}); + +test('support privacy creates its own persisted ticket after the durability probe', () => { + const history = read('progression-support-history.json').features[0]!; + const privacy = history.criteria.find(check => check.id === '612b')!.steps; + const subject = privacy.find(step => step.testid === 'support-subject')!.text; + assert(subject); + assert.notEqual(subject, history.setup.find(step => step.testid === 'support-subject')!.text); + const submit = privacy.findIndex(step => step.testid === 'support-submit'); + const reloaded = privacy.findIndex((step, index) => index > submit && step.do === 'reload'); + const positive = privacy.findIndex(step => step.do === 'expect' && step.testid === 'support-ticket' + && step.contains === subject && step.absent !== true); + assert(submit >= 0 && reloaded > submit && positive > reloaded); + assert(privacy.slice(positive + 1).some(step => step.do === 'expectReceived' && step.contains === subject)); + assert(privacy.some(step => step.actor === 'other' && step.do === 'expect' + && step.contains === subject && step.absent === true)); + assert(privacy.some(step => step.actor === 'other' && step.do === 'expectNotReceived' + && step.contains === subject)); +}); + +test('stock alerts use ready fresh account views after accepted restocks', () => { + for (const name of ['progression-stock-alerts.json', 'progression-stock-alert-delivery.json']) { + const feature = read(name).features[0]!; + for (const criterion of feature.criteria) { + for (const [index, step] of criterion.steps.entries()) { + if (step.testid !== 'notifications-toggle') continue; + assert.equal(step.unlessVisible, 'notifications-panel'); + const before = criterion.steps.slice(0, index); + const fresh = before.findLastIndex(candidate => candidate.do === 'freshClient' + && `${candidate.actor}-fresh` === step.actor); + assert(fresh >= 0); + assert(before.slice(fresh + 1).some(candidate => candidate.do === 'signIn' + && candidate.actor === step.actor)); + assert.equal(criterion.steps[index + 1]!.attribute, 'aria-busy'); + assert.equal(criterion.steps[index + 1]!.value, 'false'); + } + } + } + const delivery = read('progression-stock-alert-delivery.json').features[0]!; + const steps = delivery.criteria[0]!.steps; + const empty = steps.findIndex(step => step.do === 'expectElementCount' && step.equals === 0); + const restock = steps.findIndex(step => step.do === 'callAction' && step.action === 'restock'); + const delivered = steps.findIndex(step => step.do === 'expect' && step.testid === 'stock-alert-delivery'); + assert(empty >= 0 && empty < restock && restock < delivered); + assert.equal(steps[restock + 1]!.do, 'expectActionOutcome'); + assert.equal(steps[restock + 1]!.outcome, 'accepted'); + assert.equal(steps[restock + 2]!.do, 'wait'); + assert.equal(steps[restock + 2]!.ms, 10000); +}); + +test('cart validation sends only a negative quantity and restores actor identity after reloads', () => { + const feature = read('01-cart-boundary.json').features[0]!; + const criterion = feature.criteria.find(candidate => candidate.id === '109b')!; + const calls = criterion.steps.filter(step => step.do === 'callAction'); + // An accepted in-range quantity precedes the refusal. Zero is a convention (remove the line or + // refuse); only a negative quantity can credit money. + assert.deepEqual(calls.map(step => (step.namedAction as { args: number[] }).args[1]), [2, -3]); + for (const criterion of feature.criteria) { + criterion.steps.forEach((step, index) => { + if (step.do !== 'reload') return; + assert.equal(criterion.steps[index + 1]!.do, 'ensureSignedIn'); + assert.equal(criterion.steps[index + 1]!.actor, step.actor); + }); + // Access assertions read refreshed state after the direct write. + const result = criterion.steps.findIndex(step => step.do === 'expectActionOutcome'); + assert(result >= 0); + assert.equal(criterion.steps[result + 1]!.do, 'reload'); + assert(criterion.steps.slice(result + 1).some(step => step.do === 'expectNumber' + && step.testid === 'cart-quantity')); + } + assert(feature.criteria[0]!.steps.some(step => step.do === 'reload' && step.actor === 'stranger')); +}); + +test('later-depth checks retain their own effects and independent controls', () => { + const reorder = read('progression-automatic-reorder.json').features[0]!; + assert(!reorder.setup.some(step => step.do === 'callAction' && step.action === 'buy')); + assert(reorder.criteria.find(c => c.id === '502a')!.steps.some(step => step.do === 'callAction' && step.action === 'buy')); + const duplicate = read('progression-automatic-reorder-duplicate.json').features[0]!; + assert(duplicate.setup.some(step => step.do === 'expectElementCount' && step.testid === 'pending-restock-item' && step.equals === 1)); + assert.equal(duplicate.criteria[0]!.steps.filter(step => step.do === 'callAction' && step.action === 'buy').length, 2); + const payment = read('progression-core-business.json').features.find(f => f.id === 623)!; + assert.equal(payment.setup.find(step => step.do === 'expectCallOutcomes')!.accepted, undefined); + assert(payment.setup.some(step => step.do === 'freshClient')); + assert(payment.setup.some(step => step.do === 'expectElementCount' && step.testid === 'order-item' && step.equals === 1)); + const checkout = read('03-deferred-integrity.json').features.find(f => f.id === 314)!.criteria[0]!.steps; + assert(checkout.some(step => step.do === 'reload')); + assert(checkout.some(step => step.testid === 'order-item' && step.equals === 1)); + const filters = read('progression-faceted-filters.json').features[0]!; + assert.equal(filters.setup.filter(step => step.do === 'dbSetStock' && step.item === 'Coffee Grinder' && step.quantity === 0).length, 2); + const steps = filters.criteria[0]!.steps; + assert.equal(steps[0]!.contains, 'Coffee Grinder'); + assert.equal(steps[0]!.equals, 1); + assert.equal(steps[1]!.testid, 'in-stock-filter'); + assert(steps.some(step => step.contains === 'Coffee Grinder' && step.do === 'waitUntilAbsent')); + const boundary = read('progression-order-support-boundary.json').features[0]!.criteria[0]!.steps; + const attack = boundary.find(step => step.do === 'callAction' && step.actor === 'other')!; + assert.deepEqual((attack.input as { overrides: unknown }).overrides, { + caseId: { actor: 'other', testid: 'support-ticket', contains: 'Other order case', attribute: 'data-entity-id' }, + }); + assert(boundary.some(step => step.do === 'expectActionOutcome' && step.actor === 'owner' && step.outcome === 'accepted')); +}); + +test('progression review eligibility proves the route and reads back persisted refusal', () => { + const steps = read('progression-review-access.json').features[0]!.criteria[0]!.steps; + assert.equal(steps[0]!.do, 'callAction'); + assert.equal(steps[0]!.actor, 'owner'); + assert.equal(steps[1]!.outcome, 'accepted'); + const refused = steps.find(step => step.do === 'expectActionOutcome' && step.actor === 'stranger')!; + assert.equal(refused.outcome, 'application-refused'); + assert.equal(refused.routeProvenBy, 'owner'); + assert(steps.some(step => step.do === 'freshClient' && step.actor === 'owner')); + assert(steps.some(step => step.actor === 'owner-fresh' && step.contains === 'Original buyer review' && !step.absent)); + for (const contains of ['Nonbuyer review', 'Forged buyer review']) { + assert(steps.some(step => step.actor === 'owner-fresh' && step.contains === contains && step.absent === true)); + } +}); + + +test('countdown displays decrease from observed baselines without setup-time assumptions', () => { + for (const [source, id] of [['03-reservations.json', '305a'], ['03-scheduled-restocks.json', '302a']]) { + const criterion = read(source!).features.flatMap(f => f.criteria).find(c => c.id === id)!; + const recordIndex = criterion.steps.findIndex(step => step.do === 'recordNumber' && step.as === 'initial-countdown'); + assert(recordIndex > 0); + assert.equal(criterion.steps[recordIndex - 1]!.atLeast, 1); + assert.equal(criterion.steps[recordIndex - 1]!.atMost, 90); + assert.equal(criterion.steps[recordIndex + 1]!.ms, 1000); + const bound = criterion.steps[recordIndex + 2]!; + assert.equal(bound.relativeTo, 'initial-countdown'); + assert.equal(bound.plus, -1); + assert.equal(bound.comparison, 'atMost'); + assert.equal(bound.within, 10000); + } +}); + + +test('role revocation proves authorization before testing the same session after removal', () => { + const steps = read('progression-staff-roles.json').features[0]!.criteria.find(c => c.id === '621d')!.steps; + const login = steps.findIndex(s => s.do === 'signIn' && s.actor === 'promotedStaff'); + assert(login >= 0); + assert.equal(steps.filter(s => s.do === 'signIn' && s.actor === 'promotedStaff').length, 1); + assert(!steps.slice(login + 1).some(s => s.actor === 'promotedStaff' + && ['signIn', 'ensureSignedIn', 'reload', 'freshClient', 'closeClient'].includes(s.do))); + const calls = steps.filter(s => s.do === 'replayAs'); + assert.deepEqual(calls.map(s => [s.actor, Array.isArray(s.namedAction?.args) ? s.namedAction.args[1] : undefined]), [ + ['roleAdmin', 'admin'], ['promotedStaff', 'inventory'], + ['roleAdmin', 'staff'], ['roleAdmin', 'staff'], ['promotedStaff', 'inventory'], + ]); + const positive = steps.findIndex(s => s.do === 'expectReplayCompleted' && s.actor === 'promotedStaff'); + for (const call of calls.filter(s => s.actor === 'promotedStaff')) { + assert.equal((call.namedTarget as { testid: string }).testid, 'staff-role-account-staff2'); + } + const negative = steps.findIndex(s => s.do === 'expectReplayRejected' && s.actor === 'promotedStaff'); + assert(positive > login && negative > positive); + assert.equal(steps[positive]!.requireAccepted, true); + assert(steps.slice(positive, negative).some(s => s.do === 'expect' && s.value === 'staff')); + assert.equal(steps[negative + 1]!.do, 'reload'); + assert.equal(steps.at(-1)!.value, 'staff'); + assert.equal(steps.at(-1)!.actor, 'roleAdmin'); +}); + +test('privacy and role actors reconnect after a preceding backend restart', () => { + for (const file of ['progression-customer-profile.json', 'progression-staff-roles.json', 'progression-support-history.json']) { + const feature = read(file).features[0]!; + const disconnected = new Set<string>(); + for (const step of [...feature.setup, ...feature.criteria.flatMap(check => check.steps)]) { + if (step.do === 'restartBackend') for (const actor of feature.actors ?? []) disconnected.add(actor); + if (step.do === 'reload' && typeof step.actor === 'string') disconnected.delete(step.actor); + if (['signUp', 'signIn', 'ensureSignedIn', 'expectNotReceived'].includes(step.do)) { + assert(!disconnected.has(String(step.actor)), file + ': ' + step.do + ' used a disconnected actor ' + step.actor); + } + } + } +}); diff --git a/tools/stack-bench/tests/ecommerce-progression-campaign.contract.ts b/tools/stack-bench/tests/ecommerce-progression-campaign.contract.ts new file mode 100644 index 00000000000..fa7d552b57a --- /dev/null +++ b/tools/stack-bench/tests/ecommerce-progression-campaign.contract.ts @@ -0,0 +1,209 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { compileCampaignFile, validateCompiledCampaignPlan } + from '../src/campaigns/campaign-compiler.js'; +import { runCampaignAdmission } from '../src/campaigns/campaign-admission.js'; +import type { CampaignAdmissionPreflightRequest } + from '../src/campaigns/campaign-admission.js'; +import { attemptArgv } from '../src/campaigns/campaign-runner.js'; +import { parseBenchArguments } from '../commands/bench-arguments.js'; +import { DEFAULT_BUILD_IMAGE } from '../src/composition/product-config.js'; +import { writeArtifact } from '../src/evidence/artifacts.js'; +import { progressionEngine } from '../src/progression/progression-engine.js'; +import { compileProgressionDefinitionFile } from '../src/progression/progression-definition.js'; +import { requireRecipeRelease as resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { loadTrack } from '../src/composition/tracks.js'; + +const root = STACK_BENCH_ROOT; +const campaignPath = join(root, 'appliance', 'campaign.ecommerce-progression-reference.json'); +// One compile of the reference campaign, cloned for each test that reads it. +let referencePlanCache: ReturnType<typeof compileCampaignFile> | null = null; +const referencePlan = (): ReturnType<typeof compileCampaignFile> => + structuredClone(referencePlanCache ??= compileCampaignFile(campaignPath)); + +type UnknownRecord = Record<string, unknown>; +const record = (value: unknown): value is UnknownRecord => + value !== null && typeof value === 'object' && !Array.isArray(value); +const first = <Value>(values: readonly Value[]): Value => { + const value = values[0]; + assert(value); + return value; +}; + +function collectInputAttributes(value: unknown, attributes = new Set<string>()): Set<string> { + if (Array.isArray(value)) value.forEach(item => collectInputAttributes(item, attributes)); + else if (record(value)) { + if (record(value.input) && typeof value.input.attribute === 'string') { + attributes.add(value.input.attribute); + } + Object.values(value).forEach(item => collectInputAttributes(item, attributes)); + } + return attributes; +} + +function sourceText(directory: string): string { + return readdirSync(directory, { recursive: true, withFileTypes: true }) + .filter(entry => entry.isFile() && /\.(?:js|jsx|ts|tsx|html)$/.test(entry.name)) + .map(entry => readFileSync(join(entry.parentPath, entry.name), 'utf8')).join('\n'); +} + +test('the ecommerce reference pilot resolves the exact L1-L6 progression inputs', () => { + const plan = referencePlan(); + assert(plan.featureCatalog); + assert(plan.dependencyPolicy); + assert.deepEqual(plan.definition.levels, [1, 2, 3, 4, 5, 6]); + assert.equal(plan.featureCatalog.identity.id, 'ecommerce.questlines'); + assert.match(plan.featureCatalog.identity.contentSha256, /^[a-f0-9]{64}$/); + assert.equal(plan.dependencyPolicy.identity.id, 'dependency-graph'); + assert.deepEqual(plan.bindings.map(binding => binding.recipe.id), + Array(6).fill('ecommerce.progression-catalog')); + assert.equal(new Set(plan.bindings.map(binding => binding.recipe.contentSha256)).size, 1); + const trackRoot = join(root, 'tracks', 'ecommerce'); + assert.equal(plan.featureCatalog.definition.nodes.length, compileProgressionDefinitionFile( + join(trackRoot, 'progression', 'ecommerce.json'), { trackRoot }).nodes.length); + assert(first(plan.conditions).requested.levels.every(level => + typeof level.task.mode === 'string' && ['fresh', 'upgrade'].includes(level.task.mode))); + assert.deepEqual(plan.attempts.map(attempt => attempt.stack).sort(), + ['mongodb', 'postgres', 'spacetime']); + assert(plan.attempts.every(attempt => attempt.agentAdapter === 'reference-fixture')); + assert.equal(validateCompiledCampaignPlan(plan).contentSha256, plan.contentSha256); +}); + +test('sequential mode can run a prefix of the same ecommerce feature catalog', () => { + const directory = mkdtempSync(join(tmpdir(), 'stack-bench-sequential-prefix-')); + try { + const manifest = JSON.parse(readFileSync(campaignPath, 'utf8')); + manifest.id = 'ecommerce-sequential-prefix-proof'; + manifest.mode = { id: 'sequential' }; + manifest.repair = { selection: 'batch', budget: { total: 1 } }; + manifest.levels = [1, 2, 3]; + manifest.selection.levels = manifest.selection.levels.slice(0, 3); + const path = join(directory, 'campaign.json'); + writeFileSync(path, `${JSON.stringify(manifest, null, 2)}\n`); + const plan = compileCampaignFile(path); + const featureCatalog = plan.featureCatalog; + assert(featureCatalog); + const condition = first(plan.conditions); + assert.deepEqual(plan.definition.levels, [1, 2, 3]); + assert(plan.attempts.every(attempt => attempt.dependencyPolicy === undefined)); + assert(plan.attempts.every(attempt => { + assert(attempt.featureCatalog); + return attempt.featureCatalog.contentSha256 === featureCatalog.identity.contentSha256; + })); + assert.deepEqual(condition.requested.levels.map(level => level.level), [1, 2, 3]); + for (const level of condition.requested.levels) { + assert(level.selection.scoredChecks); + assert.deepEqual(level.selection.scoredChecks.map(check => check.stableKey).sort(), + featureCatalog.definition.nodes.filter(node => node.level === level.level) + .flatMap(node => node.gradingChecks.map(check => check.id)).sort(), + `sequential level ${level.level} must select its catalog checks exactly once`); + } + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test('dependency mode stops at the selected catalog prefix', () => { + const directory = mkdtempSync(join(tmpdir(), 'stack-bench-dependency-prefix-')); + try { + const manifest = JSON.parse(readFileSync(campaignPath, 'utf8')); + manifest.id = 'ecommerce-dependency-prefix-proof'; + manifest.levels = [1, 2, 3]; + manifest.selection.levels = manifest.selection.levels.slice(0, 3); + const sourcePath = join(directory, 'campaign.json'); + writeFileSync(sourcePath, `${JSON.stringify(manifest, null, 2)}\n`); + const plan = compileCampaignFile(sourcePath); + assert(plan.dependencyPolicy); + assert.deepEqual(plan.dependencyPolicy.definition.levels, [1, 2, 3]); + assert.deepEqual(plan.dependencyPolicy.definition.repair, + { selection: 'feature', budget: { total: 0 }, order: 'declared' }); + assert.deepEqual(plan.dependencyPolicy.definition.nodeOrder, + plan.featureCatalog?.definition.nodes.map(node => node.id)); + + const planPath = join(directory, 'plan.json'); + writeArtifact(planPath, { kind: 'campaign_plan', id: `${plan.id}-plan`, payload: plan }); + const attempt = first(plan.attempts); + const args = parseBenchArguments(['node', ...attemptArgv(plan, attempt, + join(directory, 'result'), 0, planPath)]); + assert(args.progression); + assert.deepEqual(args.levelList, [1, 2, 3]); + assert.equal(Math.max(...args.progression.definition.nodes.map(node => node.level)), 3); + + let state = progressionEngine.initialize(args.progression.definition); + const visited: number[] = []; + while (state.phase === 'active') { + const action = progressionEngine.nextAction(state); + if (typeof action.level !== 'number') throw new Error('active progression has no level'); + visited.push(action.level); + const selection = progressionEngine.gradingSelection(state); + state = progressionEngine.recordResult(state, { + attemptId: `level-${action.level}`, + outcome: 'conclusive', + nodes: selection.nodeIds.map(id => ({ id, + checks: selection.checks.filter(check => check.nodeId === id) + .map(check => ({ id: check.id, outcome: 'pass' })) })), + }); + } + assert.deepEqual(visited, [1, 2, 3]); + assert.equal(progressionEngine.nextAction(state).type, 'terminal'); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test('every progression action input is exposed by every reference app', () => { + const track = loadTrack('ecommerce'); + const binding = resolveRecipeRelease(track, 5, 'ecommerce.progression-catalog'); + const attributes = new Set<string>(); + for (const execution of binding.plan.execution) { + collectInputAttributes(JSON.parse(readFileSync(join(track.dir, execution.source), 'utf8')), + attributes); + } + assert(attributes.size > 0); + const missing: string[] = []; + for (const backend of ['mongodb', 'postgres', 'spacetime']) { + const text = sourceText(join(root, 'reference-apps', 'ecommerce', backend)); + for (const attribute of attributes) { + if (!text.includes(attribute)) missing.push(`${backend}:${attribute}`); + } + } + assert.deepEqual(missing, []); +}); + +test('campaign admission sends the exact catalog, mode, and default build image to preflight', async () => { + const output = mkdtempSync(join(tmpdir(), 'stack-bench-progression-admission-')); + try { + const plan = referencePlan(); + const featureCatalog = plan.featureCatalog; + assert(featureCatalog); + const calls: CampaignAdmissionPreflightRequest[] = []; + const admission = await runCampaignAdmission(plan, output, { + env: { STACK_BENCH_RUNNER_CAPACITY: '64', STACK_BENCH_RESOURCE_LOCK_DIR: join(output, 'locks') }, now: '2026-08-25T00:00:00.000Z', uuid: () => 'test', + preflight: request => { + calls.push(request); + return { schemaVersion: 1, generatedAt: '2026-08-25T00:00:00.000Z', + request: { backends: request.backends, track: request.track, + levels: request.levelList, runIndex: request.runIndex, + parallelism: request.parallelism, + agentAdapter: request.agentAdapter, packs: request.packIds, + checks: request.checkKeys, image: request.image, + resultsDir: request.resultsDir, smoke: request.smoke }, + ok: true, summary: { passed: 0, failed: 0, warnings: 0 }, checks: [] }; + }, + }); + assert.equal(admission.payload.ok, true); + assert.equal(calls.length, plan.summary.parallelism); + assert.equal(plan.summary.parallelism, 1); + assert(calls.every(call => call.image === DEFAULT_BUILD_IMAGE)); + assert(calls.every(call => { + assert(call.featureCatalog); + return call.featureCatalog.identity.contentSha256 === featureCatalog.identity.contentSha256; + })); + assert(calls.every(call => call.mode.id === 'dependency')); + } finally { rmSync(output, { recursive: true, force: true }); } +}); diff --git a/tools/stack-bench/tests/ecommerce-progression.contract.ts b/tools/stack-bench/tests/ecommerce-progression.contract.ts new file mode 100644 index 00000000000..dd49dd5a38f --- /dev/null +++ b/tools/stack-bench/tests/ecommerce-progression.contract.ts @@ -0,0 +1,420 @@ +import assert from 'node:assert/strict'; +import { readFileSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { compilePackDefinition, compileRecipeFile, type CompiledPackDefinition } + from '../src/composition/composition-compiler.js'; +import { loadTrack } from '../src/composition/tracks.js'; +import { requireRecipeRelease as resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; +import { buildRecipeRelease, type RecipeBinding } from '../src/composition/recipe-release.js'; +import { compileProgressionDefinition, compileProgressionDefinitionFile, + compileDependencyPolicyInput, compileFeatureCatalogInput, + dependencyRuntimeDefinition, type CompiledProgressionNode } + from '../src/progression/progression-definition.js'; +import { progressionEngine } from '../src/progression/progression-engine.js'; +import type { ProgressionWorkAction } from '../src/progression/progression-engine.js'; +import { resolveProgressionRecipeAction, resolveProgressionRepairTarget, + priorProgressionContractIds, validateProgressionRecipeBindings } from '../src/progression/progression-recipe-selection.js'; +import { resolveBoundRecipeTaskRequest } from '../src/composition/recipe-selection.js'; + +const trackRoot = join(STACK_BENCH_ROOT, 'tracks', 'ecommerce'); +const packRoot = join(trackRoot, 'composition', 'packs'); +const definitionPath = join(trackRoot, 'progression', 'ecommerce.json'); +const readJson = (path: string): unknown => JSON.parse(readFileSync(path, 'utf8')); + +function packChecks(pack: CompiledPackDefinition): string[] { + return pack.checks.flatMap(check => { + const scenario = compileScenarioDefinition(readJson(join(trackRoot, check.source)), { + source: check.source, + }); + const feature = scenario.features.find(candidate => candidate.id === check.feature); + assert(feature, `${pack.id}.${check.id} must select a feature`); + const criteria = check.criteria === undefined + ? feature.criteria + : check.criteria.map(id => { + const criterion = feature.criteria.find(candidate => candidate.id === id); + assert(criterion, `${pack.id}.${check.id} must select ${id}`); + return criterion; + }); + return criteria.map(criterion => + `${pack.stableId ?? pack.id}.${check.stableId ?? check.id}.${criterion.id}`); + }); +} + +test('every progression feature reference and scored check binds to repository data', () => { + const packs: Array<[string, CompiledPackDefinition]> = readdirSync(packRoot) + .filter(name => name.endsWith('.json')).map(name => { + const pack = compilePackDefinition(readJson(join(packRoot, name)), { source: name }); + return [pack.id, pack]; + }); + const packByRef = new Map(packs); + assert.equal(packByRef.size, packs.length, 'pack ids must be unique'); + + const definition = compileProgressionDefinitionFile(definitionPath, { trackRoot }); + for (const node of definition.nodes) { + assert.deepEqual(node.promptModules, node.featureRefs); + const owned = new Set(node.gradingChecks.map(check => check.id)); + for (const reference of node.featureRefs) { + const pack = packByRef.get(reference); + assert(pack, `${node.id} references missing ${reference}`); + assert.equal(pack.moduleType, 'feature'); + assert(packChecks(pack).every(key => owned.has(key)), + `${node.id} must own every check selected by ${reference}`); + } + } + + const recipe = compileRecipeFile(join(trackRoot, 'composition', 'recipes', + 'progression-catalog.json'), { + trackRoot, + }); + const actual = new Map(definition.nodes.flatMap(node => node.gradingChecks) + .map(check => [check.id, check.points])); + for (const check of recipe.checks.filter(item => item.points > 0)) { + if (check.stableKey.startsWith('ecommerce.feature.catalog.catalog.')) continue; + assert.equal(actual.get(check.stableKey), check.points, + `progression must preserve ${check.stableKey} from the L3 candidate`); + } + assert.deepEqual([...actual.keys()].filter(key => key.startsWith('ecommerce.feature.catalog.')).sort(), + ['ecommerce.feature.catalog.catalog-ranking.2b', + 'ecommerce.feature.catalog.catalog-search.2d', + 'ecommerce.feature.catalog.catalog-values.2a']); +}); + +test('every progression feature is a whole module and every direct graph edge is required', () => { + const packByRef = new Map<string, CompiledPackDefinition>(readdirSync(packRoot) + .filter(name => name.endsWith('.json')) + .map(name => { + const pack = compilePackDefinition(readJson(join(packRoot, name)), { source: name }); + return [pack.id, pack]; + })); + const definition = compileProgressionDefinitionFile(definitionPath, { trackRoot }); + assert(definition.nodes.every(node => Object.keys(node.dependencyReasons).length + === node.dependencies.length)); + assert(definition.questlines.every(questline => + definition.nodes.some(node => node.questline === questline.id))); + const nodeById = new Map(definition.nodes.map(node => [node.id, node])); + const ownerByRef = new Map(definition.nodes + .flatMap(node => node.featureRefs.map(reference => [reference, node.id]))); + const ancestors = (nodeId: string): Set<string> => { + const found = new Set<string>(); + const visit = (id: string): void => requiredNode(nodeById, id).dependencies.forEach(parent => { + if (found.has(parent)) return; + found.add(parent); + visit(parent); + }); + visit(nodeId); + return found; + }; + + for (const node of definition.nodes) { + const featurePacks = node.featureRefs.map(reference => packByRef.get(reference)); + const requiredPacks = featurePacks.map((pack, index) => { + assert(pack, `${node.id} references missing ${node.featureRefs[index] ?? '<unknown>'}`); + return pack; + }); + const requiredOwners = [...new Set(requiredPacks.flatMap(pack => pack.requiresPacks) + .map(reference => ownerByRef.get(reference)) + .filter((owner): owner is string => typeof owner === 'string' && owner !== node.id))]; + const directRequiredOwners = requiredOwners.filter(owner => !requiredOwners.some(other => + other !== owner && ancestors(other).has(owner))).sort(); + assert.deepEqual(node.dependencies, directRequiredOwners, + `${node.id} graph parents must be the minimal feature dependency set`); + } +}); + +test('one progression catalog binds every node and selects only current work', () => { + const definition = compileProgressionDefinitionFile(definitionPath, { trackRoot }); + const input = compileFeatureCatalogInput(definition); + const track = loadTrack('ecommerce'); + const bindings = [1, 2, 3, 4, 5, 6].map(level => ({ + level, + binding: resolveRecipeRelease(track, level, 'ecommerce.progression-catalog'), + })); + validateProgressionRecipeBindings(input, bindings); + assert.equal(new Set(bindings.map(item => item.binding.release.contentSha256)).size, 1); + + const allFeatureIds = new Set(definition.nodes.flatMap(node => node.featureRefs)); + const firstBinding = bindings[0]; + const secondBinding = bindings[1]; + assert(firstBinding && secondBinding, 'progression must bind levels 1 and 2'); + const release = firstBinding.binding.release; + assert([...allFeatureIds].every(id => release.components.packs.some(pack => pack.id === id))); + assert(definition.nodes.flatMap(node => node.gradingChecks) + .every(check => release.checkCatalog.some(item => item.stableKey === check.id))); + + const policy = compileDependencyPolicyInput({ selection: 'feature', + budget: { perFeature: 3 } }, input); + let state = progressionEngine.initialize(dependencyRuntimeDefinition(input, policy)); + const first = resolveProgressionRecipeAction(firstBinding.binding, state); + if (first.action.type === 'terminal') throw new Error('L1 must produce work'); + assert('agent' in first && 'grader' in first, 'L1 must have agent and grader selections'); + const firstFeatures = definition.nodes.filter(node => node.level === 1) + .flatMap(node => node.featureRefs).sort(); + assert.equal(first.agent.request.task.mode, 'fresh'); + assert.deepEqual(first.agent.request.selection.requested.features, firstFeatures); + assert.deepEqual([...first.grader.checkKeys].sort(), + gradingCheckIds(first.action).sort()); + assert([...allFeatureIds].filter(id => !firstFeatures.includes(id)) + .every(id => !first.agent.request.selection.promptPacks.includes(id))); + + const grading = progressionEngine.gradingSelection(state); + state = progressionEngine.recordResult(state, { + attemptId: 'level-1-pass', + outcome: 'conclusive', + nodes: grading.nodeIds.map(nodeId => ({ + id: nodeId, + checks: grading.checks.filter(check => check.nodeId === nodeId) + .map(check => ({ id: check.id, outcome: 'pass' })), + })), + }); + const second = resolveProgressionRecipeAction(secondBinding.binding, state); + if (second.action.type === 'terminal') throw new Error('L2 must produce work'); + assert('agent' in second && 'grader' in second, 'L2 must have agent and grader selections'); + const secondFeatures = definition.nodes.filter(node => node.level === 2) + .flatMap(node => node.featureRefs).sort(); + assert.equal(second.action.level, 2); + assert.equal(second.agent.request.task.mode, 'upgrade'); + assert.deepEqual(second.agent.request.selection.requested.features, secondFeatures); + assert.deepEqual([...second.grader.checkKeys].sort(), + gradingCheckIds(second.action).sort()); + const futureFeatures = definition.nodes.filter(node => node.level > 2) + .flatMap(node => node.featureRefs).map(reference => reference.slice(0, reference.lastIndexOf('@'))); + assert(futureFeatures.every(id => !second.agent.request.selection.promptPacks.includes(id))); + + const byLevel = new Map(bindings.map(item => [item.level, item.binding])); + const priorIds = priorProgressionContractIds(state, byLevel, secondBinding.binding); + assert.deepEqual(priorIds, [...first.agent.task.contractIds].sort()); + const retained = resolveProgressionRecipeAction(secondBinding.binding, state, priorIds); + assert('agent' in retained); + assert.deepEqual(retained.grader, second.grader, 'disclosure must not change grading'); + assert.equal(retained.agent.task.requirementText, second.agent.task.requirementText, + 'feature requests remain incremental'); + assert.notEqual(retained.agent.task.sha256, second.agent.task.sha256); + assert.match(retained.agent.task.contractText, /interfaces from these earlier requests/); + assert.deepEqual(new Set(retained.agent.task.contractIds), + new Set([...first.agent.task.contractIds, ...second.agent.task.contractIds])); + assert.equal(retained.agent.task.contractIds.length, new Set(retained.agent.task.contractIds).size); + assert.equal(resolveBoundRecipeTaskRequest(secondBinding.binding, retained.agent.request).task.sha256, + retained.agent.task.sha256, 'the coding CLI reconstructs the exact retained prompt'); + assert.deepEqual(priorProgressionContractIds(progressionEngine.replay(state.definition, state.events), + byLevel, secondBinding.binding), priorIds, 'resume retains the issued scope'); + const changedBinding = structuredClone(secondBinding.binding); + changedBinding.plan.recipe.task.contracts.find(item => item.id === priorIds[0])!.text += '\nchanged'; + assert.throws(() => priorProgressionContractIds(state, byLevel, changedBinding), /prior issued contract.*changed/); + + const secondGrading = progressionEngine.gradingSelection(state); + state = progressionEngine.recordResult(state, { + attemptId: 'level-2-account-regression', + outcome: 'conclusive', + nodes: secondGrading.nodeIds.map(nodeId => ({ + id: nodeId, + checks: secondGrading.checks.filter(check => check.nodeId === nodeId) + .map(check => ({ id: check.id, outcome: nodeId === 'accounts' ? 'fail' : 'pass' })), + })), + }); + const next = resolveProgressionRecipeAction(firstBinding.binding, state); + if (next.action.type === 'terminal') throw new Error('the regression must produce repair work'); + assert('agent' in next, 'the repair must have an agent selection'); + assert.equal(next.action.type, 'repair'); + assert.equal(next.action.level, 1); + assert.deepEqual(actionPromptNodeIds(next.action), ['accounts']); + assert.equal(next.agent.request.task.mode, 'upgrade'); + assert.deepEqual(priorProgressionContractIds(state, byLevel, firstBinding.binding), + [...new Set([...first.agent.task.contractIds, ...second.agent.task.contractIds])].sort(), + 'a regression must not erase previously issued contracts'); + assert.deepEqual(next.agent.request.selection.requested.features, + definition.nodes.filter(node => actionPromptNodeIds(next.action).includes(node.id)) + .flatMap(node => node.featureRefs) + .sort()); + const repairTarget = resolveProgressionRepairTarget(firstBinding.binding, state); + assert.deepEqual(repairTarget.request.selection.requested.features, + next.agent.request.selection.requested.features); + assert(repairTarget.checkKeys.length > 0); + assert(repairTarget.checkKeys.every(check => next.grader.checkKeys.includes(check))); + assert(repairTarget.checkKeys.length < next.grader.checkKeys.length); +}); + +test('all-at-once composes every selected feature into one fresh request', () => { + const definition = compileProgressionDefinitionFile(definitionPath, { trackRoot }); + const catalog = compileFeatureCatalogInput(definition); + const policy = compileDependencyPolicyInput({ selection: 'batch', budget: { total: 1 } }, + catalog, { workSelection: 'all-at-once' }); + const state = progressionEngine.initialize(dependencyRuntimeDefinition(catalog, policy)); + const binding = resolveRecipeRelease(loadTrack('ecommerce'), 6, + 'ecommerce.progression-catalog'); + const selected = resolveProgressionRecipeAction(binding, state); + if (selected.action.type === 'terminal' || !('agent' in selected)) { + throw new Error('all-at-once must produce one build request'); + } + const featureIds = definition.nodes.flatMap(node => node.featureRefs) + .sort(); + assert.equal(selected.action.level, 6); + assert.equal(selected.agent.request.task.mode, 'fresh'); + assert.deepEqual(selected.agent.request.selection.requested.features, featureIds); + assert(selected.agent.task.requirementIds.includes('ecommerce.progression.managed-support')); +}); + +test('every feature and grading check binds to the progression recipe', () => { + const definition = compileProgressionDefinitionFile(definitionPath, { trackRoot }); + const recipePath = join(trackRoot, 'composition', 'recipes', 'progression-catalog.json'); + const plan = compileRecipeFile(recipePath, { trackRoot }); + const release = buildRecipeRelease(recipePath, { trackRoot }); + const binding: RecipeBinding = { + alias: 'ecommerce.progression-catalog', + selection: { path: 'composition/dependency.json', sha256: '0'.repeat(64) }, + recipePath, plan, release, execution: [], + }; + const levels = [...new Set(definition.nodes.map(node => node.level))].sort((a, b) => a - b); + validateProgressionRecipeBindings(compileFeatureCatalogInput(definition), + levels.map(level => ({ level, binding })), { levels }); + + const owned = new Set(definition.nodes.flatMap(node => + node.gradingChecks.map(check => check.id))); + const unowned = plan.checks.filter(check => !owned.has(check.stableKey)); + assert.deepEqual(unowned.map(check => [check.stableKey, check.points]), [ + ['ecommerce.spec.external-data-sync.external-stock.901b', 0], + ['ecommerce.spec.concurrency-safety.restock-race.202-control', 0], + ], 'only zero-point controls may sit outside feature scoring'); +}); + +test('every feature pack declares a bounded, testable product contract', () => { + const definition = compileProgressionDefinitionFile(definitionPath, { trackRoot }); + const plan = compileRecipeFile(join(trackRoot, 'composition', 'recipes', + 'progression-catalog.json'), { trackRoot }); + const featurePacks = plan.packs.filter(pack => pack.moduleType === 'feature'); + assert.equal(featurePacks.length, definition.nodes.length, + 'each graph node must own one feature pack'); + for (const pack of featurePacks) { + assert(pack.capabilities.length > 0, `${pack.id} must declare its required capabilities`); + assert(pack.evidence.length > 0, `${pack.id} must declare its required evidence`); + assert.equal(pack.budget.status, 'bounded', `${pack.id} must have a bounded runtime`); + assert(pack.budget.maxRuntimeMs > 0, `${pack.id} must have a positive runtime limit`); + } +}); + +test('cross-feature grading requirements stay separate from product dependencies', () => { + // A check may need a feature outside its node's ancestry; that is declared + // grading scope, never a product edge, and the list is held explicit so it + // cannot grow unnoticed. + const definition = compileProgressionDefinitionFile(definitionPath, { trackRoot }); + const nodes = new Map(definition.nodes.map(node => [node.id, node])); + const ownerByFeature = new Map(definition.nodes.flatMap(node => node.featureRefs + .map(ref => [ref, node.id] as const))); + const ancestors = (nodeId: string, found = new Set([nodeId])): Set<string> => { + for (const dependency of nodes.get(nodeId)?.dependencies ?? []) { + if (!found.has(dependency)) { + found.add(dependency); + ancestors(dependency, found); + } + } + return found; + }; + const deferred = definition.nodes.flatMap(node => { + const productAncestors = ancestors(node.id); + return node.gradingChecks.flatMap(check => (check.requiresFeatures ?? []).flatMap(feature => { + const owner = ownerByFeature.get(feature); + return owner && !productAncestors.has(owner) ? [`${node.id}:${check.id}:${owner}`] : []; + })); + }).sort(); + assert.deepEqual(deferred, [ + 'catalog-management:ecommerce.spec.transactional-integrity.catalog-volume.622c:faceted-search', + 'catalog-management:ecommerce.spec.transactional-integrity.catalog-volume.622c:purchasing', + 'faceted-search:ecommerce.spec.search-ordering.search-ordering.402b:purchasing', + 'inventory-dashboard:ecommerce.spec.live-state.inventory-dashboard.5a:purchasing', + 'order-cancellation:ecommerce.inventory-operations.stock-conservation.202f:cart', + 'order-cancellation:ecommerce.inventory-operations.stock-conservation.202f:checkout', + 'order-cancellation:ecommerce.inventory-operations.stock-conservation.202f:stock-transfers', + 'price-history:ecommerce.returns-pricing.price-history.4a:purchasing', + 'price-history:ecommerce.returns-pricing.price-history.4c:checkout', + 'sales-dashboard:ecommerce.spec.transactional-integrity.books-balance.107a:warehouse-admin', + 'sales-dashboard:ecommerce.spec.transactional-integrity.books-balance.107b:warehouse-admin', + 'stock-transfers:ecommerce.inventory-operations.stock-conservation.202d:purchasing', + 'warehouse-admin:ecommerce.spec.access-control.warehouse-write-boundary.103b:accounts', + ]); +}); + +interface AuthoredNode { + id: string; + featureRefs: string[]; + gradingGroups: string[]; + dependencies: Array<{ id: string; reason: string }>; +} + +const authoredDefinition = (): { nodes: AuthoredNode[] } => + JSON.parse(readFileSync(definitionPath, 'utf8')) as { nodes: AuthoredNode[] }; + +test('the authored graph rejects test-driven edges, missing packs, and stray ownership', async t => { + const cases: Array<[string, (input: { nodes: AuthoredNode[] }) => void, RegExp]> = [ + ['a test-driven product edge', input => { + const dashboard = input.nodes.find(node => node.id === 'inventory-dashboard'); + assert.ok(dashboard); + dashboard.dependencies.push({ id: 'purchasing', reason: 'A grading scenario uses purchase data.' }); + }, /inventory-dashboard\.dependencies: must equal minimal product dependencies: warehouse-admin/], + ['a missing product dependency', input => { + const purchasing = input.nodes.find(node => node.id === 'purchasing'); + assert.ok(purchasing); + purchasing.dependencies = purchasing.dependencies.filter(dependency => dependency.id !== 'accounts'); + }, /purchasing\.dependencies: must equal minimal product dependencies: accounts, catalog/], + ['an unnecessary product dependency', input => { + const reviews = input.nodes.find(node => node.id === 'reviews'); + assert.ok(reviews); + reviews.dependencies.push({ id: 'accounts', reason: 'A grading scenario uses a customer.' }); + }, /reviews\.dependencies: must equal minimal product dependencies: purchasing/], + ['a missing feature pack', input => { + input.nodes[0]!.featureRefs = ['ecommerce.missing']; + }, /missing pack/], + ['a missing grading group', input => { + input.nodes[0]!.gradingGroups[0] = 'ecommerce.feature.accounts#missing'; + }, /missing group/], + ['feature checks omitted from their owner', input => { + input.nodes[0]!.gradingGroups = input.nodes[0]!.gradingGroups + .filter(reference => !reference.endsWith('#account-create')); + }, /must own feature group/], + ['a group owned twice', input => { + input.nodes[1]!.gradingGroups.push(input.nodes[0]!.gradingGroups[0]!); + }, /already owned/], + ]; + for (const [name, mutate, expected] of cases) { + await t.test(name, () => { + const input = authoredDefinition(); + mutate(input); + assert.throws(() => compileProgressionDefinition(input, { trackRoot }), expected); + }); + } +}); + +function requiredNode( + nodes: ReadonlyMap<string, CompiledProgressionNode>, + nodeId: string, +): CompiledProgressionNode { + const node = nodes.get(nodeId); + if (!node) throw new Error(`progression node ${nodeId} is required`); + return node; +} + +function gradingCheckIds(action: ProgressionWorkAction): string[] { + if (!isRecord(action.grading) || !Array.isArray(action.grading.checks)) { + throw new Error('progression work must have grading checks'); + } + return action.grading.checks.map((check, index) => { + if (!isRecord(check) || typeof check.id !== 'string') { + throw new Error(`progression grading check ${index} must have an id`); + } + return check.id; + }); +} + +function actionPromptNodeIds(action: ProgressionWorkAction): string[] { + if (!isRecord(action.prompt) || !Array.isArray(action.prompt.nodeIds) + || !action.prompt.nodeIds.every(id => typeof id === 'string')) { + throw new Error('progression work must have prompt node ids'); + } + return action.prompt.nodeIds; +} + +function isRecord(value: unknown): value is Record<string, unknown> { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} diff --git a/tools/stack-bench/tests/element-selector.test.ts b/tools/stack-bench/tests/element-selector.test.ts new file mode 100644 index 00000000000..adae0eec9cc --- /dev/null +++ b/tools/stack-bench/tests/element-selector.test.ts @@ -0,0 +1,10 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { stableElementSelector } from '../src/actions/element-selector.js'; + +test('stable selectors support application interface names', () => { + assert.equal(stableElementSelector('account-name'), + '[data-role="account-name"],#account-name'); + assert.throws(() => stableElementSelector('account name'), /invalid application interface name/); +}); diff --git a/tools/stack-bench/tests/engine-identity.contract.ts b/tools/stack-bench/tests/engine-identity.contract.ts new file mode 100644 index 00000000000..d23bbef8069 --- /dev/null +++ b/tools/stack-bench/tests/engine-identity.contract.ts @@ -0,0 +1,65 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { cpSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, relative } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import test from 'node:test'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; + +function currentIdentity(root: string): unknown { + const artifactsUrl = pathToFileURL(join(root, 'dist', 'src', 'evidence', 'artifacts.js')).href; + const result = spawnSync(process.execPath, ['--input-type=module', '--eval', + `import { currentEngineIdentity } from ${JSON.stringify(artifactsUrl)}; console.log(JSON.stringify(currentEngineIdentity()));`], + { cwd: root, encoding: 'utf8' }); + assert.equal(result.status, 0, result.stderr); + return JSON.parse(result.stdout ?? ''); +} + +function writableEngineRoot(): { temp: string; root: string } { + const temp = mkdtempSync(join(tmpdir(), 'stack-bench-engine-')); + const root = join(temp, 'stack-bench'); + const excluded = new Set(['archive', 'local-notes', 'media', 'node_modules', + 'qualification-evidence', 'reference-apps', 'results', 'tests', 'tracks', 'transcripts']); + cpSync(STACK_BENCH_ROOT, root, { recursive: true, filter: source => { + if (source === STACK_BENCH_ROOT) return true; + return !excluded.has(relative(STACK_BENCH_ROOT, source).split(/[\\/]/)[0] ?? ''); + } }); + mkdirSync(join(root, 'node_modules'), { recursive: true }); + cpSync(join(STACK_BENCH_ROOT, 'node_modules', 'zod'), join(root, 'node_modules', 'zod'), + { recursive: true }); + return { temp, root }; +} + +test('engine identity excludes generated and local-only files', () => { + const copy = writableEngineRoot(); + const before = currentIdentity(copy.root); + try { + const root = copy.root; + const runtime = mkdtempSync(join(root, '.engine-runtime-')); + writeFileSync(join(runtime, 'generated.mjs'), 'generated runtime state\n'); + writeFileSync(join(runtime, 'session.json'), '{"turns":999}\n'); + const nested = join(root, 'grader', '.generated'); + mkdirSync(nested, { recursive: true }); + writeFileSync(join(nested, 'candidate.mjs'), 'generated candidate\n'); + writeFileSync(join(root, 'grader', '.generated-report.json'), '{"generated":true}\n'); + + for (const directory of ['archive', 'local-notes', 'media', 'qualification-evidence', + 'transcripts']) { + const path = join(root, directory); + mkdirSync(path, { recursive: true }); + writeFileSync(join(path, 'generated.json'), '{"generated":true}\n'); + } + writeFileSync(join(root, 'dependency-manifest.json'), '{"schemaVersion":1}\n'); + + const path = join(root, 'grader', 'node_modules', 'generated-package'); + mkdirSync(path, { recursive: true }); + writeFileSync(join(path, 'index.js'), 'installed dependency\n'); + writeFileSync(join(path, 'package.json'), '{"name":"generated-package"}\n'); + + assert.deepEqual(currentIdentity(root), before); + } finally { + rmSync(copy.temp, { recursive: true, force: true }); + } +}); diff --git a/tools/stack-bench/tests/evidence-timing.test.ts b/tools/stack-bench/tests/evidence-timing.test.ts new file mode 100644 index 00000000000..9fbaaa1b7f4 --- /dev/null +++ b/tools/stack-bench/tests/evidence-timing.test.ts @@ -0,0 +1,13 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { evidenceNowMs } from '../src/evidence/evidence-timing.js'; + +test('evidence timestamps are epoch-shaped integers and never move backwards', () => { + const first = evidenceNowMs(); + const second = evidenceNowMs(); + + assert(Number.isInteger(first)); + assert(first > 1_000_000_000_000); + assert(second >= first); +}); diff --git a/tools/stack-bench/tests/execution-jobs.integration.ts b/tools/stack-bench/tests/execution-jobs.integration.ts new file mode 100644 index 00000000000..68e7c35f684 --- /dev/null +++ b/tools/stack-bench/tests/execution-jobs.integration.ts @@ -0,0 +1,106 @@ +import assert from 'node:assert/strict'; +import { spawn, type ChildProcess } from 'node:child_process'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { setTimeout as delay } from 'node:timers/promises'; +import { cancelExecutionJob, readExecutionJob, submitExecutionJob, workExecutionJob } from '../src/campaigns/execution-jobs.js'; +import { readCampaignState } from '../src/campaigns/campaign-scheduler.js'; +import { compiledEntrypoint, STACK_BENCH_ROOT } from '../src/package-root.js'; + +test('the worker dispatches nine attempts, cancels a job, drains on SIGTERM, and resumes its queue', { + skip: process.platform !== 'linux' ? 'Uses the isolated Linux model-free runner' : false, + timeout: 300_000, +}, async () => { + const root = mkdtempSync(join(tmpdir(), 'execution-worker-live-')); + const children: ChildProcess[] = []; + let output = ''; + const waitFor = async (condition: () => boolean) => { + const deadline = Date.now() + 120_000; + while (!condition()) { + assert(Date.now() < deadline, `worker condition timed out\n${output}`); + await delay(50); + } + }; + const start = () => { + const child = spawn(process.execPath, [compiledEntrypoint('commands', 'job-cli.js'), + 'worker', '--host', 'local-test', '--concurrency', '2', '--results', root], { + detached: true, stdio: ['ignore', 'pipe', 'pipe'], + env: { PATH: process.env.PATH, HOME: root, + PLAYWRIGHT_BROWSERS_PATH: process.env.PLAYWRIGHT_BROWSERS_PATH }, + }); + children.push(child); + child.stdout!.on('data', chunk => { output += chunk; }); + child.stderr!.on('data', chunk => { output += chunk; }); + return child; + }; + const stop = async (child: ChildProcess) => { + await waitFor(() => child.exitCode !== null || child.signalCode !== null); + assert.equal(child.exitCode, 0, output); + assert.equal(child.signalCode, null, output); + }; + try { + mkdirSync(join(root, 'plans')); + const fixture = JSON.parse(readFileSync(join(STACK_BENCH_ROOT, + 'tests/fixtures/dependency-model-free-campaign.json'), 'utf8')); + fixture.mode.retainPriorContracts = false; + writeFileSync(join(root, 'plans/single.json'), JSON.stringify(fixture)); + fixture.repetitions = fixture.parallelism = 9; + writeFileSync(join(root, 'plans/nine.json'), JSON.stringify(fixture)); + const submit = (key: string, planFile: string) => submitExecutionJob(root, + { key, planFile, hostId: 'local-test' }); + const main = submit('nine', 'nine.json'); + const cancelled = submit('cancel', 'nine.json'); + const first = start(); + await waitFor(() => [main, cancelled].every(job => { + const status = readExecutionJob(root, job.id); + assert(['queued', 'running'].includes(status.status), `${status.status}: ${status.error}\n${output}`); + if (status.status !== 'running') return false; + if (!existsSync(join(status.campaignDirectory, 'state.json'))) return false; + return readCampaignState(status.campaignDirectory).state.attempts + .filter(attempt => attempt.status === 'running').length === 9; + })); + // SIGTERM stops admission. Explicit cancellation stops only the selected job. + first.kill('SIGTERM'); + await delay(100); + const queued = submit('restart', 'single.json'); + cancelExecutionJob(root, cancelled.id); + await stop(first); + const cancelledResult = readExecutionJob(root, cancelled.id); + assert.equal(cancelledResult.status, 'cancelled', output); + assert.equal(readCampaignState(cancelledResult.campaignDirectory).state.summary.running, 0); + assert.equal(readExecutionJob(root, queued.id).status, 'queued', output); + const completed = readExecutionJob(root, main.id); + assert.equal(completed.status, 'completed', output); + const { state } = readCampaignState(completed.campaignDirectory); + assert.equal(state.attempts.length, 9); + const evidence = state.attempts.map(attempt => { + assert.equal(attempt.executions.length, 1); + assert.equal(attempt.executions[0]!.status, 'completed'); + const path = join(completed.campaignDirectory, attempt.executions[0]!.output, 'run.json'); + const content = readFileSync(path, 'utf8'); + const run = JSON.parse(content); + assert.equal(run.payload.backend, 'stub'); + assert.equal(run.payload.outcome.kind, 'app_failure'); + assert.equal(run.payload.progressionStatus.phase, 'terminal'); + return { path, content }; + }); + const restarted = start(); + await waitFor(() => readExecutionJob(root, queued.id).status === 'completed'); + restarted.kill('SIGTERM'); + await stop(restarted); + assert.equal((await workExecutionJob(root, main.id, 'local-test')).status, 'completed'); + for (const { path, content } of evidence) assert.equal(readFileSync(path, 'utf8'), content); + assert.equal(readExecutionJob(root, cancelled.id).status, 'cancelled'); + } finally { + // Each child owns its process group. Never terminate another worker's children. + for (const child of children) { + if (child.pid) { + try { process.kill(-child.pid, 'SIGKILL'); } + catch (error) { assert.equal((error as NodeJS.ErrnoException).code, 'ESRCH'); } + } + } + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/tools/stack-bench/tests/execution-jobs.test.ts b/tools/stack-bench/tests/execution-jobs.test.ts new file mode 100644 index 00000000000..da3787a34b1 --- /dev/null +++ b/tools/stack-bench/tests/execution-jobs.test.ts @@ -0,0 +1,333 @@ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import fs from 'node:fs'; +import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { syncBuiltinESMExports } from 'node:module'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { setTimeout as delay } from 'node:timers/promises'; +import { cancelExecutionJob, listExecutionJobs, readExecutionJob, submitExecutionJob, + workExecutionJob } from '../src/campaigns/execution-jobs.js'; +import { controllerInstance } from '../src/campaigns/campaign-lock.js'; +import type { executeCampaign, inspectCampaign } from '../src/campaigns/campaign-runner.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { jobCommand, resumeExecutionJob, runJobCli } from '../commands/job-cli.js'; + +test('job submission, exclusive workers, cancellation and retained failures use durable records', async () => { + const root = mkdtempSync(join(tmpdir(), 'execution-jobs-')); + try { + mkdirSync(join(root, 'plans')); + copyFileSync(join(STACK_BENCH_ROOT, 'tests/fixtures/campaign.deterministic.json'), join(root, 'plans/test.json')); + const request = { key: 'request-a', planFile: 'test.json', hostId: 'worker-a', credentials: {} }; + const job = submitExecutionJob(root, request); + assert.deepEqual(submitExecutionJob(root, request), job, 'retry returns identical submission'); + assert.throws(() => submitExecutionJob(root, { ...request, hostId: 'worker-b' }), /different submission/); + assert.throws(() => submitExecutionJob(root, { ...request, key: '../escape' })); + assert.equal(listExecutionJobs(root).jobs.length, 1); + await assert.rejects(workExecutionJob(root, job.id, 'worker-b'), /another host/); + let finish!: () => void; + const running = new Promise<void>(resolve => { finish = resolve; }); + let invocations = 0; + const execute = (async (_plan, _output, options) => { + invocations++; + assert.equal(options?.mode, 'model-free-trial'); + assert.equal(options?.capacityPolicy, 'wait'); + assert.deepEqual(options?.executionCredentials, {}); + await running; + return { summary: { completed: 9, running: 0, pending: 0 } }; + }) as typeof executeCampaign; + const first = workExecutionJob(root, job.id, 'worker-a', { execute }); + assert.equal((await workExecutionJob(root, job.id, 'worker-a', { execute })).status, 'running'); + finish(); + assert.equal((await first).status, 'completed'); + await workExecutionJob(root, job.id, 'worker-a', { execute }); + assert.equal(invocations, 1, 'duplicate claims never invoke the runner twice'); + + const cancelled = submitExecutionJob(root, { ...request, key: 'request-b' }); + cancelExecutionJob(root, cancelled.id); + assert.equal((await workExecutionJob(root, cancelled.id, 'worker-a', { execute })).status, 'cancelled'); + assert.equal(invocations, 1); + + const failed = submitExecutionJob(root, { ...request, key: 'request-c' }); + const failure = (async () => { throw new Error('owned cleanup is incomplete'); }) as typeof executeCampaign; + const outcome = await workExecutionJob(root, failed.id, 'worker-a', { execute: failure }); + assert.equal(outcome.status, 'failed'); + assert.match(outcome.error!, /cleanup/); + assert.equal((await workExecutionJob(root, failed.id, 'worker-a', { execute })).status, 'failed'); + assert.equal(invocations, 1, 'failed jobs require explicit recovery; no implicit retry'); + const invalid = submitExecutionJob(root, { ...request, key: 'request-invalid' }); + const invalidResult = (async () => ({ summary: { pending: 0, running: 0, completed: 0, invalid: 1 } })) as unknown as typeof executeCampaign; + assert.equal((await workExecutionJob(root, invalid.id, 'worker-a', { execute: invalidResult })).status, 'failed'); + const page = listExecutionJobs(root, { limit: 2 }); + assert.equal(page.jobs.length, 2); + assert.equal(listExecutionJobs(root, { after: page.next! }).jobs.length, 2); + + const modified = submitExecutionJob(root, { ...request, key: 'request-d' }); + const path = join(root, 'jobs', modified.id, 'plan.json'); + const plan = JSON.parse(readFileSync(path, 'utf8')); plan.title = 'changed'; + writeFileSync(path, JSON.stringify(plan)); + await assert.rejects(workExecutionJob(root, modified.id, 'worker-a', { execute }), /identity changed/); + assert.equal(readExecutionJob(root, modified.id).status, 'queued'); + + const stopping = submitExecutionJob(root, { ...request, key: 'request-stop' }); + let activeCalls = 0; + const untilCancelled = (async (_plan, _directory, options) => { + activeCalls++; + await new Promise<void>(resolve => options!.signal!.addEventListener('abort', () => resolve(), { once: true })); + return { summary: { pending: 1, running: 0, invalid: 0 } }; + }) as typeof executeCampaign; + const active = workExecutionJob(root, stopping.id, 'worker-a', { execute: untilCancelled }); + cancelExecutionJob(root, stopping.id); + assert.equal(readExecutionJob(root, stopping.id).cancellationRequested, true); + assert.equal((await active).status, 'cancelled'); + assert.equal((await workExecutionJob(root, stopping.id, 'worker-a', { execute: untilCancelled })).status, 'cancelled'); + assert.equal(activeCalls, 1, 'cancellation retains ownership and cannot restart the worker'); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('job reads tolerate a worker publishing its claim and result between file reads', context => { + const root = mkdtempSync(join(tmpdir(), 'execution-job-read-')); + try { + mkdirSync(join(root, 'plans')); + copyFileSync(join(STACK_BENCH_ROOT, 'tests/fixtures/campaign.deterministic.json'), join(root, 'plans/test.json')); + const job = submitExecutionJob(root, { key: 'read-race', planFile: 'test.json' }); + const directory = join(root, 'jobs', job.id); + const claimPath = join(directory, 'claim.json'); + const resultPath = join(directory, 'result.json'); + const owner = { hostId: 'worker-a', token: 'a'.repeat(64) }; + const exists = fs.existsSync; + let published = false; + context.mock.method(fs, 'existsSync', (path: Parameters<typeof fs.existsSync>[0]) => { + const present = exists(path); + if (path === claimPath && !published) { + published = true; + writeFileSync(claimPath, JSON.stringify({ ...owner, startedAt: new Date().toISOString() })); + writeFileSync(resultPath, JSON.stringify({ ...owner, status: 'completed', completedAt: new Date().toISOString() })); + } + return present; + }); + syncBuiltinESMExports(); + assert.doesNotThrow(() => readExecutionJob(root, job.id)); + assert.equal(published, true); + assert.equal(readExecutionJob(root, job.id).status, 'completed'); + writeFileSync(resultPath, JSON.stringify({ ...owner, token: 'b'.repeat(64), status: 'completed', completedAt: new Date().toISOString() })); + assert.throws(() => readExecutionJob(root, job.id), /does not belong to its worker claim/); + } finally { + context.mock.restoreAll(); + syncBuiltinESMExports(); + rmSync(root, { recursive: true, force: true }); + } +}); + +test('job resume continues the campaign with the job saved accounts and capacity policy', async () => { + const root = mkdtempSync(join(tmpdir(), 'execution-job-resume-')); + try { + mkdirSync(join(root, 'plans')); + copyFileSync(join(STACK_BENCH_ROOT, 'tests/fixtures/dependency-model-free-campaign.json'), join(root, 'plans/reference.json')); + const credentials = { adapters: { deterministic: 'team-a' } }; + const job = submitExecutionJob(root, { key: 'resume-job', planFile: 'reference.json', credentials }); + await assert.rejects(resumeExecutionJob(root, job.id), /failed or reconciled interrupted job/); + const planFile = join(root, 'jobs', job.id, 'plan.json'); + const directory = join(root, 'campaigns', `job-${job.id}`); + const failed = (async () => { throw new Error('interrupted'); }) as typeof executeCampaign; + assert.equal((await workExecutionJob(root, job.id, 'worker-a', { execute: failed })).status, 'failed'); + let status = 'completed'; + const inspect = ((path: string) => { + assert.equal(path, directory); + return { plan: { contentSha256: job.planSha256, definition: { mode: { id: 'dependency' } } }, + state: { status, attempts: [{ executions: [{}] }] } }; + }) as unknown as typeof inspectCampaign; + let calls = 0; + let finish!: () => void; + const running = new Promise<void>(resolve => { finish = resolve; }); + const execute = (async (path, output, options) => { + calls++; + assert.equal(path, planFile); + assert.equal(output, directory); + assert.equal(options?.mode, 'model-free-trial'); + assert.equal(options?.capacityPolicy, 'wait'); + assert.deepEqual(options?.executionCredentials, credentials); + await running; + return { status: 'completed', summary: { completed: 1, pending: 0, running: 0, invalid: 0 } }; + }) as typeof executeCampaign; + await assert.rejects(resumeExecutionJob(root, job.id, { execute, inspect }), /scheduled work/); + assert.equal(calls, 0); + status = 'prepared'; + const resumed = resumeExecutionJob(root, job.id, { execute, inspect }); + try { + assert.equal(readExecutionJob(root, job.id).status, 'running'); + await assert.rejects(resumeExecutionJob(root, job.id, { execute, inspect }), /live worker/); + } finally { finish(); } + assert.equal((await resumed).status, 'completed'); + assert.equal(readExecutionJob(root, job.id).status, 'completed'); + assert.match(readFileSync(join(root, 'jobs', job.id, 'result.json'), 'utf8'), /interrupted/, + 'prior failure evidence is retained'); + assert.equal(calls, 1); + await assert.rejects(resumeExecutionJob(root, job.id, { execute, inspect }), /failed or reconciled interrupted job/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('cancel reaches a resumed job and preserves its prior failure', async () => { + const root = mkdtempSync(join(tmpdir(), 'execution-job-resume-cancel-')); + try { + mkdirSync(join(root, 'plans')); + copyFileSync(join(STACK_BENCH_ROOT, 'tests/fixtures/dependency-model-free-campaign.json'), join(root, 'plans/reference.json')); + const job = submitExecutionJob(root, { key: 'resume-cancel', planFile: 'reference.json' }); + const failed = (async () => { throw new Error('first failure'); }) as typeof executeCampaign; + await workExecutionJob(root, job.id, 'worker-a', { execute: failed }); + const inspect = (() => ({ plan: { contentSha256: job.planSha256, + definition: { mode: { id: 'dependency' } } }, + state: { status: 'prepared', attempts: [{ executions: [{}] }] } })) as unknown as typeof inspectCampaign; + let abortSignal: AbortSignal | null | undefined; + const execute = (async (_path, _output, options) => { + abortSignal = options?.signal; + await new Promise<void>(resolve => abortSignal!.addEventListener('abort', () => resolve(), { once: true })); + return { status: 'prepared', summary: { completed: 0, pending: 1, running: 0, invalid: 0 } }; + }) as typeof executeCampaign; + const active = resumeExecutionJob(root, job.id, { execute, inspect }); + assert.equal(readExecutionJob(root, job.id).status, 'running'); + cancelExecutionJob(root, job.id); + assert.equal(readExecutionJob(root, job.id).cancellationRequested, true); + assert.equal((await active).status, 'cancelled'); + assert.equal(abortSignal?.aborted, true); + assert.match(readFileSync(join(root, 'jobs', job.id, 'result.json'), 'utf8'), /first failure/); + await assert.rejects(resumeExecutionJob(root, job.id, { execute, inspect }), /failed or reconciled interrupted job/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('an interrupted resume keeps its claim until its owner is proven dead', async () => { + const root = mkdtempSync(join(tmpdir(), 'execution-job-resume-owner-')); + try { + mkdirSync(join(root, 'plans')); + copyFileSync(join(STACK_BENCH_ROOT, 'tests/fixtures/dependency-model-free-campaign.json'), join(root, 'plans/reference.json')); + const job = submitExecutionJob(root, { key: 'resume-owner', planFile: 'reference.json' }); + const base = join(root, 'jobs', job.id); + const owner = { hostId: 'worker-a', token: 'a'.repeat(64), startedAt: new Date().toISOString(), + ownerPid: process.pid, ownerInstance: controllerInstance() }; + writeFileSync(join(base, 'claim.json'), JSON.stringify(owner)); + const inspect = (() => ({ plan: { contentSha256: job.planSha256, + definition: { mode: { id: 'dependency' } } }, + state: { status: 'prepared', attempts: [{ executions: [{}] }] } })) as unknown as typeof inspectCampaign; + const execute = (async () => ({ status: 'completed', summary: { completed: 1, + pending: 0, running: 0, invalid: 0 } })) as unknown as typeof executeCampaign; + await assert.rejects(resumeExecutionJob(root, job.id, { execute, inspect }), /live worker/); + assert.equal(readExecutionJob(root, job.id).status, 'running'); + writeFileSync(join(base, 'claim.json'), JSON.stringify({ ...owner, ownerPid: 2147483647 })); + assert.equal((await resumeExecutionJob(root, job.id, { execute, inspect })).status, 'completed'); + + const repeated = submitExecutionJob(root, { key: 'resume-owner-repeated', planFile: 'reference.json' }); + const failed = (async () => { throw new Error('prior failure'); }) as typeof executeCampaign; + await workExecutionJob(root, repeated.id, 'worker-a', { execute: failed }); + const repeatedBase = join(root, 'jobs', repeated.id); + mkdirSync(join(repeatedBase, 'resume-000001')); + writeFileSync(join(repeatedBase, 'resume-000001', 'claim.json'), + JSON.stringify({ ...owner, ownerPid: 2147483647 })); + const repeatedInspect = (() => ({ plan: { contentSha256: repeated.planSha256, + definition: { mode: { id: 'dependency' } } }, + state: { status: 'prepared', attempts: [{ executions: [{}] }] } })) as unknown as typeof inspectCampaign; + assert.equal((await resumeExecutionJob(root, repeated.id, + { execute, inspect: repeatedInspect })).status, 'completed'); + assert.match(readFileSync(join(repeatedBase, 'result.json'), 'utf8'), /prior failure/); + assert.equal(readdirSync(repeatedBase).includes('resume-000002'), true); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('separate processes cannot both execute one resume claim', async () => { + const root = mkdtempSync(join(tmpdir(), 'execution-job-resume-race-')); + const children: ReturnType<typeof spawn>[] = []; + try { + mkdirSync(join(root, 'plans')); + copyFileSync(join(STACK_BENCH_ROOT, 'tests/fixtures/dependency-model-free-campaign.json'), join(root, 'plans/reference.json')); + const job = submitExecutionJob(root, { key: 'resume-race', planFile: 'reference.json' }); + const failed = (async () => { throw new Error('interrupted'); }) as typeof executeCampaign; + await workExecutionJob(root, job.id, 'worker-a', { execute: failed }); + const marker = join(root, 'executing'); + const release = join(root, 'release'); + const moduleUrl = new URL('../src/campaigns/execution-jobs.js', import.meta.url).href; + const script = `import { existsSync, writeFileSync } from 'node:fs'; +import { setTimeout } from 'node:timers/promises'; +import { resumeOwnedExecutionJob } from ${JSON.stringify(moduleUrl)}; +await resumeOwnedExecutionJob(process.argv[1], process.argv[2], { execute: async () => { + writeFileSync(process.argv[3] + '.' + process.pid, 'entered'); + while (!existsSync(process.argv[4])) await setTimeout(20); + return { summary: { completed: 1, pending: 0, running: 0, invalid: 0 } }; +} });`; + const launch = () => { + const child = spawn(process.execPath, ['--input-type=module', '-e', script, + root, job.id, marker, release], { stdio: ['ignore', 'ignore', 'pipe'] }); + children.push(child); + let error = ''; + child.stderr.on('data', chunk => { error += String(chunk); }); + return { child, done: new Promise<number | null>(resolve => child.on('exit', code => resolve(code))), + error: () => error }; + }; + const first = launch(); + const second = launch(); + const deadline = Date.now() + 30_000; + const entries = () => readdirSync(root).filter(name => name.startsWith('executing.')); + while (!entries().length || (first.child.exitCode === null && second.child.exitCode === null)) { + assert(entries().length <= 1, 'both processes entered execution'); + assert(Date.now() < deadline, `${first.error()} ${second.error()}`); + await delay(20); + } + assert.equal(entries().length, 1, 'only one process entered execution'); + writeFileSync(release, 'go'); + const firstCode = await first.done; + const secondCode = await second.done; + assert(firstCode === 0 || secondCode === 0, `${first.error()} ${second.error()}`); + assert.equal(entries().length, 1, 'loser never entered execution'); + assert.equal(readExecutionJob(root, job.id).status, 'completed'); + } finally { + for (const child of children) if (child.exitCode === null) child.kill(); + rmSync(root, { recursive: true, force: true }); + } +}); + +test('job CLI rejects options its command does not use and resolves results like the dashboard', async () => { + const root = mkdtempSync(join(tmpdir(), 'execution-job-cli-')); + const cwd = process.cwd(); + try { + mkdirSync(join(root, 'plans')); + copyFileSync(join(STACK_BENCH_ROOT, 'tests/fixtures/campaign.deterministic.json'), join(root, 'plans/test.json')); + const job = submitExecutionJob(root, { key: 'cli-job', planFile: 'test.json' }); + const env = { STACK_BENCH_RESULTS_DIR: root }; + for (const argv of [['work', job.id, '--host', 'h', '--concurrency', '2'], ['status', job.id, '--host', 'h'], + ['list', '--concurrency', '2'], ['prepare', 'selection.json', '--limit', '5'], ['options', '--host', 'x'], + ['resume', job.id, '--host', 'h']]) { + await assert.rejects(jobCommand(argv, env), /does not take --/, argv.join(' ')); + } + assert.equal((await jobCommand(['status', job.id], env) as { status: string }).status, 'queued'); + assert.equal((await jobCommand(['list', '--limit', '1'], env) as { jobs: unknown[] }).jobs.length, 1); + await assert.rejects(jobCommand(['list'], { STACK_BENCH_RESULTS_DIR: 'results' }), /must be an absolute path/); + // An empty setting means the package results folder, not the working directory. + process.chdir(root); + const packageResults = await jobCommand(['list'], { STACK_BENCH_RESULTS_DIR: '' }) as + { jobs: Array<{ job: { id: string } }> }; + assert.equal(packageResults.jobs.some(entry => entry.job.id === job.id), false); + } finally { process.chdir(cwd); rmSync(root, { recursive: true, force: true }); } +}); + +test('job CLI prints one JSON document and separates usage errors from command failures', async t => { + const root = mkdtempSync(join(tmpdir(), 'execution-job-exit-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + mkdirSync(join(root, 'plans')); + copyFileSync(join(STACK_BENCH_ROOT, 'tests/fixtures/campaign.deterministic.json'), join(root, 'plans/test.json')); + const job = submitExecutionJob(root, { key: 'exit-job', planFile: 'test.json' }); + const failure = (async () => { throw new Error('synthetic failure'); }) as typeof executeCampaign; + await workExecutionJob(root, job.id, 'h', { execute: failure }); + const env = { STACK_BENCH_RESULTS_DIR: root }; + const printed = t.mock.method(console, 'log', () => {}); + t.mock.method(console, 'error', () => {}); + for (const command of ['status', 'cancel']) { + printed.mock.resetCalls(); + assert.equal(await runJobCli([command, job.id], env), 0, command); + assert.equal(printed.mock.callCount(), 1); + assert.equal(JSON.parse(printed.mock.calls[0]!.arguments[0] as string).status, 'failed'); + } + assert.equal(await runJobCli(['work', job.id, '--host', 'h'], env), 1, 'running a job to failure fails'); + assert.equal(await runJobCli(['status'], env), 2); + assert.equal(await runJobCli(['status', job.id, '--bogus'], env), 2); + assert.equal(await runJobCli(['start', 'review.json'], env), 2, 'start without a host'); + assert.equal(await runJobCli(['status', 'f'.repeat(64)], env), 1); +}); diff --git a/tools/stack-bench/tests/execution-worker.test.ts b/tools/stack-bench/tests/execution-worker.test.ts new file mode 100644 index 00000000000..8255a2f5211 --- /dev/null +++ b/tools/stack-bench/tests/execution-worker.test.ts @@ -0,0 +1,77 @@ +import assert from 'node:assert/strict'; +import { copyFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { runExecutionWorker } from '../src/campaigns/execution-worker.js'; +import { readExecutionJob, submitExecutionJob, workExecutionJob } from '../src/campaigns/execution-jobs.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import type { executeCampaign } from '../src/campaigns/campaign-runner.js'; + +test('worker discovers jobs, honors placement and concurrency, and drains without cancelling claims', { timeout: 30_000 }, async () => { + const root = mkdtempSync(join(tmpdir(), 'execution-worker-')); + const stop = new AbortController(); + let release!: () => void; + const gate = new Promise<void>(resolve => { release = resolve; }); + let started!: () => void; + const ready = new Promise<void>(resolve => { started = resolve; }); + let worker: Promise<void> | undefined; + try { + mkdirSync(join(root, 'plans')); + copyFileSync(join(STACK_BENCH_ROOT, 'tests/fixtures/campaign.deterministic.json'), join(root, 'plans/test.json')); + const request = { planFile: 'test.json', hostId: 'worker-a' }; + const foreign = submitExecutionJob(root, { ...request, key: 'foreign', hostId: 'worker-b' }); + const selected = ['one', 'two', 'three'].map(key => submitExecutionJob(root, { ...request, key })); + let calls = 0; + const execute = (async (_plan, _out, options) => { + calls++; + if (calls === 2) started(); + await gate; + assert.equal(options?.signal?.aborted, false, 'worker shutdown must not cancel a paid claim'); + return { summary: { completed: 9, pending: 0, running: 0, invalid: 0 } }; + }) as typeof executeCampaign; + worker = runExecutionWorker(root, 'worker-a', { concurrency: 2, signal: stop.signal, + work: (results, id, host, options) => workExecutionJob(results, id, host, { ...options, execute }) }); + await ready; + assert.equal(calls, 2); + stop.abort(); + release(); + await worker; + assert.equal(calls, 2, 'shutdown must not claim the pending job'); + assert.equal(readExecutionJob(root, foreign.id).status, 'queued'); + const states = selected.map(job => readExecutionJob(root, job.id).status).sort(); + assert.deepEqual(states, ['completed', 'completed', 'queued']); + await assert.rejects(runExecutionWorker(root, 'worker-a', { concurrency: 0, signal: stop.signal }), /concurrency/); + } finally { + stop.abort(); release(); await worker; + rmSync(root, { recursive: true, force: true }); + } +}); + +test('worker isolates corrupt records and pre-claim faults without fabricating results', { timeout: 15_000 }, async () => { + const root = mkdtempSync(join(tmpdir(), 'execution-worker-failure-')); + const stop = new AbortController(); + const reports: string[] = []; + try { + mkdirSync(join(root, 'plans')); + copyFileSync(join(STACK_BENCH_ROOT, 'tests/fixtures/campaign.deterministic.json'), join(root, 'plans/test.json')); + const jobs = ['first', 'second', 'third'].map(key => submitExecutionJob(root, + { key, planFile: 'test.json' })).sort((a, b) => a.id.localeCompare(b.id)); + writeFileSync(join(root, 'jobs', jobs[0]!.id, 'job.json'), '{broken'); + const calls: string[] = []; + await runExecutionWorker(root, 'worker-a', { concurrency: 1, signal: stop.signal, + report: message => reports.push(message), work: async (results, id, host, options) => { + calls.push(id); + if (id === jobs[1]!.id) throw new Error('plan identity changed'); + const result = await workExecutionJob(results, id, host, { ...options, + execute: (async (_plan, _out, _options) => ({ summary: { completed: 9, pending: 0, running: 0, invalid: 0 } })) as typeof executeCampaign }); + stop.abort(); + return result; + } }); + assert.deepEqual(calls, [jobs[1]!.id, jobs[2]!.id]); + assert.equal(readExecutionJob(root, jobs[1]!.id).status, 'queued'); + assert.equal(readExecutionJob(root, jobs[2]!.id).status, 'completed'); + assert(reports.some(line => line.includes('cannot be read'))); + assert(reports.some(line => line.includes('plan identity changed'))); + } finally { stop.abort(); rmSync(root, { recursive: true, force: true }); } +}); diff --git a/tools/stack-bench/tests/external-data-sync.mutation.ts b/tools/stack-bench/tests/external-data-sync.mutation.ts new file mode 100644 index 00000000000..b1182b17626 --- /dev/null +++ b/tools/stack-bench/tests/external-data-sync.mutation.ts @@ -0,0 +1,101 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { compileScenarioDefinition, type CompiledCriterion, type CompiledFeature } + from '../src/composition/definition-compiler.js'; +import { buildRecipeRelease } from '../src/composition/recipe-release.js'; +import { selectScenarioChecks } from '../src/composition/recipe-selection.js'; + +const ROOT = STACK_BENCH_ROOT; +const LIVE_SCENARIO = 'tracks/ecommerce/scenarios/01-external-live-sync.json'; +const RELOAD_SCENARIO = 'tracks/ecommerce/scenarios/01-external-reload-sync.json'; +const RECONNECT_SCENARIO = 'tracks/ecommerce/scenarios/01-external-reconnect-sync.json'; + +function json(relative: string): unknown { + return JSON.parse(readFileSync(join(ROOT, relative), 'utf8')); +} + +test('external synchronization scenarios are focused and state-independent', () => { + const live = compileScenarioDefinition(json(LIVE_SCENARIO), { + source: LIVE_SCENARIO, + expectedLevel: 1, + }); + const reconnect = compileScenarioDefinition(json(RECONNECT_SCENARIO), { + source: RECONNECT_SCENARIO, + expectedLevel: 1, + }); + const reload = compileScenarioDefinition(json(RELOAD_SCENARIO), { + source: RELOAD_SCENARIO, + expectedLevel: 1, + }); + + const liveFeature = requiredFeature(live.features[0], LIVE_SCENARIO); + const reconnectFeature = requiredFeature(reconnect.features[0], RECONNECT_SCENARIO); + const reloadFeature = requiredFeature(reload.features[0], RELOAD_SCENARIO); + const liveCriterion = requiredCriterion(liveFeature.criteria[0], LIVE_SCENARIO); + const reconnectCriterion = requiredCriterion(reconnectFeature.criteria[0], RECONNECT_SCENARIO); + const reloadCriterion = requiredCriterion(reloadFeature.criteria[0], RELOAD_SCENARIO); + + assert.deepEqual(liveFeature.criteria.map(criterion => criterion.id), ['901a']); + assert.deepEqual(liveCriterion.steps.map(step => step.do), + ['dbSetStock', 'expectNumber']); + assert.equal(liveCriterion.points, 1, + 'the scenario and recipe score must agree'); + assert.deepEqual(reloadCriterion.steps.map(step => step.do), + ['dbSetStock', 'reload', 'expectNumber']); + assert.equal(reloadCriterion.points, 0, + 'reload persistence is supporting evidence, not a second score'); + + const reconnectSteps = reconnectCriterion.steps; + assert.deepEqual(reconnectSteps.map(step => step.do), + ['setOffline', 'dbSetStock', 'setOffline', 'expectNumber']); + const disconnectedWrite = reconnectSteps[1]; + const reconnectResult = reconnectSteps.at(-1); + assert(disconnectedWrite && reconnectResult, 'reconnect checks must contain their boundary steps'); + assert.equal(disconnectedWrite.settleMs, 4000, + 'the external write must remain inside the disconnected window before network restoration'); + assert.equal(reconnectResult.equals, 52, + 'East 7 + untouched West 45 must not depend on another external-stock scenario'); + assert.equal(reconnectSteps.some(step => ['startAppServer', 'stopAppServer'].includes(step.do)), false); + assert.equal(reconnectCriterion.points, 1, + 'the scenario and recipe score must agree'); +}); + +test('901b is independently selectable from the current L1 recipe', () => { + const release = buildRecipeRelease(join(ROOT, 'tracks', 'ecommerce', 'composition', 'recipes', + 'sequential-l1.json')); + const key = 'ecommerce.spec.external-data-sync.external-stock.901b'; + const check = release.checkCatalog.find(candidate => candidate.stableKey === key); + assert(check, `${key} must exist in the release`); + assert.equal(check.source, 'scenarios/01-external-reload-sync.json'); + assert.equal(check.points, 0); + const reloadScenario = compileScenarioDefinition(json(RELOAD_SCENARIO), { + source: RELOAD_SCENARIO, + expectedLevel: 1, + }); + const selected = selectScenarioChecks(reloadScenario, { checks: release.checkCatalog }, [key]); + const feature = requiredFeature(selected.features[0], RELOAD_SCENARIO); + const criterion = requiredCriterion(feature.criteria[0], RELOAD_SCENARIO); + const setup = feature.setup[0]; + assert(setup, 'the selected reload scenario must have setup'); + assert.deepEqual(feature.criteria.map(item => item.id), ['901b']); + assert.equal(setup.equals, 100); + assert.deepEqual(criterion.steps.map(step => step.do), + ['dbSetStock', 'reload', 'expectNumber']); +}); + +function requiredFeature(feature: CompiledFeature | undefined, source: string): CompiledFeature { + if (!feature) throw new Error(`${source} must contain a feature`); + return feature; +} + +function requiredCriterion( + criterion: CompiledCriterion | undefined, + source: string, +): CompiledCriterion { + if (!criterion) throw new Error(`${source} must contain a criterion`); + return criterion; +} diff --git a/tools/stack-bench/tests/feature-catalog-selection.contract.ts b/tools/stack-bench/tests/feature-catalog-selection.contract.ts new file mode 100644 index 00000000000..1add84887fe --- /dev/null +++ b/tools/stack-bench/tests/feature-catalog-selection.contract.ts @@ -0,0 +1,39 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { loadTrack } from '../src/composition/tracks.js'; +import { resolveFeatureCatalog } from '../src/progression/feature-catalog-selection.js'; +import { compileFeatureCatalogInput } from '../src/progression/progression-definition.js'; + +test('feature catalogs resolve from a track-relative definition', () => { + const catalog = resolveFeatureCatalog('progression/ecommerce.json', loadTrack('ecommerce')); + assert.equal(catalog.definition.id, 'ecommerce.questlines'); + assert(catalog.definition.nodes.length > 0); + assert.match(catalog.identity.contentSha256, /^[a-f0-9]{64}$/); + assert.throws(() => resolveFeatureCatalog('../outside.json', loadTrack('ecommerce')), + /escapes the track/); +}); + +test('feature catalog identity excludes its display title only', () => { + const catalog = resolveFeatureCatalog('progression/ecommerce.json', loadTrack('ecommerce')); + const governance = compileFeatureCatalogInput({ + ...catalog.definition, + title: `${catalog.definition.title} renamed`, + }); + assert.equal(governance.identity.contentSha256, catalog.identity.contentSha256); + + const nodeTitle = structuredClone(catalog.definition); + nodeTitle.nodes[0]!.title += ' renamed'; + assert.notEqual(compileFeatureCatalogInput(nodeTitle).identity.contentSha256, + catalog.identity.contentSha256); + + const questlineTitle = structuredClone(catalog.definition); + questlineTitle.questlines[0]!.title += ' renamed'; + assert.notEqual(compileFeatureCatalogInput(questlineTitle).identity.contentSha256, + catalog.identity.contentSha256); + + const behavior = structuredClone(catalog.definition); + behavior.nodes[0]!.gradingChecks[0]!.points += 1; + assert.notEqual(compileFeatureCatalogInput(behavior).identity.contentSha256, + catalog.identity.contentSha256); +}); diff --git a/tools/stack-bench/tests/fixtures/campaign-admission-process.ts b/tools/stack-bench/tests/fixtures/campaign-admission-process.ts new file mode 100644 index 00000000000..61cfe42cf4d --- /dev/null +++ b/tools/stack-bench/tests/fixtures/campaign-admission-process.ts @@ -0,0 +1,61 @@ +import { mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { compileCampaignFile } from '../../src/campaigns/campaign-compiler.js'; +import { releaseCampaignReservation, runCampaignAdmission } from '../../src/campaigns/campaign-admission.js'; +import { readBackendLease } + from '../../src/runtime/backend-lease.js'; +import type { CampaignReservation } from '../../src/campaigns/campaign-admission.js'; +import { STACK_BENCH_ROOT } from '../../src/package-root.js'; + +const [directory, locks, workers, racers] = process.argv.slice(2); +if (!directory || !locks) process.exit(2); +const manifest = JSON.parse(readFileSync(join(STACK_BENCH_ROOT, 'tests', 'fixtures', 'campaign.deterministic.json'), 'utf8')); +manifest.parallelism = Number(workers ?? 1); +const manifestPath = join(directory, 'manifest.json'); +writeFileSync(manifestPath, JSON.stringify(manifest)); +const plan = compileCampaignFile(manifestPath); +process.send?.('ready'); +let reservation: CampaignReservation | undefined; +let joinedRace = false; +process.on('message', async message => { + if (message === 'release') { + if (reservation) releaseCampaignReservation(reservation); + reservation = undefined; + process.send?.('released'); + return; + } + try { + const result = await runCampaignAdmission(plan, directory, { env: { STACK_BENCH_RUNNER_CAPACITY: '64', STACK_BENCH_RESOURCE_LOCK_DIR: locks, + STACK_BENCH_APPLIANCE: '1' }, + probePort: () => { + if (racers && !joinedRace) { + joinedRace = true; + mkdirSync(locks, { recursive: true }); + writeFileSync(join(locks, `racer-${process.pid}`), 'ready'); + const deadline = Date.now() + 10000; + while (readdirSync(locks).filter(name => name.startsWith('racer-')).length < Number(racers)) { + if (Date.now() > deadline) throw new Error('claim race barrier timed out'); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10); + } + } + return { free: true }; + }, + preflight: request => { + writeFileSync(join(directory, 'preflight-entered'), 'entered'); + return { schemaVersion: 1, generatedAt: new Date().toISOString(), + request: { backends: request.backends, track: request.track, levels: request.levelList, + runIndex: request.runIndex, parallelism: request.parallelism, agentAdapter: request.agentAdapter, + packs: request.packIds, checks: request.checkKeys, image: request.image, + resultsDir: request.resultsDir, smoke: request.smoke }, + ok: true, summary: { passed: 0, failed: 0, warnings: 0 }, checks: [] }; + } }); + reservation = result.reservation; + { + const lease = readBackendLease(reservation!.path, { token: reservation!.token }); + process.send?.({ status: 'admitted', runIndices: result.runIndices, keys: lease.resources.locks.map(lock => lock.key) }); + } + } catch (error) { + writeFileSync(join(directory, 'admission-error'), String(error)); + process.send?.('refused'); + } +}); diff --git a/tools/stack-bench/tests/fixtures/campaign-lock-process.ts b/tools/stack-bench/tests/fixtures/campaign-lock-process.ts new file mode 100644 index 00000000000..8fd133757bf --- /dev/null +++ b/tools/stack-bench/tests/fixtures/campaign-lock-process.ts @@ -0,0 +1,17 @@ +import { acquireCampaignLock, releaseCampaignLock, watchCampaignCancellation } + from '../../src/campaigns/campaign-lock.js'; +const [root] = process.argv.slice(2); +try { + const lock = acquireCampaignLock(root!, { id: 'native-lock-proof', contentSha256: 'a'.repeat(64) }); + process.stdout.write('acquired\n'); + { + const watcher = watchCampaignCancellation(lock); + const deadline = setTimeout(() => { throw new Error('cancellation was not received'); }, 10_000); + try { await new Promise<void>(resolve => watcher.signal.addEventListener('abort', () => resolve(), { once: true })); } + finally { clearTimeout(deadline); watcher.close(); } + } + releaseCampaignLock(lock); +} catch (error) { + process.stderr.write(error instanceof Error ? error.message : String(error)); + process.exitCode = 2; +} diff --git a/tools/stack-bench/tests/fixtures/campaign.deterministic.json b/tools/stack-bench/tests/fixtures/campaign.deterministic.json new file mode 100644 index 00000000000..a21eb101053 --- /dev/null +++ b/tools/stack-bench/tests/fixtures/campaign.deterministic.json @@ -0,0 +1,115 @@ +{ + "schemaVersion": 8, + "kind": "campaign-manifest", + "id": "ecommerce-l1-example", + "version": "2.0.0", + "state": "draft", + "title": "Ecommerce L1 model-free example", + "track": "ecommerce", + "mode": { "id": "sequential" }, + "repair": { "selection": "batch", "budget": { "total": 3 } }, + "levels": [1], + "selection": { + "levels": [ + { + "level": 1, + "recipe": "ecommerce.sequential-l1", + "features": [ + "ecommerce.feature.accounts", + "ecommerce.feature.cart-checkout", + "ecommerce.feature.catalog-items", + "ecommerce.feature.catalog-discovery", + "ecommerce.feature.purchasing", + "ecommerce.feature.reviews", + "ecommerce.feature.warehouse-admin" + ], + "checks": [] + } + ] + }, + "stacks": [ + { "id": "spacetime", "adapterVersion": "1.4.0" }, + { "id": "postgres", "adapterVersion": "1.6.0" }, + { "id": "mongodb", "adapterVersion": "1.5.0" } + ], + "agents": [ + { + "adapter": "deterministic", + "adapterVersion": "1.3.0", + "model": "deterministic" + } + ], + "conditions": [ + { + "id": "prescribed", + "guidanceProfile": "prescribed", + "repairPolicy": "scored-only", + "specifications": { + "levels": [ + { + "level": 1, + "requested": [ + "ecommerce.spec.access-control", + "ecommerce.spec.concurrency-safety", + "ecommerce.spec.external-data-sync", + "ecommerce.spec.live-state", + "ecommerce.spec.state-durability", + "ecommerce.spec.transactional-integrity" + ], + "expected": [], + "observed": [] + } + ] + } + } + ], + "repetitions": 3, + "parallelism": 1, + "ordering": { + "method": "balanced-rotation", + "seed": "replace-before-measurement" + }, + "budgets": { + "attemptTimeoutMinutes": 240, + "maxCostUsdPerAttempt": null + }, + "attemptPolicy": { + "retries": 1, + "retryOn": ["provider_failure"], + "excludeFromAnalysis": ["contaminated", "harness_failure", "inconclusive", "ungraded"] + }, + "runtime": { + "releaseManifestSha256": null, + "controllerImage": null, + "buildImage": null, + "platform": "linux/amd64" + }, + "pricing": { + "currency": "USD", + "unit": "USD-per-million-tokens", + "capturedAt": "2026-08-12T00:00:00.000Z", + "source": "deterministic adapter makes no billable provider calls", + "models": { + "deterministic": { + "input": 0, + "output": 0, + "cacheWrite5m": 0, + "cacheWrite1h": 0, + "cacheRead": 0 + } + } + }, + "analysis": { + "primaryMetric": "firstBuildScoreRate", + "secondaryMetrics": [ + "finalScoreRate", + "totalCostUsd", + "totalDurationMs", + "invalidAttemptRate" + ], + "dispersion": "median-iqr", + "invalidAttempts": "report-separately", + "missingData": "no-imputation", + "comparisonUnit": "stack-agent-condition-recipe" + } +} diff --git a/tools/stack-bench/tests/fixtures/convex-native/.gitignore b/tools/stack-bench/tests/fixtures/convex-native/.gitignore new file mode 100644 index 00000000000..0157fa357eb --- /dev/null +++ b/tools/stack-bench/tests/fixtures/convex-native/.gitignore @@ -0,0 +1,12 @@ +node_modules/ +.env.local +identities.json +probe-evidence.json +auth-evidence.json +lifecycle-*.json +owned-private.json +foreign-private.json +owned-probe.json +restart-evidence.json +convex/auth.config.js +convex/_generated/ diff --git a/tools/stack-bench/tests/fixtures/convex-native/README.md b/tools/stack-bench/tests/fixtures/convex-native/README.md new file mode 100644 index 00000000000..1c3bfb52f2e --- /dev/null +++ b/tools/stack-bench/tests/fixtures/convex-native/README.md @@ -0,0 +1,191 @@ +# Convex native integration slice + +This model-free fixture tests the native protocol before Convex becomes a +selectable stack. It is not an ecommerce reference, qualified account implementation, +qualification receipt, or paid campaign. Do not supply it to a coding agent. + +Runtime pin: `ghcr.io/get-convex/convex-backend@sha256:afbf4292df387c8f031a68d00048551cf1640ddf0013c51ac704a89d7e73e743` +(Linux amd64, backend revision `4e9b83a27fb214e470c9706a0bae234e411d134f`). +The fixture lockfile pins the native client/CLI to `1.46.0`. + +Use a fresh, owned backend with its own instance secret and data directory. +Set `CONVEX_SELF_HOSTED_URL` and `CONVEX_SELF_HOSTED_ADMIN_KEY` for that backend. +The admin key is for deployment, seeding and independent observations only. +The two ordinary callers use locally signed test JWTs with different subjects; +only their public verification key is deployed. No cloud account or auth service +is used. These identities do not prove the username/password contract. + +Build StackBench once, then run from this fixture directory: + +```sh +npm ci --ignore-scripts --no-audit --no-fund +node prepare.mjs +node node_modules/convex/bin/main.js dev --once --typecheck disable +node probe.mjs +``` + +If copied outside the package, set `STACK_BENCH_CONVEX_PROTOCOL` to the file URL +of the compiled `dist/src/stacks/backends/convex-protocol.js` module. +The probe refuses to seed an existing populated fixture. It records native +responses, consistent stock/order snapshots, and the final state in +`probe-evidence.json`. Preserve failed evidence too. + +The probe checks accepted purchases, unauthorized and anonymous refusal without +effects, deliberate versus unhandled errors, missing functions, an ordinary +caller's live subscription, independent targeted stock edits, and a committed +purchase whose HTTP response is dropped. A lost response remains unknown to the +caller even when independent state proves the commit. More payloads add no points. + +Restart the owned backend with the same storage and credentials, wait for its +API to be ready, then run `node verify-restart.mjs`. Preserve the resulting +`restart-evidence.json`. This observes recovered data; it does not inject a crash +during an operation. Use a fresh owned data directory and redeploy the identical +fixture to test clean initialization again. Do not clear individual tables and +claim that scheduled work and component state were reset. + +The snapshot reader uses the documented [Data Sync API](https://docs.convex.dev/deployment-api/data-sync) +and stops only at a consistent `upToDate` result. Raw pages preserve exact native +timestamps. Its 30-page bound is for this tiny fixture, not a production limit. +The stock writer uses the vendor's internal +[`patchDocumentsFields`](https://github.com/get-convex/convex-backend/blob/4e9b83a27fb214e470c9706a0bae234e411d134f/npm-packages/system-udfs/convex/_system/frontend/patchDocumentsFields.ts) +system mutation. It is independent of fixture business handlers and preserves +document IDs and other fields. This is a pinned compatibility dependency, not a +stable public write API. Stack adoption must explicitly retain that dependency +and its regression test or choose another proven mechanism. + +Production lifecycle integration must use the existing namespace anchor and +leased stable ports. Sharing a helper's network directly with a backend container +does not preserve reachability when that container is restarted. This fixture +does not qualify firewall isolation, host-browser routing, actual UI capture, +full auth, order-interface mapping, crash controls, or concurrent snapshot updates. +Remove only owned containers/volumes and local dependency installs after the probe. + +## Local account candidate + +On a separate fresh deployment, run `node prepare-auth.mjs`, deploy with the same +CLI command, then run `node probe-auth.mjs` instead of the signed-identity probe. +The lockfile pins Convex Auth `0.0.95` and Auth Core `0.41.1`. Preparation supplies +local JWT keys to the owned deployment and uses its own HTTP endpoint for JWKS. +There is no external identity service. The browser probe uses the controller's +Playwright installation; set `PLAYWRIGHT_MODULE` to its module file if elsewhere. + +The candidate uses the documented ConvexCredentials provider and public +createAccount/retrieveAccount helpers. The account key is the exact username; +no email address is required. Existing username/password bounds are preserved. +This is fixture configuration, not new agent guidance. +The protected purchase reads the actual issued identity and live session record. +The small browser client covers signup/login, a native WebSocket purchase, reload, +the real session-token hook, signout and refused calls with independent state checks. +It does not implement refresh-token rotation or production UI. + +The unmodified pinned Password provider issues a session for a +duplicate signup when its supplied password matches the existing account. Check +1c requires a taken username to be refused. The candidate passes a fresh, +server-generated registration nonce through the supported custom user profile. +The atomic createAccount result must contain that nonce before the provider can +return a user ID and issue a session. An existing user retains its original nonce, +so a duplicate cannot sign in. There is no existence precheck or vendor patch. +Login retains retrieveAccount and the same pinned Scrypt implementation as the +library Password provider. The probe tests 36 concurrent signups across six +same/different-password races, each with one account, user and initial session. +Removing the nonce guard makes the control fail with six winners. +Convex Auth and the deprecated Lucia dependency still need an explicit +maintenance decision before promotion. This bounded probe is not qualification. + +Sources: [Password configuration](https://labs.convex.dev/auth/config/passwords), +[manual setup](https://labs.convex.dev/auth/setup/manual), +[Credentials provider](https://labs.convex.dev/auth/api_reference/providers/ConvexCredentials), +[public account helpers](https://labs.convex.dev/auth/api_reference/server), and +[custom user schema](https://labs.convex.dev/auth/setup/schema). The exact duplicate +behavior was checked in the installed package's +`src/server/implementation/mutations/createAccountFromCredentials.ts` and live. + +## Namespace, restart and reset probe + +Keep one named namespace anchor running. Publish the fixture frontend (3100), +native API (3210) and HTTP actions (3211) on that anchor, bound to host loopback. +The backend and fixture helper join `container:<anchor>`; only the backend mounts +its named data volume. Do not join the helper to the replaceable backend itself. +For this feasibility probe, Docker-assigned host ports avoid collisions. They +are not production allocation: the adapter must use StackBench's leased windows. + +With the local-account fixture deployed, run these phases from its helper: + +1. `node probe-lifecycle.mjs prepare`: create a real account and purchase, and + observe a scheduled marker execute as a positive control. +2. Keep `node probe-lifecycle.mjs serve` running in the named helper. +3. Run `probe-lifecycle-browser.mjs <page-url> HostUser <output.json>` from the + host and the same script with `ContainerUser` from the helper. Set + `PLAYWRIGHT_MODULE` to each platform's installed module. Host page URL is + `http://127.0.0.1:<frontend>/?api=http%3A%2F%2F127.0.0.1%3A<api-port>`; + helper URL is `http://127.0.0.1:3100`. Each browser signs up, purchases and reloads. +4. `capture` records independent rows and IDs. Restart only the backend with its + volume and credentials retained, wait for `/version`, then run `warm`. +5. `schedule-reset` records a pending marker due in 20 seconds. Remove only the + owned backend and its complete data volume; start the same image with a fresh + volume and deployment credentials. Run `prepare-auth.mjs` and deploy identical + fixture source, then run `reset`. It requires empty data/accounts, refuses old + credentials and observes no marker past its due time. It then checks fresh + initialization and registration. It waits at most 30 seconds. + +Keep the anchor ID and published ports unchanged through both operations. The +reset includes backend storage, not a collection of table-clearing operations. +Preserve phase receipts and source hashes, but do not export `lifecycle-private.json` +or the temporary admin environment. The fixture UI has a seed-specific item ID; +fresh production initialization must restart the app frontend too. This probe +does not establish crash-mid-operation safety, component cleanup, firewall +isolation, refresh rotation or full lifecycle integration. The scheduled marker +uses the documented [scheduler and system-table API](https://docs.convex.dev/scheduling/scheduled-functions). +Remove the helper, backend, anchor and owned volume after preserving evidence. + +## Owned lifecycle integration + +`tests/convex-owned-lifecycle.integration.ts` calls the private Convex lifecycle +through normal lease, namespace, firewall, browser-container and teardown owners. +It is opt-in with `STACK_BENCH_CONVEX_OWNED_TEST=1` inside the Linux controller. +Mount this fixture at `STACK_BENCH_CONVEX_FIXTURE`, an evidence directory at +`STACK_BENCH_CONVEX_EVIDENCE_DIR`, and the normal shared resource-lock directory. +The test claims ports 14309/14310/14311 and uses 14312/14313/14314 for one +overlapping attempt at a time. These explicit test ports do not define the +future campaign allocation. +The trusted controller deploys this fixture. Only the native client probe runs +inside the restricted browser container; its read-only/noexec policy is retained. +This does not yet test generated-app deployment from a coding container. + +`probe-owned.mjs` tests real password signup, signed native HTTP and WebSocket +calls, a purchase, and anonymous refusal without stored effects. Warm restart +must retain stored rows and the original token. Full reset must reject old +credentials and remove pending scheduled work before fresh signup and purchase. +The integration test checks two live attempts for separate accounts/data, +cross-deployment credential refusal, and blocked cross-attempt network access. +It also checks cleanup after a real port-binding failure and a startup-process +interruption after the durable container record. Each cleanup must preserve +the surviving app's stored rows and original session. + +The test checks the published API and HTTP-action JWKS endpoint, wrong lease +token refusal, stale process-record refusal, released locks and removed owned +volumes. A retained-state negative control must fail the reset observer. Public +receipts contain no admin key or session token. This slice does not qualify +Chromium UI behavior, component state, full campaign cancellation, controller +death recovery, mid-operation crash checks, or full Convex support. Convex stays +unregistered. + +## Coding-container deployment proof + +`tests/convex-coding-deploy.integration.ts` deploys the same pinned app through +the normal coding-container preparation, isolation, workspace and teardown +owners. A trusted programmatic preparation plan permits this unregistered stack; +it cannot start a paid session and adds no CLI override or registry entry. + +The test requires the normal daemon-visible `STACK_BENCH_WORK_DIR` and a pinned +`STACK_BENCH_BUILD_IMAGE`. Only app source and setup files enter `/app`. Install, +local auth setup and native deployment run as developer UID 10001 with the owned +backend's admin key. The coding container receives no grader, probe, reader, +Docker socket, provider credential or lease token. Native browser calls and an +independent stored-state reader then test the deployed app. Submitted and +deployed app hashes must match; generated auth configuration is checked +separately. Reuse must keep the same coding-container identity, and cleanup must +record workspace handback before removing the container and workspace. + +This is a model-free fixture deployment proof. It does not qualify paid-agent +execution, native grader integration, campaign allocation or full Convex support. diff --git a/tools/stack-bench/tests/fixtures/convex-native/auth-browser.js b/tools/stack-bench/tests/fixtures/convex-native/auth-browser.js new file mode 100644 index 00000000000..552389ab10e --- /dev/null +++ b/tools/stack-bench/tests/fixtures/convex-native/auth-browser.js @@ -0,0 +1,43 @@ +/* global window, document, sessionStorage -- bundled for the browser page */ +import { ConvexClient } from 'convex/browser'; +const client = new ConvexClient(window.DEPLOYMENT_URL, { unsavedChangesWarning: false }); +let token = sessionStorage.getItem('session'); +window.getSessionToken = () => token; +if (token) client.setAuth(async () => token); +const status = document.querySelector('#status'); +const current = document.querySelector('#current-user'); +async function showUser() { + const user = await client.query('accountShop:current', {}); + current.textContent = user.name; + current.hidden = false; +} +if (token) showUser().catch(() => { status.textContent = 'Session rejected'; }); +document.querySelector('form').onsubmit = async (event) => { + event.preventDefault(); + try { + const result = await client.action('auth:signIn', { provider: 'password', params: { + username: document.querySelector('#signup-username').value, + password: document.querySelector('#signup-password').value, + flow: event.submitter.value, + }}); + token = result.tokens.token; + sessionStorage.setItem('session', token); + client.setAuth(async () => token); + await showUser(); + status.textContent = 'Signed in'; + } catch { status.textContent = 'Sign in rejected'; } +}; +document.querySelector('#purchase').onclick = async () => { + try { + await client.mutation('accountShop:purchase', { itemId: window.ITEM_ID, quantity: 1 }); + status.textContent = 'Purchased'; + } catch { status.textContent = 'Purchase rejected'; } +}; +document.querySelector('#signout').onclick = async () => { + await client.action('auth:signOut', {}); + token = null; + sessionStorage.removeItem('session'); + client.setAuth(async () => null); + current.hidden = true; + status.textContent = 'Signed out'; +}; diff --git a/tools/stack-bench/tests/fixtures/convex-native/convex.json b/tools/stack-bench/tests/fixtures/convex-native/convex.json new file mode 100644 index 00000000000..0a467a517d0 --- /dev/null +++ b/tools/stack-bench/tests/fixtures/convex-native/convex.json @@ -0,0 +1 @@ +{"functions":"convex/"} diff --git a/tools/stack-bench/tests/fixtures/convex-native/convex/accountShop.js b/tools/stack-bench/tests/fixtures/convex-native/convex/accountShop.js new file mode 100644 index 00000000000..651043ea6d0 --- /dev/null +++ b/tools/stack-bench/tests/fixtures/convex-native/convex/accountShop.js @@ -0,0 +1,25 @@ +import { mutationGeneric, queryGeneric } from 'convex/server'; +import { getAuthUserId, getAuthSessionId } from '@convex-dev/auth/server'; +import { v, ConvexError } from 'convex/values'; + +async function account(ctx) { + const userId = await getAuthUserId(ctx); + const sessionId = await getAuthSessionId(ctx); + const session = sessionId && await ctx.db.get(sessionId); + if (!userId || !session || session.userId !== userId || session.expirationTime <= Date.now()) { + throw new ConvexError('Sign in'); + } + return userId; +} +export const current = queryGeneric({ args: {}, handler: async (ctx) => { + const user = await ctx.db.get(await account(ctx)); + return { id: user._id, name: user.name }; +}}); +export const purchase = mutationGeneric({ args: { itemId: v.id('items'), quantity: v.number() }, handler: async (ctx, args) => { + const buyer = await account(ctx); + if (!Number.isSafeInteger(args.quantity) || args.quantity < 1) throw new ConvexError('Invalid quantity'); + const item = await ctx.db.get(args.itemId); + if (!item || item.stock < args.quantity) throw new ConvexError('Insufficient stock'); + await ctx.db.patch(item._id, { stock: item.stock - args.quantity }); + return ctx.db.insert('orders', { itemId: item._id, buyer, quantity: args.quantity }); +}}); diff --git a/tools/stack-bench/tests/fixtures/convex-native/convex/auth.js b/tools/stack-bench/tests/fixtures/convex-native/convex/auth.js new file mode 100644 index 00000000000..31835753c8d --- /dev/null +++ b/tools/stack-bench/tests/fixtures/convex-native/convex/auth.js @@ -0,0 +1,39 @@ +import { ConvexCredentials } from '@convex-dev/auth/providers/ConvexCredentials'; +import { convexAuth, createAccount, retrieveAccount } from '@convex-dev/auth/server'; +import { ConvexError } from 'convex/values'; +import { Scrypt } from 'lucia'; + +export const { auth, signIn, signOut, store } = convexAuth({ + providers: [ConvexCredentials({ + id: 'password', + crypto: { + hashSecret: (password) => new Scrypt().hash(password), + verifySecret: (password, hash) => new Scrypt().verify(hash, password), + }, + async authorize(params, ctx) { + if (typeof params.username !== 'string' || !/^[A-Za-z0-9-]{1,48}$/.test(params.username)) { + throw new ConvexError('Invalid username'); + } + if (typeof params.password !== 'string' || params.password.length > 64) { + throw new ConvexError('Invalid password'); + } + const account = { id: params.username, secret: params.password }; + if (params.flow === 'signUp') { + // createAccount atomically returns the new OR existing account. Only this + // request's server-generated marker proves it created the user. + const registrationNonce = crypto.randomUUID(); + const { user } = await createAccount(ctx, { + provider: 'password', account, + profile: { name: params.username, registrationNonce }, + shouldLinkViaEmail: false, shouldLinkViaPhone: false, + }); + if (user.registrationNonce !== registrationNonce) throw new ConvexError('Username already exists'); + return { userId: user._id }; + } + if (params.flow !== 'signIn') throw new ConvexError('Invalid authentication flow'); + const result = await retrieveAccount(ctx, { provider: 'password', account }); + if (!result) throw new ConvexError('Invalid credentials'); + return { userId: result.user._id }; + }, + })], +}); diff --git a/tools/stack-bench/tests/fixtures/convex-native/convex/http.js b/tools/stack-bench/tests/fixtures/convex-native/convex/http.js new file mode 100644 index 00000000000..d863f60fe88 --- /dev/null +++ b/tools/stack-bench/tests/fixtures/convex-native/convex/http.js @@ -0,0 +1,5 @@ +import { httpRouter } from 'convex/server'; +import { auth } from './auth.js'; +const http = httpRouter(); +auth.addHttpRoutes(http); +export default http; diff --git a/tools/stack-bench/tests/fixtures/convex-native/convex/schema.js b/tools/stack-bench/tests/fixtures/convex-native/convex/schema.js new file mode 100644 index 00000000000..3070e6518ad --- /dev/null +++ b/tools/stack-bench/tests/fixtures/convex-native/convex/schema.js @@ -0,0 +1,16 @@ +import { defineSchema, defineTable } from 'convex/server'; +import { v } from 'convex/values'; +import { authTables } from '@convex-dev/auth/server'; +export default defineSchema({ + ...authTables, + // Public Convex Auth schema customization; preserve its default fields/indexes. + users: defineTable({ + name: v.optional(v.string()), image: v.optional(v.string()), email: v.optional(v.string()), + emailVerificationTime: v.optional(v.number()), phone: v.optional(v.string()), + phoneVerificationTime: v.optional(v.number()), isAnonymous: v.optional(v.boolean()), + registrationNonce: v.optional(v.string()), + }).index('email', ['email']).index('phone', ['phone']), + items: defineTable({ name: v.string(), stock: v.number(), marker: v.string() }), + orders: defineTable({ itemId: v.id('items'), buyer: v.string(), quantity: v.number() }), + markers: defineTable({ value: v.string() }), +}); diff --git a/tools/stack-bench/tests/fixtures/convex-native/convex/shop.js b/tools/stack-bench/tests/fixtures/convex-native/convex/shop.js new file mode 100644 index 00000000000..1c8a5f9dc47 --- /dev/null +++ b/tools/stack-bench/tests/fixtures/convex-native/convex/shop.js @@ -0,0 +1,30 @@ +import { internalMutationGeneric, mutationGeneric, queryGeneric } from 'convex/server'; +import { v, ConvexError } from 'convex/values'; +export const seed = internalMutationGeneric({ args: {}, handler: async (ctx) => { + if ((await ctx.db.query('items').collect()).length) throw new Error('Seed needs a fresh deployment'); + await ctx.db.insert('items', { name: 'Widget', stock: 10, marker: 'preserve-target' }); + await ctx.db.insert('items', { name: 'Other', stock: 31, marker: 'preserve-other' }); + return null; +}}); +export const list = queryGeneric({ args: {}, handler: async (ctx) => { + if (!await ctx.auth.getUserIdentity()) throw new ConvexError('Sign in'); + return ctx.db.query('items').collect(); +}}); +export const purchase = mutationGeneric({ args: { itemId: v.id('items'), quantity: v.number() }, handler: async (ctx, args) => { + const who = await ctx.auth.getUserIdentity(); + if (!who || who.subject !== 'buyer') throw new ConvexError('Purchase denied'); + if (!Number.isSafeInteger(args.quantity) || args.quantity < 1) throw new ConvexError('Invalid quantity'); + const item = await ctx.db.get(args.itemId); + if (!item || item.stock < args.quantity) throw new ConvexError('Insufficient stock'); + await ctx.db.patch(item._id, { stock: item.stock - args.quantity }); + return ctx.db.insert('orders', { itemId: item._id, buyer: who.subject, quantity: args.quantity }); +}}); +export const deliberateError = mutationGeneric({ args: {}, handler: () => { throw new ConvexError('C0 deliberate error'); } }); +export const unhandledError = mutationGeneric({ args: {}, handler: () => { throw new Error('C0 unhandled error'); } }); +export const writeMarker = internalMutationGeneric({ args: { value: v.string() }, handler: async (ctx, { value }) => { + await ctx.db.insert('markers', { value }); +}}); +export const scheduleMarker = internalMutationGeneric({ args: { value: v.string(), delayMillis: v.number() }, handler: async (ctx, args) => { + const jobId = await ctx.scheduler.runAfter(args.delayMillis, 'shop:writeMarker', { value: args.value }); + return await ctx.db.system.get(jobId); +}}); diff --git a/tools/stack-bench/tests/fixtures/convex-native/package-lock.json b/tools/stack-bench/tests/fixtures/convex-native/package-lock.json new file mode 100644 index 00000000000..4bd41e1cff6 --- /dev/null +++ b/tools/stack-bench/tests/fixtures/convex-native/package-lock.json @@ -0,0 +1,753 @@ +{ + "name": "stackbench-convex-c0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "stackbench-convex-c0", + "dependencies": { + "@auth/core": "0.41.1", + "@convex-dev/auth": "0.0.95", + "convex": "1.46.0", + "lucia": "3.2.2" + } + }, + "node_modules/@auth/core": { + "version": "0.41.1", + "resolved": "https://registry.npmjs.org/@auth/core/-/core-0.41.1.tgz", + "integrity": "sha512-t9cJ2zNYAdWMacGRMT6+r4xr1uybIdmYa49calBPeTqwgAFPV/88ac9TEvCR85pvATiSPt8VaNf+Gt24JIT/uw==", + "license": "ISC", + "dependencies": { + "@panva/hkdf": "^1.2.1", + "jose": "^6.0.6", + "oauth4webapi": "^3.3.0", + "preact": "10.24.3", + "preact-render-to-string": "6.5.11" + }, + "peerDependencies": { + "@simplewebauthn/browser": "^9.0.1", + "@simplewebauthn/server": "^9.0.2", + "nodemailer": "^7.0.7" + }, + "peerDependenciesMeta": { + "@simplewebauthn/browser": { + "optional": true + }, + "@simplewebauthn/server": { + "optional": true + }, + "nodemailer": { + "optional": true + } + } + }, + "node_modules/@convex-dev/auth": { + "version": "0.0.95", + "resolved": "https://registry.npmjs.org/@convex-dev/auth/-/auth-0.0.95.tgz", + "integrity": "sha512-LgUujfXRm7FcdYpTozjn0MHdVFxaxvZdWVFOQEkrL4AkY+k/8E8B4EFjJOG9S8Pq6QWfLJ8MCObeMJg0hAfKzA==", + "license": "Apache-2.0", + "dependencies": { + "@oslojs/crypto": "^1.0.1", + "@oslojs/encoding": "^1.1.0", + "cookie": "^1.0.1", + "is-network-error": "^1.1.0", + "jose": "^5.2.2", + "jwt-decode": "^4.0.0", + "lucia": "^3.2.0", + "oauth4webapi": "^3.1.2", + "path-to-regexp": "^6.3.0", + "server-only": "^0.0.1" + }, + "bin": { + "auth": "dist/bin.cjs" + }, + "peerDependencies": { + "@auth/core": "^0.41.1", + "convex": "^1.17.0", + "react": "^18.2.0 || ^19.0.0-0" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + } + } + }, + "node_modules/@convex-dev/auth/node_modules/jose": { + "version": "5.10.0", + "resolved": "https://registry.npmjs.org/jose/-/jose-5.10.0.tgz", + "integrity": "sha512-s+3Al/p9g32Iq+oqXxkW//7jk2Vig6FF1CFqzVXoTUXt2qz89YWbL+OwS17NFYEvxC35n0FKeGO2LGYSxeM2Gg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.0.tgz", + "integrity": "sha512-KuZrd2hRjz01y5JK9mEBSD3Vj3mbCvemhT466rSuJYeE/hjuBrHfjjcjMdTm/sz7au+++sdbJZJmuBwQLuw68A==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.0.tgz", + "integrity": "sha512-j67aezrPNYWJEOHUNLPj9maeJte7uSMM6gMoxfPC9hOg8N02JuQi/T7ewumf4tNvJadFkvLZMlAq73b9uwdMyQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.0.tgz", + "integrity": "sha512-CC3vt4+1xZrs97/PKDkl0yN7w8edvU2vZvAFGD16n9F0Cvniy5qvzRXjfO1l94efczkkQE6g1x0i73Qf5uthOQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.0.tgz", + "integrity": "sha512-wurMkF1nmQajBO1+0CJmcN17U4BP6GqNSROP8t0X/Jiw2ltYGLHpEksp9MpoBqkrFR3kv2/te6Sha26k3+yZ9Q==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.0.tgz", + "integrity": "sha512-uJOQKYCcHhg07DL7i8MzjvS2LaP7W7Pn/7uA0B5S1EnqAirJtbyw4yC5jQ5qcFjHK9l6o/MX9QisBg12kNkdHg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.0.tgz", + "integrity": "sha512-8mG6arH3yB/4ZXiEnXof5MK72dE6zM9cDvUcPtxhUZsDjESl9JipZYW60C3JGreKCEP+p8P/72r69m4AZGJd5g==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.0.tgz", + "integrity": "sha512-9FHtyO988CwNMMOE3YIeci+UV+x5Zy8fI2qHNpsEtSF83YPBmE8UWmfYAQg6Ux7Gsmd4FejZqnEUZCMGaNQHQw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.0.tgz", + "integrity": "sha512-zCMeMXI4HS/tXvJz8vWGexpZj2YVtRAihHLk1imZj4efx1BQzN76YFeKqlDr3bUWI26wHwLWPd3rwh6pe4EV7g==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.0.tgz", + "integrity": "sha512-t76XLQDpxgmq2cNXKTVEB7O7YMb42atj2Re2Haf45HkaUpjM2J0UuJZDuaGbPbamzZ7bawyGFUkodL+zcE+jvQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.0.tgz", + "integrity": "sha512-AS18v0V+vZiLJyi/4LphvBE+OIX682Pu7ZYNsdUHyUKSoRwdnOsMf6FDekwoAFKej14WAkOef3zAORJgAtXnlQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.0.tgz", + "integrity": "sha512-Mz1jxqm/kfgKkc/KLHC5qIujMvnnarD9ra1cEcrs7qshTUSksPihGrWHVG5+osAIQ68577Zpww7SGapmzSt4Nw==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.0.tgz", + "integrity": "sha512-QbEREjdJeIreIAbdG2hLU1yXm1uu+LTdzoq1KCo4G4pFOLlvIspBm36QrQOar9LFduavoWX2msNFAAAY9j4BDg==", + "cpu": [ + "loong64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.0.tgz", + "integrity": "sha512-sJz3zRNe4tO2wxvDpH/HYJilb6+2YJxo/ZNbVdtFiKDufzWq4JmKAiHy9iGoLjAV7r/W32VgaHGkk35cUXlNOg==", + "cpu": [ + "mips64el" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.0.tgz", + "integrity": "sha512-z9N10FBD0DCS2dmSABDBb5TLAyF1/ydVb+N4pi88T45efQ/w4ohr/F/QYCkxDPnkhkp6AIpIcQKQ8F0ANoA2JA==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.0.tgz", + "integrity": "sha512-pQdyAIZ0BWIC5GyvVFn5awDiO14TkT/19FTmFcPdDec94KJ1uZcmFs21Fo8auMXzD4Tt+diXu1LW1gHus9fhFQ==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.0.tgz", + "integrity": "sha512-hPlRWR4eIDDEci953RI1BLZitgi5uqcsjKMxwYfmi4LcwyWo2IcRP+lThVnKjNtk90pLS8nKdroXYOqW+QQH+w==", + "cpu": [ + "s390x" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.0.tgz", + "integrity": "sha512-1hBWx4OUJE2cab++aVZ7pObD6s+DK4mPGpemtnAORBvb5l/g5xFGk0vc0PjSkrDs0XaXj9yyob3d14XqvnQ4gw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.0.tgz", + "integrity": "sha512-6m0sfQfxfQfy1qRuecMkJlf1cIzTOgyaeXaiVaaki8/v+WB+U4hc6ik15ZW6TAllRlg/WuQXxWj1jx6C+dfy3w==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.0.tgz", + "integrity": "sha512-xbbOdfn06FtcJ9d0ShxxvSn2iUsGd/lgPIO2V3VZIPDbEaIj1/3nBBe1AwuEZKXVXkMmpr6LUAgMkLD/4D2PPA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.0.tgz", + "integrity": "sha512-fWgqR8uNbCQ/GGv0yhzttj6sU/9Z5/Sv/VGU3F5OuXK6J6SlriONKrQ7tNlwBrJZXRYk5jUhuWvF7GYzGguBZQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.0.tgz", + "integrity": "sha512-aCwlRdSNMNxkGGqQajMUza6uXzR/U0dIl1QmLjPtRbLOx3Gy3otfFu/VjATy4yQzo9yFDGTxYDo1FfAD9oRD2A==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.0.tgz", + "integrity": "sha512-nyvsBccxNAsNYz2jVFYwEGuRRomqZ149A39SHWk4hV0jWxKM0hjBPm3AmdxcbHiFLbBSwG6SbpIcUbXjgyECfA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.0.tgz", + "integrity": "sha512-Q1KY1iJafM+UX6CFEL+F4HRTgygmEW568YMqDA5UV97AuZSm21b7SXIrRJDwXWPzr8MGr75fUZPV67FdtMHlHA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.0.tgz", + "integrity": "sha512-W1eyGNi6d+8kOmZIwi/EDjrL9nxQIQ0MiGqe/AWc6+IaHloxHSGoeRgDRKHFISThLmsewZ5nHFvGFWdBYlgKPg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.0.tgz", + "integrity": "sha512-30z1aKL9h22kQhilnYkORFYt+3wp7yZsHWus+wSKAJR8JtdfI76LJ4SBdMsCopTR3z/ORqVu5L1vtnHZWVj4cQ==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.0.tgz", + "integrity": "sha512-aIitBcjQeyOhMTImhLZmtxfdOcuNRpwlPNmlFKPcHQYPhEssw75Cl1TSXJXpMkzaua9FUetx/4OQKq7eJul5Cg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@oslojs/asn1": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@oslojs/asn1/-/asn1-1.0.0.tgz", + "integrity": "sha512-zw/wn0sj0j0QKbIXfIlnEcTviaCzYOY3V5rAyjR6YtOByFtJiT574+8p9Wlach0lZH9fddD4yb9laEAIl4vXQA==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "license": "MIT", + "dependencies": { + "@oslojs/binary": "1.0.0" + } + }, + "node_modules/@oslojs/binary": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@oslojs/binary/-/binary-1.0.0.tgz", + "integrity": "sha512-9RCU6OwXU6p67H4NODbuxv2S3eenuQ4/WFLrsq+K/k682xrznH5EVWA7N4VFk9VYVcbFtKqur5YQQZc0ySGhsQ==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "license": "MIT" + }, + "node_modules/@oslojs/crypto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@oslojs/crypto/-/crypto-1.0.1.tgz", + "integrity": "sha512-7n08G8nWjAr/Yu3vu9zzrd0L9XnrJfpMioQcvCMxBIiF5orECHe5/3J0jmXRVvgfqMm/+4oxlQ+Sq39COYLcNQ==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "license": "MIT", + "dependencies": { + "@oslojs/asn1": "1.0.0", + "@oslojs/binary": "1.0.0" + } + }, + "node_modules/@oslojs/encoding": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@oslojs/encoding/-/encoding-1.1.0.tgz", + "integrity": "sha512-70wQhgYmndg4GCPxPPxPGevRKqTIJ2Nh4OkiMWmDAVYsTQ+Ta7Sq+rPevXyXGdzr30/qZBnyOalCszoMxlyldQ==", + "license": "MIT" + }, + "node_modules/@panva/hkdf": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@panva/hkdf/-/hkdf-1.2.1.tgz", + "integrity": "sha512-6oclG6Y3PiDFcoyk8srjLfVKyMfVCKJ27JwNPViuXziFpmdz+MZnZN/aKY0JGXgYuO/VghU0jcOAZgWXZ1Dmrw==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, + "node_modules/convex": { + "version": "1.46.0", + "resolved": "https://registry.npmjs.org/convex/-/convex-1.46.0.tgz", + "integrity": "sha512-dhBwDQTZDOhi5SUGAX17EWNehCdNq6/UFrSL0eLZuGfMkdsQbCVWVRZCzbZ5CF+/pgvoGxtApbEoTSipqL4JTw==", + "license": "Apache-2.0", + "dependencies": { + "esbuild": "0.27.0", + "prettier": "^3.0.0", + "ws": "8.21.0" + }, + "bin": { + "convex": "bin/main.js" + }, + "engines": { + "node": ">=20.0.0", + "npm": ">=7.0.0" + }, + "peerDependencies": { + "@auth0/auth0-react": "^2.0.1", + "@clerk/clerk-react": "^4.12.8 || ^5.0.0", + "@clerk/react": "^6.4.3", + "react": "^18.0.0 || ^19.0.0-0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@auth0/auth0-react": { + "optional": true + }, + "@clerk/clerk-react": { + "optional": true + }, + "@clerk/react": { + "optional": true + }, + "react": { + "optional": true + } + } + }, + "node_modules/cookie": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", + "integrity": "sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/esbuild": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.0.tgz", + "integrity": "sha512-jd0f4NHbD6cALCyGElNpGAOtWxSq46l9X/sWB0Nzd5er4Kz2YTm+Vl0qKFT9KUJvD8+fiO8AvoHhFvEatfVixA==", + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.0", + "@esbuild/android-arm": "0.27.0", + "@esbuild/android-arm64": "0.27.0", + "@esbuild/android-x64": "0.27.0", + "@esbuild/darwin-arm64": "0.27.0", + "@esbuild/darwin-x64": "0.27.0", + "@esbuild/freebsd-arm64": "0.27.0", + "@esbuild/freebsd-x64": "0.27.0", + "@esbuild/linux-arm": "0.27.0", + "@esbuild/linux-arm64": "0.27.0", + "@esbuild/linux-ia32": "0.27.0", + "@esbuild/linux-loong64": "0.27.0", + "@esbuild/linux-mips64el": "0.27.0", + "@esbuild/linux-ppc64": "0.27.0", + "@esbuild/linux-riscv64": "0.27.0", + "@esbuild/linux-s390x": "0.27.0", + "@esbuild/linux-x64": "0.27.0", + "@esbuild/netbsd-arm64": "0.27.0", + "@esbuild/netbsd-x64": "0.27.0", + "@esbuild/openbsd-arm64": "0.27.0", + "@esbuild/openbsd-x64": "0.27.0", + "@esbuild/openharmony-arm64": "0.27.0", + "@esbuild/sunos-x64": "0.27.0", + "@esbuild/win32-arm64": "0.27.0", + "@esbuild/win32-ia32": "0.27.0", + "@esbuild/win32-x64": "0.27.0" + } + }, + "node_modules/is-network-error": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/is-network-error/-/is-network-error-1.3.2.tgz", + "integrity": "sha512-PhBY86zaxNZUuWP6h13Vu5oFe0XY6/UlKzQnYFELzGVHygP3MxmvTfYSG7GN3aIab/iWudSMgjSnG9Dq+nHrgA==", + "license": "MIT", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/jose": { + "version": "6.2.12", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.12.tgz", + "integrity": "sha512-9NiFmJEex0sy2Dk58j2UGBSHgUs2ypF9eZSu4L6vjOX3Dp96Sw1F3uL+H+D1sx02jZZdzUT0HgvCy59CuvXcWw==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, + "node_modules/jwt-decode": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jwt-decode/-/jwt-decode-4.0.0.tgz", + "integrity": "sha512-+KJGIyHgkGuIq3IEBNftfhW/LfWhXUIY6OmyVWjliu5KH1y0fw7VQ8YndE2O4qZdMSd9SqbnC8GOcZEy0Om7sA==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/lucia": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/lucia/-/lucia-3.2.2.tgz", + "integrity": "sha512-P1FlFBGCMPMXu+EGdVD9W4Mjm0DqsusmKgO7Xc33mI5X1bklmsQb0hfzPhXomQr9waWIBDsiOjvr1e6BTaUqpA==", + "deprecated": "This package has been deprecated. Please see https://lucia-auth.com/lucia-v3/migrate.", + "license": "MIT", + "dependencies": { + "@oslojs/crypto": "^1.0.1", + "@oslojs/encoding": "^1.1.0" + } + }, + "node_modules/oauth4webapi": { + "version": "3.8.8", + "resolved": "https://registry.npmjs.org/oauth4webapi/-/oauth4webapi-3.8.8.tgz", + "integrity": "sha512-8N28E+a/oxfXWBgOMt+ZP/JUf/XR+IFbvkAEPP3gznXOMv9BpAAwiIj0TFNz3tGTPc0ZQ8zmWBNgN1nAys0gng==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, + "node_modules/path-to-regexp": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", + "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==", + "license": "MIT" + }, + "node_modules/preact": { + "version": "10.24.3", + "resolved": "https://registry.npmjs.org/preact/-/preact-10.24.3.tgz", + "integrity": "sha512-Z2dPnBnMUfyQfSQ+GBdsGa16hz35YmLmtTLhM169uW944hYL6xzTYkJjC07j+Wosz733pMWx0fgON3JNw1jJQA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/preact" + } + }, + "node_modules/preact-render-to-string": { + "version": "6.5.11", + "resolved": "https://registry.npmjs.org/preact-render-to-string/-/preact-render-to-string-6.5.11.tgz", + "integrity": "sha512-ubnauqoGczeGISiOh6RjX0/cdaF8v/oDXIjO85XALCQjwQP+SB4RDXXtvZ6yTYSjG+PC1QRP2AhPgCEsM2EvUw==", + "license": "MIT", + "peerDependencies": { + "preact": ">=10" + } + }, + "node_modules/prettier": { + "version": "3.9.8", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.9.8.tgz", + "integrity": "sha512-WRFq3Wn3WId7LLROfMLdH7xaFr2jR62wU8nLO6rQUOLOxNZUviyJQs1M0iIhLexSFy+L+w0ch66wtoO2jRjG0A==", + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/server-only": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/server-only/-/server-only-0.0.1.tgz", + "integrity": "sha512-qepMx2JxAa5jjfzxG79yPPq+8BuFToHd1hm7kI+Z4zAq1ftQiP7HcxMhDDItrbtwVeLg/cY2JnKnrcFkmiswNA==", + "license": "MIT" + }, + "node_modules/ws": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + } + } +} diff --git a/tools/stack-bench/tests/fixtures/convex-native/package.json b/tools/stack-bench/tests/fixtures/convex-native/package.json new file mode 100644 index 00000000000..b724c9b85d6 --- /dev/null +++ b/tools/stack-bench/tests/fixtures/convex-native/package.json @@ -0,0 +1 @@ +{"name":"stackbench-convex-c0","private":true,"type":"module","dependencies":{"@auth/core":"0.41.1","@convex-dev/auth":"0.0.95","convex":"1.46.0","lucia":"3.2.2"}} \ No newline at end of file diff --git a/tools/stack-bench/tests/fixtures/convex-native/prepare-auth.mjs b/tools/stack-bench/tests/fixtures/convex-native/prepare-auth.mjs new file mode 100644 index 00000000000..092bc3bd8cf --- /dev/null +++ b/tools/stack-bench/tests/fixtures/convex-native/prepare-auth.mjs @@ -0,0 +1,14 @@ +import assert from 'node:assert/strict'; +import { generateKeyPairSync } from 'node:crypto'; +import { writeFileSync } from 'node:fs'; +import { spawnSync } from 'node:child_process'; +const { privateKey, publicKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); +for (const [name, value] of Object.entries({ + JWT_PRIVATE_KEY: privateKey.export({ type: 'pkcs8', format: 'pem' }).trimEnd(), + JWKS: JSON.stringify({ keys: [{ ...publicKey.export({ format: 'jwk' }), use: 'sig' }] }), +})) { + const result = spawnSync(process.execPath, ['node_modules/convex/bin/main.js', 'env', 'set', name, '--', value], { encoding: 'utf8' }); + assert.equal(result.status, 0, `Could not set ${name}: ${result.stderr}`); +} +writeFileSync('convex/auth.config.js', 'export default { providers: [{ domain: process.env.CONVEX_SITE_URL, applicationID: "convex" }] };\n'); +console.log('Configured local Convex Auth signing and verification. No external identity service.'); diff --git a/tools/stack-bench/tests/fixtures/convex-native/prepare.mjs b/tools/stack-bench/tests/fixtures/convex-native/prepare.mjs new file mode 100644 index 00000000000..d8b53c2ddb3 --- /dev/null +++ b/tools/stack-bench/tests/fixtures/convex-native/prepare.mjs @@ -0,0 +1,17 @@ +// Disposable C0 identities only. This is not an account implementation or qualification. +import { generateKeyPairSync, sign } from 'node:crypto'; +import { writeFileSync } from 'node:fs'; +const { privateKey, publicKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); +const jwk = { ...publicKey.export({ format: 'jwk' }), kid: 'c0', alg: 'RS256', use: 'sig' }; +const issuer = 'https://stackbench-c0.invalid'; +const audience = 'stackbench-c0'; +const jwks = 'data:text/plain;charset=utf-8;base64,' + Buffer.from(JSON.stringify({ keys: [jwk] })).toString('base64'); +writeFileSync('convex/auth.config.js', `export default ${JSON.stringify({ providers: [{ type: 'customJwt', issuer, applicationID: audience, algorithm: 'RS256', jwks }] })};\n`); +const b64 = (value) => Buffer.from(JSON.stringify(value)).toString('base64url'); +const token = (sub) => { + const iat = Math.floor(Date.now() / 1000); + const payload = b64({ alg: 'RS256', typ: 'JWT', kid: 'c0' }) + '.' + b64({ iss: issuer, aud: audience, sub, iat, exp: iat + 7200 }); + return payload + '.' + sign('RSA-SHA256', Buffer.from(payload), privateKey).toString('base64url'); +}; +writeFileSync('identities.json', JSON.stringify({ buyer: token('buyer'), observer: token('observer') }), { mode: 0o600 }); +console.log('Wrote public auth configuration and two disposable signed actor tokens.'); diff --git a/tools/stack-bench/tests/fixtures/convex-native/probe-auth.mjs b/tools/stack-bench/tests/fixtures/convex-native/probe-auth.mjs new file mode 100644 index 00000000000..25dce31e1d5 --- /dev/null +++ b/tools/stack-bench/tests/fixtures/convex-native/probe-auth.mjs @@ -0,0 +1,150 @@ +/* global document, window -- used inside page.evaluate callbacks */ +import assert from 'node:assert/strict'; +import { writeFileSync } from 'node:fs'; +import { createServer } from 'node:http'; +import { once } from 'node:events'; +import { build } from 'esbuild'; +import { ConvexHttpClient } from 'convex/browser'; +import { snapshot } from './snapshot.mjs'; +const evidence = { result: 'running', observations: [], compatibilityFailures: [], note: 'Local account feasibility only, not qualification.' }; +const record = (name) => evidence.observations.push(name); +const save = () => writeFileSync('auth-evidence.json', JSON.stringify(evidence, null, 2)); +save(); +let browser; let server; +try { + const url = process.env.CONVEX_SELF_HOSTED_URL; + const adminKey = process.env.CONVEX_SELF_HOSTED_ADMIN_KEY; + assert(url && adminKey); + const admin = new ConvexHttpClient(url); admin.setAdminAuth(adminKey); + await admin.mutation('shop:seed', {}); + const initial = await snapshot(url, adminKey); + const item = initial.tables.items.find(row => row.name === 'Widget'); + const auth = async (username, password, flow = 'signUp') => new ConvexHttpClient(url).action('auth:signIn', { + provider: 'password', params: { username, password, flow }, + }); + const caller = (token) => { const client = new ConvexHttpClient(url); if (token) client.setAuth(token); return client; }; + const buy = (token, extra = {}) => caller(token).mutation('accountShop:purchase', { itemId: item._id, quantity: 1, ...extra }); + const noEffects = async () => assert.deepEqual((await snapshot(url, adminKey)).tables, initial.tables); + const buyer = await auth('Buyer-1', 'a'); + assert(buyer.tokens?.token); + const identity = await caller(buyer.tokens.token).query('accountShop:current', {}); + assert.equal(identity.name, 'Buyer-1'); record('one-character password and username display preserved'); + await assert.rejects(auth('Buyer-1', 'wrong', 'signIn')); + try { + const duplicate = await auth('Buyer-1', 'a'); + assert.equal((await caller(duplicate.tokens.token).query('accountShop:current', {})).id, identity.id); + evidence.compatibilityFailures.push('Taken username with the correct existing password issues a session during signUp; account check 1c requires refusal.'); + } catch (error) { + assert.match(error.message, /already exists/, 'Only a measured duplicate refusal satisfies this boundary'); + record('duplicate signup refused'); + } + await assert.rejects(auth('buyer-1', 'a', 'signIn')); + const differentCase = await auth('buyer-1', 'b'); + assert.notEqual((await caller(differentCase.tokens.token).query('accountShop:current', {})).id, identity.id); + record('wrong password refused; case variants are distinct accounts'); + evidence.registrationRaces = []; + for (const samePassword of [true, false]) { + for (let round = 0; round < 3; round++) { + const name = `Race-${samePassword ? 'same' : 'different'}-${round}`; + const passwords = Array.from({ length: 6 }, (_, i) => `RacePassword-${samePassword ? 0 : i}`); + const results = await Promise.allSettled(passwords.map(password => auth(name, password))); + const winners = results.flatMap((result, i) => result.status === 'fulfilled' ? [i] : []); + assert.equal(winners.length, 1, `Exactly one session for ${name}`); + for (const result of results) if (result.status === 'rejected') assert.match(result.reason.message, /already exists/); + const stored = (await snapshot(url, adminKey, ['users', 'authAccounts', 'authSessions'])).tables; + const accounts = stored.authAccounts.filter(row => row.providerAccountId === name); + assert.equal(accounts.length, 1); + assert.equal(stored.users.filter(row => row.name === name).length, 1); + assert.equal(stored.authSessions.filter(row => row.userId === accounts[0].userId).length, 1); + assert.equal(typeof accounts[0].secret, 'string'); + assert(!passwords.includes(accounts[0].secret), 'Store a password hash, not the supplied password'); + const winner = winners[0]; + assert.equal((await caller(results[winner].value.tokens.token).query('accountShop:current', {})).id, accounts[0].userId); + const login = await auth(name, passwords[winner], 'signIn'); + assert(login.tokens.token, 'Winning account must remain usable'); + if (!samePassword) await assert.rejects(auth(name, passwords[(winner + 1) % 6], 'signIn')); + evidence.registrationRaces.push({ samePassword, requests: 6, accepted: 1, refused: 5, accounts: 1, users: 1, initialSessions: 1 }); + } + } + await noEffects(); + record('36 concurrent signups: six same/different-password races each create exactly one account, user and initial session'); + const boundary = await auth('Z'.repeat(48), 'p'.repeat(64)); + assert(boundary.tokens.token); + const emptyPassword = await auth('EmptyPassword', ''); assert(emptyPassword.tokens.token); + for (const [name, password] of [['Z'.repeat(49), 'a'], ['Invalid_Name', 'a'], ['TooLongPassword', 'p'.repeat(65)]]) { + await assert.rejects(auth(name, password)); + } + record('48-character username and 64-character password accepted; invalid bounds refused; no password minimum added'); + await assert.rejects(buy(undefined)); + const parts = buyer.tokens.token.split('.'); + const payload = JSON.parse(Buffer.from(parts[1], 'base64url')); + parts[1] = Buffer.from(JSON.stringify({ ...payload, sub: 'forged' })).toString('base64url'); + await assert.rejects(buy(parts.join('.'))); + await assert.rejects(buy(buyer.tokens.token, { buyer: identity.id })); + await noEffects(); record('anonymous, forged token and extra caller field refused with no order or stock effects'); + await buy(buyer.tokens.token); + const after = await snapshot(url, adminKey); + assert.equal(after.tables.orders.length, 1); + assert.equal(after.tables.orders[0].buyer, identity.id); + assert.equal(after.tables.items.find(row => row._id === item._id).stock, 9); + record('real issued session purchased and stored the authenticated user ID'); + await caller(buyer.tokens.token).action('auth:signOut', {}); + await assert.rejects(buy(buyer.tokens.token)); + assert.deepEqual((await snapshot(url, adminKey)).tables, after.tables); + record('signed-out token replay refused without effects'); + + // The deployment key never enters the page; its calls use the token returned by signIn. + const { outputFiles: [{ contents: script }] } = await build({ entryPoints: ['auth-browser.js'], bundle: true, write: false, platform: 'browser', format: 'esm' }); + server = createServer((request, response) => { + if (request.url === '/app.js') { response.setHeader('Content-Type', 'text/javascript'); response.end(script); return; } + response.setHeader('Content-Type', 'text/html'); + response.end(`<form><input id="signup-username" aria-label="Username"><input id="signup-password" type="password" aria-label="Password"><button id="signup-submit" value="signUp">Sign up</button><button id="signin-submit" value="signIn">Sign in</button></form><span id="current-user" hidden></span><button id="purchase">Purchase</button><button id="signout">Sign out</button><p id="status"></p><script>window.DEPLOYMENT_URL=${JSON.stringify(url)};window.ITEM_ID=${JSON.stringify(item._id)};</script><script type="module" src="/app.js"></script>`); + }); + server.listen(0, '127.0.0.1'); await once(server, 'listening'); + const { chromium } = await import(process.env.PLAYWRIGHT_MODULE ?? '/opt/stack-bench/node_modules/playwright/index.mjs'); + browser = await chromium.launch({ headless: true }); + const page = await browser.newPage(); + await page.goto(`http://127.0.0.1:${server.address().port}`); + await page.locator('#signup-username').fill('BrowserUser'); + await page.locator('#signup-password').fill('BrowserSecret42'); + await page.locator('#signup-submit').click(); + await page.waitForFunction(() => document.querySelector('#status').textContent === 'Signed in'); + assert.equal(await page.locator('#current-user').textContent(), 'BrowserUser'); + assert(await page.evaluate(() => typeof window.getSessionToken() === 'string')); + await page.reload(); + await page.waitForFunction(() => document.querySelector('#current-user').textContent === 'BrowserUser'); + await page.locator('#purchase').click(); + await page.waitForFunction(() => document.querySelector('#status').textContent === 'Purchased'); + const browserToken = await page.evaluate(() => window.getSessionToken()); + const browserUser = await caller(browserToken).query('accountShop:current', {}); + const browserState = await snapshot(url, adminKey); + assert.equal(browserState.tables.orders.length, 2); + assert(browserState.tables.orders.some(row => row.buyer === browserUser.id)); + await page.locator('#signout').click(); + await page.waitForFunction(() => document.querySelector('#status').textContent === 'Signed out'); + assert.equal(await page.evaluate(() => window.getSessionToken()), null); + await assert.rejects(buy(browserToken)); + assert.deepEqual((await snapshot(url, adminKey)).tables, browserState.tables); + await page.locator('#signup-username').fill('BrowserUser'); + await page.locator('#signup-password').fill('WrongPassword42'); + await page.locator('#signin-submit').click(); + await page.waitForFunction(() => document.querySelector('#status').textContent === 'Sign in rejected'); + assert.equal(await page.evaluate(() => window.getSessionToken()), null); + await page.locator('#purchase').click(); + await page.waitForFunction(() => document.querySelector('#status').textContent === 'Purchase rejected'); + assert.deepEqual((await snapshot(url, adminKey)).tables, browserState.tables); + await page.locator('#signup-password').fill('BrowserSecret42'); + await page.locator('#signin-submit').click(); + await page.waitForFunction(() => document.querySelector('#status').textContent === 'Signed in'); + record('Chromium signup, native WebSocket operation, reload, session hook and signout work with actual issued token'); + record('Chromium wrong-password login creates no session and cannot purchase; correct-password login succeeds'); + evidence.result = evidence.compatibilityFailures.length ? 'incompatible' : 'passed'; evidence.final = browserState; + if (evidence.compatibilityFailures.length) process.exitCode = 1; +} catch (error) { + evidence.result = 'failed'; evidence.error = error.stack ?? String(error); process.exitCode = 1; +} finally { + try { if (browser) await browser.close(); if (server) { server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); } } + catch (error) { evidence.result = 'failed'; evidence.cleanupError = String(error); process.exitCode = 1; } + save(); +} +console.log(JSON.stringify({ result: evidence.result, observations: evidence.observations, error: evidence.error })); diff --git a/tools/stack-bench/tests/fixtures/convex-native/probe-lifecycle-browser.mjs b/tools/stack-bench/tests/fixtures/convex-native/probe-lifecycle-browser.mjs new file mode 100644 index 00000000000..31097883035 --- /dev/null +++ b/tools/stack-bench/tests/fixtures/convex-native/probe-lifecycle-browser.mjs @@ -0,0 +1,28 @@ +/* global document, window -- used inside page.evaluate callbacks */ +import assert from 'node:assert/strict'; +import { writeFileSync } from 'node:fs'; +const [url, username, output] = process.argv.slice(2); +assert(url && username && output && process.env.PLAYWRIGHT_MODULE); +const evidence = { result: 'running', url, username }; +const save = () => writeFileSync(output, JSON.stringify(evidence, null, 2)); +save(); +let browser; +try { + const { chromium } = await import(process.env.PLAYWRIGHT_MODULE); + browser = await chromium.launch({ headless: true }); + const page = await browser.newPage(); + await page.goto(url); + await page.locator('#signup-username').fill(username); + await page.locator('#signup-password').fill('LifecycleBrowser42'); + await page.locator('#signup-submit').click(); + await page.waitForFunction(() => document.querySelector('#status').textContent === 'Signed in'); + assert.equal(await page.locator('#current-user').textContent(), username); + assert(await page.evaluate(() => typeof window.getSessionToken() === 'string')); + await page.locator('#purchase').click(); + await page.waitForFunction(() => document.querySelector('#status').textContent === 'Purchased'); + await page.reload(); + await page.waitForFunction(name => document.querySelector('#current-user').textContent === name, username); + evidence.result = 'passed'; +} catch (error) { evidence.result = 'failed'; evidence.error = error.stack; process.exitCode = 1; } +finally { if (browser) await browser.close(); save(); } +console.log(JSON.stringify(evidence)); diff --git a/tools/stack-bench/tests/fixtures/convex-native/probe-lifecycle.mjs b/tools/stack-bench/tests/fixtures/convex-native/probe-lifecycle.mjs new file mode 100644 index 00000000000..64a9cf178fb --- /dev/null +++ b/tools/stack-bench/tests/fixtures/convex-native/probe-lifecycle.mjs @@ -0,0 +1,88 @@ +import assert from 'node:assert/strict'; +import { writeFileSync, readFileSync } from 'node:fs'; +import { createServer } from 'node:http'; +import { setTimeout as delay } from 'node:timers/promises'; +import { build } from 'esbuild'; +import { ConvexHttpClient } from 'convex/browser'; +import { snapshot } from './snapshot.mjs'; +const phase = process.argv[2]; +const url = process.env.CONVEX_SELF_HOSTED_URL; +const key = process.env.CONVEX_SELF_HOSTED_ADMIN_KEY; +assert(url && key); +const admin = new ConvexHttpClient(url); admin.setAdminAuth(key); +const state = () => snapshot(url, key, ['items', 'orders', 'users', 'markers']); +const auth = (flow) => new ConvexHttpClient(url).action('auth:signIn', { provider: 'password', params: { + username: 'LifecycleUser', password: 'LifecycleSecret42', flow, +}}); +const path = `lifecycle-${phase}.json`; +const evidence = { result: 'running', phase }; +const save = () => writeFileSync(path, JSON.stringify(evidence, null, 2)); +save(); +try { + if (phase === 'prepare') { + await admin.mutation('shop:seed', {}); + const tokens = (await auth('signUp')).tokens; + const actor = new ConvexHttpClient(url); actor.setAuth(tokens.token); + const user = await actor.query('accountShop:current', {}); + const item = (await state()).tables.items.find(row => row.name === 'Widget'); + await actor.mutation('accountShop:purchase', { itemId: item._id, quantity: 1 }); + const scheduled = await admin.mutation('shop:scheduleMarker', { value: 'scheduler-positive', delayMillis: 100 }); + assert.equal(scheduled.state.kind, 'pending'); + const deadline = Date.now() + 10000; + while (!(await state()).tables.markers.some(row => row.value === 'scheduler-positive')) { + assert(Date.now() < deadline, 'Scheduler positive control must execute'); await delay(100); + } + writeFileSync('lifecycle-private.json', JSON.stringify({ token: tokens.token, user, itemId: item._id }), { mode: 0o600 }); + evidence.schedulerPositive = true; + } else if (phase === 'serve') { + const saved = JSON.parse(readFileSync('lifecycle-private.json')); + const { outputFiles: [{ contents: script }] } = await build({ entryPoints: ['auth-browser.js'], bundle: true, write: false, platform: 'browser', format: 'esm' }); + createServer((request, response) => { + if (request.url === '/app.js') { response.setHeader('Content-Type', 'text/javascript'); response.end(script); return; } + const api = new URL(request.url, 'http://localhost').searchParams.get('api') ?? url; + response.setHeader('Content-Type', 'text/html'); + response.end(`<form><input id="signup-username" aria-label="Username"><input id="signup-password" type="password" aria-label="Password"><button id="signup-submit" value="signUp">Sign up</button><button id="signin-submit" value="signIn">Sign in</button></form><span id="current-user" hidden></span><button id="purchase">Purchase</button><button id="signout">Sign out</button><p id="status"></p><script>window.DEPLOYMENT_URL=${JSON.stringify(api)};window.ITEM_ID=${JSON.stringify(saved.itemId)};</script><script type="module" src="/app.js"></script>`); + }).listen(3100, '0.0.0.0'); + evidence.result = 'serving'; save(); + } else if (phase === 'capture') { + const before = await state(); + assert.equal(before.tables.orders.length, 3, 'Host, container and initial account each purchased'); + assert.equal(before.tables.items.find(row => row.name === 'Widget').stock, 7); + assert.deepEqual(before.tables.users.map(row => row.name).sort(), ['ContainerUser', 'HostUser', 'LifecycleUser']); + evidence.tables = before.tables; + } else if (phase === 'warm') { + const before = JSON.parse(readFileSync('lifecycle-capture.json')); + assert.equal(before.result, 'passed'); + assert.deepEqual((await state()).tables, before.tables); + const saved = JSON.parse(readFileSync('lifecycle-private.json')); + const actor = new ConvexHttpClient(url); actor.setAuth(saved.token); + assert.deepEqual(await actor.query('accountShop:current', {}), saved.user); + assert((await auth('signIn')).tokens.token, 'Stored password login survives restart'); + evidence.dataAndSavedSessionPreserved = true; + } else if (phase === 'schedule-reset') { + const scheduled = await admin.mutation('shop:scheduleMarker', { value: 'must-not-survive-reset', delayMillis: 20000 }); + assert.equal(scheduled.state.kind, 'pending'); + evidence.scheduled = { id: scheduled._id, time: scheduled.scheduledTime, state: scheduled.state.kind }; + } else if (phase === 'reset') { + const scheduled = JSON.parse(readFileSync('lifecycle-schedule-reset.json')); + assert.equal(scheduled.result, 'passed'); + const empty = (await state()).tables; + assert(Object.values(empty).every(rows => rows.length === 0), 'Fresh deployment must have no app data or accounts'); + const saved = JSON.parse(readFileSync('lifecycle-private.json')); + const actor = new ConvexHttpClient(url); actor.setAuth(saved.token); + await assert.rejects(actor.query('accountShop:current', {})); + await assert.rejects(auth('signIn')); + const remaining = scheduled.scheduled.time - Date.now() + 500; + assert(Number.isFinite(remaining) && remaining < 30000, 'Bound reset observation wait'); + if (remaining > 0) await delay(remaining); + assert(Object.values((await state()).tables).every(rows => rows.length === 0), 'Old scheduled work must not run after its due time'); + await admin.mutation('shop:seed', {}); + const fresh = await auth('signUp'); + assert(fresh.tokens.token, 'Same username can register on the fresh deployment'); + evidence.oldDataAccountsAndScheduledWorkAbsent = true; + evidence.waitedMillis = Math.max(0, remaining); + } else throw new Error(`Unknown phase: ${phase}`); + if (phase !== 'serve') evidence.result = 'passed'; +} catch (error) { evidence.result = 'failed'; evidence.error = error.stack; process.exitCode = 1; } +finally { save(); } +console.log(JSON.stringify(evidence)); diff --git a/tools/stack-bench/tests/fixtures/convex-native/probe-owned.mjs b/tools/stack-bench/tests/fixtures/convex-native/probe-owned.mjs new file mode 100644 index 00000000000..bc50c6ed8d0 --- /dev/null +++ b/tools/stack-bench/tests/fixtures/convex-native/probe-owned.mjs @@ -0,0 +1,140 @@ +import assert from 'node:assert/strict'; +import { readFileSync, writeFileSync } from 'node:fs'; +import { setTimeout as delay } from 'node:timers/promises'; +import { isDeepStrictEqual } from 'node:util'; +import { ConvexClient, ConvexHttpClient } from 'convex/browser'; +import { snapshot } from './snapshot.mjs'; + +const { CONVEX_SELF_HOSTED_URL: url, CONVEX_SELF_HOSTED_ADMIN_KEY: key } = JSON.parse(readFileSync('owned-private.json')); +const phase = process.argv[2] ?? 'initial'; +const evidence = { result: 'running', phase, transport: 'native HTTP and WebSocket', observations: [] }; +const state = () => snapshot(url, key, ['items', 'orders', 'users', 'markers', 'authAccounts', 'authSessions', 'authRefreshTokens']); +const auth = new ConvexHttpClient(url); +const credentials = { username: 'OwnedLifecycle', password: 'OwnedPassword42' }; +const signIn = flow => auth.action('auth:signIn', { provider: 'password', params: { ...credentials, flow } }); +const readSaved = () => JSON.parse(readFileSync('lifecycle-private.json')); +const savePrivate = value => writeFileSync('lifecycle-private.json', JSON.stringify(value), { mode: 0o600 }); +let live; +async function currentOverWebSocket(token) { + live = new ConvexClient(url, { unsavedChangesWarning: false }); + live.setAuth(async () => token); + let current; + live.onUpdate('accountShop:current', {}, value => { current = value; }); + const deadline = Date.now() + 10000; + while (!current) { assert(Date.now() < deadline, 'Signed WebSocket query did not complete'); await delay(50); } + return current; +} +try { + const admin = new ConvexHttpClient(url); admin.setAdminAuth(key); + if (phase === 'initial') { + await admin.mutation('shop:seed', {}); + const { tokens } = await signIn('signUp'); + assert(tokens?.token); + const actor = new ConvexHttpClient(url); actor.setAuth(tokens.token); + assert.equal((await actor.query('accountShop:current', {})).name, 'OwnedLifecycle'); + const item = (await state()).tables.items.find(row => row.name === 'Widget'); + const current = await currentOverWebSocket(tokens.token); + assert.equal(current.name, 'OwnedLifecycle'); + await live.mutation('accountShop:purchase', { itemId: item._id, quantity: 1 }); + await assert.rejects(auth.mutation('accountShop:purchase', { itemId: item._id, quantity: 1 })); + const after = (await state()).tables; + assert.equal(after.orders.length, 1, 'Anonymous refusal must not write an order'); + assert.equal(after.orders[0].buyer, current.id); + assert.equal(after.items.find(row => row._id === item._id).stock, 9); + evidence.observations = ['real password signup', 'signed HTTP query', 'signed WebSocket query and purchase', + 'anonymous refusal with no effect', 'independent stored order and stock']; + const scheduled = await admin.mutation('shop:scheduleMarker', { value: 'owned-scheduler-positive', delayMillis: 100 }); + assert.equal(scheduled.state.kind, 'pending'); + const schedulerDeadline = Date.now() + 10000; + while (!(await state()).tables.markers.some(row => row.value === 'owned-scheduler-positive')) { + assert(Date.now() < schedulerDeadline, 'Positive scheduler control must execute'); await delay(100); + } + savePrivate({ token: tokens.token, adminKey: key, user: current, before: (await state()).tables }); + evidence.observations.push('positive scheduled effect'); + } else if (phase === 'validation') { + const saved = readSaved(); + const item = saved.before.items.find(row => row.name === 'Widget'); + const responses = []; + const functions = await admin.query('_system/cli/modules:apiSpec', {}); + const purchase = functions.filter(fn => fn.identifier === 'accountShop.js:purchase'); + assert.equal(purchase.length, 1); + assert.equal(purchase[0].functionType, 'Mutation'); + assert.equal(purchase[0].visibility.kind, 'public'); + assert(!functions.some(fn => fn.identifier === 'accountShop.js:missing')); + evidence.functionMetadata = purchase; + for (const [path, args] of [ + ['accountShop:purchase', { itemId: item._id, quantity: 1, username: 'ForgedOwner' }], + ['accountShop:missing', { itemId: item._id, quantity: 1 }], + ['shop:unhandledError', {}], + ]) { + const response = await fetch(`${url}/api/mutation`, { method: 'POST', + headers: { Authorization: `Bearer ${saved.token}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ path, args, format: 'json' }), signal: AbortSignal.timeout(10000) }); + const text = await response.text(); + const parsed = JSON.parse(text); + assert.notEqual(parsed.status, 'success'); + responses.push({ path, status: response.status, body: parsed }); + assert(isDeepStrictEqual((await state()).tables, saved.before), 'Rejected native request changed stored data'); + } + const actor = new ConvexHttpClient(url); actor.setAuth(saved.token); + await actor.mutation('accountShop:purchase', { itemId: item._id, quantity: 1 }); + const after = (await state()).tables; + assert.equal(after.orders.length, saved.before.orders.length + 1, 'Valid operation must still work'); + assert.equal(after.items.find(row => row._id === item._id).stock, item.stock - 1); + savePrivate({ ...saved, before: after }); + evidence.responses = responses; + evidence.observations = ['extra-field schema rejection', 'missing function control', 'unhandled error control', + 'independent unchanged stored state', 'valid authenticated operation still works']; + } else if (phase === 'isolation') { + const saved = readSaved(); + const foreign = JSON.parse(readFileSync('foreign-private.json')); + const denied = new ConvexHttpClient(url); denied.setAuth(foreign.token); + const item = saved.before.items.find(row => row.name === 'Widget'); + await assert.rejects(denied.mutation('accountShop:purchase', { itemId: item._id, quantity: 1 })); + await assert.rejects(snapshot(url, foreign.adminKey)); + assert(isDeepStrictEqual((await state()).tables, saved.before), 'Foreign credentials must have no stored effect'); + assert.deepEqual(await currentOverWebSocket(saved.token), saved.user); + evidence.observations = ['other deployment JWT/admin key refused', 'exact own data unchanged', 'own original JWT works']; + } else if (phase === 'warm' || phase === 'retained') { + const saved = readSaved(); + assert(isDeepStrictEqual((await state()).tables, saved.before), 'Warm restart must retain exact stored rows'); + assert.deepEqual(await currentOverWebSocket(saved.token), saved.user); + evidence.observations = ['exact data/accounts/sessions retained', 'original JWT still works over WebSocket']; + if (phase === 'warm') { + assert((await signIn('signIn')).tokens.token, 'Password account must remain usable'); + evidence.observations.push('password login retained'); + } + } else if (phase === 'prepare-reset') { + const saved = readSaved(); + const scheduled = await admin.mutation('shop:scheduleMarker', { value: 'owned-must-not-survive-reset', delayMillis: 30000 }); + assert.equal(scheduled.state.kind, 'pending'); + savePrivate({ ...saved, scheduled: { id: scheduled._id, time: scheduled.scheduledTime } }); + evidence.observations = ['old scheduled operation is pending']; + } else if (phase === 'reset') { + const saved = readSaved(); + assert(Object.values((await state()).tables).every(rows => rows.length === 0), 'Reset requires empty stored state'); + const old = new ConvexHttpClient(url); old.setAuth(saved.token); + await assert.rejects(old.query('accountShop:current', {})); + await assert.rejects(signIn('signIn')); + await assert.rejects(snapshot(url, saved.adminKey)); + const remaining = saved.scheduled.time - Date.now() + 500; + assert(Number.isFinite(remaining) && remaining < 35000, 'Bound scheduler observation'); + if (remaining > 0) await delay(remaining); + assert(Object.values((await state()).tables).every(rows => rows.length === 0), 'Old scheduled work must remain absent'); + await admin.mutation('shop:seed', {}); + const fresh = new ConvexHttpClient(url); fresh.setAuth((await signIn('signUp')).tokens.token); + assert.equal((await fresh.query('accountShop:current', {})).name, credentials.username); + const item = (await state()).tables.items.find(row => row.name === 'Widget'); + await fresh.mutation('accountShop:purchase', { itemId: item._id, quantity: 1 }); + assert.equal((await state()).tables.orders.length, 1); + evidence.observations = ['old data/accounts/sessions absent', 'old JWT/password/admin key refused', + 'old scheduled work absent past deadline', 'fresh account and purchase usable']; + evidence.waitedMillis = Math.max(0, remaining); + } else throw new Error(`Unknown owned lifecycle probe phase ${phase}`); + evidence.result = 'passed'; +} catch (error) { evidence.result = 'failed'; evidence.error = error.stack; process.exitCode = 1; } +finally { + await live?.close(); + writeFileSync(`owned-probe${phase === 'initial' ? '' : `-${phase}`}.json`, JSON.stringify(evidence, null, 2)); +} +console.log(JSON.stringify(evidence)); diff --git a/tools/stack-bench/tests/fixtures/convex-native/probe.mjs b/tools/stack-bench/tests/fixtures/convex-native/probe.mjs new file mode 100644 index 00000000000..8b2f8251955 --- /dev/null +++ b/tools/stack-bench/tests/fixtures/convex-native/probe.mjs @@ -0,0 +1,98 @@ +import assert from 'node:assert/strict'; +import { readFileSync, writeFileSync } from 'node:fs'; +import { createServer } from 'node:http'; +import { once } from 'node:events'; +import { ConvexClient, ConvexHttpClient } from 'convex/browser'; +import { snapshot } from './snapshot.mjs'; +const evidence = { result: 'running', observations: [], snapshots: [], note: 'Pinned vendor-internal administrative write; not release qualification.' }; +writeFileSync('probe-evidence.json', JSON.stringify(evidence, null, 2)); +async function probe() { +const { convexFunctionRequest, classifyConvexFunctionResponse } = await import( + process.env.STACK_BENCH_CONVEX_PROTOCOL ?? new URL('../../../dist/src/stacks/backends/convex-protocol.js', import.meta.url)); +const url = process.env.CONVEX_SELF_HOSTED_URL; +const adminKey = process.env.CONVEX_SELF_HOSTED_ADMIN_KEY; +assert(url && adminKey, 'Use this fixture deployment URL and admin key'); +const tokens = JSON.parse(readFileSync('identities.json', 'utf8')); +const admin = new ConvexHttpClient(url); +admin.setAdminAuth(adminKey); +const state = async (table) => { + const result = await snapshot(url, adminKey); + evidence.snapshots.push(result); + return result.tables[table]; +}; +const call = async (path, args, token) => { + const request = convexFunctionRequest({ deploymentUrl: url, kind: 'mutation', path, args, token }); + const response = await fetch(request.url, { ...request, signal: AbortSignal.timeout(10000) }); + const raw = await response.text(); + const result = classifyConvexFunctionResponse(response.status, raw); + evidence.observations.push({ path, result, raw }); + return result; +}; +await admin.mutation('shop:seed', {}); +const initial = await state('items'); +const item = initial.find((x) => x.name === 'Widget'); +const other = initial.find((x) => x.name === 'Other'); +const live = new ConvexClient(url, { unsavedChangesWarning: false }); +live.setAuth(async () => tokens.observer); +let latest; const stop = live.onUpdate('shop:list', {}, (rows) => { latest = rows; }); +const waitStock = async (stock) => { + const deadline = Date.now() + 10000; + while (Date.now() < deadline) { + if (latest?.find((x) => x._id === item._id)?.stock === stock) return; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw new Error(`Subscription did not observe stock ${stock}`); +}; +try { + await waitStock(10); + assert.equal((await call('shop:purchase', { itemId: item._id, quantity: 2 }, tokens.buyer)).kind, 'accepted'); + await waitStock(8); + for (const token of [tokens.observer, undefined]) assert.equal((await call('shop:purchase', { itemId: item._id, quantity: 1 }, token)).kind, 'application-error'); + assert.equal((await call('shop:deliberateError', {}, tokens.buyer)).kind, 'application-error'); + assert.equal((await call('shop:unhandledError', {}, tokens.buyer)).kind, 'function-error'); + assert(['function-error', 'http-error'].includes((await call('shop:missing', {}, tokens.buyer)).kind)); + assert.equal((await state('orders')).length, 1); + assert.equal((await state('items')).find(x => x._id === item._id).stock, 8); + // Vendor dashboard mutation, not a fixture business handler or table replacement. + assert.deepEqual(await admin.mutation('_system/frontend/patchDocumentsFields', { table: 'items', ids: [item._id], fields: { stock: 19 }, componentId: null }), { success: true }); + await waitStock(19); + const final = await state('items'); + assert.deepEqual(final.find((x) => x._id === other._id), other); + assert.deepEqual(final.find((x) => x._id === item._id), { ...item, stock: 19 }); + assert.equal((await state('orders')).length, 1); + + // Drop a real native HTTP reply after the backend has completed the write. + // The caller sees an unknown outcome, even though independent state proves a commit. + let upstream; let proxyError; + const proxy = createServer(async (request, response) => { + try { + const chunks = []; for await (const chunk of request) chunks.push(chunk); + const reply = await fetch(`${url}/api/mutation`, { method: 'POST', + headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${tokens.buyer}` }, + body: Buffer.concat(chunks), signal: AbortSignal.timeout(10000) }); + upstream = classifyConvexFunctionResponse(reply.status, await reply.text()); + } catch (error) { proxyError = error; } + finally { response.destroy(); } + }); + proxy.listen(0, '127.0.0.1'); await once(proxy, 'listening'); + try { + const request = convexFunctionRequest({ deploymentUrl: `http://127.0.0.1:${proxy.address().port}`, + kind: 'mutation', path: 'shop:purchase', args: { itemId: item._id, quantity: 1 } }); + await assert.rejects(fetch(request.url, { ...request, signal: AbortSignal.timeout(10000) })); + assert.equal(proxyError, undefined); + assert.equal(upstream?.kind, 'accepted'); + assert.equal((await state('orders')).length, 2); + assert.equal((await state('items')).find(x => x._id === item._id).stock, 18); + await waitStock(18); + evidence.observations.push({ path: 'shop:purchase', kind: 'transport-unknown', committed: true }); + } finally { proxy.closeAllConnections(); await new Promise(resolve => proxy.close(resolve)); } + const end = await snapshot(url, adminKey); evidence.snapshots.push(end); + evidence.initial = initial; evidence.final = end.tables.items; evidence.orders = end.tables.orders; +} finally { + try { stop(); } finally { await live.close(); } +} +} +try { await probe(); evidence.result = 'passed'; } +catch (error) { evidence.result = 'failed'; evidence.error = String(error); throw error; } +finally { writeFileSync('probe-evidence.json', JSON.stringify(evidence, null, 2)); } +console.log('Convex native calls, refusal, subscriptions, targeted patch and lost-response slice passed'); diff --git a/tools/stack-bench/tests/fixtures/convex-native/snapshot.mjs b/tools/stack-bench/tests/fixtures/convex-native/snapshot.mjs new file mode 100644 index 00000000000..865d4f1eb52 --- /dev/null +++ b/tools/stack-bench/tests/fixtures/convex-native/snapshot.mjs @@ -0,0 +1,39 @@ +import assert from 'node:assert/strict'; + +// Fixture reader through the public Data Sync API, never generated app code. +// Retain raw pages because native timestamp numbers exceed JS integer precision. +export async function snapshot(url, adminKey, tableNames = ['items', 'orders']) { + const selection = { _other: 'excluded', '': { _other: 'excluded', + ...Object.fromEntries(tableNames.map(name => [name, { _other: 'included' }])) } }; + const tables = new Map(); const pages = []; let cursor; + for (let page = 0; page < 30; page++) { + const response = await fetch(`${url}/api/v1/data/sync`, { method: 'POST', + headers: { Authorization: `Convex ${adminKey}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ selection, ...(cursor ? { cursor } : {}) }), signal: AbortSignal.timeout(10000) }); + const raw = await response.text(); pages.push({ httpStatus: response.status, raw }); + assert.equal(response.status, 200, raw); + const result = JSON.parse(raw); + for (const row of result.truncates) { + assert.equal(row.component, ''); + tables.set(row.table, new Map()); + } + for (const row of result.values) { + assert.equal(row.component, ''); + assert(tables.has(row.table), 'Snapshot must declare the table before its values'); + assert.equal(typeof row.value._id, 'string'); + if (row.deleted) tables.get(row.table).delete(row.value._id); + else tables.get(row.table).set(row.value._id, row.value); + } + if (result.status.type === 'upToDate') { + const snapshotTs = /"snapshotTs"\s*:\s*(\d+)/.exec(raw)?.[1]; + assert(snapshotTs, 'Require exact snapshot timestamp'); + assert(tableNames.every(name => tables.has(name)), 'Snapshot must include every requested table'); + return { snapshotTs, pages, tables: Object.fromEntries([...tables].map(([key, rows]) => + [key, [...rows.values()].sort((a, b) => a._id.localeCompare(b._id))])) }; + } + assert(['snapshotting', 'stale'].includes(result.status.type), 'Unknown consistency state'); + assert(result.pagination.nextCursor && result.pagination.nextCursor !== cursor, 'Cursor must advance'); + cursor = result.pagination.nextCursor; + } + throw new Error('Fixture snapshot did not complete within 30 pages'); +} diff --git a/tools/stack-bench/tests/fixtures/convex-native/verify-restart.mjs b/tools/stack-bench/tests/fixtures/convex-native/verify-restart.mjs new file mode 100644 index 00000000000..dc171c5d126 --- /dev/null +++ b/tools/stack-bench/tests/fixtures/convex-native/verify-restart.mjs @@ -0,0 +1,16 @@ +import assert from 'node:assert/strict'; +import { readFileSync, writeFileSync } from 'node:fs'; +import { snapshot } from './snapshot.mjs'; + +const evidence = { result: 'running' }; +writeFileSync('restart-evidence.json', JSON.stringify(evidence)); +try { + const before = JSON.parse(readFileSync('probe-evidence.json', 'utf8')); + assert.equal(before.result, 'passed', 'Run the slice before restarting'); + const result = await snapshot(process.env.CONVEX_SELF_HOSTED_URL, process.env.CONVEX_SELF_HOSTED_ADMIN_KEY); + assert.deepEqual(result.tables.items, before.final); + assert.deepEqual(result.tables.orders, before.orders); + Object.assign(evidence, result, { result: 'passed' }); +} catch (error) { evidence.result = 'failed'; evidence.error = String(error); throw error; } +finally { writeFileSync('restart-evidence.json', JSON.stringify(evidence, null, 2)); } +console.log('Restart preserved exact stock/order rows and IDs'); diff --git a/tools/stack-bench/tests/fixtures/dashboard-fixture.ts b/tools/stack-bench/tests/fixtures/dashboard-fixture.ts new file mode 100644 index 00000000000..8f4bf573604 --- /dev/null +++ b/tools/stack-bench/tests/fixtures/dashboard-fixture.ts @@ -0,0 +1,205 @@ +import assert from 'node:assert/strict'; +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +import { ARTIFACT_FILE, emptyArtifactIdentities, writeArtifact } from '../../src/evidence/artifacts.js'; +import { claimNextAttempt, createCampaignState, finishCampaignExecution } + from '../../src/campaigns/campaign-scheduler.js'; +import type { CampaignState } from '../../src/campaigns/campaign-scheduler.js'; +import { compileCampaignFile } from '../../src/campaigns/campaign-compiler.js'; +import type { CampaignAttemptPlan, CompiledCampaignPlan } + from '../../src/campaigns/campaign-compiler.js'; +import { compileProgressionInput, dependencyRuntimeDefinition } + from '../../src/progression/progression-definition.js'; +import { createCheckEvidence } from '../../src/evidence/check-evidence.js'; +import { hashDirectory } from '../../src/evidence/provenance.js'; +import { progressionEngine } from '../../src/progression/progression-engine.js'; +import { writeProgressionState } from '../../src/progression/progression-state.js'; +import type { ProgressionState } from '../../src/progression/progression-state.js'; +import { STACK_BENCH_ROOT } from '../../src/package-root.js'; + +// The results trees the dashboard tests and the screenshot script read: a +// sequential week at appliance scale, and one dependency campaign whose +// progression state carries builds, repairs and a failure. + +export const EXAMPLE_CAMPAIGN = join(STACK_BENCH_ROOT, 'tests', 'fixtures', 'campaign.deterministic.json'); +export const DEPENDENCY_CAMPAIGN = join(STACK_BENCH_ROOT, 'appliance', + 'campaign.ecommerce-progression-reference.json'); + +export function writeCampaign(root: string, plan: CompiledCampaignPlan, state: CampaignState): void { + mkdirSync(root, { recursive: true }); + writeArtifact(join(root, 'plan.json'), { kind: 'campaign_plan', id: `${plan.id}-plan`, + identities: emptyArtifactIdentities(), payload: plan }); + writeArtifact(join(root, 'state.json'), { kind: 'campaign_state', id: `${plan.id}-state`, + identities: emptyArtifactIdentities(), payload: state }); +} + +// A results tree the size of a real appliance week: thirty campaigns, nine +// attempts each, with the artifacts the views actually read. +export const FIXTURE_CAMPAIGNS = 30; + +export function writeRunEvidence(output: string, plan: CompiledCampaignPlan, + attempt: CampaignAttemptPlan, deficit: number): void { + const agent = plan.agents.find(item => item.adapter === attempt.agentAdapter)!; + const stack = plan.stacks.find(item => item.id === attempt.stack)!; + const selection = structuredClone(attempt.condition.requested.levels + .find(item => item.level === 1)!.selection) as { sha256: string; + scoredPoints: number; scoredChecks?: Array<{ stableKey: string; points: number }> }; + const max = selection.scoredPoints; + const score = max; + const outcome = { kind: 'passed', inconclusive: [], harnessFailures: [] }; + const firstOutcome = { kind: 'app_failure', phase: 'grading', reason: null, + appFailures: [], inconclusive: [], harnessFailures: [] }; + const source = join(output, 'source'); + mkdirSync(source, { recursive: true }); + writeFileSync(join(source, 'app.js'), 'export const ready = true;\n'); + const checks = selection.scoredChecks ?? []; + const checkKeys = checks.map(check => check.stableKey); + const evidence = createCheckEvidence({ status: 'passed', code: 'completed', + phase: 'assertion', startedAtMs: 1, completedAtMs: 2 }); + writeArtifact(join(output, 'grading', 'bundle.json'), { + kind: 'grade_bundle', id: `${attempt.id}-grade-l1`, payload: { + observation: 'scored', source: { sha256: hashDirectory(source).sha256 }, + suites: { fixture: { features: [{ id: 'fixture', name: 'Fixture', + setupEvidence: createCheckEvidence({ status: 'passed', code: 'completed', + phase: 'setup', startedAtMs: 1, completedAtMs: 2 }), + criteria: checks.map(check => ({ id: check.stableKey, stableKey: check.stableKey, + desc: check.stableKey, points: check.points, evidence })) }] } }, + totals: { score, max }, + selection: { sha256: selection.sha256, checks, attemptedChecks: checkKeys, + reportedChecks: checkKeys, notRun: [] }, + }, + }); + writeArtifact(join(output, 'run.json'), { kind: 'benchmark_run', id: `${attempt.id}-run`, + attempt: { id: `${attempt.id}-run`, parentId: attempt.id }, + identities: emptyArtifactIdentities({ engine: plan.identities.engine, + experiment: { id: plan.id, version: plan.version, sha256: plan.contentSha256, + state: plan.state }, + agentAdapter: agent.identity, stackAdapter: stack }), + payload: { + mode: attempt.mode, track: plan.definition.track, backend: attempt.stack, + model: attempt.model, pricing: attempt.pricing, guidance: attempt.guidance, + condition: attempt.condition, selectionRequest: plan.definition.selection, + featureCatalog: attempt.featureCatalog ?? null, + dependencyPolicy: attempt.dependencyPolicy ?? null, progressionOwner: null, + skills: attempt.skills, runtime: { buildImage: plan.definition.runtime.buildImage }, + totals: { score, max, costUsd: 1.25, costComplete: true, durationSec: 900 }, + levels: [{ level: 1, graded: true, selection, score, max, repairs: 3, + firstBuild: { score: score - deficit, max, outcome: firstOutcome }, + repair: { status: 'corrected', limit: 3, used: 3, stopReason: null }, + durationSec: 900, buildCostUsd: 1.25, outcome }], + outcome: { kind: 'passed' }, + } }); + writeFileSync(join(output, 'process.stdout.log'), + `=== ${attempt.stack}-l1-first (${attempt.stack}) ===\n TOTAL ... ${score - 1}/${max}\n` + + `--- repair 1/3 ---\n=== ${attempt.stack}-l1-fix1 (${attempt.stack}) ===\n` + + ` TOTAL ... ${score}/${max}\n`); +} + +export function writeFixtureResults(resultsRoot: string, { running = false }: { + running?: boolean; +} = {}): void { + const plan = compileCampaignFile(EXAMPLE_CAMPAIGN); + const now = '2026-08-18T12:00:00.000Z'; + for (let index = 0; index < FIXTURE_CAMPAIGNS; index += 1) { + const directory = join(resultsRoot, 'campaigns', `fixture-run-${index}`); + let state = createCampaignState(plan, { now }); + const claims = []; + for (;;) { + const claimed = claimNextAttempt(state, { now, admissionId: `admission-${index}` }); + if (!claimed.claim) break; + claims.push(claimed.claim); + state = running && claims.length === 1 ? claimed.state + : finishCampaignExecution(claimed.state, claimed.claim.executionId, + { exitCode: 0, run: { outcome: { kind: 'passed' } } }, { now }); + } + writeCampaign(directory, plan, state); + claims.forEach((claim, ordinal) => { + const output = join(directory, claim.output); + mkdirSync(output, { recursive: true }); + writeRunEvidence(output, plan, claim.attempt, (ordinal % 3) * 4 + 1); + }); + } +} + +export function dependencyProgressionEvidence(plan: CompiledCampaignPlan, + attempt: CampaignAttemptPlan): ProgressionState { + assert.ok(plan.featureCatalog && plan.dependencyPolicy); + const input = compileProgressionInput(dependencyRuntimeDefinition( + plan.featureCatalog, plan.dependencyPolicy)); + let state = progressionEngine.initialize(input.definition); + const missed = new Set<string>(); + for (let step = 1; step <= 24; step += 1) { + if (progressionEngine.nextAction(state).type === 'terminal') break; + const selection = progressionEngine.gradingSelection(state); + // One feature per grade misses a check the first time it is graded, so the + // replay carries repairs as well as builds. + const target = selection.nodeIds.find(nodeId => !missed.has(nodeId)); + state = progressionEngine.recordResult(state, { + attemptId: `${attempt.id}-${step}`, + outcome: 'conclusive', + nodes: selection.nodeIds.map(nodeId => ({ id: nodeId, + checks: selection.checks.filter(check => check.nodeId === nodeId).map((check, index) => ({ + id: check.id, + outcome: nodeId === target && index === 0 ? 'fail' as const : 'pass' as const })) })), + }); + if (target) missed.add(target); + } + return state; +} + + +// One dependency campaign with a progression state per stack, which is what the +// graph and the replay read. +export function writeDependencyResults(resultsRoot: string, key: string): void { + const plan = compileCampaignFile(DEPENDENCY_CAMPAIGN); + assert.ok(plan.featureCatalog && plan.dependencyPolicy); + const now = '2026-08-25T12:00:00.000Z'; + const directory = join(resultsRoot, 'campaigns', key); + let state = createCampaignState(plan, { now }); + const claims = []; + for (;;) { + const claimed = claimNextAttempt(state, { now, admissionId: 'admission-1' }); + if (!claimed.claim) break; + claims.push(claimed.claim); + state = claimed.state; + } + writeCampaign(directory, plan, state); + const progression = compileProgressionInput(dependencyRuntimeDefinition( + plan.featureCatalog, plan.dependencyPolicy)); + for (const claim of claims) { + const output = join(directory, claim.output); + mkdirSync(join(output, 'source'), { recursive: true }); + writeProgressionState(join(output, ARTIFACT_FILE.progressionState), { + progression, + featureCatalogIdentity: plan.featureCatalog.identity, + dependencyPolicyIdentity: plan.dependencyPolicy.identity, + owner: { schemaVersion: 1, + campaign: { id: plan.id, version: plan.version, sha256: plan.contentSha256 }, + attempt: { id: claim.attempt.id, track: plan.definition.track, stack: claim.attempt.stack, + agentAdapter: claim.attempt.agentAdapter, model: claim.attempt.model, + conditionSha256: claim.attempt.condition.contentSha256 }, + workspace: { appDirectory: 'source' } }, + state: dependencyProgressionEvidence(plan, claim.attempt) }); + } +} + +// Three plans as the Plans page sees them: one frozen and startable, one draft, +// and one file that does not compile. +export function writePlanFixtures(plansRoot: string): void { + mkdirSync(plansRoot, { recursive: true }); + const frozen = JSON.parse(readFileSync(DEPENDENCY_CAMPAIGN, 'utf8')) as { + state: string; + budgets: { maxCostUsdPerAttempt: number | null }; + runtime: { controllerImage: string | null; buildImage: string | null }; + }; + frozen.state = 'frozen'; + frozen.budgets.maxCostUsdPerAttempt = 12; + const digest = `sha256:${'a1'.repeat(32)}`; + frozen.runtime.controllerImage = `stack-bench-controller@${digest}`; + frozen.runtime.buildImage = `stack-bench-build@${digest}`; + writeFileSync(join(plansRoot, 'ecommerce-progression-reference.json'), + JSON.stringify(frozen, null, 2)); + writeFileSync(join(plansRoot, 'campaign.example.json'), readFileSync(EXAMPLE_CAMPAIGN)); + writeFileSync(join(plansRoot, 'broken.json'), '{'); +} diff --git a/tools/stack-bench/tests/fixtures/definitions/all-actions.json b/tools/stack-bench/tests/fixtures/definitions/all-actions.json new file mode 100644 index 00000000000..30d21bf8010 --- /dev/null +++ b/tools/stack-bench/tests/fixtures/definitions/all-actions.json @@ -0,0 +1,100 @@ +{ + "schemaVersion": 1, + "level": 1, + "name": "Action language fixture", + "features": [ + { + "id": 1, + "name": "Every registered action", + "actors": ["a", "b"], + "max": 0, + "setup": [], + "criteria": [ + { + "id": "all-actions", + "desc": "Compiles one representative of every registered action", + "points": 0, + "steps": [ + { "do": "probeSignupClaims", "actor": "a", "name": "claimant", "fields": { "role": "admin" }, "branches": [[{ "do": "expect", "actor": "a", "testid": "current-user" }]] }, + { "do": "probePasswordBoundary", "actor": "a", "returningActor": "b", "impostorActor": "b", "name": "boundary", "password": "long-password-A", "wrongPassword": "long-password-B" }, + { "do": "armScriptCanary", "actor": "a" }, + { "do": "expectNoScriptExecution", "actor": "a" }, + { "do": "confirmCheckout", "actor": "a" }, + { "do": "prepareResponseLoss", "actor": "a" }, + { "do": "loseCheckoutResponse", "actor": "a", "before": "before", "prepared": "prepared", "quantity": 1 }, + { "do": "crashCheckout", "actor": "a", "before": "before", "prepared": "prepared", "quantity": 1, "requests": 1, "offsetMs": 0, "target": "database", "as": "crash" }, + { "do": "expectCrashCheckout", "from": "crash", "verdict": "atomicity" }, + { "do": "dbRecordCheckout", "account": "a", "item": "Keyboard", "as": "before" }, + { "do": "dbExpectCheckout", "before": "before", "prepared": "prepared", "quantity": 1 }, + { "do": "dbExpectOperation", "before": "before", "otherBefore": "other", "actor": "a", "operation": "buy" }, + { "do": "dbExpectCancellation", "before": "before" }, + { "do": "dbExpectPurchases", "before": {"a":"before"}, "purchases": 1 }, + { "do": "dbExpectNoPurchase", "before": "before" }, + { "do": "dbExpectPurchase", "before": "before", "actor": "a", "stockBefore": "stock" }, + { "do": "callAction", "actor": "a", "action": "buy", "input": { "testid": "item-card", "contains": "item", "attribute": "data-action-input" }, "authentication": "none", "settleMs": 1 }, + { "do": "callConcurrently", "actors": ["a", "b"], "action": "buy", "settleMs": 1 }, + { "do": "clearInput", "actor": "a" }, + { "do": "click", "actor": "a", "testid": "button" }, + { "do": "clickConcurrently", "actors": ["a", "b"], "testid": "button", "settleMs": 1 }, + { "do": "closeClient", "actor": "a" }, + { "do": "createRoom", "actor": "a", "room": "room" }, + { "do": "dbSetStock", "item": "item", "warehouse": "warehouse", "quantity": 1, "settleMs": 1 }, + { "do": "dbRecordStock", "item": "item", "as": "stored-before" }, + { "do": "dbExpectStock", "item": "item", "relativeTo": "stored-before", "plus": 0 }, + { "do": "ensureSignedIn", "actor": "a", "name": "alice" }, + { "do": "enterRoom", "actor": "a", "room": "room" }, + { "do": "expect", "actor": "a", "testid": "status" }, + { "do": "expectActionOutcome", "actor": "a", "outcome": "refused" }, + { "do": "expectActorsWith", "actors": ["a", "b"], "testid": "status", "contains": "ok", "equals": 2, "maxEach": 1 }, + { "do": "expectAgreement", "actors": ["a", "b"], "testid": "status" }, + { "do": "expectAllPresent", "actor": "a", "prefix": "message", "count": 1, "within": 1 }, + { "do": "expectCallOutcomes", "accepted": 1 }, + { "do": "expectElementCount", "actor": "a", "testid": "row", "contains": "item", "equals": 1, "within": 1 }, + { "do": "expectForgeryRejected", "actor": "a" }, + { "do": "expectNotReceived", "actor": "a", "contains": "secret" }, + { "do": "markDelivery", "actor": "a", "as": "request", "phase": "start", "idleMs": 1 }, + { "do": "expectNotDelivered", "actor": "b", "from": "a", "as": "request", "within": 1 }, + { "do": "expectNumber", "actor": "a", "testid": "count", "equals": 1 }, + { "do": "expectWarehouseInventory", "actor": "a", "items": [{ "name": "item", "stock": { "East": 1, "West": 0 } }], "within": 1 }, + { "do": "expectOrderMatches", "actors": ["a", "b"], "prefix": "message" }, + { "do": "expectReceived", "actor": "a", "contains": "hello", "within": 1 }, + { "do": "expectReplayCompleted", "actor": "a" }, + { "do": "expectReplayRejected", "actor": "a" }, + { "do": "expectSequence", "actor": "a", "testid": "row", "equals": ["one", "two"] }, + { "do": "expectStable", "actor": "a", "testid": "count", "samples": 2, "intervalMs": 1 }, + { "do": "expectUnavailable", "actor": "a", "testid": "admin" }, + { "do": "fill", "actor": "a", "testid": "input", "text": "value" }, + { "do": "forgeWrite", "actor": "a", "fromActor": "b", "settleMs": 1 }, + { "do": "freshClient", "actor": "a" }, + { "do": "openClient", "actor": "a" }, + { "do": "openItem", "actor": "a", "item": "Keyboard" }, + { "do": "pressKey", "actor": "a", "key": "Enter" }, + { "do": "race", "branches": [[{ "do": "wait", "actor": "a", "ms": 1 }], [{ "do": "wait", "actor": "b", "ms": 1 }]], "settleMs": 1 }, + { "do": "recordNumber", "actor": "a", "testid": "count", "as": "before" }, + { "do": "recordTime", "as": "started" }, + { "do": "expectElapsed", "since": "started", "atMost": 1000 }, + { "do": "reload", "actor": "a", "settleMs": 1 }, + { "do": "repeatFormWrite", "actor": "a", "match": "Item 2", "control": "Item 1", "replacement": "{value}", "fields": [{ "testid": "name", "text": "{value}" }], "submit": "save", "settleMs": 1 }, + { "do": "replayAs", "actor": "a", "from": "b", "match": "write" }, + { "do": "replayConcurrently", "actors": ["a", "b"], "settleMs": 1 }, + { "do": "restartBackend", "settleMs": 1 }, + { "do": "runScript", "script": "fixture.mjs", "args": [] }, + { "do": "send", "actor": "a", "text": "hello" }, + { "do": "sendConcurrently", "senders": [{ "actor": "a", "prefix": "message-a", "count": 1 }, { "actor": "b", "prefix": "message-b", "count": 1 }], "delayMs": 1 }, + { "do": "sendMany", "actor": "a", "prefix": "message", "count": 1, "delayMs": 1 }, + { "do": "setOffline", "actor": "a" }, + { "do": "signIn", "actor": "a", "name": "alice" }, + { "do": "signUp", "actor": "a", "name": "alice" }, + { "do": "startAppServer" }, + { "do": "stopAppServer" }, + { "do": "typeInto", "actor": "a", "text": "value" }, + { "do": "wait", "actor": "a", "ms": 1 }, + { "do": "waitUntilAbsent", "actor": "a", "testid": "status" }, + { "do": "dbExpectPurchaseCount", "before": ["before-a", "before-b"], "purchasesEach": 500 }, + { "do": "dbExpectCatalogItem", "before": "catalog-before", "name": "New product", "priceMinor": 125 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tests/fixtures/dependency-model-free-campaign.json b/tools/stack-bench/tests/fixtures/dependency-model-free-campaign.json new file mode 100644 index 00000000000..aa324f5c7a7 --- /dev/null +++ b/tools/stack-bench/tests/fixtures/dependency-model-free-campaign.json @@ -0,0 +1,139 @@ +{ + "schemaVersion": 8, + "kind": "campaign-manifest", + "id": "dependency-model-free-proof", + "version": "1.0.0", + "state": "draft", + "title": "Internal dependency campaign proof", + "track": "ecommerce", + "mode": { "id": "dependency", "workSelection": "progressive", "retainPriorContracts": false }, + "repair": { "selection": "feature", "budget": { "total": 2, "perFeature": 1 } }, + "selection": { + "levels": [ + { + "level": 1, + "recipe": "ecommerce.sequential-l1" + } + ] + }, + "featureCatalog": { + "schemaVersion": 1, + "kind": "feature-catalog", + "id": "dependency-model-free-proof", + "title": "Internal dependency campaign proof", + "nodes": [ + { + "id": "accounts", + "title": "Accounts", + "questline": "identity", + "dependencies": [], + "featureRefs": ["ecommerce.feature.accounts"], + "promptModules": [], + "gradingChecks": [ + { + "id": "ecommerce.feature.accounts.accounts.1a", + "points": 1, + "role": "feature" + } + ] + }, + { + "id": "catalog", + "title": "Catalog", + "questline": "catalog", + "dependencies": [], + "featureRefs": ["ecommerce.feature.catalog-items"], + "promptModules": [], + "gradingChecks": [ + { + "id": "ecommerce.feature.catalog.catalog-values.2a", + "points": 1, + "role": "feature" + } + ] + } + ], + "questlines": [ + { + "id": "identity", + "title": "Identity", + "nodes": ["accounts"] + }, + { + "id": "catalog", + "title": "Catalog", + "nodes": ["catalog"] + } + ] + }, + "stacks": [ + { "id": "stub", "adapterVersion": "1.1.0" } + ], + "agents": [ + { + "adapter": "deterministic", + "adapterVersion": "1.3.0", + "model": "deterministic-stall" + } + ], + "conditions": [ + { + "id": "model-free", + "guidanceProfile": "model-free-stub", + "repairPolicy": "scored-only" + } + ], + "repetitions": 1, + "parallelism": 1, + "ordering": { + "method": "balanced-rotation", + "seed": "internal-model-free-proof" + }, + "budgets": { + "attemptTimeoutMinutes": 10, + "maxCostUsdPerAttempt": null + }, + "attemptPolicy": { + "retries": 0, + "retryOn": [], + "excludeFromAnalysis": [ + "contaminated", + "harness_failure", + "inconclusive", + "ungraded" + ] + }, + "runtime": { + "releaseManifestSha256": null, + "controllerImage": null, + "buildImage": null, + "platform": "linux/amd64" + }, + "pricing": { + "currency": "USD", + "unit": "USD-per-million-tokens", + "capturedAt": "2026-08-25T00:00:00.000Z", + "source": "internal deterministic adapter makes no provider calls", + "models": { + "deterministic-stall": { + "input": 0, + "output": 0, + "cacheWrite5m": 0, + "cacheWrite1h": 0, + "cacheRead": 0 + } + } + }, + "analysis": { + "primaryMetric": "finalScoreRate", + "secondaryMetrics": [ + "totalCostUsd", + "totalDurationMs", + "invalidAttemptRate" + ], + "dispersion": "median-iqr", + "invalidAttempts": "report-separately", + "missingData": "no-imputation", + "comparisonUnit": "stack-agent-condition-recipe" + } +} diff --git a/tools/stack-bench/tests/fixtures/qualification-evidence/ecommerce-l3-7cd96d01b/postgres-reference.json b/tools/stack-bench/tests/fixtures/qualification-evidence/ecommerce-l3-7cd96d01b/postgres-reference.json new file mode 100644 index 00000000000..6cae5296064 --- /dev/null +++ b/tools/stack-bench/tests/fixtures/qualification-evidence/ecommerce-l3-7cd96d01b/postgres-reference.json @@ -0,0 +1,736 @@ +{ + "artifactSchemaVersion": 2, + "kind": "reference_qualification", + "id": "reference-live-postgres-20260917222452-31-reference", + "attempt": { + "id": "reference-live-postgres-20260917222452-31-reference", + "parentId": null + }, + "timestamps": { + "startedAt": "2026-09-17T22:24:52.439Z", + "completedAt": "2026-09-17T23:08:14.543Z" + }, + "identities": { + "engine": { + "id": "stack-bench", + "sha256": "ee2e27156ab1080f508abaf810d87c114aba21846768f33897be0581aeb7a795" + }, + "recipe": { + "id": "ecommerce.progression-catalog", + "sha256": "104589b973c1e7dd1a2410659759989eb0de08b16efeda3131777f82858a0a8b" + }, + "fixture": { + "id": "ecommerce-reference-postgres", + "sha256": "f3906fa01050f00a0ed76c69c256a6b0013bbf72eb72c03aa92d79e47b71c2b8" + }, + "calibration": { + "id": "ecommerce.dependency-l3-calibration", + "sha256": "1b0b4927efbcabbaf8975f4ffbb0ebc77d6af2679fc0f9cac9ef492d751933e0" + }, + "experiment": null, + "agentAdapter": null, + "stackAdapter": { + "id": "postgres", + "sha256": null + }, + "packs": [] + }, + "payload": { + "fixture": "ecommerce-reference-postgres", + "fixtureSha256": "f3906fa01050f00a0ed76c69c256a6b0013bbf72eb72c03aa92d79e47b71c2b8", + "requiredRepetitions": 1, + "isolation": "docker", + "runner": { + "schemaVersion": 1, + "mode": "appliance", + "platform": "linux", + "architecture": "x64", + "hostname": "docker-desktop", + "dockerEngineVersion": "29.6.2", + "dockerOs": "linux", + "dockerArchitecture": "x86_64", + "kernelVersion": "6.6.87.2-microsoft-standard-WSL2", + "cpuCount": 32, + "memoryBytes": 49232842752, + "containersRunning": 10, + "packageRegistry": "http://127.0.0.1:4873/" + }, + "qualificationScope": { + "checksSha256": "ce9efe7f66ef111ce68dc65dcb22b48e20e4e89e68bd62fb7f7776e6a379f64d", + "executableSha256": "53b1dcbe4d602e82f17999dc7e49fad71920e5af156e338e3755119e803477dd", + "kind": "reference", + "mutationSha256": null, + "recipe": { + "contentSha256": "104589b973c1e7dd1a2410659759989eb0de08b16efeda3131777f82858a0a8b", + "id": "ecommerce.progression-catalog" + }, + "schemaVersion": 3, + "stack": { + "id": "postgres", + "reference": { + "id": "ecommerce-reference-postgres", + "sourceSha256": "f3906fa01050f00a0ed76c69c256a6b0013bbf72eb72c03aa92d79e47b71c2b8" + }, + "version": "1.6.0" + }, + "sha256": "afbc23eddb27870d42bb3be22461b6dc8796a0f9e57e858a32ec9da7bd19accc" + }, + "mutationControl": false, + "runs": [ + { + "repetition": 1, + "output": "ecommerce-l3-104589b973c1-postgres-mutation.runs/r1", + "durationMs": 1929224, + "processError": null, + "harnessSha256Before": "c911977ee8f6b0894c581b7c9d95957441c8052f7404e7ebff17f00cec4d86e5", + "harnessSha256After": "c911977ee8f6b0894c581b7c9d95957441c8052f7404e7ebff17f00cec4d86e5", + "ok": true, + "failures": [], + "runId": "ecommerce-postgres-run0-20260917222453-256bef47", + "score": "183/183", + "imageId": "sha256:32ff9a9e519ba9d1af2fe72f24622caa8ca7a9f2baf2338c6b1e0435e6ac5455", + "criteria": 114, + "zeroPointCriteria": 0, + "fingerprint": "3197a361efd9c27179e97087a9e9e342476520f9b1ae1fef4e358b08893058a8", + "outcome": "passed", + "packRuntime": { + "schemaVersion": 1, + "metric": "pack-check-wall-clock-sum-v1", + "packs": [ + { + "id": "ecommerce.feature.accounts", + "checkCount": 4, + "setupRuntimeMs": 1019, + "criterionRuntimeMs": 15352, + "measuredRuntimeMs": 16371, + "budget": { + "status": "bounded", + "maxRuntimeMs": 18000 + }, + "exceeded": false + }, + { + "id": "ecommerce.feature.cart", + "checkCount": 1, + "setupRuntimeMs": 497, + "criterionRuntimeMs": 6057, + "measuredRuntimeMs": 6554, + "budget": { + "status": "bounded", + "maxRuntimeMs": 42000 + }, + "exceeded": false + }, + { + "id": "ecommerce.feature.catalog-discovery", + "checkCount": 2, + "setupRuntimeMs": 1, + "criterionRuntimeMs": 1976, + "measuredRuntimeMs": 1977, + "budget": { + "status": "bounded", + "maxRuntimeMs": 30000 + }, + "exceeded": false + }, + { + "id": "ecommerce.feature.catalog-items", + "checkCount": 1, + "setupRuntimeMs": 0, + "criterionRuntimeMs": 227, + "measuredRuntimeMs": 227, + "budget": { + "status": "bounded", + "maxRuntimeMs": 30000 + }, + "exceeded": false + }, + { + "id": "ecommerce.feature.checkout", + "checkCount": 1, + "setupRuntimeMs": 497, + "criterionRuntimeMs": 6013, + "measuredRuntimeMs": 6510, + "budget": { + "status": "bounded", + "maxRuntimeMs": 42000 + }, + "exceeded": false + }, + { + "id": "ecommerce.feature.purchasing", + "checkCount": 1, + "setupRuntimeMs": 331, + "criterionRuntimeMs": 1180, + "measuredRuntimeMs": 1511, + "budget": { + "status": "bounded", + "maxRuntimeMs": 24000 + }, + "exceeded": false + }, + { + "id": "ecommerce.feature.reviews", + "checkCount": 1, + "setupRuntimeMs": 1732, + "criterionRuntimeMs": 432, + "measuredRuntimeMs": 2164, + "budget": { + "status": "bounded", + "maxRuntimeMs": 22000 + }, + "exceeded": false + }, + { + "id": "ecommerce.feature.warehouse-admin", + "checkCount": 2, + "setupRuntimeMs": 1344, + "criterionRuntimeMs": 4349, + "measuredRuntimeMs": 5693, + "budget": { + "status": "bounded", + "maxRuntimeMs": 30000 + }, + "exceeded": false + }, + { + "id": "ecommerce.l2.inventory-dashboard", + "checkCount": 1, + "setupRuntimeMs": 3548, + "criterionRuntimeMs": 9, + "measuredRuntimeMs": 3557, + "budget": { + "status": "bounded", + "maxRuntimeMs": 86000 + }, + "exceeded": false + }, + { + "id": "ecommerce.l2.order-cancellation-features", + "checkCount": 2, + "setupRuntimeMs": 1075, + "criterionRuntimeMs": 11260, + "measuredRuntimeMs": 12335, + "budget": { + "status": "bounded", + "maxRuntimeMs": 60000 + }, + "exceeded": false + }, + { + "id": "ecommerce.l2.recommendations", + "checkCount": 1, + "setupRuntimeMs": 338, + "criterionRuntimeMs": 5177, + "measuredRuntimeMs": 5515, + "budget": { + "status": "bounded", + "maxRuntimeMs": 86000 + }, + "exceeded": false + }, + { + "id": "ecommerce.l2.sales-dashboard", + "checkCount": 2, + "setupRuntimeMs": 2055, + "criterionRuntimeMs": 5062, + "measuredRuntimeMs": 7117, + "budget": { + "status": "bounded", + "maxRuntimeMs": 86000 + }, + "exceeded": false + }, + { + "id": "ecommerce.l2.stock-transfers-features", + "checkCount": 1, + "setupRuntimeMs": 533, + "criterionRuntimeMs": 1057, + "measuredRuntimeMs": 1590, + "budget": { + "status": "bounded", + "maxRuntimeMs": 86000 + }, + "exceeded": false + }, + { + "id": "ecommerce.l3.deferred-access-specifications", + "checkCount": 1, + "setupRuntimeMs": 20929, + "criterionRuntimeMs": 4291, + "measuredRuntimeMs": 25220, + "budget": { + "status": "bounded", + "maxRuntimeMs": 60000 + }, + "exceeded": false + }, + { + "id": "ecommerce.l3.deferred-durability-specifications", + "checkCount": 1, + "setupRuntimeMs": 126364, + "criterionRuntimeMs": 41064, + "measuredRuntimeMs": 167428, + "budget": { + "status": "bounded", + "maxRuntimeMs": 720000 + }, + "exceeded": false + }, + { + "id": "ecommerce.l3.deferred-integrity-specifications", + "checkCount": 1, + "setupRuntimeMs": 42261, + "criterionRuntimeMs": 38490, + "measuredRuntimeMs": 80751, + "budget": { + "status": "bounded", + "maxRuntimeMs": 400000 + }, + "exceeded": false + }, + { + "id": "ecommerce.l3.scheduled-restocks-features", + "checkCount": 3, + "setupRuntimeMs": 61539, + "criterionRuntimeMs": 70376, + "measuredRuntimeMs": 131915, + "budget": { + "status": "bounded", + "maxRuntimeMs": 150000 + }, + "exceeded": false + }, + { + "id": "ecommerce.l3.server-time-specifications", + "checkCount": 1, + "setupRuntimeMs": 27295, + "criterionRuntimeMs": 112908, + "measuredRuntimeMs": 140203, + "budget": { + "status": "bounded", + "maxRuntimeMs": 300000 + }, + "exceeded": false + }, + { + "id": "ecommerce.progression.cancellation-accounting-specifications", + "checkCount": 1, + "setupRuntimeMs": 936, + "criterionRuntimeMs": 9579, + "measuredRuntimeMs": 10515, + "budget": { + "status": "bounded", + "maxRuntimeMs": 24000 + }, + "exceeded": false + }, + { + "id": "ecommerce.progression.catalog-management", + "checkCount": 2, + "setupRuntimeMs": 3407, + "criterionRuntimeMs": 22, + "measuredRuntimeMs": 3429, + "budget": { + "status": "bounded", + "maxRuntimeMs": 30000 + }, + "exceeded": false + }, + { + "id": "ecommerce.progression.customer-profile", + "checkCount": 1, + "setupRuntimeMs": 527, + "criterionRuntimeMs": 5072, + "measuredRuntimeMs": 5599, + "budget": { + "status": "bounded", + "maxRuntimeMs": 30000 + }, + "exceeded": false + }, + { + "id": "ecommerce.progression.faceted-search", + "checkCount": 2, + "setupRuntimeMs": 42702, + "criterionRuntimeMs": 25568, + "measuredRuntimeMs": 68270, + "budget": { + "status": "bounded", + "maxRuntimeMs": 141000 + }, + "exceeded": false + }, + { + "id": "ecommerce.progression.fulfilment-queue", + "checkCount": 2, + "setupRuntimeMs": 1272, + "criterionRuntimeMs": 26980, + "measuredRuntimeMs": 28252, + "budget": { + "status": "bounded", + "maxRuntimeMs": 76000 + }, + "exceeded": false + }, + { + "id": "ecommerce.progression.inventory-conservation-specifications", + "checkCount": 5, + "setupRuntimeMs": 2700, + "criterionRuntimeMs": 45224, + "measuredRuntimeMs": 47924, + "budget": { + "status": "bounded", + "maxRuntimeMs": 138000 + }, + "exceeded": false + }, + { + "id": "ecommerce.progression.managed-support", + "checkCount": 1, + "setupRuntimeMs": 1105, + "criterionRuntimeMs": 6204, + "measuredRuntimeMs": 7309, + "budget": { + "status": "bounded", + "maxRuntimeMs": 55000 + }, + "exceeded": false + }, + { + "id": "ecommerce.progression.notification-preferences", + "checkCount": 1, + "setupRuntimeMs": 677, + "criterionRuntimeMs": 5073, + "measuredRuntimeMs": 5750, + "budget": { + "status": "bounded", + "maxRuntimeMs": 40000 + }, + "exceeded": false + }, + { + "id": "ecommerce.progression.operations-access-specifications", + "checkCount": 3, + "setupRuntimeMs": 10011, + "criterionRuntimeMs": 34156, + "measuredRuntimeMs": 44167, + "budget": { + "status": "bounded", + "maxRuntimeMs": 98000 + }, + "exceeded": false + }, + { + "id": "ecommerce.progression.promotion-rules", + "checkCount": 1, + "setupRuntimeMs": 589, + "criterionRuntimeMs": 375, + "measuredRuntimeMs": 964, + "budget": { + "status": "bounded", + "maxRuntimeMs": 45000 + }, + "exceeded": false + }, + { + "id": "ecommerce.progression.review-access-specifications", + "checkCount": 2, + "setupRuntimeMs": 8895, + "criterionRuntimeMs": 9541, + "measuredRuntimeMs": 18436, + "budget": { + "status": "bounded", + "maxRuntimeMs": 82000 + }, + "exceeded": false + }, + { + "id": "ecommerce.progression.staff-access", + "checkCount": 1, + "setupRuntimeMs": 0, + "criterionRuntimeMs": 782, + "measuredRuntimeMs": 782, + "budget": { + "status": "bounded", + "maxRuntimeMs": 30000 + }, + "exceeded": false + }, + { + "id": "ecommerce.progression.staff-roles", + "checkCount": 1, + "setupRuntimeMs": 772, + "criterionRuntimeMs": 26, + "measuredRuntimeMs": 798, + "budget": { + "status": "bounded", + "maxRuntimeMs": 30000 + }, + "exceeded": false + }, + { + "id": "ecommerce.progression.stock-alerts", + "checkCount": 1, + "setupRuntimeMs": 3943, + "criterionRuntimeMs": 11163, + "measuredRuntimeMs": 15106, + "budget": { + "status": "bounded", + "maxRuntimeMs": 60000 + }, + "exceeded": false + }, + { + "id": "ecommerce.progression.support-history", + "checkCount": 1, + "setupRuntimeMs": 859, + "criterionRuntimeMs": 1135, + "measuredRuntimeMs": 1994, + "budget": { + "status": "bounded", + "maxRuntimeMs": 50000 + }, + "exceeded": false + }, + { + "id": "ecommerce.progression.support-intake", + "checkCount": 1, + "setupRuntimeMs": 0, + "criterionRuntimeMs": 441, + "measuredRuntimeMs": 441, + "budget": { + "status": "bounded", + "maxRuntimeMs": 30000 + }, + "exceeded": false + }, + { + "id": "ecommerce.progression.support-triage", + "checkCount": 3, + "setupRuntimeMs": 927, + "criterionRuntimeMs": 10071, + "measuredRuntimeMs": 10998, + "budget": { + "status": "bounded", + "maxRuntimeMs": 45000 + }, + "exceeded": false + }, + { + "id": "ecommerce.spec.access-control", + "checkCount": 21, + "setupRuntimeMs": 27625, + "criterionRuntimeMs": 212621, + "measuredRuntimeMs": 240246, + "budget": { + "status": "bounded", + "maxRuntimeMs": 464000 + }, + "exceeded": false + }, + { + "id": "ecommerce.spec.concurrency-safety", + "checkCount": 6, + "setupRuntimeMs": 24565, + "criterionRuntimeMs": 58044, + "measuredRuntimeMs": 82609, + "budget": { + "status": "bounded", + "maxRuntimeMs": 125000 + }, + "exceeded": false + }, + { + "id": "ecommerce.spec.external-data-sync", + "checkCount": 3, + "setupRuntimeMs": 670, + "criterionRuntimeMs": 28234, + "measuredRuntimeMs": 28904, + "budget": { + "status": "bounded", + "maxRuntimeMs": 105000 + }, + "exceeded": false + }, + { + "id": "ecommerce.spec.live-state", + "checkCount": 11, + "setupRuntimeMs": 20495, + "criterionRuntimeMs": 6782, + "measuredRuntimeMs": 27277, + "budget": { + "status": "bounded", + "maxRuntimeMs": 184000 + }, + "exceeded": false + }, + { + "id": "ecommerce.spec.search-ordering", + "checkCount": 1, + "setupRuntimeMs": 20800, + "criterionRuntimeMs": 5194, + "measuredRuntimeMs": 25994, + "budget": { + "status": "bounded", + "maxRuntimeMs": 60000 + }, + "exceeded": false + }, + { + "id": "ecommerce.spec.state-durability", + "checkCount": 10, + "setupRuntimeMs": 11999, + "criterionRuntimeMs": 64392, + "measuredRuntimeMs": 76391, + "budget": { + "status": "bounded", + "maxRuntimeMs": 768000 + }, + "exceeded": false + }, + { + "id": "ecommerce.spec.transactional-integrity", + "checkCount": 6, + "setupRuntimeMs": 24082, + "criterionRuntimeMs": 30891, + "measuredRuntimeMs": 54973, + "budget": { + "status": "bounded", + "maxRuntimeMs": 100000 + }, + "exceeded": false + } + ] + }, + "mutations": null + } + ], + "stable": true, + "sameImage": true, + "sameHarness": true, + "harnessSha256": "c911977ee8f6b0894c581b7c9d95957441c8052f7404e7ebff17f00cec4d86e5", + "qualifiedCheckKeys": [ + "ecommerce.feature.accounts.accounts.1a", + "ecommerce.feature.accounts.accounts.1b", + "ecommerce.feature.accounts.accounts.1c", + "ecommerce.feature.accounts.accounts.1d", + "ecommerce.feature.cart-checkout.cart.4a", + "ecommerce.feature.cart-checkout.cart.4d", + "ecommerce.feature.catalog.catalog-ranking.2b", + "ecommerce.feature.catalog.catalog-search.2d", + "ecommerce.feature.catalog.catalog-values.2a", + "ecommerce.feature.purchasing.purchase-order.3c", + "ecommerce.feature.reviews.reviews.6a", + "ecommerce.feature.warehouse-admin.admin-write.103a", + "ecommerce.feature.warehouse-admin.warehouse-view.7b", + "ecommerce.inventory-operations.operational-views.5c", + "ecommerce.inventory-operations.operational-views.5d", + "ecommerce.inventory-operations.operational-views.5e", + "ecommerce.inventory-operations.operational-views.5f", + "ecommerce.inventory-operations.shipping-accounting.202e", + "ecommerce.inventory-operations.stock-conservation.202a", + "ecommerce.inventory-operations.stock-conservation.202b", + "ecommerce.inventory-operations.stock-conservation.202c", + "ecommerce.inventory-operations.stock-conservation.202d", + "ecommerce.inventory-operations.warehouse-transfer.2a", + "ecommerce.l3.deferred-access.scheduled-work-access.317a", + "ecommerce.l3.deferred-durability.restart-survival.311a", + "ecommerce.l3.deferred-integrity.exactly-once.311a", + "ecommerce.l3.scheduled-restocks.scheduled-restocks.302a", + "ecommerce.l3.scheduled-restocks.scheduled-restocks.305a", + "ecommerce.l3.scheduled-restocks.scheduled-restocks.306a", + "ecommerce.l3.server-time.server-time.312a", + "ecommerce.operations-access.fulfilment-queue.1b", + "ecommerce.operations-access.fulfilment-queue.1c", + "ecommerce.operations-access.operator-authorization.201a", + "ecommerce.operations-access.operator-authorization.201c", + "ecommerce.operations-access.order-owner.204a", + "ecommerce.progression.catalog-management.catalog-management.622a", + "ecommerce.progression.catalog-management.catalog-management.622b", + "ecommerce.progression.customer-profile.customer-profile.620c", + "ecommerce.progression.faceted-search.faceted-search.401a", + "ecommerce.progression.faceted-search.faceted-search.402a", + "ecommerce.progression.managed-support.managed-support.613c", + "ecommerce.progression.notification-preferences.notification-preferences.630c", + "ecommerce.progression.promotion-rules.promotion-rule-values.620a", + "ecommerce.progression.review-access-specifications.review-eligibility-direct.618a", + "ecommerce.progression.review-access-specifications.stored-review-script.9180a", + "ecommerce.progression.staff-access.staff-access.601a", + "ecommerce.progression.staff-roles.staff-roles.621c", + "ecommerce.progression.stock-alerts.stock-alert-delivery.631c", + "ecommerce.progression.support-history.support-history.612c", + "ecommerce.progression.support-intake.support-intake.610a", + "ecommerce.progression.support-triage.support-assignment.611a", + "ecommerce.progression.support-triage.support-priority.611b", + "ecommerce.progression.support-triage.support-status.611c", + "ecommerce.returns-pricing.cancellation-and-return.3a", + "ecommerce.returns-pricing.cancellation-and-return.3b", + "ecommerce.returns-pricing.refund-accounting.203a", + "ecommerce.spec.access-control.cart-boundary.109a", + "ecommerce.spec.access-control.cart-boundary.109b", + "ecommerce.spec.access-control.customer-profile-privacy.620b", + "ecommerce.spec.access-control.fulfilment-area-boundary.1d", + "ecommerce.spec.access-control.managed-support-privacy.613b", + "ecommerce.spec.access-control.notification-preferences-privacy.630b", + "ecommerce.spec.access-control.order-ownership.106a", + "ecommerce.spec.access-control.promotion-management-boundary.620b", + "ecommerce.spec.access-control.purchase-attribution.102a", + "ecommerce.spec.access-control.purchase-session.101a", + "ecommerce.spec.access-control.review-eligibility.108a", + "ecommerce.spec.access-control.review-eligibility.108b", + "ecommerce.spec.access-control.signed-out-purchase.3a", + "ecommerce.spec.access-control.staff-area-boundary.601b", + "ecommerce.spec.access-control.staff-role-boundary.621b", + "ecommerce.spec.access-control.staff-role-revocation.621d", + "ecommerce.spec.access-control.stock-alert-privacy.631b", + "ecommerce.spec.access-control.support-history-logout.612d", + "ecommerce.spec.access-control.support-history-privacy.612b", + "ecommerce.spec.access-control.warehouse-area-boundary.7a", + "ecommerce.spec.access-control.warehouse-write-boundary.103b", + "ecommerce.spec.concurrency-safety.duplicate-checkout.203a", + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b", + "ecommerce.spec.concurrency-safety.last-unit.201a", + "ecommerce.spec.concurrency-safety.last-unit.201b", + "ecommerce.spec.concurrency-safety.last-unit.201c", + "ecommerce.spec.concurrency-safety.restock-race.202a", + "ecommerce.spec.external-data-sync.external-stock.901a", + "ecommerce.spec.external-data-sync.external-stock.901c", + "ecommerce.spec.external-data-sync.external-stock.901d", + "ecommerce.spec.live-state.fulfilment-queue.1a", + "ecommerce.spec.live-state.inventory-dashboard.5a", + "ecommerce.spec.live-state.managed-support.613a", + "ecommerce.spec.live-state.open-list.902a", + "ecommerce.spec.live-state.purchase-stock.3b", + "ecommerce.spec.live-state.ranking.2c", + "ecommerce.spec.live-state.rating.6c", + "ecommerce.spec.live-state.sales-dashboard.5b", + "ecommerce.spec.live-state.shared-cart.4c", + "ecommerce.spec.live-state.stock-transfers.2b", + "ecommerce.spec.live-state.warehouse-stock.7c", + "ecommerce.spec.search-ordering.search-ordering.402b", + "ecommerce.spec.state-durability.account-state-recovery.105a", + "ecommerce.spec.state-durability.account-state-recovery.105b", + "ecommerce.spec.state-durability.cart-reload.4b", + "ecommerce.spec.state-durability.checkout-crash-durability.910b", + "ecommerce.spec.state-durability.checkout-crash-integrity.910a", + "ecommerce.spec.state-durability.customer-profile-reload.620a", + "ecommerce.spec.state-durability.notification-preferences-reload.630a", + "ecommerce.spec.state-durability.session-reload.1e", + "ecommerce.spec.state-durability.staff-role-reload.621a", + "ecommerce.spec.state-durability.support-history-reload.612a", + "ecommerce.spec.transactional-integrity.books-balance.107a", + "ecommerce.spec.transactional-integrity.books-balance.107b", + "ecommerce.spec.transactional-integrity.server-price.104a", + "ecommerce.spec.transactional-integrity.stock-alert-deduplication.631a", + "ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c", + "ecommerce.spec.transactional-integrity.unique-review.6b" + ], + "featureCatalog": { + "contentSha256": "8671f7883c2e5a24474a546ef1517407cd9334258e1a7cfe91e292a9af1952f2", + "id": "ecommerce.questlines" + }, + "diagnostic": false, + "ok": true + } +} diff --git a/tools/stack-bench/tests/fixtures/qualification-evidence/ecommerce-l3-e804c1302/current-inputs.json b/tools/stack-bench/tests/fixtures/qualification-evidence/ecommerce-l3-e804c1302/current-inputs.json new file mode 100644 index 00000000000..28866766bcd --- /dev/null +++ b/tools/stack-bench/tests/fixtures/qualification-evidence/ecommerce-l3-e804c1302/current-inputs.json @@ -0,0 +1 @@ +{"documents":{"release":{"capabilities":["backend-lifecycle","browser","concurrent-actors","database-observation","database-read","direct-database-write","direct-server-call","process-crash","request-replay"],"checkCatalog":[{"category":"feature","checkGroupId":"accounts","criterionId":"1a","description":"a visitor can create an account and is signed in as it","executionId":"selected-source-001","featureId":1,"packId":"ecommerce.feature.accounts","points":1,"role":"feature","source":"scenarios/01-account-create.json","stableKey":"ecommerce.feature.accounts.accounts.1a"},{"category":"production","checkGroupId":"accounts","criterionId":"1b","description":"a taken username is refused and does not sign the visitor in as the existing account","executionId":"selected-source-002","featureId":1,"packId":"ecommerce.feature.accounts","points":1,"role":"feature","source":"scenarios/01-account-duplicate.json","stableKey":"ecommerce.feature.accounts.accounts.1b"},{"category":"production","checkGroupId":"accounts","criterionId":"1c","description":"a wrong password is refused","executionId":"selected-source-003","featureId":1,"packId":"ecommerce.feature.accounts","points":1,"role":"feature","source":"scenarios/01-account-password.json","stableKey":"ecommerce.feature.accounts.accounts.1c"},{"category":"production","checkGroupId":"session-reload","criterionId":"1e","description":"the session survives a reload","executionId":"selected-source-004","featureId":1,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":1,"requiresFeatures":["ecommerce.feature.accounts"],"role":"guarantee","source":"scenarios/01-account-reload.json","stableKey":"ecommerce.spec.state-durability.session-reload.1e"},{"category":"feature","checkGroupId":"accounts","criterionId":"1d","description":"signing out and back in returns the same account","executionId":"selected-source-005","featureId":1,"packId":"ecommerce.feature.accounts","points":1,"role":"feature","source":"scenarios/01-account-signout.json","stableKey":"ecommerce.feature.accounts.accounts.1d"},{"category":"feature","checkGroupId":"admin-write","criterionId":"103a","description":"an administrator can restock a warehouse","executionId":"selected-source-006","featureId":103,"packId":"ecommerce.feature.warehouse-admin","points":1,"role":"feature","source":"scenarios/01-admin-write-staff.json","stableKey":"ecommerce.feature.warehouse-admin.admin-write.103a"},{"category":"production","checkGroupId":"warehouse-write-boundary","criterionId":"103b","description":"the server refuses a warehouse write from staff","executionId":"selected-source-006","featureId":103,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-admin-write-staff.json","stableKey":"ecommerce.spec.access-control.warehouse-write-boundary.103b"},{"category":"production","checkGroupId":"purchase-stock","criterionId":"3b","description":"buying reduces the stock every other client sees, without a reload","executionId":"selected-source-007","featureId":3,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":2,"requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/01-buying.json","stableKey":"ecommerce.spec.live-state.purchase-stock.3b"},{"category":"production","checkGroupId":"cart-boundary","criterionId":"109a","description":"the same cart action run by another customer changes only that customer's cart","executionId":"selected-source-008","featureId":109,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.feature.cart"],"role":"guarantee","source":"scenarios/01-cart-boundary.json","stableKey":"ecommerce.spec.access-control.cart-boundary.109a"},{"category":"production","checkGroupId":"cart-boundary","criterionId":"109b","description":"a negative quantity is refused and leaves the cart unchanged","executionId":"selected-source-008","featureId":109,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"requiresFeatures":["ecommerce.feature.cart"],"role":"guarantee","source":"scenarios/01-cart-boundary.json","stableKey":"ecommerce.spec.access-control.cart-boundary.109b"},{"category":"production","checkGroupId":"cart-reload","criterionId":"4b","description":"the cart survives a reload","executionId":"selected-source-009","featureId":4,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":1,"requiresFeatures":["ecommerce.feature.cart"],"role":"guarantee","source":"scenarios/01-cart.json","stableKey":"ecommerce.spec.state-durability.cart-reload.4b"},{"category":"production","checkGroupId":"shared-cart","criterionId":"4c","description":"the same account signed in elsewhere sees one cart, live","executionId":"selected-source-009","featureId":4,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":2,"requiresFeatures":["ecommerce.feature.cart"],"role":"guarantee","source":"scenarios/01-cart.json","stableKey":"ecommerce.spec.live-state.shared-cart.4c"},{"category":"feature","checkGroupId":"catalog-ranking","criterionId":"2b","description":"the storefront shows the exact alphabetical top ten before any purchase","executionId":"selected-source-010","featureId":2,"packId":"ecommerce.feature.catalog-discovery","points":1,"role":"feature","source":"scenarios/01-catalog-ranking.json","stableKey":"ecommerce.feature.catalog.catalog-ranking.2b","stablePackId":"ecommerce.feature.catalog"},{"category":"feature","checkGroupId":"catalog-search","criterionId":"2d","description":"case-insensitive partial search finds an item outside the storefront top ten","executionId":"selected-source-011","featureId":2,"packId":"ecommerce.feature.catalog-discovery","points":1,"role":"feature","source":"scenarios/01-catalog-search.json","stableKey":"ecommerce.feature.catalog.catalog-search.2d","stablePackId":"ecommerce.feature.catalog"},{"category":"feature","checkGroupId":"catalog-values","criterionId":"2a","description":"a signed-out visitor sees the seeded item name, price, and total stock","executionId":"selected-source-012","featureId":2,"packId":"ecommerce.feature.catalog-items","points":1,"role":"feature","source":"scenarios/01-catalog-values.json","stableKey":"ecommerce.feature.catalog.catalog-values.2a","stablePackId":"ecommerce.feature.catalog"},{"category":"production","checkGroupId":"ranking","criterionId":"2c","description":"a purchase moves the bought item to the front of the ranking, live","executionId":"selected-source-013","featureId":2,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":1,"requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/01-core.json","stableKey":"ecommerce.spec.live-state.ranking.2c"},{"category":"production","checkGroupId":"duplicate-checkout","criterionId":"203a","description":"the same item added from two tabs at once becomes one line of two","executionId":"selected-source-014","featureId":203,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":1,"requiresFeatures":["ecommerce.feature.checkout"],"role":"guarantee","source":"scenarios/01-duplicate-checkout.json","stableKey":"ecommerce.spec.concurrency-safety.duplicate-checkout.203a"},{"category":"production","checkGroupId":"duplicate-checkout","criterionId":"203b","description":"checking the same cart out twice at once produces one order","executionId":"selected-source-014","featureId":203,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":1,"requiresFeatures":["ecommerce.feature.checkout"],"role":"guarantee","source":"scenarios/01-duplicate-checkout.json","stableKey":"ecommerce.spec.concurrency-safety.duplicate-checkout.203b"},{"category":"production","checkGroupId":"external-stock","criterionId":"901a","description":"a direct database write sets Desk Lamp's East stock to 5, and the already-open storefront updates from 100 to 50 without a reload or page action","executionId":"selected-source-015","featureId":901,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.external-data-sync","points":1,"requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-external-live-sync.json","stableKey":"ecommerce.spec.external-data-sync.external-stock.901a"},{"category":"production","checkGroupId":"external-stock","criterionId":"901d","description":"while the storefront is offline, a direct database write sets Desk Lamp's East stock to 7; after reconnecting, the same page catches up from 100 to the authoritative total of 52","executionId":"selected-source-016","featureId":901,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.external-data-sync","points":1,"requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-external-reconnect-sync.json","stableKey":"ecommerce.spec.external-data-sync.external-stock.901d"},{"checkGroupId":"external-stock","criterionId":"901b","description":"after a direct database write sets Desk Lamp's East stock to 5, a reload reads the persisted total of 50","executionId":"selected-source-017","featureId":901,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.external-data-sync","points":0,"requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-external-reload-sync.json","stableKey":"ecommerce.spec.external-data-sync.external-stock.901b"},{"category":"production","checkGroupId":"external-stock","criterionId":"901c","description":"a stock correction lands while the app server is stopped, and the already-open storefront shows the authoritative total of 65 after the server returns without a reload","executionId":"selected-source-018","featureId":901,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.external-data-sync","points":1,"requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-external-server-restart-sync.json","stableKey":"ecommerce.spec.external-data-sync.external-stock.901c"},{"category":"production","checkGroupId":"last-unit","criterionId":"201a","description":"after six customers try to buy the last three units, each warehouse stores zero stock and all observed clients show zero stock","executionId":"selected-source-019","featureId":201,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":1,"requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-last-unit.json","stableKey":"ecommerce.spec.concurrency-safety.last-unit.201a"},{"category":"production","checkGroupId":"last-unit","criterionId":"201c","description":"revenue increases by exactly three sales, not six","executionId":"selected-source-019","featureId":201,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":1,"requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-last-unit.json","stableKey":"ecommerce.spec.concurrency-safety.last-unit.201c"},{"category":"production","checkGroupId":"last-unit","criterionId":"201b","description":"the last three units create complete orders for the successful buyers, and all four affordable purchases succeed when stock is sufficient","executionId":"selected-source-019","featureId":201,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":1,"requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-last-unit.json","stableKey":"ecommerce.spec.concurrency-safety.last-unit.201b"},{"category":"production","checkGroupId":"order-ownership","criterionId":"106a","description":"a working order history contains the customer's own order and not another customer's order","executionId":"selected-source-020","featureId":106,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/01-order-ownership.json","stableKey":"ecommerce.spec.access-control.order-ownership.106a"},{"category":"production","checkGroupId":"purchase-attribution","criterionId":"102a","description":"a direct purchase is attributed to the authenticated caller, not another account","executionId":"selected-source-021","featureId":102,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/01-purchase-attribution.json","stableKey":"ecommerce.spec.access-control.purchase-attribution.102a"},{"category":"production","checkGroupId":"purchase-session","criterionId":"101a","description":"a valid direct purchase works for the buyer but is refused without a session","executionId":"selected-source-022","featureId":101,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/01-purchase-session.json","stableKey":"ecommerce.spec.access-control.purchase-session.101a"},{"category":"feature","checkGroupId":"restock-race","criterionId":"202-control","description":"an uncontended restock of five is stored by the server and shows on the storefront","executionId":"selected-source-023","featureId":202,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":0,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-restock-race.json","stableKey":"ecommerce.spec.concurrency-safety.restock-race.202-control"},{"category":"production","checkGroupId":"restock-race","criterionId":"202a","description":"restocking during purchases preserves stock, complete buyer orders and their warehouse allocations","executionId":"selected-source-023","featureId":202,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":1,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-restock-race.json","stableKey":"ecommerce.spec.concurrency-safety.restock-race.202a"},{"category":"production","checkGroupId":"review-eligibility","criterionId":"108a","description":"someone who never bought the item cannot review it","executionId":"selected-source-024","featureId":108,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","source":"scenarios/01-review-eligibility.json","stableKey":"ecommerce.spec.access-control.review-eligibility.108a"},{"category":"production","checkGroupId":"review-eligibility","criterionId":"108b","description":"buying the item earns the right to review it","executionId":"selected-source-024","featureId":108,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","source":"scenarios/01-review-eligibility.json","stableKey":"ecommerce.spec.access-control.review-eligibility.108b"},{"category":"production","checkGroupId":"rating","criterionId":"6c","description":"the average rating reflects both reviewers and updates live","executionId":"selected-source-025","featureId":6,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":1,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","source":"scenarios/01-review-rating-live.json","stableKey":"ecommerce.spec.live-state.rating.6c"},{"category":"production","checkGroupId":"unique-review","criterionId":"6b","description":"a later review submission does not create a duplicate for the same customer and item","executionId":"selected-source-026","featureId":6,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":1,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","source":"scenarios/01-review-uniqueness.json","stableKey":"ecommerce.spec.transactional-integrity.unique-review.6b"},{"category":"feature","checkGroupId":"reviews","criterionId":"6a","description":"a customer can review an item and everyone sees it, signed out included","executionId":"selected-source-027","featureId":6,"packId":"ecommerce.feature.reviews","points":2,"role":"feature","source":"scenarios/01-review-visibility.json","stableKey":"ecommerce.feature.reviews.reviews.6a"},{"category":"production","checkGroupId":"server-price","criterionId":"104a","description":"direct purchases of two differently priced items persist exactly one correctly priced order each","executionId":"selected-source-028","featureId":104,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":2,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/01-server-price.json","stableKey":"ecommerce.spec.transactional-integrity.server-price.104a"},{"category":"production","checkGroupId":"warehouse-area-boundary","criterionId":"7a","description":"the administrator area stays unavailable to other staff","executionId":"selected-source-029","featureId":7,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-warehouse-admin-staff.json","stableKey":"ecommerce.spec.access-control.warehouse-area-boundary.7a"},{"category":"feature","checkGroupId":"warehouse-view","criterionId":"7b","description":"admin lists every item, every warehouse, and what each warehouse holds","executionId":"selected-source-029","featureId":7,"packId":"ecommerce.feature.warehouse-admin","points":1,"role":"feature","source":"scenarios/01-warehouse-admin-staff.json","stableKey":"ecommerce.feature.warehouse-admin.warehouse-view.7b"},{"category":"production","checkGroupId":"warehouse-stock","criterionId":"7c","description":"the storefront stock is the sum across warehouses, and a restock raises it live","executionId":"selected-source-030","featureId":7,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":2,"requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-warehouse-stock-live-staff.json","stableKey":"ecommerce.spec.live-state.warehouse-stock.7c"},{"category":"production","checkGroupId":"cancellation-and-return","criterionId":"3d","description":"cancelling a pending order removes it from the fulfilment queue","executionId":"selected-source-031","featureId":3,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.cancellation-queue-specifications","points":1,"requiresFeatures":["ecommerce.l2.order-cancellation-features","ecommerce.progression.fulfilment-queue"],"role":"guarantee","source":"scenarios/02-cancellation-queue.json","stableKey":"ecommerce.returns-pricing.cancellation-and-return.3d","stablePackId":"ecommerce.returns-pricing"},{"category":"production","checkGroupId":"fulfilment-area-boundary","criterionId":"1d","description":"staff and administrators can open fulfilment while customers cannot","executionId":"selected-source-032","featureId":1,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"requiresFeatures":["ecommerce.progression.fulfilment-queue"],"role":"guarantee","source":"scenarios/02-fulfilment-access.json","stableKey":"ecommerce.spec.access-control.fulfilment-area-boundary.1d"},{"category":"production","checkGroupId":"fulfilment-queue","criterionId":"1a","description":"an order placed by a customer appears in the staff queue without a reload","executionId":"selected-source-033","featureId":1,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":2,"requiresFeatures":["ecommerce.progression.fulfilment-queue"],"role":"guarantee","source":"scenarios/02-fulfilment-live.json","stableKey":"ecommerce.spec.live-state.fulfilment-queue.1a"},{"category":"feature","checkGroupId":"fulfilment-queue","criterionId":"1c","description":"shipping removes the order from the queue and marks the customer's order shipped","executionId":"selected-source-034","featureId":1,"packId":"ecommerce.progression.fulfilment-queue","points":2,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"feature","source":"scenarios/02-fulfilment-ship.json","stableKey":"ecommerce.operations-access.fulfilment-queue.1c","stablePackId":"ecommerce.operations-access"},{"category":"production","checkGroupId":"refund-accounting","criterionId":"203a","description":"concurrent cancellation restores original stock and the booked amount once, while revenue returns to its prior value","executionId":"selected-source-035","featureId":203,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.cancellation-accounting-specifications","points":3,"requiresFeatures":["ecommerce.l2.order-cancellation-features"],"role":"guarantee","source":"scenarios/02-invariants.json","stableKey":"ecommerce.returns-pricing.refund-accounting.203a","stablePackId":"ecommerce.returns-pricing"},{"category":"production","checkGroupId":"refund-accounting","criterionId":"203b","description":"a price change does not rewrite revenue already earned","executionId":"selected-source-035","featureId":203,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.price-accounting-specifications","points":2,"requiresFeatures":["ecommerce.l2.price-history-features"],"role":"guarantee","source":"scenarios/02-invariants.json","stableKey":"ecommerce.returns-pricing.refund-accounting.203b","stablePackId":"ecommerce.returns-pricing"},{"category":"production","checkGroupId":"price-history","criterionId":"4b","description":"the new price reaches a signed-out visitor without a reload","executionId":"selected-source-036","featureId":4,"packId":"ecommerce.l2.price-history-features","points":2,"role":"feature","source":"scenarios/02-live-price.json","stableKey":"ecommerce.returns-pricing.price-history.4b","stablePackId":"ecommerce.returns-pricing"},{"category":"feature","checkGroupId":"operational-views","criterionId":"5e","description":"the dashboard lists a current low-stock item","executionId":"selected-source-037","featureId":5,"packId":"ecommerce.l2.inventory-dashboard","points":1,"role":"feature","source":"scenarios/02-low-stock.json","stableKey":"ecommerce.inventory-operations.operational-views.5e","stablePackId":"ecommerce.inventory-operations"},{"category":"production","checkGroupId":"inventory-dashboard","criterionId":"5a","description":"an item falling to ten units or fewer joins the low-stock list, live","executionId":"selected-source-037","featureId":5,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":3,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.inventory-dashboard"],"role":"guarantee","source":"scenarios/02-low-stock.json","stableKey":"ecommerce.spec.live-state.inventory-dashboard.5a"},{"category":"feature","checkGroupId":"operational-views","criterionId":"5d","description":"a signed-out visitor sees a best seller in the recommendations list","executionId":"selected-source-038","featureId":5,"packId":"ecommerce.l2.sales-dashboard","points":1,"role":"feature","source":"scenarios/02-operational-best-sellers.json","stableKey":"ecommerce.inventory-operations.operational-views.5d","stablePackId":"ecommerce.inventory-operations"},{"category":"feature","checkGroupId":"operational-views","criterionId":"5f","description":"the dashboard shows category units and revenue","executionId":"selected-source-039","featureId":5,"packId":"ecommerce.l2.sales-dashboard","points":1,"role":"feature","source":"scenarios/02-operational-category-totals.json","stableKey":"ecommerce.inventory-operations.operational-views.5f","stablePackId":"ecommerce.inventory-operations"},{"category":"production","checkGroupId":"sales-dashboard","criterionId":"5b","description":"a purchase updates that category's units and revenue live","executionId":"selected-source-039","featureId":5,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":3,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.sales-dashboard"],"role":"guarantee","source":"scenarios/02-operational-category-totals.json","stableKey":"ecommerce.spec.live-state.sales-dashboard.5b"},{"category":"feature","checkGroupId":"operational-views","criterionId":"5c","description":"a purchase recommends another item from that category and excludes an item in the cart","executionId":"selected-source-040","featureId":5,"packId":"ecommerce.l2.recommendations","points":2,"role":"feature","source":"scenarios/02-operational-recommendations.json","stableKey":"ecommerce.inventory-operations.operational-views.5c","stablePackId":"ecommerce.inventory-operations"},{"category":"production","checkGroupId":"cancellation-and-return","criterionId":"3a","description":"cancelling a pending order restores its stock and revenue","executionId":"selected-source-041","featureId":3,"packId":"ecommerce.l2.order-cancellation-features","points":2,"role":"feature","source":"scenarios/02-order-cancellation-core.json","stableKey":"ecommerce.returns-pricing.cancellation-and-return.3a","stablePackId":"ecommerce.returns-pricing"},{"category":"feature","checkGroupId":"cancellation-and-return","criterionId":"3b","description":"a cancelled order is shown as cancelled in the customer's history","executionId":"selected-source-042","featureId":3,"packId":"ecommerce.l2.order-cancellation-features","points":1,"role":"feature","source":"scenarios/02-order-cancellation-history.json","stableKey":"ecommerce.returns-pricing.cancellation-and-return.3b","stablePackId":"ecommerce.returns-pricing"},{"category":"production","checkGroupId":"price-history","criterionId":"4a","description":"a price change updates the live catalog but leaves the customer's exact paid price unchanged","executionId":"selected-source-043","featureId":4,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.price-history-specifications","points":3,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.price-history-features"],"role":"guarantee","source":"scenarios/02-paid-price-history.json","stableKey":"ecommerce.returns-pricing.price-history.4a","stablePackId":"ecommerce.returns-pricing"},{"category":"feature","checkGroupId":"fulfilment-queue","criterionId":"1b","description":"the queue names the warehouse the order will ship from","executionId":"selected-source-044","featureId":1,"packId":"ecommerce.progression.fulfilment-queue","points":1,"role":"feature","source":"scenarios/02-queue-warehouse.json","stableKey":"ecommerce.operations-access.fulfilment-queue.1b","stablePackId":"ecommerce.operations-access"},{"category":"production","checkGroupId":"stock-conservation","criterionId":"202b","description":"a sale and its cancellation leave the shelf exactly as they found it","executionId":"selected-source-045","featureId":202,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","points":2,"requiresFeatures":["ecommerce.l2.order-cancellation-features"],"role":"guarantee","source":"scenarios/02-self-contained.json","stableKey":"ecommerce.inventory-operations.stock-conservation.202b","stablePackId":"ecommerce.inventory-operations"},{"category":"production","checkGroupId":"stock-conservation","criterionId":"202c","description":"a fresh client sees the restored total after a sale is cancelled","executionId":"selected-source-045","featureId":202,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","points":1,"requiresFeatures":["ecommerce.l2.order-cancellation-features"],"role":"guarantee","source":"scenarios/02-self-contained.json","stableKey":"ecommerce.inventory-operations.stock-conservation.202c","stablePackId":"ecommerce.inventory-operations"},{"category":"production","checkGroupId":"operator-authorization","criterionId":"201c","description":"the server refuses a customer's direct attempt to ship their own pending order","executionId":"selected-source-046","featureId":201,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.operations-access-specifications","points":2,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"guarantee","source":"scenarios/02-server-actions.json","stableKey":"ecommerce.operations-access.operator-authorization.201c","stablePackId":"ecommerce.operations-access"},{"category":"production","checkGroupId":"stock-conservation","criterionId":"202d","description":"a direct transfer racing a direct purchase leaves the exact starting total minus the sold unit","executionId":"selected-source-046","featureId":202,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","points":2,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.stock-transfers-features"],"role":"guarantee","source":"scenarios/02-server-actions.json","stableKey":"ecommerce.inventory-operations.stock-conservation.202d","stablePackId":"ecommerce.inventory-operations"},{"category":"production","checkGroupId":"order-owner","criterionId":"204a","description":"the server refuses one customer trying to cancel another customer's still-pending order","executionId":"selected-source-046","featureId":204,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.operations-access-specifications","points":2,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.order-cancellation-features"],"role":"guarantee","source":"scenarios/02-server-actions.json","stableKey":"ecommerce.operations-access.order-owner.204a","stablePackId":"ecommerce.operations-access"},{"category":"production","checkGroupId":"warehouse-transfer","criterionId":"2a","description":"a transfer decreases the source, increases the destination, and preserves the item's exact total","executionId":"selected-source-047","featureId":2,"packId":"ecommerce.l2.stock-transfers-features","points":3,"role":"feature","source":"scenarios/02-strengthened.json","stableKey":"ecommerce.inventory-operations.warehouse-transfer.2a","stablePackId":"ecommerce.inventory-operations"},{"category":"production","checkGroupId":"operator-authorization","criterionId":"201a","description":"the server refuses a customer's direct transfer and neither warehouse nor the item total changes","executionId":"selected-source-047","featureId":201,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.operations-access-specifications","points":2,"requiresFeatures":["ecommerce.l2.stock-transfers-features"],"role":"guarantee","source":"scenarios/02-strengthened.json","stableKey":"ecommerce.operations-access.operator-authorization.201a","stablePackId":"ecommerce.operations-access"},{"category":"production","checkGroupId":"operator-authorization","criterionId":"201b","description":"the server refuses a customer's direct price change and the last accepted price remains exact","executionId":"selected-source-047","featureId":201,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.operations-access-specifications","points":2,"requiresFeatures":["ecommerce.l2.price-history-features"],"role":"guarantee","source":"scenarios/02-strengthened.json","stableKey":"ecommerce.operations-access.operator-authorization.201b","stablePackId":"ecommerce.operations-access"},{"category":"production","checkGroupId":"stock-conservation","criterionId":"202a","description":"a transfer decreases East, increases West, and leaves the item's exact total unchanged","executionId":"selected-source-047","featureId":202,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","points":2,"requiresFeatures":["ecommerce.l2.stock-transfers-features"],"role":"guarantee","source":"scenarios/02-strengthened.json","stableKey":"ecommerce.inventory-operations.stock-conservation.202a","stablePackId":"ecommerce.inventory-operations"},{"category":"production","checkGroupId":"stock-transfer-overdraw","criterionId":"2c","description":"a transfer that would overdraw a warehouse is refused and changes neither warehouse nor the item total","executionId":"selected-source-048","featureId":2,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":2,"requiresFeatures":["ecommerce.l2.stock-transfers-features"],"role":"guarantee","source":"scenarios/02-transfer-overdraw.json","stableKey":"ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c"},{"category":"production","checkGroupId":"stock-transfers","criterionId":"2b","description":"both warehouse totals move live and in opposite directions as stock is transferred","executionId":"selected-source-049","featureId":2,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":2,"requiresFeatures":["ecommerce.l2.stock-transfers-features"],"role":"guarantee","source":"scenarios/02-transfer-totals.json","stableKey":"ecommerce.spec.live-state.stock-transfers.2b"},{"category":"production","checkGroupId":"cart-expiration","criterionId":"304a","description":"an inactive cart expires without a browser, releases stock, and returns empty","executionId":"selected-source-050","featureId":304,"packId":"ecommerce.l3.cart-expiration-features","points":4,"role":"feature","source":"scenarios/03-cart-expiration.json","stableKey":"ecommerce.l3.cart-expiration.cart-expiration.304a","stablePackId":"ecommerce.l3.cart-expiration"},{"category":"production","checkGroupId":"scheduled-work-access","criterionId":"317a","description":"the server refuses customer scheduling and cancellation of restocks","executionId":"selected-source-051","featureId":317,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-access-specifications","points":3,"requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"role":"guarantee","source":"scenarios/03-deferred-access.json","stableKey":"ecommerce.l3.deferred-access.scheduled-work-access.317a","stablePackId":"ecommerce.l3.deferred-access"},{"category":"production","checkGroupId":"restart-survival","criterionId":"311a","description":"a restock scheduled before restart still applies","executionId":"selected-source-052","featureId":311,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-durability-specifications","points":4,"requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"role":"guarantee","source":"scenarios/03-deferred-durability.json","stableKey":"ecommerce.l3.deferred-durability.restart-survival.311a","stablePackId":"ecommerce.l3.deferred-durability"},{"category":"production","checkGroupId":"restart-survival","criterionId":"314a","description":"a reservation pending before restart still expires and returns stock","executionId":"selected-source-052","featureId":314,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-durability-specifications","points":4,"requiresFeatures":["ecommerce.l3.reservations-features"],"role":"guarantee","source":"scenarios/03-deferred-durability.json","stableKey":"ecommerce.l3.deferred-durability.restart-survival.314a","stablePackId":"ecommerce.l3.deferred-durability"},{"category":"production","checkGroupId":"restart-survival","criterionId":"315a","description":"an order shipped before restart still becomes delivered","executionId":"selected-source-052","featureId":315,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-durability-specifications","points":4,"requiresFeatures":["ecommerce.l3.order-delivery-features"],"role":"guarantee","source":"scenarios/03-deferred-durability.json","stableKey":"ecommerce.l3.deferred-durability.restart-survival.315a","stablePackId":"ecommerce.l3.deferred-durability"},{"category":"production","checkGroupId":"restart-survival","criterionId":"316a","description":"a cart survives restart and expires near its original five-minute deadline","executionId":"selected-source-052","featureId":316,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-durability-specifications","points":4,"requiresFeatures":["ecommerce.l3.cart-expiration-features"],"role":"guarantee","source":"scenarios/03-deferred-durability.json","stableKey":"ecommerce.l3.deferred-durability.restart-survival.316a","stablePackId":"ecommerce.l3.deferred-durability"},{"category":"production","checkGroupId":"exactly-once","criterionId":"311a","description":"restart cannot replay a completed restock","executionId":"selected-source-053","featureId":311,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-integrity-specifications","points":3,"requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"role":"guarantee","source":"scenarios/03-deferred-integrity.json","stableKey":"ecommerce.l3.deferred-integrity.exactly-once.311a","stablePackId":"ecommerce.l3.deferred-integrity"},{"category":"production","checkGroupId":"exactly-once","criterionId":"312a","description":"restart leaves one delivered order record","executionId":"selected-source-053","featureId":312,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-integrity-specifications","points":3,"requiresFeatures":["ecommerce.l3.order-delivery-features"],"role":"guarantee","source":"scenarios/03-deferred-integrity.json","stableKey":"ecommerce.l3.deferred-integrity.exactly-once.312a","stablePackId":"ecommerce.l3.deferred-integrity"},{"category":"production","checkGroupId":"stock-conservation","criterionId":"313a","description":"expiry returns exactly the unit reserved","executionId":"selected-source-053","featureId":313,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-integrity-specifications","points":3,"requiresFeatures":["ecommerce.l3.reservations-features"],"role":"guarantee","source":"scenarios/03-deferred-integrity.json","stableKey":"ecommerce.l3.deferred-integrity.stock-conservation.313a","stablePackId":"ecommerce.l3.deferred-integrity"},{"category":"production","checkGroupId":"stock-conservation","criterionId":"314a","description":"checkout does not decrement stock after the reservation already did","executionId":"selected-source-053","featureId":314,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-integrity-specifications","points":3,"requiresFeatures":["ecommerce.l3.reservations-features"],"role":"guarantee","source":"scenarios/03-deferred-integrity.json","stableKey":"ecommerce.l3.deferred-integrity.stock-conservation.314a","stablePackId":"ecommerce.l3.deferred-integrity"},{"category":"production","checkGroupId":"order-delivery","criterionId":"303a","description":"a shipped order becomes delivered in customer and staff views","executionId":"selected-source-054","featureId":303,"packId":"ecommerce.l3.order-delivery-features","points":3,"role":"feature","source":"scenarios/03-order-delivery.json","stableKey":"ecommerce.l3.order-delivery.order-delivery.303a","stablePackId":"ecommerce.l3.order-delivery"},{"category":"production","checkGroupId":"order-delivery","criterionId":"305a","description":"a cancelled order remains cancelled after the delivery interval","executionId":"selected-source-054","featureId":305,"packId":"ecommerce.l3.order-delivery-features","points":2,"role":"feature","source":"scenarios/03-order-delivery.json","stableKey":"ecommerce.l3.order-delivery.order-delivery.305a","stablePackId":"ecommerce.l3.order-delivery"},{"category":"production","checkGroupId":"reservations","criterionId":"301a","description":"adding an item reserves one unit for every open viewer","executionId":"selected-source-055","featureId":301,"packId":"ecommerce.l3.reservations-features","points":2,"role":"feature","source":"scenarios/03-reservations.json","stableKey":"ecommerce.l3.reservations.reservations.301a","stablePackId":"ecommerce.l3.reservations"},{"category":"interface","checkGroupId":"reservations","criterionId":"305a","description":"the reservation timer decreases","executionId":"selected-source-055","featureId":305,"packId":"ecommerce.l3.reservations-features","points":1,"role":"feature","source":"scenarios/03-reservations.json","stableKey":"ecommerce.l3.reservations.reservations.305a","stablePackId":"ecommerce.l3.reservations"},{"category":"feature","checkGroupId":"reservations","criterionId":"306a","description":"checkout converts the reservation into an order and empties the cart","executionId":"selected-source-055","featureId":306,"packId":"ecommerce.l3.reservations-features","points":2,"role":"feature","source":"scenarios/03-reservations.json","stableKey":"ecommerce.l3.reservations.reservations.306a","stablePackId":"ecommerce.l3.reservations"},{"category":"feature","checkGroupId":"reservations","criterionId":"307a","description":"an expired reservation marks its cart line","executionId":"selected-source-055","featureId":307,"packId":"ecommerce.l3.reservations-features","points":3,"role":"feature","source":"scenarios/03-reservations.json","stableKey":"ecommerce.l3.reservations.reservations.307a","stablePackId":"ecommerce.l3.reservations"},{"category":"feature","checkGroupId":"reservations","criterionId":"308a","description":"raising quantity starts a new reservation window","executionId":"selected-source-055","featureId":308,"packId":"ecommerce.l3.reservations-features","points":2,"role":"feature","source":"scenarios/03-reservations.json","stableKey":"ecommerce.l3.reservations.reservations.308a","stablePackId":"ecommerce.l3.reservations"},{"category":"feature","checkGroupId":"scheduled-restocks","criterionId":"305a","description":"a due restock updates stock and moves to the ledger","executionId":"selected-source-056","featureId":305,"packId":"ecommerce.l3.scheduled-restocks-features","points":3,"role":"feature","source":"scenarios/03-scheduled-restock-apply.json","stableKey":"ecommerce.l3.scheduled-restocks.scheduled-restocks.305a","stablePackId":"ecommerce.l3.scheduled-restocks"},{"category":"production","checkGroupId":"scheduled-restocks","criterionId":"306a","description":"a cancelled restock never applies","executionId":"selected-source-057","featureId":306,"packId":"ecommerce.l3.scheduled-restocks-features","points":2,"role":"feature","source":"scenarios/03-scheduled-restock-cancel.json","stableKey":"ecommerce.l3.scheduled-restocks.scheduled-restocks.306a","stablePackId":"ecommerce.l3.scheduled-restocks"},{"category":"feature","checkGroupId":"scheduled-restocks","criterionId":"302a","description":"a scheduled restock is pending and its remaining time decreases","executionId":"selected-source-058","featureId":302,"packId":"ecommerce.l3.scheduled-restocks-features","points":2,"role":"feature","source":"scenarios/03-scheduled-restocks.json","stableKey":"ecommerce.l3.scheduled-restocks.scheduled-restocks.302a","stablePackId":"ecommerce.l3.scheduled-restocks"},{"category":"production","checkGroupId":"server-time","criterionId":"312a","description":"restart preserves the due time and the work later completes","executionId":"selected-source-059","featureId":312,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.server-time-specifications","points":3,"requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"role":"guarantee","source":"scenarios/03-server-time.json","stableKey":"ecommerce.l3.server-time.server-time.312a","stablePackId":"ecommerce.l3.server-time"},{"category":"production","checkGroupId":"server-time","criterionId":"313a","description":"a reservation expires while its browser is closed","executionId":"selected-source-059","featureId":313,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.server-time-specifications","points":3,"requiresFeatures":["ecommerce.l3.reservations-features"],"role":"guarantee","source":"scenarios/03-server-time.json","stableKey":"ecommerce.l3.server-time.server-time.313a","stablePackId":"ecommerce.l3.server-time"},{"category":"production","checkGroupId":"account-state-recovery","criterionId":"105b","description":"the same account and cart survive the connection dropping and coming back","executionId":"selected-source-060","featureId":105,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":1,"requiresFeatures":["ecommerce.feature.accounts","ecommerce.feature.checkout"],"role":"guarantee","source":"scenarios/progression-account-state-reconnect.json","stableKey":"ecommerce.spec.state-durability.account-state-recovery.105b"},{"category":"production","checkGroupId":"account-state-recovery","criterionId":"105a","description":"cart and order history survive reload and backend restart, including a fresh account login","executionId":"selected-source-061","featureId":105,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":1,"requiresFeatures":["ecommerce.feature.accounts","ecommerce.feature.checkout"],"role":"guarantee","source":"scenarios/progression-account-state-reload.json","stableKey":"ecommerce.spec.state-durability.account-state-recovery.105a"},{"category":"production","checkGroupId":"automatic-reorder-access","criterionId":"502c","description":"a customer cannot see or replay automatic reorder management","executionId":"selected-source-062","featureId":502,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.progression.automatic-reorder"],"role":"guarantee","source":"scenarios/progression-automatic-reorder-access.json","stableKey":"ecommerce.spec.access-control.automatic-reorder-access.502c"},{"category":"production","checkGroupId":"automatic-reorder-deduplication","criterionId":"502b","description":"more sales do not duplicate a pending restock","executionId":"selected-source-063","featureId":502,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":2,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.automatic-reorder"],"role":"guarantee","source":"scenarios/progression-automatic-reorder-duplicate.json","stableKey":"ecommerce.spec.transactional-integrity.automatic-reorder-deduplication.502b"},{"category":"feature","checkGroupId":"automatic-reorder","criterionId":"502a","description":"crossing the threshold creates one pending restock","executionId":"selected-source-064","featureId":502,"packId":"ecommerce.progression.automatic-reorder","points":3,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"feature","source":"scenarios/progression-automatic-reorder.json","stableKey":"ecommerce.progression.automatic-reorder.automatic-reorder.502a"},{"category":"production","checkGroupId":"books-balance","criterionId":"107a","description":"revenue rises by exactly what was bought","executionId":"selected-source-065","featureId":107,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":1,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/progression-books-balance.json","stableKey":"ecommerce.spec.transactional-integrity.books-balance.107a"},{"category":"production","checkGroupId":"books-balance","criterionId":"107b","description":"what the store sold is what left the warehouses, and a fresh client agrees","executionId":"selected-source-065","featureId":107,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":1,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/progression-books-balance.json","stableKey":"ecommerce.spec.transactional-integrity.books-balance.107b"},{"category":"feature","checkGroupId":"bundle-checkout","criterionId":"741a","description":"adding a bundle reserves its components and checkout records the bundle price once","executionId":"selected-source-066","featureId":741,"packId":"ecommerce.feature.bundle-checkout","points":2,"role":"feature","source":"scenarios/progression-bundle-checkout.json","stableKey":"ecommerce.feature.bundle-checkout.bundle-checkout.741a"},{"category":"production","checkGroupId":"bundle-744","criterionId":"744a","description":"two competing reservations accept exactly one whole bundle without consuming extra components","executionId":"selected-source-066","featureId":744,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","points":3,"requiresFeatures":["ecommerce.feature.bundle-checkout"],"role":"guarantee","source":"scenarios/progression-bundle-checkout.json","stableKey":"ecommerce.spec.bundle-integrity.bundle-744.744a"},{"category":"production","checkGroupId":"bundle-745","criterionId":"745a","description":"a missing component refuses the reservation without taking available stock or adding a cart line","executionId":"selected-source-066","featureId":745,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","points":3,"requiresFeatures":["ecommerce.feature.bundle-checkout"],"role":"guarantee","source":"scenarios/progression-bundle-checkout.json","stableKey":"ecommerce.spec.bundle-integrity.bundle-745.745a"},{"category":"production","checkGroupId":"bundle-746","criterionId":"746a","description":"an expired reservation releases each component once across a backend restart","executionId":"selected-source-066","featureId":746,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","points":3,"requiresFeatures":["ecommerce.feature.bundle-checkout"],"role":"guarantee","source":"scenarios/progression-bundle-checkout.json","stableKey":"ecommerce.spec.bundle-integrity.bundle-746.746a"},{"category":"production","checkGroupId":"bundle-747","criterionId":"747a","description":"two checkout requests consume one reservation and create one paid bundle","executionId":"selected-source-066","featureId":747,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","points":3,"requiresFeatures":["ecommerce.feature.bundle-checkout"],"role":"guarantee","source":"scenarios/progression-bundle-checkout.json","stableKey":"ecommerce.spec.bundle-integrity.bundle-747.747a"},{"category":"feature","checkGroupId":"bundle-returns","criterionId":"742a","description":"returning a shipped bundle refunds the paid price and restores original components after its definition changes","executionId":"selected-source-067","featureId":742,"packId":"ecommerce.feature.bundle-returns","points":2,"role":"feature","source":"scenarios/progression-bundle-returns.json","stableKey":"ecommerce.feature.bundle-returns.bundle-returns.742a"},{"category":"production","checkGroupId":"bundle-742","criterionId":"742b","description":"replaying a completed bundle return after restart does not refund or restock it twice","executionId":"selected-source-067","featureId":742,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","points":3,"requiresFeatures":["ecommerce.feature.bundle-returns"],"role":"guarantee","source":"scenarios/progression-bundle-returns.json","stableKey":"ecommerce.spec.bundle-integrity.bundle-742.742b"},{"category":"production","checkGroupId":"bundle-748","criterionId":"748a","description":"another customer cannot return a paid bundle by submitting its order ID","executionId":"selected-source-067","featureId":748,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","points":3,"requiresFeatures":["ecommerce.feature.bundle-returns"],"role":"guarantee","source":"scenarios/progression-bundle-returns.json","stableKey":"ecommerce.spec.bundle-integrity.bundle-748.748a"},{"category":"feature","checkGroupId":"cart","criterionId":"4a","description":"adding the same item twice raises its quantity instead of adding a second line","executionId":"selected-source-068","featureId":4,"packId":"ecommerce.feature.cart","points":1,"role":"feature","source":"scenarios/progression-cart-checkout.json","stableKey":"ecommerce.feature.cart-checkout.cart.4a","stablePackId":"ecommerce.feature.cart-checkout"},{"category":"feature","checkGroupId":"cart","criterionId":"4d","description":"checkout creates one order, reduces stock, and empties the cart","executionId":"selected-source-068","featureId":4,"packId":"ecommerce.feature.checkout","points":2,"role":"feature","source":"scenarios/progression-cart-checkout.json","stableKey":"ecommerce.feature.cart-checkout.cart.4d","stablePackId":"ecommerce.feature.cart-checkout"},{"category":"feature","checkGroupId":"cart-recovery","criterionId":"503a","description":"restoring an expired cart reserves available items again","executionId":"selected-source-069","featureId":503,"packId":"ecommerce.progression.cart-recovery","points":3,"role":"feature","source":"scenarios/progression-cart-recovery.json","stableKey":"ecommerce.progression.cart-recovery.cart-recovery.503a"},{"category":"feature","checkGroupId":"cart-recovery","criterionId":"503b","description":"a partial restore keeps available items and names each unavailable item","executionId":"selected-source-069","featureId":503,"packId":"ecommerce.progression.cart-recovery","points":3,"role":"feature","source":"scenarios/progression-cart-recovery.json","stableKey":"ecommerce.progression.cart-recovery.cart-recovery.503b"},{"category":"feature","checkGroupId":"catalog-management","criterionId":"622a","description":"a new product reaches the public catalog","executionId":"selected-source-070","featureId":622,"packId":"ecommerce.progression.catalog-management","points":2,"role":"feature","source":"scenarios/progression-catalog-management.json","stableKey":"ecommerce.progression.catalog-management.catalog-management.622a"},{"category":"feature","checkGroupId":"catalog-management","criterionId":"622b","description":"the product exposes its named variants","executionId":"selected-source-070","featureId":622,"packId":"ecommerce.progression.catalog-management","points":2,"role":"feature","source":"scenarios/progression-catalog-management.json","stableKey":"ecommerce.progression.catalog-management.catalog-management.622b"},{"category":"production","checkGroupId":"checkout-crash-integrity","criterionId":"910a","description":"interrupted checkout recovers to the prepared cart or one complete order with the cart cleared after each independent process crash","executionId":"selected-source-071","featureId":910,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":1,"requiresFeatures":["ecommerce.feature.checkout"],"role":"guarantee","source":"scenarios/progression-checkout-crash.json","stableKey":"ecommerce.spec.state-durability.checkout-crash-integrity.910a"},{"category":"production","checkGroupId":"checkout-crash-durability","criterionId":"910b","description":"acknowledged checkout is not rolled back and earlier orders remain unchanged after each independent process crash","executionId":"selected-source-071","featureId":910,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":1,"requiresFeatures":["ecommerce.feature.checkout"],"role":"guarantee","source":"scenarios/progression-checkout-crash.json","stableKey":"ecommerce.spec.state-durability.checkout-crash-durability.910b"},{"category":"production","checkGroupId":"payment-records","criterionId":"623a","description":"checkout records the exact paid amount","executionId":"selected-source-072","featureId":623,"packId":"ecommerce.progression.payment-records","points":3,"role":"feature","source":"scenarios/progression-core-business.json","stableKey":"ecommerce.progression.payment-records.payment-records.623a"},{"category":"production","checkGroupId":"payment-deduplication","criterionId":"623b","description":"one checkout has one payment record","executionId":"selected-source-072","featureId":623,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":2,"requiresFeatures":["ecommerce.progression.payment-records"],"role":"guarantee","source":"scenarios/progression-core-business.json","stableKey":"ecommerce.spec.transactional-integrity.payment-deduplication.623b"},{"category":"feature","checkGroupId":"customer-profile","criterionId":"620c","description":"the owner can save and view a customer profile","executionId":"selected-source-073","featureId":620,"packId":"ecommerce.progression.customer-profile","points":1,"role":"feature","source":"scenarios/progression-customer-profile.json","stableKey":"ecommerce.progression.customer-profile.customer-profile.620c"},{"category":"production","checkGroupId":"customer-profile-reload","criterionId":"620a","description":"the saved profile survives reload and backend restart in a fresh browser","executionId":"selected-source-073","featureId":620,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":2,"requiresFeatures":["ecommerce.progression.customer-profile"],"role":"guarantee","source":"scenarios/progression-customer-profile.json","stableKey":"ecommerce.spec.state-durability.customer-profile-reload.620a"},{"category":"production","checkGroupId":"customer-profile-privacy","criterionId":"620b","description":"another customer neither sees nor receives the owner's private address","executionId":"selected-source-073","featureId":620,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.progression.customer-profile"],"role":"guarantee","source":"scenarios/progression-customer-profile.json","stableKey":"ecommerce.spec.access-control.customer-profile-privacy.620b"},{"category":"feature","checkGroupId":"delivery-notification-delivery","criterionId":"501a","description":"the order owner receives one delivery notification","executionId":"selected-source-074","featureId":501,"packId":"ecommerce.progression.delivery-notifications","points":3,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"feature","source":"scenarios/progression-delivery-notifications.json","stableKey":"ecommerce.progression.delivery-notifications.delivery-notification-delivery.501a"},{"category":"production","checkGroupId":"delivery-notification-privacy","criterionId":"501b","description":"another customer cannot see the delivery notification","executionId":"selected-source-074","featureId":501,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.delivery-notifications","ecommerce.progression.fulfilment-queue"],"role":"guarantee","source":"scenarios/progression-delivery-notifications.json","stableKey":"ecommerce.spec.access-control.delivery-notification-privacy.501b"},{"category":"feature","checkGroupId":"faceted-search","criterionId":"401a","description":"category, price, and availability filters apply together","executionId":"selected-source-075","featureId":401,"packId":"ecommerce.progression.faceted-search","points":3,"role":"feature","source":"scenarios/progression-faceted-filters.json","stableKey":"ecommerce.progression.faceted-search.faceted-search.401a"},{"category":"feature","checkGroupId":"faceted-search","criterionId":"402a","description":"moving between pages returns the same ordered items without duplicates","executionId":"selected-source-076","featureId":402,"packId":"ecommerce.progression.faceted-search","points":3,"role":"feature","source":"scenarios/progression-faceted-pagination.json","stableKey":"ecommerce.progression.faceted-search.faceted-search.402a"},{"category":"production","checkGroupId":"managed-support-privacy","criterionId":"613b","description":"another customer cannot read or reply to the managed case","executionId":"selected-source-077","featureId":613,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"requiresFeatures":["ecommerce.progression.managed-support"],"role":"guarantee","source":"scenarios/progression-managed-support-privacy.json","stableKey":"ecommerce.spec.access-control.managed-support-privacy.613b"},{"category":"feature","checkGroupId":"managed-support","criterionId":"613c","description":"staff can update a support case and the customer can reply","executionId":"selected-source-078","featureId":613,"packId":"ecommerce.progression.managed-support","points":1,"role":"feature","source":"scenarios/progression-managed-support-shared.json","stableKey":"ecommerce.progression.managed-support.managed-support.613c"},{"category":"production","checkGroupId":"managed-support","criterionId":"613a","description":"the customer and staff see the same replies and status live","executionId":"selected-source-078","featureId":613,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":2,"requiresFeatures":["ecommerce.progression.managed-support"],"role":"guarantee","source":"scenarios/progression-managed-support-shared.json","stableKey":"ecommerce.spec.live-state.managed-support.613a"},{"category":"feature","checkGroupId":"notification-preferences","criterionId":"630c","description":"the customer can save a notification choice","executionId":"selected-source-079","featureId":630,"packId":"ecommerce.progression.notification-preferences","points":1,"role":"feature","source":"scenarios/progression-notification-preferences.json","stableKey":"ecommerce.progression.notification-preferences.notification-preferences.630c"},{"category":"production","checkGroupId":"notification-preferences-reload","criterionId":"630a","description":"notification choices survive reload and backend restart in a fresh browser","executionId":"selected-source-079","featureId":630,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":2,"requiresFeatures":["ecommerce.progression.notification-preferences"],"role":"guarantee","source":"scenarios/progression-notification-preferences.json","stableKey":"ecommerce.spec.state-durability.notification-preferences-reload.630a"},{"category":"production","checkGroupId":"notification-preferences-privacy","criterionId":"630b","description":"the owner's choice does not change another account","executionId":"selected-source-079","featureId":630,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.progression.notification-preferences"],"role":"guarantee","source":"scenarios/progression-notification-preferences.json","stableKey":"ecommerce.spec.access-control.notification-preferences-privacy.630b"},{"category":"production","checkGroupId":"open-list","criterionId":"902a","description":"one customer has the Keyboard's reviews open before another posts one; the already-open view shows that review exactly once","executionId":"selected-source-080","featureId":902,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":1,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","source":"scenarios/progression-open-list-live.json","stableKey":"ecommerce.spec.live-state.open-list.902a"},{"category":"interface","checkGroupId":"cancellation-and-return","criterionId":"3e","description":"a pending order does not offer a return button","executionId":"selected-source-081","featureId":331,"packId":"ecommerce.l3.order-returns-features","points":1,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"feature","source":"scenarios/progression-order-return-boundary.json","stableKey":"ecommerce.returns-pricing.cancellation-and-return.3e","stablePackId":"ecommerce.returns-pricing"},{"category":"production","checkGroupId":"cancellation-and-return","criterionId":"3f","description":"the server refuses a pending return without changing stock or revenue","executionId":"selected-source-081","featureId":332,"packId":"ecommerce.l3.order-returns-features","points":2,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"feature","source":"scenarios/progression-order-return-boundary.json","stableKey":"ecommerce.returns-pricing.cancellation-and-return.3f","stablePackId":"ecommerce.returns-pricing"},{"category":"production","checkGroupId":"cancellation-and-return","criterionId":"3c","description":"returning a shipped item restores stock and revenue and marks the item returned","executionId":"selected-source-082","featureId":330,"packId":"ecommerce.l3.order-returns-features","points":3,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"feature","source":"scenarios/progression-order-return-complete.json","stableKey":"ecommerce.returns-pricing.cancellation-and-return.3c","stablePackId":"ecommerce.returns-pricing"},{"category":"production","checkGroupId":"order-support-ownership","criterionId":"614b","description":"another customer cannot attach or inspect the owner's order","executionId":"selected-source-083","featureId":614,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.progression.order-support"],"role":"guarantee","source":"scenarios/progression-order-support-boundary.json","stableKey":"ecommerce.spec.access-control.order-support-ownership.614b"},{"category":"feature","checkGroupId":"order-support-owned","criterionId":"614a","description":"the customer can link their order and staff can inspect it","executionId":"selected-source-084","featureId":614,"packId":"ecommerce.progression.order-support","points":3,"role":"feature","source":"scenarios/progression-order-support-owned.json","stableKey":"ecommerce.progression.order-support.order-support-owned.614a"},{"category":"feature","checkGroupId":"personalized-recommendations","criterionId":"403a","description":"recommendations follow the customer's categories, global sales, and name tie-break","executionId":"selected-source-085","featureId":403,"packId":"ecommerce.progression.personalized-recommendations","points":4,"role":"feature","source":"scenarios/progression-personalized-recommendations.json","stableKey":"ecommerce.progression.personalized-recommendations.personalized-recommendations.403a"},{"category":"production","checkGroupId":"recommendation-profile-isolation","criterionId":"403b","description":"one customer's activity does not replace another customer's recommendations","executionId":"selected-source-085","featureId":403,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.progression.personalized-recommendations"],"role":"guarantee","source":"scenarios/progression-personalized-recommendations.json","stableKey":"ecommerce.spec.access-control.recommendation-profile-isolation.403b"},{"category":"production","checkGroupId":"price-history","criterionId":"4c","description":"a price change updates an open cart and direct checkout persists the new total","executionId":"selected-source-086","featureId":420,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.price-history-specifications","points":3,"requiresFeatures":["ecommerce.feature.checkout","ecommerce.l2.price-history-features"],"role":"guarantee","source":"scenarios/progression-price-cart-checkout.json","stableKey":"ecommerce.returns-pricing.price-history.4c","stablePackId":"ecommerce.returns-pricing"},{"category":"feature","checkGroupId":"product-bundles","criterionId":"740a","description":"a saved bundle shows the exact price and component quantities after reopening the application","executionId":"selected-source-087","featureId":740,"packId":"ecommerce.feature.product-bundles","points":2,"role":"feature","source":"scenarios/progression-product-bundles.json","stableKey":"ecommerce.feature.product-bundles.product-bundles.740a"},{"category":"production","checkGroupId":"bundle-743","criterionId":"743a","description":"a customer cannot replace a staff-created bundle through the application write","executionId":"selected-source-087","featureId":743,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","points":3,"requiresFeatures":["ecommerce.feature.product-bundles"],"role":"guarantee","source":"scenarios/progression-product-bundles.json","stableKey":"ecommerce.spec.bundle-integrity.bundle-743.743a"},{"category":"feature","checkGroupId":"promotion-checkout-active","criterionId":"621a","description":"an active promotion changes checkout and is recorded on the order","executionId":"selected-source-088","featureId":621,"packId":"ecommerce.progression.promotion-checkout","points":3,"role":"feature","source":"scenarios/progression-promotion-checkout.json","stableKey":"ecommerce.progression.promotion-checkout.promotion-checkout-active.621a"},{"category":"feature","checkGroupId":"promotion-checkout-expired","criterionId":"621b","description":"an expired promotion is refused","executionId":"selected-source-088","featureId":621,"packId":"ecommerce.progression.promotion-checkout","points":2,"role":"feature","source":"scenarios/progression-promotion-checkout.json","stableKey":"ecommerce.progression.promotion-checkout.promotion-checkout-expired.621b"},{"category":"feature","checkGroupId":"promotion-checkout-exhausted","criterionId":"621c","description":"a fully redeemed promotion is refused","executionId":"selected-source-088","featureId":621,"packId":"ecommerce.progression.promotion-checkout","points":2,"role":"feature","source":"scenarios/progression-promotion-checkout.json","stableKey":"ecommerce.progression.promotion-checkout.promotion-checkout-exhausted.621c"},{"category":"feature","checkGroupId":"promotion-report-redemptions","criterionId":"622a","description":"the promotion report has the exact redemption count","executionId":"selected-source-089","featureId":622,"packId":"ecommerce.progression.promotion-reporting","points":1,"role":"feature","source":"scenarios/progression-promotion-reporting.json","stableKey":"ecommerce.progression.promotion-reporting.promotion-report-redemptions.622a"},{"category":"feature","checkGroupId":"promotion-report-revenue","criterionId":"622b","description":"the promotion report has the exact discounted revenue","executionId":"selected-source-089","featureId":622,"packId":"ecommerce.progression.promotion-reporting","points":2,"role":"feature","source":"scenarios/progression-promotion-reporting.json","stableKey":"ecommerce.progression.promotion-reporting.promotion-report-revenue.622b"},{"category":"feature","checkGroupId":"promotion-rule-values","criterionId":"620a","description":"staff can save every bounded promotion value","executionId":"selected-source-090","featureId":620,"packId":"ecommerce.progression.promotion-rules","points":2,"role":"feature","source":"scenarios/progression-promotion-rules.json","stableKey":"ecommerce.progression.promotion-rules.promotion-rule-values.620a"},{"category":"production","checkGroupId":"promotion-management-boundary","criterionId":"620b","description":"customers cannot open promotion management","executionId":"selected-source-090","featureId":620,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.progression.promotion-rules"],"role":"guarantee","source":"scenarios/progression-promotion-rules.json","stableKey":"ecommerce.spec.access-control.promotion-management-boundary.620b"},{"category":"feature","checkGroupId":"purchase-order","criterionId":"3c","description":"the purchase is recorded in the buyer's order history at the price paid","executionId":"selected-source-091","featureId":3,"packId":"ecommerce.feature.purchasing","points":1,"role":"feature","source":"scenarios/progression-purchasing.json","stableKey":"ecommerce.feature.purchasing.purchase-order.3c"},{"category":"feature","checkGroupId":"recommendation-feedback","criterionId":"504a","description":"dismissing a recommendation removes it from the customer view","executionId":"selected-source-092","featureId":504,"packId":"ecommerce.progression.recommendation-feedback","points":2,"role":"feature","source":"scenarios/progression-recommendation-feedback.json","stableKey":"ecommerce.progression.recommendation-feedback.recommendation-feedback.504a"},{"category":"production","checkGroupId":"recommendation-feedback-privacy","criterionId":"504b","description":"one customer's dismissal does not hide another customer's result","executionId":"selected-source-092","featureId":504,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.progression.recommendation-feedback"],"role":"guarantee","source":"scenarios/progression-recommendation-feedback.json","stableKey":"ecommerce.spec.access-control.recommendation-feedback-privacy.504b"},{"category":"production","checkGroupId":"recommendation-feedback-restart","criterionId":"504c","description":"a dismissed recommendation stays absent after reload and backend restart in a fresh browser","executionId":"selected-source-092","featureId":504,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":2,"requiresFeatures":["ecommerce.progression.recommendation-feedback"],"role":"guarantee","source":"scenarios/progression-recommendation-feedback.json","stableKey":"ecommerce.spec.state-durability.recommendation-feedback-restart.504c"},{"category":"production","checkGroupId":"review-eligibility-direct","criterionId":"618a","description":"a nonbuyer cannot submit or replace a review by claiming a buyer's username","executionId":"selected-source-093","featureId":618,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.review-access-specifications","points":2,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","source":"scenarios/progression-review-access.json","stableKey":"ecommerce.progression.review-access-specifications.review-eligibility-direct.618a"},{"category":"production","checkGroupId":"stored-review-script","criterionId":"9180a","description":"review input is rejected or displayed without executing script in another customer session","executionId":"selected-source-094","featureId":9180,"observations":["unmentioned"],"packId":"ecommerce.progression.review-access-specifications","points":1,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","source":"scenarios/progression-review-script.json","stableKey":"ecommerce.progression.review-access-specifications.stored-review-script.9180a"},{"category":"production","checkGroupId":"search-ordering","criterionId":"402b","description":"active filters and text searches remain alphabetical after purchases; clearing them restores purchase ranking","executionId":"selected-source-095","featureId":402,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.search-ordering","points":1,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.faceted-search"],"role":"guarantee","source":"scenarios/progression-search-ordering.json","stableKey":"ecommerce.spec.search-ordering.search-ordering.402b"},{"category":"production","checkGroupId":"shipping-accounting","criterionId":"202e","description":"shipping a purchased order does not deduct stock or add revenue again","executionId":"selected-source-096","featureId":202,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","points":2,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin","ecommerce.progression.fulfilment-queue"],"role":"guarantee","source":"scenarios/progression-shipping-accounting.json","stableKey":"ecommerce.inventory-operations.shipping-accounting.202e","stablePackId":"ecommerce.inventory-operations"},{"category":"production","checkGroupId":"signed-out-purchase","criterionId":"3a","description":"using the purchase control while signed out does not buy an item","executionId":"selected-source-097","featureId":3,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/progression-signed-out-purchase.json","stableKey":"ecommerce.spec.access-control.signed-out-purchase.3a"},{"category":"feature","checkGroupId":"split-tender-refunds-751","criterionId":"751a","description":"Full refund restores each original payment portion","executionId":"selected-source-098","featureId":751,"packId":"ecommerce.feature.split-tender-refunds","points":2,"requiresFeatures":["ecommerce.feature.split-tender-refunds","ecommerce.feature.store-credit","ecommerce.progression.support-refunds"],"role":"feature","source":"scenarios/progression-split-tender-refunds.json","stableKey":"ecommerce.feature.split-tender-refunds.split-tender-refunds-751.751a"},{"category":"production","checkGroupId":"production-756","criterionId":"756a","description":"Concurrent refunds restore the original credit and external amounts once, including after restart","executionId":"selected-source-098","featureId":756,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.split-tender-refunds","points":2,"requiresFeatures":["ecommerce.feature.split-tender-refunds","ecommerce.feature.store-credit","ecommerce.progression.support-refunds"],"role":"guarantee","source":"scenarios/progression-split-tender-refunds.json","stableKey":"ecommerce.spec.split-tender-refunds.production-756.756a"},{"category":"feature","checkGroupId":"staff-access","criterionId":"601a","description":"staff and administrators can sign in and open staff tools","executionId":"selected-source-099","featureId":601,"packId":"ecommerce.progression.staff-access","points":2,"role":"feature","source":"scenarios/progression-staff-access.json","stableKey":"ecommerce.progression.staff-access.staff-access.601a"},{"category":"production","checkGroupId":"staff-area-boundary","criterionId":"601b","description":"customers cannot open staff tools","executionId":"selected-source-099","featureId":601,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.progression.staff-access"],"role":"guarantee","source":"scenarios/progression-staff-access.json","stableKey":"ecommerce.spec.access-control.staff-area-boundary.601b"},{"category":"feature","checkGroupId":"staff-activity","criterionId":"624a","description":"an administrative change records its actor, action, subject, and time","executionId":"selected-source-100","featureId":624,"packId":"ecommerce.progression.staff-activity","points":3,"role":"feature","source":"scenarios/progression-staff-activity.json","stableKey":"ecommerce.progression.staff-activity.staff-activity.624a"},{"category":"production","checkGroupId":"staff-activity-privacy","criterionId":"624b","description":"customers cannot open staff activity history","executionId":"selected-source-100","featureId":624,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.progression.staff-activity"],"role":"guarantee","source":"scenarios/progression-staff-activity.json","stableKey":"ecommerce.spec.access-control.staff-activity-privacy.624b"},{"category":"feature","checkGroupId":"staff-roles","criterionId":"621c","description":"an administrator can assign a staff role","executionId":"selected-source-101","featureId":621,"packId":"ecommerce.progression.staff-roles","points":1,"role":"feature","source":"scenarios/progression-staff-roles.json","stableKey":"ecommerce.progression.staff-roles.staff-roles.621c"},{"category":"production","checkGroupId":"staff-role-reload","criterionId":"621a","description":"an assigned staff role survives reload and backend restart in a fresh browser","executionId":"selected-source-101","featureId":621,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":2,"requiresFeatures":["ecommerce.progression.staff-roles"],"role":"guarantee","source":"scenarios/progression-staff-roles.json","stableKey":"ecommerce.spec.state-durability.staff-role-reload.621a"},{"category":"production","checkGroupId":"staff-role-boundary","criterionId":"621b","description":"a staff member cannot assign roles through the UI or a replayed request","executionId":"selected-source-101","featureId":621,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.progression.staff-roles"],"role":"guarantee","source":"scenarios/progression-staff-roles.json","stableKey":"ecommerce.spec.access-control.staff-role-boundary.621b"},{"category":"production","checkGroupId":"staff-role-revocation","criterionId":"621d","description":"removing administrator access blocks a previously authorized session without changing the target role","executionId":"selected-source-101","featureId":621,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.progression.staff-roles"],"role":"guarantee","source":"scenarios/progression-staff-roles.json","stableKey":"ecommerce.spec.access-control.staff-role-revocation.621d"},{"category":"feature","checkGroupId":"stock-alert-delivery","criterionId":"631c","description":"restored stock sends the requested alert","executionId":"selected-source-102","featureId":631,"packId":"ecommerce.progression.stock-alerts","points":1,"requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"feature","source":"scenarios/progression-stock-alert-delivery.json","stableKey":"ecommerce.progression.stock-alerts.stock-alert-delivery.631c"},{"category":"production","checkGroupId":"stock-alert-deduplication","criterionId":"631a","description":"restored stock sends one alert and later restocks do not duplicate it","executionId":"selected-source-103","featureId":631,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":2,"requiresFeatures":["ecommerce.feature.warehouse-admin","ecommerce.progression.stock-alerts"],"role":"guarantee","source":"scenarios/progression-stock-alerts.json","stableKey":"ecommerce.spec.transactional-integrity.stock-alert-deduplication.631a"},{"category":"production","checkGroupId":"stock-alert-privacy","criterionId":"631b","description":"a customer who did not request the alert cannot see it","executionId":"selected-source-103","featureId":631,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"requiresFeatures":["ecommerce.feature.warehouse-admin","ecommerce.progression.stock-alerts"],"role":"guarantee","source":"scenarios/progression-stock-alerts.json","stableKey":"ecommerce.spec.access-control.stock-alert-privacy.631b"},{"category":"production","checkGroupId":"stock-limit","criterionId":"3d","description":"an item sells out visibly, and a further purchase is refused without changing stock","executionId":"selected-source-104","featureId":3,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":2,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/progression-stock-limit.json","stableKey":"ecommerce.spec.concurrency-safety.stock-limit.3d"},{"category":"feature","checkGroupId":"store-credit-750","criterionId":"750a","description":"Credit checkout records both payment portions","executionId":"selected-source-105","featureId":750,"packId":"ecommerce.feature.store-credit","points":2,"role":"feature","source":"scenarios/progression-store-credit.json","stableKey":"ecommerce.feature.store-credit.store-credit-750.750a"},{"category":"production","checkGroupId":"production-752","criterionId":"752a","description":"Repeating a grant reference does not increase the balance twice","executionId":"selected-source-105","featureId":752,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.store-credit","points":2,"requiresFeatures":["ecommerce.feature.store-credit"],"role":"guarantee","source":"scenarios/progression-store-credit.json","stableKey":"ecommerce.spec.store-credit.production-752.752a"},{"category":"production","checkGroupId":"production-753","criterionId":"753a","description":"A customer cannot grant credit","executionId":"selected-source-105","featureId":753,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.store-credit","points":2,"requiresFeatures":["ecommerce.feature.store-credit"],"role":"guarantee","source":"scenarios/progression-store-credit.json","stableKey":"ecommerce.spec.store-credit.production-753.753a"},{"category":"production","checkGroupId":"production-754","criterionId":"754a","description":"Concurrent checkout consumes one cart and one credit allocation","executionId":"selected-source-105","featureId":754,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.store-credit","points":2,"requiresFeatures":["ecommerce.feature.store-credit"],"role":"guarantee","source":"scenarios/progression-store-credit.json","stableKey":"ecommerce.spec.store-credit.production-754.754a"},{"category":"production","checkGroupId":"production-755","criterionId":"755a","description":"Issued credit survives a backend restart","executionId":"selected-source-105","featureId":755,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.store-credit","points":2,"requiresFeatures":["ecommerce.feature.store-credit"],"role":"guarantee","source":"scenarios/progression-store-credit.json","stableKey":"ecommerce.spec.store-credit.production-755.755a"},{"category":"feature","checkGroupId":"subscriptions-760","criterionId":"760a","description":"A subscription creates exactly its requested deliveries and payments","executionId":"selected-source-106","featureId":760,"packId":"ecommerce.feature.subscriptions","points":2,"role":"feature","source":"scenarios/progression-subscriptions.json","stableKey":"ecommerce.feature.subscriptions.subscriptions-760.760a"},{"category":"production","checkGroupId":"production-761","criterionId":"761a","description":"A pending subscription continues after backend restart without duplicate deliveries","executionId":"selected-source-106","featureId":761,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.subscriptions","points":2,"requiresFeatures":["ecommerce.feature.subscriptions"],"role":"guarantee","source":"scenarios/progression-subscriptions.json","stableKey":"ecommerce.spec.subscriptions.production-761.761a"},{"category":"production","checkGroupId":"production-762","criterionId":"762a","description":"Another customer cannot cancel an active subscription","executionId":"selected-source-106","featureId":762,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.subscriptions","points":2,"requiresFeatures":["ecommerce.feature.subscriptions"],"role":"guarantee","source":"scenarios/progression-subscriptions.json","stableKey":"ecommerce.spec.subscriptions.production-762.762a"},{"category":"production","checkGroupId":"production-763","criterionId":"763a","description":"Pause survives a restart and resume completes the remaining deliveries","executionId":"selected-source-106","featureId":763,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.subscriptions","points":2,"requiresFeatures":["ecommerce.feature.subscriptions"],"role":"guarantee","source":"scenarios/progression-subscriptions.json","stableKey":"ecommerce.spec.subscriptions.production-763.763a"},{"category":"feature","checkGroupId":"support-history","criterionId":"612c","description":"the customer can view their support ticket history","executionId":"selected-source-107","featureId":612,"packId":"ecommerce.progression.support-history","points":1,"role":"feature","source":"scenarios/progression-support-history.json","stableKey":"ecommerce.progression.support-history.support-history.612c"},{"category":"production","checkGroupId":"support-history-reload","criterionId":"612a","description":"support history survives reload and backend restart in a fresh browser","executionId":"selected-source-107","featureId":612,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":2,"requiresFeatures":["ecommerce.progression.support-history"],"role":"guarantee","source":"scenarios/progression-support-history.json","stableKey":"ecommerce.spec.state-durability.support-history-reload.612a"},{"category":"production","checkGroupId":"support-history-privacy","criterionId":"612b","description":"another customer neither sees nor receives the private ticket","executionId":"selected-source-107","featureId":612,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.progression.support-history"],"role":"guarantee","source":"scenarios/progression-support-history.json","stableKey":"ecommerce.spec.access-control.support-history-privacy.612b"},{"category":"production","checkGroupId":"support-history-logout","criterionId":"612d","description":"after logout the same browser storage no longer grants access to private support history","executionId":"selected-source-107","featureId":612,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.progression.support-history"],"role":"guarantee","source":"scenarios/progression-support-history.json","stableKey":"ecommerce.spec.access-control.support-history-logout.612d"},{"category":"feature","checkGroupId":"support-intake","criterionId":"610a","description":"a visitor can submit a support ticket and receives a reference","executionId":"selected-source-108","featureId":610,"packId":"ecommerce.progression.support-intake","points":2,"role":"feature","source":"scenarios/progression-support-intake.json","stableKey":"ecommerce.progression.support-intake.support-intake.610a"},{"category":"production","checkGroupId":"support-refund-access","criterionId":"615c","description":"a customer cannot issue a refund or change its records","executionId":"selected-source-109","featureId":615,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"requiresFeatures":["ecommerce.progression.support-refunds"],"role":"guarantee","source":"scenarios/progression-support-refunds-access.json","stableKey":"ecommerce.spec.access-control.support-refund-access.615c"},{"category":"production","checkGroupId":"support-refund-accounting","criterionId":"615b","description":"the refund equals the paid total, cannot be applied twice, and leaves another order unrefunded","executionId":"selected-source-110","featureId":615,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":1,"requiresFeatures":["ecommerce.progression.support-refunds"],"role":"guarantee","source":"scenarios/progression-support-refunds-accounting.json","stableKey":"ecommerce.spec.transactional-integrity.support-refund-accounting.615b"},{"category":"feature","checkGroupId":"support-refunds-resolution","criterionId":"615a","description":"an authorized refund resolves the case and updates the order","executionId":"selected-source-111","featureId":615,"packId":"ecommerce.progression.support-refunds","points":2,"role":"feature","source":"scenarios/progression-support-refunds-resolution.json","stableKey":"ecommerce.progression.support-refunds.support-refunds-resolution.615a"},{"category":"production","checkGroupId":"return-refund-interaction","criterionId":"757a","description":"a support refund followed by physical return restores each warehouse once and refunds only the price paid","executionId":"selected-source-112","featureId":757,"packId":"ecommerce.feature.split-tender-refunds","points":2,"requiresFeatures":["ecommerce.l3.order-returns-features","ecommerce.progression.fulfilment-queue","ecommerce.progression.support-refunds"],"role":"feature","source":"scenarios/progression-support-return-interaction.json","stableKey":"ecommerce.feature.split-tender-refunds.return-refund-interaction.757a"},{"category":"production","checkGroupId":"return-refund-interaction","criterionId":"757b","description":"a physical return followed by support refund restores each warehouse once and refunds only the price paid","executionId":"selected-source-112","featureId":757,"packId":"ecommerce.feature.split-tender-refunds","points":2,"requiresFeatures":["ecommerce.l3.order-returns-features","ecommerce.progression.fulfilment-queue","ecommerce.progression.support-refunds"],"role":"feature","source":"scenarios/progression-support-return-interaction.json","stableKey":"ecommerce.feature.split-tender-refunds.return-refund-interaction.757b"},{"category":"feature","checkGroupId":"support-assignment","criterionId":"611a","description":"staff can assign a new ticket","executionId":"selected-source-113","featureId":611,"packId":"ecommerce.progression.support-triage","points":1,"role":"feature","source":"scenarios/progression-support-triage.json","stableKey":"ecommerce.progression.support-triage.support-assignment.611a"},{"category":"feature","checkGroupId":"support-priority","criterionId":"611b","description":"staff can set a ticket priority","executionId":"selected-source-113","featureId":611,"packId":"ecommerce.progression.support-triage","points":1,"role":"feature","source":"scenarios/progression-support-triage.json","stableKey":"ecommerce.progression.support-triage.support-priority.611b"},{"category":"feature","checkGroupId":"support-status","criterionId":"611c","description":"staff can change a ticket status","executionId":"selected-source-113","featureId":611,"packId":"ecommerce.progression.support-triage","points":1,"role":"feature","source":"scenarios/progression-support-triage.json","stableKey":"ecommerce.progression.support-triage.support-status.611c"}],"components":{"fixture":{"id":"ecommerce.operations","path":"composition/fixtures/operations.json","sha256":"d06444b72dc94fe1ef5e08867d875e1f3bbaa5cd82c35a3558f399c1fcb5ceae"},"packs":[{"id":"ecommerce.feature.accounts","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/feature-accounts.json","requiresPacks":[],"sha256":"9eec7949c45ab6af816008b41245dd1e13e9164a6e10d42895eb93ad40664fbf"},{"id":"ecommerce.feature.bundle-checkout","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/feature-bundle-checkout.json","requiresPacks":["ecommerce.feature.product-bundles","ecommerce.l3.reservations-features"],"sha256":"2905f7ee7d8ae6d1086d2194649242b68ef871d10ec60e1bc0382baf5cb6e7df"},{"id":"ecommerce.feature.bundle-returns","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/feature-bundle-returns.json","requiresPacks":["ecommerce.feature.bundle-checkout","ecommerce.l3.order-returns-features"],"sha256":"c0f9424066d11efd12bf2c88ec3607ac1152ba5b1e34d7fe598529c03035e352"},{"id":"ecommerce.feature.cart","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/feature-cart.json","requiresPacks":["ecommerce.feature.accounts","ecommerce.feature.catalog-items"],"sha256":"217d2a7af92bc0551e17c12376b15668431619cc42f919f9d5a0a709f11eba78","stableId":"ecommerce.feature.cart-checkout"},{"id":"ecommerce.feature.catalog-discovery","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/feature-catalog-discovery.json","requiresPacks":["ecommerce.feature.catalog-items"],"sha256":"9aeea0db86cdacc386e5a26fed12b5604d674047c57c792e9aac3a5eabf14b42","stableId":"ecommerce.feature.catalog"},{"id":"ecommerce.feature.catalog-items","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/feature-catalog-items.json","requiresPacks":[],"sha256":"fce1929f7932d508f463793be463896fbb13ad6da2b7c57e0838188579f7a95f","stableId":"ecommerce.feature.catalog"},{"id":"ecommerce.feature.checkout","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/feature-checkout.json","requiresPacks":["ecommerce.feature.cart"],"sha256":"660aaf39db20b4fd6e405948b45f816c6733a2be37b8190b989a4d18c59eb214","stableId":"ecommerce.feature.cart-checkout"},{"id":"ecommerce.feature.product-bundles","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/feature-product-bundles.json","requiresPacks":["ecommerce.progression.catalog-management"],"sha256":"e768568827d4febae15fe2ec1d25410d86f78ada77debf9cb047d70fe3fdb12c"},{"id":"ecommerce.feature.purchasing","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/feature-purchasing.json","requiresPacks":["ecommerce.feature.accounts","ecommerce.feature.catalog-items"],"sha256":"d21746b5e7246b1771ee9f72d0cc66ef2c2cc9573485ffc839f6ca4b1b01ddff"},{"id":"ecommerce.feature.reviews","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/feature-reviews.json","requiresPacks":["ecommerce.feature.accounts","ecommerce.feature.catalog-items","ecommerce.feature.purchasing"],"sha256":"385771e4c68dbc7afbc0d68af897d698a66f744f50e4f580a813db580cf4615c"},{"id":"ecommerce.feature.split-tender-refunds","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/feature-split-tender-refunds.json","requiresPacks":["ecommerce.feature.store-credit","ecommerce.l3.order-returns-features","ecommerce.progression.support-refunds"],"sha256":"8d12162887644dc6bca1fbcccfd06ebd679bd8017b9234cc5632e6a1f508eba8"},{"id":"ecommerce.feature.store-credit","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/feature-store-credit.json","requiresPacks":["ecommerce.progression.payment-records","ecommerce.progression.staff-roles"],"sha256":"f252e74cb95d87a6a7cec629d59bc7a37cac133e6abee255738ff9a1fca50fcd"},{"id":"ecommerce.feature.subscriptions","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/feature-subscriptions.json","requiresPacks":["ecommerce.progression.payment-records"],"sha256":"31bbe7f11ac99af6eeeeadcd62c49bc635bb20384a6b5e3724b8ba9124530799"},{"id":"ecommerce.feature.warehouse-admin","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/feature-warehouse-admin.json","requiresPacks":["ecommerce.feature.catalog-items","ecommerce.progression.staff-access"],"sha256":"9230ca4ddfe048fd3903f3b90aee16a4711c6d53fbe37478917b16a6e550d66d"},{"id":"ecommerce.l2.inventory-dashboard","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/l2-inventory-dashboard.json","requiresPacks":["ecommerce.feature.warehouse-admin"],"sha256":"98d6ac14b32e61e976377df4687a13d6a780a0e96d2e49a05ce2a386e4588778","stableId":"ecommerce.inventory-operations"},{"id":"ecommerce.l2.order-cancellation-features","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/l2-order-cancellation-features.json","requiresPacks":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"sha256":"21f8663b36ea0871493903776ed32fae619183dc5fc0f74a8c1a5e8bc932f561","stableId":"ecommerce.returns-pricing"},{"id":"ecommerce.l2.price-history-features","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/l2-price-history-features.json","requiresPacks":["ecommerce.progression.catalog-management"],"sha256":"786b9f48b9670aa9cb02a439b7eda2feff11c5163a4d49939c892d604206e924","stableId":"ecommerce.returns-pricing"},{"id":"ecommerce.l2.recommendations","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/l2-recommendations.json","requiresPacks":["ecommerce.feature.cart","ecommerce.feature.purchasing"],"sha256":"a21e16ece7b0c4c6e2f0871428af4e4311ae5a91c46174a7ad644b6b93f2c18b","stableId":"ecommerce.inventory-operations"},{"id":"ecommerce.l2.sales-dashboard","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/l2-sales-dashboard.json","requiresPacks":["ecommerce.feature.purchasing"],"sha256":"5e9b1693dcea799edd672e276257a87ad6d7d88099d60be072e952e22e75ca77","stableId":"ecommerce.inventory-operations"},{"id":"ecommerce.l2.stock-transfers-features","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/l2-stock-transfers-features.json","requiresPacks":["ecommerce.feature.warehouse-admin"],"sha256":"fa9514e71b90d38623ede3aa4108cb8a53ca72ed95fc69858733ad38da7235ad","stableId":"ecommerce.inventory-operations"},{"id":"ecommerce.l3.cart-expiration-features","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/l3-cart-expiration-features.json","requiresPacks":["ecommerce.l3.reservations-features"],"sha256":"38029c5ea6d34974b155ef66c311dc500fcece1f82746cffa57ec5f0695f6a5d","stableId":"ecommerce.l3.cart-expiration"},{"id":"ecommerce.l3.deferred-access-specifications","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/l3-deferred-access-specifications.json","requiresPacks":[],"sha256":"cc47732a893439472253b8e209d653959966ca7be20ef0616ab24f8695096814","stableId":"ecommerce.l3.deferred-access"},{"id":"ecommerce.l3.deferred-durability-specifications","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/l3-deferred-durability-specifications.json","requiresPacks":[],"sha256":"9f5109ceb2a8f5e68df317910cea1911dfe79e8c42de51c83b60b760f4075d76","stableId":"ecommerce.l3.deferred-durability"},{"id":"ecommerce.l3.deferred-integrity-specifications","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/l3-deferred-integrity-specifications.json","requiresPacks":[],"sha256":"1a3d27f25be0b41f87afff3412b6f2c93617a68306309f471e0120d1ea359056","stableId":"ecommerce.l3.deferred-integrity"},{"id":"ecommerce.l3.order-delivery-features","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/l3-order-delivery-features.json","requiresPacks":["ecommerce.l2.order-cancellation-features","ecommerce.progression.fulfilment-queue"],"sha256":"7f86ba1b2a2791ba38ba6abeeee6078b061bdf03ea951647d79a74f6fe435b95","stableId":"ecommerce.l3.order-delivery"},{"id":"ecommerce.l3.order-returns-features","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/l3-order-returns-features.json","requiresPacks":["ecommerce.feature.warehouse-admin","ecommerce.l3.order-delivery-features"],"sha256":"cd5dc23d3c7514df91d0a54d92d97e7604e6828b1a287d0c79c714724cdc2206","stableId":"ecommerce.returns-pricing"},{"id":"ecommerce.l3.reservations-features","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/l3-reservations-features.json","requiresPacks":["ecommerce.feature.checkout","ecommerce.feature.warehouse-admin"],"sha256":"972b14948d5dea26cadab3567fc5bbd2c42d4feed5c428c004117b723acecd12","stableId":"ecommerce.l3.reservations"},{"id":"ecommerce.l3.scheduled-restocks-features","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/l3-scheduled-restocks-features.json","requiresPacks":["ecommerce.feature.warehouse-admin"],"sha256":"305c2cf78bc84c63b20cb3e5531760174355128332a365d741dbbd0d64c938cb","stableId":"ecommerce.l3.scheduled-restocks"},{"id":"ecommerce.l3.server-time-specifications","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/l3-server-time-specifications.json","requiresPacks":[],"sha256":"75ad830a0023bf1fe8022695f0776854d75cdbd87f5120a25312c7de33bf57c9","stableId":"ecommerce.l3.server-time"},{"id":"ecommerce.progression.automatic-reorder","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-automatic-reorder.json","requiresPacks":["ecommerce.feature.purchasing","ecommerce.l3.scheduled-restocks-features","ecommerce.progression.staff-roles"],"sha256":"a1d14fcb9cd6c9e96b71d2bacdd4865badaeda913ba9cb3cb5fab8f1eb8497e7"},{"id":"ecommerce.progression.cancellation-accounting-specifications","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/progression-cancellation-accounting-specifications.json","requiresPacks":[],"sha256":"29b290a1356e63a9d2e937c7f92ad540403b27ebcef94df15451d12e59ac86cc","stableId":"ecommerce.returns-pricing"},{"id":"ecommerce.progression.cancellation-queue-specifications","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/progression-cancellation-queue-specifications.json","requiresPacks":[],"sha256":"adf9f45a20d13a400a7812fe825a73de6035f980813c19e3698128e49085b58d","stableId":"ecommerce.returns-pricing"},{"id":"ecommerce.progression.cart-recovery","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-cart-recovery.json","requiresPacks":["ecommerce.l3.cart-expiration-features"],"sha256":"ca6d95040f5b81478546b9afedcc428114e795ced11b8364eb2542c258aec794"},{"id":"ecommerce.progression.catalog-management","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-catalog-management.json","requiresPacks":["ecommerce.feature.catalog-discovery","ecommerce.progression.staff-roles"],"sha256":"f12e7e37d031f470683ff74671b22c1de08eb8f7fe0e9cf03cbcfb4604dd2f06"},{"id":"ecommerce.progression.customer-profile","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-customer-profile.json","requiresPacks":["ecommerce.feature.accounts"],"sha256":"731eeced13c1053432560d4149d2de29fa2cd2b0051759d705cb36e92ad16bb3"},{"id":"ecommerce.progression.delivery-notifications","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-delivery-notifications.json","requiresPacks":["ecommerce.l3.order-delivery-features","ecommerce.progression.notification-preferences"],"sha256":"60b58c4327d74f710f94ffc0deae1b51ef40ca1d2146e45b8e02d1b28e1146bc"},{"id":"ecommerce.progression.faceted-search","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-faceted-search.json","requiresPacks":["ecommerce.feature.catalog-discovery"],"sha256":"5a967859348f967bdc43e0e2490289a859f568e4e8d809c21d5f4404d85e4dd7"},{"id":"ecommerce.progression.fulfilment-queue","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-fulfilment-queue.json","requiresPacks":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"sha256":"811eeb166f261e32b5338e118b0851aa1ffeb06b3107dafb6861057e11004db4","stableId":"ecommerce.operations-access"},{"id":"ecommerce.progression.inventory-conservation-specifications","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/progression-inventory-conservation-specifications.json","requiresPacks":[],"sha256":"b3bff8b33e6069bf3cfd2e68389bbd50f0ec9c981d766fd2d7717dcf892463dc","stableId":"ecommerce.inventory-operations"},{"id":"ecommerce.progression.managed-support","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-managed-support.json","requiresPacks":["ecommerce.progression.support-history","ecommerce.progression.support-triage"],"sha256":"6546f37be296605cad2e06bbe71d0cf014d1025a5a4b3b8997f4c38cf6329841"},{"id":"ecommerce.progression.notification-preferences","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-notification-preferences.json","requiresPacks":["ecommerce.feature.accounts"],"sha256":"e8423fe1a2a615a1f2b5bd7073e9cc7040c8d2e2eadc0ef466d50e5d21aaf134"},{"id":"ecommerce.progression.operations-access-specifications","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/progression-operations-access-specifications.json","requiresPacks":[],"sha256":"796806b8a647fd367f53f6246d51aa998b9aa9e2a8959356d237b6411be96937","stableId":"ecommerce.operations-access"},{"id":"ecommerce.progression.order-support","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-order-support.json","requiresPacks":["ecommerce.feature.purchasing","ecommerce.progression.managed-support"],"sha256":"1e1a1722adc4c72fdbf31aa34261b73975df53ffab27b28134d71b09ada1b27f"},{"id":"ecommerce.progression.payment-records","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-payment-records.json","requiresPacks":["ecommerce.feature.checkout","ecommerce.feature.purchasing"],"sha256":"f9203b2def92ee787077b88ae9e31425c03cff7b43a039823c160bc69303cc3f"},{"id":"ecommerce.progression.personalized-recommendations","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-personalized-recommendations.json","requiresPacks":["ecommerce.l2.recommendations"],"sha256":"fafda848a453566e61af3513659a8884979ae49218e490830774bb4c25f7b383"},{"id":"ecommerce.progression.price-accounting-specifications","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/progression-price-accounting-specifications.json","requiresPacks":[],"sha256":"f49fa571d5e63617537742620e92e95a9671ee87f52fd1abc15c2daaf2a6a195","stableId":"ecommerce.returns-pricing"},{"id":"ecommerce.progression.price-history-specifications","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/progression-price-history-specifications.json","requiresPacks":[],"sha256":"5882d2b0daee49ff6f3b8b6aace9fd63f6dca52b18f4dc72f8329e59ffcb9664","stableId":"ecommerce.returns-pricing"},{"id":"ecommerce.progression.promotion-checkout","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-promotion-checkout.json","requiresPacks":["ecommerce.feature.checkout","ecommerce.progression.promotion-rules"],"sha256":"ead1c8b0ab75a29bc55eaf71ceb216f881655ce20321dc0e36ee8e72271383a9"},{"id":"ecommerce.progression.promotion-reporting","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-promotion-reporting.json","requiresPacks":["ecommerce.progression.promotion-checkout"],"sha256":"a0c28a936e0929584ccf418dbbe5f53ba6b6beccd33a5a8255bbfb334a3ddca8"},{"id":"ecommerce.progression.promotion-rules","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-promotion-rules.json","requiresPacks":["ecommerce.feature.catalog-items","ecommerce.progression.staff-access"],"sha256":"4275d81b85379c0030abd92208b732f616a53a2bad8054e0ee6c7f745488105c"},{"id":"ecommerce.progression.recommendation-feedback","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-recommendation-feedback.json","requiresPacks":["ecommerce.progression.personalized-recommendations"],"sha256":"e7e703471cb30e8caf6acf89e3a0ad089dcde8d9320ceaf4fba7132add4007e5"},{"id":"ecommerce.progression.review-access-specifications","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/progression-review-access-specifications.json","requiresPacks":[],"sha256":"d89b6e559244faed30648f2ebdb462913c0b882c614ffe51821c1eb446e66d55"},{"id":"ecommerce.progression.staff-access","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-staff-access.json","requiresPacks":[],"sha256":"440a036f482be11c4954abbb478693a995aea0fe0843f1959a641f3ab6733539"},{"id":"ecommerce.progression.staff-activity","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-staff-activity.json","requiresPacks":["ecommerce.progression.catalog-management","ecommerce.progression.staff-roles"],"sha256":"129e48d11c4b24ae94d45350ffc93e190265dd69eb7948fc34c24c49b438db2e"},{"id":"ecommerce.progression.staff-roles","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-staff-roles.json","requiresPacks":["ecommerce.progression.staff-access"],"sha256":"341efe45e33b4e2fd94a19036034963393b6f9d3749c63faf0bbff0415d0ead8"},{"id":"ecommerce.progression.stock-alerts","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-stock-alerts.json","requiresPacks":["ecommerce.feature.warehouse-admin","ecommerce.progression.notification-preferences"],"sha256":"837f8fcb2c0c023704372810efdc8532246df2b795b94b8503f9d17dae415db9"},{"id":"ecommerce.progression.support-history","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-support-history.json","requiresPacks":["ecommerce.feature.accounts","ecommerce.progression.support-intake"],"sha256":"23d9dd3e5132a88f3ed0d1c65afac0f6ca769f886b6fb531718831358699b90a"},{"id":"ecommerce.progression.support-intake","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-support-intake.json","requiresPacks":[],"sha256":"6458979c3421deb9916903854a1889580b122813500e3437c475fd3bf73d7721"},{"id":"ecommerce.progression.support-refunds","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-support-refunds.json","requiresPacks":["ecommerce.l2.order-cancellation-features","ecommerce.progression.order-support"],"sha256":"fd05f39442798d4bae9e96c65230268bd80be46f40a57007ede06d02d5a28e7c"},{"id":"ecommerce.progression.support-triage","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-support-triage.json","requiresPacks":["ecommerce.progression.staff-access","ecommerce.progression.support-intake"],"sha256":"2c90c09870de8922db55a9fd8a1b07553c60fd3aebc679f3e4ff1649c752df85"},{"id":"ecommerce.spec.access-control","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/spec-access-control.json","requiresPacks":[],"sha256":"e8999511e92036535ee412cee4be2739aceedd565bcee9ef54c291bb0dbeaf48"},{"id":"ecommerce.spec.bundle-integrity","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/spec-bundle-integrity.json","requiresPacks":[],"sha256":"e0ca540f434e643c5ddbe7fb204fe1edad39376c719d48d4d544caef559595f9"},{"id":"ecommerce.spec.concurrency-safety","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/spec-concurrency-safety.json","requiresPacks":[],"sha256":"350390ddaf5b65e1af7e734cb69404940ec8ab7a632d7201d334a25eb06e2e4c"},{"id":"ecommerce.spec.external-data-sync","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/spec-external-data-sync.json","requiresPacks":[],"sha256":"f4c03de56a2e98f3d4ceb35063b9b5624f7842a4ba89915fcadd9c8a267faef2"},{"id":"ecommerce.spec.live-state","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/spec-live-state.json","requiresPacks":[],"sha256":"49d210d3c5a7aed24d5b296d3a63fe18e87cb9d33821c1f2cdd872a8e06e92f3"},{"id":"ecommerce.spec.search-ordering","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/spec-search-ordering.json","requiresPacks":[],"sha256":"f33275ca6a9593135c94170bd262203eaf65d980e6f810c393f6d07a28bffa50"},{"id":"ecommerce.spec.split-tender-refunds","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/spec-split-tender-refunds.json","requiresPacks":[],"sha256":"91ca0be0e80ea7d001cdd9a2959c7b2c00be3b87aeb5dc06c7db8b9d179ead8c"},{"id":"ecommerce.spec.state-durability","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/spec-state-durability.json","requiresPacks":[],"sha256":"df7c5808d25201493e6eafd46e37091a5f3b9d8f70f2c6ac708d59395137019d"},{"id":"ecommerce.spec.store-credit","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/spec-store-credit.json","requiresPacks":[],"sha256":"4d858d568aabb314ad118b3e2b5003e43710585fdbf4a85b71b9971ca636843c"},{"id":"ecommerce.spec.subscriptions","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/spec-subscriptions.json","requiresPacks":[],"sha256":"7dac820cfeec043b619d468be0c6b0d7e022093851e7884ad573a80aeeb468f2"},{"id":"ecommerce.spec.transactional-integrity","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/spec-transactional-integrity.json","requiresPacks":[],"sha256":"3894b3d9967c3cbb65ea3a3d5b0f8eb88f2673c06b9122f75acc945c8ca06f5f"}]},"contentSha256":"53fbb8093b2335837d88c246f04da99858c40b57ce24d17fbd2b6c815abdb62d","executionSha256":"558ea7e4033dd1e08c76c2ecc475751f2cabd210f4e18cc0f2c6e3e3531a2498","id":"ecommerce.progression-catalog","meaningSha256":"79dc9a2f8ee5854d3f4edf68862942778d0fa0bd667b35554f065d025726aa76","recipeReleaseSchemaVersion":3,"scoring":{"checks":190,"mode":"source-points","points":359},"sequence":null,"sourceManifest":[{"kinds":["fixture"],"path":"composition/fixtures/operations.json","sha256":"d06444b72dc94fe1ef5e08867d875e1f3bbaa5cd82c35a3558f399c1fcb5ceae"},{"kinds":["pack"],"path":"composition/packs/feature-accounts.json","sha256":"9eec7949c45ab6af816008b41245dd1e13e9164a6e10d42895eb93ad40664fbf"},{"kinds":["pack"],"path":"composition/packs/feature-bundle-checkout.json","sha256":"2905f7ee7d8ae6d1086d2194649242b68ef871d10ec60e1bc0382baf5cb6e7df"},{"kinds":["pack"],"path":"composition/packs/feature-bundle-returns.json","sha256":"c0f9424066d11efd12bf2c88ec3607ac1152ba5b1e34d7fe598529c03035e352"},{"kinds":["pack"],"path":"composition/packs/feature-cart.json","sha256":"217d2a7af92bc0551e17c12376b15668431619cc42f919f9d5a0a709f11eba78"},{"kinds":["pack"],"path":"composition/packs/feature-catalog-discovery.json","sha256":"9aeea0db86cdacc386e5a26fed12b5604d674047c57c792e9aac3a5eabf14b42"},{"kinds":["pack"],"path":"composition/packs/feature-catalog-items.json","sha256":"fce1929f7932d508f463793be463896fbb13ad6da2b7c57e0838188579f7a95f"},{"kinds":["pack"],"path":"composition/packs/feature-checkout.json","sha256":"660aaf39db20b4fd6e405948b45f816c6733a2be37b8190b989a4d18c59eb214"},{"kinds":["pack"],"path":"composition/packs/feature-product-bundles.json","sha256":"e768568827d4febae15fe2ec1d25410d86f78ada77debf9cb047d70fe3fdb12c"},{"kinds":["pack"],"path":"composition/packs/feature-purchasing.json","sha256":"d21746b5e7246b1771ee9f72d0cc66ef2c2cc9573485ffc839f6ca4b1b01ddff"},{"kinds":["pack"],"path":"composition/packs/feature-reviews.json","sha256":"385771e4c68dbc7afbc0d68af897d698a66f744f50e4f580a813db580cf4615c"},{"kinds":["pack"],"path":"composition/packs/feature-split-tender-refunds.json","sha256":"8d12162887644dc6bca1fbcccfd06ebd679bd8017b9234cc5632e6a1f508eba8"},{"kinds":["pack"],"path":"composition/packs/feature-store-credit.json","sha256":"f252e74cb95d87a6a7cec629d59bc7a37cac133e6abee255738ff9a1fca50fcd"},{"kinds":["pack"],"path":"composition/packs/feature-subscriptions.json","sha256":"31bbe7f11ac99af6eeeeadcd62c49bc635bb20384a6b5e3724b8ba9124530799"},{"kinds":["pack"],"path":"composition/packs/feature-warehouse-admin.json","sha256":"9230ca4ddfe048fd3903f3b90aee16a4711c6d53fbe37478917b16a6e550d66d"},{"kinds":["pack"],"path":"composition/packs/l2-inventory-dashboard.json","sha256":"98d6ac14b32e61e976377df4687a13d6a780a0e96d2e49a05ce2a386e4588778"},{"kinds":["pack"],"path":"composition/packs/l2-order-cancellation-features.json","sha256":"21f8663b36ea0871493903776ed32fae619183dc5fc0f74a8c1a5e8bc932f561"},{"kinds":["pack"],"path":"composition/packs/l2-price-history-features.json","sha256":"786b9f48b9670aa9cb02a439b7eda2feff11c5163a4d49939c892d604206e924"},{"kinds":["pack"],"path":"composition/packs/l2-recommendations.json","sha256":"a21e16ece7b0c4c6e2f0871428af4e4311ae5a91c46174a7ad644b6b93f2c18b"},{"kinds":["pack"],"path":"composition/packs/l2-sales-dashboard.json","sha256":"5e9b1693dcea799edd672e276257a87ad6d7d88099d60be072e952e22e75ca77"},{"kinds":["pack"],"path":"composition/packs/l2-stock-transfers-features.json","sha256":"fa9514e71b90d38623ede3aa4108cb8a53ca72ed95fc69858733ad38da7235ad"},{"kinds":["pack"],"path":"composition/packs/l3-cart-expiration-features.json","sha256":"38029c5ea6d34974b155ef66c311dc500fcece1f82746cffa57ec5f0695f6a5d"},{"kinds":["pack"],"path":"composition/packs/l3-deferred-access-specifications.json","sha256":"cc47732a893439472253b8e209d653959966ca7be20ef0616ab24f8695096814"},{"kinds":["pack"],"path":"composition/packs/l3-deferred-durability-specifications.json","sha256":"9f5109ceb2a8f5e68df317910cea1911dfe79e8c42de51c83b60b760f4075d76"},{"kinds":["pack"],"path":"composition/packs/l3-deferred-integrity-specifications.json","sha256":"1a3d27f25be0b41f87afff3412b6f2c93617a68306309f471e0120d1ea359056"},{"kinds":["pack"],"path":"composition/packs/l3-order-delivery-features.json","sha256":"7f86ba1b2a2791ba38ba6abeeee6078b061bdf03ea951647d79a74f6fe435b95"},{"kinds":["pack"],"path":"composition/packs/l3-order-returns-features.json","sha256":"cd5dc23d3c7514df91d0a54d92d97e7604e6828b1a287d0c79c714724cdc2206"},{"kinds":["pack"],"path":"composition/packs/l3-reservations-features.json","sha256":"972b14948d5dea26cadab3567fc5bbd2c42d4feed5c428c004117b723acecd12"},{"kinds":["pack"],"path":"composition/packs/l3-scheduled-restocks-features.json","sha256":"305c2cf78bc84c63b20cb3e5531760174355128332a365d741dbbd0d64c938cb"},{"kinds":["pack"],"path":"composition/packs/l3-server-time-specifications.json","sha256":"75ad830a0023bf1fe8022695f0776854d75cdbd87f5120a25312c7de33bf57c9"},{"kinds":["pack"],"path":"composition/packs/progression-automatic-reorder.json","sha256":"a1d14fcb9cd6c9e96b71d2bacdd4865badaeda913ba9cb3cb5fab8f1eb8497e7"},{"kinds":["pack"],"path":"composition/packs/progression-cancellation-accounting-specifications.json","sha256":"29b290a1356e63a9d2e937c7f92ad540403b27ebcef94df15451d12e59ac86cc"},{"kinds":["pack"],"path":"composition/packs/progression-cancellation-queue-specifications.json","sha256":"adf9f45a20d13a400a7812fe825a73de6035f980813c19e3698128e49085b58d"},{"kinds":["pack"],"path":"composition/packs/progression-cart-recovery.json","sha256":"ca6d95040f5b81478546b9afedcc428114e795ced11b8364eb2542c258aec794"},{"kinds":["pack"],"path":"composition/packs/progression-catalog-management.json","sha256":"f12e7e37d031f470683ff74671b22c1de08eb8f7fe0e9cf03cbcfb4604dd2f06"},{"kinds":["pack"],"path":"composition/packs/progression-customer-profile.json","sha256":"731eeced13c1053432560d4149d2de29fa2cd2b0051759d705cb36e92ad16bb3"},{"kinds":["pack"],"path":"composition/packs/progression-delivery-notifications.json","sha256":"60b58c4327d74f710f94ffc0deae1b51ef40ca1d2146e45b8e02d1b28e1146bc"},{"kinds":["pack"],"path":"composition/packs/progression-faceted-search.json","sha256":"5a967859348f967bdc43e0e2490289a859f568e4e8d809c21d5f4404d85e4dd7"},{"kinds":["pack"],"path":"composition/packs/progression-fulfilment-queue.json","sha256":"811eeb166f261e32b5338e118b0851aa1ffeb06b3107dafb6861057e11004db4"},{"kinds":["pack"],"path":"composition/packs/progression-inventory-conservation-specifications.json","sha256":"b3bff8b33e6069bf3cfd2e68389bbd50f0ec9c981d766fd2d7717dcf892463dc"},{"kinds":["pack"],"path":"composition/packs/progression-managed-support.json","sha256":"6546f37be296605cad2e06bbe71d0cf014d1025a5a4b3b8997f4c38cf6329841"},{"kinds":["pack"],"path":"composition/packs/progression-notification-preferences.json","sha256":"e8423fe1a2a615a1f2b5bd7073e9cc7040c8d2e2eadc0ef466d50e5d21aaf134"},{"kinds":["pack"],"path":"composition/packs/progression-operations-access-specifications.json","sha256":"796806b8a647fd367f53f6246d51aa998b9aa9e2a8959356d237b6411be96937"},{"kinds":["pack"],"path":"composition/packs/progression-order-support.json","sha256":"1e1a1722adc4c72fdbf31aa34261b73975df53ffab27b28134d71b09ada1b27f"},{"kinds":["pack"],"path":"composition/packs/progression-payment-records.json","sha256":"f9203b2def92ee787077b88ae9e31425c03cff7b43a039823c160bc69303cc3f"},{"kinds":["pack"],"path":"composition/packs/progression-personalized-recommendations.json","sha256":"fafda848a453566e61af3513659a8884979ae49218e490830774bb4c25f7b383"},{"kinds":["pack"],"path":"composition/packs/progression-price-accounting-specifications.json","sha256":"f49fa571d5e63617537742620e92e95a9671ee87f52fd1abc15c2daaf2a6a195"},{"kinds":["pack"],"path":"composition/packs/progression-price-history-specifications.json","sha256":"5882d2b0daee49ff6f3b8b6aace9fd63f6dca52b18f4dc72f8329e59ffcb9664"},{"kinds":["pack"],"path":"composition/packs/progression-promotion-checkout.json","sha256":"ead1c8b0ab75a29bc55eaf71ceb216f881655ce20321dc0e36ee8e72271383a9"},{"kinds":["pack"],"path":"composition/packs/progression-promotion-reporting.json","sha256":"a0c28a936e0929584ccf418dbbe5f53ba6b6beccd33a5a8255bbfb334a3ddca8"},{"kinds":["pack"],"path":"composition/packs/progression-promotion-rules.json","sha256":"4275d81b85379c0030abd92208b732f616a53a2bad8054e0ee6c7f745488105c"},{"kinds":["pack"],"path":"composition/packs/progression-recommendation-feedback.json","sha256":"e7e703471cb30e8caf6acf89e3a0ad089dcde8d9320ceaf4fba7132add4007e5"},{"kinds":["pack"],"path":"composition/packs/progression-review-access-specifications.json","sha256":"d89b6e559244faed30648f2ebdb462913c0b882c614ffe51821c1eb446e66d55"},{"kinds":["pack"],"path":"composition/packs/progression-staff-access.json","sha256":"440a036f482be11c4954abbb478693a995aea0fe0843f1959a641f3ab6733539"},{"kinds":["pack"],"path":"composition/packs/progression-staff-activity.json","sha256":"129e48d11c4b24ae94d45350ffc93e190265dd69eb7948fc34c24c49b438db2e"},{"kinds":["pack"],"path":"composition/packs/progression-staff-roles.json","sha256":"341efe45e33b4e2fd94a19036034963393b6f9d3749c63faf0bbff0415d0ead8"},{"kinds":["pack"],"path":"composition/packs/progression-stock-alerts.json","sha256":"837f8fcb2c0c023704372810efdc8532246df2b795b94b8503f9d17dae415db9"},{"kinds":["pack"],"path":"composition/packs/progression-support-history.json","sha256":"23d9dd3e5132a88f3ed0d1c65afac0f6ca769f886b6fb531718831358699b90a"},{"kinds":["pack"],"path":"composition/packs/progression-support-intake.json","sha256":"6458979c3421deb9916903854a1889580b122813500e3437c475fd3bf73d7721"},{"kinds":["pack"],"path":"composition/packs/progression-support-refunds.json","sha256":"fd05f39442798d4bae9e96c65230268bd80be46f40a57007ede06d02d5a28e7c"},{"kinds":["pack"],"path":"composition/packs/progression-support-triage.json","sha256":"2c90c09870de8922db55a9fd8a1b07553c60fd3aebc679f3e4ff1649c752df85"},{"kinds":["pack"],"path":"composition/packs/spec-access-control.json","sha256":"e8999511e92036535ee412cee4be2739aceedd565bcee9ef54c291bb0dbeaf48"},{"kinds":["pack"],"path":"composition/packs/spec-bundle-integrity.json","sha256":"e0ca540f434e643c5ddbe7fb204fe1edad39376c719d48d4d544caef559595f9"},{"kinds":["pack"],"path":"composition/packs/spec-concurrency-safety.json","sha256":"350390ddaf5b65e1af7e734cb69404940ec8ab7a632d7201d334a25eb06e2e4c"},{"kinds":["pack"],"path":"composition/packs/spec-external-data-sync.json","sha256":"f4c03de56a2e98f3d4ceb35063b9b5624f7842a4ba89915fcadd9c8a267faef2"},{"kinds":["pack"],"path":"composition/packs/spec-live-state.json","sha256":"49d210d3c5a7aed24d5b296d3a63fe18e87cb9d33821c1f2cdd872a8e06e92f3"},{"kinds":["pack"],"path":"composition/packs/spec-search-ordering.json","sha256":"f33275ca6a9593135c94170bd262203eaf65d980e6f810c393f6d07a28bffa50"},{"kinds":["pack"],"path":"composition/packs/spec-split-tender-refunds.json","sha256":"91ca0be0e80ea7d001cdd9a2959c7b2c00be3b87aeb5dc06c7db8b9d179ead8c"},{"kinds":["pack"],"path":"composition/packs/spec-state-durability.json","sha256":"df7c5808d25201493e6eafd46e37091a5f3b9d8f70f2c6ac708d59395137019d"},{"kinds":["pack"],"path":"composition/packs/spec-store-credit.json","sha256":"4d858d568aabb314ad118b3e2b5003e43710585fdbf4a85b71b9971ca636843c"},{"kinds":["pack"],"path":"composition/packs/spec-subscriptions.json","sha256":"7dac820cfeec043b619d468be0c6b0d7e022093851e7884ad573a80aeeb468f2"},{"kinds":["pack"],"path":"composition/packs/spec-transactional-integrity.json","sha256":"3894b3d9967c3cbb65ea3a3d5b0f8eb88f2673c06b9122f75acc945c8ca06f5f"},{"kinds":["recipe"],"path":"composition/recipes/progression-catalog.json","sha256":"a869c95b1d5a222dc679d1529a789d623be3d0a82beff818e6b94702a1791ee9"},{"kinds":["contract-source"],"path":"contracts/accounts.md","sha256":"19b453ded62710d998bce1b169497342209feefccd9c4b1cef3bbb9442f01488"},{"kinds":["contract-source"],"path":"contracts/application-interface.md","sha256":"b770db6c1dbf92c106c7501a17028876a30a63655c8f88b5667b44557f401692"},{"kinds":["contract-source"],"path":"contracts/bundle-checkout.md","sha256":"2c4e70b620934dc927ad4c84c17e54b6e9cef665ba3b098f9f2fc9e79a3c4c3e"},{"kinds":["contract-source"],"path":"contracts/bundle-returns.md","sha256":"69f8bf6afc325a98bab38235e49eaa2875bc674e6a61c7559692593018c9f527"},{"kinds":["contract-source"],"path":"contracts/cart-expiration.md","sha256":"56c0966494cd19b629140f8b8d53e8b9838e34668a5cab6b7cccb41b78734c50"},{"kinds":["contract-source"],"path":"contracts/cart.md","sha256":"bb1b8741b9eb47b5766a330bb1fc23005f7540fdc79d20efdaefb183cf50918d"},{"kinds":["contract-source"],"path":"contracts/catalog-discovery.md","sha256":"d13ea495f78e3827b7a243b97466aefbf0252980c76edb432b6b0506854e8450"},{"kinds":["contract-source"],"path":"contracts/catalog-items.md","sha256":"596ec08b1f4a6b290595570f3bcf2847a1f60feeb20d90f98d25761666bfe9f2"},{"kinds":["contract-source"],"path":"contracts/catalog-management.md","sha256":"5123971185ded331deaef2e323fad31e6133d55fc839289e2c4a0dfc5292ee4e"},{"kinds":["contract-source"],"path":"contracts/checkout.md","sha256":"49416683e06d996a28980ff650a3281f89cae062269b7de761899e71759c731c"},{"kinds":["contract-source"],"path":"contracts/customer-profile.md","sha256":"7451bf14ada950c9b7ad269d63467154c388e9ea18b97f479037cdd397ca519f"},{"kinds":["contract-source"],"path":"contracts/delivery-notifications.md","sha256":"4bf565ea751bb5278bae5910420beeb531a2eb88fb5755e5a06dda68860af490"},{"kinds":["contract-source"],"path":"contracts/faceted-search.md","sha256":"08ace4d2d1947f16ceb4da95e6993086c9bd77a0977c258fef1f0a96c7e7879d"},{"kinds":["contract-source"],"path":"contracts/inventory-dashboard.md","sha256":"8c3de4ff8a958776ca087febbbbf5f950eaef9efacb96b7c4fa49357b2d0a371"},{"kinds":["contract-source"],"path":"contracts/managed-support.md","sha256":"458b89a7f786086a95957c778943660068cd6c00b30635fd1cd786c37df76398"},{"kinds":["contract-source"],"path":"contracts/notification-preferences.md","sha256":"d6b4231f67d5e70a03ec331545b6419acf766b52841c8520b94963c16aa13fec"},{"kinds":["contract-source"],"path":"contracts/operations-access.md","sha256":"fd9c31482b484dc1f9bf3c521d247971334e1f52f51a7548a987bdebc65dbdb5"},{"kinds":["contract-source"],"path":"contracts/order-cancellation.md","sha256":"fc274b1409c50de8edcf797ab83cf8e8c46a54db30ada859f854e1cd40c7dee1"},{"kinds":["contract-source"],"path":"contracts/order-data.md","sha256":"7b7cfc6b102f4836235c7f23344865f60156b3e4ac96c3e5cbb0e298390b3b06"},{"kinds":["contract-source"],"path":"contracts/order-delivery.md","sha256":"74bd21b39c8d5f020ff62cc408992c15788e48eba8e05a76a0cddc4b4920a98e"},{"kinds":["contract-source"],"path":"contracts/order-returns.md","sha256":"fe0668ab3bf00d4c167794dbecb75b26b23cb62dfc0a1b79636a013edb237329"},{"kinds":["contract-source"],"path":"contracts/order-support.md","sha256":"a2901e66af870055b6c3c26f240132ae08f3cb93d0c52a7b2573c9dc153e1e03"},{"kinds":["contract-source"],"path":"contracts/payment-records.md","sha256":"8c752352c7f47a0a65d1823f9ec74009dc95088fc6c2a6cc30d1a1dc7fa11fe0"},{"kinds":["contract-source"],"path":"contracts/price-history-orders.md","sha256":"c3a86a7de94e345c578d6b9ba3527c82c137797f385ca4aca3cf2b7ee39b9d40"},{"kinds":["contract-source"],"path":"contracts/price-history.md","sha256":"3465714f91e0e978607900b19377a134e9beb704848e2462b1438465cc2e98ee"},{"kinds":["contract-source"],"path":"contracts/product-bundles.md","sha256":"b27905002b693e6c63532cd8193c8e3fd1fdeaad85041d84b8240a2092bf1c57"},{"kinds":["contract-source"],"path":"contracts/progression-automatic-reorder.md","sha256":"3e70361965e6ddeb7b05cc7ad5348e363d58d628734de96b8a9792e68e71cc72"},{"kinds":["contract-source"],"path":"contracts/progression-cart-recovery.md","sha256":"daf04aea3082483caa2ebf3a3c89a438329600cb6aa5d74fb47d97bc244e5747"},{"kinds":["contract-source"],"path":"contracts/progression-personalized-recommendations.md","sha256":"42389a6d284dcfdb53a610ce5ce77d40dbc2b9d5e177bbd3f6bccc40079c7d98"},{"kinds":["contract-source"],"path":"contracts/promotion-checkout.md","sha256":"b74e703dc77c9eeec26f5beb41cab102570c8314d817015c858557e9946293aa"},{"kinds":["contract-source"],"path":"contracts/promotion-reporting.md","sha256":"81cded5fd9a8822c8756ea50bedb879971c94f8727d176f22fa53d2a6a1efbb0"},{"kinds":["contract-source"],"path":"contracts/promotion-rules.md","sha256":"f352f8bb938aa874125b10bbb6182643f04b6d1ce82cde41512bd16835a7b90b"},{"kinds":["contract-source"],"path":"contracts/purchasing.md","sha256":"c2d59da67b400d055ab90bf53f2e46a3af9605a1fc68534c2b64d8db2ed3409c"},{"kinds":["contract-source"],"path":"contracts/recommendation-feedback.md","sha256":"00a96f594c8da1ffb4a933c6f8d8d0219216ab6cc2c781cfd17dd9c97c517d58"},{"kinds":["contract-source"],"path":"contracts/recommendations.md","sha256":"ab5417cbf78f43464e82a543e50e9601580bb2b3a57826a9e73da9899f39de16"},{"kinds":["contract-source"],"path":"contracts/reservations.md","sha256":"e1c4af7a67dd200d5f18d550e5b4c30e7a4bbaf60600b434108ebff7bdf90627"},{"kinds":["contract-source"],"path":"contracts/reviews.md","sha256":"6be3567258c0c4bf3331e609f63be26ad0bb43ceca3e7da4accaaad8c9ac6541"},{"kinds":["contract-source"],"path":"contracts/sales-dashboard.md","sha256":"ff9ed3890aa5c8303a0badecf946e7d98da3fbbb0a2890a693099bab1a2056c0"},{"kinds":["contract-source"],"path":"contracts/scheduled-restocks.md","sha256":"a946c0954a641d4fe64b933761e2b92d546ae08ee434d676c6faed3ed8b03c8e"},{"kinds":["contract-source"],"path":"contracts/split-tender-refunds.md","sha256":"62b8b3c233d33858e9b696d06d94c70f6524f8cbed77028acc0d2affdd780841"},{"kinds":["contract-source"],"path":"contracts/staff-access.md","sha256":"dcd7b740d5f65160ddf0b4221d29f84426ee89f04560e953add0856cf0bca87a"},{"kinds":["contract-source"],"path":"contracts/staff-activity.md","sha256":"7e7c1846445f82aa78ad8e32606e6830f9d326ebfea75d13b077c073695c9594"},{"kinds":["contract-source"],"path":"contracts/staff-roles.md","sha256":"cce6145182485fafcd2adc29cec90af20523a2afd9c952800eefb5d15a49f96e"},{"kinds":["contract-source"],"path":"contracts/stock-alerts.md","sha256":"405f106569c84b3062f8159452730f16adadf12d804c554a570e39b958b63421"},{"kinds":["contract-source"],"path":"contracts/stock-transfers.md","sha256":"01f28ebb3487559d4689919f0991e43f74fcc85609fa4c810e92adde30e84096"},{"kinds":["contract-source"],"path":"contracts/store-credit.md","sha256":"505ca48dfc4f9f2d6ea38e53e1bd382910264d0195be8580b5fb01f7e7074713"},{"kinds":["contract-source"],"path":"contracts/subscriptions.md","sha256":"dc533bc7f3b9be03ae7d9e18c2e02ffa996bf934b9ba230826d492b4cefa045e"},{"kinds":["contract-source"],"path":"contracts/support-history.md","sha256":"610022f964c2aa00b409fb2723f91b885e306edde5bf73390e880cffd2a5583d"},{"kinds":["contract-source"],"path":"contracts/support-intake.md","sha256":"fd9f732acf67d6c772f4c05d626e4e9e472818fc5226b7ef3d7223b85a0e0bc7"},{"kinds":["contract-source"],"path":"contracts/support-refunds.md","sha256":"da5d0e69bbf3a028dc79f207b940bbd4a79e7f82c68d89f252eb8c73ea16b8bc"},{"kinds":["contract-source"],"path":"contracts/support-triage.md","sha256":"8079080a299674bb0558a6550243051360be1c0f18252708128c3f71af899a0d"},{"kinds":["contract-source"],"path":"contracts/warehouse-administration.md","sha256":"ea037935ce439d10f4ccea3296bb16aa2ddee4e0c13b2b7ade0638ff7cb9b8d6"},{"kinds":["requirement-source"],"path":"prompts/modular/accounts.md","sha256":"0a8dee0847a02da7777c11c4175577c4533fafa538fc09e5124cf4d825e4c96f"},{"kinds":["requirement-source"],"path":"prompts/modular/bundle-checkout.md","sha256":"d3f21ac08991619b5c9aa6ac99a7554340bae991d5b31a25cdb048cdb314936e"},{"kinds":["requirement-source"],"path":"prompts/modular/bundle-returns.md","sha256":"34b3bee882e6a1f6a1527e67d12f8cbdcfd74d026108f9805159123c41015bf2"},{"kinds":["requirement-source"],"path":"prompts/modular/cancellation-conservation.md","sha256":"f383c20f16d75b77dfe2c0ca10dc1c77c998feb80b7c327616512b486c4714f8"},{"kinds":["requirement-source"],"path":"prompts/modular/cancellation-queue.md","sha256":"7e1adf98391a930017acb5c37d46c9b0d17d89e58c939fc57dc52f22e2f86f31"},{"kinds":["requirement-source"],"path":"prompts/modular/cart-expiration.md","sha256":"7c66696f044249d5f7f31487efb85e83f4c06b862fbaa3dd142ef2aef6afc2b7"},{"kinds":["requirement-source"],"path":"prompts/modular/cart.md","sha256":"da024de4032661a46574c8ac8dbe3d1baf45ef6ea7115ba1976d7d5c1044afbe"},{"kinds":["requirement-source"],"path":"prompts/modular/catalog-discovery.md","sha256":"9a0785c55d17e577b84ef257de482bcd8bc8c961e9d0760c38472936a1a0062b"},{"kinds":["requirement-source"],"path":"prompts/modular/catalog-items.md","sha256":"36a72da77f42d856b691c93dc587fee6d9d7f88b722a9893fc226b1d5bb5d3fe"},{"kinds":["requirement-source"],"path":"prompts/modular/catalog-management.md","sha256":"93fece29e1d68513732398c815fe4ebc83079e705e2b9de40df543281439f390"},{"kinds":["requirement-source"],"path":"prompts/modular/checkout-recovery-specification.md","sha256":"0e851fea800d81cb558f8004f04e628e5ccf9f6e5589a9d2f6fceaf33e50f207"},{"kinds":["requirement-source"],"path":"prompts/modular/checkout.md","sha256":"bb834dec7858349a17a8980eb13459a606cdbd2b1c534f468f666459c1a24ba7"},{"kinds":["requirement-source"],"path":"prompts/modular/customer-profile.md","sha256":"f4ffc5d2a642bbccec9eb5c428a37b2d9bfaa1c9f96648d8544bfcf2fe8187bb"},{"kinds":["requirement-source"],"path":"prompts/modular/delivery-notifications.md","sha256":"12d6db9061749779dcb3db699a492781b34ae955c4c11cbb3e5dc2c8e0300ef3"},{"kinds":["requirement-source"],"path":"prompts/modular/faceted-search.md","sha256":"5dc13b47abcbc85654c82148f44b854979d868578db50db42e060c773edc2207"},{"kinds":["requirement-source"],"path":"prompts/modular/inventory-dashboard.md","sha256":"d2190c809ce459141cfdb716d25d38b01f7aca2ce8b8f74128e731e4e3fede07"},{"kinds":["requirement-source"],"path":"prompts/modular/l1-external-sync-specifications.md","sha256":"7f0486045ef1b18271d1b004515f3a1dbf76d09ed62eb7c0683b099100b7f903"},{"kinds":["requirement-source"],"path":"prompts/modular/l1-specifications.md","sha256":"bc3ab63f33d9bbc59fde776c7fdc83ecfdf3d087be78e2596538b7d6945fdd8a"},{"kinds":["requirement-source"],"path":"prompts/modular/l3-specifications.md","sha256":"e6b9877c0f28ec287e510367a810221d09f9b48bb506f7c3cde4eb4450a3eaf4"},{"kinds":["requirement-source"],"path":"prompts/modular/later-specifications.md","sha256":"097745468f48a0589fd9dd81ad98d3ca0ce9455678156fc74d2331e403a77a39"},{"kinds":["requirement-source"],"path":"prompts/modular/managed-support.md","sha256":"33abfe9e7514c6826df60572557d54f2e1570477afeae14c64bb3c5f3bad5fb6"},{"kinds":["requirement-source"],"path":"prompts/modular/notification-preferences.md","sha256":"496baa70ef300e6874f6dbf58ccd2bac594e53101875fd68740e27779d11871b"},{"kinds":["requirement-source"],"path":"prompts/modular/operations-access.md","sha256":"1699a8d0466e115874e49b8ae73bb0004c3c620a03d3aa04dd83818b6c798003"},{"kinds":["requirement-source"],"path":"prompts/modular/order-accounting.md","sha256":"c83044972a6f53a723dcba9ef9dec157eeacb4d2059b790a426da255134364d0"},{"kinds":["requirement-source"],"path":"prompts/modular/order-cancellation.md","sha256":"7d429d4e8db546961e7ec9a12fb5dcba5efd0562c8059ab354121fd5ed13ea84"},{"kinds":["requirement-source"],"path":"prompts/modular/order-delivery.md","sha256":"40483d0bf91c81efe57a61d62dbda756498792383bed9af595cdd8878a0b0cb8"},{"kinds":["requirement-source"],"path":"prompts/modular/order-ownership.md","sha256":"1f4e5149768d1d0a4917ca9b711f1f40bf2c783ee5d59a42ab307f01babb052e"},{"kinds":["requirement-source"],"path":"prompts/modular/order-returns.md","sha256":"7277266f1e8c7d660ee38ba55ac1359200e56c695862c6d7d755d6ec6a6cb808"},{"kinds":["requirement-source"],"path":"prompts/modular/order-support.md","sha256":"f515fe9827373e110633245df158ce8f78ca040f6c25a059d3885182515abb9a"},{"kinds":["requirement-source"],"path":"prompts/modular/payment-records.md","sha256":"ab5c9c913041e324cad82fe6a135d78906818afa3c9a5c204289142c1d696f09"},{"kinds":["requirement-source"],"path":"prompts/modular/price-authorization.md","sha256":"6e9f949bd45f3bc816336de6e7db6dd18254ceebe6ecacc8e0abfd5f3887b8b5"},{"kinds":["requirement-source"],"path":"prompts/modular/price-history-orders.md","sha256":"005e1db32c0c5e4c4f52c1834f82c3d35c7118912c86580b4295a34e9ff12d3e"},{"kinds":["requirement-source"],"path":"prompts/modular/price-history.md","sha256":"9edb3db98af12f3a08cae7887dd6da91c746be49597f086457fc2ecb70c558b8"},{"kinds":["requirement-source"],"path":"prompts/modular/product-bundles.md","sha256":"a0805178ebbb20ea290994de65b4ded503c37798af59905d47c1cb2fd4e2c467"},{"kinds":["requirement-source"],"path":"prompts/modular/production-specifications.md","sha256":"a49cc5aae32bfb035b878e297ee67c1aa8b83ca90e7c73c3dd0cf9f919ae1cf6"},{"kinds":["requirement-source"],"path":"prompts/modular/progression-automatic-reorder.md","sha256":"64751064e699c709a6550daba6fee8f019f445b817910e1ed0d8831632fa2471"},{"kinds":["requirement-source"],"path":"prompts/modular/progression-cart-recovery.md","sha256":"4dc64aaa87d54a05bd7cce31bdc98254e605251d14f1688b859e9aa908da6a8b"},{"kinds":["requirement-source"],"path":"prompts/modular/progression-framing.md","sha256":"33042f5721843cd3796c8545c2a01a32e6d9b341192036321f6532f5ed0e03d2"},{"kinds":["requirement-source"],"path":"prompts/modular/progression-personalized-recommendations.md","sha256":"6c78b3963318b18da11173f96bed5cb8e5537f49e0008342ff11209fada3e099"},{"kinds":["requirement-source"],"path":"prompts/modular/promotion-checkout.md","sha256":"1979ce8991b38fab30c4ea0d0fb7adba51d79bb82981059b0dafded82fed843a"},{"kinds":["requirement-source"],"path":"prompts/modular/promotion-reporting.md","sha256":"3285eb0a50ceeed40b3415bdab8d6f5fc4138e496d2062a08c5ee2fac70157c6"},{"kinds":["requirement-source"],"path":"prompts/modular/promotion-rules.md","sha256":"7a43b632e8fd43b8060100f9614476e41003e47afa04dd1b7c3b09639f6280b2"},{"kinds":["requirement-source"],"path":"prompts/modular/purchasing.md","sha256":"059073391cf92eef2fb1edb710ec1a62747e82115c2bd0251274111b92f93e46"},{"kinds":["requirement-source"],"path":"prompts/modular/recommendation-feedback.md","sha256":"072a548ae797773041e86cbc62f39011559ca7ab40441bee6b77ec1854cd7f51"},{"kinds":["requirement-source"],"path":"prompts/modular/recommendations.md","sha256":"0eed3feb6fc6acf86260d74eeff557b15837aaa09a857c63cf9dee848cd7201f"},{"kinds":["requirement-source"],"path":"prompts/modular/reservations.md","sha256":"e45b53568066c7200486453f78fdbbe89a116259a4d83d9435bde350316595e0"},{"kinds":["requirement-source"],"path":"prompts/modular/review-access.md","sha256":"430fe60b2f028f67eab9f23b572238d4dacc60f7b89d74f8030618368455f371"},{"kinds":["requirement-source"],"path":"prompts/modular/reviews.md","sha256":"1649822528e1bbcfedc59b3c8c5d7ff1b5f0e42e1e82b81ee7e59361cbefab33"},{"kinds":["requirement-source"],"path":"prompts/modular/sales-dashboard.md","sha256":"c65aaadb1f8da6e2b5f7967cc3bcf07beb0e7a860351e866373667bc33012dbd"},{"kinds":["requirement-source"],"path":"prompts/modular/scheduled-restocks.md","sha256":"ba32547038c681f7214219e5a10df75dbf925ee19d4ba60db5b291570f9c16ec"},{"kinds":["requirement-source"],"path":"prompts/modular/search-ordering-specification.md","sha256":"77d88f2b03c71b6cdc8350020ffd7815cf0bee87e4155c1aa00beb9bb81a83f8"},{"kinds":["requirement-source"],"path":"prompts/modular/shipping-authorization.md","sha256":"f336e9c8e1ea425c3dc44e72fabbc5e9e0e2a3541cab3750dd8e112b6c7997f8"},{"kinds":["requirement-source"],"path":"prompts/modular/split-tender-refunds.md","sha256":"c21e00af3623136de4ae45f089b45b457a34563eb37a756d5b8b3563d1f19734"},{"kinds":["requirement-source"],"path":"prompts/modular/staff-access.md","sha256":"10c6aedbc5c60b440d6d0b5c38bb57363ae021c54d62ee2ffe2294d7f6741550"},{"kinds":["requirement-source"],"path":"prompts/modular/staff-activity.md","sha256":"0cb274acefc0d75cd82415de0eb167133db0412e1de4f0fd59f62983d29a1476"},{"kinds":["requirement-source"],"path":"prompts/modular/staff-roles.md","sha256":"951f617a191161df8b9061b304ad568bb5a942070e72142ddf4b93c3dd099951"},{"kinds":["requirement-source"],"path":"prompts/modular/stock-alerts.md","sha256":"3478629a2831e04baca5eec1d3a2a3695c03f6044c36b43acb7c8ca7e35e9867"},{"kinds":["requirement-source"],"path":"prompts/modular/stock-conservation.md","sha256":"4551438e045bb65b44127d3f92dcb79548c78dc1ecc42d2c7807a599d6c59c8f"},{"kinds":["requirement-source"],"path":"prompts/modular/stock-transfers.md","sha256":"6e2f0735d81fd5376b851893404873af4807b83e807e078cca0b8c9a9920e627"},{"kinds":["requirement-source"],"path":"prompts/modular/store-credit.md","sha256":"d769da6c0ef4eb2fa6ac241aeddb61e8e2d85a0f1fcfd595fbf2c6660f67cd7b"},{"kinds":["requirement-source"],"path":"prompts/modular/subscriptions.md","sha256":"ca93ab01c0d23c2fb3c035981372dc1254b219f1875b9566c0d01123176cc7c1"},{"kinds":["requirement-source"],"path":"prompts/modular/support-history.md","sha256":"df2ca2664901a90814eab17d5f7c029595ef614d76ee29eeb4204d9f4f371429"},{"kinds":["requirement-source"],"path":"prompts/modular/support-intake.md","sha256":"048f11b71b66dadf80c77b5d9aef8bbff2b2030171eeda5012dce74ae16421ed"},{"kinds":["requirement-source"],"path":"prompts/modular/support-refunds.md","sha256":"79090320de03a74660418da9e9adbf8170f8c574be990faeaf3d7df652221aa6"},{"kinds":["requirement-source"],"path":"prompts/modular/support-triage.md","sha256":"bbb35abcbae9b8d3782ee3eb94f3175428edfe3fa564d6b0e3af2e0050e10dee"},{"kinds":["requirement-source"],"path":"prompts/modular/transfer-authorization.md","sha256":"6d1e33c7a6aa209ad3f2705607a839dde5c915b65d78a8bb10b9e0da29172c0b"},{"kinds":["requirement-source"],"path":"prompts/modular/warehouse-administration.md","sha256":"8fa2d7b54b3c3d9601ae69bbfee5bdf389f68fea0d0ef4032201af12d3f424a2"},{"kinds":["scenario"],"path":"scenarios/01-account-create.json","sha256":"f37b83a76ff787e12aae1c86fa91b5a2483bfb1aef9497c291e2a374a4376f45"},{"kinds":["scenario"],"path":"scenarios/01-account-duplicate.json","sha256":"f699c90ca3b1af3eda1b73e3c38a4a7cce19b779afa2f9b208c9ead6fdbf15ba"},{"kinds":["scenario"],"path":"scenarios/01-account-password.json","sha256":"35ea9a3a824444096d9d0a51388296c7c35fc833a532b5252f1df9fc5cfab125"},{"kinds":["scenario"],"path":"scenarios/01-account-reload.json","sha256":"574de181769f0d2fa1a88f1131b2a033385fdd576f9aa5088e95781cbd2266d4"},{"kinds":["scenario"],"path":"scenarios/01-account-signout.json","sha256":"7ce90ffda1338d99e37a87ea5beecaeaaa56adecbeb2f46707d1cf0934e596d1"},{"kinds":["scenario"],"path":"scenarios/01-admin-write-staff.json","sha256":"caf94ac7cbffea4438a343c0883322e090cf83aad743a882fb747c287c89f2a2"},{"kinds":["scenario"],"path":"scenarios/01-buying.json","sha256":"48168c76a51b5e1e6d15573a0891cf90889f99aab78b638a24892f70c55ff257"},{"kinds":["scenario"],"path":"scenarios/01-cart-boundary.json","sha256":"76b9c484f70ac94f0dc1787a1a4e61851bf6586e8d1facc9f970791d47a81777"},{"kinds":["scenario"],"path":"scenarios/01-cart.json","sha256":"88cd43f3f48ae86eeec891677a2817f42f8f6554bad299cfae92889d00262d3c"},{"kinds":["scenario"],"path":"scenarios/01-catalog-ranking.json","sha256":"05436b81ee1fefb5e781f7a68060d41690bcdb7b1b9cb0e7543d183b738f1e14"},{"kinds":["scenario"],"path":"scenarios/01-catalog-search.json","sha256":"a633a4afcdbc5e19cc3189ab5e93c358fad44672a4ec2570fa4569c0e9ee0d02"},{"kinds":["scenario"],"path":"scenarios/01-catalog-values.json","sha256":"6216be4761d0db8ac4480ad1fa1a6d4a628d3a4c022278eb7ebf25105db50ea2"},{"kinds":["scenario"],"path":"scenarios/01-core.json","sha256":"1d7f0cf801d41fb01a4f63b2a414c9d723b0749130d1eefff84730567f9a1614"},{"kinds":["scenario"],"path":"scenarios/01-duplicate-checkout.json","sha256":"b6f470b48e0f504d479376ec25895c1d9c293b1ab52cc9d27a1f626519513290"},{"kinds":["scenario"],"path":"scenarios/01-external-live-sync.json","sha256":"3a3dbe302f7e300ac7e82d680d6fb7c6ee277ec9e175d590f68c1c86569ab752"},{"kinds":["scenario"],"path":"scenarios/01-external-reconnect-sync.json","sha256":"5a9e3822bf5c888709d8fc6e255c29e429ba52be3460d688b8ce0a77ef72e79c"},{"kinds":["scenario"],"path":"scenarios/01-external-reload-sync.json","sha256":"80d9eeb9995c30ae4af156c3d93d42475e8b7de0c1e7dca69d4f882536a5cf86"},{"kinds":["scenario"],"path":"scenarios/01-external-server-restart-sync.json","sha256":"c4dab98e2623e35045fa9dd9f4d251392adf316c7ccacc41982fe16b9a8aa574"},{"kinds":["scenario"],"path":"scenarios/01-last-unit.json","sha256":"499778db939f0392d5576e9ea8feba62cd3f9d7ae07130283cbc3babebc81405"},{"kinds":["scenario"],"path":"scenarios/01-order-ownership.json","sha256":"6bef82c8e28f962d06479e452b2019bf107e4b89cfcda5a11e927f9c518c647b"},{"kinds":["scenario"],"path":"scenarios/01-purchase-attribution.json","sha256":"6e1974179f5bb3c475fd84dc051b3e6f37cf4e02cde17928bc1a68275ce798cc"},{"kinds":["scenario"],"path":"scenarios/01-purchase-session.json","sha256":"219f67e5959209f89ee7179a584bcde6e8ac340bd121cbeb0e25895604d1c614"},{"kinds":["scenario"],"path":"scenarios/01-restock-race.json","sha256":"d37c5071bdb221db025cbe43c6377cc7eca99c62ee6e4720737a32eb60decffa"},{"kinds":["scenario"],"path":"scenarios/01-review-eligibility.json","sha256":"ad387c41d1902fab24915cf9cdeed8a2cd0d7aae26ffc379ae784e61d5c3a277"},{"kinds":["scenario"],"path":"scenarios/01-review-rating-live.json","sha256":"f6d4d0c9ea3bbd1213d130ee1d706e326ffbab33e5b9fd6633fa6ff90fea3e3c"},{"kinds":["scenario"],"path":"scenarios/01-review-uniqueness.json","sha256":"49873dc6cf26204dcebd1038b71bfc5213b091d3d13f5f37c5980998a7f3f8f6"},{"kinds":["scenario"],"path":"scenarios/01-review-visibility.json","sha256":"81360bc6b9735251c7672ee26148912aebcc1426b0be2eca9eecd5e607b927b8"},{"kinds":["scenario"],"path":"scenarios/01-server-price.json","sha256":"24481cb30af8c57ebb26b2adb393b8987642e60ddecdbeff2c0381c122825fdc"},{"kinds":["scenario"],"path":"scenarios/01-warehouse-admin-staff.json","sha256":"5434bf92e5b343862dfcb0e6b26593a71db86c0635222ebb3101f60e00fbea5a"},{"kinds":["scenario"],"path":"scenarios/01-warehouse-stock-live-staff.json","sha256":"708a7bd1df90dba203d6c1236d83a3e6deaaecb6e69b4c2faa4208f773966414"},{"kinds":["scenario"],"path":"scenarios/02-cancellation-queue.json","sha256":"d8758cc074fec24a5d8d1d4395e2edb2a7f376d14d91427df6f70b9a5e8e49ce"},{"kinds":["scenario"],"path":"scenarios/02-fulfilment-access.json","sha256":"ccd33eb7a0fbf80f71c86c963f3fe747581808622a33f842e3aa85d940c20012"},{"kinds":["scenario"],"path":"scenarios/02-fulfilment-live.json","sha256":"54455bebda5a1ef78421b81229d63298fcddae0a8315379c173cca0436e0638f"},{"kinds":["scenario"],"path":"scenarios/02-fulfilment-ship.json","sha256":"3996ecb875ae0ca32eb68053650d0d415246487aac909c5de901789f39d64913"},{"kinds":["scenario"],"path":"scenarios/02-invariants.json","sha256":"23b929b83386165a6fff61ee3cbe6845d5da66e5707cdc1ce8f5ead3207018b4"},{"kinds":["scenario"],"path":"scenarios/02-live-price.json","sha256":"7a905177cae6af15f682ebdb909f695bdee85d3124ee91917cb65e8209a73803"},{"kinds":["scenario"],"path":"scenarios/02-low-stock.json","sha256":"a985505feaa58e6f80f08865ea08658da174a01d08a1e70b832b5dd8579e5c88"},{"kinds":["scenario"],"path":"scenarios/02-operational-best-sellers.json","sha256":"f887d0f4dd660fb85a80f10d6e6e1c9fcf6f2bf25e79fbbd7ba86227968e8ace"},{"kinds":["scenario"],"path":"scenarios/02-operational-category-totals.json","sha256":"263411fec43db53777309f2d64fcbdeef1b551a2d981ee8aa8b4caa0ea2faa16"},{"kinds":["scenario"],"path":"scenarios/02-operational-recommendations.json","sha256":"7ea0fa0f317f963c9d2d37adef945be9b8f19eb8618f387bda0a5cd6b92f30b1"},{"kinds":["scenario"],"path":"scenarios/02-order-cancellation-core.json","sha256":"a490889daee2a92eeb18eb35cb4050a47df179d02e031286d70848a4f2595787"},{"kinds":["scenario"],"path":"scenarios/02-order-cancellation-history.json","sha256":"304c8eb8477bc63fda3a1d90f11e3a03fd6391001d8a12ec405c14aeaced4ccb"},{"kinds":["scenario"],"path":"scenarios/02-paid-price-history.json","sha256":"84896fdea36b4f267443000306a6a349fc7221772278d96c751516b0330dff5d"},{"kinds":["scenario"],"path":"scenarios/02-queue-warehouse.json","sha256":"212e8e907b85aff9e4047e7f6f4258d3078aa78ac6bc1df21abb980759322cf2"},{"kinds":["scenario"],"path":"scenarios/02-self-contained.json","sha256":"aad3ce91dae1a94c0f230bab33b0c7a7942337db51f4f9cde06d1d9e2cc99300"},{"kinds":["scenario"],"path":"scenarios/02-server-actions.json","sha256":"a4f22b10cd76a69e6d761818668c7ec52ad064a0a5b1c391a30fde95849b7211"},{"kinds":["scenario"],"path":"scenarios/02-strengthened.json","sha256":"15fc5e537422dbe8dd875cae0b82ac5b5afbf4b65d45ff4545aaea38dd8a9db8"},{"kinds":["scenario"],"path":"scenarios/02-transfer-overdraw.json","sha256":"df6655ff0bb3c9553384ac37f3659af66d6153fef05d31db5e3d3a1d8165f096"},{"kinds":["scenario"],"path":"scenarios/02-transfer-totals.json","sha256":"81462d3b8d4e78f2dd0e207faf285dca472974cd2cab95f723ce735fc3a5c589"},{"kinds":["scenario"],"path":"scenarios/03-cart-expiration.json","sha256":"22a10c3255506dd1bd50ab7caef5187bd0fc3bad3400ad4af9221c811b570d40"},{"kinds":["scenario"],"path":"scenarios/03-deferred-access.json","sha256":"91ed336253f239ed05cb74bf639be7d9ec0f29bb9b372261295ea7d9babc7098"},{"kinds":["scenario"],"path":"scenarios/03-deferred-durability.json","sha256":"fc425c650e26f4be6134cfdbf590423073fd9e2ebac5ee852b646df7a599a954"},{"kinds":["scenario"],"path":"scenarios/03-deferred-integrity.json","sha256":"5bc1faefe4b3d2547b19afc8a5fafb8dbbfef05d45dcb71b2979e0777eb0bcb0"},{"kinds":["scenario"],"path":"scenarios/03-order-delivery.json","sha256":"880e518338a1615825595b983daf64c6e1d68db924ad5fd8c1b205df7e3ff1aa"},{"kinds":["scenario"],"path":"scenarios/03-reservations.json","sha256":"c2460d2b1f1336d1e4c918b6c27ac962d680f8df897a8ca8fd52bdbd403d17ca"},{"kinds":["scenario"],"path":"scenarios/03-scheduled-restock-apply.json","sha256":"74f49d25eea210b9e0209a57261809a585ca28380edfd50084933beff743d2c1"},{"kinds":["scenario"],"path":"scenarios/03-scheduled-restock-cancel.json","sha256":"462df740d24017bc487d961b1547cafdf0e2bc1cea7805dc669d68ff245261bc"},{"kinds":["scenario"],"path":"scenarios/03-scheduled-restocks.json","sha256":"510a8b147738a249b30dce4900e431ca32880de5ea990600aca063c38535b228"},{"kinds":["scenario"],"path":"scenarios/03-server-time.json","sha256":"667e580173187cb4b0f9538336a7e12108e14b5273c9ac663faa5f2325b006be"},{"kinds":["scenario"],"path":"scenarios/progression-account-state-reconnect.json","sha256":"1ead0e72e9375a95cd03caa09bd1fefd858bef7bc910dd6762a53bcee968ce55"},{"kinds":["scenario"],"path":"scenarios/progression-account-state-reload.json","sha256":"fd464513460c640ddcec6ed7e30f9c434a982ae300d16f5233551c14bff45995"},{"kinds":["scenario"],"path":"scenarios/progression-automatic-reorder-access.json","sha256":"ed57cd01a22e7b7a964108ccf1f971f2e8877921015fdd952cb28109cb7be32d"},{"kinds":["scenario"],"path":"scenarios/progression-automatic-reorder-duplicate.json","sha256":"41fe868171047997d33fe6f424bf771fff8d91fc32a19ef749510fa6f41abb57"},{"kinds":["scenario"],"path":"scenarios/progression-automatic-reorder.json","sha256":"91827fedd89c2e268b2b9102d62a89d39dc8d4e07af619d82073c07595c10d57"},{"kinds":["scenario"],"path":"scenarios/progression-books-balance.json","sha256":"6f8d35c97296d36f26ebe07b9b6abc7d73531e4dab495c370a13e80d43adde46"},{"kinds":["scenario"],"path":"scenarios/progression-bundle-checkout.json","sha256":"b26a777560d208a1e415b58dfd46b3662669c4546208d141f0954033ff575a2f"},{"kinds":["scenario"],"path":"scenarios/progression-bundle-returns.json","sha256":"e8d4c983cb11934ac323abbe0ddf55715d75a832133bfd664197677da4911ec8"},{"kinds":["scenario"],"path":"scenarios/progression-cart-checkout.json","sha256":"b49e07cca26e9461db2467cfce31eb43c7b10fb1481a7252353f08d31bf67abd"},{"kinds":["scenario"],"path":"scenarios/progression-cart-recovery.json","sha256":"8c00fd1d2d10091754577acab63319dd593c137ccb75bcbb45cb43bf000b8298"},{"kinds":["scenario"],"path":"scenarios/progression-catalog-management.json","sha256":"a18d83be7f697b89976b062112acd31755b93a702e12b2bd0f6de993f1c9b0e4"},{"kinds":["scenario"],"path":"scenarios/progression-checkout-crash.json","sha256":"024b80278cdcb30ac53a448db82b45ada850d7c22a04ace3aecf296f03b2b1ef"},{"kinds":["scenario"],"path":"scenarios/progression-core-business.json","sha256":"deac867579938f0e41fb16c02e302416d5dcbfd2d20d11a7374236d58c35b8c1"},{"kinds":["scenario"],"path":"scenarios/progression-customer-profile.json","sha256":"7590ab78f8f475fb16fe7959e625da404d29f1e58f9ceb11f51f557648b91829"},{"kinds":["scenario"],"path":"scenarios/progression-delivery-notifications.json","sha256":"62a30dc8ec24d1758e49f90cdaf60856cc465136595347550ae78fe880d2a8a6"},{"kinds":["scenario"],"path":"scenarios/progression-faceted-filters.json","sha256":"b4610dc373d96ad2eeba7c38fb33173ec74250d8ef4941897bb3c48a98cb5368"},{"kinds":["scenario"],"path":"scenarios/progression-faceted-pagination.json","sha256":"c27b5edbd219e2f440ffa29dbad77dbe2da3485437ec9c0071c2d8a99017aeee"},{"kinds":["scenario"],"path":"scenarios/progression-managed-support-privacy.json","sha256":"74da98e1252bb77f0c3b8bf2a9ac788acb7a0104405d786c9a65c51048af7aad"},{"kinds":["scenario"],"path":"scenarios/progression-managed-support-shared.json","sha256":"712852c3d0289ca15efe348797449fcc3b301df5d3d0a9150b188623fced206d"},{"kinds":["scenario"],"path":"scenarios/progression-notification-preferences.json","sha256":"cb494fbc2543347407abe54f74d9cc3cc698f135690f063ebbbff756aec8c807"},{"kinds":["scenario"],"path":"scenarios/progression-open-list-live.json","sha256":"9abcf41a6919dab036d26869c75de0129b12814e9d49bcf80653fa6e61eff1b0"},{"kinds":["scenario"],"path":"scenarios/progression-order-return-boundary.json","sha256":"c404ce6d68113bd69e290bfcfae4b72b14c0a19f63de349dcf79e47563b1f255"},{"kinds":["scenario"],"path":"scenarios/progression-order-return-complete.json","sha256":"02c41f76b4bc60ba0b0e624df540fcd77e7407c90a2e66487672ed6f9c5ed9f2"},{"kinds":["scenario"],"path":"scenarios/progression-order-support-boundary.json","sha256":"9f19a26a9e4d4fe4385f94150bdef6c4e16913f79c5b0fbaede02984ce098362"},{"kinds":["scenario"],"path":"scenarios/progression-order-support-owned.json","sha256":"dda5208d3dbe3a151e683a8cd238f414118553ea0812584f00a82655255cf1a5"},{"kinds":["scenario"],"path":"scenarios/progression-personalized-recommendations.json","sha256":"8bb96bf7c8a81df356cc0e4bd248184dc13924d78bfe85bf173f840899350e1d"},{"kinds":["scenario"],"path":"scenarios/progression-price-cart-checkout.json","sha256":"0e0f6383af25fa70701c7cc99a4b934da2318a68d168ea897333c8be943c8274"},{"kinds":["scenario"],"path":"scenarios/progression-product-bundles.json","sha256":"7d517f3c1265dae451ed4b0ed0f97186b340717561a10f030edd622f5a50163c"},{"kinds":["scenario"],"path":"scenarios/progression-promotion-checkout.json","sha256":"3e6baa221f1ab203893a2a105c809a3f221d985251a64f115d47e2ab58c180ec"},{"kinds":["scenario"],"path":"scenarios/progression-promotion-reporting.json","sha256":"8c9efe46cdeee1d1dd8d802a222bd1f58b06e997dc53a44fd8e19ff45586673e"},{"kinds":["scenario"],"path":"scenarios/progression-promotion-rules.json","sha256":"e49ea26a881a6d49c485a63d7f1f5cca55fb314522d9d948ed5c2f7eb2a84306"},{"kinds":["scenario"],"path":"scenarios/progression-purchasing.json","sha256":"b2aff32949f55c4819bde7b670c00896aacdbfe59f478fa46df9423da455bd12"},{"kinds":["scenario"],"path":"scenarios/progression-recommendation-feedback.json","sha256":"6a6bae970c29cd443a9e6c66919c909a9f0792a8c37a7d00d2c63f5e4be872c3"},{"kinds":["scenario"],"path":"scenarios/progression-review-access.json","sha256":"c416df2ced8caa0cc58002e333ba72c6e1d3dc83a24681240dfe2a2215e8530b"},{"kinds":["scenario"],"path":"scenarios/progression-review-script.json","sha256":"267a9134ddb8cd25d3e9d76b5595e97debd2a8071f8ff8e568aa7285b608d30f"},{"kinds":["scenario"],"path":"scenarios/progression-search-ordering.json","sha256":"b96f50c9ec28d68aa1fe4e6d6ea7631f6399f439025c6fe74c1d9a3ce8a55803"},{"kinds":["scenario"],"path":"scenarios/progression-shipping-accounting.json","sha256":"2e310fb4489c154617660b3d247136646c04faa903e958cf2561e9072615b779"},{"kinds":["scenario"],"path":"scenarios/progression-signed-out-purchase.json","sha256":"59de2a84876ae4542d783dca533764eb4f06bc390a67a37b25f0967477ee44fe"},{"kinds":["scenario"],"path":"scenarios/progression-split-tender-refunds.json","sha256":"39771047c0b0a84a6de5c4178e387c95581667cba5741e24d7a3555bc6a19018"},{"kinds":["scenario"],"path":"scenarios/progression-staff-access.json","sha256":"67de6a963d3cdb33dfb119ffacfdd1024d7f9c4d095bf90d553ef1c6efe235c9"},{"kinds":["scenario"],"path":"scenarios/progression-staff-activity.json","sha256":"28600137357ce5620f9f4919b25eb1ff27ba5aff4ed02285349e4d82cd1585a4"},{"kinds":["scenario"],"path":"scenarios/progression-staff-roles.json","sha256":"d87a2f7f0890a84614f538d7656de92082ba1c5843a40d3b2cf0f7f2d9097e11"},{"kinds":["scenario"],"path":"scenarios/progression-stock-alert-delivery.json","sha256":"e4899fb49e87edd9787997e9abb173c890604397fe096186ea0f601fd2e2f67f"},{"kinds":["scenario"],"path":"scenarios/progression-stock-alerts.json","sha256":"582bcb0bea4f109da56e761ad27c809cfb2da38edec41a9e4a528ea603c020f1"},{"kinds":["scenario"],"path":"scenarios/progression-stock-limit.json","sha256":"e07cd6c2b949b392cbb10f0f7cf8b2e8b74a0ef2ed17d563187ad28387eebc4a"},{"kinds":["scenario"],"path":"scenarios/progression-store-credit.json","sha256":"04a1fc19d215155c90a5382e91e15cbf01b400d6b2f8600b5a0e25b42132c820"},{"kinds":["scenario"],"path":"scenarios/progression-subscriptions.json","sha256":"de31264cfc1890ab6c3ebaead98c41964da70d135d34a5f4944ac41108998e94"},{"kinds":["scenario"],"path":"scenarios/progression-support-history.json","sha256":"86928df21aa68fec5c20993ec13e3b1c57291f38a35167996a99ef34f363a2b7"},{"kinds":["scenario"],"path":"scenarios/progression-support-intake.json","sha256":"c8d01303a3316f28de956fb2a72ab289905a46f5d689fe40b8193e12cea1753a"},{"kinds":["scenario"],"path":"scenarios/progression-support-refunds-access.json","sha256":"52f03c8a7ab8b1c7554f7d637bd5d46a19e24f4be5666eac14398ed1f1b2707b"},{"kinds":["scenario"],"path":"scenarios/progression-support-refunds-accounting.json","sha256":"38be5988391c96acfa7eac25248df674e0fe95e2b58f22ea47fa909a02f01576"},{"kinds":["scenario"],"path":"scenarios/progression-support-refunds-resolution.json","sha256":"2627a122fd791ed4ed2c4c08e7fea283e017bfe59dd1a682899207cc89d7ee67"},{"kinds":["scenario"],"path":"scenarios/progression-support-return-interaction.json","sha256":"7c50a90555a616e9e8ce7fefb771e79237ed2890b28545a5951da2fa52902820"},{"kinds":["scenario"],"path":"scenarios/progression-support-triage.json","sha256":"a0ad2546ae46257154a75fa3250353cd443f1c6dc93ff52b4e666e99444101f5"},{"kinds":["track-manifest"],"path":"track.json","sha256":"ae917ab791cfc59a6ece01cc23ab433a235cc5fb20ab37974d8ed36c494a67d9"}],"sourceManifestSha256":"aab07ac3bb812f5c50efbda67ac9e1605a8e97a6a39b6baa219d762ffc65722f","task":{"baseRecipe":null,"composedSha256":"d063ef87268f9c5a4bb2e7c0f4570cd2ac6b50e9f45e1ca1c1df76049731f650","contractSha256":"e432e08fc81f67f95f38c44e05fc2adc393f10cd29d67af78d0f1676e73b7578","contracts":[{"from":null,"id":"ecommerce.application-interface","modes":["fresh","upgrade"],"order":0,"owners":["recipe"],"path":"contracts/application-interface.md","sha256":"b770db6c1dbf92c106c7501a17028876a30a63655c8f88b5667b44557f401692","until":null},{"from":null,"id":"ecommerce.progression.staff-access-hooks","modes":["fresh","upgrade"],"order":1900,"owners":["ecommerce.progression.staff-access"],"path":"contracts/staff-access.md","sha256":"dcd7b740d5f65160ddf0b4221d29f84426ee89f04560e953add0856cf0bca87a","until":null},{"from":null,"id":"ecommerce.progression.customer-profile-hooks","modes":["upgrade"],"order":1950,"owners":["ecommerce.progression.customer-profile"],"path":"contracts/customer-profile.md","sha256":"7451bf14ada950c9b7ad269d63467154c388e9ea18b97f479037cdd397ca519f","until":null},{"from":null,"id":"ecommerce.progression.staff-role-hooks","modes":["upgrade"],"order":1960,"owners":["ecommerce.progression.staff-roles"],"path":"contracts/staff-roles.md","sha256":"cce6145182485fafcd2adc29cec90af20523a2afd9c952800eefb5d15a49f96e","until":null},{"from":null,"id":"ecommerce.progression.catalog-management-hooks","modes":["upgrade"],"order":1970,"owners":["ecommerce.progression.catalog-management"],"path":"contracts/catalog-management.md","sha256":"5123971185ded331deaef2e323fad31e6133d55fc839289e2c4a0dfc5292ee4e","until":null},{"from":null,"id":"ecommerce.progression.payment-record-hooks","modes":["upgrade"],"order":1980,"owners":["ecommerce.progression.payment-records"],"path":"contracts/payment-records.md","sha256":"8c752352c7f47a0a65d1823f9ec74009dc95088fc6c2a6cc30d1a1dc7fa11fe0","until":null},{"from":null,"id":"ecommerce.progression.staff-activity-hooks","modes":["upgrade"],"order":1990,"owners":["ecommerce.progression.staff-activity"],"path":"contracts/staff-activity.md","sha256":"7e7c1846445f82aa78ad8e32606e6830f9d326ebfea75d13b077c073695c9594","until":null},{"from":null,"id":"ecommerce.feature.catalog-items.hooks","modes":["fresh","upgrade"],"order":2000,"owners":["ecommerce.feature.catalog-items"],"path":"contracts/catalog-items.md","sha256":"596ec08b1f4a6b290595570f3bcf2847a1f60feeb20d90f98d25761666bfe9f2","until":null},{"from":null,"id":"ecommerce.feature.catalog-discovery.hooks","modes":["fresh","upgrade"],"order":2010,"owners":["ecommerce.feature.catalog-discovery"],"path":"contracts/catalog-discovery.md","sha256":"d13ea495f78e3827b7a243b97466aefbf0252980c76edb432b6b0506854e8450","until":null},{"from":null,"id":"ecommerce.l2.transfer-hooks","modes":["upgrade"],"order":2031,"owners":["ecommerce.l2.stock-transfers-features"],"path":"contracts/stock-transfers.md","sha256":"01f28ebb3487559d4689919f0991e43f74fcc85609fa4c810e92adde30e84096","until":null},{"from":null,"id":"ecommerce.l2.price-hooks","modes":["upgrade"],"order":2041,"owners":["ecommerce.l2.price-history-features"],"path":"contracts/price-history.md","sha256":"3465714f91e0e978607900b19377a134e9beb704848e2462b1438465cc2e98ee","until":null},{"from":"# Price history completed-order interface","id":"ecommerce.progression.price-history-order-hooks","modes":["upgrade"],"order":2042,"owners":["ecommerce.progression.price-history-specifications"],"path":"contracts/price-history-orders.md","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.price-history-features"],"sha256":"45d360052ede67e50e12b5c0e64904dbbdd1dd17a430c38130a2b8a70d274278","until":"# Price history cart and checkout interface"},{"from":"# Price history cart and checkout interface","id":"ecommerce.progression.price-history-cart-hooks","modes":["upgrade"],"order":2043,"owners":["ecommerce.progression.price-history-specifications"],"path":"contracts/price-history-orders.md","requiresFeatures":["ecommerce.feature.checkout","ecommerce.l2.price-history-features"],"sha256":"0d80d8f3bb9ced280a6203d2e8974b1a4db923b2d423205969ca1d44d002fc8b","until":null},{"from":null,"id":"ecommerce.l2.inventory-dashboard-hooks","modes":["upgrade"],"order":2050,"owners":["ecommerce.l2.inventory-dashboard"],"path":"contracts/inventory-dashboard.md","sha256":"8c3de4ff8a958776ca087febbbbf5f950eaef9efacb96b7c4fa49357b2d0a371","until":null},{"from":null,"id":"ecommerce.l2.order-cancellation-hooks","modes":["upgrade"],"order":2060,"owners":["ecommerce.l2.order-cancellation-features"],"path":"contracts/order-cancellation.md","sha256":"fc274b1409c50de8edcf797ab83cf8e8c46a54db30ada859f854e1cd40c7dee1","until":null},{"from":null,"id":"ecommerce.l2.sales-dashboard-hooks","modes":["upgrade"],"order":2060,"owners":["ecommerce.l2.sales-dashboard"],"path":"contracts/sales-dashboard.md","sha256":"ff9ed3890aa5c8303a0badecf946e7d98da3fbbb0a2890a693099bab1a2056c0","until":null},{"from":null,"id":"ecommerce.l2.recommendations-hooks","modes":["upgrade"],"order":2070,"owners":["ecommerce.l2.recommendations"],"path":"contracts/recommendations.md","sha256":"ab5417cbf78f43464e82a543e50e9601580bb2b3a57826a9e73da9899f39de16","until":null},{"from":null,"id":"ecommerce.feature.accounts.hooks","modes":["fresh","upgrade"],"order":2100,"owners":["ecommerce.feature.accounts"],"path":"contracts/accounts.md","sha256":"e281aa0cff56c23117203b13d11f6787a7f8163cb8d1b7bc0d2d86b5390cfff6","until":null},{"from":null,"id":"ecommerce.feature.purchasing.hooks","modes":["fresh","upgrade"],"order":2200,"owners":["ecommerce.feature.purchasing"],"path":"contracts/purchasing.md","sha256":"c2d59da67b400d055ab90bf53f2e46a3af9605a1fc68534c2b64d8db2ed3409c","until":null},{"from":null,"id":"ecommerce.feature.cart.hooks","modes":["fresh","upgrade"],"order":2300,"owners":["ecommerce.feature.cart"],"path":"contracts/cart.md","sha256":"bb1b8741b9eb47b5766a330bb1fc23005f7540fdc79d20efdaefb183cf50918d","until":null},{"from":null,"id":"ecommerce.feature.checkout.hooks","modes":["fresh","upgrade"],"order":2310,"owners":["ecommerce.feature.checkout"],"path":"contracts/checkout.md","sha256":"49416683e06d996a28980ff650a3281f89cae062269b7de761899e71759c731c","until":null},{"from":null,"id":"ecommerce.orders.data","modes":["fresh","upgrade"],"order":2315,"owners":["ecommerce.feature.checkout","ecommerce.feature.purchasing"],"path":"contracts/order-data.md","sha256":"7b7cfc6b102f4836235c7f23344865f60156b3e4ac96c3e5cbb0e298390b3b06","until":null},{"from":null,"id":"ecommerce.feature.reviews.hooks","modes":["fresh","upgrade"],"order":2400,"owners":["ecommerce.feature.reviews"],"path":"contracts/reviews.md","sha256":"6be3567258c0c4bf3331e609f63be26ad0bb43ceca3e7da4accaaad8c9ac6541","until":null},{"from":null,"id":"ecommerce.feature.warehouse-admin.hooks","modes":["fresh","upgrade"],"order":2500,"owners":["ecommerce.feature.warehouse-admin"],"path":"contracts/warehouse-administration.md","sha256":"ea037935ce439d10f4ccea3296bb16aa2ddee4e0c13b2b7ade0638ff7cb9b8d6","until":null},{"from":null,"id":"ecommerce.progression.support-history-hooks","modes":["upgrade"],"order":2900,"owners":["ecommerce.progression.support-history"],"path":"contracts/support-history.md","sha256":"610022f964c2aa00b409fb2723f91b885e306edde5bf73390e880cffd2a5583d","until":null},{"from":null,"id":"ecommerce.progression.support-intake-hooks","modes":["fresh","upgrade"],"order":2900,"owners":["ecommerce.progression.support-intake"],"path":"contracts/support-intake.md","sha256":"fd9f732acf67d6c772f4c05d626e4e9e472818fc5226b7ef3d7223b85a0e0bc7","until":null},{"from":null,"id":"ecommerce.progression.support-triage-hooks","modes":["upgrade"],"order":2900,"owners":["ecommerce.progression.support-triage"],"path":"contracts/support-triage.md","sha256":"8079080a299674bb0558a6550243051360be1c0f18252708128c3f71af899a0d","until":null},{"from":null,"id":"ecommerce.progression.fulfilment-queue-hooks","modes":["upgrade"],"order":2920,"owners":["ecommerce.progression.fulfilment-queue"],"path":"contracts/operations-access.md","sha256":"fd9c31482b484dc1f9bf3c521d247971334e1f52f51a7548a987bdebc65dbdb5","until":null},{"from":null,"id":"ecommerce.progression.promotion-rules-hooks","modes":["upgrade"],"order":2950,"owners":["ecommerce.progression.promotion-rules"],"path":"contracts/promotion-rules.md","sha256":"f352f8bb938aa874125b10bbb6182643f04b6d1ce82cde41512bd16835a7b90b","until":null},{"from":null,"id":"ecommerce.progression.notification-preferences-hooks","modes":["upgrade"],"order":2990,"owners":["ecommerce.progression.notification-preferences"],"path":"contracts/notification-preferences.md","sha256":"d6b4231f67d5e70a03ec331545b6419acf766b52841c8520b94963c16aa13fec","until":null},{"from":null,"id":"ecommerce.l3.reservation-hooks","modes":["upgrade"],"order":3900,"owners":["ecommerce.l3.reservations-features"],"path":"contracts/reservations.md","sha256":"e1c4af7a67dd200d5f18d550e5b4c30e7a4bbaf60600b434108ebff7bdf90627","until":null},{"from":null,"id":"ecommerce.progression.managed-support-hooks","modes":["upgrade"],"order":3900,"owners":["ecommerce.progression.managed-support"],"path":"contracts/managed-support.md","sha256":"458b89a7f786086a95957c778943660068cd6c00b30635fd1cd786c37df76398","until":null},{"from":null,"id":"ecommerce.l3.scheduled-restock-hooks","modes":["upgrade"],"order":3910,"owners":["ecommerce.l3.scheduled-restocks-features"],"path":"contracts/scheduled-restocks.md","sha256":"a946c0954a641d4fe64b933761e2b92d546ae08ee434d676c6faed3ed8b03c8e","until":null},{"from":null,"id":"ecommerce.l3.order-delivery-hooks","modes":["upgrade"],"order":3920,"owners":["ecommerce.l3.order-delivery-features"],"path":"contracts/order-delivery.md","sha256":"74bd21b39c8d5f020ff62cc408992c15788e48eba8e05a76a0cddc4b4920a98e","until":null},{"from":null,"id":"ecommerce.l3.cart-expiration-hooks","modes":["upgrade"],"order":3930,"owners":["ecommerce.l3.cart-expiration-features"],"path":"contracts/cart-expiration.md","sha256":"56c0966494cd19b629140f8b8d53e8b9838e34668a5cab6b7cccb41b78734c50","until":null},{"from":null,"id":"ecommerce.progression.promotion-checkout-hooks","modes":["upgrade"],"order":3950,"owners":["ecommerce.progression.promotion-checkout"],"path":"contracts/promotion-checkout.md","sha256":"b74e703dc77c9eeec26f5beb41cab102570c8314d817015c858557e9946293aa","until":null},{"from":null,"id":"ecommerce.progression.stock-alert-hooks","modes":["upgrade"],"order":3990,"owners":["ecommerce.progression.stock-alerts"],"path":"contracts/stock-alerts.md","sha256":"405f106569c84b3062f8159452730f16adadf12d804c554a570e39b958b63421","until":null},{"from":null,"id":"ecommerce.l3.order-return-hooks","modes":["upgrade"],"order":4041,"owners":["ecommerce.l3.order-returns-features"],"path":"contracts/order-returns.md","sha256":"fe0668ab3bf00d4c167794dbecb75b26b23cb62dfc0a1b79636a013edb237329","until":null},{"from":null,"id":"ecommerce.progression.faceted-search-hooks","modes":["upgrade"],"order":4900,"owners":["ecommerce.progression.faceted-search"],"path":"contracts/faceted-search.md","sha256":"08ace4d2d1947f16ceb4da95e6993086c9bd77a0977c258fef1f0a96c7e7879d","until":null},{"from":null,"id":"ecommerce.progression.order-support-hooks","modes":["upgrade"],"order":4900,"owners":["ecommerce.progression.order-support"],"path":"contracts/order-support.md","sha256":"a2901e66af870055b6c3c26f240132ae08f3cb93d0c52a7b2573c9dc153e1e03","until":null},{"from":null,"id":"ecommerce.progression.personalized-recommendation-hooks","modes":["upgrade"],"order":4910,"owners":["ecommerce.progression.personalized-recommendations"],"path":"contracts/progression-personalized-recommendations.md","sha256":"42389a6d284dcfdb53a610ce5ce77d40dbc2b9d5e177bbd3f6bccc40079c7d98","until":null},{"from":null,"id":"ecommerce.progression.promotion-reporting-hooks","modes":["upgrade"],"order":4950,"owners":["ecommerce.progression.promotion-reporting"],"path":"contracts/promotion-reporting.md","sha256":"81cded5fd9a8822c8756ea50bedb879971c94f8727d176f22fa53d2a6a1efbb0","until":null},{"from":null,"id":"ecommerce.feature.store-credit.interface","modes":["upgrade"],"order":5900,"owners":["ecommerce.feature.store-credit"],"path":"contracts/store-credit.md","sha256":"505ca48dfc4f9f2d6ea38e53e1bd382910264d0195be8580b5fb01f7e7074713","until":null},{"from":null,"id":"ecommerce.feature.subscriptions.interface","modes":["upgrade"],"order":5900,"owners":["ecommerce.feature.subscriptions"],"path":"contracts/subscriptions.md","sha256":"dc533bc7f3b9be03ae7d9e18c2e02ffa996bf934b9ba230826d492b4cefa045e","until":null},{"from":null,"id":"ecommerce.progression.delivery-notification-hooks","modes":["upgrade"],"order":5900,"owners":["ecommerce.progression.delivery-notifications"],"path":"contracts/delivery-notifications.md","sha256":"4bf565ea751bb5278bae5910420beeb531a2eb88fb5755e5a06dda68860af490","until":null},{"from":null,"id":"ecommerce.progression.support-refund-hooks","modes":["upgrade"],"order":5900,"owners":["ecommerce.progression.support-refunds"],"path":"contracts/support-refunds.md","sha256":"da5d0e69bbf3a028dc79f207b940bbd4a79e7f82c68d89f252eb8c73ea16b8bc","until":null},{"from":null,"id":"ecommerce.progression.automatic-reorder-hooks","modes":["upgrade"],"order":5910,"owners":["ecommerce.progression.automatic-reorder"],"path":"contracts/progression-automatic-reorder.md","sha256":"3e70361965e6ddeb7b05cc7ad5348e363d58d628734de96b8a9792e68e71cc72","until":null},{"from":null,"id":"ecommerce.progression.cart-recovery-hooks","modes":["upgrade"],"order":5920,"owners":["ecommerce.progression.cart-recovery"],"path":"contracts/progression-cart-recovery.md","sha256":"daf04aea3082483caa2ebf3a3c89a438329600cb6aa5d74fb47d97bc244e5747","until":null},{"from":null,"id":"ecommerce.progression.recommendation-feedback-hooks","modes":["upgrade"],"order":5930,"owners":["ecommerce.progression.recommendation-feedback"],"path":"contracts/recommendation-feedback.md","sha256":"00a96f594c8da1ffb4a933c6f8d8d0219216ab6cc2c781cfd17dd9c97c517d58","until":null},{"from":null,"id":"ecommerce.feature.split-tender-refunds.interface","modes":["upgrade"],"order":6900,"owners":["ecommerce.feature.split-tender-refunds"],"path":"contracts/split-tender-refunds.md","sha256":"62b8b3c233d33858e9b696d06d94c70f6524f8cbed77028acc0d2affdd780841","until":null},{"from":null,"id":"ecommerce.interface.product-bundles","modes":["upgrade"],"order":8104,"owners":["ecommerce.feature.product-bundles"],"path":"contracts/product-bundles.md","sha256":"b27905002b693e6c63532cd8193c8e3fd1fdeaad85041d84b8240a2092bf1c57","until":null},{"from":null,"id":"ecommerce.interface.bundle-checkout","modes":["upgrade"],"order":8105,"owners":["ecommerce.feature.bundle-checkout"],"path":"contracts/bundle-checkout.md","sha256":"2c4e70b620934dc927ad4c84c17e54b6e9cef665ba3b098f9f2fc9e79a3c4c3e","until":null},{"from":null,"id":"ecommerce.interface.bundle-returns","modes":["upgrade"],"order":8106,"owners":["ecommerce.feature.bundle-returns"],"path":"contracts/bundle-returns.md","sha256":"69f8bf6afc325a98bab38235e49eaa2875bc674e6a61c7559692593018c9f527","until":null}],"mode":"action","requirementSha256":"bd057cbfbcae06db5bdd6a7eb20d03ae00f9c16cff9841199b342a7df5b19279","requirements":[{"from":null,"id":"ecommerce.progression.fresh","modes":["fresh"],"order":0,"owners":["recipe"],"path":"prompts/modular/progression-framing.md","sha256":"7c3267ef4ef6a454ba26217d8a0efc47f1cbe655970fbb1ddfd2ae7637e17df1","until":"## Existing application"},{"from":"## Existing application","id":"ecommerce.progression.upgrade","modes":["upgrade"],"order":0,"owners":["recipe"],"path":"prompts/modular/progression-framing.md","sha256":"ad2dc327dea4410a715b013d9455c37060a193b844c7de7967ce760ef1c2c118","until":null},{"from":null,"id":"ecommerce.l2.inventory-dashboard","modes":["upgrade"],"order":50,"owners":["ecommerce.l2.inventory-dashboard"],"path":"prompts/modular/inventory-dashboard.md","sha256":"d2190c809ce459141cfdb716d25d38b01f7aca2ce8b8f74128e731e4e3fede07","until":null},{"from":null,"id":"ecommerce.l2.sales-dashboard","modes":["upgrade"],"order":60,"owners":["ecommerce.l2.sales-dashboard"],"path":"prompts/modular/sales-dashboard.md","sha256":"c65aaadb1f8da6e2b5f7967cc3bcf07beb0e7a860351e866373667bc33012dbd","until":null},{"from":null,"id":"ecommerce.l2.recommendations","modes":["upgrade"],"order":70,"owners":["ecommerce.l2.recommendations"],"path":"prompts/modular/recommendations.md","sha256":"0eed3feb6fc6acf86260d74eeff557b15837aaa09a857c63cf9dee848cd7201f","until":null},{"from":null,"id":"ecommerce.feature.accounts.requirement","modes":["fresh","upgrade"],"order":100,"owners":["ecommerce.feature.accounts"],"path":"prompts/modular/accounts.md","sha256":"0a8dee0847a02da7777c11c4175577c4533fafa538fc09e5124cf4d825e4c96f","until":null},{"from":null,"id":"ecommerce.feature.catalog-items.requirement","modes":["fresh","upgrade"],"order":200,"owners":["ecommerce.feature.catalog-items"],"path":"prompts/modular/catalog-items.md","sha256":"36a72da77f42d856b691c93dc587fee6d9d7f88b722a9893fc226b1d5bb5d3fe","until":null},{"from":null,"id":"ecommerce.feature.catalog-discovery.requirement","modes":["fresh","upgrade"],"order":210,"owners":["ecommerce.feature.catalog-discovery"],"path":"prompts/modular/catalog-discovery.md","sha256":"9a0785c55d17e577b84ef257de482bcd8bc8c961e9d0760c38472936a1a0062b","until":null},{"from":null,"id":"ecommerce.feature.purchasing.requirement","modes":["fresh","upgrade"],"order":300,"owners":["ecommerce.feature.purchasing"],"path":"prompts/modular/purchasing.md","sha256":"059073391cf92eef2fb1edb710ec1a62747e82115c2bd0251274111b92f93e46","until":null},{"from":null,"id":"ecommerce.feature.cart.requirement","modes":["fresh","upgrade"],"order":400,"owners":["ecommerce.feature.cart"],"path":"prompts/modular/cart.md","sha256":"da024de4032661a46574c8ac8dbe3d1baf45ef6ea7115ba1976d7d5c1044afbe","until":null},{"from":null,"id":"ecommerce.feature.checkout.requirement","modes":["fresh","upgrade"],"order":410,"owners":["ecommerce.feature.checkout"],"path":"prompts/modular/checkout.md","sha256":"bb834dec7858349a17a8980eb13459a606cdbd2b1c534f468f666459c1a24ba7","until":null},{"from":null,"id":"ecommerce.feature.reviews.requirement","modes":["fresh","upgrade"],"order":500,"owners":["ecommerce.feature.reviews"],"path":"prompts/modular/reviews.md","sha256":"1649822528e1bbcfedc59b3c8c5d7ff1b5f0e42e1e82b81ee7e59361cbefab33","until":null},{"from":null,"id":"ecommerce.feature.warehouse-admin.requirement","modes":["fresh","upgrade"],"order":600,"owners":["ecommerce.feature.warehouse-admin"],"path":"prompts/modular/warehouse-administration.md","sha256":"8fa2d7b54b3c3d9601ae69bbfee5bdf389f68fea0d0ef4032201af12d3f424a2","until":null},{"from":"## Access control: purchasing","id":"ecommerce.spec.access-control.purchasing","modes":["fresh"],"order":1000,"owners":["ecommerce.spec.access-control"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.purchasing"],"sha256":"45c2ac9e423af1431c6a9d8b7de707b7d6a8402c004c5472c56d7efe387e61a1","until":"## Access control: warehouse administration"},{"from":"## Access control: warehouse administration","id":"ecommerce.spec.access-control.warehouse-admin","modes":["fresh"],"order":1010,"owners":["ecommerce.spec.access-control"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.warehouse-admin"],"sha256":"02500e3bded8076287f0b4c7264fe48ff3c766b0ab9aa386497c5cc66fc725f4","until":"## Access control: reviews"},{"from":"## Access control: reviews","id":"ecommerce.spec.access-control.reviews","modes":["fresh"],"order":1020,"owners":["ecommerce.spec.access-control"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.reviews"],"sha256":"10d4140e8f649ddf46843100231683f13143c09226195cfe487827b662333f4d","until":"## Access control: cart"},{"from":"## Access control: cart","id":"ecommerce.spec.access-control.cart","modes":["fresh"],"order":1030,"owners":["ecommerce.spec.access-control"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.cart"],"sha256":"353e3d5daa87c988456b392ad53fd93bdf473b5a860049104cd5acdaddf05cee","until":"## State durability: accounts"},{"from":null,"id":"ecommerce.progression.staff-access","modes":["fresh","upgrade"],"order":1100,"owners":["ecommerce.progression.staff-access"],"path":"prompts/modular/staff-access.md","sha256":"10c6aedbc5c60b440d6d0b5c38bb57363ae021c54d62ee2ffe2294d7f6741550","until":null},{"from":"## State durability: accounts","id":"ecommerce.spec.state-durability.accounts","modes":["fresh"],"order":1100,"owners":["ecommerce.spec.state-durability"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.accounts"],"sha256":"3466c8854dd16c38beafe194fb402f8f5c5539defb90a9b856c357347255c3ee","until":"## State durability: account data"},{"from":"## State durability: account data","id":"ecommerce.spec.state-durability.account-data","modes":["fresh"],"order":1110,"owners":["ecommerce.spec.state-durability"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.accounts","ecommerce.feature.cart"],"sha256":"db7e1e0fbf6e6f2956f28ef00dfc7191c90ca05349a605f246776ff3c6f4e2d9","until":"## Live state: catalog and purchasing"},{"from":null,"id":"ecommerce.spec.state-durability.checkout-crash","modes":["fresh","upgrade"],"order":1120,"owners":["ecommerce.spec.state-durability"],"path":"prompts/modular/checkout-recovery-specification.md","requiresFeatures":["ecommerce.feature.checkout"],"sha256":"0e851fea800d81cb558f8004f04e628e5ccf9f6e5589a9d2f6fceaf33e50f207","until":null},{"from":null,"id":"ecommerce.progression.customer-profile","modes":["upgrade"],"order":1200,"owners":["ecommerce.progression.customer-profile"],"path":"prompts/modular/customer-profile.md","sha256":"f4ffc5d2a642bbccec9eb5c428a37b2d9bfaa1c9f96648d8544bfcf2fe8187bb","until":null},{"from":null,"id":"ecommerce.progression.support-intake","modes":["fresh","upgrade"],"order":1200,"owners":["ecommerce.progression.support-intake"],"path":"prompts/modular/support-intake.md","sha256":"048f11b71b66dadf80c77b5d9aef8bbff2b2030171eeda5012dce74ae16421ed","until":null},{"from":"## Live state: catalog and purchasing","id":"ecommerce.spec.live-state.catalog-purchasing","modes":["fresh"],"order":1200,"owners":["ecommerce.spec.live-state"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing"],"sha256":"3bfd8cf26509ffeeb475df0eee040f4743cc31acd9e8292e4144b46055686b55","until":"## Live state: cart"},{"from":"## Live state: cart","id":"ecommerce.spec.live-state.cart","modes":["fresh"],"order":1210,"owners":["ecommerce.spec.live-state"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.cart"],"sha256":"244ee5cf7387c74dd1776e659964e782f042229eff84918de2517781ad095045","until":"## Live state: reviews"},{"from":"## Live state: reviews","id":"ecommerce.spec.live-state.reviews","modes":["fresh"],"order":1220,"owners":["ecommerce.spec.live-state"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.reviews"],"sha256":"aeeab5a0dda0f1df6b1ab3ae78d3c1c6a9ff2f108ac1d09e904c27b7ee71100d","until":"## Live state: warehouse administration"},{"from":"## Live state: warehouse administration","id":"ecommerce.spec.live-state.warehouse-admin","modes":["fresh"],"order":1230,"owners":["ecommerce.spec.live-state"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.warehouse-admin"],"sha256":"362d4ef3422a09d060cdd5b09837b0a4a1f67b8fb108f59dcf7afb4a12d2a4e5","until":"## Concurrency safety: purchasing"},{"from":null,"id":"ecommerce.progression.staff-roles","modes":["upgrade"],"order":1250,"owners":["ecommerce.progression.staff-roles"],"path":"prompts/modular/staff-roles.md","sha256":"175a323c92238509111aa9c366e4ec302d508edc872077ecd26d8acbcbe21ba9","until":null},{"from":null,"id":"ecommerce.progression.catalog-management","modes":["upgrade"],"order":1300,"owners":["ecommerce.progression.catalog-management"],"path":"prompts/modular/catalog-management.md","sha256":"93fece29e1d68513732398c815fe4ebc83079e705e2b9de40df543281439f390","until":null},{"from":"## Concurrency safety: purchasing","id":"ecommerce.spec.concurrency-safety.purchasing","modes":["fresh"],"order":1300,"owners":["ecommerce.spec.concurrency-safety"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.purchasing"],"sha256":"e2995c00f9971e1389bc03b56faf4e03da9c83a84cb205ddfeb84964bd4a905f","until":"## Concurrency safety: restocking"},{"from":"## Concurrency safety: restocking","id":"ecommerce.spec.concurrency-safety.restocking","modes":["fresh"],"order":1310,"owners":["ecommerce.spec.concurrency-safety"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"sha256":"958e12514579d21f92a73806ba6983912f1828f40f51efeb16947e7a6c3d539e","until":"## Concurrency safety: checkout"},{"from":"## Concurrency safety: checkout","id":"ecommerce.spec.concurrency-safety.checkout","modes":["fresh"],"order":1320,"owners":["ecommerce.spec.concurrency-safety"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.checkout"],"sha256":"0bb644068b430d491cfd6c6f5af00cf16cff7ec930a1d8b3150493e4d0bebbe5","until":"## Transactional integrity: reviews"},{"from":null,"id":"ecommerce.progression.payment-records","modes":["upgrade"],"order":1350,"owners":["ecommerce.progression.payment-records"],"path":"prompts/modular/payment-records.md","sha256":"ab5c9c913041e324cad82fe6a135d78906818afa3c9a5c204289142c1d696f09","until":null},{"from":null,"id":"ecommerce.progression.staff-activity","modes":["upgrade"],"order":1400,"owners":["ecommerce.progression.staff-activity"],"path":"prompts/modular/staff-activity.md","sha256":"0cb274acefc0d75cd82415de0eb167133db0412e1de4f0fd59f62983d29a1476","until":null},{"from":"## Transactional integrity: reviews","id":"ecommerce.spec.transactional-integrity.reviews","modes":["fresh"],"order":1400,"owners":["ecommerce.spec.transactional-integrity"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.reviews"],"sha256":"715afabddf7a8ed221f38fdba2b89dad7d0b656c62f5e607153ef39ec52e2fc0","until":"## Transactional integrity: purchasing"},{"from":"## Transactional integrity: purchasing","id":"ecommerce.spec.transactional-integrity.purchasing","modes":["fresh"],"order":1410,"owners":["ecommerce.spec.transactional-integrity"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.purchasing"],"sha256":"1452a3c92e5a35b5bed9273e146de8d1dda3011105ec5d79821462629ef56c8d","until":"## Transactional integrity: warehouse accounting"},{"from":"## Transactional integrity: warehouse accounting","id":"ecommerce.spec.transactional-integrity.warehouse-accounting","modes":["fresh"],"order":1420,"owners":["ecommerce.spec.transactional-integrity"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"sha256":"667b2197f1e9c20e22f03abc7cc756b98b421377adfdb3fd06f35005837c3948","until":"## External data synchronization"},{"from":"## External data synchronization","id":"ecommerce.spec.external-data-sync.requirement","modes":["fresh","upgrade"],"order":1500,"owners":["ecommerce.spec.external-data-sync"],"path":"prompts/modular/l1-external-sync-specifications.md","requiresFeatures":["ecommerce.feature.warehouse-admin"],"sha256":"0e3db28232aedb57b3fb2fbb9afa3ca65fcad28b8c050e1965feee4db32e4239","until":null},{"from":null,"id":"ecommerce.progression.fulfilment-queue","modes":["upgrade"],"order":2020,"owners":["ecommerce.progression.fulfilment-queue"],"path":"prompts/modular/operations-access.md","sha256":"1699a8d0466e115874e49b8ae73bb0004c3c620a03d3aa04dd83818b6c798003","until":null},{"from":null,"id":"ecommerce.l2.stock-transfer","modes":["upgrade"],"order":2030,"owners":["ecommerce.l2.stock-transfers-features"],"path":"prompts/modular/stock-transfers.md","sha256":"6e2f0735d81fd5376b851893404873af4807b83e807e078cca0b8c9a9920e627","until":null},{"from":null,"id":"ecommerce.l2.order-cancellation","modes":["upgrade"],"order":2040,"owners":["ecommerce.l2.order-cancellation-features"],"path":"prompts/modular/order-cancellation.md","sha256":"7d429d4e8db546961e7ec9a12fb5dcba5efd0562c8059ab354121fd5ed13ea84","until":null},{"from":null,"id":"ecommerce.l2.price-history","modes":["upgrade"],"order":2040,"owners":["ecommerce.l2.price-history-features"],"path":"prompts/modular/price-history.md","sha256":"9edb3db98af12f3a08cae7887dd6da91c746be49597f086457fc2ecb70c558b8","until":null},{"from":"## Price history: completed orders","id":"ecommerce.progression.price-history-orders","modes":["upgrade"],"order":2041,"owners":["ecommerce.progression.price-history-specifications"],"path":"prompts/modular/price-history-orders.md","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.price-history-features"],"sha256":"12abf44d08a49b076c721b62c4fc240a82c0fb350b10a3eb052afcf2ee78c6b6","until":"## Price history: cart and checkout"},{"from":"## Price history: cart and checkout","id":"ecommerce.progression.price-history-cart-checkout","modes":["upgrade"],"order":2042,"owners":["ecommerce.progression.price-history-specifications"],"path":"prompts/modular/price-history-orders.md","requiresFeatures":["ecommerce.feature.checkout","ecommerce.l2.price-history-features"],"sha256":"86caeac95b2d3cb1fc0ec0d467ee52303218d087ca30c24eb38c7c7714c8ec2b","until":null},{"from":null,"id":"ecommerce.progression.cancellation-queue","modes":["upgrade"],"order":2045,"owners":["ecommerce.progression.cancellation-queue-specifications"],"path":"prompts/modular/cancellation-queue.md","requiresFeatures":["ecommerce.l2.order-cancellation-features","ecommerce.progression.fulfilment-queue"],"sha256":"7e1adf98391a930017acb5c37d46c9b0d17d89e58c939fc57dc52f22e2f86f31","until":null},{"from":"## Cancellation accounting","id":"ecommerce.progression.cancellation-accounting","modes":["upgrade"],"order":2100,"owners":["ecommerce.progression.cancellation-accounting-specifications"],"path":"prompts/modular/order-accounting.md","requiresFeatures":["ecommerce.l2.order-cancellation-features"],"sha256":"a31e83bef57e79d545be549b9905d1b1f178e40a34e676ec18a62dfddf916d8d","until":"## Price accounting"},{"from":null,"id":"ecommerce.progression.support-triage","modes":["upgrade"],"order":2100,"owners":["ecommerce.progression.support-triage"],"path":"prompts/modular/support-triage.md","sha256":"bbb35abcbae9b8d3782ee3eb94f3175428edfe3fa564d6b0e3af2e0050e10dee","until":null},{"from":"## Price accounting","id":"ecommerce.progression.price-accounting","modes":["upgrade"],"order":2110,"owners":["ecommerce.progression.price-accounting-specifications"],"path":"prompts/modular/order-accounting.md","requiresFeatures":["ecommerce.l2.price-history-features"],"sha256":"64f96277738b0a33453e69bdcb1d79971d979a57ec27b8caacc96b499bc43de5","until":null},{"from":null,"id":"ecommerce.progression.support-history","modes":["upgrade"],"order":2110,"owners":["ecommerce.progression.support-history"],"path":"prompts/modular/support-history.md","sha256":"df2ca2664901a90814eab17d5f7c029595ef614d76ee29eeb4204d9f4f371429","until":null},{"from":null,"id":"ecommerce.progression.promotion-rules","modes":["upgrade"],"order":2200,"owners":["ecommerce.progression.promotion-rules"],"path":"prompts/modular/promotion-rules.md","sha256":"7a43b632e8fd43b8060100f9614476e41003e47afa04dd1b7c3b09639f6280b2","until":null},{"from":null,"id":"ecommerce.progression.notification-preferences","modes":["upgrade"],"order":2300,"owners":["ecommerce.progression.notification-preferences"],"path":"prompts/modular/notification-preferences.md","sha256":"496baa70ef300e6874f6dbf58ccd2bac594e53101875fd68740e27779d11871b","until":null},{"from":null,"id":"ecommerce.progression.transfer-authorization","modes":["upgrade"],"order":2930,"owners":["ecommerce.progression.operations-access-specifications"],"path":"prompts/modular/transfer-authorization.md","requiresFeatures":["ecommerce.l2.stock-transfers-features"],"sha256":"6d1e33c7a6aa209ad3f2705607a839dde5c915b65d78a8bb10b9e0da29172c0b","until":null},{"from":null,"id":"ecommerce.progression.price-authorization","modes":["upgrade"],"order":2940,"owners":["ecommerce.progression.operations-access-specifications"],"path":"prompts/modular/price-authorization.md","requiresFeatures":["ecommerce.l2.price-history-features"],"sha256":"6e9f949bd45f3bc816336de6e7db6dd18254ceebe6ecacc8e0abfd5f3887b8b5","until":null},{"from":null,"id":"ecommerce.progression.shipping-authorization","modes":["upgrade"],"order":2950,"owners":["ecommerce.progression.operations-access-specifications"],"path":"prompts/modular/shipping-authorization.md","requiresFeatures":["ecommerce.progression.fulfilment-queue"],"sha256":"f336e9c8e1ea425c3dc44e72fabbc5e9e0e2a3541cab3750dd8e112b6c7997f8","until":null},{"from":null,"id":"ecommerce.progression.order-ownership","modes":["upgrade"],"order":2960,"owners":["ecommerce.progression.operations-access-specifications"],"path":"prompts/modular/order-ownership.md","requiresFeatures":["ecommerce.l2.order-cancellation-features"],"sha256":"1f4e5149768d1d0a4917ca9b711f1f40bf2c783ee5d59a42ab307f01babb052e","until":null},{"from":null,"id":"ecommerce.progression.review-access","modes":["upgrade"],"order":2970,"owners":["ecommerce.progression.review-access-specifications"],"path":"prompts/modular/review-access.md","requiresFeatures":["ecommerce.feature.reviews"],"sha256":"430fe60b2f028f67eab9f23b572238d4dacc60f7b89d74f8030618368455f371","until":null},{"from":null,"id":"ecommerce.progression.transfer-conservation","modes":["upgrade"],"order":2970,"owners":["ecommerce.progression.inventory-conservation-specifications"],"path":"prompts/modular/stock-conservation.md","requiresFeatures":["ecommerce.l2.stock-transfers-features"],"sha256":"4551438e045bb65b44127d3f92dcb79548c78dc1ecc42d2c7807a599d6c59c8f","until":null},{"from":null,"id":"ecommerce.progression.cancellation-conservation","modes":["upgrade"],"order":2980,"owners":["ecommerce.progression.inventory-conservation-specifications"],"path":"prompts/modular/cancellation-conservation.md","requiresFeatures":["ecommerce.l2.order-cancellation-features"],"sha256":"f383c20f16d75b77dfe2c0ca10dc1c77c998feb80b7c327616512b486c4714f8","until":null},{"from":null,"id":"ecommerce.l3.reservations","modes":["upgrade"],"order":3000,"owners":["ecommerce.l3.reservations-features"],"path":"prompts/modular/reservations.md","sha256":"e45b53568066c7200486453f78fdbbe89a116259a4d83d9435bde350316595e0","until":null},{"from":null,"id":"ecommerce.l3.scheduled-restocks","modes":["upgrade"],"order":3010,"owners":["ecommerce.l3.scheduled-restocks-features"],"path":"prompts/modular/scheduled-restocks.md","sha256":"ba32547038c681f7214219e5a10df75dbf925ee19d4ba60db5b291570f9c16ec","until":null},{"from":null,"id":"ecommerce.l3.order-delivery","modes":["upgrade"],"order":3020,"owners":["ecommerce.l3.order-delivery-features"],"path":"prompts/modular/order-delivery.md","sha256":"40483d0bf91c81efe57a61d62dbda756498792383bed9af595cdd8878a0b0cb8","until":null},{"from":null,"id":"ecommerce.l3.cart-expiration","modes":["upgrade"],"order":3030,"owners":["ecommerce.l3.cart-expiration-features"],"path":"prompts/modular/cart-expiration.md","sha256":"7c66696f044249d5f7f31487efb85e83f4c06b862fbaa3dd142ef2aef6afc2b7","until":null},{"from":"## Durable reservations","id":"ecommerce.l3.durable-reservations","modes":["upgrade"],"order":3100,"owners":["ecommerce.l3.deferred-durability-specifications"],"path":"prompts/modular/l3-specifications.md","requiresFeatures":["ecommerce.l3.reservations-features"],"sha256":"18ec31e622e6d392c68eacef275c6595125aa9bc45296e6064be50353b228bc6","until":"## Durable restocks"},{"from":null,"id":"ecommerce.progression.managed-support","modes":["upgrade"],"order":3100,"owners":["ecommerce.progression.managed-support"],"path":"prompts/modular/managed-support.md","sha256":"33abfe9e7514c6826df60572557d54f2e1570477afeae14c64bb3c5f3bad5fb6","until":null},{"from":"## Durable restocks","id":"ecommerce.l3.durable-restocks","modes":["upgrade"],"order":3101,"owners":["ecommerce.l3.deferred-durability-specifications"],"path":"prompts/modular/l3-specifications.md","requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"sha256":"40c3fac2bde0e10b85416906bcd0080e9748c916820ca44918074f2d342f7deb","until":"## Durable order delivery"},{"from":"## Durable order delivery","id":"ecommerce.l3.durable-order-delivery","modes":["upgrade"],"order":3102,"owners":["ecommerce.l3.deferred-durability-specifications"],"path":"prompts/modular/l3-specifications.md","requiresFeatures":["ecommerce.l3.order-delivery-features"],"sha256":"56a0e9d3113f8c49d563d6a99940e8ee272c32adb841c1dbb5533b17705a1e69","until":"## Durable cart expiration"},{"from":"## Durable cart expiration","id":"ecommerce.l3.durable-cart-expiration","modes":["upgrade"],"order":3103,"owners":["ecommerce.l3.deferred-durability-specifications"],"path":"prompts/modular/l3-specifications.md","requiresFeatures":["ecommerce.l3.cart-expiration-features"],"sha256":"1c81d51182e1a7648dc8e1e34cf1e80aade556a1eaeb63f79ba49b50bf2c2997","until":"## Exactly-once restocks"},{"from":"## Exactly-once restocks","id":"ecommerce.l3.exactly-once-restocks","modes":["upgrade"],"order":3110,"owners":["ecommerce.l3.deferred-integrity-specifications"],"path":"prompts/modular/l3-specifications.md","requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"sha256":"2fe8adca82a8fd5bb0fa1ba02cbb88e9f4ae3c799c49cf0980451d85603b6515","until":"## Exactly-once delivery"},{"from":"## Exactly-once delivery","id":"ecommerce.l3.exactly-once-delivery","modes":["upgrade"],"order":3111,"owners":["ecommerce.l3.deferred-integrity-specifications"],"path":"prompts/modular/l3-specifications.md","requiresFeatures":["ecommerce.l3.order-delivery-features"],"sha256":"70fa7a5b7e6a074950652a574684c4230545cd3b3b21645e568edf45dea5d44f","until":"## Server-timed restocks"},{"from":"## Server-timed restocks","id":"ecommerce.l3.server-timed-restocks","modes":["upgrade"],"order":3120,"owners":["ecommerce.l3.server-time-specifications"],"path":"prompts/modular/l3-specifications.md","requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"sha256":"ffb402340b8e0afa1973da971a48ed9534bf615e36c213dc892f7dc7260a9da3","until":"## Server-timed reservations"},{"from":"## Server-timed reservations","id":"ecommerce.l3.server-timed-reservations","modes":["upgrade"],"order":3121,"owners":["ecommerce.l3.server-time-specifications"],"path":"prompts/modular/l3-specifications.md","requiresFeatures":["ecommerce.l3.reservations-features"],"sha256":"108cb53bdda3f4a651347cec50a2236bb5cbbb01b9f92943fd6d287f151789bc","until":"## Deferred-work access"},{"from":"## Deferred-work access","id":"ecommerce.l3.deferred-access","modes":["upgrade"],"order":3130,"owners":["ecommerce.l3.deferred-access-specifications"],"path":"prompts/modular/l3-specifications.md","requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"sha256":"72874aad78d6da1f467443015f53f82514e45866b09a17c6435b16ec72b4e26f","until":"## Stock conservation"},{"from":"## Stock conservation","id":"ecommerce.l3.stock-conservation","modes":["upgrade"],"order":3140,"owners":["ecommerce.l3.deferred-integrity-specifications"],"path":"prompts/modular/l3-specifications.md","requiresFeatures":["ecommerce.l3.reservations-features"],"sha256":"cf7ef3381b7b3d4beee90d314b4a00d667a89909b67d6fd57f453329a6c48be3","until":null},{"from":null,"id":"ecommerce.progression.promotion-checkout","modes":["upgrade"],"order":3200,"owners":["ecommerce.progression.promotion-checkout"],"path":"prompts/modular/promotion-checkout.md","sha256":"1979ce8991b38fab30c4ea0d0fb7adba51d79bb82981059b0dafded82fed843a","until":null},{"from":null,"id":"ecommerce.progression.stock-alerts","modes":["upgrade"],"order":3300,"owners":["ecommerce.progression.stock-alerts"],"path":"prompts/modular/stock-alerts.md","sha256":"3478629a2831e04baca5eec1d3a2a3695c03f6044c36b43acb7c8ca7e35e9867","until":null},{"from":null,"id":"ecommerce.progression.faceted-search","modes":["upgrade"],"order":4000,"owners":["ecommerce.progression.faceted-search"],"path":"prompts/modular/faceted-search.md","sha256":"5dc13b47abcbc85654c82148f44b854979d868578db50db42e060c773edc2207","until":null},{"from":null,"id":"ecommerce.progression.personalized-recommendations","modes":["upgrade"],"order":4010,"owners":["ecommerce.progression.personalized-recommendations"],"path":"prompts/modular/progression-personalized-recommendations.md","sha256":"6c78b3963318b18da11173f96bed5cb8e5537f49e0008342ff11209fada3e099","until":null},{"from":null,"id":"ecommerce.spec.search-ordering","modes":["fresh","upgrade"],"order":4010,"owners":["ecommerce.spec.search-ordering"],"path":"prompts/modular/search-ordering-specification.md","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.faceted-search"],"sha256":"77d88f2b03c71b6cdc8350020ffd7815cf0bee87e4155c1aa00beb9bb81a83f8","until":null},{"from":null,"id":"ecommerce.l3.order-returns","modes":["upgrade"],"order":4040,"owners":["ecommerce.l3.order-returns-features"],"path":"prompts/modular/order-returns.md","sha256":"7277266f1e8c7d660ee38ba55ac1359200e56c695862c6d7d755d6ec6a6cb808","until":null},{"from":null,"id":"ecommerce.progression.order-support","modes":["upgrade"],"order":4100,"owners":["ecommerce.progression.order-support"],"path":"prompts/modular/order-support.md","sha256":"f515fe9827373e110633245df158ce8f78ca040f6c25a059d3885182515abb9a","until":null},{"from":null,"id":"ecommerce.progression.promotion-reporting","modes":["upgrade"],"order":4200,"owners":["ecommerce.progression.promotion-reporting"],"path":"prompts/modular/promotion-reporting.md","sha256":"3285eb0a50ceeed40b3415bdab8d6f5fc4138e496d2062a08c5ee2fac70157c6","until":null},{"from":null,"id":"ecommerce.progression.delivery-notifications","modes":["upgrade"],"order":5000,"owners":["ecommerce.progression.delivery-notifications"],"path":"prompts/modular/delivery-notifications.md","sha256":"12d6db9061749779dcb3db699a492781b34ae955c4c11cbb3e5dc2c8e0300ef3","until":null},{"from":null,"id":"ecommerce.progression.automatic-reorder","modes":["upgrade"],"order":5010,"owners":["ecommerce.progression.automatic-reorder"],"path":"prompts/modular/progression-automatic-reorder.md","sha256":"64751064e699c709a6550daba6fee8f019f445b817910e1ed0d8831632fa2471","until":null},{"from":null,"id":"ecommerce.progression.cart-recovery","modes":["upgrade"],"order":5020,"owners":["ecommerce.progression.cart-recovery"],"path":"prompts/modular/progression-cart-recovery.md","sha256":"4dc64aaa87d54a05bd7cce31bdc98254e605251d14f1688b859e9aa908da6a8b","until":null},{"from":null,"id":"ecommerce.progression.recommendation-feedback","modes":["upgrade"],"order":5030,"owners":["ecommerce.progression.recommendation-feedback"],"path":"prompts/modular/recommendation-feedback.md","sha256":"072a548ae797773041e86cbc62f39011559ca7ab40441bee6b77ec1854cd7f51","until":null},{"from":null,"id":"ecommerce.feature.store-credit","modes":["upgrade"],"order":5100,"owners":["ecommerce.feature.store-credit"],"path":"prompts/modular/store-credit.md","sha256":"d769da6c0ef4eb2fa6ac241aeddb61e8e2d85a0f1fcfd595fbf2c6660f67cd7b","until":null},{"from":null,"id":"ecommerce.feature.subscriptions","modes":["upgrade"],"order":5100,"owners":["ecommerce.feature.subscriptions"],"path":"prompts/modular/subscriptions.md","sha256":"ca93ab01c0d23c2fb3c035981372dc1254b219f1875b9566c0d01123176cc7c1","until":null},{"from":null,"id":"ecommerce.progression.support-refunds","modes":["upgrade"],"order":5100,"owners":["ecommerce.progression.support-refunds"],"path":"prompts/modular/support-refunds.md","sha256":"79090320de03a74660418da9e9adbf8170f8c574be990faeaf3d7df652221aa6","until":null},{"from":"## automatic-reorder-access","id":"ecommerce.spec.access-control.automatic-reorder-access","modes":["upgrade"],"order":6001,"owners":["ecommerce.spec.access-control"],"path":"prompts/modular/production-specifications.md","requiresFeatures":["ecommerce.progression.automatic-reorder"],"sha256":"f93257ed878a9fc27e9a0b80ceb050ab44e5ced5f6ba3d868939e8dee9ed49b1","until":"## recommendation-profile-isolation"},{"from":"## recommendation-profile-isolation","id":"ecommerce.spec.access-control.recommendation-profile-isolation","modes":["upgrade"],"order":6002,"owners":["ecommerce.spec.access-control"],"path":"prompts/modular/production-specifications.md","requiresFeatures":["ecommerce.progression.personalized-recommendations"],"sha256":"36fe3efec8aa2529e48f5f2ef809256a97957554e2a349e669fddcc3d6942a5a","until":"## staff-activity-privacy"},{"from":"## staff-activity-privacy","id":"ecommerce.spec.access-control.staff-activity-privacy","modes":["upgrade"],"order":6003,"owners":["ecommerce.spec.access-control"],"path":"prompts/modular/production-specifications.md","requiresFeatures":["ecommerce.progression.staff-activity"],"sha256":"3d9a91bd3c10abf3594a2dc690077f5a3b1ed95ddb7257c7c49f8e666448949c","until":"## order-support-ownership"},{"from":"## order-support-ownership","id":"ecommerce.spec.access-control.order-support-ownership","modes":["upgrade"],"order":6004,"owners":["ecommerce.spec.access-control"],"path":"prompts/modular/production-specifications.md","requiresFeatures":["ecommerce.progression.order-support"],"sha256":"5e090901a9bbfe7b0f1869be90a16d8d9df30b735effc5cd55e63784443aad05","until":"## delivery-notification-privacy"},{"from":"## delivery-notification-privacy","id":"ecommerce.spec.access-control.delivery-notification-privacy","modes":["upgrade"],"order":6005,"owners":["ecommerce.spec.access-control"],"path":"prompts/modular/production-specifications.md","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.delivery-notifications","ecommerce.progression.fulfilment-queue"],"sha256":"4fbdbdc513c7cc363198ce55780a421261c057f54ac8a44e3a69908d792d2e22","until":"## support-refund-access"},{"from":"## support-refund-access","id":"ecommerce.spec.access-control.support-refund-access","modes":["upgrade"],"order":6006,"owners":["ecommerce.spec.access-control"],"path":"prompts/modular/production-specifications.md","requiresFeatures":["ecommerce.progression.support-refunds"],"sha256":"a4200b390cf7c85d467425834af2642ceb7a80fa9ee53cccdfaa565d6ff69684","until":"## recommendation-feedback-privacy"},{"from":"## recommendation-feedback-privacy","id":"ecommerce.spec.access-control.recommendation-feedback-privacy","modes":["upgrade"],"order":6007,"owners":["ecommerce.spec.access-control"],"path":"prompts/modular/production-specifications.md","requiresFeatures":["ecommerce.progression.recommendation-feedback"],"sha256":"3df273f53ccd54520a89fedb64952b990ff4f61c36d16a50bd1f28e8523471b2","until":"## recommendation-feedback-restart"},{"from":"## recommendation-feedback-restart","id":"ecommerce.spec.state-durability.recommendation-feedback-restart","modes":["upgrade"],"order":6008,"owners":["ecommerce.spec.state-durability"],"path":"prompts/modular/production-specifications.md","requiresFeatures":["ecommerce.progression.recommendation-feedback"],"sha256":"5531638e5123f8c7c76c6f095ba3dadb52128482acc6d325de997ec280656d7c","until":"## automatic-reorder-deduplication"},{"from":"## automatic-reorder-deduplication","id":"ecommerce.spec.transactional-integrity.automatic-reorder-deduplication","modes":["upgrade"],"order":6009,"owners":["ecommerce.spec.transactional-integrity"],"path":"prompts/modular/production-specifications.md","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.automatic-reorder"],"sha256":"de82b9029ce5ae8c90dc1ea20a024da19d8ea55eac6ab36f6fa1d84b8b241b01","until":"## payment-deduplication"},{"from":"## payment-deduplication","id":"ecommerce.spec.transactional-integrity.payment-deduplication","modes":["upgrade"],"order":6010,"owners":["ecommerce.spec.transactional-integrity"],"path":"prompts/modular/production-specifications.md","requiresFeatures":["ecommerce.progression.payment-records"],"sha256":"f33fb61f31726bbe7e8b617ae12d970a8d876d98d30d49191dd0043a58557760","until":"## support-refund-accounting"},{"from":"## support-refund-accounting","id":"ecommerce.spec.transactional-integrity.support-refund-accounting","modes":["upgrade"],"order":6011,"owners":["ecommerce.spec.transactional-integrity"],"path":"prompts/modular/production-specifications.md","requiresFeatures":["ecommerce.progression.support-refunds"],"sha256":"bd44a5a8c9211493896869df9b086177b022d9840f9a362d0c609e3c6bff6877","until":null},{"from":null,"id":"ecommerce.feature.split-tender-refunds","modes":["upgrade"],"order":6100,"owners":["ecommerce.feature.split-tender-refunds"],"path":"prompts/modular/split-tender-refunds.md","sha256":"c21e00af3623136de4ae45f089b45b457a34563eb37a756d5b8b3563d1f19734","until":null},{"from":"## Product bundles","id":"ecommerce.spec.bundle-integrity.product-bundles","modes":["fresh","upgrade"],"order":6500,"owners":["ecommerce.spec.bundle-integrity"],"path":"prompts/modular/later-specifications.md","requiresFeatures":["ecommerce.feature.product-bundles"],"sha256":"7eb9ee1c794134867f6b27ad3b773bbd99d4c860774618d1286d39b4347470fb","until":"## Bundle checkout"},{"from":"## Bundle checkout","id":"ecommerce.spec.bundle-integrity.bundle-checkout","modes":["fresh","upgrade"],"order":6501,"owners":["ecommerce.spec.bundle-integrity"],"path":"prompts/modular/later-specifications.md","requiresFeatures":["ecommerce.feature.bundle-checkout"],"sha256":"630eead6de0dc2d9672e83e64beb8cb5036eccd9ba4aaf7c9284d91c0e42144b","until":"## Bundle returns"},{"from":"## Bundle returns","id":"ecommerce.spec.bundle-integrity.bundle-returns","modes":["fresh","upgrade"],"order":6502,"owners":["ecommerce.spec.bundle-integrity"],"path":"prompts/modular/later-specifications.md","requiresFeatures":["ecommerce.feature.bundle-returns"],"sha256":"7a2264753c8088c30ecc87cb1f98940198fde663b5a4a1bb8de83c410e01e5b2","until":"## Store credit"},{"from":"## Store credit","id":"ecommerce.spec.store-credit.store-credit","modes":["fresh","upgrade"],"order":6503,"owners":["ecommerce.spec.store-credit"],"path":"prompts/modular/later-specifications.md","requiresFeatures":["ecommerce.feature.store-credit"],"sha256":"ee64f04e0f30fc269021083c4b02c26f9fc87321d41d7a7262e68bf6926b7c83","until":"## Split-tender refunds"},{"from":"## Split-tender refunds","id":"ecommerce.spec.split-tender-refunds.split-tender-refunds","modes":["fresh","upgrade"],"order":6504,"owners":["ecommerce.spec.split-tender-refunds"],"path":"prompts/modular/later-specifications.md","requiresFeatures":["ecommerce.feature.split-tender-refunds"],"sha256":"860929486a2d9e283be8486c85b1ee2086936b22de28db17febf5a1e65bc10ff","until":"## Scheduled purchases"},{"from":"## Scheduled purchases","id":"ecommerce.spec.subscriptions.subscriptions","modes":["fresh","upgrade"],"order":6505,"owners":["ecommerce.spec.subscriptions"],"path":"prompts/modular/later-specifications.md","requiresFeatures":["ecommerce.feature.subscriptions"],"sha256":"69f32aaf688858b89cb8fc56352b1674e13fd18b7b9454da3ba9a025e24c90ab","until":null},{"from":null,"id":"ecommerce.feature.product-bundles","modes":["upgrade"],"order":8004,"owners":["ecommerce.feature.product-bundles"],"path":"prompts/modular/product-bundles.md","sha256":"a0805178ebbb20ea290994de65b4ded503c37798af59905d47c1cb2fd4e2c467","until":null},{"from":null,"id":"ecommerce.feature.bundle-checkout","modes":["upgrade"],"order":8005,"owners":["ecommerce.feature.bundle-checkout"],"path":"prompts/modular/bundle-checkout.md","sha256":"d3f21ac08991619b5c9aa6ac99a7554340bae991d5b31a25cdb048cdb314936e","until":null},{"from":null,"id":"ecommerce.feature.bundle-returns","modes":["upgrade"],"order":8006,"owners":["ecommerce.feature.bundle-returns"],"path":"prompts/modular/bundle-returns.md","sha256":"34b3bee882e6a1f6a1527e67d12f8cbdcfd74d026108f9805159123c41015bf2","until":null}]},"title":"Ecommerce progression catalog","track":"ecommerce"},"meaning":{"checks":[{"category":"feature","checkGroupId":"accounts","criterionId":"1a","description":"a visitor can create an account and is signed in as it","featureId":1,"featureName":"Accounts","note":null,"packId":"ecommerce.feature.accounts","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"signUp","name":"ann"},{"contains":"ann","do":"expect"}],"source":"scenarios/01-account-create.json","stableKey":"ecommerce.feature.accounts.accounts.1a","statedBy":"a visitor can create an account with a username and password","withheld":null},{"category":"production","checkGroupId":"accounts","criterionId":"1b","description":"a taken username is refused and does not sign the visitor in as the existing account","featureId":1,"featureName":"Accounts","note":null,"packId":"ecommerce.feature.accounts","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"signUp","expectFailure":true,"name":"ann","password":"different-pw"},{"do":"expect"},{"absent":true,"do":"expect"}],"source":"scenarios/01-account-duplicate.json","stableKey":"ecommerce.feature.accounts.accounts.1b","statedBy":"signing up with a taken username fails with a visible error and must never sign the visitor in as the existing account","withheld":null},{"category":"production","checkGroupId":"accounts","criterionId":"1c","description":"a wrong password is refused","featureId":1,"featureName":"Accounts","note":null,"packId":"ecommerce.feature.accounts","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"signIn","expectFailure":true,"name":"ann","password":"wrong-pw"},{"do":"expect"},{"absent":true,"do":"expect"}],"source":"scenarios/01-account-password.json","stableKey":"ecommerce.feature.accounts.accounts.1c","statedBy":"signing in with a wrong password fails with a visible error","withheld":null},{"category":"production","checkGroupId":"session-reload","criterionId":"1e","description":"the session survives a reload","featureId":1,"featureName":"Accounts","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.accounts"],"role":"guarantee","semantics":[{"do":"reload"},{"contains":"ann","do":"expect"}],"source":"scenarios/01-account-reload.json","stableKey":"ecommerce.spec.state-durability.session-reload.1e","statedBy":"a signed-in session persists across a page reload","withheld":null},{"category":"feature","checkGroupId":"accounts","criterionId":"1d","description":"signing out and back in returns the same account","featureId":1,"featureName":"Accounts","note":null,"packId":"ecommerce.feature.accounts","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"click","unlessVisible":"signout"},{"do":"click"},{"do":"waitUntilAbsent"},{"do":"signIn","name":"ann"},{"contains":"ann","do":"expect"}],"source":"scenarios/01-account-signout.json","stableKey":"ecommerce.feature.accounts.accounts.1d","statedBy":"a signed-in user can sign out, returning to the signed-out state","withheld":null},{"category":"feature","checkGroupId":"admin-write","criterionId":"103a","description":"an administrator can restock a warehouse","featureId":103,"featureName":"Only an administrator can restock","note":null,"packId":"ecommerce.feature.warehouse-admin","points":1,"provenBy":null,"role":"feature","semantics":[{"as":"purifier-before-control","do":"recordNumber"},{"action":"restock","do":"callAction","input":{"attribute":"data-restock-input","contains":"Air Purifier","testid":"admin-location-row"},"namedAction":{"args":[0,0,1],"id":"restock","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"warehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/restock","reducer":"admin_restock"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"expectNumber","plus":1,"relativeTo":"purifier-before-control"}],"source":"scenarios/01-admin-write-staff.json","stableKey":"ecommerce.feature.warehouse-admin.admin-write.103a","statedBy":"An administrator can add units to a selected item and warehouse.","withheld":null},{"category":"production","checkGroupId":"warehouse-write-boundary","criterionId":"103b","description":"the server refuses a warehouse write from staff","featureId":103,"featureName":"Only an administrator can restock","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","semantics":[{"action":"restock","do":"callAction","input":{"attribute":"data-restock-input","contains":"Air Purifier","testid":"admin-location-row"},"namedAction":{"args":[0,0,1],"id":"restock","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"warehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/restock","reducer":"admin_restock"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"as":"purifier-before-refusal","do":"recordNumber"},{"action":"restock","do":"callAction","from":"admin","input":{"attribute":"data-restock-input","contains":"Air Purifier","testid":"admin-location-row"},"namedAction":{"args":[0,0,1],"id":"restock","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"warehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/restock","reducer":"admin_restock"}},{"do":"expectActionOutcome","outcome":"refused","routeProvenBy":"admin"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"expectNumber","plus":0,"relativeTo":"purifier-before-refusal"}],"source":"scenarios/01-admin-write-staff.json","stableKey":"ecommerce.spec.access-control.warehouse-write-boundary.103b","statedBy":"Only administrators can change warehouse stock.","withheld":null},{"category":"production","checkGroupId":"purchase-stock","criterionId":"3b","description":"buying reduces the stock every other client sees, without a reload","featureId":3,"featureName":"Buying","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing"],"role":"guarantee","semantics":[{"do":"expectNumber","equals":100},{"do":"click"},{"do":"expectNumber","equals":99},{"do":"expectNumber","equals":99}],"source":"scenarios/01-buying.json","stableKey":"ecommerce.spec.live-state.purchase-stock.3b","statedBy":"buying an item reduces its stock by one for everyone","withheld":null},{"category":"production","checkGroupId":"cart-boundary","criterionId":"109a","description":"the same cart action run by another customer changes only that customer's cart","featureId":109,"featureName":"A cart is nobody else's business","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.cart"],"role":"guarantee","semantics":[{"action":"cart-add","do":"callAction","input":{"attribute":"data-buy-input","contains":"Coffee Grinder","testid":"item-card"},"namedAction":{"args":[0],"id":"cart-add","params":[{"in":"body","name":"itemId","wireType":"u64"}],"path":"/api/cart","reducer":"add_to_cart"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"reload"},{"do":"ensureSignedIn","name":"vic"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"cart-total"},{"do":"reload"},{"do":"ensureSignedIn","name":"wes"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"contains":"Coffee Grinder","count":1,"do":"expect"},{"do":"expectNumber","equals":1},{"do":"click","unlessVisible":"cart-total"},{"contains":"Coffee Grinder","count":1,"do":"expect"},{"do":"expectNumber","equals":1}],"source":"scenarios/01-cart-boundary.json","stableKey":"ecommerce.spec.access-control.cart-boundary.109a","statedBy":"one customer cannot read or change another customer's cart","withheld":null},{"category":"production","checkGroupId":"cart-boundary","criterionId":"109b","description":"a negative quantity is refused and leaves the cart unchanged","featureId":109,"featureName":"A cart is nobody else's business","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.cart"],"role":"guarantee","semantics":[{"as":"cart-total-before-invalid","do":"recordNumber"},{"as":"cart-quantity-before-invalid","do":"recordNumber"},{"action":"cart-set-quantity","do":"callAction","input":{"attribute":"data-cart-input","contains":"Coffee Grinder","testid":"cart-item"},"namedAction":{"args":[0,-3],"id":"cart-set-quantity","method":"PATCH","params":[{"in":"path","name":"itemId","placeholder":":itemId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/cart/:itemId","reducer":"update_cart_quantity"}},{"do":"expectActionOutcome","outcome":"validation-refused"},{"do":"reload"},{"do":"ensureSignedIn","name":"vic"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"cart-total"},{"do":"expectNumber","plus":0,"relativeTo":"cart-total-before-invalid"},{"do":"expectNumber","plus":0,"relativeTo":"cart-quantity-before-invalid"}],"source":"scenarios/01-cart-boundary.json","stableKey":"ecommerce.spec.access-control.cart-boundary.109b","statedBy":"a request carrying a negative quantity is refused and changes nothing","withheld":null},{"category":"production","checkGroupId":"cart-reload","criterionId":"4b","description":"the cart survives a reload","featureId":4,"featureName":"Cart belongs to the account","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.cart"],"role":"guarantee","semantics":[{"do":"click"},{"do":"click","unlessVisible":"cart-total"},{"contains":"Laptop Stand","do":"expect"},{"do":"reload"},{"do":"ensureSignedIn","name":"omar"},{"do":"click","unlessVisible":"cart-total"},{"contains":"Laptop Stand","do":"expect"}],"source":"scenarios/01-cart.json","stableKey":"ecommerce.spec.state-durability.cart-reload.4b","statedBy":"the cart survives a reload","withheld":null},{"category":"production","checkGroupId":"shared-cart","criterionId":"4c","description":"the same account signed in elsewhere sees one cart, live","featureId":4,"featureName":"Cart belongs to the account","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.cart"],"role":"guarantee","semantics":[{"do":"signIn","name":"pia"},{"do":"click","unlessVisible":"cart-total"},{"do":"click"},{"contains":"Induction Cooktop","do":"expect"}],"source":"scenarios/01-cart.json","stableKey":"ecommerce.spec.live-state.shared-cart.4c","statedBy":"the same account signed in twice sees one cart, and a change made in one place appears in the other without a reload","withheld":null},{"category":"feature","checkGroupId":"catalog-ranking","criterionId":"2b","description":"the storefront shows the exact alphabetical top ten before any purchase","featureId":2,"featureName":"Public catalog ranking","note":null,"packId":"ecommerce.feature.catalog-discovery","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"expectSequence","equals":["Air Purifier","Bluetooth Speaker","Coffee Grinder","Desk Lamp","Espresso Machine","Gaming Mouse","Headphones","Induction Cooktop","Keyboard","Laptop Stand"]}],"source":"scenarios/01-catalog-ranking.json","stableKey":"ecommerce.feature.catalog.catalog-ranking.2b","statedBy":"Show the ten most-purchased items and break ties by item name","withheld":null},{"category":"feature","checkGroupId":"catalog-search","criterionId":"2d","description":"case-insensitive partial search finds an item outside the storefront top ten","featureId":2,"featureName":"Public catalog search","note":null,"packId":"ecommerce.feature.catalog-discovery","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"fill","enter":true,"text":"mirrorLESS"},{"contains":"Mirrorless Camera","do":"expect"}],"source":"scenarios/01-catalog-search.json","stableKey":"ecommerce.feature.catalog.catalog-search.2d","statedBy":"Search matches any part of an item name without regard to case across the full catalog","withheld":null},{"category":"feature","checkGroupId":"catalog-values","criterionId":"2a","description":"a signed-out visitor sees the seeded item name, price, and total stock","featureId":2,"featureName":"Public catalog values","note":null,"packId":"ecommerce.feature.catalog-items","points":1,"provenBy":null,"role":"feature","semantics":[{"contains":"Air Purifier","do":"expect"},{"do":"expectNumber","equals":189},{"do":"expectNumber","equals":100}],"source":"scenarios/01-catalog-values.json","stableKey":"ecommerce.feature.catalog.catalog-values.2a","statedBy":"Each item shows its name, price, and total stock","withheld":null},{"category":"production","checkGroupId":"ranking","criterionId":"2c","description":"a purchase moves the bought item to the front of the ranking, live","featureId":2,"featureName":"Storefront is public and live","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing"],"role":"guarantee","semantics":[{"do":"click"},{"do":"expectSequence","equals":["Coffee Grinder","Air Purifier","Bluetooth Speaker","Desk Lamp","Espresso Machine","Gaming Mouse","Headphones","Induction Cooktop","Keyboard","Laptop Stand"]},{"do":"expectAgreement","numeric":true}],"source":"scenarios/01-core.json","stableKey":"ecommerce.spec.live-state.ranking.2c","statedBy":"a purchase immediately changes the ranking for every open client","withheld":null},{"category":"production","checkGroupId":"duplicate-checkout","criterionId":"203a","description":"the same item added from two tabs at once becomes one line of two","featureId":203,"featureName":"One cart, two tabs, one checkout","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.checkout"],"role":"guarantee","semantics":[{"contains":"Gaming Mouse","count":1,"do":"expect"},{"do":"expectNumber","equals":2}],"source":"scenarios/01-duplicate-checkout.json","stableKey":"ecommerce.spec.concurrency-safety.duplicate-checkout.203a","statedBy":"raises its quantity rather than adding a second line","withheld":null},{"category":"production","checkGroupId":"duplicate-checkout","criterionId":"203b","description":"checking the same cart out twice at once produces one order","featureId":203,"featureName":"One cart, two tabs, one checkout","note":"callConcurrently resolves the track's named checkout action through the selected backend adapter and issues it at the same time with tab1 and tab2's own session credentials. The filler prepares the shared Keyboard cart. Native order evidence requires one complete order for that account, the booked price and quantity, a cleared cart, and unchanged prior orders. Checkout can precede warehouse administration, so this check reads no warehouse data and retains its visible stock reduction assertion. The second call may succeed idempotently or deliberately refuse; server errors and no progress fail.","observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":1,"provenBy":"The L1 2.3 candidate set binds an exact 203b defect on every reference stack: MongoDB bypasses the atomic cart claim-and-clear, while PostgreSQL and SpacetimeDB leave checked-out cart lines behind. These defects let both named checkout calls reuse the same cart, which the call-outcome and final order assertions are designed to catch. Exact Docker mutation qualification remains the promotion gate.","requiresFeatures":["ecommerce.feature.checkout"],"role":"guarantee","semantics":[{"do":"click"},{"do":"wait"},{"as":"keyboard-before-checkout","do":"recordNumber"},{"account":"{user:twin}","as":"checkout-before","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"do":"click"},{"do":"wait"},{"account":"{user:twin}","as":"checkout-prepared","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"action":"checkout","do":"callConcurrently"},{"do":"expectCallOutcomes"},{"before":"checkout-before","do":"dbExpectCheckout","prepared":"checkout-prepared","quantity":1},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Keyboard","count":1,"do":"expect"},{"do":"reload"},{"do":"ensureSignedIn","name":"twin"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"expectNumber","plus":-1,"relativeTo":"keyboard-before-checkout"}],"source":"scenarios/01-duplicate-checkout.json","stableKey":"ecommerce.spec.concurrency-safety.duplicate-checkout.203b","statedBy":"checking out twice must not produce two orders","withheld":"The named checkout action has an HTTP route for MongoDB and PostgreSQL and a reducer mapping for SpacetimeDB. The stack adapter issues the corresponding credentialed request for each actor, giving all three stacks the same one-cart, two-call expectation."},{"category":"production","checkGroupId":"external-stock","criterionId":"901a","description":"a direct database write sets Desk Lamp's East stock to 5, and the already-open storefront updates from 100 to 50 without a reload or page action","featureId":901,"featureName":"An open storefront follows a direct database write","note":"This proposed score remains draft until the focused pristine and mutation controls are qualified.","observations":["requested","unmentioned"],"packId":"ecommerce.spec.external-data-sync","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","semantics":[{"do":"dbSetStock","item":"Desk Lamp","quantity":5,"warehouse":"East"},{"do":"expectNumber","equals":50}],"source":"scenarios/01-external-live-sync.json","stableKey":"ecommerce.spec.external-data-sync.external-stock.901a","statedBy":"The storefront reflects current database values after changes made outside the application server.","withheld":null},{"category":"production","checkGroupId":"external-stock","criterionId":"901d","description":"while the storefront is offline, a direct database write sets Desk Lamp's East stock to 7; after reconnecting, the same page catches up from 100 to the authoritative total of 52","featureId":901,"featureName":"A reconnecting storefront catches up to an external write","note":"This proposed score remains draft until the focused pristine and mutation controls are qualified.","observations":["requested","unmentioned"],"packId":"ecommerce.spec.external-data-sync","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","semantics":[{"do":"setOffline","offline":true},{"do":"dbSetStock","item":"Desk Lamp","quantity":7,"warehouse":"East"},{"do":"setOffline","offline":false},{"do":"expectNumber","equals":52}],"source":"scenarios/01-external-reconnect-sync.json","stableKey":"ecommerce.spec.external-data-sync.external-stock.901d","statedBy":"The storefront reflects current database values after changes made outside the application server.","withheld":null},{"checkGroupId":"external-stock","criterionId":"901b","description":"after a direct database write sets Desk Lamp's East stock to 5, a reload reads the persisted total of 50","featureId":901,"featureName":"A direct database write survives reload","note":"Supporting evidence for 901a rather than a second score. It rules out a page-only patch but has no distinct calibrated failure mode.","observations":["requested","unmentioned"],"packId":"ecommerce.spec.external-data-sync","points":0,"provenBy":null,"requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","semantics":[{"do":"dbSetStock","item":"Desk Lamp","quantity":5,"warehouse":"East"},{"do":"reload"},{"do":"expectNumber","equals":50}],"source":"scenarios/01-external-reload-sync.json","stableKey":"ecommerce.spec.external-data-sync.external-stock.901b","statedBy":"The storefront reflects current database values after changes made outside the application server.","withheld":null},{"category":"production","checkGroupId":"external-stock","criterionId":"901c","description":"a stock correction lands while the app server is stopped, and the already-open storefront shows the authoritative total of 65 after the server returns without a reload","featureId":901,"featureName":"An open storefront catches up after its server restarts","note":"Preserves the previously promoted score while removing its dependency on 901a changing East stock first.","observations":["requested","unmentioned"],"packId":"ecommerce.spec.external-data-sync","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","semantics":[{"do":"stopAppServer"},{"do":"dbSetStock","item":"Desk Lamp","quantity":10,"warehouse":"West"},{"do":"startAppServer"},{"do":"expectNumber","equals":65}],"source":"scenarios/01-external-server-restart-sync.json","stableKey":"ecommerce.spec.external-data-sync.external-stock.901c","statedBy":"a restart must end up showing the current numbers once the server is back","withheld":null},{"category":"production","checkGroupId":"last-unit","criterionId":"201a","description":"after six customers try to buy the last three units, each warehouse stores zero stock and all observed clients show zero stock","featureId":201,"featureName":"The last unit is sold once","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":1,"provenBy":"Prior qualification covers the client observations only. The added stored-stock observations need matching three-stack reference and defect-control evidence before promotion.","requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","semantics":[{"do":"dbExpectStock","equals":0,"item":"Air Purifier","warehouse":"East"},{"do":"dbExpectStock","equals":0,"item":"Air Purifier","warehouse":"West"},{"do":"expectNumber","equals":0},{"do":"expectAgreement","numeric":true}],"source":"scenarios/01-last-unit.json","stableKey":"ecommerce.spec.concurrency-safety.last-unit.201a","statedBy":"stock may never go negative, and two customers must never both get the last unit","withheld":null},{"category":"production","checkGroupId":"last-unit","criterionId":"201c","description":"revenue increases by exactly three sales, not six","featureId":201,"featureName":"The last unit is sold once","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"admin-revenue"},{"do":"expectNumber","plus":567,"relativeTo":"revenue-before-last-unit"}],"source":"scenarios/01-last-unit.json","stableKey":"ecommerce.spec.concurrency-safety.last-unit.201c","statedBy":"stock must never leave a warehouse without a corresponding order","withheld":null},{"category":"production","checkGroupId":"last-unit","criterionId":"201b","description":"the last three units create complete orders for the successful buyers, and all four affordable purchases succeed when stock is sufficient","featureId":201,"featureName":"The last unit is sold once","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":1,"provenBy":"Draft: native order reconciliation and the sufficient-stock progress case require matching reference and defect controls. This criterion runs after the revenue assertion so its extra purchases cannot alter that assertion's baseline.","requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","semantics":[{"before":{"a":"buy-a","b":"buy-b","c":"buy-c","d":"buy-d","e":"buy-e","f":"buy-f"},"do":"dbExpectPurchases","purchases":3},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Air Purifier","do":"expectActorsWith","equals":3,"maxEach":1},{"do":"dbSetStock","item":"Air Purifier","quantity":2,"warehouse":"East"},{"do":"dbSetStock","item":"Air Purifier","quantity":2,"warehouse":"West"},{"do":"reload"},{"do":"ensureSignedIn","name":"c1"},{"do":"click","ifAvailable":true},{"account":"{user:c1}","as":"ample-a","do":"dbRecordCheckout","item":"Air Purifier","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"account":"{user:c2}","as":"ample-b","do":"dbRecordCheckout","item":"Air Purifier","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"action":"buy","do":"callConcurrently","from":"a","input":{"attribute":"data-buy-input","contains":"Air Purifier","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"},"requests":4},{"do":"expectCallOutcomes"},{"before":{"a":"ample-a","b":"ample-b"},"do":"dbExpectPurchases","purchases":4}],"source":"scenarios/01-last-unit.json","stableKey":"ecommerce.spec.concurrency-safety.last-unit.201b","statedBy":"two customers must never both get the last unit","withheld":null},{"category":"production","checkGroupId":"order-ownership","criterionId":"106a","description":"a working order history contains the customer's own order and not another customer's order","featureId":106,"featureName":"One customer's orders are not another's","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","semantics":[{"do":"click"},{"do":"click"},{"do":"wait"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Desk Lamp","count":1,"do":"expect"},{"absent":true,"contains":"Coffee Grinder","do":"expect"}],"source":"scenarios/01-order-ownership.json","stableKey":"ecommerce.spec.access-control.order-ownership.106a","statedBy":"a customer sees only their own orders","withheld":null},{"category":"production","checkGroupId":"purchase-attribution","criterionId":"102a","description":"a direct purchase is attributed to the authenticated caller, not another account","featureId":102,"featureName":"Purchases are attributed to whoever made them","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","semantics":[{"action":"buy","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Coffee Grinder","count":1,"do":"expect"},{"absent":true,"contains":"Desk Lamp","do":"expect"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Desk Lamp","count":1,"do":"expect"}],"source":"scenarios/01-purchase-attribution.json","stableKey":"ecommerce.spec.access-control.purchase-attribution.102a","statedBy":"the authenticated caller, not a client-supplied identity, owns the order","withheld":null},{"category":"production","checkGroupId":"purchase-session","criterionId":"101a","description":"a valid direct purchase works for the buyer but is refused without a session","featureId":101,"featureName":"Purchase requires an account","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","semantics":[{"action":"buy","authentication":"none","do":"callAction","from":"buyer","input":{"attribute":"data-buy-input","contains":"Bluetooth Speaker","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"do":"expectActionOutcome","outcome":"refused","routeProvenBy":"buyer"},{"do":"dbExpectStock","item":"Bluetooth Speaker","plus":-1,"relativeTo":"speaker-before-control"}],"source":"scenarios/01-purchase-session.json","stableKey":"ecommerce.spec.access-control.purchase-session.101a","statedBy":"the server refuses a purchase without an authenticated customer","withheld":null},{"category":"feature","checkGroupId":"restock-race","criterionId":"202-control","description":"an uncontended restock of five is stored by the server and shows on the storefront","featureId":202,"featureName":"A restock during a rush is not lost","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":0,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","semantics":[{"do":"dbExpectStock","item":"Bluetooth Speaker","plus":5,"relativeTo":"stored-before-control"},{"do":"expectNumber","plus":5,"relativeTo":"storefront-before"}],"source":"scenarios/01-restock-race.json","stableKey":"ecommerce.spec.concurrency-safety.restock-race.202-control","statedBy":null,"withheld":"The ordinary restock is a setup prerequisite retained when only the race is selected. This zero-point control verifies that setup; it does not add product credit."},{"category":"production","checkGroupId":"restock-race","criterionId":"202a","description":"restocking during purchases preserves stock, complete buyer orders and their warehouse allocations","featureId":202,"featureName":"A restock during a rush is not lost","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","semantics":[{"as":"stored-before-rush","do":"dbRecordStock","item":"Bluetooth Speaker"},{"do":"fill","text":"5"},{"as":"rush-before","do":"recordNumber"},{"branches":[[{"do":"clickConcurrently"}],[{"do":"click"}]],"do":"race"},{"do":"dbExpectStock","item":"Bluetooth Speaker","plus":2,"relativeTo":"stored-before-rush"},{"atLeast":0,"do":"dbExpectStock","item":"Bluetooth Speaker","warehouse":"East"},{"atLeast":0,"do":"dbExpectStock","item":"Bluetooth Speaker","warehouse":"West"},{"do":"reload"},{"do":"click"},{"do":"expectNumber","plus":2,"relativeTo":"rush-before"},{"do":"ensureSignedIn","name":"r1"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Bluetooth Speaker","count":2,"do":"expect"},{"do":"reload"},{"do":"ensureSignedIn","name":"r2"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Bluetooth Speaker","count":1,"do":"expect"},{"do":"reload"},{"do":"ensureSignedIn","name":"r3"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Bluetooth Speaker","count":1,"do":"expect"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"expectNumber","plus":2,"relativeTo":"stored-before-rush"},{"do":"reload"},{"do":"click"},{"do":"expectNumber","plus":2,"relativeTo":"stored-before-rush"},{"do":"ensureSignedIn","name":"r1"},{"account":"{user:r1}","as":"mixed-a","do":"dbRecordCheckout","item":"Bluetooth Speaker","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"account":"{user:r2}","as":"mixed-b","do":"dbRecordCheckout","item":"Bluetooth Speaker","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"account":"{user:r3}","as":"mixed-c","do":"dbRecordCheckout","item":"Bluetooth Speaker","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"action":"buy","alongside":[{"action":"restock","actors":["admin"],"input":{"attribute":"data-restock-input","contains":"Bluetooth Speaker","testid":"admin-location-row"},"namedAction":{"args":[0,0,1],"id":"restock","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"warehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/restock","reducer":"admin_restock"},"requests":1}],"do":"callConcurrently","from":"a","input":{"attribute":"data-buy-input","contains":"Bluetooth Speaker","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"},"requests":3},{"accepted":4,"do":"expectCallOutcomes"},{"before":{"a":"mixed-a","b":"mixed-b","c":"mixed-c"},"do":"dbExpectPurchases","purchases":3}],"source":"scenarios/01-restock-race.json","stableKey":"ecommerce.spec.concurrency-safety.restock-race.202a","statedBy":"a restock in one warehouse raises the storefront number live","withheld":null},{"category":"production","checkGroupId":"review-eligibility","criterionId":"108a","description":"someone who never bought the item cannot review it","featureId":108,"featureName":"A review is a claim about a purchase","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","semantics":[{"action":"buy","do":"callAction","input":{"attribute":"data-buy-input","contains":"Air Purifier","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Air Purifier","do":"expect"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"action":"submitReview","do":"callAction","from":"owner","input":{"attribute":"data-buy-input","contains":"Air Purifier","testid":"item-card"},"namedAction":{"args":[0,4,"eligible review control"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"}],"path":"/api/items/:id/reviews","reducer":"submit_review"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"openItem","item":"Air Purifier"},{"contains":"eligible review control","do":"expect"},{"action":"submitReview","do":"callAction","from":"owner","input":{"attribute":"data-buy-input","contains":"Air Purifier","testid":"item-card"},"namedAction":{"args":[0,4,"never bought this"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"}],"path":"/api/items/:id/reviews","reducer":"submit_review"}},{"do":"expectActionOutcome","outcome":"application-refused","routeProvenBy":"owner"},{"do":"reload"},{"do":"ensureSignedIn","name":"uma"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"openItem","item":"Air Purifier"},{"contains":"eligible review control","do":"expect"},{"absent":true,"contains":"never bought this","do":"expect"}],"source":"scenarios/01-review-eligibility.json","stableKey":"ecommerce.spec.access-control.review-eligibility.108a","statedBy":"the server refuses a review from someone who has never ordered the item","withheld":null},{"category":"production","checkGroupId":"review-eligibility","criterionId":"108b","description":"buying the item earns the right to review it","featureId":108,"featureName":"A review is a claim about a purchase","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","semantics":[{"action":"buy","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"openItem","item":"Keyboard"},{"do":"click","ifAvailable":true,"unlessVisible":"review-rating"},{"do":"fill","text":"4"},{"do":"fill","text":"bought and used it"},{"do":"click"},{"contains":"bought and used it","do":"expect"}],"source":"scenarios/01-review-eligibility.json","stableKey":"ecommerce.spec.access-control.review-eligibility.108b","statedBy":"a customer can review an item they bought","withheld":null},{"category":"production","checkGroupId":"rating","criterionId":"6c","description":"the average rating reflects both reviewers and updates live","featureId":6,"featureName":"Reviews","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","semantics":[{"do":"openItem","item":"Gaming Mouse"},{"do":"click","ifAvailable":true,"unlessVisible":"review-rating"},{"do":"fill","text":"4"},{"do":"fill","text":"does the job"},{"do":"click"},{"do":"expectNumber","equals":3},{"do":"expectAgreement","numeric":true}],"source":"scenarios/01-review-rating-live.json","stableKey":"ecommerce.spec.live-state.rating.6c","statedBy":"each item shows its average rating, which updates live as reviews arrive","withheld":null},{"category":"production","checkGroupId":"unique-review","criterionId":"6b","description":"a later review submission does not create a duplicate for the same customer and item","featureId":6,"featureName":"Reviews","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","semantics":[{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"action":"submitReview","do":"callAction","from":"author","input":{"attribute":"data-buy-input","contains":"Air Purifier","testid":"item-card"},"namedAction":{"args":[0,4,"changed my mind"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"}],"path":"/api/items/:id/reviews","reducer":"submit_review"}},{"do":"expectActionOutcome","outcome":"completed"},{"do":"reload"},{"do":"ensureSignedIn","name":"kira"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"openItem","item":"Air Purifier"},{"count":1,"do":"expect"}],"source":"scenarios/01-review-uniqueness.json","stableKey":"ecommerce.spec.transactional-integrity.unique-review.6b","statedBy":"one customer has at most one review per item","withheld":null},{"category":"feature","checkGroupId":"reviews","criterionId":"6a","description":"a customer can review an item and everyone sees it, signed out included","featureId":6,"featureName":"Reviews","note":null,"packId":"ecommerce.feature.reviews","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"openItem","item":"Induction Cooktop"},{"do":"click","ifAvailable":true,"unlessVisible":"review-rating"},{"do":"fill","text":"4"},{"do":"fill","text":"solid mold"},{"do":"click"},{"contains":"solid mold","do":"expect"},{"do":"openItem","item":"Induction Cooktop"},{"contains":"solid mold","do":"expect"}],"source":"scenarios/01-review-visibility.json","stableKey":"ecommerce.feature.reviews.reviews.6a","statedBy":"reviews are visible to everyone, including signed-out visitors","withheld":null},{"category":"production","checkGroupId":"server-price","criterionId":"104a","description":"direct purchases of two differently priced items persist exactly one correctly priced order each","featureId":104,"featureName":"The price is the store's to set","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","semantics":[{"action":"buy","do":"callAction","input":{"attribute":"data-buy-input","contains":"Espresso Machine","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"do":"expectActionOutcome","outcome":"accepted"},{"action":"buy","do":"callAction","input":{"attribute":"data-buy-input","contains":"Coffee Grinder","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"freshClient"},{"do":"signIn","name":"oli"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"count":2,"do":"expect"},{"contains":"Espresso Machine","count":1,"do":"expect"},{"do":"expectNumber","equals":449},{"contains":"Coffee Grinder","count":1,"do":"expect"},{"do":"expectNumber","equals":64}],"source":"scenarios/01-server-price.json","stableKey":"ecommerce.spec.transactional-integrity.server-price.104a","statedBy":"the server uses the current stored price when it creates an order","withheld":null},{"category":"production","checkGroupId":"warehouse-area-boundary","criterionId":"7a","description":"the administrator area stays unavailable to other staff","featureId":7,"featureName":"Admin and warehouses","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","semantics":[{"do":"expect"},{"do":"click","ifAvailable":true},{"absent":true,"do":"expect"}],"source":"scenarios/01-warehouse-admin-staff.json","stableKey":"ecommerce.spec.access-control.warehouse-area-boundary.7a","statedBy":"Administrators can open the administration area; other staff cannot.","withheld":null},{"category":"feature","checkGroupId":"warehouse-view","criterionId":"7b","description":"admin lists every item, every warehouse, and what each warehouse holds","featureId":7,"featureName":"Admin and warehouses","note":null,"packId":"ecommerce.feature.warehouse-admin","points":1,"provenBy":null,"role":"feature","semantics":[{"count":13,"do":"expect"},{"count":26,"do":"expect"},{"contains":"East","do":"expect"},{"contains":"West","do":"expect"},{"do":"expectNumber","equals":100}],"source":"scenarios/01-warehouse-admin-staff.json","stableKey":"ecommerce.feature.warehouse-admin.warehouse-view.7b","statedBy":"admin lists every item with its stock, every warehouse, and the stock of each item in each warehouse","withheld":null},{"category":"production","checkGroupId":"warehouse-stock","criterionId":"7c","description":"the storefront stock is the sum across warehouses, and a restock raises it live","featureId":7,"featureName":"Warehouse stock stays live","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.warehouse-admin"],"role":"guarantee","semantics":[{"do":"fill","text":"25"},{"do":"click"},{"do":"expectNumber","equals":125},{"do":"expectNumber","equals":125}],"source":"scenarios/01-warehouse-stock-live-staff.json","stableKey":"ecommerce.spec.live-state.warehouse-stock.7c","statedBy":"an item's stock on the storefront is the sum of that item's units across all warehouses","withheld":null},{"category":"production","checkGroupId":"cancellation-and-return","criterionId":"3d","description":"cancelling a pending order removes it from the fulfilment queue","featureId":3,"featureName":"Cancellation and fulfilment","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.cancellation-queue-specifications","points":1,"provenBy":null,"requiresFeatures":["ecommerce.l2.order-cancellation-features","ecommerce.progression.fulfilment-queue"],"role":"guarantee","semantics":[{"do":"click"},{"contains":"Coffee Grinder","do":"expect"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"do":"click"},{"do":"waitUntilAbsent"}],"source":"scenarios/02-cancellation-queue.json","stableKey":"ecommerce.returns-pricing.cancellation-and-return.3d","statedBy":"the order leaves the fulfilment queue","withheld":null},{"category":"production","checkGroupId":"fulfilment-area-boundary","criterionId":"1d","description":"staff and administrators can open fulfilment while customers cannot","featureId":1,"featureName":"Fulfilment area access","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"provenBy":null,"requiresFeatures":["ecommerce.progression.fulfilment-queue"],"role":"guarantee","semantics":[{"do":"click"},{"do":"expect"},{"do":"click"},{"do":"expect"},{"do":"click","ifAvailable":true},{"absent":true,"do":"expect"}],"source":"scenarios/02-fulfilment-access.json","stableKey":"ecommerce.spec.access-control.fulfilment-area-boundary.1d","statedBy":"Customers cannot open the fulfilment area","withheld":null},{"category":"production","checkGroupId":"fulfilment-queue","criterionId":"1a","description":"an order placed by a customer appears in the staff queue without a reload","featureId":1,"featureName":"Live fulfilment queue","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.fulfilment-queue"],"role":"guarantee","semantics":[{"as":"depth-before","do":"recordNumber"},{"do":"click"},{"contains":"Desk Lamp","do":"expect"},{"do":"expectNumber","plus":1,"relativeTo":"depth-before"}],"source":"scenarios/02-fulfilment-live.json","stableKey":"ecommerce.spec.live-state.fulfilment-queue.1a","statedBy":"New orders appear in the queue without a reload","withheld":null},{"category":"feature","checkGroupId":"fulfilment-queue","criterionId":"1c","description":"shipping removes the order from the queue and marks the customer's order shipped","featureId":1,"featureName":"Ship a pending order","note":null,"packId":"ecommerce.progression.fulfilment-queue","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"feature","semantics":[{"action":"buy","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"pending"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click","ifAvailable":true},{"contains":"Keyboard","do":"expect"},{"do":"click"},{"attribute":"data-submit-state","do":"expect","value":"succeeded"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click","ifAvailable":true},{"absent":true,"contains":"Keyboard","do":"expect"},{"do":"reload"},{"do":"ensureSignedIn","name":"fq-ship"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"shipped"}],"source":"scenarios/02-fulfilment-ship.json","stableKey":"ecommerce.operations-access.fulfilment-queue.1c","statedBy":"Staff can mark an order as shipped; show the new status in the fulfilment area and the customer's order history.","withheld":null},{"category":"production","checkGroupId":"refund-accounting","criterionId":"203a","description":"concurrent cancellation restores original stock and the booked amount once, while revenue returns to its prior value","featureId":203,"featureName":"The books still balance once money can flow backwards","note":"Draft: native order, refund and original warehouse allocation evidence supplements the retained revenue assertions. Four calls from two sessions of the owner must cancel once; a repeated call may deliberately refuse or succeed without extra effects. Qualification pending.","observations":["requested","unmentioned"],"packId":"ecommerce.progression.cancellation-accounting-specifications","points":3,"provenBy":null,"requiresFeatures":["ecommerce.l2.order-cancellation-features"],"role":"guarantee","semantics":[{"as":"rev-start","do":"recordNumber"},{"do":"click"},{"do":"wait"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"admin-revenue"},{"do":"expectNumber","plus":42,"relativeTo":"rev-start"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"pending"},{"account":"{user:books}","as":"cancel-before","do":"dbRecordCheckout","item":"Desk Lamp","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"action":"cancel","do":"callConcurrently","from":"customer","input":{"attribute":"data-cancel-input","contains":"Desk Lamp","testid":"order-item"},"namedAction":{"args":[0],"id":"cancel","params":[{"in":"path","name":"orderId","placeholder":":id","wireType":"u64"}],"path":"/api/orders/:id/cancel","reducer":"cancel_order"},"requests":4},{"do":"expectCallOutcomes"},{"before":"cancel-before","do":"dbExpectCancellation"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"admin-revenue"},{"do":"expectNumber","plus":0,"relativeTo":"rev-start"}],"source":"scenarios/02-invariants.json","stableKey":"ecommerce.returns-pricing.refund-accounting.203a","statedBy":"revenue always equals the sum of orders that are still standing","withheld":null},{"category":"production","checkGroupId":"refund-accounting","criterionId":"203b","description":"a price change does not rewrite revenue already earned","featureId":203,"featureName":"The books still balance once money can flow backwards","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.price-accounting-specifications","points":2,"provenBy":null,"requiresFeatures":["ecommerce.l2.price-history-features"],"role":"guarantee","semantics":[{"as":"history-revenue-before-sale","do":"recordNumber"},{"do":"pressKey","key":"Escape"},{"do":"click"},{"do":"wait"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"admin-revenue"},{"do":"expectNumber","plus":79.5,"relativeTo":"history-revenue-before-sale"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Bluetooth Speaker","count":1,"do":"expect"},{"as":"rev-after-sale","do":"recordNumber"},{"do":"fill","text":"5.00"},{"do":"click"},{"do":"wait"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"expectNumber","equals":5},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"admin-revenue"},{"do":"expectNumber","plus":0,"relativeTo":"rev-after-sale"}],"source":"scenarios/02-invariants.json","stableKey":"ecommerce.returns-pricing.refund-accounting.203b","statedBy":"changing a price never alters the history, the revenue already recorded","withheld":null},{"category":"production","checkGroupId":"price-history","criterionId":"4b","description":"the new price reaches a signed-out visitor without a reload","featureId":4,"featureName":"Live catalog price","note":null,"packId":"ecommerce.l2.price-history-features","points":2,"provenBy":null,"role":"feature","semantics":[{"atLeast":2,"do":"expectNumber"},{"do":"fill","text":"1.00"},{"do":"click"},{"do":"expectNumber","equals":1}],"source":"scenarios/02-live-price.json","stableKey":"ecommerce.returns-pricing.price-history.4b","statedBy":"the storefront shows the new price immediately, to everyone","withheld":null},{"category":"feature","checkGroupId":"operational-views","criterionId":"5e","description":"the dashboard lists a current low-stock item","featureId":5,"featureName":"The low-stock view","note":null,"packId":"ecommerce.l2.inventory-dashboard","points":1,"provenBy":null,"role":"feature","semantics":[{"contains":"Air Purifier","do":"expect"}],"source":"scenarios/02-low-stock.json","stableKey":"ecommerce.inventory-operations.operational-views.5e","statedBy":"It lists items with 10 units or fewer, most urgent first.","withheld":null},{"category":"production","checkGroupId":"inventory-dashboard","criterionId":"5a","description":"an item falling to ten units or fewer joins the low-stock list, live","featureId":5,"featureName":"The low-stock view","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":3,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.inventory-dashboard"],"role":"guarantee","semantics":[{"contains":"Air Purifier","do":"expect"},{"do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click"},{"do":"fill","text":"8"},{"do":"click"},{"do":"waitUntilAbsent"},{"do":"click"},{"contains":"Air Purifier","do":"expect"}],"source":"scenarios/02-low-stock.json","stableKey":"ecommerce.spec.live-state.inventory-dashboard.5a","statedBy":"items enter and leave this list as stock moves, sells, is restocked, cancelled or returned","withheld":null},{"category":"feature","checkGroupId":"operational-views","criterionId":"5d","description":"a signed-out visitor sees a best seller in the recommendations list","featureId":5,"featureName":"Signed-out best sellers","note":null,"packId":"ecommerce.l2.sales-dashboard","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"reload"},{"do":"expectNumber","equals":1}],"source":"scenarios/02-operational-best-sellers.json","stableKey":"ecommerce.inventory-operations.operational-views.5d","statedBy":"Signed-out visitors see best sellers","withheld":null},{"category":"feature","checkGroupId":"operational-views","criterionId":"5f","description":"the dashboard shows category units and revenue","featureId":5,"featureName":"Category sales totals","note":null,"packId":"ecommerce.l2.sales-dashboard","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"click","ifAvailable":true,"unlessVisible":"category-row"},{"as":"audio-core-units","do":"recordNumber"},{"as":"audio-core-revenue","do":"recordNumber"},{"do":"click"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click"},{"do":"click","ifAvailable":true,"unlessVisible":"category-row"},{"do":"expectNumber","plus":1,"relativeTo":"audio-core-units"},{"do":"expectNumber","plus":79.5,"relativeTo":"audio-core-revenue"}],"source":"scenarios/02-operational-category-totals.json","stableKey":"ecommerce.inventory-operations.operational-views.5f","statedBy":"Category totals show units sold and revenue for each category.","withheld":null},{"category":"production","checkGroupId":"sales-dashboard","criterionId":"5b","description":"a purchase updates that category's units and revenue live","featureId":5,"featureName":"Category sales totals","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":3,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.sales-dashboard"],"role":"guarantee","semantics":[{"do":"click","ifAvailable":true,"unlessVisible":"category-row"},{"as":"audio-units","do":"recordNumber"},{"as":"audio-revenue","do":"recordNumber"},{"do":"click"},{"do":"expectNumber","plus":1,"relativeTo":"audio-units"},{"do":"expectNumber","plus":79.5,"relativeTo":"audio-revenue"}],"source":"scenarios/02-operational-category-totals.json","stableKey":"ecommerce.spec.live-state.sales-dashboard.5b","statedBy":"Category totals show units sold and revenue for each category","withheld":null},{"category":"feature","checkGroupId":"operational-views","criterionId":"5c","description":"a purchase recommends another item from that category and excludes an item in the cart","featureId":5,"featureName":"Customer recommendations","note":null,"packId":"ecommerce.l2.recommendations","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"contains":"Headphones","do":"expect"},{"do":"click"},{"do":"waitUntilAbsent"}],"source":"scenarios/02-operational-recommendations.json","stableKey":"ecommerce.inventory-operations.operational-views.5c","statedBy":"Recommend items from categories the customer bought from and exclude items already in the cart","withheld":null},{"category":"production","checkGroupId":"cancellation-and-return","criterionId":"3a","description":"cancelling a pending order restores its stock and revenue","featureId":3,"featureName":"Cancel a pending order","note":null,"packId":"ecommerce.l2.order-cancellation-features","points":2,"provenBy":null,"role":"feature","semantics":[{"as":"revenue-before","do":"recordNumber"},{"as":"stock-before","do":"recordNumber"},{"do":"click"},{"do":"expectNumber","plus":-1,"relativeTo":"stock-before"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"expectNumber","plus":64,"relativeTo":"revenue-before"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"pending"},{"do":"click"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"expectNumber","plus":0,"relativeTo":"revenue-before"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"expectNumber","plus":0,"relativeTo":"stock-before"}],"source":"scenarios/02-order-cancellation-core.json","stableKey":"ecommerce.returns-pricing.cancellation-and-return.3a","statedBy":"the stock goes back to the warehouse it came from and revenue falls","withheld":null},{"category":"feature","checkGroupId":"cancellation-and-return","criterionId":"3b","description":"a cancelled order is shown as cancelled in the customer's history","featureId":3,"featureName":"Cancellation history","note":null,"packId":"ecommerce.l2.order-cancellation-features","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"click"},{"do":"expect","ignoreCase":true,"value":"cancelled"}],"source":"scenarios/02-order-cancellation-history.json","stableKey":"ecommerce.returns-pricing.cancellation-and-return.3b","statedBy":"a customer can cancel an order that has not shipped","withheld":null},{"category":"production","checkGroupId":"price-history","criterionId":"4a","description":"a price change updates the live catalog but leaves the customer's exact paid price unchanged","featureId":4,"featureName":"Prices change, history does not","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.price-history-specifications","points":3,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.price-history-features"],"role":"guarantee","semantics":[{"do":"fill","enter":true,"text":"Air Purifier"},{"as":"air-purifier-paid","do":"recordNumber"},{"do":"click"},{"do":"fill","enter":true,"text":"Air Purifier"},{"do":"fill","text":"1.00"},{"do":"click"},{"do":"expectNumber","equals":1},{"do":"reload"},{"do":"ensureSignedIn","name":"history-persisted"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"do":"expectNumber","plus":0,"relativeTo":"air-purifier-paid"}],"source":"scenarios/02-paid-price-history.json","stableKey":"ecommerce.returns-pricing.price-history.4a","statedBy":"past orders keep the price that was paid","withheld":null},{"category":"feature","checkGroupId":"fulfilment-queue","criterionId":"1b","description":"the queue names the warehouse the order will ship from","featureId":1,"featureName":"Fulfilment queue","note":null,"packId":"ecommerce.progression.fulfilment-queue","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"warehouse":"West"},{"do":"click"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click","ifAvailable":true},{"contains":"Desk Lamp","do":"expect"},{"contains":"East","do":"expect"}],"source":"scenarios/02-queue-warehouse.json","stableKey":"ecommerce.operations-access.fulfilment-queue.1b","statedBy":"which warehouse each will ship from","withheld":null},{"category":"production","checkGroupId":"stock-conservation","criterionId":"202b","description":"a sale and its cancellation leave the shelf exactly as they found it","featureId":202,"featureName":"Stock recovery is durable across clients","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","points":2,"provenBy":null,"requiresFeatures":["ecommerce.l2.order-cancellation-features"],"role":"guarantee","semantics":[{"as":"east-before-cancel-202b","do":"dbRecordStock","item":"Induction Cooktop","warehouse":"East"},{"as":"west-before-cancel-202b","do":"dbRecordStock","item":"Induction Cooktop","warehouse":"West"},{"as":"stored-before-cancel-202b","do":"dbRecordStock","item":"Induction Cooktop"},{"as":"cancel-stock-before","do":"recordNumber"},{"do":"click"},{"do":"expectNumber","plus":-1,"relativeTo":"cancel-stock-before"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Induction Cooktop","count":1,"do":"expect"},{"do":"dbExpectStock","item":"Induction Cooktop","plus":-1,"relativeTo":"stored-before-cancel-202b"},{"do":"click"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"expectNumber","plus":0,"relativeTo":"cancel-stock-before"},{"do":"dbExpectStock","item":"Induction Cooktop","plus":0,"relativeTo":"stored-before-cancel-202b"},{"do":"dbExpectStock","item":"Induction Cooktop","plus":0,"relativeTo":"east-before-cancel-202b","warehouse":"East"},{"do":"dbExpectStock","item":"Induction Cooktop","plus":0,"relativeTo":"west-before-cancel-202b","warehouse":"West"}],"source":"scenarios/02-self-contained.json","stableKey":"ecommerce.inventory-operations.stock-conservation.202b","statedBy":"the stock goes back to the warehouse it came from","withheld":null},{"category":"production","checkGroupId":"stock-conservation","criterionId":"202c","description":"a fresh client sees the restored total after a sale is cancelled","featureId":202,"featureName":"Stock recovery is durable across clients","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","points":1,"provenBy":null,"requiresFeatures":["ecommerce.l2.order-cancellation-features"],"role":"guarantee","semantics":[{"as":"east-before-cancel-202c","do":"dbRecordStock","item":"Headphones","warehouse":"East"},{"as":"west-before-cancel-202c","do":"dbRecordStock","item":"Headphones","warehouse":"West"},{"as":"stored-before-cancel-202c","do":"dbRecordStock","item":"Headphones"},{"as":"fresh-stock-before","do":"recordNumber"},{"do":"click"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Headphones","count":1,"do":"expect"},{"do":"dbExpectStock","item":"Headphones","plus":-1,"relativeTo":"stored-before-cancel-202c"},{"do":"click"},{"do":"freshClient"},{"do":"expectNumber","plus":0,"relativeTo":"fresh-stock-before"},{"do":"dbExpectStock","item":"Headphones","plus":0,"relativeTo":"stored-before-cancel-202c"},{"do":"dbExpectStock","item":"Headphones","plus":0,"relativeTo":"east-before-cancel-202c","warehouse":"East"},{"do":"dbExpectStock","item":"Headphones","plus":0,"relativeTo":"west-before-cancel-202c","warehouse":"West"}],"source":"scenarios/02-self-contained.json","stableKey":"ecommerce.inventory-operations.stock-conservation.202c","statedBy":"every one of these numbers is the same for every person looking at it","withheld":null},{"category":"production","checkGroupId":"operator-authorization","criterionId":"201c","description":"the server refuses a customer's direct attempt to ship their own pending order","featureId":201,"featureName":"Shipping requires an operator","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.operations-access-specifications","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"guarantee","semantics":[{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"click"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Coffee Grinder","count":1,"do":"expect"},{"action":"ship","do":"callAction","from":"customer","input":{"attribute":"data-ship-input","contains":"Coffee Grinder","testid":"order-item"},"namedAction":{"args":[0],"id":"ship","params":[{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/fulfilment/ship","reducer":"ship_order"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"reload"},{"do":"ensureSignedIn","name":"direct-notstaff"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"shipped"},{"action":"ship","do":"callAction","input":{"attribute":"data-ship-input","contains":"Laptop Stand","testid":"order-item"},"namedAction":{"args":[0],"id":"ship","params":[{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/fulfilment/ship","reducer":"ship_order"}},{"do":"expectActionOutcome","outcome":"refused","routeProvenBy":"staff"},{"do":"reload"},{"do":"ensureSignedIn","name":"direct-notstaff"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"pending"}],"source":"scenarios/02-server-actions.json","stableKey":"ecommerce.operations-access.operator-authorization.201c","statedBy":"Staff mark an order shipped","withheld":null},{"category":"production","checkGroupId":"stock-conservation","criterionId":"202d","description":"a direct transfer racing a direct purchase leaves the exact starting total minus the sold unit","featureId":202,"featureName":"Stock is conserved while operations overlap","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.stock-transfers-features"],"role":"guarantee","semantics":[{"do":"dbExpectStock","equals":60,"item":"Headphones","warehouse":"East"},{"do":"dbExpectStock","equals":40,"item":"Headphones","warehouse":"West"},{"as":"direct-race-stock-before","do":"dbRecordStock","item":"Headphones"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"reload"},{"do":"ensureSignedIn","name":"direct-conserve"},{"do":"fill","text":"East"},{"do":"fill","text":"West"},{"do":"fill","text":"25"},{"branches":[[{"action":"transfer","do":"callAction","input":{"attribute":"data-transfer-input","contains":"Headphones","testid":"admin-item-row"},"namedAction":{"args":[0,0,0,25],"id":"transfer","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"fromWarehouseId","wireType":"u64"},{"in":"body","name":"toWarehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/transfer","reducer":"admin_transfer_stock"}}],[{"action":"buy","do":"callAction","input":{"attribute":"data-buy-input","contains":"Headphones","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}}]],"do":"race"},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"expectNumber","plus":-1,"relativeTo":"direct-race-stock-before"},{"do":"reload"},{"do":"ensureSignedIn","name":"direct-conserve"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"expectNumber","plus":-1,"relativeTo":"direct-race-stock-before"},{"atLeast":34,"atMost":35,"do":"dbExpectStock","item":"Headphones","warehouse":"East"},{"atLeast":64,"atMost":65,"do":"dbExpectStock","item":"Headphones","warehouse":"West"},{"do":"dbExpectStock","item":"Headphones","plus":-1,"relativeTo":"direct-race-stock-before"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Headphones","count":1,"do":"expect"}],"source":"scenarios/02-server-actions.json","stableKey":"ecommerce.inventory-operations.stock-conservation.202d","statedBy":"a transfer moves stock, it does not create or destroy it","withheld":null},{"category":"production","checkGroupId":"order-owner","criterionId":"204a","description":"the server refuses one customer trying to cancel another customer's still-pending order","featureId":204,"featureName":"An order belongs to the person who placed it","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.operations-access-specifications","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.order-cancellation-features"],"role":"guarantee","semantics":[{"action":"cancel","do":"callAction","input":{"attribute":"data-cancel-input","contains":"Desk Lamp","testid":"order-item"},"namedAction":{"args":[0],"id":"cancel","params":[{"in":"path","name":"orderId","placeholder":":id","wireType":"u64"}],"path":"/api/orders/:id/cancel","reducer":"cancel_order"}},{"do":"expectActionOutcome","outcome":"accepted"},{"action":"cancel","do":"callAction","from":"owner","input":{"attribute":"data-cancel-input","contains":"Keyboard","testid":"order-item"},"namedAction":{"args":[0],"id":"cancel","params":[{"in":"path","name":"orderId","placeholder":":id","wireType":"u64"}],"path":"/api/orders/:id/cancel","reducer":"cancel_order"}},{"do":"expectActionOutcome","outcome":"refused","routeProvenBy":"owner"},{"do":"reload"},{"do":"ensureSignedIn","name":"direct-owner"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"pending"}],"source":"scenarios/02-server-actions.json","stableKey":"ecommerce.operations-access.order-owner.204a","statedBy":"A customer can cancel an order that has not shipped","withheld":null},{"category":"production","checkGroupId":"warehouse-transfer","criterionId":"2a","description":"a transfer decreases the source, increases the destination, and preserves the item's exact total","featureId":2,"featureName":"Moving stock between warehouses","note":null,"packId":"ecommerce.l2.stock-transfers-features","points":3,"provenBy":null,"role":"feature","semantics":[{"as":"product-East","do":"dbRecordStock","item":"Headphones","warehouse":"East"},{"as":"product-West","do":"dbRecordStock","item":"Headphones","warehouse":"West"},{"as":"transfer-item-before","do":"recordNumber"},{"contains":"East","count":1,"do":"expect"},{"as":"transfer-east-before","do":"recordNumber"},{"contains":"West","count":1,"do":"expect"},{"as":"transfer-west-before","do":"recordNumber"},{"do":"fill","text":"East"},{"do":"fill","text":"West"},{"do":"fill","text":"10"},{"do":"click"},{"contains":"East","count":1,"do":"expect"},{"do":"dbExpectStock","item":"Headphones","plus":-10,"relativeTo":"product-East","warehouse":"East"},{"do":"dbExpectStock","item":"Headphones","plus":10,"relativeTo":"product-West","warehouse":"West"},{"do":"expectNumber","plus":-10,"relativeTo":"transfer-east-before"},{"contains":"West","count":1,"do":"expect"},{"do":"expectNumber","plus":10,"relativeTo":"transfer-west-before"},{"do":"expectNumber","plus":0,"relativeTo":"transfer-item-before"}],"source":"scenarios/02-strengthened.json","stableKey":"ecommerce.inventory-operations.warehouse-transfer.2a","statedBy":"a transfer moves stock, it does not create or destroy it","withheld":null},{"category":"production","checkGroupId":"operator-authorization","criterionId":"201a","description":"the server refuses a customer's direct transfer and neither warehouse nor the item total changes","featureId":201,"featureName":"Operating the store requires authorization","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.operations-access-specifications","points":2,"provenBy":null,"requiresFeatures":["ecommerce.l2.stock-transfers-features"],"role":"guarantee","semantics":[{"as":"authorized-transfer-east","do":"dbRecordStock","item":"Headphones","warehouse":"East"},{"as":"authorized-transfer-west","do":"dbRecordStock","item":"Headphones","warehouse":"West"},{"do":"fill","text":"East"},{"do":"fill","text":"West"},{"do":"fill","text":"25"},{"action":"transfer","do":"callAction","input":{"attribute":"data-transfer-input","contains":"Headphones","testid":"admin-item-row"},"namedAction":{"args":[0,0,0,25],"id":"transfer","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"fromWarehouseId","wireType":"u64"},{"in":"body","name":"toWarehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/transfer","reducer":"admin_transfer_stock"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"dbExpectStock","item":"Headphones","plus":-25,"relativeTo":"authorized-transfer-east","warehouse":"East"},{"do":"dbExpectStock","item":"Headphones","plus":25,"relativeTo":"authorized-transfer-west","warehouse":"West"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"reload"},{"do":"ensureSignedIn","name":"not-operator"},{"as":"unauthorized-item-before","do":"recordNumber"},{"contains":"East","count":1,"do":"expect"},{"as":"unauthorized-east-before","do":"recordNumber"},{"contains":"West","count":1,"do":"expect"},{"as":"unauthorized-west-before","do":"recordNumber"},{"do":"fill","text":"East"},{"do":"fill","text":"West"},{"as":"refused-East","do":"dbRecordStock","item":"Headphones","warehouse":"East"},{"as":"refused-West","do":"dbRecordStock","item":"Headphones","warehouse":"West"},{"action":"transfer","do":"callAction","from":"admin","input":{"attribute":"data-transfer-input","contains":"Headphones","testid":"admin-item-row"},"namedAction":{"args":[0,0,0,25],"id":"transfer","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"fromWarehouseId","wireType":"u64"},{"in":"body","name":"toWarehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/transfer","reducer":"admin_transfer_stock"}},{"do":"expectActionOutcome","outcome":"refused","routeProvenBy":"admin"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"reload"},{"do":"ensureSignedIn","name":"not-operator"},{"contains":"East","count":1,"do":"expect"},{"do":"dbExpectStock","item":"Headphones","plus":0,"relativeTo":"refused-East","warehouse":"East"},{"do":"dbExpectStock","item":"Headphones","plus":0,"relativeTo":"refused-West","warehouse":"West"},{"do":"expectNumber","plus":0,"relativeTo":"unauthorized-east-before"},{"contains":"West","count":1,"do":"expect"},{"do":"expectNumber","plus":0,"relativeTo":"unauthorized-west-before"},{"do":"expectNumber","plus":0,"relativeTo":"unauthorized-item-before"}],"source":"scenarios/02-strengthened.json","stableKey":"ecommerce.operations-access.operator-authorization.201a","statedBy":"An admin can transfer a number of units of an item from one warehouse to another","withheld":null},{"category":"production","checkGroupId":"operator-authorization","criterionId":"201b","description":"the server refuses a customer's direct price change and the last accepted price remains exact","featureId":201,"featureName":"Operating the store requires authorization","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.operations-access-specifications","points":2,"provenBy":null,"requiresFeatures":["ecommerce.l2.price-history-features"],"role":"guarantee","semantics":[{"do":"fill","text":"77.00"},{"action":"price","do":"callAction","input":{"attribute":"data-price-input","contains":"Gaming Mouse","testid":"admin-item-row"},"namedAction":{"args":[0,1],"id":"price","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"price"}],"path":"/api/admin/price","reducer":"admin_change_price"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"reload"},{"do":"ensureSignedIn","name":"not-operator"},{"do":"expectNumber","equals":77},{"do":"fill","text":"1.00"},{"action":"price","do":"callAction","from":"admin","input":{"attribute":"data-price-input","contains":"Gaming Mouse","testid":"admin-item-row"},"namedAction":{"args":[0,1],"id":"price","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"price"}],"path":"/api/admin/price","reducer":"admin_change_price"}},{"do":"expectActionOutcome","outcome":"refused","routeProvenBy":"admin"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"reload"},{"do":"ensureSignedIn","name":"not-operator"},{"do":"expectNumber","equals":77}],"source":"scenarios/02-strengthened.json","stableKey":"ecommerce.operations-access.operator-authorization.201b","statedBy":"an admin can change an item's price","withheld":null},{"category":"production","checkGroupId":"stock-conservation","criterionId":"202a","description":"a transfer decreases East, increases West, and leaves the item's exact total unchanged","featureId":202,"featureName":"Stock is conserved however it moves","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","points":2,"provenBy":null,"requiresFeatures":["ecommerce.l2.stock-transfers-features"],"role":"guarantee","semantics":[{"as":"product-East","do":"dbRecordStock","item":"Espresso Machine","warehouse":"East"},{"as":"product-West","do":"dbRecordStock","item":"Espresso Machine","warehouse":"West"},{"as":"conservation-item-before","do":"recordNumber"},{"contains":"East","count":1,"do":"expect"},{"as":"conservation-east-before","do":"recordNumber"},{"contains":"West","count":1,"do":"expect"},{"as":"conservation-west-before","do":"recordNumber"},{"do":"fill","text":"East"},{"do":"fill","text":"West"},{"do":"fill","text":"17"},{"do":"click"},{"contains":"East","count":1,"do":"expect"},{"do":"dbExpectStock","item":"Espresso Machine","plus":-17,"relativeTo":"product-East","warehouse":"East"},{"do":"dbExpectStock","item":"Espresso Machine","plus":17,"relativeTo":"product-West","warehouse":"West"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","unlessVisible":"admin-revenue"},{"do":"expectNumber","plus":-17,"relativeTo":"conservation-east-before"},{"contains":"West","count":1,"do":"expect"},{"do":"expectNumber","plus":17,"relativeTo":"conservation-west-before"},{"do":"expectNumber","plus":0,"relativeTo":"conservation-item-before"}],"source":"scenarios/02-strengthened.json","stableKey":"ecommerce.inventory-operations.stock-conservation.202a","statedBy":"a transfer moves stock, it does not create or destroy it","withheld":null},{"category":"production","checkGroupId":"stock-transfer-overdraw","criterionId":"2c","description":"a transfer that would overdraw a warehouse is refused and changes neither warehouse nor the item total","featureId":2,"featureName":"Moving stock between warehouses","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":2,"provenBy":null,"requiresFeatures":["ecommerce.l2.stock-transfers-features"],"role":"guarantee","semantics":[{"as":"product-East","do":"dbRecordStock","item":"Headphones","warehouse":"East"},{"as":"product-West","do":"dbRecordStock","item":"Headphones","warehouse":"West"},{"as":"overdraw-item-before","do":"recordNumber"},{"as":"overdraw-east-before","do":"recordNumber"},{"as":"overdraw-west-before","do":"recordNumber"},{"do":"fill","text":"East"},{"do":"fill","text":"West"},{"do":"fill","text":"99999"},{"do":"click"},{"do":"expect"},{"do":"dbExpectStock","item":"Headphones","plus":0,"relativeTo":"product-East","warehouse":"East"},{"do":"dbExpectStock","item":"Headphones","plus":0,"relativeTo":"product-West","warehouse":"West"},{"do":"expectNumber","plus":0,"relativeTo":"overdraw-east-before"},{"do":"expectNumber","plus":0,"relativeTo":"overdraw-west-before"},{"do":"expectNumber","plus":0,"relativeTo":"overdraw-item-before"}],"source":"scenarios/02-transfer-overdraw.json","stableKey":"ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c","statedBy":"a transfer that would leave a warehouse short is refused and changes nothing","withheld":null},{"category":"production","checkGroupId":"stock-transfers","criterionId":"2b","description":"both warehouse totals move live and in opposite directions as stock is transferred","featureId":2,"featureName":"Warehouse totals","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":2,"provenBy":null,"requiresFeatures":["ecommerce.l2.stock-transfers-features"],"role":"guarantee","semantics":[{"as":"product-East","do":"dbRecordStock","item":"Headphones","warehouse":"East"},{"as":"product-West","do":"dbRecordStock","item":"Headphones","warehouse":"West"},{"contains":"East","count":1,"do":"expect"},{"as":"warehouse-east-before","do":"recordNumber"},{"contains":"West","count":1,"do":"expect"},{"as":"warehouse-west-before","do":"recordNumber"},{"do":"fill","text":"East"},{"do":"fill","text":"West"},{"do":"fill","text":"10"},{"do":"click"},{"contains":"East","count":1,"do":"expect"},{"do":"expectNumber","plus":-10,"relativeTo":"warehouse-east-before"},{"contains":"West","count":1,"do":"expect"},{"do":"expectNumber","plus":10,"relativeTo":"warehouse-west-before"},{"do":"dbExpectStock","item":"Headphones","plus":-10,"relativeTo":"product-East","warehouse":"East"},{"do":"dbExpectStock","item":"Headphones","plus":10,"relativeTo":"product-West","warehouse":"West"}],"source":"scenarios/02-transfer-totals.json","stableKey":"ecommerce.spec.live-state.stock-transfers.2b","statedBy":"the per-warehouse numbers staff and admins see both move at once","withheld":null},{"category":"production","checkGroupId":"cart-expiration","criterionId":"304a","description":"an inactive cart expires without a browser, releases stock, and returns empty","featureId":304,"featureName":"An inactive cart expires","note":null,"packId":"ecommerce.l3.cart-expiration-features","points":4,"provenBy":null,"role":"feature","semantics":[{"do":"wait"},{"do":"reload"},{"do":"expectNumber","plus":0,"relativeTo":"before"},{"do":"openClient"},{"do":"expect"},{"do":"expectNumber","equals":0}],"source":"scenarios/03-cart-expiration.json","stableKey":"ecommerce.l3.cart-expiration.cart-expiration.304a","statedBy":"A cart with no activity for five minutes expires and releases its reservations.","withheld":null},{"category":"production","checkGroupId":"scheduled-work-access","criterionId":"317a","description":"the server refuses customer scheduling and cancellation of restocks","featureId":317,"featureName":"Customers cannot manage scheduled restocks","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-access-specifications","points":3,"provenBy":null,"requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"role":"guarantee","semantics":[{"do":"fill","text":"Webcam"},{"do":"fill","text":"West"},{"do":"fill","text":"3"},{"do":"fill","text":"180"},{"action":"scheduleRestock","authentication":"actor","do":"callAction","from":"admin","input":{"attribute":"data-action-input","testid":"schedule-restock-submit"},"namedAction":{"args":["","",0,0],"id":"scheduleRestock","method":"POST","params":[{"in":"body","name":"item"},{"in":"body","name":"warehouse"},{"in":"body","name":"quantity"},{"in":"body","name":"delaySeconds"}],"path":"/api/admin/scheduled-restocks","reducer":"schedule_restock"}},{"do":"expectActionOutcome","outcome":"refused"},{"do":"replayAs","from":"admin","match":"DELETE","namedAction":{"args":[0],"id":"cancelScheduledRestock","method":"DELETE","params":[{"in":"path","name":"restockId","placeholder":"{restockId}","wireType":"u64"}],"path":"/api/admin/scheduled-restocks/{restockId}","reducer":"cancel_scheduled_restock"},"namedTarget":{"attribute":"data-entity-id","testid":"pending-restock-item","valueType":"string"}},{"do":"expectReplayRejected"},{"count":1,"do":"expect"},{"do":"click"}],"source":"scenarios/03-deferred-access.json","stableKey":"ecommerce.l3.deferred-access.scheduled-work-access.317a","statedBy":"Only an admin can schedule or cancel a restock.","withheld":null},{"category":"production","checkGroupId":"restart-survival","criterionId":"311a","description":"a restock scheduled before restart still applies","featureId":311,"featureName":"A scheduled restock survives restart","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-durability-specifications","points":4,"provenBy":null,"requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"role":"guarantee","semantics":[{"do":"dbExpectStock","item":"Air Purifier","plus":5,"relativeTo":"before"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"expect"},{"absent":true,"do":"expect"}],"source":"scenarios/03-deferred-durability.json","stableKey":"ecommerce.l3.deferred-durability.restart-survival.311a","statedBy":"Pending restocks survive a backend restart.","withheld":null},{"category":"production","checkGroupId":"restart-survival","criterionId":"314a","description":"a reservation pending before restart still expires and returns stock","featureId":314,"featureName":"A reservation survives restart","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-durability-specifications","points":4,"provenBy":null,"requiresFeatures":["ecommerce.l3.reservations-features"],"role":"guarantee","semantics":[{"atMost":70000,"do":"expectElapsed","since":"pending-314"},{"do":"reload"},{"atMost":70000,"do":"expectElapsed","since":"pending-314"},{"do":"expectNumber","plus":-1,"relativeTo":"before"},{"atMost":70000,"do":"expectElapsed","since":"pending-314"},{"do":"wait","since":"pending-314-accepted"},{"do":"reload"},{"do":"expectNumber","plus":0,"relativeTo":"before"},{"do":"reload"},{"do":"ensureSignedIn","name":"durable-reservation"},{"do":"click","unlessVisible":"cart-item"},{"do":"expect"}],"source":"scenarios/03-deferred-durability.json","stableKey":"ecommerce.l3.deferred-durability.restart-survival.314a","statedBy":"Pending reservations survive a backend restart.","withheld":null},{"category":"production","checkGroupId":"restart-survival","criterionId":"315a","description":"an order shipped before restart still becomes delivered","featureId":315,"featureName":"An order transition survives restart","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-durability-specifications","points":4,"provenBy":null,"requiresFeatures":["ecommerce.l3.order-delivery-features"],"role":"guarantee","semantics":[{"do":"wait","since":"delivery-start-accepted"},{"do":"reload"},{"do":"ensureSignedIn","name":"durable-delivery"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"delivered"}],"source":"scenarios/03-deferred-durability.json","stableKey":"ecommerce.l3.deferred-durability.restart-survival.315a","statedBy":"Pending order delivery survives a backend restart.","withheld":null},{"category":"production","checkGroupId":"restart-survival","criterionId":"316a","description":"a cart survives restart and expires near its original five-minute deadline","featureId":316,"featureName":"Cart expiration survives restart","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-durability-specifications","points":4,"provenBy":null,"requiresFeatures":["ecommerce.l3.cart-expiration-features"],"role":"guarantee","semantics":[{"atMost":250000,"do":"expectElapsed","since":"pending-316"},{"do":"reload"},{"do":"ensureSignedIn","name":"durable-cart"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"cart-item"},{"atMost":250000,"do":"expectElapsed","since":"pending-316"},{"do":"expectNumber","equals":1},{"atMost":250000,"do":"expectElapsed","since":"pending-316"},{"do":"wait","since":"pending-316-accepted"},{"do":"reload"},{"do":"ensureSignedIn","name":"durable-cart"},{"do":"click"},{"do":"expectNumber","equals":0},{"do":"expect"},{"do":"reload"},{"do":"expectNumber","plus":0,"relativeTo":"before"}],"source":"scenarios/03-deferred-durability.json","stableKey":"ecommerce.l3.deferred-durability.restart-survival.316a","statedBy":"Pending cart expiration survives a backend restart.","withheld":null},{"category":"production","checkGroupId":"exactly-once","criterionId":"311a","description":"restart cannot replay a completed restock","featureId":311,"featureName":"A restock applies once","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-integrity-specifications","points":3,"provenBy":null,"requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"dbExpectStock","item":"Air Purifier","plus":5,"relativeTo":"before"},{"do":"restartBackend"},{"do":"reload"},{"do":"wait"},{"do":"dbExpectStock","item":"Air Purifier","plus":5,"relativeTo":"before"}],"source":"scenarios/03-deferred-integrity.json","stableKey":"ecommerce.l3.deferred-integrity.exactly-once.311a","statedBy":"Restarting the backend cannot apply a restock more than once.","withheld":null},{"category":"production","checkGroupId":"exactly-once","criterionId":"312a","description":"restart leaves one delivered order record","featureId":312,"featureName":"Delivery applies once","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-integrity-specifications","points":3,"provenBy":null,"requiresFeatures":["ecommerce.l3.order-delivery-features"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click","ifAvailable":true},{"contains":"Desk Lamp","count":1,"do":"expect"},{"do":"expect","ignoreCase":true,"value":"delivered"}],"source":"scenarios/03-deferred-integrity.json","stableKey":"ecommerce.l3.deferred-integrity.exactly-once.312a","statedBy":"A delivered order is not duplicated by a backend restart.","withheld":null},{"category":"production","checkGroupId":"stock-conservation","criterionId":"313a","description":"expiry returns exactly the unit reserved","featureId":313,"featureName":"Reservation expiry conserves stock","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-integrity-specifications","points":3,"provenBy":null,"requiresFeatures":["ecommerce.l3.reservations-features"],"role":"guarantee","semantics":[{"do":"wait"},{"do":"reload"},{"do":"expectNumber","plus":0,"relativeTo":"before"}],"source":"scenarios/03-deferred-integrity.json","stableKey":"ecommerce.l3.deferred-integrity.stock-conservation.313a","statedBy":"Reservation expiry returns exactly the stock that the reservation took.","withheld":null},{"category":"production","checkGroupId":"stock-conservation","criterionId":"314a","description":"checkout does not decrement stock after the reservation already did","featureId":314,"featureName":"Checkout conserves reserved stock","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-integrity-specifications","points":3,"provenBy":null,"requiresFeatures":["ecommerce.l3.reservations-features"],"role":"guarantee","semantics":[{"do":"click"},{"do":"expectNumber","equals":0},{"do":"reload"},{"do":"ensureSignedIn","name":"conserve-checkout"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Headphones","do":"expectElementCount","equals":1},{"do":"reload"},{"do":"expectNumber","plus":-1,"relativeTo":"before"}],"source":"scenarios/03-deferred-integrity.json","stableKey":"ecommerce.l3.deferred-integrity.stock-conservation.314a","statedBy":"Checkout does not take reserved stock twice.","withheld":null},{"category":"production","checkGroupId":"order-delivery","criterionId":"303a","description":"a shipped order becomes delivered in customer and staff views","featureId":303,"featureName":"A shipped order becomes delivered","note":null,"packId":"ecommerce.l3.order-delivery-features","points":3,"provenBy":null,"role":"feature","semantics":[{"do":"wait"},{"do":"reload"},{"do":"ensureSignedIn","name":"delivery-live"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"delivered"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"completed-order-item"},{"do":"expect","ignoreCase":true,"value":"delivered"}],"source":"scenarios/03-order-delivery.json","stableKey":"ecommerce.l3.order-delivery.order-delivery.303a","statedBy":"A shipped order becomes delivered after 60 seconds.","withheld":null},{"category":"production","checkGroupId":"order-delivery","criterionId":"305a","description":"a cancelled order remains cancelled after the delivery interval","featureId":305,"featureName":"Cancellation is final","note":null,"packId":"ecommerce.l3.order-delivery-features","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"wait"},{"do":"reload"},{"do":"ensureSignedIn","name":"delivery-cancel"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"cancelled"}],"source":"scenarios/03-order-delivery.json","stableKey":"ecommerce.l3.order-delivery.order-delivery.305a","statedBy":"A cancelled order never advances.","withheld":null},{"category":"production","checkGroupId":"reservations","criterionId":"301a","description":"adding an item reserves one unit for every open viewer","featureId":301,"featureName":"A cart reserves stock","note":null,"packId":"ecommerce.l3.reservations-features","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"do":"expectNumber","plus":-1,"relativeTo":"before"}],"source":"scenarios/03-reservations.json","stableKey":"ecommerce.l3.reservations.reservations.301a","statedBy":"Adding an item to a cart reserves its stock immediately for 90 seconds.","withheld":null},{"category":"interface","checkGroupId":"reservations","criterionId":"305a","description":"the reservation timer decreases","featureId":305,"featureName":"Reservation time is visible","note":null,"packId":"ecommerce.l3.reservations-features","points":1,"provenBy":null,"role":"feature","semantics":[{"atLeast":1,"atMost":90,"do":"expectNumber"},{"as":"initial-countdown","do":"recordNumber"},{"do":"wait"},{"comparison":"atMost","do":"expectNumber","plus":-1,"relativeTo":"initial-countdown"}],"source":"scenarios/03-reservations.json","stableKey":"ecommerce.l3.reservations.reservations.305a","statedBy":"The cart shows the remaining reservation time.","withheld":null},{"category":"feature","checkGroupId":"reservations","criterionId":"306a","description":"checkout converts the reservation into an order and empties the cart","featureId":306,"featureName":"Checkout consumes a reservation","note":null,"packId":"ecommerce.l3.reservations-features","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"do":"expectNumber","equals":0},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Keyboard","count":1,"do":"expect"}],"source":"scenarios/03-reservations.json","stableKey":"ecommerce.l3.reservations.reservations.306a","statedBy":"Checkout converts a live reservation into a sale.","withheld":null},{"category":"feature","checkGroupId":"reservations","criterionId":"307a","description":"an expired reservation marks its cart line","featureId":307,"featureName":"A reservation expires","note":null,"packId":"ecommerce.l3.reservations-features","points":3,"provenBy":null,"role":"feature","semantics":[{"do":"wait"},{"do":"expect"}],"source":"scenarios/03-reservations.json","stableKey":"ecommerce.l3.reservations.reservations.307a","statedBy":"An expired reservation remains visible as expired.","withheld":null},{"category":"feature","checkGroupId":"reservations","criterionId":"308a","description":"raising quantity starts a new reservation window","featureId":308,"featureName":"Changing quantity renews a reservation","note":null,"packId":"ecommerce.l3.reservations-features","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"wait"},{"atLeast":35,"do":"expectNumber"},{"absent":true,"do":"expect"}],"source":"scenarios/03-reservations.json","stableKey":"ecommerce.l3.reservations.reservations.308a","statedBy":"Adding the item again renews the reservation.","withheld":null},{"category":"feature","checkGroupId":"scheduled-restocks","criterionId":"305a","description":"a due restock updates stock and moves to the ledger","featureId":305,"featureName":"A due restock applies","note":null,"packId":"ecommerce.l3.scheduled-restocks-features","points":3,"provenBy":null,"role":"feature","semantics":[{"as":"ledger-before","count":true,"do":"recordNumber"},{"do":"fill","text":"Keyboard"},{"do":"fill","text":"West"},{"do":"fill","text":"7"},{"do":"fill","text":"15"},{"do":"click"},{"do":"wait"},{"do":"reload"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"expectNumber","plus":7,"relativeTo":"before"},{"absent":true,"do":"expect"},{"do":"expectElementCount","plus":1,"relativeTo":"ledger-before"}],"source":"scenarios/03-scheduled-restock-apply.json","stableKey":"ecommerce.l3.scheduled-restocks.scheduled-restocks.305a","statedBy":"A due restock updates stock, leaves the pending list, and enters the stock ledger.","withheld":null},{"category":"production","checkGroupId":"scheduled-restocks","criterionId":"306a","description":"a cancelled restock never applies","featureId":306,"featureName":"A scheduled restock can be cancelled","note":null,"packId":"ecommerce.l3.scheduled-restocks-features","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"fill","text":"Desk Lamp"},{"do":"fill","text":"East"},{"do":"fill","text":"9"},{"do":"fill","text":"15"},{"do":"click"},{"count":1,"do":"expect"},{"do":"click"},{"do":"wait"},{"do":"reload"},{"do":"expectNumber","plus":0,"relativeTo":"before"}],"source":"scenarios/03-scheduled-restock-cancel.json","stableKey":"ecommerce.l3.scheduled-restocks.scheduled-restocks.306a","statedBy":"An admin can schedule and cancel a restock.","withheld":null},{"category":"feature","checkGroupId":"scheduled-restocks","criterionId":"302a","description":"a scheduled restock is pending and its remaining time decreases","featureId":302,"featureName":"A restock is pending before it is due","note":null,"packId":"ecommerce.l3.scheduled-restocks-features","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"fill","text":"Webcam"},{"do":"fill","text":"East"},{"do":"fill","text":"7"},{"do":"fill","text":"90"},{"do":"click"},{"count":1,"do":"expect"},{"atLeast":1,"atMost":90,"do":"expectNumber"},{"as":"initial-countdown","do":"recordNumber"},{"do":"wait"},{"comparison":"atMost","do":"expectNumber","plus":-1,"relativeTo":"initial-countdown"},{"do":"click"}],"source":"scenarios/03-scheduled-restocks.json","stableKey":"ecommerce.l3.scheduled-restocks.scheduled-restocks.302a","statedBy":"A pending restock shows its remaining time.","withheld":null},{"category":"production","checkGroupId":"server-time","criterionId":"312a","description":"restart preserves the due time and the work later completes","featureId":312,"featureName":"Restart does not run work early","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.server-time-specifications","points":3,"provenBy":null,"requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"role":"guarantee","semantics":[{"do":"wait"},{"do":"reload"},{"atMost":100000,"do":"expectElapsed","since":"restock-start"},{"do":"dbExpectStock","item":"Espresso Machine","plus":0,"relativeTo":"before"},{"count":1,"do":"expect"},{"atMost":100000,"do":"expectElapsed","since":"restock-start"},{"do":"wait","since":"restock-start-accepted"},{"do":"dbExpectStock","item":"Espresso Machine","plus":4,"relativeTo":"before"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"expect"},{"absent":true,"do":"expect"}],"source":"scenarios/03-server-time.json","stableKey":"ecommerce.l3.server-time.server-time.312a","statedBy":"A pending restock does not run early after a restart.","withheld":null},{"category":"production","checkGroupId":"server-time","criterionId":"313a","description":"a reservation expires while its browser is closed","featureId":313,"featureName":"A browser is not the clock","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.server-time-specifications","points":3,"provenBy":null,"requiresFeatures":["ecommerce.l3.reservations-features"],"role":"guarantee","semantics":[{"do":"wait"},{"do":"reload"},{"do":"expectNumber","plus":0,"relativeTo":"before"}],"source":"scenarios/03-server-time.json","stableKey":"ecommerce.l3.server-time.server-time.313a","statedBy":"A reservation expires without an open browser.","withheld":null},{"category":"production","checkGroupId":"account-state-recovery","criterionId":"105b","description":"the same account and cart survive the connection dropping and coming back","featureId":105,"featureName":"An account keeps what belongs to it","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.accounts","ecommerce.feature.checkout"],"role":"guarantee","semantics":[{"do":"setOffline","offline":true},{"do":"click"},{"do":"setOffline","offline":false},{"contains":"pat","do":"expect"},{"do":"click","unlessVisible":"cart-total"},{"contains":"Keyboard","do":"expect"},{"contains":"Headphones","do":"expect"}],"source":"scenarios/progression-account-state-reconnect.json","stableKey":"ecommerce.spec.state-durability.account-state-recovery.105b","statedBy":"the signed-in account and its current data remain available after reconnecting","withheld":null},{"category":"production","checkGroupId":"account-state-recovery","criterionId":"105a","description":"cart and order history survive reload and backend restart, including a fresh account login","featureId":105,"featureName":"An account keeps what belongs to it","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.accounts","ecommerce.feature.checkout"],"role":"guarantee","semantics":[{"do":"click"},{"do":"click","unlessVisible":"cart-total"},{"do":"click"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"click"},{"do":"click","unlessVisible":"cart-total"},{"contains":"Keyboard","do":"expect"},{"do":"reload"},{"contains":"pat","do":"expect"},{"do":"click","unlessVisible":"cart-total"},{"contains":"Keyboard","do":"expect"},{"do":"reload"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Headphones","count":1,"do":"expect"},{"do":"restartBackend"},{"do":"freshClient"},{"do":"signIn","name":"pat"},{"do":"click","unlessVisible":"cart-total"},{"contains":"Keyboard","count":1,"do":"expect"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"order-item"},{"contains":"Headphones","count":1,"do":"expect"}],"source":"scenarios/progression-account-state-reload.json","stableKey":"ecommerce.spec.state-durability.account-state-recovery.105a","statedBy":"the signed-in account, cart, and orders persist across a page reload","withheld":null},{"category":"production","checkGroupId":"automatic-reorder-access","criterionId":"502c","description":"a customer cannot see or replay automatic reorder management","featureId":502,"featureName":"Warehouse staff manage automatic reorder rules","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.automatic-reorder"],"role":"guarantee","semantics":[{"absent":true,"do":"expect"},{"do":"fill","text":"Desk Lamp"},{"do":"fill","text":"1"},{"do":"fill","text":"9"},{"action":"saveReorderRule","do":"callAction","from":"staff","input":{"attribute":"data-action-input","testid":"reorder-submit"},"namedAction":{"args":[0,1,9],"id":"saveReorderRule","method":"PUT","params":[{"in":"path","name":"itemId","placeholder":"{itemId}","wireType":"u64"},{"in":"body","name":"threshold"},{"in":"body","name":"quantity"}],"path":"/api/reorders/{itemId}","reducer":"save_reorder_rule"}},{"do":"expectActionOutcome","outcome":"refused","routeProvenBy":"staff"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"reorder-submit"},{"attribute":"data-threshold","contains":"Desk Lamp","count":1,"do":"expect","value":"2"},{"attribute":"data-quantity","contains":"Desk Lamp","do":"expect","value":"5"}],"source":"scenarios/progression-automatic-reorder-access.json","stableKey":"ecommerce.spec.access-control.automatic-reorder-access.502c","statedBy":"Only warehouse staff can manage automatic reorder rules.","withheld":null},{"category":"production","checkGroupId":"automatic-reorder-deduplication","criterionId":"502b","description":"more sales do not duplicate a pending restock","featureId":502,"featureName":"Warehouse staff manage automatic reorder rules","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.automatic-reorder"],"role":"guarantee","semantics":[{"action":"buy","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"do":"expectActionOutcome","outcome":"accepted"},{"action":"buy","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"dbExpectStock","equals":0,"item":"Desk Lamp"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"admin-revenue"},{"do":"click","ifAvailable":true},{"do":"expect"},{"do":"expectElementCount","equals":1},{"attribute":"data-quantity","contains":"Desk Lamp","count":1,"do":"expect","value":"5"}],"source":"scenarios/progression-automatic-reorder-duplicate.json","stableKey":"ecommerce.spec.transactional-integrity.automatic-reorder-deduplication.502b","statedBy":"A pending automatic restock is not scheduled twice.","withheld":null},{"category":"feature","checkGroupId":"automatic-reorder","criterionId":"502a","description":"crossing the threshold creates one pending restock","featureId":502,"featureName":"Warehouse staff manage automatic reorder rules","note":null,"packId":"ecommerce.progression.automatic-reorder","points":3,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"feature","semantics":[{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"admin-revenue"},{"do":"click","ifAvailable":true},{"do":"expect"},{"do":"expectElementCount","equals":0},{"action":"buy","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"dbExpectStock","equals":2,"item":"Desk Lamp"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"admin-revenue"},{"do":"click","ifAvailable":true},{"do":"expect"},{"do":"expectElementCount","equals":1},{"attribute":"data-quantity","contains":"Desk Lamp","count":1,"do":"expect","value":"5"}],"source":"scenarios/progression-automatic-reorder.json","stableKey":"ecommerce.progression.automatic-reorder.automatic-reorder.502a","statedBy":"Crossing a reorder threshold schedules a restock.","withheld":null},{"category":"production","checkGroupId":"books-balance","criterionId":"107a","description":"revenue rises by exactly what was bought","featureId":107,"featureName":"The books balance","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"admin-revenue"},{"do":"expectNumber","plus":58,"relativeTo":"revenue-before"}],"source":"scenarios/progression-books-balance.json","stableKey":"ecommerce.spec.transactional-integrity.books-balance.107a","statedBy":"total revenue equals the sum of order totals","withheld":null},{"category":"production","checkGroupId":"books-balance","criterionId":"107b","description":"what the store sold is what left the warehouses, and a fresh client agrees","featureId":107,"featureName":"The books balance","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"expectNumber","plus":-2,"relativeTo":"stand-before"},{"do":"freshClient"},{"do":"expectNumber","plus":-2,"relativeTo":"stand-before"}],"source":"scenarios/progression-books-balance.json","stableKey":"ecommerce.spec.transactional-integrity.books-balance.107b","statedBy":"warehouse totals and a fresh storefront agree with completed sales","withheld":null},{"category":"feature","checkGroupId":"bundle-checkout","criterionId":"741a","description":"adding a bundle reserves its components and checkout records the bundle price once","featureId":741,"featureName":"Bundle checkout","note":null,"packId":"ecommerce.feature.bundle-checkout","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"do":"dbExpectStock","equals":4,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":3,"item":"Desk Lamp","warehouse":"East"},{"do":"click"},{"do":"click"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Checkout bundle","count":1,"do":"expect"},{"do":"expectNumber","equals":75},{"do":"dbExpectStock","equals":4,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":3,"item":"Desk Lamp","warehouse":"East"},{"do":"dbExpectStock","equals":0,"item":"Keyboard","warehouse":"West"},{"do":"dbExpectStock","equals":0,"item":"Desk Lamp","warehouse":"West"}],"source":"scenarios/progression-bundle-checkout.json","stableKey":"ecommerce.feature.bundle-checkout.bundle-checkout.741a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"bundle-744","criterionId":"744a","description":"two competing reservations accept exactly one whole bundle without consuming extra components","featureId":744,"featureName":"Competing bundle reservations","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","points":3,"provenBy":null,"requiresFeatures":["ecommerce.feature.bundle-checkout"],"role":"guarantee","semantics":[{"action":"addBundleToCart","do":"callConcurrently","input":{"attribute":"data-bundle-input","contains":"Scarce bundle","testid":"bundle-card"},"namedAction":{"args":[0],"id":"addBundleToCart","params":[{"in":"body","name":"bundleId","wireType":"u64"}],"path":"/api/cart/bundles","reducer":"add_bundle_to_cart"}},{"accepted":1,"do":"expectCallOutcomes"},{"do":"dbExpectStock","equals":0,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":0,"item":"Desk Lamp","warehouse":"East"},{"do":"reload"},{"do":"reload"},{"do":"click"},{"do":"click"},{"contains":"Scarce bundle","do":"expectActorsWith","equals":1,"maxEach":1},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"dbExpectStock","equals":2,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":1,"item":"Desk Lamp","warehouse":"East"}],"source":"scenarios/progression-bundle-checkout.json","stableKey":"ecommerce.spec.bundle-integrity.bundle-744.744a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"bundle-745","criterionId":"745a","description":"a missing component refuses the reservation without taking available stock or adding a cart line","featureId":745,"featureName":"Incomplete bundle stock","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","points":3,"provenBy":null,"requiresFeatures":["ecommerce.feature.bundle-checkout"],"role":"guarantee","semantics":[{"action":"addBundleToCart","do":"callAction","input":{"attribute":"data-bundle-input","contains":"Unavailable bundle","testid":"bundle-card"},"namedAction":{"args":[0],"id":"addBundleToCart","params":[{"in":"body","name":"bundleId","wireType":"u64"}],"path":"/api/cart/bundles","reducer":"add_bundle_to_cart"}},{"do":"expectActionOutcome","outcome":"application-refused"},{"do":"dbExpectStock","equals":2,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":0,"item":"Desk Lamp","warehouse":"East"},{"do":"click"},{"absent":true,"contains":"Unavailable bundle","do":"expect"}],"source":"scenarios/progression-bundle-checkout.json","stableKey":"ecommerce.spec.bundle-integrity.bundle-745.745a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"bundle-746","criterionId":"746a","description":"an expired reservation releases each component once across a backend restart","featureId":746,"featureName":"Bundle reservation restart","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","points":3,"provenBy":null,"requiresFeatures":["ecommerce.feature.bundle-checkout"],"role":"guarantee","semantics":[{"do":"click"},{"do":"dbExpectStock","equals":4,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":3,"item":"Desk Lamp","warehouse":"East"},{"do":"restartBackend"},{"do":"wait"},{"do":"reload"},{"do":"ensureSignedIn","name":"bundle-expiry"},{"do":"click"},{"do":"expect"},{"do":"dbExpectStock","equals":6,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":4,"item":"Desk Lamp","warehouse":"East"},{"do":"restartBackend"},{"do":"dbExpectStock","equals":6,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":4,"item":"Desk Lamp","warehouse":"East"}],"source":"scenarios/progression-bundle-checkout.json","stableKey":"ecommerce.spec.bundle-integrity.bundle-746.746a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"bundle-747","criterionId":"747a","description":"two checkout requests consume one reservation and create one paid bundle","featureId":747,"featureName":"Repeated bundle checkout","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","points":3,"provenBy":null,"requiresFeatures":["ecommerce.feature.bundle-checkout"],"role":"guarantee","semantics":[{"action":"checkout","do":"callConcurrently"},{"accepted":1,"do":"expectCallOutcomes"},{"do":"reload"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Repeated bundle","count":1,"do":"expect"},{"do":"expectNumber","equals":75},{"do":"dbExpectStock","equals":4,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":3,"item":"Desk Lamp","warehouse":"East"}],"source":"scenarios/progression-bundle-checkout.json","stableKey":"ecommerce.spec.bundle-integrity.bundle-747.747a","statedBy":null,"withheld":null},{"category":"feature","checkGroupId":"bundle-returns","criterionId":"742a","description":"returning a shipped bundle refunds the paid price and restores original components after its definition changes","featureId":742,"featureName":"Historical bundle return","note":null,"packId":"ecommerce.feature.bundle-returns","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"fill","text":"Historical bundle"},{"do":"fill","text":"9.00"},{"do":"fill","text":"[{\"item\":\"Keyboard\",\"quantity\":1},{\"item\":\"Desk Lamp\",\"quantity\":3}]"},{"do":"click"},{"contains":"Historical bundle","do":"expect"},{"do":"click"},{"do":"expect","ignoreCase":true,"value":"returned"},{"do":"expectNumber","equals":75},{"do":"dbExpectStock","equals":6,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":4,"item":"Desk Lamp","warehouse":"East"},{"do":"dbExpectStock","equals":0,"item":"Keyboard","warehouse":"West"},{"do":"dbExpectStock","equals":0,"item":"Desk Lamp","warehouse":"West"}],"source":"scenarios/progression-bundle-returns.json","stableKey":"ecommerce.feature.bundle-returns.bundle-returns.742a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"bundle-742","criterionId":"742b","description":"replaying a completed bundle return after restart does not refund or restock it twice","featureId":742,"featureName":"Historical bundle return","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","points":3,"provenBy":null,"requiresFeatures":["ecommerce.feature.bundle-returns"],"role":"guarantee","semantics":[{"do":"restartBackend"},{"do":"reload"},{"do":"ensureSignedIn","name":"Historical bundle"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"action":"returnBundle","do":"callAction","input":{"attribute":"data-bundle-return-input","contains":"Historical bundle","testid":"order-item"},"namedAction":{"args":[0],"id":"returnBundle","params":[{"in":"path","name":"orderId","placeholder":":orderId","wireType":"u64"}],"path":"/api/bundle-orders/:orderId/return","reducer":"return_bundle"}},{"do":"reload"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"do":"expectNumber","equals":75},{"do":"dbExpectStock","equals":6,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":4,"item":"Desk Lamp","warehouse":"East"}],"source":"scenarios/progression-bundle-returns.json","stableKey":"ecommerce.spec.bundle-integrity.bundle-742.742b","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"bundle-748","criterionId":"748a","description":"another customer cannot return a paid bundle by submitting its order ID","featureId":748,"featureName":"Bundle return ownership","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","points":3,"provenBy":null,"requiresFeatures":["ecommerce.feature.bundle-returns"],"role":"guarantee","semantics":[{"action":"returnBundle","do":"callAction","from":"buyer","input":{"attribute":"data-bundle-return-input","contains":"Private bundle","testid":"order-item"},"namedAction":{"args":[0],"id":"returnBundle","params":[{"in":"path","name":"orderId","placeholder":":orderId","wireType":"u64"}],"path":"/api/bundle-orders/:orderId/return","reducer":"return_bundle"}},{"do":"expectActionOutcome","outcome":"application-refused"},{"do":"reload"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"do":"expect","ignoreCase":true,"value":"shipped"},{"do":"dbExpectStock","equals":4,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":3,"item":"Desk Lamp","warehouse":"East"},{"do":"click"},{"do":"expectNumber","equals":75},{"do":"dbExpectStock","equals":6,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":4,"item":"Desk Lamp","warehouse":"East"}],"source":"scenarios/progression-bundle-returns.json","stableKey":"ecommerce.spec.bundle-integrity.bundle-748.748a","statedBy":null,"withheld":null},{"category":"feature","checkGroupId":"cart","criterionId":"4a","description":"adding the same item twice raises its quantity instead of adding a second line","featureId":4,"featureName":"Account cart and checkout","note":null,"packId":"ecommerce.feature.cart","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"do":"wait"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"click"},{"do":"click","unlessVisible":"cart-total"},{"contains":"Headphones","count":1,"do":"expect"},{"do":"expectNumber","equals":2}],"source":"scenarios/progression-cart-checkout.json","stableKey":"ecommerce.feature.cart-checkout.cart.4a","statedBy":"Adding an existing cart item increases its quantity.","withheld":null},{"category":"feature","checkGroupId":"cart","criterionId":"4d","description":"checkout creates one order, reduces stock, and empties the cart","featureId":4,"featureName":"Account cart and checkout","note":null,"packId":"ecommerce.feature.checkout","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"expectNumber","equals":100},{"do":"click"},{"do":"click","unlessVisible":"cart-total"},{"do":"expectNumber","equals":1},{"do":"click"},{"do":"wait"},{"do":"reload"},{"do":"ensureSignedIn","name":"cart-checkout"},{"do":"click","unlessVisible":"cart-total"},{"do":"expectNumber","equals":0},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"expectNumber","equals":99},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Desk Lamp","count":1,"do":"expect"}],"source":"scenarios/progression-cart-checkout.json","stableKey":"ecommerce.feature.cart-checkout.cart.4d","statedBy":"Checkout creates one order, reduces stock, and empties the cart.","withheld":null},{"category":"feature","checkGroupId":"cart-recovery","criterionId":"503a","description":"restoring an expired cart reserves available items again","featureId":503,"featureName":"Expired carts restore only available items","note":null,"packId":"ecommerce.progression.cart-recovery","points":3,"provenBy":null,"role":"feature","semantics":[{"do":"reload"},{"do":"ensureSignedIn","name":"cart-recovery-available"},{"do":"expect"},{"do":"expectNumber","equals":0},{"as":"restore-available-shopper","do":"recordNumber"},{"do":"click"},{"do":"click"},{"contains":"Keyboard","do":"expect"},{"absent":true,"do":"expect"},{"do":"expectNumber","equals":1},{"do":"reload"},{"do":"ensureSignedIn","name":"cart-recovery-available"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"item-card"},{"do":"expectNumber","plus":-1,"relativeTo":"restore-available-shopper"}],"source":"scenarios/progression-cart-recovery.json","stableKey":"ecommerce.progression.cart-recovery.cart-recovery.503a","statedBy":"A customer can restore the available items from an expired cart.","withheld":null},{"category":"feature","checkGroupId":"cart-recovery","criterionId":"503b","description":"a partial restore keeps available items and names each unavailable item","featureId":503,"featureName":"Expired carts restore only available items","note":null,"packId":"ecommerce.progression.cart-recovery","points":3,"provenBy":null,"role":"feature","semantics":[{"do":"reload"},{"do":"ensureSignedIn","name":"cart-recovery-partial"},{"do":"expect"},{"do":"expectNumber","equals":0},{"as":"restore-partial-shopper","do":"recordNumber"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"warehouse":"West"},{"do":"click"},{"do":"click"},{"contains":"Gaming Mouse","do":"expect"},{"absent":true,"contains":"Desk Lamp","do":"expect"},{"contains":"Desk Lamp","do":"expect"},{"do":"expectNumber","equals":1},{"do":"reload"},{"do":"ensureSignedIn","name":"cart-recovery-partial"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"item-card"},{"do":"expectNumber","plus":-1,"relativeTo":"restore-partial-shopper"}],"source":"scenarios/progression-cart-recovery.json","stableKey":"ecommerce.progression.cart-recovery.cart-recovery.503b","statedBy":"Restore the available items and list each item that could not be restored.","withheld":null},{"category":"feature","checkGroupId":"catalog-management","criterionId":"622a","description":"a new product reaches the public catalog","featureId":622,"featureName":"Catalog management","note":null,"packId":"ecommerce.progression.catalog-management","points":2,"provenBy":null,"role":"feature","semantics":[{"contains":"Travel Mug","do":"expect"}],"source":"scenarios/progression-catalog-management.json","stableKey":"ecommerce.progression.catalog-management.catalog-management.622a","statedBy":"Authorized staff can add products.","withheld":null},{"category":"feature","checkGroupId":"catalog-management","criterionId":"622b","description":"the product exposes its named variants","featureId":622,"featureName":"Catalog management","note":null,"packId":"ecommerce.progression.catalog-management","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"openItem","item":"Travel Mug","unlessVisible":"item-variant"},{"contains":"Black","do":"expectElementCount","equals":1},{"contains":"Silver","do":"expectElementCount","equals":1}],"source":"scenarios/progression-catalog-management.json","stableKey":"ecommerce.progression.catalog-management.catalog-management.622b","statedBy":"Products can have named variants.","withheld":null},{"category":"production","checkGroupId":"checkout-crash-integrity","criterionId":"910a","description":"interrupted checkout recovers to the prepared cart or one complete order with the cart cleared after each independent process crash","featureId":910,"featureName":"Checkout crash recovery","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.checkout"],"role":"guarantee","semantics":[{"do":"expectCrashCheckout","from":"database-crash-observation","verdict":"atomicity"},{"do":"expectCrashCheckout","from":"application-crash-observation","verdict":"atomicity"}],"source":"scenarios/progression-checkout-crash.json","stableKey":"ecommerce.spec.state-durability.checkout-crash-integrity.910a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"checkout-crash-durability","criterionId":"910b","description":"acknowledged checkout is not rolled back and earlier orders remain unchanged after each independent process crash","featureId":910,"featureName":"Checkout crash recovery","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.checkout"],"role":"guarantee","semantics":[{"do":"expectCrashCheckout","from":"database-crash-observation","verdict":"durability"},{"do":"expectCrashCheckout","from":"application-crash-observation","verdict":"durability"}],"source":"scenarios/progression-checkout-crash.json","stableKey":"ecommerce.spec.state-durability.checkout-crash-durability.910b","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"payment-records","criterionId":"623a","description":"checkout records the exact paid amount","featureId":623,"featureName":"Payment records","note":null,"packId":"ecommerce.progression.payment-records","points":3,"provenBy":null,"role":"feature","semantics":[{"do":"expect","ignoreCase":true,"value":"paid"},{"do":"expectNumber","plus":0,"relativeTo":"payment-total"}],"source":"scenarios/progression-core-business.json","stableKey":"ecommerce.progression.payment-records.payment-records.623a","statedBy":"Checkout records the amount paid on the order.","withheld":null},{"category":"production","checkGroupId":"payment-deduplication","criterionId":"623b","description":"one checkout has one payment record","featureId":623,"featureName":"Payment records","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.payment-records"],"role":"guarantee","semantics":[{"do":"expectElementCount","equals":1}],"source":"scenarios/progression-core-business.json","stableKey":"ecommerce.spec.transactional-integrity.payment-deduplication.623b","statedBy":"A checkout does not create duplicate payments.","withheld":null},{"category":"feature","checkGroupId":"customer-profile","criterionId":"620c","description":"the owner can save and view a customer profile","featureId":620,"featureName":"Customer profile","note":null,"packId":"ecommerce.progression.customer-profile","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"profile-address-summary"},{"contains":"14 Market Street {user:profilemarker}","do":"expect"}],"source":"scenarios/progression-customer-profile.json","stableKey":"ecommerce.progression.customer-profile.customer-profile.620c","statedBy":"A signed-in customer can save and view their name and shipping address.","withheld":null},{"category":"production","checkGroupId":"customer-profile-reload","criterionId":"620a","description":"the saved profile survives reload and backend restart in a fresh browser","featureId":620,"featureName":"Customer profile","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.customer-profile"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"ensureSignedIn","name":"profile-owner"},{"do":"click"},{"contains":"14 Market Street {user:profilemarker}","do":"expect"},{"do":"restartBackend"},{"do":"freshClient"},{"do":"signIn","name":"profile-owner"},{"do":"click"},{"contains":"14 Market Street {user:profilemarker}","do":"expect"}],"source":"scenarios/progression-customer-profile.json","stableKey":"ecommerce.spec.state-durability.customer-profile-reload.620a","statedBy":"The saved profile remains after a reload.","withheld":null},{"category":"production","checkGroupId":"customer-profile-privacy","criterionId":"620b","description":"another customer neither sees nor receives the owner's private address","featureId":620,"featureName":"Customer profile","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.customer-profile"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"signUp","name":"profile-private-owner"},{"do":"click"},{"do":"fill","text":"Avery Stone"},{"do":"fill","text":"14 Market Street {user:profilemarker}"},{"do":"click"},{"do":"reload"},{"do":"ensureSignedIn","name":"profile-private-owner"},{"do":"click"},{"contains":"14 Market Street {user:profilemarker}","do":"expect"},{"contains":"14 Market Street {user:profilemarker}","do":"expectReceived"},{"do":"reload"},{"do":"signUp","name":"profile-other"},{"do":"click"},{"absent":true,"contains":"14 Market Street {user:profilemarker}","do":"expect"},{"contains":"14 Market Street {user:profilemarker}","do":"expectNotReceived"}],"source":"scenarios/progression-customer-profile.json","stableKey":"ecommerce.spec.access-control.customer-profile-privacy.620b","statedBy":"Each customer can see only their own profile.","withheld":null},{"category":"feature","checkGroupId":"delivery-notification-delivery","criterionId":"501a","description":"the order owner receives one delivery notification","featureId":501,"featureName":"Delivery creates one private notification","note":null,"packId":"ecommerce.progression.delivery-notifications","points":3,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"feature","semantics":[{"do":"reload"},{"do":"ensureSignedIn","name":"delivery-owner"},{"do":"click","unlessVisible":"notifications-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"contains":"Desk Lamp","do":"expectElementCount","equals":1}],"source":"scenarios/progression-delivery-notifications.json","stableKey":"ecommerce.progression.delivery-notifications.delivery-notification-delivery.501a","statedBy":"A delivered order creates one notification for its owner.","withheld":null},{"category":"production","checkGroupId":"delivery-notification-privacy","criterionId":"501b","description":"another customer cannot see the delivery notification","featureId":501,"featureName":"Delivery creates one private notification","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.delivery-notifications","ecommerce.progression.fulfilment-queue"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"ensureSignedIn","name":"delivery-owner"},{"do":"click","unlessVisible":"notifications-panel"},{"contains":"Desk Lamp","do":"expectElementCount","equals":1},{"do":"click","unlessVisible":"notifications-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"absent":true,"contains":"Desk Lamp","do":"expect"}],"source":"scenarios/progression-delivery-notifications.json","stableKey":"ecommerce.spec.access-control.delivery-notification-privacy.501b","statedBy":"Delivery notifications are private to the order owner.","withheld":null},{"category":"feature","checkGroupId":"faceted-search","criterionId":"401a","description":"category, price, and availability filters apply together","featureId":401,"featureName":"Filters compose","note":null,"packId":"ecommerce.progression.faceted-search","points":3,"provenBy":null,"role":"feature","semantics":[{"contains":"Coffee Grinder","do":"expectElementCount","equals":1},{"do":"click"},{"do":"click","ifAvailable":true},{"do":"waitUntilAbsent"},{"contains":"Air Purifier","do":"expectElementCount","equals":1},{"absent":true,"contains":"USB Cable","do":"expect"},{"absent":true,"contains":"Desk Lamp","do":"expect"},{"absent":true,"contains":"Espresso Machine","do":"expect"},{"absent":true,"contains":"Gaming Mouse","do":"expect"}],"source":"scenarios/progression-faceted-filters.json","stableKey":"ecommerce.progression.faceted-search.faceted-search.401a","statedBy":"Apply all selected filters together","withheld":null},{"category":"feature","checkGroupId":"faceted-search","criterionId":"402a","description":"moving between pages returns the same ordered items without duplicates","featureId":402,"featureName":"Pages are stable","note":null,"packId":"ecommerce.progression.faceted-search","points":3,"provenBy":null,"role":"feature","semantics":[{"do":"expectSequence","equals":["Air Purifier","Bluetooth Speaker","Coffee Grinder","Desk Lamp","Espresso Machine","Gaming Mouse","Headphones","Induction Cooktop","Keyboard","Laptop Stand"]},{"do":"click"},{"do":"expectSequence","equals":["Mirrorless Camera","USB Cable","Webcam"]},{"do":"click"},{"do":"expectSequence","equals":["Air Purifier","Bluetooth Speaker","Coffee Grinder","Desk Lamp","Espresso Machine","Gaming Mouse","Headphones","Induction Cooktop","Keyboard","Laptop Stand"]}],"source":"scenarios/progression-faceted-pagination.json","stableKey":"ecommerce.progression.faceted-search.faceted-search.402a","statedBy":"Moving between pages must not omit or repeat an item","withheld":null},{"category":"production","checkGroupId":"managed-support-privacy","criterionId":"613b","description":"another customer cannot read or reply to the managed case","featureId":613,"featureName":"Managed support privacy","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"provenBy":null,"requiresFeatures":["ecommerce.progression.managed-support"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"ensureSignedIn","name":"managed-private-owner"},{"do":"click"},{"contains":"Private managed case {user:casemarker}","do":"expect"},{"contains":"Private managed case {user:casemarker}","do":"expectReceived"},{"absent":true,"contains":"Private managed case {user:casemarker}","do":"expect"},{"contains":"Private managed case {user:casemarker}","do":"expectNotReceived"},{"do":"fill","text":"Owner-only update"},{"do":"click"},{"do":"replayAs","from":"owner","match":"Owner-only update","namedAction":{"args":[0,"Owner-only update"],"id":"replySupport","method":"POST","params":[{"in":"path","name":"ticketId","placeholder":":id","wireType":"u64"},{"in":"body","name":"body"}],"path":"/api/support/:id/replies","reducer":"reply_support"},"namedTarget":{"attribute":"data-entity-id","contains":"Private managed case {user:casemarker}","testid":"support-ticket","valueType":"string"}},{"allowNotFound":true,"do":"expectReplayRejected"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click","ifAvailable":true,"unlessVisible":"support-assignee"},{"do":"click","ifAvailable":true,"unlessVisible":"support-assignee"},{"contains":"Private managed case {user:casemarker}","do":"expect"},{"contains":"Owner-only update","do":"expectElementCount","equals":1},{"contains":"Owner-only update","do":"expectReceived"},{"contains":"Owner-only update","do":"expectNotReceived"}],"source":"scenarios/progression-managed-support-privacy.json","stableKey":"ecommerce.spec.access-control.managed-support-privacy.613b","statedBy":"A customer can access only cases that belong to their account.","withheld":null},{"category":"feature","checkGroupId":"managed-support","criterionId":"613c","description":"staff can update a support case and the customer can reply","featureId":613,"featureName":"Shared managed support case","note":null,"packId":"ecommerce.progression.managed-support","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"fill","text":"in progress"},{"do":"click"},{"do":"fill","text":"Case received."},{"do":"click"},{"do":"reload"},{"do":"ensureSignedIn","name":"managed-shared-owner"},{"do":"click"},{"contains":"in progress","do":"expect"},{"contains":"Case received.","do":"expect"},{"do":"fill","text":"Thank you."},{"do":"click"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click","unlessVisible":"support-assignee"},{"do":"click","ifAvailable":true,"unlessVisible":"support-assignee"},{"contains":"Thank you.","do":"expect"}],"source":"scenarios/progression-managed-support-shared.json","stableKey":"ecommerce.progression.managed-support.managed-support.613c","statedBy":"Customers and staff can exchange replies and update the status of a support case.","withheld":null},{"category":"production","checkGroupId":"managed-support","criterionId":"613a","description":"the customer and staff see the same replies and status live","featureId":613,"featureName":"Shared managed support case","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.managed-support"],"role":"guarantee","semantics":[{"do":"fill","text":"open"},{"do":"click"},{"contains":"open","do":"expect"},{"do":"reload"},{"do":"ensureSignedIn","name":"managed-shared-owner"},{"do":"click","unlessVisible":"support-ticket"},{"contains":"open","do":"expect"},{"do":"fill","text":"in progress"},{"do":"click"},{"do":"fill","text":"We are investigating."},{"do":"click"},{"contains":"in progress","do":"expect"},{"contains":"We are investigating.","do":"expect"},{"do":"fill","text":"Thank you for the update."},{"do":"click"},{"contains":"Thank you for the update.","do":"expect"}],"source":"scenarios/progression-managed-support-shared.json","stableKey":"ecommerce.spec.live-state.managed-support.613a","statedBy":"Customers and authorized staff use one shared support case.","withheld":null},{"category":"feature","checkGroupId":"notification-preferences","criterionId":"630c","description":"the customer can save a notification choice","featureId":630,"featureName":"Account notification preferences","note":null,"packId":"ecommerce.progression.notification-preferences","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"notification-order"},{"attribute":"data-state","do":"expect","value":"on"}],"source":"scenarios/progression-notification-preferences.json","stableKey":"ecommerce.progression.notification-preferences.notification-preferences.630c","statedBy":"Signed-in customers can turn order and stock notifications on or off.","withheld":null},{"category":"production","checkGroupId":"notification-preferences-reload","criterionId":"630a","description":"notification choices survive reload and backend restart in a fresh browser","featureId":630,"featureName":"Account notification preferences","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.notification-preferences"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"ensureSignedIn","name":"notification-owner"},{"do":"click","unlessVisible":"notification-order"},{"attribute":"data-state","do":"expect","value":"on"},{"do":"restartBackend"},{"do":"freshClient"},{"do":"signIn","name":"notification-owner"},{"do":"click","unlessVisible":"notification-order"},{"attribute":"data-state","do":"expect","value":"on"}],"source":"scenarios/progression-notification-preferences.json","stableKey":"ecommerce.spec.state-durability.notification-preferences-reload.630a","statedBy":"Notification choices persist for the account.","withheld":null},{"category":"production","checkGroupId":"notification-preferences-privacy","criterionId":"630b","description":"the owner's choice does not change another account","featureId":630,"featureName":"Account notification preferences","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.notification-preferences"],"role":"guarantee","semantics":[{"do":"click","unlessVisible":"notification-order"},{"attribute":"data-state","do":"expect","value":"off"}],"source":"scenarios/progression-notification-preferences.json","stableKey":"ecommerce.spec.access-control.notification-preferences-privacy.630b","statedBy":"One customer's notification choices do not affect another customer.","withheld":null},{"category":"production","checkGroupId":"open-list","criterionId":"902a","description":"one customer has the Keyboard's reviews open before another posts one; the already-open view shows that review exactly once","featureId":902,"featureName":"An open list stays current","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","semantics":[{"do":"openItem","item":"Keyboard"},{"do":"expect"},{"do":"click","ifAvailable":true,"unlessVisible":"review-rating"},{"do":"fill","text":"live-review-kbd"},{"do":"click"},{"contains":"live-review-kbd","do":"expectElementCount","equals":1},{"contains":"live-review-kbd","do":"expectElementCount","equals":1}],"source":"scenarios/progression-open-list-live.json","stableKey":"ecommerce.spec.live-state.open-list.902a","statedBy":"A view opened while a review is submitted converges to the current review list.","withheld":null},{"category":"interface","checkGroupId":"cancellation-and-return","criterionId":"3e","description":"a pending order does not offer a return button","featureId":331,"featureName":"Pending order return boundary","note":null,"packId":"ecommerce.l3.order-returns-features","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"feature","semantics":[{"absent":true,"do":"expect"}],"source":"scenarios/progression-order-return-boundary.json","stableKey":"ecommerce.returns-pricing.cancellation-and-return.3e","statedBy":"An item from a pending order cannot be returned.","withheld":null},{"category":"production","checkGroupId":"cancellation-and-return","criterionId":"3f","description":"the server refuses a pending return without changing stock or revenue","featureId":332,"featureName":"Pending return server boundary","note":null,"packId":"ecommerce.l3.order-returns-features","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"feature","semantics":[{"action":"returnItem","do":"callAction","input":{"attribute":"data-return-input","contains":"Desk Lamp","testid":"order-line"},"namedAction":{"args":[0,0],"id":"returnItem","method":"POST","params":[{"in":"path","name":"orderId","placeholder":"{orderId}","wireType":"u64"},{"in":"path","name":"itemId","placeholder":"{itemId}","wireType":"u64"}],"path":"/api/orders/{orderId}/items/{itemId}/return","reducer":"return_order_item"}},{"do":"expectActionOutcome","outcome":"validation-refused"},{"do":"reload"},{"do":"ensureSignedIn","name":"return-boundary"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"pending"},{"do":"dbExpectStock","item":"Desk Lamp","plus":0,"relativeTo":"pending-East","warehouse":"East"},{"do":"dbExpectStock","item":"Desk Lamp","plus":0,"relativeTo":"pending-West","warehouse":"West"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"admin-revenue"},{"do":"expectNumber","plus":0,"relativeTo":"pending-revenue"}],"source":"scenarios/progression-order-return-boundary.json","stableKey":"ecommerce.returns-pricing.cancellation-and-return.3f","statedBy":"Only shipped items can be returned.","withheld":null},{"category":"production","checkGroupId":"cancellation-and-return","criterionId":"3c","description":"returning a shipped item restores stock and revenue and marks the item returned","featureId":330,"featureName":"Completed order return","note":null,"packId":"ecommerce.l3.order-returns-features","points":3,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"feature","semantics":[{"do":"click"},{"contains":"Keyboard","containsText":"returned","do":"expect","ignoreCase":true},{"do":"freshClient"},{"do":"signIn","name":"return-complete"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Keyboard","containsText":"returned","do":"expect","ignoreCase":true},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","unlessVisible":"admin-revenue"},{"do":"expectNumber","plus":0,"relativeTo":"return-revenue-before"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"dbExpectStock","item":"Keyboard","plus":0,"relativeTo":"return-stock-before"}],"source":"scenarios/progression-order-return-complete.json","stableKey":"ecommerce.returns-pricing.cancellation-and-return.3c","statedBy":"An accepted return marks the item as returned, restores stock, and reduces revenue by the price paid.","withheld":null},{"category":"production","checkGroupId":"order-support-ownership","criterionId":"614b","description":"another customer cannot attach or inspect the owner's order","featureId":614,"featureName":"Order support ownership boundary","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.order-support"],"role":"guarantee","semantics":[{"absent":true,"contains":"Desk Lamp","do":"expect"},{"action":"linkSupportOrder","authentication":"actor","do":"callAction","input":{"attribute":"data-action-input","testid":"support-link-order"},"namedAction":{"args":[0,0],"id":"linkSupportOrder","method":"POST","params":[{"in":"path","name":"caseId","placeholder":"{caseId}","wireType":"u64"},{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/support/cases/{caseId}/order","reducer":"link_support_order"}},{"do":"expectActionOutcome","outcome":"accepted"},{"contains":"Desk Lamp","do":"expectElementCount","equals":1},{"action":"linkSupportOrder","authentication":"actor","do":"callAction","from":"owner","input":{"attribute":"data-action-input","overrides":{"caseId":{"actor":"other","attribute":"data-entity-id","contains":"Other order case","testid":"support-ticket"}},"testid":"support-link-order"},"namedAction":{"args":[0,0],"id":"linkSupportOrder","method":"POST","params":[{"in":"path","name":"caseId","placeholder":"{caseId}","wireType":"u64"},{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/support/cases/{caseId}/order","reducer":"link_support_order"}},{"do":"expectActionOutcome","outcome":"refused"},{"do":"freshClient"},{"do":"signIn","name":"order-boundary-other"},{"do":"click"},{"absent":true,"contains":"Desk Lamp","do":"expect"},{"contains":"Desk Lamp","do":"expectElementCount","equals":1}],"source":"scenarios/progression-order-support-boundary.json","stableKey":"ecommerce.spec.access-control.order-support-ownership.614b","statedBy":"A customer cannot attach or inspect another customer's order.","withheld":null},{"category":"feature","checkGroupId":"order-support-owned","criterionId":"614a","description":"the customer can link their order and staff can inspect it","featureId":614,"featureName":"Owned order support link","note":null,"packId":"ecommerce.progression.order-support","points":3,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"do":"click"},{"do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click","unlessVisible":"support-assignee"},{"do":"click","ifAvailable":true,"unlessVisible":"support-assignee"},{"contains":"Desk Lamp","do":"expect"}],"source":"scenarios/progression-order-support-owned.json","stableKey":"ecommerce.progression.order-support.order-support-owned.614a","statedBy":"A customer can attach one of their orders to a support case.","withheld":null},{"category":"feature","checkGroupId":"personalized-recommendations","criterionId":"403a","description":"recommendations follow the customer's categories, global sales, and name tie-break","featureId":403,"featureName":"Recommendations use customer activity","note":null,"packId":"ecommerce.progression.personalized-recommendations","points":4,"provenBy":null,"role":"feature","semantics":[{"contains":"Headphones","do":"expect"},{"do":"expectNumber","equals":1},{"do":"expectSequence","equals":["Gaming Mouse","Laptop Stand","Webcam"]}],"source":"scenarios/progression-personalized-recommendations.json","stableKey":"ecommerce.progression.personalized-recommendations.personalized-recommendations.403a","statedBy":"Order the remaining items by global units sold, highest first, then by item name.","withheld":null},{"category":"production","checkGroupId":"recommendation-profile-isolation","criterionId":"403b","description":"one customer's activity does not replace another customer's recommendations","featureId":403,"featureName":"Recommendations use customer activity","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.personalized-recommendations"],"role":"guarantee","semantics":[{"do":"click"},{"do":"waitUntilAbsent"},{"do":"expectNumber","equals":1},{"do":"expectSequence","equals":["Gaming Mouse","Laptop Stand","Webcam"]}],"source":"scenarios/progression-personalized-recommendations.json","stableKey":"ecommerce.spec.access-control.recommendation-profile-isolation.403b","statedBy":"Customer recommendation profiles are isolated.","withheld":null},{"category":"production","checkGroupId":"price-history","criterionId":"4c","description":"a price change updates an open cart and direct checkout persists the new total","featureId":420,"featureName":"Open cart price changes","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.price-history-specifications","points":3,"provenBy":null,"requiresFeatures":["ecommerce.feature.checkout","ecommerce.l2.price-history-features"],"role":"guarantee","semantics":[{"do":"fill","enter":true,"text":"Desk Lamp"},{"do":"click"},{"do":"click"},{"do":"expectNumber","equals":42},{"do":"fill","text":"52.00"},{"do":"click"},{"do":"expectNumber","equals":52},{"action":"checkout","do":"callAction","namedAction":{"args":[],"id":"checkout","method":"POST","path":"/api/checkout","reducer":"checkout"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"freshClient"},{"do":"signIn","name":"price-cart"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Desk Lamp","count":1,"do":"expect"},{"do":"expectNumber","equals":52}],"source":"scenarios/progression-price-cart-checkout.json","stableKey":"ecommerce.returns-pricing.price-history.4c","statedBy":"The public catalog and every open cart show the new price without a reload. Checkout uses the current price.","withheld":null},{"category":"feature","checkGroupId":"product-bundles","criterionId":"740a","description":"a saved bundle shows the exact price and component quantities after reopening the application","featureId":740,"featureName":"Bundle definitions","note":null,"packId":"ecommerce.feature.product-bundles","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"fill","text":"Office bundle"},{"do":"fill","text":"75.00"},{"do":"fill","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"},{"do":"click"},{"contains":"Office bundle","do":"expect"},{"do":"reload"},{"do":"click","unlessVisible":"bundle-card"},{"do":"expectNumber","equals":75},{"do":"expectElementCount","equals":2},{"attribute":"data-quantity","contains":"Keyboard","do":"expect","value":"2"},{"attribute":"data-quantity","contains":"Desk Lamp","do":"expect","value":"1"}],"source":"scenarios/progression-product-bundles.json","stableKey":"ecommerce.feature.product-bundles.product-bundles.740a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"bundle-743","criterionId":"743a","description":"a customer cannot replace a staff-created bundle through the application write","featureId":743,"featureName":"Bundle management authorization","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","points":3,"provenBy":null,"requiresFeatures":["ecommerce.feature.product-bundles"],"role":"guarantee","semantics":[{"do":"fill","text":"1.00"},{"action":"saveBundle","do":"callAction","from":"admin","input":{"attribute":"data-bundle-save-input","testid":"bundle-save"},"namedAction":{"args":["Protected bundle",1,"[{\"item\":\"Keyboard\",\"quantity\":1}]"],"id":"saveBundle","params":[{"in":"body","name":"name"},{"in":"body","name":"price"},{"in":"body","name":"componentsJson"}],"path":"/api/bundles","reducer":"save_bundle"}},{"do":"expectActionOutcome","outcome":"application-refused"},{"do":"reload"},{"do":"click","unlessVisible":"bundle-card"},{"do":"expectNumber","equals":75},{"attribute":"data-quantity","contains":"Keyboard","do":"expect","value":"2"}],"source":"scenarios/progression-product-bundles.json","stableKey":"ecommerce.spec.bundle-integrity.bundle-743.743a","statedBy":null,"withheld":null},{"category":"feature","checkGroupId":"promotion-checkout-active","criterionId":"621a","description":"an active promotion changes checkout and is recorded on the order","featureId":621,"featureName":"Bounded promotions at checkout","note":null,"packId":"ecommerce.progression.promotion-checkout","points":3,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"do":"click"},{"do":"fill","text":"LIVE10"},{"do":"click"},{"do":"click"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"do":"expectNumber","equals":8.9}],"source":"scenarios/progression-promotion-checkout.json","stableKey":"ecommerce.progression.promotion-checkout.promotion-checkout-active.621a","statedBy":"An active promotion applies to the cart and the order records its discount.","withheld":null},{"category":"feature","checkGroupId":"promotion-checkout-expired","criterionId":"621b","description":"an expired promotion is refused","featureId":621,"featureName":"Bounded promotions at checkout","note":null,"packId":"ecommerce.progression.promotion-checkout","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"do":"click"},{"do":"fill","text":"OLD10"},{"do":"click"},{"do":"expect"}],"source":"scenarios/progression-promotion-checkout.json","stableKey":"ecommerce.progression.promotion-checkout.promotion-checkout-expired.621b","statedBy":"Expired promotions cannot be applied.","withheld":null},{"category":"feature","checkGroupId":"promotion-checkout-exhausted","criterionId":"621c","description":"a fully redeemed promotion is refused","featureId":621,"featureName":"Bounded promotions at checkout","note":null,"packId":"ecommerce.progression.promotion-checkout","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"do":"click"},{"do":"fill","text":"ONCE10"},{"do":"click"},{"do":"click"},{"do":"click"},{"do":"click"},{"do":"fill","text":"ONCE10"},{"do":"click"},{"do":"expect"}],"source":"scenarios/progression-promotion-checkout.json","stableKey":"ecommerce.progression.promotion-checkout.promotion-checkout-exhausted.621c","statedBy":"A promotion cannot be used after its redemption limit is reached.","withheld":null},{"category":"feature","checkGroupId":"promotion-report-redemptions","criterionId":"622a","description":"the promotion report has the exact redemption count","featureId":622,"featureName":"Exact promotion totals","note":null,"packId":"ecommerce.progression.promotion-reporting","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"expectNumber","equals":1}],"source":"scenarios/progression-promotion-reporting.json","stableKey":"ecommerce.progression.promotion-reporting.promotion-report-redemptions.622a","statedBy":"Promotion redemption counts match orders that used the promotion.","withheld":null},{"category":"feature","checkGroupId":"promotion-report-revenue","criterionId":"622b","description":"the promotion report has the exact discounted revenue","featureId":622,"featureName":"Exact promotion totals","note":null,"packId":"ecommerce.progression.promotion-reporting","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"expectNumber","equals":80.1}],"source":"scenarios/progression-promotion-reporting.json","stableKey":"ecommerce.progression.promotion-reporting.promotion-report-revenue.622b","statedBy":"Promotion revenue after discounts matches orders that used the promotion.","withheld":null},{"category":"feature","checkGroupId":"promotion-rule-values","criterionId":"620a","description":"staff can save every bounded promotion value","featureId":620,"featureName":"Staff-managed promotion rules","note":null,"packId":"ecommerce.progression.promotion-rules","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"click","unlessVisible":"promotion-code"},{"do":"fill","text":"SAVE10"},{"do":"fill","text":"10"},{"do":"fill","text":"2099-01-01"},{"do":"fill","text":"2099-12-31"},{"do":"fill","text":"2"},{"do":"click"},{"do":"expectNumber","equals":10},{"contains":"2099-01-01","do":"expect"},{"contains":"2099-12-31","do":"expect"},{"do":"expectNumber","equals":2}],"source":"scenarios/progression-promotion-rules.json","stableKey":"ecommerce.progression.promotion-rules.promotion-rule-values.620a","statedBy":"Staff can create a promotion code with a discount, active period, and redemption limit.","withheld":null},{"category":"production","checkGroupId":"promotion-management-boundary","criterionId":"620b","description":"customers cannot open promotion management","featureId":620,"featureName":"Staff-managed promotion rules","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.promotion-rules"],"role":"guarantee","semantics":[{"do":"click","unlessVisible":"promotion-code"},{"do":"fill","text":"ACCESS10"},{"do":"fill","text":"10"},{"do":"fill","text":"2099-01-01"},{"do":"fill","text":"2099-12-31"},{"do":"fill","text":"2"},{"do":"click"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click","ifAvailable":true,"unlessVisible":"promotion-code"},{"do":"click","unlessVisible":"promotion-code"},{"contains":"ACCESS10","do":"expect"},{"absent":true,"do":"expect"},{"do":"replayAs","from":"staff","match":"ACCESS10","namedAction":{"args":["ACCESS10",10,4070908800000000,4102444740000000,2],"id":"createPromotion","params":[{"in":"body","name":"code"},{"in":"body","name":"discountPercent"},{"in":"body","name":"startMicros"},{"in":"body","name":"endMicros"},{"in":"body","name":"usageLimit"}],"path":"/api/promotions","reducer":"create_promotion"},"swap":{"find":"ACCESS10","with":"HACK10"}},{"do":"expectReplayRejected"},{"absent":true,"contains":"HACK10","do":"expect"}],"source":"scenarios/progression-promotion-rules.json","stableKey":"ecommerce.spec.access-control.promotion-management-boundary.620b","statedBy":"Customers cannot manage promotion rules.","withheld":null},{"category":"feature","checkGroupId":"purchase-order","criterionId":"3c","description":"the purchase is recorded in the buyer's order history at the price paid","featureId":3,"featureName":"Purchase order history","note":null,"packId":"ecommerce.feature.purchasing","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Coffee Grinder","do":"expect"},{"do":"expectNumber","equals":64}],"source":"scenarios/progression-purchasing.json","stableKey":"ecommerce.feature.purchasing.purchase-order.3c","statedBy":"A purchase creates an order for the customer at the price paid.","withheld":null},{"category":"feature","checkGroupId":"recommendation-feedback","criterionId":"504a","description":"dismissing a recommendation removes it from the customer view","featureId":504,"featureName":"Recommendation dismissal","note":null,"packId":"ecommerce.progression.recommendation-feedback","points":2,"provenBy":null,"role":"feature","semantics":[{"absent":true,"contains":"Headphones","do":"expect"}],"source":"scenarios/progression-recommendation-feedback.json","stableKey":"ecommerce.progression.recommendation-feedback.recommendation-feedback.504a","statedBy":"Customers can dismiss a recommendation.","withheld":null},{"category":"production","checkGroupId":"recommendation-feedback-privacy","criterionId":"504b","description":"one customer's dismissal does not hide another customer's result","featureId":504,"featureName":"Recommendation dismissal","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.recommendation-feedback"],"role":"guarantee","semantics":[{"do":"waitUntilAbsent"},{"contains":"Headphones","do":"expect"}],"source":"scenarios/progression-recommendation-feedback.json","stableKey":"ecommerce.spec.access-control.recommendation-feedback-privacy.504b","statedBy":"Recommendation feedback belongs to one customer.","withheld":null},{"category":"production","checkGroupId":"recommendation-feedback-restart","criterionId":"504c","description":"a dismissed recommendation stays absent after reload and backend restart in a fresh browser","featureId":504,"featureName":"Recommendation dismissal","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.recommendation-feedback"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"ensureSignedIn","name":"feedback-owner"},{"absent":true,"contains":"Headphones","do":"expect"},{"do":"restartBackend"},{"do":"freshClient"},{"do":"signIn","name":"feedback-owner"},{"do":"expect"},{"absent":true,"contains":"Headphones","do":"expect"}],"source":"scenarios/progression-recommendation-feedback.json","stableKey":"ecommerce.spec.state-durability.recommendation-feedback-restart.504c","statedBy":"A dismissed recommendation stays dismissed for that customer.","withheld":null},{"category":"production","checkGroupId":"review-eligibility-direct","criterionId":"618a","description":"a nonbuyer cannot submit or replace a review by claiming a buyer's username","featureId":618,"featureName":"Review access","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.review-access-specifications","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","semantics":[{"action":"submitReview","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0,5,"Original buyer review"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"}],"path":"/api/items/:id/reviews","reducer":"submit_review"}},{"do":"expectActionOutcome","outcome":"accepted"},{"action":"submitReview","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0,5,"Nonbuyer review"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"}],"path":"/api/items/:id/reviews","reducer":"submit_review"}},{"do":"expectActionOutcome","outcome":"application-refused","routeProvenBy":"owner"},{"action":"submitReview","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0,5,"Forged buyer review","claim-review-owner"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"},{"in":"body","name":"username"}],"path":"/api/items/:id/reviews","reducer":"submit_review"}},{"do":"expectActionOutcome","outcome":"application-refused","routeProvenBy":"owner"},{"do":"freshClient"},{"do":"signIn","exact":true,"name":"claim-review-owner"},{"do":"openItem","item":"Keyboard"},{"contains":"Original buyer review","do":"expect"},{"absent":true,"contains":"Nonbuyer review","do":"expect"},{"absent":true,"contains":"Forged buyer review","do":"expect"}],"source":"scenarios/progression-review-access.json","stableKey":"ecommerce.progression.review-access-specifications.review-eligibility-direct.618a","statedBy":"Only a customer who bought an item can review it.","withheld":null},{"category":"production","checkGroupId":"stored-review-script","criterionId":"9180a","description":"review input is rejected or displayed without executing script in another customer session","featureId":9180,"featureName":"Stored review content","note":null,"observations":["unmentioned"],"packId":"ecommerce.progression.review-access-specifications","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","semantics":[{"action":"submitReview","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0,5,"Benign stored review"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"}],"path":"/api/items/:id/reviews","reducer":"submit_review"}},{"do":"expectActionOutcome","outcome":"accepted"},{"action":"submitReview","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0,5,"Stored review marker <img src=\"data:,bad-image\" onerror=\"window.__stackBenchScriptCanary()\">"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"}],"path":"/api/items/:id/reviews","reducer":"submit_review"}},{"do":"expectActionOutcome","outcome":"completed","routeProvenBy":"owner"},{"do":"openItem","item":"Keyboard"},{"contains":"Benign stored review","do":"expect"},{"do":"wait"},{"do":"expectNoScriptExecution"},{"do":"freshClient"},{"do":"signIn","name":"script-review-reader"},{"do":"armScriptCanary"},{"do":"openItem","item":"Keyboard"},{"contains":"Benign stored review","do":"expect"},{"do":"wait"},{"do":"expectNoScriptExecution"}],"source":"scenarios/progression-review-script.json","stableKey":"ecommerce.progression.review-access-specifications.stored-review-script.9180a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"search-ordering","criterionId":"402b","description":"active filters and text searches remain alphabetical after purchases; clearing them restores purchase ranking","featureId":402,"featureName":"Purchases preserve search ordering","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.search-ordering","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.faceted-search"],"role":"guarantee","semantics":[{"do":"expectSequence","equals":["Air Purifier","Bluetooth Speaker","Coffee Grinder","Desk Lamp","Espresso Machine","Gaming Mouse","Headphones","Induction Cooktop","Keyboard","Laptop Stand"]},{"do":"fill","enter":true,"text":"e"},{"do":"fill","text":""},{"do":"click","ifAvailable":true},{"do":"expectSequence","equals":["Air Purifier","Bluetooth Speaker","Coffee Grinder","Desk Lamp","Espresso Machine","Gaming Mouse","Headphones","Keyboard","Mirrorless Camera","USB Cable"]},{"do":"fill","enter":true,"text":""},{"do":"expectSequence","equals":["Headphones","Air Purifier","Bluetooth Speaker","Coffee Grinder","Desk Lamp","Espresso Machine","Gaming Mouse","Induction Cooktop","Keyboard","Laptop Stand"]}],"source":"scenarios/progression-search-ordering.json","stableKey":"ecommerce.spec.search-ordering.search-ordering.402b","statedBy":"Active search and filter results use item-name order; clearing them restores purchase ranking","withheld":null},{"category":"production","checkGroupId":"shipping-accounting","criterionId":"202e","description":"shipping a purchased order does not deduct stock or add revenue again","featureId":202,"featureName":"Shipping preserves completed purchase accounting","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin","ecommerce.progression.fulfilment-queue"],"role":"guarantee","semantics":[{"as":"stock-before-purchase","do":"dbRecordStock","item":"Keyboard"},{"as":"revenue-before-purchase","do":"recordNumber"},{"do":"click"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"pending"},{"do":"dbExpectStock","item":"Keyboard","plus":-1,"relativeTo":"stock-before-purchase"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"expectNumber","plus":89,"relativeTo":"revenue-before-purchase"},{"as":"revenue-before-ship","do":"recordNumber"},{"as":"East-before-ship","do":"dbRecordStock","item":"Keyboard","warehouse":"East"},{"as":"West-before-ship","do":"dbRecordStock","item":"Keyboard","warehouse":"West"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click","ifAvailable":true},{"action":"ship","do":"callAction","from":"customer","input":{"attribute":"data-ship-input","contains":"Keyboard","testid":"order-item"},"namedAction":{"args":[0],"id":"ship","params":[{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/fulfilment/ship","reducer":"ship_order"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"reload"},{"do":"ensureSignedIn","name":"shipping-accounting"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"shipped"},{"do":"dbExpectStock","item":"Keyboard","plus":0,"relativeTo":"East-before-ship","warehouse":"East"},{"do":"dbExpectStock","item":"Keyboard","plus":0,"relativeTo":"West-before-ship","warehouse":"West"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"expectNumber","plus":0,"relativeTo":"revenue-before-ship"}],"source":"scenarios/progression-shipping-accounting.json","stableKey":"ecommerce.inventory-operations.shipping-accounting.202e","statedBy":"Shipping preserves the stock and revenue recorded for the purchase.","withheld":null},{"category":"production","checkGroupId":"signed-out-purchase","criterionId":"3a","description":"using the purchase control while signed out does not buy an item","featureId":3,"featureName":"Buying","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","semantics":[{"as":"keyboard-before-guest","do":"recordNumber"},{"do":"click","ifAvailable":true},{"do":"reload"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"expectNumber","plus":0,"relativeTo":"keyboard-before-guest"}],"source":"scenarios/progression-signed-out-purchase.json","stableKey":"ecommerce.spec.access-control.signed-out-purchase.3a","statedBy":"Unauthenticated callers cannot purchase.","withheld":null},{"category":"feature","checkGroupId":"split-tender-refunds-751","criterionId":"751a","description":"Full refund restores each original payment portion","featureId":751,"featureName":"Full refund restores each original payment portion","note":null,"packId":"ecommerce.feature.split-tender-refunds","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.split-tender-refunds","ecommerce.feature.store-credit","ecommerce.progression.support-refunds"],"role":"feature","semantics":[{"do":"click"},{"do":"freshClient"},{"do":"signIn","name":"credit-owner-751"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Desk Lamp","do":"expectElementCount","equals":1},{"do":"expectNumber","equals":42},{"do":"expectNumber","equals":10},{"do":"expectNumber","equals":32},{"do":"click"},{"attribute":"aria-busy","do":"expect","value":"false"},{"do":"expectNumber","equals":10}],"source":"scenarios/progression-split-tender-refunds.json","stableKey":"ecommerce.feature.split-tender-refunds.split-tender-refunds-751.751a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"production-756","criterionId":"756a","description":"Concurrent refunds restore the original credit and external amounts once, including after restart","featureId":756,"featureName":"Concurrent refunds do not duplicate credit after restart","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.split-tender-refunds","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.split-tender-refunds","ecommerce.feature.store-credit","ecommerce.progression.support-refunds"],"role":"guarantee","semantics":[{"action":"supportRefund","do":"callConcurrently","from":"staff","input":{"attribute":"data-refund-input","contains":"Split refund 756","testid":"support-ticket"},"namedAction":{"args":[0],"id":"supportRefund","method":"POST","params":[{"in":"path","name":"caseId","placeholder":"{caseId}","wireType":"u64"}],"path":"/api/support/cases/{caseId}/refund","reducer":"support_refund"}},{"do":"expectCallOutcomes"},{"do":"freshClient"},{"do":"signIn","name":"credit-owner-756"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Desk Lamp","do":"expectElementCount","equals":1},{"do":"expectNumber","equals":42},{"do":"expectNumber","equals":10},{"do":"expectNumber","equals":32},{"do":"click"},{"attribute":"aria-busy","do":"expect","value":"false"},{"do":"expectNumber","equals":10},{"do":"restartBackend"},{"do":"freshClient"},{"do":"signIn","name":"credit-owner-756"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Desk Lamp","do":"expectElementCount","equals":1},{"do":"expectNumber","equals":42},{"do":"expectNumber","equals":10},{"do":"expectNumber","equals":32},{"do":"click"},{"attribute":"aria-busy","do":"expect","value":"false"},{"do":"expectNumber","equals":10}],"source":"scenarios/progression-split-tender-refunds.json","stableKey":"ecommerce.spec.split-tender-refunds.production-756.756a","statedBy":null,"withheld":null},{"category":"feature","checkGroupId":"staff-access","criterionId":"601a","description":"staff and administrators can sign in and open staff tools","featureId":601,"featureName":"Staff access","note":null,"packId":"ecommerce.progression.staff-access","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click"},{"do":"expect"},{"do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click"},{"do":"expect"}],"source":"scenarios/progression-staff-access.json","stableKey":"ecommerce.progression.staff-access.staff-access.601a","statedBy":"Staff and administrators can sign in and use staff areas.","withheld":null},{"category":"production","checkGroupId":"staff-area-boundary","criterionId":"601b","description":"customers cannot open staff tools","featureId":601,"featureName":"Staff access","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.staff-access"],"role":"guarantee","semantics":[{"do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click"},{"do":"expect"},{"do":"signIn","exact":true,"name":"customer","password":"stackbench-customer-2026"},{"do":"click","ifAvailable":true},{"absent":true,"do":"expect"}],"source":"scenarios/progression-staff-access.json","stableKey":"ecommerce.spec.access-control.staff-area-boundary.601b","statedBy":"Customers cannot open staff tools.","withheld":null},{"category":"feature","checkGroupId":"staff-activity","criterionId":"624a","description":"an administrative change records its actor, action, subject, and time","featureId":624,"featureName":"Attributable staff activity","note":null,"packId":"ecommerce.progression.staff-activity","points":3,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"do":"click"},{"contains":"Activity Mug","do":"expect"},{"contains":"admin","do":"expect"},{"contains":"creat","do":"expect"},{"contains":"Activity Mug","do":"expect"},{"do":"expect"}],"source":"scenarios/progression-staff-activity.json","stableKey":"ecommerce.progression.staff-activity.staff-activity.624a","statedBy":"Each administrative change identifies its actor, action, subject, and time.","withheld":null},{"category":"production","checkGroupId":"staff-activity-privacy","criterionId":"624b","description":"customers cannot open staff activity history","featureId":624,"featureName":"Attributable staff activity","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.staff-activity"],"role":"guarantee","semantics":[{"absent":true,"do":"expect"}],"source":"scenarios/progression-staff-activity.json","stableKey":"ecommerce.spec.access-control.staff-activity-privacy.624b","statedBy":"Customers cannot open staff activity history.","withheld":null},{"category":"feature","checkGroupId":"staff-roles","criterionId":"621c","description":"an administrator can assign a staff role","featureId":621,"featureName":"Staff roles","note":null,"packId":"ecommerce.progression.staff-roles","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"expect","value":"inventory"}],"source":"scenarios/progression-staff-roles.json","stableKey":"ecommerce.progression.staff-roles.staff-roles.621c","statedBy":"An administrator can assign a role to an existing staff account.","withheld":null},{"category":"production","checkGroupId":"staff-role-reload","criterionId":"621a","description":"an assigned staff role survives reload and backend restart in a fresh browser","featureId":621,"featureName":"Staff roles","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.staff-roles"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true,"unlessVisible":"staff-role-account-staff"},{"do":"expect","value":"inventory"},{"do":"restartBackend"},{"do":"freshClient"},{"do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","unlessVisible":"staff-role-account-staff"},{"do":"expect","value":"inventory"}],"source":"scenarios/progression-staff-roles.json","stableKey":"ecommerce.spec.state-durability.staff-role-reload.621a","statedBy":"Assigned staff roles persist.","withheld":null},{"category":"production","checkGroupId":"staff-role-boundary","criterionId":"621b","description":"a staff member cannot assign roles through the UI or a replayed request","featureId":621,"featureName":"Staff roles","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.staff-roles"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"expect"},{"do":"click","unlessVisible":"staff-role-account-staff"},{"do":"fill","text":"staff"},{"do":"click"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true,"unlessVisible":"staff-role-account-staff"},{"do":"expect","value":"staff"},{"do":"reload"},{"do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click","ifAvailable":true},{"absent":true,"do":"expect"},{"do":"replayAs","from":"replayAdmin","match":"role","namedAction":{"args":[0,"inventory"],"id":"assignStaffRole","method":"PUT","params":[{"in":"path","name":"accountId","placeholder":":id","wireType":"u64"},{"in":"body","name":"role"}],"path":"/api/staff/:id/role","reducer":"assign_staff_role"},"namedTarget":{"attribute":"data-account-id","testid":"staff-role-account-staff","valueType":"string"},"swap":{"find":"\"role\":\"staff\"","with":"\"role\":\"inventory\""}},{"do":"expectReplayRejected"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true,"unlessVisible":"staff-role-account-staff"},{"do":"expect","value":"staff"}],"source":"scenarios/progression-staff-roles.json","stableKey":"ecommerce.spec.access-control.staff-role-boundary.621b","statedBy":"A staff member cannot assign or change roles.","withheld":null},{"category":"production","checkGroupId":"staff-role-revocation","criterionId":"621d","description":"removing administrator access blocks a previously authorized session without changing the target role","featureId":621,"featureName":"Staff roles","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.staff-roles"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","unlessVisible":"staff-role-account-staff"},{"do":"replayAs","from":"roleAdmin","match":"__role-revocation-named-action__","namedAction":{"args":[0,"admin"],"id":"assignStaffRole","method":"PUT","params":[{"in":"path","name":"accountId","placeholder":":id","wireType":"u64"},{"in":"body","name":"role"}],"path":"/api/staff/:id/role","reducer":"assign_staff_role"},"namedTarget":{"attribute":"data-account-id","testid":"staff-role-account-staff","valueType":"string"}},{"do":"expectReplayCompleted","requireAccepted":true},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true,"unlessVisible":"staff-role-account-staff"},{"do":"expect","value":"admin"},{"do":"reload"},{"do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"replayAs","from":"roleAdmin","match":"__role-revocation-named-action__","namedAction":{"args":[0,"admin"],"id":"assignStaffRole","method":"PUT","params":[{"in":"path","name":"accountId","placeholder":":id","wireType":"u64"},{"in":"body","name":"role"}],"path":"/api/staff/:id/role","reducer":"assign_staff_role"},"namedTarget":{"attribute":"data-account-id","testid":"staff-role-account-staff","valueType":"string"}},{"do":"expectReplayCompleted","requireAccepted":true},{"do":"replayAs","from":"roleAdmin","match":"__role-revocation-named-action__","namedAction":{"args":[0,"staff"],"id":"assignStaffRole","method":"PUT","params":[{"in":"path","name":"accountId","placeholder":":id","wireType":"u64"},{"in":"body","name":"role"}],"path":"/api/staff/:id/role","reducer":"assign_staff_role"},"namedTarget":{"attribute":"data-account-id","testid":"staff-role-account-staff","valueType":"string"}},{"do":"expectReplayCompleted","requireAccepted":true},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true,"unlessVisible":"staff-role-account-staff"},{"do":"expect","value":"staff"},{"do":"replayAs","from":"roleAdmin","match":"__role-revocation-named-action__","namedAction":{"args":[0,"admin"],"id":"assignStaffRole","method":"PUT","params":[{"in":"path","name":"accountId","placeholder":":id","wireType":"u64"},{"in":"body","name":"role"}],"path":"/api/staff/:id/role","reducer":"assign_staff_role"},"namedTarget":{"attribute":"data-account-id","testid":"staff-role-account-staff","valueType":"string"}},{"do":"expectReplayRejected"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true,"unlessVisible":"staff-role-account-staff"},{"do":"expect","value":"staff"}],"source":"scenarios/progression-staff-roles.json","stableKey":"ecommerce.spec.access-control.staff-role-revocation.621d","statedBy":"Only the admin role grants administrator access.","withheld":null},{"category":"feature","checkGroupId":"stock-alert-delivery","criterionId":"631c","description":"restored stock sends the requested alert","featureId":631,"featureName":"Stock alert delivery","note":null,"packId":"ecommerce.progression.stock-alerts","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"feature","semantics":[{"do":"freshClient"},{"do":"signIn","name":"stock-subscriber"},{"do":"click","unlessVisible":"notifications-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"contains":"Air Purifier","do":"expectElementCount","equals":0},{"do":"click"},{"action":"restock","do":"callAction","input":{"attribute":"data-restock-input","contains":"Air Purifier","testid":"admin-location-row"},"namedAction":{"args":[0,0,1],"id":"restock","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"warehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/restock","reducer":"admin_restock"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"wait"},{"do":"freshClient"},{"do":"signIn","name":"stock-subscriber"},{"do":"click","unlessVisible":"notifications-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"contains":"Air Purifier","do":"expect"}],"source":"scenarios/progression-stock-alert-delivery.json","stableKey":"ecommerce.progression.stock-alerts.stock-alert-delivery.631c","statedBy":"Show an alert when stock returns.","withheld":null},{"category":"production","checkGroupId":"stock-alert-deduplication","criterionId":"631a","description":"restored stock sends one alert and later restocks do not duplicate it","featureId":631,"featureName":"Private one-time stock alerts","note":"Sample a fresh account view 10 seconds after the second accepted restock. This catches persistent duplicates delivered by that sample, not all later or transient duplicates.","observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.warehouse-admin","ecommerce.progression.stock-alerts"],"role":"guarantee","semantics":[{"do":"freshClient"},{"do":"signIn","name":"stock-subscriber"},{"do":"click","unlessVisible":"notifications-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"contains":"Air Purifier","do":"expectElementCount","equals":1},{"action":"restock","do":"callAction","input":{"attribute":"data-restock-input","contains":"Air Purifier","testid":"admin-location-row"},"namedAction":{"args":[0,0,1],"id":"restock","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"warehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/restock","reducer":"admin_restock"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"wait"},{"do":"freshClient"},{"do":"signIn","name":"stock-subscriber"},{"do":"click","unlessVisible":"notifications-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"contains":"Air Purifier","do":"expectElementCount","equals":1}],"source":"scenarios/progression-stock-alerts.json","stableKey":"ecommerce.spec.transactional-integrity.stock-alert-deduplication.631a","statedBy":"A stock request creates one alert when stock returns.","withheld":null},{"category":"production","checkGroupId":"stock-alert-privacy","criterionId":"631b","description":"a customer who did not request the alert cannot see it","featureId":631,"featureName":"Private one-time stock alerts","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.warehouse-admin","ecommerce.progression.stock-alerts"],"role":"guarantee","semantics":[{"do":"freshClient"},{"do":"signIn","name":"stock-subscriber"},{"do":"click","unlessVisible":"notifications-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"contains":"Air Purifier","do":"expect"},{"do":"freshClient"},{"do":"signIn","name":"stock-other"},{"do":"click","unlessVisible":"notifications-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"absent":true,"contains":"Air Purifier","do":"expect"}],"source":"scenarios/progression-stock-alerts.json","stableKey":"ecommerce.spec.access-control.stock-alert-privacy.631b","statedBy":"Stock alerts are private to the requesting customer.","withheld":null},{"category":"production","checkGroupId":"stock-limit","criterionId":"3d","description":"an item sells out visibly, and a further purchase is refused without changing stock","featureId":3,"featureName":"Buying","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","semantics":[{"do":"click"},{"do":"wait"},{"do":"click"},{"do":"wait"},{"do":"click"},{"do":"expectNumber","equals":0},{"do":"expect"},{"action":"buy","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"do":"expectActionOutcome","outcome":"validation-refused"},{"do":"reload"},{"do":"ensureSignedIn","name":"eli"},{"do":"expectNumber","equals":0}],"source":"scenarios/progression-stock-limit.json","stableKey":"ecommerce.spec.concurrency-safety.stock-limit.3d","statedBy":"an item with zero stock cannot be bought","withheld":null},{"category":"feature","checkGroupId":"store-credit-750","criterionId":"750a","description":"Credit checkout records both payment portions","featureId":750,"featureName":"Credit checkout records both payment portions","note":null,"packId":"ecommerce.feature.store-credit","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"do":"freshClient"},{"do":"signIn","name":"credit-owner-750"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Desk Lamp","do":"expectElementCount","equals":1},{"do":"expectNumber","equals":10},{"do":"expectNumber","equals":32},{"do":"expectNumber","equals":42},{"do":"click","unlessVisible":"credit-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"do":"expectNumber","equals":0}],"source":"scenarios/progression-store-credit.json","stableKey":"ecommerce.feature.store-credit.store-credit-750.750a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"production-752","criterionId":"752a","description":"Repeating a grant reference does not increase the balance twice","featureId":752,"featureName":"Repeating a grant reference does not increase the balance twice","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.store-credit","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.store-credit"],"role":"guarantee","semantics":[{"action":"grantCredit","do":"callAction","from":"staff","input":{"attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","attribute":"data-account-id","testid":"credit-panel"}},"testid":"credit-grant"},"namedAction":{"args":[0,1000,""],"id":"grantCredit","method":"POST","params":[{"in":"body","name":"accountId","wireType":"u64"},{"in":"body","name":"amountMinor"},{"in":"body","name":"reference"}],"path":"/api/staff/credit","reducer":"grant_credit"}},{"do":"expectActionOutcome","outcome":"completed"},{"do":"freshClient"},{"do":"signIn","name":"credit-owner-752"},{"do":"click","unlessVisible":"credit-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"do":"expectNumber","equals":10}],"source":"scenarios/progression-store-credit.json","stableKey":"ecommerce.spec.store-credit.production-752.752a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"production-753","criterionId":"753a","description":"A customer cannot grant credit","featureId":753,"featureName":"A customer cannot grant credit","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.store-credit","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.store-credit"],"role":"guarantee","semantics":[{"do":"fill","text":"unauthorized-credit-753"},{"action":"grantCredit","do":"callAction","from":"staff","input":{"attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","attribute":"data-account-id","testid":"credit-panel"}},"testid":"credit-grant"},"namedAction":{"args":[0,1000,""],"id":"grantCredit","method":"POST","params":[{"in":"body","name":"accountId","wireType":"u64"},{"in":"body","name":"amountMinor"},{"in":"body","name":"reference"}],"path":"/api/staff/credit","reducer":"grant_credit"}},{"do":"expectActionOutcome","outcome":"refused"},{"do":"freshClient"},{"do":"signIn","name":"credit-owner-753"},{"do":"click","unlessVisible":"credit-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"do":"expectNumber","equals":10}],"source":"scenarios/progression-store-credit.json","stableKey":"ecommerce.spec.store-credit.production-753.753a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"production-754","criterionId":"754a","description":"Concurrent checkout consumes one cart and one credit allocation","featureId":754,"featureName":"Concurrent checkout consumes one cart and one credit allocation","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.store-credit","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.store-credit"],"role":"guarantee","semantics":[{"action":"checkoutCredit","do":"callConcurrently","namedAction":{"args":[],"id":"checkoutCredit","method":"POST","path":"/api/checkout/credit","reducer":"checkout_credit"}},{"do":"expectCallOutcomes"},{"do":"freshClient"},{"do":"signIn","name":"credit-owner-754"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Desk Lamp","do":"expectElementCount","equals":1},{"do":"expectNumber","equals":10},{"do":"expectNumber","equals":32},{"do":"expectNumber","equals":42},{"do":"click","unlessVisible":"credit-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"do":"expectNumber","equals":0}],"source":"scenarios/progression-store-credit.json","stableKey":"ecommerce.spec.store-credit.production-754.754a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"production-755","criterionId":"755a","description":"Issued credit survives a backend restart","featureId":755,"featureName":"Issued credit survives a backend restart","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.store-credit","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.store-credit"],"role":"guarantee","semantics":[{"do":"restartBackend"},{"do":"freshClient"},{"do":"signIn","name":"credit-owner-755"},{"do":"click","unlessVisible":"credit-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"do":"expectNumber","equals":10}],"source":"scenarios/progression-store-credit.json","stableKey":"ecommerce.spec.store-credit.production-755.755a","statedBy":null,"withheld":null},{"category":"feature","checkGroupId":"subscriptions-760","criterionId":"760a","description":"A subscription creates exactly its requested deliveries and payments","featureId":760,"featureName":"A subscription creates exactly its requested deliveries and payments","note":null,"packId":"ecommerce.feature.subscriptions","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"freshClient"},{"do":"signIn","name":"subscription-760"},{"do":"click","unlessVisible":"subscriptions-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"do":"expect","value":"complete"},{"do":"expectElementCount","equals":2},{"do":"expectSequence","equals":["paid","paid"]},{"do":"expectNumber","equals":84},{"do":"dbExpectStock","equals":8,"item":"Desk Lamp"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Desk Lamp","do":"expectElementCount","equals":2},{"do":"expectElementCount","equals":2}],"source":"scenarios/progression-subscriptions.json","stableKey":"ecommerce.feature.subscriptions.subscriptions-760.760a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"production-761","criterionId":"761a","description":"A pending subscription continues after backend restart without duplicate deliveries","featureId":761,"featureName":"A pending subscription continues after backend restart without duplicate deliveries","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.subscriptions","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.subscriptions"],"role":"guarantee","semantics":[{"do":"restartBackend"},{"do":"freshClient"},{"do":"signIn","name":"subscription-761"},{"do":"click","unlessVisible":"subscriptions-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"do":"expect","value":"complete"},{"do":"expectElementCount","equals":2},{"do":"expectSequence","equals":["paid","paid"]},{"do":"expectNumber","equals":84},{"do":"dbExpectStock","equals":8,"item":"Desk Lamp"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Desk Lamp","do":"expectElementCount","equals":2},{"do":"expectElementCount","equals":2},{"do":"restartBackend"},{"do":"freshClient"},{"do":"signIn","name":"subscription-761"},{"do":"click","unlessVisible":"subscriptions-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"do":"expect","value":"complete"},{"do":"expectElementCount","equals":2},{"do":"expectSequence","equals":["paid","paid"]},{"do":"expectNumber","equals":84},{"do":"dbExpectStock","equals":8,"item":"Desk Lamp"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Desk Lamp","do":"expectElementCount","equals":2},{"do":"expectElementCount","equals":2}],"source":"scenarios/progression-subscriptions.json","stableKey":"ecommerce.spec.subscriptions.production-761.761a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"production-762","criterionId":"762a","description":"Another customer cannot cancel an active subscription","featureId":762,"featureName":"Another customer cannot cancel an active subscription","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.subscriptions","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.subscriptions"],"role":"guarantee","semantics":[{"action":"cancelSubscription","do":"callAction","from":"owner","input":{"attribute":"data-action-input","testid":"subscription-cancel"},"namedAction":{"args":[0],"id":"cancelSubscription","method":"POST","params":[{"in":"path","name":"subscriptionId","placeholder":"{subscriptionId}","wireType":"u64"}],"path":"/api/subscriptions/{subscriptionId}/cancel","reducer":"cancel_subscription"}},{"do":"expectActionOutcome","outcome":"refused"},{"do":"expect","value":"active"},{"do":"click"},{"do":"freshClient"},{"do":"signIn","name":"subscription-762"},{"do":"click","unlessVisible":"subscriptions-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"do":"expect","value":"cancelled"},{"as":"deliveries-at-cancel","count":true,"do":"recordNumber"},{"as":"stock-at-cancel","do":"dbRecordStock","item":"Desk Lamp"},{"do":"wait"},{"do":"expectElementCount","plus":0,"relativeTo":"deliveries-at-cancel"},{"do":"dbExpectStock","item":"Desk Lamp","plus":0,"relativeTo":"stock-at-cancel"}],"source":"scenarios/progression-subscriptions.json","stableKey":"ecommerce.spec.subscriptions.production-762.762a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"production-763","criterionId":"763a","description":"Pause survives a restart and resume completes the remaining deliveries","featureId":763,"featureName":"Pause survives a restart and resume completes the remaining deliveries","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.subscriptions","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.subscriptions"],"role":"guarantee","semantics":[{"do":"click"},{"do":"expect","value":"paused"},{"do":"restartBackend"},{"do":"freshClient"},{"do":"signIn","name":"subscription-763"},{"do":"click","unlessVisible":"subscriptions-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"do":"expect","value":"paused"},{"as":"deliveries-at-pause","count":true,"do":"recordNumber"},{"as":"stock-at-pause","do":"dbRecordStock","item":"Desk Lamp"},{"do":"wait"},{"do":"expectElementCount","plus":0,"relativeTo":"deliveries-at-pause"},{"do":"dbExpectStock","item":"Desk Lamp","plus":0,"relativeTo":"stock-at-pause"},{"do":"click"},{"do":"freshClient"},{"do":"signIn","name":"subscription-763"},{"do":"click","unlessVisible":"subscriptions-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"do":"expect","value":"complete"},{"do":"expectElementCount","equals":2},{"do":"expectSequence","equals":["paid","paid"]},{"do":"expectNumber","equals":84},{"do":"dbExpectStock","equals":8,"item":"Desk Lamp"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Desk Lamp","do":"expectElementCount","equals":2},{"do":"expectElementCount","equals":2}],"source":"scenarios/progression-subscriptions.json","stableKey":"ecommerce.spec.subscriptions.production-763.763a","statedBy":null,"withheld":null},{"category":"feature","checkGroupId":"support-history","criterionId":"612c","description":"the customer can view their support ticket history","featureId":612,"featureName":"Customer support history","note":null,"packId":"ecommerce.progression.support-history","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"reload"},{"do":"ensureSignedIn","name":"support-owner"},{"do":"click","unlessVisible":"support-ticket"},{"contains":"Owner ticket {user:ticketmarker}","do":"expect"}],"source":"scenarios/progression-support-history.json","stableKey":"ecommerce.progression.support-history.support-history.612c","statedBy":"Signed-in customers can see their support ticket history.","withheld":null},{"category":"production","checkGroupId":"support-history-reload","criterionId":"612a","description":"support history survives reload and backend restart in a fresh browser","featureId":612,"featureName":"Customer support history","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.support-history"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"ensureSignedIn","name":"support-owner"},{"do":"click"},{"contains":"Owner ticket {user:ticketmarker}","do":"expect"},{"do":"restartBackend"},{"do":"freshClient"},{"do":"signIn","name":"support-owner"},{"do":"click"},{"contains":"Owner ticket {user:ticketmarker}","do":"expect"}],"source":"scenarios/progression-support-history.json","stableKey":"ecommerce.spec.state-durability.support-history-reload.612a","statedBy":"A customer's support history persists.","withheld":null},{"category":"production","checkGroupId":"support-history-privacy","criterionId":"612b","description":"another customer neither sees nor receives the private ticket","featureId":612,"featureName":"Customer support history","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.support-history"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"ensureSignedIn","name":"support-owner"},{"do":"click"},{"do":"fill","text":"owner@example.com"},{"do":"fill","text":"Private ticket {user:privateticketmarker}"},{"do":"fill","text":"Private account issue."},{"do":"click"},{"do":"expect","nonEmpty":true},{"do":"reload"},{"do":"ensureSignedIn","name":"support-owner"},{"do":"click"},{"contains":"Private ticket {user:privateticketmarker}","do":"expect"},{"contains":"Private ticket {user:privateticketmarker}","do":"expectReceived"},{"do":"reload"},{"do":"ensureSignedIn","name":"support-other"},{"do":"click"},{"absent":true,"contains":"Private ticket {user:privateticketmarker}","do":"expect"},{"contains":"Private ticket {user:privateticketmarker}","do":"expectNotReceived"}],"source":"scenarios/progression-support-history.json","stableKey":"ecommerce.spec.access-control.support-history-privacy.612b","statedBy":"Customers cannot see another customer's support tickets.","withheld":null},{"category":"production","checkGroupId":"support-history-logout","criterionId":"612d","description":"after logout the same browser storage no longer grants access to private support history","featureId":612,"featureName":"Customer support history","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.support-history"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"ensureSignedIn","name":"support-owner"},{"do":"click"},{"do":"fill","text":"owner@example.com"},{"do":"fill","text":"Logout ticket {user:logoutticketmarker}"},{"do":"fill","text":"Private account issue."},{"do":"click"},{"do":"expect","nonEmpty":true},{"do":"reload"},{"do":"ensureSignedIn","name":"support-owner"},{"do":"click","unlessVisible":"support-ticket"},{"contains":"Logout ticket {user:logoutticketmarker}","do":"expect"},{"contains":"Logout ticket {user:logoutticketmarker}","do":"expectReceived"},{"do":"reload"},{"do":"ensureSignedIn","name":"support-owner"},{"do":"click","unlessVisible":"signout"},{"do":"click"},{"do":"waitUntilAbsent"},{"do":"freshClient","preserveStorage":true},{"absent":true,"do":"expect"},{"do":"click"},{"do":"expect"},{"absent":true,"contains":"Logout ticket {user:logoutticketmarker}","do":"expect"},{"contains":"Logout ticket {user:logoutticketmarker}","do":"expectNotReceived"}],"source":"scenarios/progression-support-history.json","stableKey":"ecommerce.spec.access-control.support-history-logout.612d","statedBy":"Signed-out visitors cannot access private support tickets.","withheld":null},{"category":"feature","checkGroupId":"support-intake","criterionId":"610a","description":"a visitor can submit a support ticket and receives a reference","featureId":610,"featureName":"Support intake","note":null,"packId":"ecommerce.progression.support-intake","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"do":"fill","text":"visitor@example.com"},{"do":"fill","text":"Damaged package"},{"do":"fill","text":"The package arrived damaged."},{"do":"click"},{"do":"expect","nonEmpty":true}],"source":"scenarios/progression-support-intake.json","stableKey":"ecommerce.progression.support-intake.support-intake.610a","statedBy":"Anyone can open a support ticket and receives a reference.","withheld":null},{"category":"production","checkGroupId":"support-refund-access","criterionId":"615c","description":"a customer cannot issue a refund or change its records","featureId":615,"featureName":"Support refund access","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"provenBy":null,"requiresFeatures":["ecommerce.progression.support-refunds"],"role":"guarantee","semantics":[{"absent":true,"do":"expect"},{"action":"supportRefund","authentication":"actor","do":"callAction","from":"staff","input":{"attribute":"data-action-input","testid":"support-refund"},"namedAction":{"args":[0],"id":"supportRefund","method":"POST","params":[{"in":"path","name":"caseId","placeholder":"{caseId}","wireType":"u64"}],"path":"/api/support/cases/{caseId}/refund","reducer":"support_refund"}},{"do":"expectActionOutcome","outcome":"refused"},{"do":"reload"},{"do":"ensureSignedIn","name":"refund-access-owner"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"support-subject"},{"do":"expect","notContains":"resolved"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"do":"expect","value":"pending"},{"absent":true,"contains":"Keyboard","do":"expect"}],"source":"scenarios/progression-support-refunds-access.json","stableKey":"ecommerce.spec.access-control.support-refund-access.615c","statedBy":"Only authorized staff can refund an order from a support case.","withheld":null},{"category":"production","checkGroupId":"support-refund-accounting","criterionId":"615b","description":"the refund equals the paid total, cannot be applied twice, and leaves another order unrefunded","featureId":615,"featureName":"Support refund accounting","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":1,"provenBy":null,"requiresFeatures":["ecommerce.progression.support-refunds"],"role":"guarantee","semantics":[{"do":"click"},{"do":"reload"},{"do":"ensureSignedIn","name":"refund-accounting-owner"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"support-subject"},{"do":"expectNumber","plus":0,"relativeTo":"paid-total"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"expectNumber","plus":0,"relativeTo":"paid-total"},{"contains":"Keyboard","do":"expectElementCount","equals":1},{"do":"replayAs","from":"staff","match":"refund","namedAction":{"args":[0],"id":"supportRefund","method":"POST","params":[{"in":"path","name":"caseId","placeholder":"{caseId}","wireType":"u64"}],"path":"/api/support/cases/{caseId}/refund","reducer":"support_refund"},"namedTarget":{"attribute":"data-entity-id","contains":"Accounting refund case","testid":"support-ticket","valueType":"string"}},{"do":"expectReplayCompleted"},{"do":"freshClient"},{"do":"signIn","name":"refund-accounting-owner"},{"do":"click"},{"do":"expectNumber","plus":0,"relativeTo":"paid-total"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"expectNumber","plus":0,"relativeTo":"paid-total"},{"contains":"Keyboard","do":"expectElementCount","equals":1},{"do":"expectNumber","equals":0},{"contains":"Mouse","do":"expectElementCount","equals":0}],"source":"scenarios/progression-support-refunds-accounting.json","stableKey":"ecommerce.spec.transactional-integrity.support-refund-accounting.615b","statedBy":"The recorded refund equals the amount paid and is applied only once.","withheld":null},{"category":"feature","checkGroupId":"support-refunds-resolution","criterionId":"615a","description":"an authorized refund resolves the case and updates the order","featureId":615,"featureName":"Support refund resolution","note":null,"packId":"ecommerce.progression.support-refunds","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"contains":"resolved","do":"expect"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"refunded","do":"expect"}],"source":"scenarios/progression-support-refunds-resolution.json","stableKey":"ecommerce.progression.support-refunds.support-refunds-resolution.615a","statedBy":"Authorized staff can refund an order from its support case.","withheld":null},{"category":"production","checkGroupId":"return-refund-interaction","criterionId":"757a","description":"a support refund followed by physical return restores each warehouse once and refunds only the price paid","featureId":757,"featureName":"Return and support refund accounting","note":null,"packId":"ecommerce.feature.split-tender-refunds","points":2,"provenBy":null,"requiresFeatures":["ecommerce.l3.order-returns-features","ecommerce.progression.fulfilment-queue","ecommerce.progression.support-refunds"],"role":"feature","semantics":[{"do":"reload"},{"do":"ensureSignedIn","name":"return-refund-owner"},{"do":"click","ifAvailable":true},{"as":"757atotal","do":"dbRecordStock","item":"Keyboard"},{"as":"757aEast","do":"dbRecordStock","item":"Keyboard","warehouse":"East"},{"as":"757aWest","do":"dbRecordStock","item":"Keyboard","warehouse":"West"},{"as":"757arevenue","do":"recordNumber"},{"do":"click"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"pending"},{"as":"757apaid","do":"recordNumber"},{"action":"ship","do":"callAction","from":"owner","input":{"attribute":"data-ship-input","contains":"Keyboard","testid":"order-item"},"namedAction":{"args":[0],"id":"ship","params":[{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/fulfilment/ship","reducer":"ship_order"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"expect","ignoreCase":true,"value":"shipped"},{"do":"dbExpectStock","item":"Keyboard","plus":-1,"relativeTo":"757atotal"},{"do":"reload"},{"do":"ensureSignedIn","name":"return-refund-owner"},{"do":"click"},{"do":"fill","text":"Return refund 757a"},{"do":"fill","text":"Please refund this order."},{"do":"click"},{"contains":"Return refund 757a","do":"expect"},{"do":"click"},{"do":"click"},{"do":"expect"},{"action":"supportRefund","do":"callAction","from":"owner","input":{"attribute":"data-refund-input","contains":"Return refund 757a","testid":"support-ticket"},"namedAction":{"args":[0],"id":"supportRefund","method":"POST","params":[{"in":"path","name":"caseId","placeholder":"{caseId}","wireType":"u64"}],"path":"/api/support/cases/{caseId}/refund","reducer":"support_refund"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"dbExpectStock","item":"Keyboard","plus":-1,"relativeTo":"757atotal"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"click"},{"contains":"Keyboard","containsText":"returned","do":"expect","ignoreCase":true},{"do":"freshClient"},{"do":"signIn","name":"return-refund-owner"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Keyboard","containsText":"returned","do":"expect","ignoreCase":true},{"do":"expectNumber","plus":0,"relativeTo":"757apaid"},{"do":"dbExpectStock","item":"Keyboard","plus":0,"relativeTo":"757aEast","warehouse":"East"},{"do":"dbExpectStock","item":"Keyboard","plus":0,"relativeTo":"757aWest","warehouse":"West"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","unlessVisible":"admin-revenue"},{"do":"expectNumber","plus":0,"relativeTo":"757arevenue"}],"source":"scenarios/progression-support-return-interaction.json","stableKey":"ecommerce.feature.split-tender-refunds.return-refund-interaction.757a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"return-refund-interaction","criterionId":"757b","description":"a physical return followed by support refund restores each warehouse once and refunds only the price paid","featureId":757,"featureName":"Return and support refund accounting","note":null,"packId":"ecommerce.feature.split-tender-refunds","points":2,"provenBy":null,"requiresFeatures":["ecommerce.l3.order-returns-features","ecommerce.progression.fulfilment-queue","ecommerce.progression.support-refunds"],"role":"feature","semantics":[{"do":"reload"},{"do":"ensureSignedIn","name":"return-refund-owner"},{"do":"click","ifAvailable":true},{"as":"757btotal","do":"dbRecordStock","item":"Desk Lamp"},{"as":"757bEast","do":"dbRecordStock","item":"Desk Lamp","warehouse":"East"},{"as":"757bWest","do":"dbRecordStock","item":"Desk Lamp","warehouse":"West"},{"as":"757brevenue","do":"recordNumber"},{"do":"click"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"pending"},{"as":"757bpaid","do":"recordNumber"},{"action":"ship","do":"callAction","from":"owner","input":{"attribute":"data-ship-input","contains":"Desk Lamp","testid":"order-item"},"namedAction":{"args":[0],"id":"ship","params":[{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/fulfilment/ship","reducer":"ship_order"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"expect","ignoreCase":true,"value":"shipped"},{"do":"dbExpectStock","item":"Desk Lamp","plus":-1,"relativeTo":"757btotal"},{"do":"reload"},{"do":"ensureSignedIn","name":"return-refund-owner"},{"do":"click"},{"do":"fill","text":"Return refund 757b"},{"do":"fill","text":"Please refund this order."},{"do":"click"},{"contains":"Return refund 757b","do":"expect"},{"do":"click"},{"do":"click"},{"do":"expect"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"click"},{"contains":"Desk Lamp","containsText":"returned","do":"expect","ignoreCase":true},{"action":"supportRefund","do":"callAction","from":"owner","input":{"attribute":"data-refund-input","contains":"Return refund 757b","testid":"support-ticket"},"namedAction":{"args":[0],"id":"supportRefund","method":"POST","params":[{"in":"path","name":"caseId","placeholder":"{caseId}","wireType":"u64"}],"path":"/api/support/cases/{caseId}/refund","reducer":"support_refund"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"freshClient"},{"do":"signIn","name":"return-refund-owner"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Desk Lamp","containsText":"returned","do":"expect","ignoreCase":true},{"do":"expectNumber","plus":0,"relativeTo":"757bpaid"},{"do":"dbExpectStock","item":"Desk Lamp","plus":0,"relativeTo":"757bEast","warehouse":"East"},{"do":"dbExpectStock","item":"Desk Lamp","plus":0,"relativeTo":"757bWest","warehouse":"West"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","unlessVisible":"admin-revenue"},{"do":"expectNumber","plus":0,"relativeTo":"757brevenue"}],"source":"scenarios/progression-support-return-interaction.json","stableKey":"ecommerce.feature.split-tender-refunds.return-refund-interaction.757b","statedBy":null,"withheld":null},{"category":"feature","checkGroupId":"support-assignment","criterionId":"611a","description":"staff can assign a new ticket","featureId":611,"featureName":"Support triage","note":null,"packId":"ecommerce.progression.support-triage","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"fill","text":"staff"},{"do":"click"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click","ifAvailable":true,"unlessVisible":"support-assignee"},{"do":"click","ifAvailable":true,"unlessVisible":"support-assignee"},{"do":"expect","value":"staff"}],"source":"scenarios/progression-support-triage.json","stableKey":"ecommerce.progression.support-triage.support-assignment.611a","statedBy":"Staff can assign support tickets.","withheld":null},{"category":"feature","checkGroupId":"support-priority","criterionId":"611b","description":"staff can set a ticket priority","featureId":611,"featureName":"Support triage","note":null,"packId":"ecommerce.progression.support-triage","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"fill","text":"high"},{"do":"click"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click","ifAvailable":true,"unlessVisible":"support-assignee"},{"do":"click","ifAvailable":true,"unlessVisible":"support-assignee"},{"do":"expect","value":"high"}],"source":"scenarios/progression-support-triage.json","stableKey":"ecommerce.progression.support-triage.support-priority.611b","statedBy":"Staff can set support ticket priority.","withheld":null},{"category":"feature","checkGroupId":"support-status","criterionId":"611c","description":"staff can change a ticket status","featureId":611,"featureName":"Support triage","note":null,"packId":"ecommerce.progression.support-triage","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"fill","text":"in progress"},{"do":"click"},{"contains":"in progress","do":"expect"}],"source":"scenarios/progression-support-triage.json","stableKey":"ecommerce.progression.support-triage.support-status.611c","statedBy":"Staff can change support ticket status.","withheld":null}],"schemaVersion":3,"task":{"baseMeaningSha256":null,"contracts":[{"id":"ecommerce.application-interface","owners":["recipe"],"text":"# Application interface\n\nUse each interface name below as an exact HTML attribute value. Use `id` for\na one-off element. Use `data-role` when the same interface can\nappear more than once. These attributes do not prescribe the layout, data\nmodel, libraries, or transport.\n\n<!-- interface:http -->\nFor error responses, use HTTP 401 or 403 for access errors and 400, 409, or 422 for input errors.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nReport reducer errors by failing the call.\n<!-- /interface -->\n\nHuman-readable status text is case-insensitive. Machine identifiers and protocol values keep their specified spelling.\n"},{"id":"ecommerce.progression.staff-access-hooks","owners":["ecommerce.progression.staff-access"],"text":"## Staff access controls\n\nUse the existing account sign-in controls. Use `staff-link` for the staff area and `admin-link`\nfor the administrator area. Use `staff-area` for the staff tools, not for a loading or error message.\n\nUse these provided accounts:\n\n- staff: `staff` / `stackbench-staff-2026`\n- administrator: `admin` / `stackbench-admin-2026`\n- customer: `customer` / `stackbench-customer-2026`\n"},{"id":"ecommerce.progression.customer-profile-hooks","owners":["ecommerce.progression.customer-profile"],"text":"# Customer profile application interface\n\nUse `catalog-link` to return to the catalog. If a profile overlay blocks navigation,\nexpose `overlay-close` to dismiss it. Saving may leave the profile open.\n\nMake `profile-link` available from the catalog while signed in, without first opening another\narea. Use it to open the profile. Use `profile-name` and `profile-address` for the editable\nvalues. Use `profile-save` to save them. Use `profile-address-summary` to display\nthe saved address in the profile view.\n"},{"id":"ecommerce.progression.staff-role-hooks","owners":["ecommerce.progression.staff-roles"],"text":"# Staff role application interface\n\nPut role management in the administrator area opened by `admin-link`.\nUse `staff-role-row` for each staff account and set `data-account-id` to that account's server\nidentifier. Put `staff-role-select` and `staff-role-save` inside the row.\nAlso set the row's HTML `id` to `staff-role-account-` followed by\n`encodeURIComponent(username)`, using the exact account username without changing its case.\nFor example, username `staff` has row ID `staff-role-account-staff`. This identifies the\naccount independently of the role options or other text in the row.\n\nThe staff sign-in and staff-area controls come from the staff access feature.\n\nExpose the same role assignment used by `staff-role-save`.\n\n<!-- interface:http -->\nUse `PUT /api/staff/:id/role`, where `:id` is the account identifier from `data-account-id`.\nThe JSON body is `{ \"role\": \"<selected role>\" }`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nUse the `assign_staff_role` reducer with arguments in this order: `accountId: u64`,\n`role: string`. Render `data-account-id` as the decimal account identifier without precision loss.\n<!-- /interface -->\n\n`staff-role-select` offers the roles `staff`, `inventory`, and `admin` as its option values.\n"},{"id":"ecommerce.progression.catalog-management-hooks","owners":["ecommerce.progression.catalog-management"],"text":"# Catalog management application interface\n\nUse `admin-link` to open the administrator area containing the product controls.\nUse `catalog-name`, `catalog-category`, `catalog-price`, and `catalog-variants` for the product\nvalues; `catalog-category` accepts a new category name as text, and `catalog-variants` accepts\ncomma-separated variant names. Use `catalog-save` to add the product. Use `item-variant` for each named variant shown\nto a visitor.\n"},{"id":"ecommerce.progression.payment-record-hooks","owners":["ecommerce.progression.payment-records"],"text":"# Payment record application interface\n\nUse `payment-record`, `payment-status`, and `payment-amount` inside the matching `order-item`.\n"},{"id":"ecommerce.progression.staff-activity-hooks","owners":["ecommerce.progression.staff-activity"],"text":"# Staff activity application interface\n\nUse `activity-link` to open staff activity history and `activity-entry` for each change. Inside\neach entry, use `activity-actor`, `activity-action`, `activity-subject`, and `activity-time`.\n"},{"id":"ecommerce.feature.catalog-items.hooks","owners":["ecommerce.feature.catalog-items"],"text":"# Catalog item application interface\n\n| Element ID | Required element |\n| --- | --- |\n| `item-list` | Contains the public catalog items. |\n| `item-card` | Shows one catalog item. |\n| `item-name` | Shows the item name inside its `item-card`; activating it opens `item-detail`. |\n| `item-price` | Shows the numeric item price inside its `item-card`. |\n| `item-stock` | Shows total stock inside its `item-card`. |\n| `item-detail` | Contains the selected item's details. |\n"},{"id":"ecommerce.feature.catalog-discovery.hooks","owners":["ecommerce.feature.catalog-discovery"],"text":"# Catalog discovery application interface\n\nNew catalog data starts with zero purchases. Do not seed sample orders or purchase counts.\nWhen adding this feature to an existing app, preserve purchases made through the app.\n\n| Element ID | Required element |\n| --- | --- |\n| `item-list` | Contains exactly the ten ranked storefront items. |\n| `item-card` | Shows one storefront or search result. |\n| `item-name` | Shows the item name inside its `item-card`. |\n| `search-input` | Searches the full catalog as the visitor types or when they press Enter; no separate control runs the search. |\n| `search-results` | Contains matching `item-card` results. |\n"},{"id":"ecommerce.l2.transfer-hooks","owners":["ecommerce.l2.stock-transfers-features"],"text":"# Stock transfer application interface\n\nPut `transfer-from`, `transfer-to`, `transfer-qty`, and `transfer-submit` inside the applicable\n`admin-item-row`. Use `warehouse-total` inside each `admin-warehouse-item` for its numeric stock\ntotal. Show `order-error` when a transfer is refused.\n\nPut `data-transfer-input` on each `admin-item-row`. Its value is a JSON object with exactly\n`itemId`, `fromWarehouseId`, and `toWarehouseId` for the currently selected source and destination.\nIdentifiers can be JSON numbers or strings.\n\n<!-- interface:http -->\nExpose `POST /api/admin/transfer`. The JSON body has `itemId`, `fromWarehouseId`,\n`toWarehouseId`, and `quantity`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nExpose `admin_transfer_stock` with arguments in this order: `itemId: u64`,\n`fromWarehouseId: u64`, `toWarehouseId: u64`, `quantity`.\n<!-- /interface -->\n"},{"id":"ecommerce.l2.price-hooks","owners":["ecommerce.l2.price-history-features"],"text":"# Price history application interface\n\nPut `price-input` and `price-submit` inside the applicable `admin-item-row`.\n\nPut a `data-price-input` attribute on each `admin-item-row`. Its value is a JSON object with\n`itemId` and numeric `price` from the current price input. Identifiers can be\nJSON numbers or strings.\n\n<!-- interface:http -->\nExpose `POST /api/admin/price`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nExpose the `admin_change_price` reducer.\n<!-- /interface -->\n\nUse the same action as the visible price control.\n"},{"id":"ecommerce.progression.price-history-order-hooks","owners":["ecommerce.progression.price-history-specifications"],"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.price-history-features"],"text":"# Price history completed-order interface\n\nUse `orders-toggle`, `order-item`, and `order-total` to inspect the order created at checkout.\nUse `buy-now` inside an `item-card` to create a paid order before a price change.\n\n"},{"id":"ecommerce.progression.price-history-cart-hooks","owners":["ecommerce.progression.price-history-specifications"],"requiresFeatures":["ecommerce.feature.checkout","ecommerce.l2.price-history-features"],"text":"# Price history cart and checkout interface\n\nUse `add-to-cart` inside an `item-card`, `cart-toggle`, `cart-total`, and `checkout-submit`.\n"},{"id":"ecommerce.l2.inventory-dashboard-hooks","owners":["ecommerce.l2.inventory-dashboard"],"text":"# Inventory dashboard application interface\n\nUse `admin-link` to open the administrator area. If `low-stock-list` is on a separate screen\nwithin it, expose `low-stock-link` there to reach it. Use `low-stock-list` for items with 10 units\nor fewer and `low-stock-item` for each item. Use\n`buy-now` inside an `item-card` to create sales that change available stock.\n"},{"id":"ecommerce.l2.order-cancellation-hooks","owners":["ecommerce.l2.order-cancellation-features"],"text":"# Order cancellation application interface\n\nUse `order-status` for an order's state inside its `order-item`. Use `cancel-order` on a pending\norder. Use `catalog-link` to return to the catalog.\n`order-status` reads `pending` until the order ships, `shipped` once it has, and `cancelled`\nafter a cancellation. Later features may add further states after `shipped`.\n\nEach customer `order-item` must have `data-cancel-input` containing a JSON object with\nexactly `orderId`. Use the identifier representation required by the selected stack.\n\n<!-- interface:http -->\nUse `POST /api/orders/:id/cancel`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nUse the `cancel_order` reducer.\n<!-- /interface -->\n"},{"id":"ecommerce.l2.sales-dashboard-hooks","owners":["ecommerce.l2.sales-dashboard"],"text":"# Sales dashboard application interface\n\nOpen the administrator area with `admin-link`. If category totals are on a\nseparate tab or screen, expose `sales-link` there to open them. Omit this control\nwhen the totals are already shown.\n\nUse `category-row` for each product category, including a category with no sales yet, whose\nunits and revenue read 0. Use `category-units` and `category-revenue` inside each row. Use `recommended-list` for signed-out\nbest sellers and `recommended-item` for each item. Use `buy-now` inside an `item-card` to create\nsales activity for the dashboard. Each best seller contains its one-based `recommendation-rank`.\n"},{"id":"ecommerce.l2.recommendations-hooks","owners":["ecommerce.l2.recommendations"],"text":"# Recommendations application interface\n\nUse `recommended-list` for recommendations and `recommended-item` for each item in the list.\nUse `catalog-link` to open the catalog with this list visible.\n"},{"id":"ecommerce.feature.accounts.hooks","owners":["ecommerce.feature.accounts"],"text":"# Account application interface\n\nUse these exact `id` attributes on the corresponding visible controls. They do not prescribe UI\nstructure, data modeling, libraries, or implementation strategy.\n\nFrom the signed-out page, show the sign-up inputs, `signup-toggle`, or\n`signin-toggle`. If sign-up is inside the sign-in dialog, opening `signin-toggle`\nmust reveal the sign-up inputs or `signup-toggle`. That control must reveal the\nsign-up form. No other navigation is required to reach it.\nShow the sign-in inputs or `signin-toggle` on the signed-out page.\nWhile signed in, show `signout` directly or reveal it by clicking `current-user`.\nNo other navigation is required to reach sign-out.\n\n| Element ID | Observable element |\n|---|---|\n| `signup-username` | sign-up username input |\n| `signup-password` | sign-up password input |\n| `signup-submit` | sign-up submit control |\n| `signin-toggle` | control that reveals sign-in |\n| `signup-toggle` | reveals sign-up; available on the signed-out page or after opening `signin-toggle`; omit when sign-up inputs are already visible |\n| `signin-username` | sign-in username input |\n| `signin-password` | sign-in password input |\n| `signin-submit` | sign-in submit control |\n| `current-user` | Active account name, present only while signed in. Do not use this hook for a signed-out message. |\n| `signout` | sign-out control |\n| `auth-error` | visible account error |\n\nAccept any username of up to 48 characters made of letters, digits, and hyphens, and any\npassword of up to 64 characters.\n\nExpose the same account writes used by the UI.\n\nFor bearer-token authentication, expose `window.getSessionToken()` as a synchronous\nfunction that returns the current session's existing token, or `null` when signed out.\nThis hook does not prescribe credential storage. Return the caller's real credential;\ndo not create a separate identity.\n\n<!-- interface:http -->\nUse `POST /api/auth/signup` and `POST /api/auth/signin`. Both accept JSON with\n`username` and `password` fields.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nUse the `signUp` and `signIn` reducers. Both take `username` and `password`, in that order.\n<!-- /interface -->\n"},{"id":"ecommerce.feature.purchasing.hooks","owners":["ecommerce.feature.purchasing"],"text":"# Purchasing application interface\n\nUse `catalog-link` to return to the catalog. Use `buy-now` inside an `item-card` to buy one unit. Use `orders-toggle` to open order history.\nIf an overlay blocks catalog navigation, expose a visible `overlay-close` control\nthat dismisses it before `catalog-link` is used. Screens without a blocking\noverlay need no such control. Dialogs, panels, and ordinary page layouts are all allowed.\nUse `order-item` for each order, containing the names of its purchased items. Inside that\n`order-item`, use `order-total` for its numeric total and `order-status` for its current state. Show `out-of-stock` inside an `item-card` once that item's stock reaches zero.\nUse `buy-error` for a failed purchase.\n\nPut `data-buy-input` on each `item-card`. Its value is a JSON object containing that item's\nserver identifier, for example `{\"itemId\":42}`. The identifier may be a JSON number or string.\nUse the same identifier for the visible buy action.\n\nExpose the same purchase used by `buy-now`.\n\n<!-- interface:http -->\nUse `POST /api/items/:id/buy`, where `:id` is the item identifier.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nUse the `buy_now` reducer with the item identifier.\n<!-- /interface -->\n"},{"id":"ecommerce.feature.cart.hooks","owners":["ecommerce.feature.cart"],"text":"# Cart application interface\n\nUse `catalog-link` to return to the catalog. Use `add-to-cart` inside an `item-card` to add one unit. Use `cart-toggle` to open the cart.\nIf an overlay blocks catalog navigation, expose a visible `overlay-close` control\nthat dismisses it before `catalog-link` is used. Screens without a blocking\noverlay need no such control. Dialogs, panels, and ordinary page layouts are all allowed.\nUse `cart-count` for the total units, `cart-item` for each line, `cart-quantity` for its\nquantity, and `cart-total` for the numeric total. Use `cart-remove` to remove a line and\n`empty-cart` for an empty cart. Keep `cart-count` visible, showing 0, while the cart is empty.\n\nPut `data-buy-input` on each `item-card`. Its value is a JSON object containing that item's\nserver identifier, for example `{\"itemId\":42}`. Put `data-cart-input` on each `cart-item`.\nIts value contains the item identifier, for example `{\"itemId\":42}`. The identifier may be a\nJSON number or string.\n\nExpose the same add and quantity-update operations used by the cart controls.\n\n<!-- interface:http -->\nUse `POST /api/cart`. Put `itemId` in the JSON body.\nUse `PATCH /api/cart/:itemId`. Put `quantity` in the JSON body.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nUse the `add_to_cart` reducer with the item identifier.\nUse the `update_cart_quantity` reducer with the item identifier and quantity.\n<!-- /interface -->\n\nUse `checkout-submit` to check out. Use `orders-toggle` to open order history and `order-item`\nfor each order created by checkout.\n"},{"id":"ecommerce.feature.checkout.hooks","owners":["ecommerce.feature.checkout"],"text":"# Checkout application interface\n\nUse `checkout-submit` to check out and `buy-error` for a failed checkout. Use `orders-toggle`\nto open order history and `order-item` for each order created by checkout.\n\nExpose the same checkout used by `checkout-submit`.\n\n<!-- interface:http -->\nUse `POST /api/checkout`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nUse the `checkout` reducer.\n<!-- /interface -->\n"},{"id":"ecommerce.orders.data","owners":["ecommerce.feature.checkout","ecommerce.feature.purchasing"],"text":"# Order data interface\n\nExpose the following data through the database's native read tools. These names\ndescribe a read interface; tables, collections, or database-native views over the\napplication's current records are allowed. Do not maintain separate copies for\nthis interface. Extra columns are allowed.\n\n- `item(id, name, price)` identifies catalog items and their current prices. These are the same items shown in the catalog.\n- `order_account(id, username)` identifies customer accounts. Do not include passwords or tokens in this interface.\n- `order_header(id, account_id, total, refunded, status)` contains every order from direct purchase or checkout, including cancelled orders. `total` is the amount booked for the order. `refunded` is the amount refunded so far, initially zero.\n- `order_line(id, order_id, item_id, quantity, unit_price)` contains each order's purchased lines and their booked unit prices.\n- When carts are available, `order_cart(account_id, item_id, quantity)` contains their current lines. An empty cart has no lines.\n- When carts and warehouse stock are available, `order_reservation(account_id, item_id, warehouse_id, quantity)` contains any stock held for those carts and already deducted from available `stock.quantity`. If the app does not hold stock for carts, this read interface is empty. This does not require adding stock reservations to the app.\n- When warehouse stock is available, `order_allocation(order_line_id, warehouse_id, quantity)` contains the original warehouse quantities used for each order line. Keep these quantities available after cancellation.\n\nEach `id` is a nonempty string or an exact nonnegative integer. Related identifiers\nrefer to that same `id`; a document may use `_id` when it has no `id`. Item and warehouse\nidentifiers match the existing `item` and `warehouse` data interfaces and the visible\napplication actions. Quantities are whole numbers. Money fields use the same currency\nunits as displayed prices, with at most two decimal places. Status uses the states in\nthe order interface.\n\nThe names and fields above must remain readable by the supplied database credentials\nas features are added. Customer screens and writes must use the same underlying records.\nThis does not require making customer data available to unauthenticated app users.\n"},{"id":"ecommerce.feature.reviews.hooks","owners":["ecommerce.feature.reviews"],"text":"# Review application interface\n\nOn item details, show the review form for eligible customers or expose\n`review-toggle` to open it. Omit this control when the form is already shown.\n\nUse `review-rating` on an input or select with values 1 through 5, `review-input` for the comment, and\n`review-submit` to submit it. Put the item's server identifier in\n`data-review-item-id` on `review-submit`. Use `review-average` for the numeric average,\n`review-item` for each visible review, and `review-error` for a failed submission.\n\nExpose the same review operation used by `review-submit`.\n\n<!-- interface:http -->\nUse `POST /api/items/:id/reviews`, where `:id` is the item identifier. Send `rating` and\n`comment` in the request body.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nUse the `submit_review` reducer with the item identifier, rating, and comment.\n<!-- /interface -->\n"},{"id":"ecommerce.feature.warehouse-admin.hooks","owners":["ecommerce.feature.warehouse-admin"],"text":"# Warehouse administration application interface\n\nUse `admin-link` to open `admin-panel`. Use `admin-item-row` for each item and `admin-stock` for\nits numeric total stock. Use `admin-warehouse-item` for each warehouse. Use `admin-location-row`\nfor every item in every warehouse, including zero quantities; the row shows the item name and\nthe warehouse name, with\n`admin-location-qty` for its quantity. Use\n`restock-input` and `restock-submit` inside that row. Use `admin-revenue` for numeric total\nrevenue.\n\nKeep all item, warehouse, and holding rows available in the open admin panel, without pagination.\n\nPut a `data-restock-input` attribute on each `admin-location-row`. Its value is a JSON object\nwith exactly `itemId`, `warehouseId`, and a valid one-unit `quantity`. Identifiers can be JSON numbers or\nstrings.\n\nUse the same restock action as the visible control.\n\n## Stock data interface\n\nExpose singular tables `item(id, name, price)`, `warehouse(id, name)`, and\n`stock(item_id, warehouse_id, quantity)` for direct database access.\n`stock.item_id` and `stock.warehouse_id` reference `item.id` and `warehouse.id`; in a document\nstore they hold the referenced document's `id` value, or its `_id` when it has no `id`. Keep\nthese tables readable and writable with the database's own tools.\n\n<!-- interface:http -->\nExpose `POST /api/admin/restock`. The JSON body has the same fields as `data-restock-input`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nExpose `admin_restock` with arguments in this order: `itemId: u64`, `warehouseId: u64`,\n`quantity`.\n<!-- /interface -->\n"},{"id":"ecommerce.progression.support-history-hooks","owners":["ecommerce.progression.support-history"],"text":"## Customer support history controls\n\n`support-link` opens support. Use `support-ticket` for each ticket in the history, showing\nits subject. The intake form stays reachable from the same control.\n"},{"id":"ecommerce.progression.support-intake-hooks","owners":["ecommerce.progression.support-intake"],"text":"## Support intake controls\n\nUse `support-link` to open support. Use `support-email`, `support-subject`, and\n`support-message` for the ticket fields. Use `support-submit` to submit the ticket and\n`support-reference` to show its reference.\n"},{"id":"ecommerce.progression.support-triage-hooks","owners":["ecommerce.progression.support-triage"],"text":"## Support triage controls\n\nOpen the staff area with `staff-link`. If its ticket queue is on a separate tab or\nscreen, expose `support-queue-link` there to open it. Omit this control when the\neditable ticket controls are already shown.\n\nUse `support-ticket` for each ticket in the staff view. Within a ticket, use\n`support-assignee`, `support-priority`, and `support-status-input` for the editable fields.\nUse `support-update` to apply the changes. Use `support-status` to show the current status.\n`support-assignee` takes the assignee's username; if it is a select, its option values are the\nusernames. `support-priority` offers `low`, `normal`, and `high` as its option values.\n\n`support-status-input` offers the statuses `open`, `in progress`, and `resolved` as its option\nvalues; `support-status` shows the one chosen.\n"},{"id":"ecommerce.progression.fulfilment-queue-hooks","owners":["ecommerce.progression.fulfilment-queue"],"text":"# Fulfilment application interface\n\n| Element ID | Required element |\n| --- | --- |\n| `staff-link` | Opens the fulfilment area. |\n| `fulfilment-panel` | Contains fulfilment tools, including an empty queue; not a loading or error message. |\n| `queue-depth` | Shows the number of pending orders. |\n| `queue-item` | Shows one pending order and names its items. |\n| `queue-warehouse` | Shows the selected warehouse inside its `queue-item`. |\n| `ship-submit` | Marks the order in its `queue-item` as shipped. |\n\nOn `fulfilment-panel`, expose `data-submit-state` for the latest shipping submission:\n`idle` initially, `pending` immediately when submitted, `succeeded` only after the server\nconfirms success, or `failed` after rejection or transport failure. Keep the terminal state\non the panel when the shipped row disappears. A new submission must replace the old state.\n\n`order-status` reads `pending` until the order ships, `shipped` once it has, and `cancelled`\nafter a cancellation. Later features may add further states after `shipped`.\n\nEach customer `order-item` must have `data-ship-input` containing a JSON object with\nexactly `orderId`.\n\nUse the identifier representation required by the selected stack.\n\n<!-- interface:http -->\nUse `POST /api/fulfilment/ship` with `{ \"orderId\": ... }`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nUse the `ship_order` reducer.\n<!-- /interface -->\n"},{"id":"ecommerce.progression.promotion-rules-hooks","owners":["ecommerce.progression.promotion-rules"],"text":"# Promotion rule application interface\n\nUse `staff-link` to open the staff area. In that area, use `promotions-link` for promotion\nmanagement. Use `promotion-code`, `promotion-discount`,\n`promotion-start`, `promotion-end`, `promotion-limit`, and `promotion-submit` to create a rule.\nList rules as `promotion-item` elements and expose the saved values with the matching field IDs.\n\nExpose the same rule creation used by `promotion-submit`.\n\n<!-- interface:http -->\nUse `POST /api/promotions` with a JSON object containing `code` (string),\n`discountPercent` (number), `startMicros` and `endMicros` (integer numbers of microseconds\nsince the Unix epoch), and `usageLimit` (positive integer).\n<!-- /interface -->\n\n<!-- interface:reducer -->\nUse the `create_promotion` reducer with arguments in this order: `code: string`,\n`discountPercent: f64`, `startMicros: i64`, `endMicros: i64`, `usageLimit: u32`.\nBoth time arguments are microseconds since the Unix epoch.\n<!-- /interface -->\n\nOn a listed rule, `promotion-start` and `promotion-end` show the dates as entered, in ISO\n`YYYY-MM-DD` form.\n"},{"id":"ecommerce.progression.notification-preferences-hooks","owners":["ecommerce.progression.notification-preferences"],"text":"# Notification preference application interface\n\nSaving may leave settings open. Use `catalog-link` to return to the catalog. If\na settings overlay blocks navigation, expose `overlay-close` to dismiss it.\n\nMake `notification-settings` available from the catalog while signed in, without first opening another area. Use it to open the settings. Use `notification-order` and\n`notification-stock` for the choices, and `notification-save` to save them. Each choice exposes\nits current state in `data-state` as `on` or `off`.\nBoth choices start `off` for a new account. Activating `notification-order` or\n`notification-stock` switches it between `on` and `off`.\n"},{"id":"ecommerce.l3.reservation-hooks","owners":["ecommerce.l3.reservations-features"],"text":"# Reservation application interface\n\nUse `cart-reservation-timer` inside a `cart-item` for the remaining reservation time in seconds.\nUse `cart-item-expired` inside an expired cart line and `cart-expired-notice` after cart\nexpiration.\n"},{"id":"ecommerce.progression.managed-support-hooks","owners":["ecommerce.progression.managed-support"],"text":"# Managed support application interface\n\n## Managed support controls\n\nUse `support-ticket` for each case and set `data-entity-id` to that case's server identifier.\nWithin a case, use `support-status` for the current status, `support-reply` for the reply field,\n`support-reply-submit` to send a reply, and `support-reply-item` for each reply.\n\nExpose the same reply operation used by `support-reply-submit`.\n\n<!-- interface:http -->\nUse `POST /api/support/:id/replies`, where `:id` is the case identifier from `data-entity-id`.\nThe JSON body is `{ \"body\": \"<reply text>\" }`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nUse the `reply_support` reducer with arguments in this order: `ticketId: u64`, `body: string`.\nRender `data-entity-id` as the decimal case identifier without precision loss.\n<!-- /interface -->\n"},{"id":"ecommerce.l3.scheduled-restock-hooks","owners":["ecommerce.l3.scheduled-restocks-features"],"text":"# Scheduled restock application interface\n\nUse `admin-link` to open the administrator area. If these controls are on a separate screen within\nit, expose `restocks-link` there to reach them. Use `schedule-restock-item`, `schedule-restock-warehouse`, `schedule-restock-qty`, and\n`schedule-restock-delay` for the inputs. Use `schedule-restock-submit` to schedule the restock.\nSet its `data-action-input` to a JSON object with exactly `item`, `warehouse`, `quantity`, and\n`delaySeconds`. `item` and `warehouse` are their names as strings; `quantity` and\n`delaySeconds` are JSON integers. Use `pending-restock-item` for each pending row and set its\n`data-entity-id` to the restock's server identifier, written as a decimal number.\nEach row contains the item name and sets `data-quantity` to its integer quantity. Use\n`pending-restock-remaining` for its remaining seconds, `pending-restock-cancel` to cancel it,\nand `stock-ledger-entry` for a completed stock movement.\n\n<!-- interface:http -->\nExpose `POST /api/admin/scheduled-restocks` and `DELETE /api/admin/scheduled-restocks/:id`.\nThe POST body has the same fields as `data-action-input`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nExpose `schedule_restock` with arguments in this order: `item: string`, `warehouse: string`,\n`quantity: u32`, `delaySeconds: u32`; and `cancel_scheduled_restock` with `restockId: u64`.\n<!-- /interface -->\n"},{"id":"ecommerce.l3.order-delivery-hooks","owners":["ecommerce.l3.order-delivery-features"],"text":"# Order delivery application interface\n\nUse `completed-order-item` for each completed order in the staff view. Use\n`completed-order-status` inside it for the current state. This extends the order lifecycle:\nafter `shipped`, `order-status` and `completed-order-status` read `delivered`.\n"},{"id":"ecommerce.l3.cart-expiration-hooks","owners":["ecommerce.l3.cart-expiration-features"],"text":"# Cart expiration application interface\n\nUse `cart-reservation-timer` inside a `cart-item` for the remaining reservation time in seconds.\nUse `cart-item-expired` inside an expired cart line and `cart-expired-notice` after cart\nexpiration.\n"},{"id":"ecommerce.progression.promotion-checkout-hooks","owners":["ecommerce.progression.promotion-checkout"],"text":"# Promotion checkout application interface\n\nUse `cart-promotion` for the code and `apply-promotion` to apply it. Use `promotion-error` when a\ncode is refused. Expose the saved discount as `order-discount` inside its `order-item`.\n"},{"id":"ecommerce.progression.stock-alert-hooks","owners":["ecommerce.progression.stock-alerts"],"text":"# Stock alert application interface\n\nUse `stock-alert` inside an unavailable `item-card` to request an alert. Use\n`notifications-toggle` to open notifications and `notification-item` for each alert.\nOn that `item-card`, expose `data-submit-state` for the latest stock-alert request:\n`idle` initially, `pending` immediately when submitted, `succeeded` only after the server\nconfirms success, or `failed` after rejection or transport failure. Keep the terminal state\nif the request button disappears. A new submission must replace the old state.\nUse `notifications-panel` for the opened notification view, including while its contents\nload. Set its `aria-busy` attribute to `false` only when the signed-in account's contents\nhave loaded successfully, including an empty list; keep it `true` while loading or after\na failed read. Its toggle may also close it.\nWithin a delivered stock alert, expose `stock-alert-delivery` containing the item's\ndisplayed name. A pending request must not expose `stock-alert-delivery`.\n\nUse `catalog-link` to return to the catalog. If an overlay blocks navigation,\nexpose a visible `overlay-close` control that dismisses it before navigation.\nScreens without a blocking overlay do not need this control.\n"},{"id":"ecommerce.l3.order-return-hooks","owners":["ecommerce.l3.order-returns-features"],"text":"# Order return application interface\n\nUse `return-item` inside an `order-item` for each item that can be returned. After a return,\nthe same `order-item` contains the word `returned`.\n\nEach ordinary item has an `order-line` containing its name, including while pending.\nSet `data-return-input` on that line to JSON with `orderId` and `itemId`, using the\nidentifiers accepted by the return action.\n\n<!-- interface:http -->\n`returnItem` is `POST /api/orders/{orderId}/items/{itemId}/return`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\n`returnItem` is `return_order_item(orderId, itemId)`.\n<!-- /interface -->\n\nThe existing `orders-toggle`, `order-item`, `item-stock`, `admin-revenue`, and `catalog-link`\ninterfaces expose the order, stock, and accounting results.\n"},{"id":"ecommerce.progression.faceted-search-hooks","owners":["ecommerce.progression.faceted-search"],"text":"# Faceted search application interface\n\n| Element ID | Required element |\n| --- | --- |\n| `category-filter` | Sets the category filter; a text input, or a `select` whose option values are the category names. |\n| `minimum-price` | Sets the inclusive minimum price. |\n| `maximum-price` | Sets the inclusive maximum price. |\n| `in-stock-filter` | Toggles the in-stock-only filter, which starts off. |\n| `search-results` | Contains the filtered page; with no search text and no filter selected, the current page of the full catalog. |\n| `item-card` | Shows one result inside `search-results`. |\n| `search-next-page` | Opens the next page. |\n| `search-previous-page` | Opens the previous page. |\n\nIf a `filter-apply` control exists, activating it applies the filters; otherwise results update\nas each filter changes.\n\nSearch text or any active filter selects alphabetical ordering. With neither, use\npurchase ranking and break ties by item name. Clearing all search text and filters\nrestores purchase ranking. Both modes can use the same rendered list.\n"},{"id":"ecommerce.progression.order-support-hooks","owners":["ecommerce.progression.order-support"],"text":"# Order-linked support application interface\n\n## Order-linked support controls\n\nWithin a `support-ticket`, use `support-order-option` for each order that the customer can attach,\n`support-link-order` to attach the selected order, and `support-order` for the attached order. The\nlink action must expose its input in `data-action-input` for the named\n`linkSupportOrder` application action.\n\n<!-- interface:http -->\n`linkSupportOrder` is `POST /api/support/cases/{caseId}/order`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\n`linkSupportOrder` is the `link_support_order(caseId, orderId)` reducer.\n<!-- /interface -->\n"},{"id":"ecommerce.progression.personalized-recommendation-hooks","owners":["ecommerce.progression.personalized-recommendations"],"text":"# Personalized recommendation application interface\n\nUse `recommendations` for the ordered recommendation list. Use `recommended-item` for each\nitem in that list. Each item contains its item name. Use `recommendation-rank` inside each item\nfor its one-based position in the list.\n"},{"id":"ecommerce.progression.promotion-reporting-hooks","owners":["ecommerce.progression.promotion-reporting"],"text":"# Promotion reporting application interface\n\nUse `promotions-link` to open the staff view. Use `promotion-report` for each promotion and\n`promotion-redemptions` and `promotion-revenue` for its totals.\n"},{"id":"ecommerce.feature.store-credit.interface","owners":["ecommerce.feature.store-credit"],"text":"# Store credit interface\n\nOpen customer credit with `credit-link`. The `credit-panel` exposes the signed-in account's `data-account-id` and `aria-busy=\"false\"` when loaded. Show `credit-balance` in major currency units and one `credit-entry` per movement.\n\nStaff use `credit-customer`, `credit-amount-input` (major units), `credit-reference-input`, and `credit-grant`. The grant control exposes `data-action-input` as JSON with `accountId`, `amountMinor`, and `reference`.\n\nUse `credit-checkout` in the cart. Each `order-item` shows `payment-credit-amount` and `payment-external-amount` in major units. Their sum is `payment-amount`.\n\n<!-- interface:http -->\n`grantCredit` is `POST /api/staff/credit` with `accountId`, `amountMinor`, and `reference`.\n`checkoutCredit` is `POST /api/checkout/credit` with no body fields.\n<!-- /interface -->\n\n<!-- interface:reducer -->\n`grantCredit` is `grant_credit(accountId, amountMinor, reference)`.\n`checkoutCredit` is `checkout_credit()`.\n<!-- /interface -->\n"},{"id":"ecommerce.feature.subscriptions.interface","owners":["ecommerce.feature.subscriptions"],"text":"# Subscription interface\n\nOpen with `subscriptions-link`. Use `subscription-item-input` (item name), `subscription-quantity-input`, `subscription-interval-input` (whole seconds, minimum 30), `subscription-deliveries-input` (1–12), and `subscription-create`.\n\nThe loaded `subscriptions-panel` has `aria-busy=\"false\"`. Each `subscription-row` includes the item name and `subscription-status`: `active`, `paused`, `cancelled`, or `complete`. Use `subscription-pause`, `subscription-resume`, and `subscription-cancel`; each exposes `data-action-input` with `subscriptionId`. Each processed slot has one `subscription-delivery` inside the row, with `subscription-delivery-status` (`paid` or `skipped`). `subscription-total` shows the sum of its recorded payments in major currency units. Ordinary orders and payment records include their item names.\n\n<!-- interface:http -->\n`pauseSubscription` is `POST /api/subscriptions/{subscriptionId}/pause`.\n`resumeSubscription` is `POST /api/subscriptions/{subscriptionId}/resume`.\n`cancelSubscription` is `POST /api/subscriptions/{subscriptionId}/cancel`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\n`pauseSubscription` is `pause_subscription(subscriptionId)`.\n`resumeSubscription` is `resume_subscription(subscriptionId)`.\n`cancelSubscription` is `cancel_subscription(subscriptionId)`.\n<!-- /interface -->\n"},{"id":"ecommerce.progression.delivery-notification-hooks","owners":["ecommerce.progression.delivery-notifications"],"text":"# Delivery notification application interface\n\nUse `notifications-toggle` to open notifications. Use `notification-item` for each notification\nand `notification-unread-count` for the unread total.\n\nThe `notifications-panel` has `aria-busy=\"false\"` only when the signed-in account's\nnotifications have loaded. Keep the panel present when the list is empty.\n"},{"id":"ecommerce.progression.support-refund-hooks","owners":["ecommerce.progression.support-refunds"],"text":"# Support refund application interface\n\n## Support refund controls\n\nWithin a `support-ticket`, use `support-refund` for the refund action and\n`support-refund-total` for the recorded refund amount. The refund action must expose its input in\n`data-action-input` for the named `supportRefund` application action. The `support-ticket` also\nexposes the same JSON `{ \"caseId\": \"...\" }` in `data-refund-input` so the case remains addressable\nafter its refund button is disabled or removed. Within an `order-item`, use\n`order-refund-total` for the refunded amount and `refund-entry` for each refund record. Each\n`refund-entry` includes the order item name.\n\n<!-- interface:http -->\n`supportRefund` is `POST /api/support/cases/{caseId}/refund`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\n`supportRefund` is the `support_refund(caseId)` reducer.\n<!-- /interface -->\n"},{"id":"ecommerce.progression.automatic-reorder-hooks","owners":["ecommerce.progression.automatic-reorder"],"text":"# Automatic reorder application interface\n\nUse these application interface names:\n\n- `reorder-link` opens the automatic reorder rules for warehouse staff.\n- `reorder-item`, `reorder-threshold`, and `reorder-quantity` identify the rule inputs.\n- `reorder-submit` saves the rule.\n- `reorder-rule-item` identifies each saved rule, sets `data-entity-id` to the rule's item\n identifier, and contains its item name, threshold, quantity, and current state.\n Set `data-threshold` and `data-quantity` to their integer values. Set\n `data-action-input` on `reorder-submit` to JSON with `itemId`, `threshold`, and `quantity`\n for the current form values.\n\nSaving a rule is the named `saveReorderRule` application action.\n\n<!-- interface:http -->\n`saveReorderRule` is `PUT /api/reorders/{itemId}` with `threshold` and `quantity` in the body.\n<!-- /interface -->\n\n<!-- interface:reducer -->\n`saveReorderRule` is the `save_reorder_rule(itemId, threshold, quantity)` reducer.\n<!-- /interface -->\n\nUse `buy-now` inside an `item-card` to create stock changes that evaluate a reorder rule.\n"},{"id":"ecommerce.progression.cart-recovery-hooks","owners":["ecommerce.progression.cart-recovery"],"text":"# Cart recovery application interface\n\nUse these application interface names:\n\n- `expired-cart` identifies the expired cart.\n- `restore-cart` restores that cart.\n- `cart-restore-warning` lists the names of items that could not be restored.\n\nThe existing `cart-item` control identifies each item restored to the active cart.\n"},{"id":"ecommerce.progression.recommendation-feedback-hooks","owners":["ecommerce.progression.recommendation-feedback"],"text":"# Recommendation feedback application interface\n\nUse `dismiss-recommendation` inside each `recommended-item`.\n"},{"id":"ecommerce.feature.split-tender-refunds.interface","owners":["ecommerce.feature.split-tender-refunds"],"text":"# Split refund interface\n\nReuse `support-refund` and its existing named action. Each `refund-entry` shows `refund-credit-amount` and `refund-external-amount` in major currency units. Show the restored credit in the existing credit balance and history.\n"},{"id":"ecommerce.interface.product-bundles","owners":["ecommerce.feature.product-bundles"],"text":"# Product bundle interface\n\nThe catalog has a `bundles-link`. The bundle panel contains `bundle-card` rows with\n`bundle-name`, numeric `bundle-price`, and `bundle-component` rows. Each component row has\n`bundle-component-name` and numeric `bundle-component-quantity`. Each component row also\nexposes its quantity in `data-quantity`.\n\nCatalog staff use `bundle-name-input`, `bundle-price-input` (currency units), and\n`bundle-components-input` (JSON array of `{ \"item\": \"product name\", \"quantity\": 1 }`),\nthen `bundle-save`. Saving an existing name edits that bundle. Each `bundle-card` exposes\n`data-bundle-input` as JSON `{ \"bundleId\": \"...\" }`.\nThe save button exposes `data-bundle-save-input` with `{ name, price, componentsJson }`\nfrom the current form values.\n\nUse the same application write for the visible form and this named action:\n\n<!-- interface:http -->\nSave a bundle with `POST /api/bundles` and `{ name, price, componentsJson }`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nSave a bundle with `save_bundle(name: string, price: number, componentsJson: string)`.\n<!-- /interface -->\n\nEach component uses an existing product's exact name. `componentsJson` contains the\ncomponent array as a JSON string.\n"},{"id":"ecommerce.interface.bundle-checkout","owners":["ecommerce.feature.bundle-checkout"],"text":"# Bundle checkout interface\n\nUse `bundle-add-to-cart` inside `bundle-card`. A bundle cart line uses the existing\n`cart-item`, `cart-reservation-timer`, and `cart-item-expired` interfaces, with\n`bundle-remove` to remove it. `checkout-submit` buys the cart through the existing checkout\naction. The existing `order-item` and `payment-amount` show the bundle name and price paid.\n\n<!-- interface:http -->\nAdd one bundle with `POST /api/cart/bundles` and `{ bundleId }`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nAdd one bundle with `add_bundle_to_cart(bundleId: u64)`.\n<!-- /interface -->\n\nUse the same application action as the visible control. The `data-bundle-input` attribute\nsupplies its bundle ID. The cart and order interfaces remain shared with individual products.\n"},{"id":"ecommerce.interface.bundle-returns","owners":["ecommerce.feature.bundle-returns"],"text":"# Bundle return interface\n\nUse `return-bundle` inside the existing `order-item`. Mark each returned bundle line\n`returned`. The order's `order-status` reads `returned` when all its lines have been\nreturned; otherwise keep its current fulfilment status. `bundle-refund-amount` shows\nthe refunded amount in currency units.\nEach bundle `order-item` exposes `data-bundle-return-input` as JSON `{ \"orderId\": \"...\" }`.\n\n<!-- interface:http -->\nReturn a whole bundle with `POST /api/bundle-orders/:orderId/return`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nReturn a whole bundle with `return_bundle(orderId: u64)`.\n<!-- /interface -->\n\nUse the same application action as the visible control.\n"}],"mode":"action","requirements":[{"id":"ecommerce.progression.fresh","owners":["recipe"],"text":"## New application\n\nBuild an ecommerce application from the product work below. Use `Storefront`\nas the visible page title. Use the application interface names where they are\nprovided. Start with no orders or purchase history.\n\n"},{"id":"ecommerce.progression.upgrade","owners":["recipe"],"text":"## Existing application\n\nAdd the current product work to the existing ecommerce application. Preserve its data;\ndo not add sample orders or purchase history while adding features.\n"},{"id":"ecommerce.l2.inventory-dashboard","owners":["ecommerce.l2.inventory-dashboard"],"text":"## Inventory dashboard\n\nGive administrators a low-stock view. It lists items with 10 units or fewer, most urgent first.\n"},{"id":"ecommerce.l2.sales-dashboard","owners":["ecommerce.l2.sales-dashboard"],"text":"## Sales dashboard\n\nGive customers and administrators sales views. Category totals show units sold and revenue for\neach category. Signed-out visitors see best sellers on the storefront.\n"},{"id":"ecommerce.l2.recommendations","owners":["ecommerce.l2.recommendations"],"text":"## Recommendations\n\nShow customers a recommended-for-you list on the catalog page. It recommends items from categories the customer has\nbought from and excludes items already in the cart.\n"},{"id":"ecommerce.feature.accounts.requirement","owners":["ecommerce.feature.accounts"],"text":"## Accounts\n\nVisitors can create an account with a username and password. Returning users can sign in with\nthose credentials, see which account is active, and sign out. Show a useful error for a taken\nusername or an incorrect password.\n"},{"id":"ecommerce.feature.catalog-items.requirement","owners":["ecommerce.feature.catalog-items"],"text":"## Catalog items\n\nShow public catalog items. Each item shows its name, price, and total stock.\n"},{"id":"ecommerce.feature.catalog-discovery.requirement","owners":["ecommerce.feature.catalog-discovery"],"text":"## Catalog discovery\n\nShow the ten most-purchased items. Break ties by item name. Search matches any part of\nan item name, without regard to case, across the full catalog.\n"},{"id":"ecommerce.feature.purchasing.requirement","owners":["ecommerce.feature.purchasing"],"text":"## Purchasing and orders\n\nA signed-in customer can buy one unit of an available item. The purchase reduces stock and\ncreates an order for that customer at the price paid. Their order history shows the newest\norders first, including items, quantities, prices, and totals.\n"},{"id":"ecommerce.feature.cart.requirement","owners":["ecommerce.feature.cart"],"text":"## Cart\n\nA signed-in customer has one cart. They can add an item, change its quantity, remove it, and\nsee the total. Adding the same item again increases the existing line quantity.\n"},{"id":"ecommerce.feature.checkout.requirement","owners":["ecommerce.feature.checkout"],"text":"## Checkout\n\nCheckout creates one order, reduces stock for every line, and empties the cart. Show an\nexplanation if checkout fails.\n"},{"id":"ecommerce.feature.reviews.requirement","owners":["ecommerce.feature.reviews"],"text":"## Reviews\n\nCustomers can rate items they purchased from one to five and add a comment. Show reviews\nand the average rating on the item detail.\n"},{"id":"ecommerce.feature.warehouse-admin.requirement","owners":["ecommerce.feature.warehouse-admin"],"text":"## Warehouse administration\n\nProvide an administration area that lists every item, both warehouses, and the quantity held in\neach warehouse. An administrator can add units to a selected item and warehouse. The area also\nshows total revenue across orders.\n"},{"id":"ecommerce.spec.access-control.purchasing","owners":["ecommerce.spec.access-control"],"requiresFeatures":["ecommerce.feature.purchasing"],"text":"## Access control: purchasing\n\nTreat identity as server-enforced authority, not UI decoration. Unauthenticated\ncallers cannot purchase, one account cannot place an order for another account,\nand order history is visible only to its owner. Knowing a username never grants\naccess to that account.\n\n"},{"id":"ecommerce.spec.access-control.warehouse-admin","owners":["ecommerce.spec.access-control"],"requiresFeatures":["ecommerce.feature.warehouse-admin"],"text":"## Access control: warehouse administration\n\nCustomer accounts cannot perform warehouse-administration writes. Enforce this\non the server even if the corresponding controls are hidden in the UI.\n\n"},{"id":"ecommerce.spec.access-control.reviews","owners":["ecommerce.spec.access-control"],"requiresFeatures":["ecommerce.feature.reviews"],"text":"## Access control: reviews\n\nOnly a customer who purchased an item may review it. Enforce this on the server\ninstead of relying on whether the review form is visible.\n\n"},{"id":"ecommerce.spec.access-control.cart","owners":["ecommerce.spec.access-control"],"requiresFeatures":["ecommerce.feature.cart"],"text":"## Access control: cart\n\nOne account cannot read or change another account's cart. Refuse a cart request\nwhose quantity is negative without changing state.\n\n"},{"id":"ecommerce.progression.staff-access","owners":["ecommerce.progression.staff-access"],"text":"## Staff access\n\nStaff and administrators can sign in and use staff areas.\n"},{"id":"ecommerce.spec.state-durability.accounts","owners":["ecommerce.spec.state-durability"],"requiresFeatures":["ecommerce.feature.accounts"],"text":"## State durability: accounts\n\nA signed-in session survives a page reload as the same account.\n\n"},{"id":"ecommerce.spec.state-durability.account-data","owners":["ecommerce.spec.state-durability"],"requiresFeatures":["ecommerce.feature.accounts","ecommerce.feature.cart"],"text":"## State durability: account data\n\nThe same account keeps its cart and orders across reload and connection loss.\nAfter reconnect it has current state without another sign-in. Restarting the\napplication must not duplicate starting data or reset state users changed.\n\n"},{"id":"ecommerce.spec.state-durability.checkout-crash","owners":["ecommerce.spec.state-durability"],"requiresFeatures":["ecommerce.feature.checkout"],"text":"After the application or database process restarts, an interrupted checkout leaves\neither its unchanged cart or one complete order with the cart cleared. A checkout\nreported as complete and the account's earlier orders remain recorded correctly.\nThe application can accept new checkouts after recovery.\n"},{"id":"ecommerce.progression.customer-profile","owners":["ecommerce.progression.customer-profile"],"text":"## Customer profile\n\nA signed-in customer can save and view their name and shipping address.\n"},{"id":"ecommerce.progression.support-intake","owners":["ecommerce.progression.support-intake"],"text":"## Support intake\n\nAnyone can open a support ticket with contact details, a subject, and a message. Return a\nreference that the visitor can use to identify the ticket.\n"},{"id":"ecommerce.spec.live-state.catalog-purchasing","owners":["ecommerce.spec.live-state"],"requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing"],"text":"## Live state: catalog and purchasing\n\nStock and best-seller ranking update every affected open storefront without a\nreload, including signed-out storefronts.\n\n"},{"id":"ecommerce.spec.live-state.cart","owners":["ecommerce.spec.live-state"],"requiresFeatures":["ecommerce.feature.cart"],"text":"## Live state: cart\n\nThe same account open in two clients sees one current cart; a change in either\nclient reaches the other without a reload.\n\n"},{"id":"ecommerce.spec.live-state.reviews","owners":["ecommerce.spec.live-state"],"requiresFeatures":["ecommerce.feature.reviews"],"text":"## Live state: reviews\n\nReviews and average ratings update affected open item views without a reload.\nA view opened while a review is submitted converges to the current review list.\n\n"},{"id":"ecommerce.spec.live-state.warehouse-admin","owners":["ecommerce.spec.live-state"],"requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.warehouse-admin"],"text":"## Live state: warehouse administration\n\nRestocking updates warehouse quantities, total item stock, and open storefronts\nwithout a reload.\n\n"},{"id":"ecommerce.progression.staff-roles","owners":["ecommerce.progression.staff-roles"],"text":"## Staff roles\n\nAn administrator can assign a role to an existing staff account. The `admin` role grants\nadministrator access. The `staff` and `inventory` roles grant staff access without\nadministrator access.\n"},{"id":"ecommerce.progression.catalog-management","owners":["ecommerce.progression.catalog-management"],"text":"## Catalog management\n\nAuthorized staff can add a product with named variants. The new product and its variants appear\nin the public catalog.\n"},{"id":"ecommerce.spec.concurrency-safety.purchasing","owners":["ecommerce.spec.concurrency-safety"],"requiresFeatures":["ecommerce.feature.purchasing"],"text":"## Concurrency safety: purchasing\n\nStock never becomes negative and only one customer can receive the last unit.\n\n"},{"id":"ecommerce.spec.concurrency-safety.restocking","owners":["ecommerce.spec.concurrency-safety"],"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"text":"## Concurrency safety: restocking\n\nConcurrent restocks and purchases preserve every accepted stock change.\n\n"},{"id":"ecommerce.spec.concurrency-safety.checkout","owners":["ecommerce.spec.concurrency-safety"],"requiresFeatures":["ecommerce.feature.checkout"],"text":"## Concurrency safety: checkout\n\nRepeating or racing checkout for the same cart creates only one order.\n\n"},{"id":"ecommerce.progression.payment-records","owners":["ecommerce.progression.payment-records"],"text":"## Payment records\n\nShow the payment status and amount paid on each order.\n"},{"id":"ecommerce.progression.staff-activity","owners":["ecommerce.progression.staff-activity"],"text":"## Staff activity history\n\nStaff can inspect a history of administrative changes. Each entry shows the staff member,\naction, subject, and time.\n"},{"id":"ecommerce.spec.transactional-integrity.reviews","owners":["ecommerce.spec.transactional-integrity"],"requiresFeatures":["ecommerce.feature.reviews"],"text":"## Transactional integrity: reviews\n\nA customer has at most one review per item. A later submission must not create\na duplicate; it may update the existing review or be refused.\n\n"},{"id":"ecommerce.spec.transactional-integrity.purchasing","owners":["ecommerce.spec.transactional-integrity"],"requiresFeatures":["ecommerce.feature.purchasing"],"text":"## Transactional integrity: purchasing\n\nThe server controls prices and order attribution rather than trusting client\nvalues. Historical order prices do not change.\n\n"},{"id":"ecommerce.spec.transactional-integrity.warehouse-accounting","owners":["ecommerce.spec.transactional-integrity"],"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"text":"## Transactional integrity: warehouse accounting\n\nEvery stock reduction caused by a sale has its matching order, revenue equals\nthe orders' recorded totals, and fresh clients agree with those results.\n\n"},{"id":"ecommerce.spec.external-data-sync.requirement","owners":["ecommerce.spec.external-data-sync"],"requiresFeatures":["ecommerce.feature.warehouse-admin"],"text":"## External data synchronization\n\nOther systems may write stock directly without calling the application. Open\npages and newly loaded pages must converge to a direct stock correction,\nincluding a correction made while the application server is down.\n"},{"id":"ecommerce.progression.fulfilment-queue","owners":["ecommerce.progression.fulfilment-queue"],"text":"# Fulfilment operations\n\nAdd a fulfilment area for staff and administrators. Show pending orders, their items, and the\nwarehouse that will ship them. Staff and administrators can mark an order as shipped. Show the\nnew status in the fulfilment area and the customer's order history.\n"},{"id":"ecommerce.l2.stock-transfer","owners":["ecommerce.l2.stock-transfers-features"],"text":"## Stock transfers\n\nAn administrator can move units of an item from one warehouse to another.\n"},{"id":"ecommerce.l2.order-cancellation","owners":["ecommerce.l2.order-cancellation-features"],"text":"## Order cancellation\n\nA customer can cancel an order before it ships. Refund the purchase, return its stock to\nthe supplying warehouse, and show its cancelled status in order history.\n"},{"id":"ecommerce.l2.price-history","owners":["ecommerce.l2.price-history-features"],"text":"## Price history\n\nAn administrator can change an item's price. Show the price in the public catalog.\n"},{"id":"ecommerce.progression.price-history-orders","owners":["ecommerce.progression.price-history-specifications"],"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.price-history-features"],"text":"## Price history: completed orders\n\nCompleted orders keep the price paid at checkout. A later price change does not alter a receipt\nor revenue already recorded.\n\n"},{"id":"ecommerce.progression.price-history-cart-checkout","owners":["ecommerce.progression.price-history-specifications"],"requiresFeatures":["ecommerce.feature.checkout","ecommerce.l2.price-history-features"],"text":"## Price history: cart and checkout\n\nEvery open cart shows the new price without a reload. Checkout uses the current price.\n"},{"id":"ecommerce.progression.cancellation-queue","owners":["ecommerce.progression.cancellation-queue-specifications"],"requiresFeatures":["ecommerce.l2.order-cancellation-features","ecommerce.progression.fulfilment-queue"],"text":"## Fulfilment queue integration\n\nWhen fulfilment is available, cancelling a pending order also removes it from the fulfilment\nqueue.\n"},{"id":"ecommerce.progression.cancellation-accounting","owners":["ecommerce.progression.cancellation-accounting-specifications"],"requiresFeatures":["ecommerce.l2.order-cancellation-features"],"text":"## Cancellation accounting\n\nRevenue equals the sum of orders that remain paid. Cancellation removes the cancelled amount.\n\n"},{"id":"ecommerce.progression.support-triage","owners":["ecommerce.progression.support-triage"],"text":"## Support triage\n\nStaff can view new support tickets, assign a ticket, set its priority, and change its status.\n"},{"id":"ecommerce.progression.price-accounting","owners":["ecommerce.progression.price-accounting-specifications"],"requiresFeatures":["ecommerce.l2.price-history-features"],"text":"## Price accounting\n\nLater catalog price changes do not change the amount recorded on an existing order.\n"},{"id":"ecommerce.progression.support-history","owners":["ecommerce.progression.support-history"],"text":"## Customer support history\n\nSigned-in customers can see their support ticket history.\n"},{"id":"ecommerce.progression.promotion-rules","owners":["ecommerce.progression.promotion-rules"],"text":"## Promotion rules\n\nStaff can create promotion codes with a percentage discount, a start date, an end date, and a\nredemption limit.\n"},{"id":"ecommerce.progression.notification-preferences","owners":["ecommerce.progression.notification-preferences"],"text":"## Notification preferences\n\nSigned-in customers can turn order and stock notifications on or off.\n"},{"id":"ecommerce.progression.transfer-authorization","owners":["ecommerce.progression.operations-access-specifications"],"requiresFeatures":["ecommerce.l2.stock-transfers-features"],"text":"## Transfer authorization\n\nThe server allows only staff and administrators to transfer warehouse stock.\n"},{"id":"ecommerce.progression.price-authorization","owners":["ecommerce.progression.operations-access-specifications"],"requiresFeatures":["ecommerce.l2.price-history-features"],"text":"## Price authorization\n\nThe server allows only administrators to change catalog prices.\n"},{"id":"ecommerce.progression.shipping-authorization","owners":["ecommerce.progression.operations-access-specifications"],"requiresFeatures":["ecommerce.progression.fulfilment-queue"],"text":"## Shipping authorization\n\nThe server allows only staff and administrators to mark orders as shipped.\n"},{"id":"ecommerce.progression.order-ownership","owners":["ecommerce.progression.operations-access-specifications"],"requiresFeatures":["ecommerce.l2.order-cancellation-features"],"text":"## Order ownership\n\nThe server allows a customer to act only on that customer's own orders.\n"},{"id":"ecommerce.progression.review-access","owners":["ecommerce.progression.review-access-specifications"],"requiresFeatures":["ecommerce.feature.reviews"],"text":"## Review access\n\nOnly a customer who bought an item can review it.\n"},{"id":"ecommerce.progression.transfer-conservation","owners":["ecommerce.progression.inventory-conservation-specifications"],"requiresFeatures":["ecommerce.l2.stock-transfers-features"],"text":"## Stock conservation\n\nStock moves between warehouses without changing the total. A refused transfer changes nothing.\nWhen a purchase and a transfer overlap, the final total reflects the sold units exactly once.\n"},{"id":"ecommerce.progression.cancellation-conservation","owners":["ecommerce.progression.inventory-conservation-specifications"],"requiresFeatures":["ecommerce.l2.order-cancellation-features"],"text":"## Cancellation conservation\n\nCancelling an order restores its stock to the warehouse that supplied it. The restored total is\nthe same for current and newly opened clients.\n"},{"id":"ecommerce.l3.reservations","owners":["ecommerce.l3.reservations-features"],"text":"## Reservations\n\nAdding an item to a cart reserves its stock for 90 seconds. The cart shows the remaining time.\nCheckout consumes a live reservation. An expired reservation releases its stock and remains\nvisible as expired. Adding the item again renews the reservation.\n"},{"id":"ecommerce.l3.scheduled-restocks","owners":["ecommerce.l3.scheduled-restocks-features"],"text":"## Scheduled restocks\n\nAn admin can schedule and cancel a restock. Show pending restocks and their remaining\ntime. Show completed stock movements in a stock ledger.\n"},{"id":"ecommerce.l3.order-delivery","owners":["ecommerce.l3.order-delivery-features"],"text":"## Order delivery\n\nA shipped order becomes delivered 60 seconds after shipping. Show its status in the\ncustomer's order history and the staff view.\n"},{"id":"ecommerce.l3.cart-expiration","owners":["ecommerce.l3.cart-expiration-features"],"text":"## Cart expiration\n\nA cart with no activity for five minutes expires and releases its reservations. The customer\nsees an empty cart and an expiration notice.\n"},{"id":"ecommerce.l3.durable-reservations","owners":["ecommerce.l3.deferred-durability-specifications"],"requiresFeatures":["ecommerce.l3.reservations-features"],"text":"## Durable reservations\n\nPending reservations survive a backend restart.\n\n"},{"id":"ecommerce.progression.managed-support","owners":["ecommerce.progression.managed-support"],"text":"## Managed support cases\n\nCustomers and staff can exchange replies and update the status of a support case.\n"},{"id":"ecommerce.l3.durable-restocks","owners":["ecommerce.l3.deferred-durability-specifications"],"requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"text":"## Durable restocks\n\nPending restocks survive a backend restart.\n\n"},{"id":"ecommerce.l3.durable-order-delivery","owners":["ecommerce.l3.deferred-durability-specifications"],"requiresFeatures":["ecommerce.l3.order-delivery-features"],"text":"## Durable order delivery\n\nPending order delivery survives a backend restart.\n\n"},{"id":"ecommerce.l3.durable-cart-expiration","owners":["ecommerce.l3.deferred-durability-specifications"],"requiresFeatures":["ecommerce.l3.cart-expiration-features"],"text":"## Durable cart expiration\n\nPending cart expiration survives a backend restart.\n\n"},{"id":"ecommerce.l3.exactly-once-restocks","owners":["ecommerce.l3.deferred-integrity-specifications"],"requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"text":"## Exactly-once restocks\n\nRestarting the backend cannot apply a restock more than once.\n\n"},{"id":"ecommerce.l3.exactly-once-delivery","owners":["ecommerce.l3.deferred-integrity-specifications"],"requiresFeatures":["ecommerce.l3.order-delivery-features"],"text":"## Exactly-once delivery\n\nRestarting the backend cannot apply an order transition more than once.\n\n"},{"id":"ecommerce.l3.server-timed-restocks","owners":["ecommerce.l3.server-time-specifications"],"requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"text":"## Server-timed restocks\n\nA pending restock does not run early after a restart.\n\n"},{"id":"ecommerce.l3.server-timed-reservations","owners":["ecommerce.l3.server-time-specifications"],"requiresFeatures":["ecommerce.l3.reservations-features"],"text":"## Server-timed reservations\n\nA reservation expires without an open browser.\n\n"},{"id":"ecommerce.l3.deferred-access","owners":["ecommerce.l3.deferred-access-specifications"],"requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"text":"## Deferred-work access\n\nOnly an admin can schedule or cancel a restock.\n\n"},{"id":"ecommerce.l3.stock-conservation","owners":["ecommerce.l3.deferred-integrity-specifications"],"requiresFeatures":["ecommerce.l3.reservations-features"],"text":"## Stock conservation\n\nReservation expiry returns exactly the stock that the reservation took. Checkout does not take reserved stock twice.\n"},{"id":"ecommerce.progression.promotion-checkout","owners":["ecommerce.progression.promotion-checkout"],"text":"## Promotion checkout\n\nCustomers can apply an active promotion code to a cart. The final order records the applied\ndiscount. Expired and fully redeemed promotions are refused.\n"},{"id":"ecommerce.progression.stock-alerts","owners":["ecommerce.progression.stock-alerts"],"text":"## Stock alerts\n\nA signed-in customer can request an alert for an unavailable item. Show an alert when stock\nreturns.\n"},{"id":"ecommerce.progression.faceted-search","owners":["ecommerce.progression.faceted-search"],"text":"# Faceted search\n\nLet visitors filter the catalog by category, minimum price, maximum price, and availability.\nCombine the selected filters. While search text or any filter is active, order matching\nitems by name, with ten results per page. With no search text or active filter, use\nthe storefront's purchase ranking, breaking ties by item name. Clearing all search\ntext and filters returns to that ranking.\n"},{"id":"ecommerce.progression.personalized-recommendations","owners":["ecommerce.progression.personalized-recommendations"],"text":"# Personalized recommendations\n\nRecommend items from categories in the signed-in customer's purchase history. Exclude\nitems they already purchased. Order by global units sold, highest first, then by item name.\n"},{"id":"ecommerce.spec.search-ordering","owners":["ecommerce.spec.search-ordering"],"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.faceted-search"],"text":"# Search ordering after purchases\n\nPurchases must not change the alphabetical order of active search or filter results.\nClearing all search text and filters restores the current purchase ranking.\n"},{"id":"ecommerce.l3.order-returns","owners":["ecommerce.l3.order-returns-features"],"text":"## Order returns\n\nA customer can return an item after the order ships for a refund of its purchase price.\nRestock the item and mark it returned in order history.\n"},{"id":"ecommerce.progression.order-support","owners":["ecommerce.progression.order-support"],"text":"## Order-linked support\n\nA customer can attach an order to a support case. Staff can inspect the linked order\nfrom the case.\n"},{"id":"ecommerce.progression.promotion-reporting","owners":["ecommerce.progression.promotion-reporting"],"text":"## Promotion reporting\n\nStaff can see each promotion's redemption count and revenue after discounts.\n"},{"id":"ecommerce.progression.delivery-notifications","owners":["ecommerce.progression.delivery-notifications"],"text":"## Delivery notifications\n\nNotify customers when their orders are delivered, using their notification preferences.\n"},{"id":"ecommerce.progression.automatic-reorder","owners":["ecommerce.progression.automatic-reorder"],"text":"# Automatic reorder rules\n\nWarehouse staff can manage automatic reorder rules. A rule names an item, a stock\nthreshold, and a restock quantity. Schedule a restock when stock falls from above the\nthreshold to the threshold or below.\nThe automatic restock is due 60 seconds after it is scheduled.\n"},{"id":"ecommerce.progression.cart-recovery","owners":["ecommerce.progression.cart-recovery"],"text":"# Cart recovery\n\nLet a signed-in customer restore an expired cart. Reserve each item again only when its full\nquantity is available. Restore the available items and list each item that could not be restored.\n"},{"id":"ecommerce.progression.recommendation-feedback","owners":["ecommerce.progression.recommendation-feedback"],"text":"## Recommendation feedback\n\nA customer can dismiss a recommendation from their list.\n"},{"id":"ecommerce.feature.store-credit","owners":["ecommerce.feature.store-credit"],"text":"## Store credit\n\nStaff can issue customer credit with a reference. Customers can see their balance and history and choose credit at cart checkout. Use credit first, up to the order total; record any remainder as the existing payment. Amounts use the shop currency and whole minor units. A reference identifies one grant.\n"},{"id":"ecommerce.feature.subscriptions","owners":["ecommerce.feature.subscriptions"],"text":"## Scheduled purchases\n\nCustomers can subscribe to an individual catalog item (not a bundle) and quantity for a chosen number of deliveries at a chosen interval. Use the item price at subscription creation. The first delivery is due after one interval. Each delivery creates an ordinary order and payment. Skip an unavailable delivery without charging; it still uses one delivery slot. Customers can pause, resume, or cancel future deliveries; a pause moves future due times by the pause duration.\n"},{"id":"ecommerce.progression.support-refunds","owners":["ecommerce.progression.support-refunds"],"text":"## Support refunds\n\nStaff can refund an entire order from its support case. A successful refund resolves\nthe case. Show the refund amount and case status.\n\nA refund does not prevent a later physical return of shipped goods. Refund only\nthe amount not already refunded, and restock goods when they are returned.\n"},{"id":"ecommerce.spec.access-control.automatic-reorder-access","owners":["ecommerce.spec.access-control"],"requiresFeatures":["ecommerce.progression.automatic-reorder"],"text":"## automatic-reorder-access\n\nOnly warehouse staff can manage automatic reorder rules.\n\n"},{"id":"ecommerce.spec.access-control.recommendation-profile-isolation","owners":["ecommerce.spec.access-control"],"requiresFeatures":["ecommerce.progression.personalized-recommendations"],"text":"## recommendation-profile-isolation\n\nCustomer recommendation profiles are isolated.\n\n"},{"id":"ecommerce.spec.access-control.staff-activity-privacy","owners":["ecommerce.spec.access-control"],"requiresFeatures":["ecommerce.progression.staff-activity"],"text":"## staff-activity-privacy\n\nCustomers cannot open staff activity history.\n\n"},{"id":"ecommerce.spec.access-control.order-support-ownership","owners":["ecommerce.spec.access-control"],"requiresFeatures":["ecommerce.progression.order-support"],"text":"## order-support-ownership\n\nA customer cannot attach or inspect another customer's order.\n\n"},{"id":"ecommerce.spec.access-control.delivery-notification-privacy","owners":["ecommerce.spec.access-control"],"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.delivery-notifications","ecommerce.progression.fulfilment-queue"],"text":"## delivery-notification-privacy\n\nDelivery notifications are private to the order owner.\n\n"},{"id":"ecommerce.spec.access-control.support-refund-access","owners":["ecommerce.spec.access-control"],"requiresFeatures":["ecommerce.progression.support-refunds"],"text":"## support-refund-access\n\nOnly authorized staff can refund an order from a support case.\n\n"},{"id":"ecommerce.spec.access-control.recommendation-feedback-privacy","owners":["ecommerce.spec.access-control"],"requiresFeatures":["ecommerce.progression.recommendation-feedback"],"text":"## recommendation-feedback-privacy\n\nRecommendation feedback belongs to one customer.\n\n"},{"id":"ecommerce.spec.state-durability.recommendation-feedback-restart","owners":["ecommerce.spec.state-durability"],"requiresFeatures":["ecommerce.progression.recommendation-feedback"],"text":"## recommendation-feedback-restart\n\nDismissed recommendations stay dismissed after reconnecting or restarting the server.\n\n"},{"id":"ecommerce.spec.transactional-integrity.automatic-reorder-deduplication","owners":["ecommerce.spec.transactional-integrity"],"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.automatic-reorder"],"text":"## automatic-reorder-deduplication\n\nA pending automatic restock is not scheduled twice.\n\n"},{"id":"ecommerce.spec.transactional-integrity.payment-deduplication","owners":["ecommerce.spec.transactional-integrity"],"requiresFeatures":["ecommerce.progression.payment-records"],"text":"## payment-deduplication\n\nA checkout does not create duplicate payments.\n\n"},{"id":"ecommerce.spec.transactional-integrity.support-refund-accounting","owners":["ecommerce.spec.transactional-integrity"],"requiresFeatures":["ecommerce.progression.support-refunds"],"text":"## support-refund-accounting\n\nThe recorded refund equals the amount paid and is applied only once.\n"},{"id":"ecommerce.feature.split-tender-refunds","owners":["ecommerce.feature.split-tender-refunds"],"text":"## Refunds with store credit\n\nExtend support refunds and item returns to orders paid with credit and another payment. Return each part to its original source. A full refund restores the original credit portion to the customer balance and refunds the original external portion.\n"},{"id":"ecommerce.spec.bundle-integrity.product-bundles","owners":["ecommerce.spec.bundle-integrity"],"requiresFeatures":["ecommerce.feature.product-bundles"],"text":"## Product bundles\n\nOnly authorized staff can change bundles.\n\n"},{"id":"ecommerce.spec.bundle-integrity.bundle-checkout","owners":["ecommerce.spec.bundle-integrity"],"requiresFeatures":["ecommerce.feature.bundle-checkout"],"text":"## Bundle checkout\n\nReserve all components or none. Competing purchases share the same stock. Release component reservations when the cart expires or the bundle is removed.\n\n"},{"id":"ecommerce.spec.bundle-integrity.bundle-returns","owners":["ecommerce.spec.bundle-integrity"],"requiresFeatures":["ecommerce.feature.bundle-returns"],"text":"## Bundle returns\n\nReturn the original component allocations and price paid. Repeating a return must not change stock or refunds again. Customers cannot return another account's order.\n\n"},{"id":"ecommerce.spec.store-credit.store-credit","owners":["ecommerce.spec.store-credit"],"requiresFeatures":["ecommerce.feature.store-credit"],"text":"## Store credit\n\nOnly authorized staff can grant credit. Repeating a reference must not issue credit twice. Concurrent checkout must not duplicate the order or credit use. Accepted credit survives a backend restart.\n\n"},{"id":"ecommerce.spec.split-tender-refunds.split-tender-refunds","owners":["ecommerce.spec.split-tender-refunds"],"requiresFeatures":["ecommerce.feature.split-tender-refunds"],"text":"## Split-tender refunds\n\nConcurrent or repeated refunds must restore each original payment portion only once. The resulting balance and refund records survive a backend restart.\n\n"},{"id":"ecommerce.spec.subscriptions.subscriptions","owners":["ecommerce.spec.subscriptions"],"requiresFeatures":["ecommerce.feature.subscriptions"],"text":"## Scheduled purchases\n\nPending deliveries and pauses survive a backend restart. Process elapsed pending slots after recovery. Completed delivery slots must not run again. Customers cannot change another account's subscription.\n"},{"id":"ecommerce.feature.product-bundles","owners":["ecommerce.feature.product-bundles"],"text":"## Product bundles\n\nCatalog staff can create and edit a named bundle of existing products, with a quantity\nfor each component and one bundle price. Show bundles and their components in the catalog.\nBundles contain products only, not other bundles.\n"},{"id":"ecommerce.feature.bundle-checkout","owners":["ecommerce.feature.bundle-checkout"],"text":"## Bundle checkout\n\nCustomers can add a whole bundle to their cart and buy it through checkout. Its components\nuse the existing stock reservations and reservation lifetime. Removing a bundle releases\nits reservation. Show the bundle as one order line at its bundle price.\n"},{"id":"ecommerce.feature.bundle-returns","owners":["ecommerce.feature.bundle-returns"],"text":"## Bundle returns\n\nCustomers can return a shipped bundle as a whole. Restore the purchased component quantities\nto their original warehouses and refund the price paid. Show the return and refund on the\norder. Partial bundle returns are not supported.\n"}]},"track":"ecommerce"},"execution":{"capabilities":["backend-lifecycle","browser","concurrent-actors","database-observation","database-read","direct-database-write","direct-server-call","process-crash","request-replay"],"execution":[{"checkGroups":[{"checkGroupId":"accounts","feature":{"actors":["shopper"],"criteria":[{"id":"1a","steps":[{"actor":"shopper","do":"signUp","name":"ann"},{"actor":"shopper","contains":"ann","do":"expect","testid":"current-user","within":6000}]}],"id":1,"setup":[]},"packId":"ecommerce.feature.accounts","role":"feature","source":"scenarios/01-account-create.json"}],"id":"selected-source-001","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-account-create.json"},{"checkGroups":[{"checkGroupId":"accounts","feature":{"actors":["owner","impostor"],"criteria":[{"id":"1b","steps":[{"actor":"impostor","do":"signUp","expectFailure":true,"name":"ann","password":"different-pw"},{"actor":"impostor","do":"expect","testid":"auth-error","within":6000},{"absent":true,"actor":"impostor","do":"expect","testid":"current-user"}]}],"id":1,"setup":[{"actor":"owner","do":"signUp","name":"ann"}]},"packId":"ecommerce.feature.accounts","role":"feature","source":"scenarios/01-account-duplicate.json"}],"id":"selected-source-002","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-account-duplicate.json"},{"checkGroups":[{"checkGroupId":"accounts","feature":{"actors":["owner","impostor"],"criteria":[{"id":"1c","steps":[{"actor":"impostor","do":"signIn","expectFailure":true,"name":"ann","password":"wrong-pw"},{"actor":"impostor","do":"expect","testid":"auth-error","within":6000},{"absent":true,"actor":"impostor","do":"expect","testid":"current-user"}]}],"id":1,"setup":[{"actor":"owner","do":"signUp","name":"ann"}]},"packId":"ecommerce.feature.accounts","role":"feature","source":"scenarios/01-account-password.json"}],"id":"selected-source-003","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-account-password.json"},{"checkGroups":[{"checkGroupId":"session-reload","feature":{"actors":["shopper"],"criteria":[{"id":"1e","steps":[{"actor":"shopper","do":"reload","settleMs":4000},{"actor":"shopper","contains":"ann","do":"expect","testid":"current-user","within":6000}]}],"id":1,"setup":[{"actor":"shopper","do":"signUp","name":"ann"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","requiresFeatures":["ecommerce.feature.accounts"],"role":"guarantee","source":"scenarios/01-account-reload.json"}],"id":"selected-source-004","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-account-reload.json"},{"checkGroups":[{"checkGroupId":"accounts","feature":{"actors":["shopper"],"criteria":[{"id":"1d","steps":[{"actor":"shopper","do":"click","testid":"current-user","unlessVisible":"signout"},{"actor":"shopper","do":"click","testid":"signout"},{"actor":"shopper","do":"waitUntilAbsent","testid":"current-user","within":6000},{"actor":"shopper","do":"signIn","name":"ann"},{"actor":"shopper","contains":"ann","do":"expect","testid":"current-user","within":6000}]}],"id":1,"setup":[{"actor":"shopper","do":"signUp","name":"ann"}]},"packId":"ecommerce.feature.accounts","role":"feature","source":"scenarios/01-account-signout.json"}],"id":"selected-source-005","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-account-signout.json"},{"checkGroups":[{"checkGroupId":"admin-write","feature":{"actors":["admin","staff"],"criteria":[{"id":"103a","steps":[{"actor":"staff","as":"purifier-before-control","do":"recordNumber","in":{"contains":"Air Purifier","testid":"item-card"},"testid":"item-stock"},{"action":"restock","actor":"admin","do":"callAction","input":{"attribute":"data-restock-input","contains":"Air Purifier","testid":"admin-location-row"},"namedAction":{"args":[0,0,1],"id":"restock","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"warehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/restock","reducer":"admin_restock"},"settleMs":2000},{"actor":"admin","do":"expectActionOutcome","outcome":"accepted"},{"actor":"staff","do":"reload","settleMs":2000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"expectNumber","in":{"contains":"Air Purifier","testid":"item-card"},"plus":1,"relativeTo":"purifier-before-control","testid":"item-stock"}]}],"id":103,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"admin","do":"click","testid":"admin-link"}]},"packId":"ecommerce.feature.warehouse-admin","role":"feature","source":"scenarios/01-admin-write-staff.json"},{"checkGroupId":"warehouse-write-boundary","feature":{"actors":["admin","staff"],"criteria":[{"id":"103b","steps":[{"action":"restock","actor":"admin","do":"callAction","input":{"attribute":"data-restock-input","contains":"Air Purifier","testid":"admin-location-row"},"namedAction":{"args":[0,0,1],"id":"restock","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"warehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/restock","reducer":"admin_restock"},"settleMs":2000},{"actor":"admin","do":"expectActionOutcome","outcome":"accepted"},{"actor":"staff","do":"reload","settleMs":2000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","as":"purifier-before-refusal","do":"recordNumber","in":{"contains":"Air Purifier","testid":"item-card"},"testid":"item-stock"},{"action":"restock","actor":"staff","do":"callAction","from":"admin","input":{"attribute":"data-restock-input","contains":"Air Purifier","testid":"admin-location-row"},"namedAction":{"args":[0,0,1],"id":"restock","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"warehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/restock","reducer":"admin_restock"},"settleMs":2000},{"actor":"staff","do":"expectActionOutcome","outcome":"refused","routeProvenBy":"admin"},{"actor":"staff","do":"reload","settleMs":2000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"expectNumber","in":{"contains":"Air Purifier","testid":"item-card"},"plus":0,"relativeTo":"purifier-before-refusal","testid":"item-stock"}]}],"id":103,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"admin","do":"click","testid":"admin-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-admin-write-staff.json"}],"id":"selected-source-006","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-admin-write-staff.json"},{"checkGroups":[{"checkGroupId":"purchase-stock","feature":{"actors":["buyer","watcher","visitor"],"criteria":[{"id":"3b","steps":[{"actor":"watcher","do":"expectNumber","equals":100,"in":{"contains":"Espresso Machine","testid":"item-card"},"testid":"item-stock"},{"actor":"buyer","do":"click","in":{"contains":"Espresso Machine","testid":"item-card"},"testid":"buy-now"},{"actor":"watcher","do":"expectNumber","equals":99,"in":{"contains":"Espresso Machine","testid":"item-card"},"testid":"item-stock"},{"actor":"visitor","do":"expectNumber","equals":99,"in":{"contains":"Espresso Machine","testid":"item-card"},"testid":"item-stock"}]}],"id":3,"setup":[{"actor":"buyer","do":"signUp","name":"eli"},{"actor":"watcher","do":"signUp","name":"fay"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/01-buying.json"}],"id":"selected-source-007","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-buying.json"},{"checkGroups":[{"checkGroupId":"cart-boundary","feature":{"actors":["owner","stranger"],"criteria":[{"id":"109a","steps":[{"action":"cart-add","actor":"stranger","do":"callAction","input":{"attribute":"data-buy-input","contains":"Coffee Grinder","testid":"item-card"},"namedAction":{"args":[0],"id":"cart-add","params":[{"in":"body","name":"itemId","wireType":"u64"}],"path":"/api/cart","reducer":"add_to_cart"},"settleMs":2000},{"actor":"stranger","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"reload","settleMs":1500},{"actor":"owner","do":"ensureSignedIn","name":"vic","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"owner","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"},{"actor":"stranger","do":"reload","settleMs":1500},{"actor":"stranger","do":"ensureSignedIn","name":"wes","readyTestid":"current-user"},{"actor":"stranger","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"stranger","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"owner","contains":"Coffee Grinder","count":1,"do":"expect","testid":"cart-item"},{"actor":"owner","do":"expectNumber","equals":1,"in":{"contains":"Coffee Grinder","testid":"cart-item"},"testid":"cart-quantity"},{"actor":"stranger","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"},{"actor":"stranger","contains":"Coffee Grinder","count":1,"do":"expect","testid":"cart-item"},{"actor":"stranger","do":"expectNumber","equals":1,"in":{"contains":"Coffee Grinder","testid":"cart-item"},"testid":"cart-quantity"}]},{"id":"109b","steps":[{"actor":"owner","as":"cart-total-before-invalid","do":"recordNumber","testid":"cart-total"},{"actor":"owner","as":"cart-quantity-before-invalid","do":"recordNumber","in":{"contains":"Coffee Grinder","testid":"cart-item"},"testid":"cart-quantity"},{"action":"cart-set-quantity","actor":"owner","do":"callAction","input":{"attribute":"data-cart-input","contains":"Coffee Grinder","testid":"cart-item"},"namedAction":{"args":[0,-3],"id":"cart-set-quantity","method":"PATCH","params":[{"in":"path","name":"itemId","placeholder":":itemId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/cart/:itemId","reducer":"update_cart_quantity"},"settleMs":2000},{"actor":"owner","do":"expectActionOutcome","outcome":"validation-refused"},{"actor":"owner","do":"reload","settleMs":1500},{"actor":"owner","do":"ensureSignedIn","name":"vic","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"owner","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"},{"actor":"owner","do":"expectNumber","plus":0,"relativeTo":"cart-total-before-invalid","testid":"cart-total"},{"actor":"owner","do":"expectNumber","in":{"contains":"Coffee Grinder","testid":"cart-item"},"plus":0,"relativeTo":"cart-quantity-before-invalid","testid":"cart-quantity"}]}],"id":109,"setup":[{"actor":"owner","do":"signUp","name":"vic"},{"actor":"stranger","do":"signUp","name":"wes"},{"actor":"owner","do":"click","in":{"contains":"Coffee Grinder","testid":"item-card"},"testid":"add-to-cart"},{"actor":"owner","do":"wait","ms":1500},{"actor":"owner","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.feature.cart"],"role":"guarantee","source":"scenarios/01-cart-boundary.json"}],"id":"selected-source-008","scenario":{"level":1,"writeUrlPattern":"\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses"},"source":"scenarios/01-cart-boundary.json"},{"checkGroups":[{"checkGroupId":"cart-reload","feature":{"actors":["quantity","reload","live1","live2","checkout"],"criteria":[{"id":"4b","steps":[{"actor":"reload","do":"click","in":{"contains":"Laptop Stand","testid":"item-card"},"testid":"add-to-cart"},{"actor":"reload","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"},{"actor":"reload","contains":"Laptop Stand","do":"expect","testid":"cart-item","within":10000},{"actor":"reload","do":"reload","settleMs":3000},{"actor":"reload","do":"ensureSignedIn","name":"omar","readyTestid":"current-user"},{"actor":"reload","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"},{"actor":"reload","contains":"Laptop Stand","do":"expect","testid":"cart-item"}]}],"id":4,"setup":[{"actor":"quantity","do":"signUp","name":"nora"},{"actor":"reload","do":"signUp","name":"omar"},{"actor":"live1","do":"signUp","name":"pia"},{"actor":"checkout","do":"signUp","name":"quinn"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","requiresFeatures":["ecommerce.feature.cart"],"role":"guarantee","source":"scenarios/01-cart.json"},{"checkGroupId":"shared-cart","feature":{"actors":["quantity","reload","live1","live2","checkout"],"criteria":[{"id":"4c","steps":[{"actor":"live2","do":"signIn","name":"pia"},{"actor":"live2","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"},{"actor":"live1","do":"click","in":{"contains":"Induction Cooktop","testid":"item-card"},"testid":"add-to-cart"},{"actor":"live2","contains":"Induction Cooktop","do":"expect","testid":"cart-item","within":10000}]}],"id":4,"setup":[{"actor":"quantity","do":"signUp","name":"nora"},{"actor":"reload","do":"signUp","name":"omar"},{"actor":"live1","do":"signUp","name":"pia"},{"actor":"checkout","do":"signUp","name":"quinn"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","requiresFeatures":["ecommerce.feature.cart"],"role":"guarantee","source":"scenarios/01-cart.json"}],"id":"selected-source-009","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-cart.json"},{"checkGroups":[{"checkGroupId":"catalog-ranking","feature":{"actors":["visitor"],"criteria":[{"id":"2b","steps":[{"actor":"visitor","do":"expectSequence","equals":["Air Purifier","Bluetooth Speaker","Coffee Grinder","Desk Lamp","Espresso Machine","Gaming Mouse","Headphones","Induction Cooktop","Keyboard","Laptop Stand"],"in":{"testid":"item-list"},"testid":"item-name"}]}],"id":2,"setup":[]},"packId":"ecommerce.feature.catalog-discovery","role":"feature","source":"scenarios/01-catalog-ranking.json","stablePackId":"ecommerce.feature.catalog"}],"id":"selected-source-010","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-catalog-ranking.json"},{"checkGroups":[{"checkGroupId":"catalog-search","feature":{"actors":["visitor"],"criteria":[{"id":"2d","steps":[{"actor":"visitor","do":"fill","enter":true,"settleMs":1500,"testid":"search-input","text":"mirrorLESS"},{"actor":"visitor","contains":"Mirrorless Camera","do":"expect","in":{"testid":"search-results"},"testid":"item-card"}]}],"id":2,"setup":[]},"packId":"ecommerce.feature.catalog-discovery","role":"feature","source":"scenarios/01-catalog-search.json","stablePackId":"ecommerce.feature.catalog"}],"id":"selected-source-011","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-catalog-search.json"},{"checkGroups":[{"checkGroupId":"catalog-values","feature":{"actors":["visitor"],"criteria":[{"id":"2a","steps":[{"actor":"visitor","contains":"Air Purifier","do":"expect","testid":"item-card"},{"actor":"visitor","do":"expectNumber","equals":189,"in":{"contains":"Air Purifier","testid":"item-card"},"testid":"item-price"},{"actor":"visitor","do":"expectNumber","equals":100,"in":{"contains":"Air Purifier","testid":"item-card"},"testid":"item-stock"}]}],"id":2,"setup":[]},"packId":"ecommerce.feature.catalog-items","role":"feature","source":"scenarios/01-catalog-values.json","stablePackId":"ecommerce.feature.catalog"}],"id":"selected-source-012","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-catalog-values.json"},{"checkGroups":[{"checkGroupId":"ranking","feature":{"actors":["buyer","visitor","inspector"],"criteria":[{"id":"2c","steps":[{"actor":"buyer","do":"click","in":{"contains":"Coffee Grinder","testid":"item-card"},"testid":"buy-now"},{"actor":"visitor","do":"expectSequence","equals":["Coffee Grinder","Air Purifier","Bluetooth Speaker","Desk Lamp","Espresso Machine","Gaming Mouse","Headphones","Induction Cooktop","Keyboard","Laptop Stand"],"in":{"testid":"item-list"},"testid":"item-name","within":10000},{"actors":["buyer","visitor"],"do":"expectAgreement","in":{"contains":"Coffee Grinder","testid":"item-card"},"numeric":true,"testid":"item-stock"}]}],"id":2,"setup":[{"actor":"buyer","do":"signUp","name":"dov"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/01-core.json"}],"id":"selected-source-013","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-core.json"},{"checkGroups":[{"checkGroupId":"duplicate-checkout","feature":{"actors":["tab1","tab2","filler"],"criteria":[{"id":"203a","steps":[{"actor":"tab1","contains":"Gaming Mouse","count":1,"do":"expect","testid":"cart-item"},{"actor":"tab1","do":"expectNumber","equals":2,"in":{"contains":"Gaming Mouse","testid":"cart-item"},"testid":"cart-quantity"}]},{"id":"203b","steps":[{"actor":"tab1","do":"click","testid":"checkout-submit"},{"actor":"tab1","do":"wait","ms":2500},{"actor":"filler","as":"keyboard-before-checkout","do":"recordNumber","in":{"contains":"Keyboard","testid":"item-card"},"testid":"item-stock"},{"account":"{user:twin}","as":"checkout-before","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"filler","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"actor":"filler","do":"wait","ms":2000},{"account":"{user:twin}","as":"checkout-prepared","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"action":"checkout","actors":["tab1","tab2"],"do":"callConcurrently","settleMs":5000},{"do":"expectCallOutcomes"},{"before":"checkout-before","do":"dbExpectCheckout","prepared":"checkout-prepared","quantity":1},{"actor":"filler","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"filler","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"filler","contains":"Keyboard","count":1,"do":"expect","testid":"order-item"},{"actor":"filler","do":"reload","settleMs":2000},{"actor":"filler","do":"ensureSignedIn","name":"twin","readyTestid":"current-user"},{"actor":"filler","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"filler","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"filler","do":"expectNumber","in":{"contains":"Keyboard","testid":"item-card"},"plus":-1,"relativeTo":"keyboard-before-checkout","testid":"item-stock"}]}],"id":203,"setup":[{"actor":"tab1","do":"signUp","name":"twin"},{"actor":"tab2","do":"signIn","name":"twin"},{"actor":"filler","do":"signIn","name":"twin"},{"actors":["tab1","tab2"],"do":"clickConcurrently","in":{"contains":"Gaming Mouse","testid":"item-card"},"settleMs":4000,"testid":"add-to-cart"},{"actor":"tab1","do":"reload","settleMs":2500},{"actor":"tab1","do":"ensureSignedIn","name":"twin","readyTestid":"current-user"},{"actor":"tab1","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","requiresFeatures":["ecommerce.feature.checkout"],"role":"guarantee","source":"scenarios/01-duplicate-checkout.json"}],"id":"selected-source-014","scenario":{"level":1,"writeUrlPattern":"\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses"},"source":"scenarios/01-duplicate-checkout.json"},{"checkGroups":[{"checkGroupId":"external-stock","feature":{"actors":["viewer"],"criteria":[{"id":"901a","steps":[{"do":"dbSetStock","item":"Desk Lamp","quantity":5,"settleMs":4000,"warehouse":"East"},{"actor":"viewer","do":"expectNumber","equals":50,"in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock","within":10000}]}],"id":901,"setup":[{"actor":"viewer","do":"expectNumber","equals":100,"in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.external-data-sync","requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-external-live-sync.json"}],"id":"selected-source-015","scenario":{"level":1,"writeUrlPattern":"\\/api\\/|\\/items|\\/stock|\\/warehouses"},"source":"scenarios/01-external-live-sync.json"},{"checkGroups":[{"checkGroupId":"external-stock","feature":{"actors":["viewer"],"criteria":[{"id":"901d","steps":[{"actor":"viewer","do":"setOffline","offline":true,"settleMs":1000},{"do":"dbSetStock","item":"Desk Lamp","quantity":7,"settleMs":4000,"warehouse":"East"},{"actor":"viewer","do":"setOffline","offline":false,"settleMs":1000},{"actor":"viewer","do":"expectNumber","equals":52,"in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock","within":20000}]}],"id":901,"setup":[{"actor":"viewer","do":"expectNumber","equals":100,"in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.external-data-sync","requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-external-reconnect-sync.json"}],"id":"selected-source-016","scenario":{"level":1,"writeUrlPattern":"\\/api\\/|\\/items|\\/stock|\\/warehouses"},"source":"scenarios/01-external-reconnect-sync.json"},{"checkGroups":[{"checkGroupId":"external-stock","feature":{"actors":["viewer"],"criteria":[{"id":"901b","steps":[{"do":"dbSetStock","item":"Desk Lamp","quantity":5,"settleMs":1000,"warehouse":"East"},{"actor":"viewer","do":"reload","settleMs":3000},{"actor":"viewer","do":"expectNumber","equals":50,"in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock","within":10000}]}],"id":901,"setup":[{"actor":"viewer","do":"expectNumber","equals":100,"in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.external-data-sync","requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-external-reload-sync.json"}],"id":"selected-source-017","scenario":{"level":1,"writeUrlPattern":"\\/api\\/|\\/items|\\/stock|\\/warehouses"},"source":"scenarios/01-external-reload-sync.json"},{"checkGroups":[{"checkGroupId":"external-stock","feature":{"actors":["viewer"],"criteria":[{"id":"901c","steps":[{"do":"stopAppServer"},{"do":"dbSetStock","item":"Desk Lamp","quantity":10,"settleMs":4000,"warehouse":"West"},{"do":"startAppServer"},{"actor":"viewer","do":"expectNumber","equals":65,"in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock","within":20000}]}],"id":901,"setup":[{"actor":"viewer","do":"expectNumber","equals":100,"in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.external-data-sync","requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-external-server-restart-sync.json"}],"id":"selected-source-018","scenario":{"level":1,"writeUrlPattern":"\\/api\\/|\\/items|\\/stock|\\/warehouses"},"source":"scenarios/01-external-server-restart-sync.json"},{"checkGroups":[{"checkGroupId":"last-unit","feature":{"actors":["admin","a","b","c","d","e","f"],"criteria":[{"id":"201a","steps":[{"do":"dbExpectStock","equals":0,"item":"Air Purifier","warehouse":"East"},{"do":"dbExpectStock","equals":0,"item":"Air Purifier","warehouse":"West"},{"actor":"a","do":"expectNumber","equals":0,"in":{"contains":"Air Purifier","testid":"item-card"},"testid":"item-stock"},{"actors":["a","b","c","d","e","f"],"do":"expectAgreement","in":{"contains":"Air Purifier","testid":"item-card"},"numeric":true,"testid":"item-stock"}]},{"id":"201c","steps":[{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"expectNumber","plus":567,"relativeTo":"revenue-before-last-unit","testid":"admin-revenue","within":10000}]},{"id":"201b","steps":[{"before":{"a":"buy-a","b":"buy-b","c":"buy-c","d":"buy-d","e":"buy-e","f":"buy-f"},"do":"dbExpectPurchases","purchases":3},{"actor":"a","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"a","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"b","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"b","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"c","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"c","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"d","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"d","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"e","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"e","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"f","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"f","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actors":["a","b","c","d","e","f"],"contains":"Air Purifier","do":"expectActorsWith","equals":3,"maxEach":1,"testid":"order-item"},{"do":"dbSetStock","item":"Air Purifier","quantity":2,"settleMs":250,"warehouse":"East"},{"do":"dbSetStock","item":"Air Purifier","quantity":2,"settleMs":250,"warehouse":"West"},{"actor":"a","do":"reload","settleMs":1000},{"actor":"a","do":"ensureSignedIn","name":"c1","readyTestid":"current-user"},{"actor":"a","do":"click","ifAvailable":true,"testid":"catalog-link"},{"account":"{user:c1}","as":"ample-a","do":"dbRecordCheckout","item":"Air Purifier","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"account":"{user:c2}","as":"ample-b","do":"dbRecordCheckout","item":"Air Purifier","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"action":"buy","actors":["a","b"],"do":"callConcurrently","from":"a","input":{"attribute":"data-buy-input","contains":"Air Purifier","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"},"requests":4,"settleMs":3000},{"do":"expectCallOutcomes"},{"before":{"a":"ample-a","b":"ample-b"},"do":"dbExpectPurchases","purchases":4}]}],"id":201,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","as":"revenue-before-last-unit","do":"recordNumber","testid":"admin-revenue"},{"do":"dbSetStock","item":"Air Purifier","quantity":2,"settleMs":250,"warehouse":"East"},{"do":"dbSetStock","item":"Air Purifier","quantity":1,"settleMs":250,"warehouse":"West"},{"actor":"a","do":"signUp","name":"c1"},{"actor":"b","do":"signUp","name":"c2"},{"actor":"c","do":"signUp","name":"c3"},{"actor":"d","do":"signUp","name":"c4"},{"actor":"e","do":"signUp","name":"c5"},{"actor":"f","do":"signUp","name":"c6"},{"account":"{user:c1}","as":"buy-a","do":"dbRecordCheckout","item":"Air Purifier","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"account":"{user:c2}","as":"buy-b","do":"dbRecordCheckout","item":"Air Purifier","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"account":"{user:c3}","as":"buy-c","do":"dbRecordCheckout","item":"Air Purifier","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"account":"{user:c4}","as":"buy-d","do":"dbRecordCheckout","item":"Air Purifier","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"account":"{user:c5}","as":"buy-e","do":"dbRecordCheckout","item":"Air Purifier","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"account":"{user:c6}","as":"buy-f","do":"dbRecordCheckout","item":"Air Purifier","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"action":"buy","actors":["a","b","c","d","e","f"],"do":"callConcurrently","input":{"attribute":"data-buy-input","contains":"Air Purifier","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"},"settleMs":6000},{"do":"expectCallOutcomes"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-last-unit.json"}],"id":"selected-source-019","scenario":{"level":1,"writeUrlPattern":"\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses"},"source":"scenarios/01-last-unit.json"},{"checkGroups":[{"checkGroupId":"order-ownership","feature":{"actors":["one","two"],"criteria":[{"id":"106a","steps":[{"actor":"one","do":"click","in":{"contains":"Coffee Grinder","testid":"item-card"},"testid":"buy-now"},{"actor":"two","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"buy-now"},{"actor":"two","do":"wait","ms":2000},{"actor":"two","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"two","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"two","contains":"Desk Lamp","count":1,"do":"expect","testid":"order-item"},{"absent":true,"actor":"two","contains":"Coffee Grinder","do":"expect","testid":"order-item"}]}],"id":106,"setup":[{"actor":"one","do":"signUp","name":"quin"},{"actor":"two","do":"signUp","name":"ros"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/01-order-ownership.json"}],"id":"selected-source-020","scenario":{"level":1,"writeUrlPattern":"\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses"},"source":"scenarios/01-order-ownership.json"},{"checkGroups":[{"checkGroupId":"purchase-attribution","feature":{"actors":["victim","attacker"],"criteria":[{"id":"102a","steps":[{"action":"buy","actor":"attacker","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"},"settleMs":2000},{"actor":"attacker","do":"expectActionOutcome","outcome":"accepted"},{"actor":"victim","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"victim","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"victim","contains":"Coffee Grinder","count":1,"do":"expect","testid":"order-item"},{"absent":true,"actor":"victim","contains":"Desk Lamp","do":"expect","testid":"order-item"},{"actor":"attacker","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"attacker","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"attacker","contains":"Desk Lamp","count":1,"do":"expect","testid":"order-item"}]}],"id":102,"setup":[{"actor":"victim","do":"signUp","name":"lee"},{"actor":"attacker","do":"signUp","name":"mel"},{"actor":"victim","do":"click","in":{"contains":"Coffee Grinder","testid":"item-card"},"testid":"buy-now"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/01-purchase-attribution.json"}],"id":"selected-source-021","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-purchase-attribution.json"},{"checkGroups":[{"checkGroupId":"purchase-session","feature":{"actors":["buyer","guest"],"criteria":[{"id":"101a","steps":[{"action":"buy","actor":"guest","authentication":"none","do":"callAction","from":"buyer","input":{"attribute":"data-buy-input","contains":"Bluetooth Speaker","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"},"settleMs":2000},{"actor":"guest","do":"expectActionOutcome","outcome":"refused","routeProvenBy":"buyer"},{"do":"dbExpectStock","item":"Bluetooth Speaker","plus":-1,"relativeTo":"speaker-before-control"}]}],"id":101,"setup":[{"actor":"buyer","do":"signUp","name":"kim"},{"as":"speaker-before-control","do":"dbRecordStock","item":"Bluetooth Speaker"},{"action":"buy","actor":"buyer","do":"callAction","input":{"attribute":"data-buy-input","contains":"Bluetooth Speaker","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"},"settleMs":2000},{"actor":"buyer","do":"expectActionOutcome","outcome":"accepted"},{"do":"dbExpectStock","item":"Bluetooth Speaker","plus":-1,"relativeTo":"speaker-before-control"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/01-purchase-session.json"}],"id":"selected-source-022","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-purchase-session.json"},{"checkGroups":[{"checkGroupId":"restock-race","feature":{"actors":["admin","a","b","c"],"criteria":[{"id":"202-control","steps":[{"do":"dbExpectStock","item":"Bluetooth Speaker","plus":5,"relativeTo":"stored-before-control"},{"actor":"a","do":"expectNumber","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"plus":5,"relativeTo":"storefront-before","testid":"item-stock","within":8000}]},{"id":"202a","steps":[{"as":"stored-before-rush","do":"dbRecordStock","item":"Bluetooth Speaker"},{"actor":"admin","do":"fill","in":{"contains":"Bluetooth Speaker","testid":"admin-location-row"},"testid":"restock-input","text":"5"},{"actor":"a","as":"rush-before","do":"recordNumber","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"item-stock"},{"branches":[[{"actors":["a","b","c"],"do":"clickConcurrently","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"settleMs":500,"testid":"buy-now"}],[{"actor":"admin","do":"click","in":{"contains":"Bluetooth Speaker","testid":"admin-location-row"},"testid":"restock-submit"}]],"do":"race","settleMs":6000},{"do":"dbExpectStock","item":"Bluetooth Speaker","plus":2,"relativeTo":"stored-before-rush"},{"atLeast":0,"do":"dbExpectStock","item":"Bluetooth Speaker","warehouse":"East"},{"atLeast":0,"do":"dbExpectStock","item":"Bluetooth Speaker","warehouse":"West"},{"actor":"a","do":"reload","settleMs":1000},{"actor":"a","do":"click","testid":"catalog-link"},{"actor":"a","do":"expectNumber","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"plus":2,"relativeTo":"rush-before","testid":"item-stock","within":10000},{"actor":"a","do":"ensureSignedIn","name":"r1","readyTestid":"current-user"},{"actor":"a","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"a","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"a","contains":"Bluetooth Speaker","count":2,"do":"expect","testid":"order-item","within":10000},{"actor":"b","do":"reload","settleMs":1000},{"actor":"b","do":"ensureSignedIn","name":"r2","readyTestid":"current-user"},{"actor":"b","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"b","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"b","contains":"Bluetooth Speaker","count":1,"do":"expect","testid":"order-item","within":10000},{"actor":"c","do":"reload","settleMs":1000},{"actor":"c","do":"ensureSignedIn","name":"r3","readyTestid":"current-user"},{"actor":"c","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"c","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"c","contains":"Bluetooth Speaker","count":1,"do":"expect","testid":"order-item","within":10000},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link"},{"actor":"admin","do":"expectNumber","in":{"contains":"Bluetooth Speaker","testid":"admin-item-row"},"plus":2,"relativeTo":"stored-before-rush","testid":"admin-stock","within":10000},{"actor":"a","do":"reload","settleMs":1000},{"actor":"a","do":"click","testid":"catalog-link"},{"actor":"a","do":"expectNumber","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"plus":2,"relativeTo":"stored-before-rush","testid":"item-stock","within":10000},{"actor":"a","do":"ensureSignedIn","name":"r1","readyTestid":"current-user"},{"account":"{user:r1}","as":"mixed-a","do":"dbRecordCheckout","item":"Bluetooth Speaker","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"account":"{user:r2}","as":"mixed-b","do":"dbRecordCheckout","item":"Bluetooth Speaker","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"account":"{user:r3}","as":"mixed-c","do":"dbRecordCheckout","item":"Bluetooth Speaker","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"action":"buy","actors":["a","b","c"],"alongside":[{"action":"restock","actors":["admin"],"input":{"attribute":"data-restock-input","contains":"Bluetooth Speaker","testid":"admin-location-row"},"namedAction":{"args":[0,0,1],"id":"restock","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"warehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/restock","reducer":"admin_restock"},"requests":1}],"do":"callConcurrently","from":"a","input":{"attribute":"data-buy-input","contains":"Bluetooth Speaker","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"},"requests":3,"settleMs":3000},{"accepted":4,"do":"expectCallOutcomes"},{"before":{"a":"mixed-a","b":"mixed-b","c":"mixed-c"},"do":"dbExpectPurchases","purchases":3}]}],"id":202,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"a","do":"signUp","name":"r1"},{"actor":"b","do":"signUp","name":"r2"},{"actor":"c","do":"signUp","name":"r3"},{"as":"stored-before-serial-purchase","do":"dbRecordStock","item":"Bluetooth Speaker"},{"actor":"a","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"buy-now"},{"actor":"a","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"a","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"a","contains":"Bluetooth Speaker","count":1,"do":"expect","testid":"order-item","within":10000},{"do":"dbExpectStock","item":"Bluetooth Speaker","plus":-1,"relativeTo":"stored-before-serial-purchase"},{"actor":"a","do":"reload","settleMs":1000},{"actor":"a","do":"ensureSignedIn","name":"r1","readyTestid":"current-user"},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"expectNumber","in":{"contains":"Bluetooth Speaker","testid":"admin-item-row"},"plus":-1,"relativeTo":"stored-before-serial-purchase","testid":"admin-stock","within":10000},{"as":"stored-before-control","do":"dbRecordStock","item":"Bluetooth Speaker"},{"actor":"admin","as":"warehouse-before","do":"recordNumber","in":{"contains":"Bluetooth Speaker","testid":"admin-location-row"},"testid":"admin-location-qty"},{"actor":"a","do":"click","testid":"catalog-link"},{"actor":"a","as":"storefront-before","do":"recordNumber","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"item-stock"},{"actor":"admin","do":"fill","in":{"contains":"Bluetooth Speaker","testid":"admin-location-row"},"testid":"restock-input","text":"5"},{"actor":"admin","do":"click","in":{"contains":"Bluetooth Speaker","testid":"admin-location-row"},"testid":"restock-submit"},{"do":"dbExpectStock","item":"Bluetooth Speaker","plus":5,"relativeTo":"stored-before-control","within":8000},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"expectNumber","in":{"contains":"Bluetooth Speaker","testid":"admin-location-row"},"plus":5,"relativeTo":"warehouse-before","testid":"admin-location-qty","within":8000},{"actor":"a","do":"reload","settleMs":1000},{"actor":"a","do":"click","testid":"catalog-link"},{"actor":"a","do":"expectNumber","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"plus":5,"relativeTo":"storefront-before","testid":"item-stock","within":8000},{"do":"dbExpectStock","item":"Bluetooth Speaker","plus":5,"relativeTo":"stored-before-control"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-restock-race.json"}],"id":"selected-source-023","scenario":{"level":1,"writeUrlPattern":"\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses"},"source":"scenarios/01-restock-race.json"},{"checkGroups":[{"checkGroupId":"review-eligibility","feature":{"actors":["owner","stranger"],"criteria":[{"id":"108a","steps":[{"action":"buy","actor":"owner","do":"callAction","input":{"attribute":"data-buy-input","contains":"Air Purifier","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"owner","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner","contains":"Air Purifier","do":"expect","testid":"order-item"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"catalog-link"},{"action":"submitReview","actor":"owner","do":"callAction","from":"owner","input":{"attribute":"data-buy-input","contains":"Air Purifier","testid":"item-card"},"namedAction":{"args":[0,4,"eligible review control"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"}],"path":"/api/items/:id/reviews","reducer":"submit_review"},"settleMs":1500},{"actor":"owner","do":"expectActionOutcome","outcome":"accepted"},{"actor":"stranger","do":"openItem","item":"Air Purifier"},{"actor":"stranger","contains":"eligible review control","do":"expect","testid":"review-item"},{"action":"submitReview","actor":"stranger","do":"callAction","from":"owner","input":{"attribute":"data-buy-input","contains":"Air Purifier","testid":"item-card"},"namedAction":{"args":[0,4,"never bought this"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"}],"path":"/api/items/:id/reviews","reducer":"submit_review"},"settleMs":1500},{"actor":"stranger","do":"expectActionOutcome","outcome":"application-refused","routeProvenBy":"owner"},{"actor":"stranger","do":"reload","settleMs":1500},{"actor":"stranger","do":"ensureSignedIn","name":"uma","readyTestid":"current-user"},{"actor":"stranger","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"stranger","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"stranger","do":"openItem","item":"Air Purifier"},{"actor":"stranger","contains":"eligible review control","do":"expect","testid":"review-item"},{"absent":true,"actor":"stranger","contains":"never bought this","do":"expect","testid":"review-item"}]},{"id":"108b","steps":[{"action":"buy","actor":"owner","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"owner","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"openItem","item":"Keyboard"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"review-toggle","unlessVisible":"review-rating"},{"actor":"owner","do":"fill","testid":"review-rating","text":"4"},{"actor":"owner","do":"fill","testid":"review-input","text":"bought and used it"},{"actor":"owner","do":"click","testid":"review-submit"},{"actor":"owner","contains":"bought and used it","do":"expect","testid":"review-item","within":8000}]}],"id":108,"setup":[{"actor":"owner","do":"signUp","name":"tam"},{"actor":"stranger","do":"signUp","name":"uma"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","source":"scenarios/01-review-eligibility.json"}],"id":"selected-source-024","scenario":{"level":1,"writeUrlPattern":"\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses"},"source":"scenarios/01-review-eligibility.json"},{"checkGroups":[{"checkGroupId":"rating","feature":{"actors":["author","other"],"criteria":[{"id":"6c","steps":[{"actor":"other","do":"openItem","item":"Gaming Mouse"},{"actor":"other","do":"click","ifAvailable":true,"testid":"review-toggle","unlessVisible":"review-rating"},{"actor":"other","do":"fill","testid":"review-rating","text":"4"},{"actor":"other","do":"fill","testid":"review-input","text":"does the job"},{"actor":"other","do":"click","testid":"review-submit"},{"actor":"other","do":"expectNumber","equals":3,"testid":"review-average","within":10000},{"actors":["author","other"],"do":"expectAgreement","numeric":true,"testid":"review-average","within":10000}]}],"id":6,"setup":[{"actor":"author","do":"signUp","name":"leon"},{"actor":"other","do":"signUp","name":"maya"},{"action":"buy","actor":"author","do":"callAction","input":{"attribute":"data-buy-input","contains":"Gaming Mouse","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"author","do":"expectActionOutcome","outcome":"accepted"},{"actor":"author","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"author","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"author","contains":"Gaming Mouse","do":"expect","testid":"order-item","within":10000},{"actor":"author","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"author","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"author","do":"openItem","item":"Gaming Mouse"},{"actor":"author","do":"click","ifAvailable":true,"testid":"review-toggle","unlessVisible":"review-rating"},{"actor":"author","do":"fill","testid":"review-rating","text":"2"},{"actor":"author","do":"fill","testid":"review-input","text":"works for travel"},{"actor":"author","do":"click","testid":"review-submit"},{"actor":"author","contains":"works for travel","do":"expect","testid":"review-item"},{"action":"buy","actor":"other","do":"callAction","input":{"attribute":"data-buy-input","contains":"Gaming Mouse","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"other","do":"expectActionOutcome","outcome":"accepted"},{"actor":"other","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"other","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"other","contains":"Gaming Mouse","do":"expect","testid":"order-item","within":10000},{"actor":"other","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"other","do":"click","ifAvailable":true,"testid":"catalog-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","source":"scenarios/01-review-rating-live.json"}],"id":"selected-source-025","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-review-rating-live.json"},{"checkGroups":[{"checkGroupId":"unique-review","feature":{"actors":["author"],"criteria":[{"id":"6b","steps":[{"actor":"author","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"author","do":"click","ifAvailable":true,"testid":"catalog-link"},{"action":"submitReview","actor":"author","do":"callAction","from":"author","input":{"attribute":"data-buy-input","contains":"Air Purifier","testid":"item-card"},"namedAction":{"args":[0,4,"changed my mind"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"}],"path":"/api/items/:id/reviews","reducer":"submit_review"},"settleMs":1500},{"actor":"author","do":"expectActionOutcome","outcome":"completed"},{"actor":"author","do":"reload","settleMs":2500},{"actor":"author","do":"ensureSignedIn","name":"kira","readyTestid":"current-user"},{"actor":"author","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"author","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"author","do":"openItem","item":"Air Purifier"},{"actor":"author","count":1,"do":"expect","testid":"review-item","within":10000}]}],"id":6,"setup":[{"actor":"author","do":"signUp","name":"kira"},{"action":"buy","actor":"author","do":"callAction","input":{"attribute":"data-buy-input","contains":"Air Purifier","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"author","do":"expectActionOutcome","outcome":"accepted"},{"actor":"author","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"author","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"author","contains":"Air Purifier","do":"expect","testid":"order-item","within":10000},{"actor":"author","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"author","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"author","do":"openItem","item":"Air Purifier"},{"actor":"author","do":"click","ifAvailable":true,"testid":"review-toggle","unlessVisible":"review-rating"},{"actor":"author","do":"fill","testid":"review-rating","text":"4"},{"actor":"author","do":"fill","testid":"review-input","text":"quiet and effective"},{"actor":"author","do":"click","testid":"review-submit"},{"actor":"author","contains":"quiet and effective","do":"expect","testid":"review-item"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","source":"scenarios/01-review-uniqueness.json"}],"id":"selected-source-026","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-review-uniqueness.json"},{"checkGroups":[{"checkGroupId":"reviews","feature":{"actors":["author","visitor"],"criteria":[{"id":"6a","steps":[{"actor":"author","do":"openItem","item":"Induction Cooktop"},{"actor":"author","do":"click","ifAvailable":true,"testid":"review-toggle","unlessVisible":"review-rating"},{"actor":"author","do":"fill","testid":"review-rating","text":"4"},{"actor":"author","do":"fill","testid":"review-input","text":"solid mold"},{"actor":"author","do":"click","testid":"review-submit"},{"actor":"author","contains":"solid mold","do":"expect","testid":"review-item"},{"actor":"visitor","do":"openItem","item":"Induction Cooktop"},{"actor":"visitor","contains":"solid mold","do":"expect","testid":"review-item","within":10000}]}],"id":6,"setup":[{"actor":"author","do":"signUp","name":"hal"},{"actor":"author","do":"click","in":{"contains":"Induction Cooktop","testid":"item-card"},"testid":"buy-now"},{"actor":"author","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"author","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"author","contains":"Induction Cooktop","do":"expect","testid":"order-item","within":10000},{"actor":"author","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"author","do":"click","ifAvailable":true,"testid":"catalog-link"}]},"packId":"ecommerce.feature.reviews","role":"feature","source":"scenarios/01-review-visibility.json"}],"id":"selected-source-027","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-review-visibility.json"},{"checkGroups":[{"checkGroupId":"server-price","feature":{"actors":["buyer"],"criteria":[{"id":"104a","steps":[{"action":"buy","actor":"buyer","do":"callAction","input":{"attribute":"data-buy-input","contains":"Espresso Machine","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"},"settleMs":2000},{"actor":"buyer","do":"expectActionOutcome","outcome":"accepted"},{"action":"buy","actor":"buyer","do":"callAction","input":{"attribute":"data-buy-input","contains":"Coffee Grinder","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"},"settleMs":2000},{"actor":"buyer","do":"expectActionOutcome","outcome":"accepted"},{"actor":"buyer","do":"freshClient"},{"actor":"buyer-fresh","do":"signIn","name":"oli"},{"actor":"buyer-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"buyer-fresh","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"buyer-fresh","count":2,"do":"expect","testid":"order-item"},{"actor":"buyer-fresh","contains":"Espresso Machine","count":1,"do":"expect","testid":"order-item"},{"actor":"buyer-fresh","do":"expectNumber","equals":449,"in":{"contains":"Espresso Machine","testid":"order-item"},"testid":"order-total"},{"actor":"buyer-fresh","contains":"Coffee Grinder","count":1,"do":"expect","testid":"order-item"},{"actor":"buyer-fresh","do":"expectNumber","equals":64,"in":{"contains":"Coffee Grinder","testid":"order-item"},"testid":"order-total"}]}],"id":104,"setup":[{"actor":"buyer","do":"signUp","name":"oli"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/01-server-price.json"}],"id":"selected-source-028","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-server-price.json"},{"checkGroups":[{"checkGroupId":"warehouse-area-boundary","feature":{"actors":["admin","staff"],"criteria":[{"id":"7a","steps":[{"actor":"admin","do":"expect","testid":"admin-item-row"},{"actor":"staff","do":"click","ifAvailable":true,"settleMs":1500,"testid":"admin-link"},{"absent":true,"actor":"staff","do":"expect","testid":"admin-item-row"}]}],"id":7,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"admin","do":"click","testid":"admin-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-warehouse-admin-staff.json"},{"checkGroupId":"warehouse-view","feature":{"actors":["admin","staff"],"criteria":[{"id":"7b","steps":[{"actor":"admin","count":13,"do":"expect","testid":"admin-item-row"},{"actor":"admin","count":26,"do":"expect","testid":"admin-location-row"},{"actor":"admin","contains":"East","do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","contains":"West","do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","do":"expectNumber","equals":100,"in":{"contains":"Gaming Mouse","testid":"admin-item-row"},"testid":"admin-stock"}]}],"id":7,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"admin","do":"click","testid":"admin-link"}]},"packId":"ecommerce.feature.warehouse-admin","role":"feature","source":"scenarios/01-warehouse-admin-staff.json"}],"id":"selected-source-029","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-warehouse-admin-staff.json"},{"checkGroups":[{"checkGroupId":"warehouse-stock","feature":{"actors":["admin","visitor"],"criteria":[{"id":"7c","steps":[{"actor":"admin","do":"fill","in":{"contains":"Gaming Mouse","testid":"admin-location-row"},"testid":"restock-input","text":"25"},{"actor":"admin","do":"click","in":{"contains":"Gaming Mouse","testid":"admin-location-row"},"testid":"restock-submit"},{"actor":"visitor","do":"expectNumber","equals":125,"in":{"contains":"Gaming Mouse","testid":"item-card"},"testid":"item-stock","within":10000},{"actor":"admin","do":"expectNumber","equals":125,"in":{"contains":"Gaming Mouse","testid":"admin-item-row"},"testid":"admin-stock"}]}],"id":7,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"visitor","do":"expectNumber","equals":100,"in":{"contains":"Gaming Mouse","testid":"item-card"},"testid":"item-stock"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-warehouse-stock-live-staff.json"}],"id":"selected-source-030","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-warehouse-stock-live-staff.json"},{"checkGroups":[{"checkGroupId":"cancellation-and-return","feature":{"actors":["customer","staff"],"criteria":[{"id":"3d","steps":[{"actor":"customer","do":"click","in":{"contains":"Coffee Grinder","testid":"item-card"},"testid":"buy-now"},{"actor":"staff","contains":"Coffee Grinder","do":"expect","testid":"queue-item","within":10000},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle"},{"actor":"customer","do":"click","in":{"contains":"Coffee Grinder","testid":"order-item"},"testid":"cancel-order"},{"actor":"staff","contains":"Coffee Grinder","do":"waitUntilAbsent","testid":"queue-item","within":10000}]}],"id":3,"setup":[{"actor":"customer","do":"signUp","name":"cancel-queue"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.cancellation-queue-specifications","requiresFeatures":["ecommerce.l2.order-cancellation-features","ecommerce.progression.fulfilment-queue"],"role":"guarantee","source":"scenarios/02-cancellation-queue.json","stablePackId":"ecommerce.returns-pricing"}],"id":"selected-source-031","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-cancellation-queue.json"},{"checkGroups":[{"checkGroupId":"fulfilment-area-boundary","feature":{"actors":["customer","staff","admin"],"criteria":[{"id":"1d","steps":[{"actor":"staff","do":"click","testid":"staff-link"},{"actor":"staff","do":"expect","testid":"fulfilment-panel"},{"actor":"admin","do":"click","testid":"staff-link"},{"actor":"admin","do":"expect","testid":"fulfilment-panel"},{"actor":"customer","do":"click","ifAvailable":true,"settleMs":1500,"testid":"staff-link"},{"absent":true,"actor":"customer","do":"expect","testid":"fulfilment-panel"}]}],"id":1,"setup":[{"actor":"customer","do":"signUp","name":"fq-customer"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.fulfilment-queue"],"role":"guarantee","source":"scenarios/02-fulfilment-access.json"}],"id":"selected-source-032","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-fulfilment-access.json"},{"checkGroups":[{"checkGroupId":"fulfilment-queue","feature":{"actors":["customer","staff"],"criteria":[{"id":"1a","steps":[{"actor":"staff","as":"depth-before","do":"recordNumber","testid":"queue-depth"},{"actor":"customer","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"buy-now"},{"actor":"staff","contains":"Desk Lamp","do":"expect","testid":"queue-item","within":10000},{"actor":"staff","do":"expectNumber","plus":1,"relativeTo":"depth-before","testid":"queue-depth"}]}],"id":1,"setup":[{"actor":"customer","do":"signUp","name":"fq-live"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","requiresFeatures":["ecommerce.progression.fulfilment-queue"],"role":"guarantee","source":"scenarios/02-fulfilment-live.json"}],"id":"selected-source-033","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-fulfilment-live.json"},{"checkGroups":[{"checkGroupId":"fulfilment-queue","feature":{"actors":["customer","staff"],"criteria":[{"id":"1c","steps":[{"action":"buy","actor":"customer","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"customer","do":"expectActionOutcome","outcome":"accepted"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"pending","within":10000},{"actor":"staff","do":"reload","settleMs":2000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"staff-link"},{"actor":"staff","contains":"Keyboard","do":"expect","testid":"queue-item","within":10000},{"actor":"staff","do":"click","in":{"contains":"Keyboard","testid":"queue-item"},"testid":"ship-submit"},{"actor":"staff","attribute":"data-submit-state","do":"expect","testid":"fulfilment-panel","value":"succeeded","within":10000},{"actor":"staff","do":"reload","settleMs":2000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"staff-link"},{"absent":true,"actor":"staff","contains":"Keyboard","do":"expect","testid":"queue-item","within":10000},{"actor":"customer","do":"reload","settleMs":2000},{"actor":"customer","do":"ensureSignedIn","name":"fq-ship","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"shipped","within":10000}]}],"id":1,"setup":[{"actor":"customer","do":"signUp","name":"fq-ship"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link"}]},"packId":"ecommerce.progression.fulfilment-queue","requiresFeatures":["ecommerce.feature.purchasing"],"role":"feature","source":"scenarios/02-fulfilment-ship.json","stablePackId":"ecommerce.operations-access"}],"id":"selected-source-034","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-fulfilment-ship.json"},{"checkGroups":[{"checkGroupId":"refund-accounting","feature":{"actors":["admin","customer","customer2"],"criteria":[{"id":"203a","steps":[{"actor":"admin","as":"rev-start","do":"recordNumber","testid":"admin-revenue"},{"actor":"customer","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"buy-now"},{"actor":"customer","do":"wait","ms":2500},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"expectNumber","plus":42,"relativeTo":"rev-start","testid":"admin-revenue","within":10000},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-status","value":"pending","within":10000},{"account":"{user:books}","as":"cancel-before","do":"dbRecordCheckout","item":"Desk Lamp","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"action":"cancel","actors":["customer","customer2"],"do":"callConcurrently","from":"customer","input":{"attribute":"data-cancel-input","contains":"Desk Lamp","testid":"order-item"},"namedAction":{"args":[0],"id":"cancel","params":[{"in":"path","name":"orderId","placeholder":":id","wireType":"u64"}],"path":"/api/orders/:id/cancel","reducer":"cancel_order"},"requests":4,"settleMs":3000},{"do":"expectCallOutcomes"},{"before":"cancel-before","do":"dbExpectCancellation"},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"expectNumber","plus":0,"relativeTo":"rev-start","testid":"admin-revenue","within":10000}]}],"id":203,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"customer","do":"signUp","name":"books"},{"actor":"customer2","do":"signIn","name":"books"},{"actor":"admin","do":"click","testid":"admin-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.cancellation-accounting-specifications","requiresFeatures":["ecommerce.l2.order-cancellation-features"],"role":"guarantee","source":"scenarios/02-invariants.json","stablePackId":"ecommerce.returns-pricing"},{"checkGroupId":"refund-accounting","feature":{"actors":["admin","customer","customer2"],"criteria":[{"id":"203b","steps":[{"actor":"admin","as":"history-revenue-before-sale","do":"recordNumber","testid":"admin-revenue"},{"actor":"customer","do":"pressKey","key":"Escape"},{"actor":"customer","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"buy-now"},{"actor":"customer","do":"wait","ms":2500},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"expectNumber","plus":79.5,"relativeTo":"history-revenue-before-sale","testid":"admin-revenue","within":10000},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","contains":"Bluetooth Speaker","count":1,"do":"expect","testid":"order-item","within":10000},{"actor":"admin","as":"rev-after-sale","do":"recordNumber","testid":"admin-revenue"},{"actor":"admin","do":"fill","in":{"contains":"Bluetooth Speaker","testid":"admin-item-row"},"testid":"price-input","text":"5.00"},{"actor":"admin","do":"click","in":{"contains":"Bluetooth Speaker","testid":"admin-item-row"},"testid":"price-submit"},{"actor":"admin","do":"wait","ms":3000},{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link","within":1000},{"actor":"admin","do":"expectNumber","equals":5,"in":{"contains":"Bluetooth Speaker","testid":"admin-item-row"},"testid":"price-input","within":10000},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"expectNumber","plus":0,"relativeTo":"rev-after-sale","testid":"admin-revenue"}]}],"id":203,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"customer","do":"signUp","name":"books"},{"actor":"customer2","do":"signIn","name":"books"},{"actor":"admin","do":"click","testid":"admin-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.price-accounting-specifications","requiresFeatures":["ecommerce.l2.price-history-features"],"role":"guarantee","source":"scenarios/02-invariants.json","stablePackId":"ecommerce.returns-pricing"}],"id":"selected-source-035","scenario":{"level":2,"writeUrlPattern":"\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses|\\/transfer|\\/cancel|\\/return|\\/ship|\\/price|\\/fulfil"},"source":"scenarios/02-invariants.json"},{"checkGroups":[{"checkGroupId":"price-history","feature":{"actors":["admin","visitor"],"criteria":[{"id":"4b","steps":[{"actor":"visitor","atLeast":2,"do":"expectNumber","in":{"contains":"Air Purifier","testid":"item-card"},"testid":"item-price"},{"actor":"admin","do":"fill","in":{"contains":"Air Purifier","testid":"admin-item-row"},"testid":"price-input","text":"1.00"},{"actor":"admin","do":"click","in":{"contains":"Air Purifier","testid":"admin-item-row"},"testid":"price-submit"},{"actor":"visitor","do":"expectNumber","equals":1,"in":{"contains":"Air Purifier","testid":"item-card"},"testid":"item-price","within":10000}]}],"id":4,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"visitor","do":"fill","enter":true,"settleMs":1500,"testid":"search-input","text":"Air Purifier"}]},"packId":"ecommerce.l2.price-history-features","role":"feature","source":"scenarios/02-live-price.json","stablePackId":"ecommerce.returns-pricing"}],"id":"selected-source-036","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-live-price.json"},{"checkGroups":[{"checkGroupId":"operational-views","feature":{"actors":["admin","customer","restocker"],"criteria":[{"id":"5e","steps":[{"actor":"admin","contains":"Air Purifier","do":"expect","testid":"low-stock-item","within":8000}]}],"id":5,"setup":[{"do":"dbSetStock","item":"Air Purifier","quantity":2,"settleMs":250,"warehouse":"East"},{"do":"dbSetStock","item":"Air Purifier","quantity":1,"settleMs":250,"warehouse":"West"},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"customer","do":"signUp","name":"low-stock-focused"},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"low-stock-focused","readyTestid":"current-user"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"low-stock-link","unlessVisible":"low-stock-item","within":10000}]},"packId":"ecommerce.l2.inventory-dashboard","role":"feature","source":"scenarios/02-low-stock.json","stablePackId":"ecommerce.inventory-operations"},{"checkGroupId":"inventory-dashboard","feature":{"actors":["admin","customer","restocker"],"criteria":[{"id":"5a","steps":[{"actor":"admin","contains":"Air Purifier","do":"expect","testid":"low-stock-item","within":8000},{"actor":"restocker","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"restocker","do":"click","testid":"admin-link"},{"actor":"restocker","do":"fill","in":{"contains":"Air Purifier","testid":"admin-location-row"},"testid":"restock-input","text":"8"},{"actor":"restocker","do":"click","in":{"contains":"Air Purifier","testid":"admin-location-row"},"testid":"restock-submit"},{"actor":"admin","contains":"Air Purifier","do":"waitUntilAbsent","testid":"low-stock-item","within":10000},{"actor":"customer","do":"click","in":{"contains":"Air Purifier","testid":"item-card"},"testid":"buy-now"},{"actor":"admin","contains":"Air Purifier","do":"expect","testid":"low-stock-item","within":10000}]}],"id":5,"setup":[{"do":"dbSetStock","item":"Air Purifier","quantity":2,"settleMs":250,"warehouse":"East"},{"do":"dbSetStock","item":"Air Purifier","quantity":1,"settleMs":250,"warehouse":"West"},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"customer","do":"signUp","name":"low-stock-focused"},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"low-stock-focused","readyTestid":"current-user"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"low-stock-link","unlessVisible":"low-stock-item","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.inventory-dashboard"],"role":"guarantee","source":"scenarios/02-low-stock.json"}],"id":"selected-source-037","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-low-stock.json"},{"checkGroups":[{"checkGroupId":"operational-views","feature":{"actors":["buyer","visitor"],"criteria":[{"id":"5d","steps":[{"actor":"visitor","do":"reload","settleMs":2500},{"actor":"visitor","do":"expectNumber","equals":1,"in":{"contains":"Gaming Mouse","testid":"recommended-item"},"testid":"recommendation-rank","within":10000}]}],"id":5,"setup":[{"actor":"buyer","do":"signUp","name":"best-seller-buyer"},{"actor":"buyer","do":"click","in":{"contains":"Gaming Mouse","testid":"item-card"},"testid":"buy-now"},{"actor":"buyer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"buyer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"buyer","contains":"Gaming Mouse","do":"expect","testid":"order-item","within":10000}]},"packId":"ecommerce.l2.sales-dashboard","role":"feature","source":"scenarios/02-operational-best-sellers.json","stablePackId":"ecommerce.inventory-operations"}],"id":"selected-source-038","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-operational-best-sellers.json"},{"checkGroups":[{"checkGroupId":"operational-views","feature":{"actors":["admin","customer"],"criteria":[{"id":"5f","steps":[{"actor":"admin","do":"click","ifAvailable":true,"testid":"sales-link","unlessVisible":"category-row"},{"actor":"admin","as":"audio-core-units","do":"recordNumber","in":{"contains":"Audio","testid":"category-row"},"testid":"category-units"},{"actor":"admin","as":"audio-core-revenue","do":"recordNumber","in":{"contains":"Audio","testid":"category-row"},"testid":"category-revenue"},{"actor":"customer","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"buy-now"},{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"sales-link","unlessVisible":"category-row"},{"actor":"admin","do":"expectNumber","in":{"contains":"Audio","testid":"category-row"},"plus":1,"relativeTo":"audio-core-units","testid":"category-units"},{"actor":"admin","do":"expectNumber","in":{"contains":"Audio","testid":"category-row"},"plus":79.5,"relativeTo":"audio-core-revenue","testid":"category-revenue"}]}],"id":5,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"customer","do":"signUp","name":"category-totals"},{"actor":"admin","do":"click","testid":"admin-link"}]},"packId":"ecommerce.l2.sales-dashboard","role":"feature","source":"scenarios/02-operational-category-totals.json","stablePackId":"ecommerce.inventory-operations"},{"checkGroupId":"sales-dashboard","feature":{"actors":["admin","customer"],"criteria":[{"id":"5b","steps":[{"actor":"admin","do":"click","ifAvailable":true,"testid":"sales-link","unlessVisible":"category-row"},{"actor":"admin","as":"audio-units","do":"recordNumber","in":{"contains":"Audio","testid":"category-row"},"testid":"category-units"},{"actor":"admin","as":"audio-revenue","do":"recordNumber","in":{"contains":"Audio","testid":"category-row"},"testid":"category-revenue"},{"actor":"customer","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"buy-now"},{"actor":"admin","do":"expectNumber","in":{"contains":"Audio","testid":"category-row"},"plus":1,"relativeTo":"audio-units","testid":"category-units","within":10000},{"actor":"admin","do":"expectNumber","in":{"contains":"Audio","testid":"category-row"},"plus":79.5,"relativeTo":"audio-revenue","testid":"category-revenue","within":10000}]}],"id":5,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"customer","do":"signUp","name":"category-totals"},{"actor":"admin","do":"click","testid":"admin-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.sales-dashboard"],"role":"guarantee","source":"scenarios/02-operational-category-totals.json"}],"id":"selected-source-039","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-operational-category-totals.json"},{"checkGroups":[{"checkGroupId":"operational-views","feature":{"actors":["customer"],"criteria":[{"id":"5c","steps":[{"actor":"customer","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"buy-now"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"customer","contains":"Headphones","do":"expect","in":{"testid":"recommended-list"},"testid":"recommended-item","within":10000},{"actor":"customer","do":"click","in":{"contains":"Headphones","testid":"item-card"},"testid":"add-to-cart"},{"actor":"customer","contains":"Headphones","do":"waitUntilAbsent","in":{"testid":"recommended-list"},"testid":"recommended-item","within":10000}]}],"id":5,"setup":[{"actor":"customer","do":"signUp","name":"recommendations"}]},"packId":"ecommerce.l2.recommendations","role":"feature","source":"scenarios/02-operational-recommendations.json","stablePackId":"ecommerce.inventory-operations"}],"id":"selected-source-040","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-operational-recommendations.json"},{"checkGroups":[{"checkGroupId":"cancellation-and-return","feature":{"actors":["customer","admin"],"criteria":[{"id":"3a","steps":[{"actor":"admin","as":"revenue-before","do":"recordNumber","testid":"admin-revenue"},{"actor":"customer","as":"stock-before","do":"recordNumber","in":{"contains":"Coffee Grinder","testid":"item-card"},"testid":"item-stock"},{"actor":"customer","do":"click","in":{"contains":"Coffee Grinder","testid":"item-card"},"testid":"buy-now"},{"actor":"customer","do":"expectNumber","in":{"contains":"Coffee Grinder","testid":"item-card"},"plus":-1,"relativeTo":"stock-before","testid":"item-stock","within":10000},{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link"},{"actor":"admin","do":"expectNumber","plus":64,"relativeTo":"revenue-before","testid":"admin-revenue","within":10000},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","settleMs":2000,"testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Coffee Grinder","testid":"order-item"},"testid":"order-status","value":"pending"},{"actor":"customer","do":"click","in":{"contains":"Coffee Grinder","testid":"order-item"},"testid":"cancel-order"},{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link"},{"actor":"admin","do":"expectNumber","plus":0,"relativeTo":"revenue-before","testid":"admin-revenue","within":10000},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"customer","do":"expectNumber","in":{"contains":"Coffee Grinder","testid":"item-card"},"plus":0,"relativeTo":"stock-before","testid":"item-stock","within":10000}]}],"id":3,"setup":[{"actor":"customer","do":"signUp","name":"cancel-core"},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"}]},"packId":"ecommerce.l2.order-cancellation-features","role":"feature","source":"scenarios/02-order-cancellation-core.json","stablePackId":"ecommerce.returns-pricing"}],"id":"selected-source-041","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-order-cancellation-core.json"},{"checkGroups":[{"checkGroupId":"cancellation-and-return","feature":{"actors":["customer"],"criteria":[{"id":"3b","steps":[{"actor":"customer","do":"click","in":{"contains":"Coffee Grinder","testid":"item-card"},"settleMs":2000,"testid":"buy-now"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"click","in":{"contains":"Coffee Grinder","testid":"order-item"},"testid":"cancel-order"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Coffee Grinder","testid":"order-item"},"testid":"order-status","value":"cancelled","within":10000}]}],"id":3,"setup":[{"actor":"customer","do":"signUp","name":"cancel-focused"}]},"packId":"ecommerce.l2.order-cancellation-features","role":"feature","source":"scenarios/02-order-cancellation-history.json","stablePackId":"ecommerce.returns-pricing"}],"id":"selected-source-042","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-order-cancellation-history.json"},{"checkGroups":[{"checkGroupId":"price-history","feature":{"actors":["admin","customer","visitor"],"criteria":[{"id":"4a","steps":[{"actor":"customer","do":"fill","enter":true,"settleMs":1500,"testid":"search-input","text":"Air Purifier"},{"actor":"customer","as":"air-purifier-paid","do":"recordNumber","in":{"contains":"Air Purifier","testid":"item-card"},"testid":"item-price"},{"actor":"customer","do":"click","in":{"contains":"Air Purifier","testid":"item-card"},"settleMs":2000,"testid":"buy-now"},{"actor":"visitor","do":"fill","enter":true,"settleMs":1500,"testid":"search-input","text":"Air Purifier"},{"actor":"admin","do":"fill","in":{"contains":"Air Purifier","testid":"admin-item-row"},"testid":"price-input","text":"1.00"},{"actor":"admin","do":"click","in":{"contains":"Air Purifier","testid":"admin-item-row"},"testid":"price-submit"},{"actor":"visitor","do":"expectNumber","equals":1,"in":{"contains":"Air Purifier","testid":"item-card"},"testid":"item-price","within":10000},{"actor":"customer","do":"reload","settleMs":2000},{"actor":"customer","do":"ensureSignedIn","name":"history-persisted","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle"},{"actor":"customer","do":"expectNumber","in":{"contains":"Air Purifier","testid":"order-item"},"plus":0,"relativeTo":"air-purifier-paid","testid":"order-total"}]}],"id":4,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"customer","do":"signUp","name":"history-persisted"},{"actor":"admin","do":"click","testid":"admin-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.price-history-specifications","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.price-history-features"],"role":"guarantee","source":"scenarios/02-paid-price-history.json","stablePackId":"ecommerce.returns-pricing"}],"id":"selected-source-043","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-paid-price-history.json"},{"checkGroups":[{"checkGroupId":"fulfilment-queue","feature":{"actors":["customer","staff"],"criteria":[{"id":"1b","steps":[{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":2000,"warehouse":"West"},{"actor":"customer","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"settleMs":1500,"testid":"buy-now"},{"actor":"staff","do":"reload","settleMs":2000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"staff-link"},{"actor":"staff","contains":"Desk Lamp","do":"expect","testid":"queue-item","within":10000},{"actor":"staff","contains":"East","do":"expect","in":{"contains":"Desk Lamp","testid":"queue-item"},"testid":"queue-warehouse"}]}],"id":1,"setup":[{"actor":"customer","do":"signUp","name":"fq-warehouse"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link"}]},"packId":"ecommerce.progression.fulfilment-queue","role":"feature","source":"scenarios/02-queue-warehouse.json","stablePackId":"ecommerce.operations-access"}],"id":"selected-source-044","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-queue-warehouse.json"},{"checkGroups":[{"checkGroupId":"stock-conservation","feature":{"actors":["customer"],"criteria":[{"id":"202b","steps":[{"as":"east-before-cancel-202b","do":"dbRecordStock","item":"Induction Cooktop","warehouse":"East"},{"as":"west-before-cancel-202b","do":"dbRecordStock","item":"Induction Cooktop","warehouse":"West"},{"as":"stored-before-cancel-202b","do":"dbRecordStock","item":"Induction Cooktop"},{"actor":"customer","as":"cancel-stock-before","do":"recordNumber","in":{"contains":"Induction Cooktop","testid":"item-card"},"testid":"item-stock"},{"actor":"customer","do":"click","in":{"contains":"Induction Cooktop","testid":"item-card"},"settleMs":2000,"testid":"buy-now"},{"actor":"customer","do":"expectNumber","in":{"contains":"Induction Cooktop","testid":"item-card"},"plus":-1,"relativeTo":"cancel-stock-before","testid":"item-stock"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","contains":"Induction Cooktop","count":1,"do":"expect","testid":"order-item","within":10000},{"do":"dbExpectStock","item":"Induction Cooktop","plus":-1,"relativeTo":"stored-before-cancel-202b"},{"actor":"customer","do":"click","in":{"contains":"Induction Cooktop","testid":"order-item"},"settleMs":2000,"testid":"cancel-order"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"customer","do":"expectNumber","in":{"contains":"Induction Cooktop","testid":"item-card"},"plus":0,"relativeTo":"cancel-stock-before","testid":"item-stock"},{"do":"dbExpectStock","item":"Induction Cooktop","plus":0,"relativeTo":"stored-before-cancel-202b"},{"do":"dbExpectStock","item":"Induction Cooktop","plus":0,"relativeTo":"east-before-cancel-202b","warehouse":"East"},{"do":"dbExpectStock","item":"Induction Cooktop","plus":0,"relativeTo":"west-before-cancel-202b","warehouse":"West"}]}],"id":202,"setup":[{"actor":"customer","do":"signUp","name":"fresh-stock"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","requiresFeatures":["ecommerce.l2.order-cancellation-features"],"role":"guarantee","source":"scenarios/02-self-contained.json","stablePackId":"ecommerce.inventory-operations"},{"checkGroupId":"stock-conservation","feature":{"actors":["customer"],"criteria":[{"id":"202c","steps":[{"as":"east-before-cancel-202c","do":"dbRecordStock","item":"Headphones","warehouse":"East"},{"as":"west-before-cancel-202c","do":"dbRecordStock","item":"Headphones","warehouse":"West"},{"as":"stored-before-cancel-202c","do":"dbRecordStock","item":"Headphones"},{"actor":"customer","as":"fresh-stock-before","do":"recordNumber","in":{"contains":"Headphones","testid":"item-card"},"testid":"item-stock"},{"actor":"customer","do":"click","in":{"contains":"Headphones","testid":"item-card"},"settleMs":2000,"testid":"buy-now"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","contains":"Headphones","count":1,"do":"expect","testid":"order-item","within":10000},{"do":"dbExpectStock","item":"Headphones","plus":-1,"relativeTo":"stored-before-cancel-202c"},{"actor":"customer","do":"click","in":{"contains":"Headphones","testid":"order-item"},"settleMs":2000,"testid":"cancel-order"},{"actor":"customer","do":"freshClient"},{"actor":"customer-fresh","do":"expectNumber","in":{"contains":"Headphones","testid":"item-card"},"plus":0,"relativeTo":"fresh-stock-before","testid":"item-stock"},{"do":"dbExpectStock","item":"Headphones","plus":0,"relativeTo":"stored-before-cancel-202c"},{"do":"dbExpectStock","item":"Headphones","plus":0,"relativeTo":"east-before-cancel-202c","warehouse":"East"},{"do":"dbExpectStock","item":"Headphones","plus":0,"relativeTo":"west-before-cancel-202c","warehouse":"West"}]}],"id":202,"setup":[{"actor":"customer","do":"signUp","name":"fresh-stock"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","requiresFeatures":["ecommerce.l2.order-cancellation-features"],"role":"guarantee","source":"scenarios/02-self-contained.json","stablePackId":"ecommerce.inventory-operations"}],"id":"selected-source-045","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-self-contained.json"},{"checkGroups":[{"checkGroupId":"operator-authorization","feature":{"actors":["customer","staff"],"criteria":[{"id":"201c","steps":[{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","ifAvailable":true,"testid":"catalog-link","within":1000},{"actor":"customer","do":"click","in":{"contains":"Coffee Grinder","testid":"item-card"},"settleMs":2000,"testid":"buy-now"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","contains":"Coffee Grinder","count":1,"do":"expect","testid":"order-item","within":10000},{"action":"ship","actor":"staff","do":"callAction","from":"customer","input":{"attribute":"data-ship-input","contains":"Coffee Grinder","testid":"order-item"},"namedAction":{"args":[0],"id":"ship","params":[{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/fulfilment/ship","reducer":"ship_order"},"settleMs":2000},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"customer","do":"reload","settleMs":2000},{"actor":"customer","do":"ensureSignedIn","name":"direct-notstaff","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Coffee Grinder","testid":"order-item"},"testid":"order-status","value":"shipped","within":10000},{"action":"ship","actor":"customer","do":"callAction","input":{"attribute":"data-ship-input","contains":"Laptop Stand","testid":"order-item"},"namedAction":{"args":[0],"id":"ship","params":[{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/fulfilment/ship","reducer":"ship_order"},"settleMs":2000},{"actor":"customer","do":"expectActionOutcome","outcome":"refused","routeProvenBy":"staff"},{"actor":"customer","do":"reload","settleMs":2000},{"actor":"customer","do":"ensureSignedIn","name":"direct-notstaff","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Laptop Stand","testid":"order-item"},"testid":"order-status","value":"pending"}]}],"id":201,"setup":[{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"customer","do":"signUp","name":"direct-notstaff"},{"actor":"customer","do":"click","in":{"contains":"Laptop Stand","testid":"item-card"},"settleMs":2000,"testid":"buy-now"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.operations-access-specifications","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"guarantee","source":"scenarios/02-server-actions.json","stablePackId":"ecommerce.operations-access"},{"checkGroupId":"stock-conservation","feature":{"actors":["admin","customer"],"criteria":[{"id":"202d","steps":[{"do":"dbExpectStock","equals":60,"item":"Headphones","warehouse":"East"},{"do":"dbExpectStock","equals":40,"item":"Headphones","warehouse":"West"},{"as":"direct-race-stock-before","do":"dbRecordStock","item":"Headphones"},{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link","within":1000},{"actor":"customer","do":"reload","settleMs":2000},{"actor":"customer","do":"ensureSignedIn","name":"direct-conserve","readyTestid":"current-user"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-from","text":"East"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-to","text":"West"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-qty","text":"25"},{"branches":[[{"action":"transfer","actor":"admin","do":"callAction","input":{"attribute":"data-transfer-input","contains":"Headphones","testid":"admin-item-row"},"namedAction":{"args":[0,0,0,25],"id":"transfer","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"fromWarehouseId","wireType":"u64"},{"in":"body","name":"toWarehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/transfer","reducer":"admin_transfer_stock"}}],[{"action":"buy","actor":"customer","do":"callAction","input":{"attribute":"data-buy-input","contains":"Headphones","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}}]],"do":"race","settleMs":5000},{"actor":"admin","do":"expectActionOutcome","outcome":"accepted"},{"actor":"customer","do":"expectActionOutcome","outcome":"accepted"},{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link","within":1000},{"actor":"admin","do":"expectNumber","in":{"contains":"Headphones","testid":"admin-item-row"},"plus":-1,"relativeTo":"direct-race-stock-before","testid":"admin-stock","within":12000},{"actor":"customer","do":"reload","settleMs":2000},{"actor":"customer","do":"ensureSignedIn","name":"direct-conserve","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","ifAvailable":true,"testid":"catalog-link","within":1000},{"actor":"customer","do":"expectNumber","in":{"contains":"Headphones","testid":"item-card"},"plus":-1,"relativeTo":"direct-race-stock-before","testid":"item-stock","within":12000},{"atLeast":34,"atMost":35,"do":"dbExpectStock","item":"Headphones","warehouse":"East"},{"atLeast":64,"atMost":65,"do":"dbExpectStock","item":"Headphones","warehouse":"West"},{"do":"dbExpectStock","item":"Headphones","plus":-1,"relativeTo":"direct-race-stock-before"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","contains":"Headphones","count":1,"do":"expect","testid":"order-item","within":10000}]}],"id":202,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"customer","do":"signUp","name":"direct-conserve"},{"actor":"admin","do":"click","testid":"admin-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.stock-transfers-features"],"role":"guarantee","source":"scenarios/02-server-actions.json","stablePackId":"ecommerce.inventory-operations"},{"checkGroupId":"order-owner","feature":{"actors":["owner","other"],"criteria":[{"id":"204a","steps":[{"action":"cancel","actor":"owner","do":"callAction","input":{"attribute":"data-cancel-input","contains":"Desk Lamp","testid":"order-item"},"namedAction":{"args":[0],"id":"cancel","params":[{"in":"path","name":"orderId","placeholder":":id","wireType":"u64"}],"path":"/api/orders/:id/cancel","reducer":"cancel_order"},"settleMs":2000},{"actor":"owner","do":"expectActionOutcome","outcome":"accepted"},{"action":"cancel","actor":"other","do":"callAction","from":"owner","input":{"attribute":"data-cancel-input","contains":"Keyboard","testid":"order-item"},"namedAction":{"args":[0],"id":"cancel","params":[{"in":"path","name":"orderId","placeholder":":id","wireType":"u64"}],"path":"/api/orders/:id/cancel","reducer":"cancel_order"},"settleMs":2000},{"actor":"other","do":"expectActionOutcome","outcome":"refused","routeProvenBy":"owner"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"direct-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner","do":"expect","ignoreCase":true,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"pending"}]}],"id":204,"setup":[{"actor":"owner","do":"signUp","name":"direct-owner"},{"actor":"other","do":"signUp","name":"direct-other"},{"action":"buy","actor":"owner","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"owner","do":"expectActionOutcome","outcome":"accepted"},{"action":"buy","actor":"owner","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"owner","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.operations-access-specifications","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.order-cancellation-features"],"role":"guarantee","source":"scenarios/02-server-actions.json","stablePackId":"ecommerce.operations-access"}],"id":"selected-source-046","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-server-actions.json"},{"checkGroups":[{"checkGroupId":"warehouse-transfer","feature":{"actors":["admin","visitor"],"criteria":[{"id":"2a","steps":[{"as":"product-East","do":"dbRecordStock","item":"Headphones","warehouse":"East"},{"as":"product-West","do":"dbRecordStock","item":"Headphones","warehouse":"West"},{"actor":"visitor","as":"transfer-item-before","do":"recordNumber","in":{"contains":"Headphones","testid":"item-card"},"testid":"item-stock"},{"actor":"admin","contains":"East","count":1,"do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","as":"transfer-east-before","do":"recordNumber","in":{"contains":"East","testid":"admin-warehouse-item"},"testid":"warehouse-total"},{"actor":"admin","contains":"West","count":1,"do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","as":"transfer-west-before","do":"recordNumber","in":{"contains":"West","testid":"admin-warehouse-item"},"testid":"warehouse-total"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-from","text":"East"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-to","text":"West"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-qty","text":"10"},{"actor":"admin","do":"click","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-submit"},{"actor":"admin","contains":"East","count":1,"do":"expect","testid":"admin-warehouse-item"},{"do":"dbExpectStock","item":"Headphones","plus":-10,"relativeTo":"product-East","warehouse":"East","within":10000},{"do":"dbExpectStock","item":"Headphones","plus":10,"relativeTo":"product-West","warehouse":"West","within":10000},{"actor":"admin","do":"expectNumber","in":{"contains":"East","testid":"admin-warehouse-item"},"plus":-10,"relativeTo":"transfer-east-before","testid":"warehouse-total","within":10000},{"actor":"admin","contains":"West","count":1,"do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","do":"expectNumber","in":{"contains":"West","testid":"admin-warehouse-item"},"plus":10,"relativeTo":"transfer-west-before","testid":"warehouse-total","within":10000},{"actor":"visitor","do":"expectNumber","in":{"contains":"Headphones","testid":"item-card"},"plus":0,"relativeTo":"transfer-item-before","testid":"item-stock","within":10000}]}],"id":2,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"}]},"packId":"ecommerce.l2.stock-transfers-features","role":"feature","source":"scenarios/02-strengthened.json","stablePackId":"ecommerce.inventory-operations"},{"checkGroupId":"operator-authorization","feature":{"actors":["admin","customer"],"criteria":[{"id":"201a","steps":[{"as":"authorized-transfer-east","do":"dbRecordStock","item":"Headphones","warehouse":"East"},{"as":"authorized-transfer-west","do":"dbRecordStock","item":"Headphones","warehouse":"West"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-from","text":"East"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-to","text":"West"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-qty","text":"25"},{"action":"transfer","actor":"admin","do":"callAction","input":{"attribute":"data-transfer-input","contains":"Headphones","testid":"admin-item-row"},"namedAction":{"args":[0,0,0,25],"id":"transfer","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"fromWarehouseId","wireType":"u64"},{"in":"body","name":"toWarehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/transfer","reducer":"admin_transfer_stock"},"settleMs":2000},{"actor":"admin","do":"expectActionOutcome","outcome":"accepted"},{"do":"dbExpectStock","item":"Headphones","plus":-25,"relativeTo":"authorized-transfer-east","warehouse":"East"},{"do":"dbExpectStock","item":"Headphones","plus":25,"relativeTo":"authorized-transfer-west","warehouse":"West"},{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link","within":1000},{"actor":"customer","do":"reload","settleMs":2000},{"actor":"customer","do":"ensureSignedIn","name":"not-operator","readyTestid":"current-user"},{"actor":"customer","as":"unauthorized-item-before","do":"recordNumber","in":{"contains":"Headphones","testid":"item-card"},"testid":"item-stock"},{"actor":"admin","contains":"East","count":1,"do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","as":"unauthorized-east-before","do":"recordNumber","in":{"contains":"East","testid":"admin-warehouse-item"},"testid":"warehouse-total"},{"actor":"admin","contains":"West","count":1,"do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","as":"unauthorized-west-before","do":"recordNumber","in":{"contains":"West","testid":"admin-warehouse-item"},"testid":"warehouse-total"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-from","text":"East"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-to","text":"West"},{"as":"refused-East","do":"dbRecordStock","item":"Headphones","warehouse":"East"},{"as":"refused-West","do":"dbRecordStock","item":"Headphones","warehouse":"West"},{"action":"transfer","actor":"customer","do":"callAction","from":"admin","input":{"attribute":"data-transfer-input","contains":"Headphones","testid":"admin-item-row"},"namedAction":{"args":[0,0,0,25],"id":"transfer","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"fromWarehouseId","wireType":"u64"},{"in":"body","name":"toWarehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/transfer","reducer":"admin_transfer_stock"},"settleMs":2000},{"actor":"customer","do":"expectActionOutcome","outcome":"refused","routeProvenBy":"admin"},{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link","within":1000},{"actor":"customer","do":"reload","settleMs":2000},{"actor":"customer","do":"ensureSignedIn","name":"not-operator","readyTestid":"current-user"},{"actor":"admin","contains":"East","count":1,"do":"expect","testid":"admin-warehouse-item"},{"do":"dbExpectStock","item":"Headphones","plus":0,"relativeTo":"refused-East","warehouse":"East"},{"do":"dbExpectStock","item":"Headphones","plus":0,"relativeTo":"refused-West","warehouse":"West"},{"actor":"admin","do":"expectNumber","in":{"contains":"East","testid":"admin-warehouse-item"},"plus":0,"relativeTo":"unauthorized-east-before","testid":"warehouse-total"},{"actor":"admin","contains":"West","count":1,"do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","do":"expectNumber","in":{"contains":"West","testid":"admin-warehouse-item"},"plus":0,"relativeTo":"unauthorized-west-before","testid":"warehouse-total"},{"actor":"customer","do":"expectNumber","in":{"contains":"Headphones","testid":"item-card"},"plus":0,"relativeTo":"unauthorized-item-before","testid":"item-stock"}]}],"id":201,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"customer","do":"signUp","name":"not-operator"},{"actor":"admin","do":"click","testid":"admin-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.operations-access-specifications","requiresFeatures":["ecommerce.l2.stock-transfers-features"],"role":"guarantee","source":"scenarios/02-strengthened.json","stablePackId":"ecommerce.operations-access"},{"checkGroupId":"operator-authorization","feature":{"actors":["admin","customer"],"criteria":[{"id":"201b","steps":[{"actor":"admin","do":"fill","in":{"contains":"Gaming Mouse","testid":"admin-item-row"},"testid":"price-input","text":"77.00"},{"action":"price","actor":"admin","do":"callAction","input":{"attribute":"data-price-input","contains":"Gaming Mouse","testid":"admin-item-row"},"namedAction":{"args":[0,1],"id":"price","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"price"}],"path":"/api/admin/price","reducer":"admin_change_price"},"settleMs":2000},{"actor":"admin","do":"expectActionOutcome","outcome":"accepted"},{"actor":"customer","do":"reload","settleMs":2000},{"actor":"customer","do":"ensureSignedIn","name":"not-operator","readyTestid":"current-user"},{"actor":"customer","do":"expectNumber","equals":77,"in":{"contains":"Gaming Mouse","testid":"item-card"},"testid":"item-price","within":10000},{"actor":"admin","do":"fill","in":{"contains":"Gaming Mouse","testid":"admin-item-row"},"testid":"price-input","text":"1.00"},{"action":"price","actor":"customer","do":"callAction","from":"admin","input":{"attribute":"data-price-input","contains":"Gaming Mouse","testid":"admin-item-row"},"namedAction":{"args":[0,1],"id":"price","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"price"}],"path":"/api/admin/price","reducer":"admin_change_price"},"settleMs":2000},{"actor":"customer","do":"expectActionOutcome","outcome":"refused","routeProvenBy":"admin"},{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link","within":1000},{"actor":"customer","do":"reload","settleMs":2000},{"actor":"customer","do":"ensureSignedIn","name":"not-operator","readyTestid":"current-user"},{"actor":"customer","do":"expectNumber","equals":77,"in":{"contains":"Gaming Mouse","testid":"item-card"},"testid":"item-price","within":10000}]}],"id":201,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"customer","do":"signUp","name":"not-operator"},{"actor":"admin","do":"click","testid":"admin-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.operations-access-specifications","requiresFeatures":["ecommerce.l2.price-history-features"],"role":"guarantee","source":"scenarios/02-strengthened.json","stablePackId":"ecommerce.operations-access"},{"checkGroupId":"stock-conservation","feature":{"actors":["admin","customer"],"criteria":[{"id":"202a","steps":[{"as":"product-East","do":"dbRecordStock","item":"Espresso Machine","warehouse":"East"},{"as":"product-West","do":"dbRecordStock","item":"Espresso Machine","warehouse":"West"},{"actor":"customer","as":"conservation-item-before","do":"recordNumber","in":{"contains":"Espresso Machine","testid":"item-card"},"testid":"item-stock"},{"actor":"admin","contains":"East","count":1,"do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","as":"conservation-east-before","do":"recordNumber","in":{"contains":"East","testid":"admin-warehouse-item"},"testid":"warehouse-total"},{"actor":"admin","contains":"West","count":1,"do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","as":"conservation-west-before","do":"recordNumber","in":{"contains":"West","testid":"admin-warehouse-item"},"testid":"warehouse-total"},{"actor":"admin","do":"fill","in":{"contains":"Espresso Machine","testid":"admin-item-row"},"testid":"transfer-from","text":"East"},{"actor":"admin","do":"fill","in":{"contains":"Espresso Machine","testid":"admin-item-row"},"testid":"transfer-to","text":"West"},{"actor":"admin","do":"fill","in":{"contains":"Espresso Machine","testid":"admin-item-row"},"testid":"transfer-qty","text":"17"},{"actor":"admin","do":"click","in":{"contains":"Espresso Machine","testid":"admin-item-row"},"testid":"transfer-submit"},{"actor":"admin","contains":"East","count":1,"do":"expect","testid":"admin-warehouse-item"},{"do":"dbExpectStock","item":"Espresso Machine","plus":-17,"relativeTo":"product-East","warehouse":"East","within":10000},{"do":"dbExpectStock","item":"Espresso Machine","plus":17,"relativeTo":"product-West","warehouse":"West","within":10000},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"expectNumber","in":{"contains":"East","testid":"admin-warehouse-item"},"plus":-17,"relativeTo":"conservation-east-before","testid":"warehouse-total","within":10000},{"actor":"admin","contains":"West","count":1,"do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","do":"expectNumber","in":{"contains":"West","testid":"admin-warehouse-item"},"plus":17,"relativeTo":"conservation-west-before","testid":"warehouse-total","within":10000},{"actor":"customer","do":"expectNumber","in":{"contains":"Espresso Machine","testid":"item-card"},"plus":0,"relativeTo":"conservation-item-before","testid":"item-stock","within":10000}]}],"id":202,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"customer","do":"signUp","name":"directional-stock"},{"actor":"admin","do":"click","testid":"admin-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","requiresFeatures":["ecommerce.l2.stock-transfers-features"],"role":"guarantee","source":"scenarios/02-strengthened.json","stablePackId":"ecommerce.inventory-operations"}],"id":"selected-source-047","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-strengthened.json"},{"checkGroups":[{"checkGroupId":"stock-transfer-overdraw","feature":{"actors":["admin","visitor"],"criteria":[{"id":"2c","steps":[{"as":"product-East","do":"dbRecordStock","item":"Headphones","warehouse":"East"},{"as":"product-West","do":"dbRecordStock","item":"Headphones","warehouse":"West"},{"actor":"visitor","as":"overdraw-item-before","do":"recordNumber","in":{"contains":"Headphones","testid":"item-card"},"testid":"item-stock"},{"actor":"admin","as":"overdraw-east-before","do":"recordNumber","in":{"contains":"East","testid":"admin-warehouse-item"},"testid":"warehouse-total"},{"actor":"admin","as":"overdraw-west-before","do":"recordNumber","in":{"contains":"West","testid":"admin-warehouse-item"},"testid":"warehouse-total"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-from","text":"East"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-to","text":"West"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-qty","text":"99999"},{"actor":"admin","do":"click","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-submit"},{"actor":"admin","do":"expect","testid":"order-error","within":6000},{"do":"dbExpectStock","item":"Headphones","plus":0,"relativeTo":"product-East","warehouse":"East"},{"do":"dbExpectStock","item":"Headphones","plus":0,"relativeTo":"product-West","warehouse":"West"},{"actor":"admin","do":"expectNumber","in":{"contains":"East","testid":"admin-warehouse-item"},"plus":0,"relativeTo":"overdraw-east-before","testid":"warehouse-total"},{"actor":"admin","do":"expectNumber","in":{"contains":"West","testid":"admin-warehouse-item"},"plus":0,"relativeTo":"overdraw-west-before","testid":"warehouse-total"},{"actor":"visitor","do":"expectNumber","in":{"contains":"Headphones","testid":"item-card"},"plus":0,"relativeTo":"overdraw-item-before","testid":"item-stock"}]}],"id":2,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","requiresFeatures":["ecommerce.l2.stock-transfers-features"],"role":"guarantee","source":"scenarios/02-transfer-overdraw.json"}],"id":"selected-source-048","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-transfer-overdraw.json"},{"checkGroups":[{"checkGroupId":"stock-transfers","feature":{"actors":["admin"],"criteria":[{"id":"2b","steps":[{"as":"product-East","do":"dbRecordStock","item":"Headphones","warehouse":"East"},{"as":"product-West","do":"dbRecordStock","item":"Headphones","warehouse":"West"},{"actor":"admin","contains":"East","count":1,"do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","as":"warehouse-east-before","do":"recordNumber","in":{"contains":"East","testid":"admin-warehouse-item"},"testid":"warehouse-total"},{"actor":"admin","contains":"West","count":1,"do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","as":"warehouse-west-before","do":"recordNumber","in":{"contains":"West","testid":"admin-warehouse-item"},"testid":"warehouse-total"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-from","text":"East"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-to","text":"West"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-qty","text":"10"},{"actor":"admin","do":"click","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-submit"},{"actor":"admin","contains":"East","count":1,"do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","do":"expectNumber","in":{"contains":"East","testid":"admin-warehouse-item"},"plus":-10,"relativeTo":"warehouse-east-before","testid":"warehouse-total","within":10000},{"actor":"admin","contains":"West","count":1,"do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","do":"expectNumber","in":{"contains":"West","testid":"admin-warehouse-item"},"plus":10,"relativeTo":"warehouse-west-before","testid":"warehouse-total","within":10000},{"do":"dbExpectStock","item":"Headphones","plus":-10,"relativeTo":"product-East","warehouse":"East","within":10000},{"do":"dbExpectStock","item":"Headphones","plus":10,"relativeTo":"product-West","warehouse":"West","within":10000}]}],"id":2,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","requiresFeatures":["ecommerce.l2.stock-transfers-features"],"role":"guarantee","source":"scenarios/02-transfer-totals.json"}],"id":"selected-source-049","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-transfer-totals.json"},{"checkGroups":[{"checkGroupId":"cart-expiration","feature":{"actors":["customer","watcher"],"criteria":[{"id":"304a","steps":[{"actor":"watcher","do":"wait","ms":310000},{"actor":"watcher","do":"reload","settleMs":1000},{"actor":"watcher","do":"expectNumber","in":{"contains":"Air Purifier","testid":"item-card"},"plus":0,"relativeTo":"before","testid":"item-stock","within":30000},{"actor":"customer","do":"openClient","settleMs":3000},{"actor":"customer","do":"expect","testid":"cart-expired-notice","within":10000},{"actor":"customer","do":"expectNumber","equals":0,"testid":"cart-count"}]}],"id":304,"setup":[{"actor":"customer","do":"signUp","name":"cart-expiry"},{"actor":"watcher","as":"before","do":"recordNumber","in":{"contains":"Air Purifier","testid":"item-card"},"testid":"item-stock"},{"actor":"customer","do":"click","in":{"contains":"Air Purifier","testid":"item-card"},"testid":"add-to-cart"},{"actor":"watcher","do":"reload","settleMs":1000},{"actor":"watcher","do":"expectNumber","in":{"contains":"Air Purifier","testid":"item-card"},"plus":-1,"relativeTo":"before","testid":"item-stock","within":10000},{"actor":"customer","do":"closeClient"}]},"packId":"ecommerce.l3.cart-expiration-features","role":"feature","source":"scenarios/03-cart-expiration.json","stablePackId":"ecommerce.l3.cart-expiration"}],"id":"selected-source-050","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/03-cart-expiration.json"},{"checkGroups":[{"checkGroupId":"scheduled-work-access","feature":{"actors":["admin","customer"],"criteria":[{"id":"317a","steps":[{"actor":"admin","do":"fill","testid":"schedule-restock-item","text":"Webcam"},{"actor":"admin","do":"fill","testid":"schedule-restock-warehouse","text":"West"},{"actor":"admin","do":"fill","testid":"schedule-restock-qty","text":"3"},{"actor":"admin","do":"fill","testid":"schedule-restock-delay","text":"180"},{"action":"scheduleRestock","actor":"customer","authentication":"actor","do":"callAction","from":"admin","input":{"attribute":"data-action-input","testid":"schedule-restock-submit"},"namedAction":{"args":["","",0,0],"id":"scheduleRestock","method":"POST","params":[{"in":"body","name":"item"},{"in":"body","name":"warehouse"},{"in":"body","name":"quantity"},{"in":"body","name":"delaySeconds"}],"path":"/api/admin/scheduled-restocks","reducer":"schedule_restock"},"settleMs":2000},{"actor":"customer","do":"expectActionOutcome","outcome":"refused"},{"actor":"customer","do":"replayAs","from":"admin","match":"DELETE","namedAction":{"args":[0],"id":"cancelScheduledRestock","method":"DELETE","params":[{"in":"path","name":"restockId","placeholder":"{restockId}","wireType":"u64"}],"path":"/api/admin/scheduled-restocks/{restockId}","reducer":"cancel_scheduled_restock"},"namedTarget":{"attribute":"data-entity-id","testid":"pending-restock-item","valueType":"string"},"settleMs":2000},{"actor":"customer","do":"expectReplayRejected"},{"actor":"admin","count":1,"do":"expect","testid":"pending-restock-item"},{"actor":"admin","do":"click","in":{"testid":"pending-restock-item"},"testid":"pending-restock-cancel"}]}],"id":317,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link"},{"actor":"customer","do":"signUp","name":"restock-outsider"},{"actor":"admin","do":"fill","testid":"schedule-restock-item","text":"Webcam"},{"actor":"admin","do":"fill","testid":"schedule-restock-warehouse","text":"West"},{"actor":"admin","do":"fill","testid":"schedule-restock-qty","text":"3"},{"actor":"admin","do":"fill","testid":"schedule-restock-delay","text":"180"},{"actor":"admin","do":"click","testid":"schedule-restock-submit"},{"actor":"admin","count":1,"do":"expect","testid":"pending-restock-item"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-access-specifications","requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"role":"guarantee","source":"scenarios/03-deferred-access.json","stablePackId":"ecommerce.l3.deferred-access"}],"id":"selected-source-051","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/03-deferred-access.json"},{"checkGroups":[{"checkGroupId":"restart-survival","feature":{"actors":["admin"],"criteria":[{"id":"311a","steps":[{"do":"dbExpectStock","item":"Air Purifier","plus":5,"relativeTo":"before","within":70000},{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link"},{"actor":"admin","do":"expect","testid":"schedule-restock-submit"},{"absent":true,"actor":"admin","do":"expect","testid":"pending-restock-item"}]}],"id":311,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link"},{"as":"ordinaryBefore","do":"dbRecordStock","item":"Air Purifier"},{"actor":"admin","do":"fill","testid":"schedule-restock-item","text":"Air Purifier"},{"actor":"admin","do":"fill","testid":"schedule-restock-warehouse","text":"East"},{"actor":"admin","do":"fill","testid":"schedule-restock-qty","text":"5"},{"actor":"admin","do":"fill","testid":"schedule-restock-delay","text":"45"},{"actor":"admin","do":"click","testid":"schedule-restock-submit"},{"actor":"admin","count":1,"do":"expect","testid":"pending-restock-item","within":10000},{"do":"dbExpectStock","item":"Air Purifier","plus":5,"relativeTo":"ordinaryBefore","within":70000},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link"},{"actor":"admin","do":"expect","testid":"schedule-restock-submit"},{"absent":true,"actor":"admin","do":"expect","testid":"pending-restock-item"},{"as":"before","do":"dbRecordStock","item":"Air Purifier"},{"actor":"admin","do":"fill","testid":"schedule-restock-item","text":"Air Purifier"},{"actor":"admin","do":"fill","testid":"schedule-restock-warehouse","text":"East"},{"actor":"admin","do":"fill","testid":"schedule-restock-qty","text":"5"},{"actor":"admin","do":"fill","testid":"schedule-restock-delay","text":"45"},{"actor":"admin","do":"click","testid":"schedule-restock-submit"},{"actor":"admin","count":1,"do":"expect","testid":"pending-restock-item","within":10000},{"do":"dbExpectStock","item":"Air Purifier","plus":0,"relativeTo":"before"},{"do":"restartBackend","settleMs":15000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-durability-specifications","requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"role":"guarantee","source":"scenarios/03-deferred-durability.json","stablePackId":"ecommerce.l3.deferred-durability"},{"checkGroupId":"restart-survival","feature":{"actors":["customer","watcher"],"criteria":[{"id":"314a","steps":[{"atMost":70000,"do":"expectElapsed","since":"pending-314"},{"actor":"watcher","do":"reload","settleMs":1000},{"atMost":70000,"do":"expectElapsed","since":"pending-314"},{"actor":"watcher","do":"expectNumber","in":{"contains":"Desk Lamp","testid":"item-card"},"plus":-1,"relativeTo":"before","testid":"item-stock","within":1000},{"atMost":70000,"do":"expectElapsed","since":"pending-314"},{"actor":"customer","do":"wait","ms":110000,"since":"pending-314-accepted"},{"actor":"watcher","do":"reload","settleMs":2000},{"actor":"watcher","do":"expectNumber","in":{"contains":"Desk Lamp","testid":"item-card"},"plus":0,"relativeTo":"before","testid":"item-stock","within":10000},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"durable-reservation","readyTestid":"current-user"},{"actor":"customer","do":"click","testid":"cart-toggle","unlessVisible":"cart-item"},{"actor":"customer","do":"expect","in":{"contains":"Desk Lamp","testid":"cart-item"},"testid":"cart-item-expired","within":10000}]}],"id":314,"setup":[{"actor":"customer","do":"signUp","name":"durable-reservation"},{"actor":"watcher","as":"before","do":"recordNumber","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock"},{"as":"pending-314","do":"recordTime"},{"actor":"customer","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"add-to-cart"},{"actor":"watcher","do":"reload","settleMs":1000},{"actor":"watcher","do":"expectNumber","in":{"contains":"Desk Lamp","testid":"item-card"},"plus":-1,"relativeTo":"before","testid":"item-stock","within":10000},{"as":"pending-314-accepted","do":"recordTime"},{"actor":"customer","do":"wait","ms":30000},{"do":"restartBackend","settleMs":15000},{"actor":"watcher","do":"reload","settleMs":3000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-durability-specifications","requiresFeatures":["ecommerce.l3.reservations-features"],"role":"guarantee","source":"scenarios/03-deferred-durability.json","stablePackId":"ecommerce.l3.deferred-durability"},{"checkGroupId":"restart-survival","feature":{"actors":["customer","staff"],"criteria":[{"id":"315a","steps":[{"actor":"customer","do":"wait","ms":75000,"since":"delivery-start-accepted"},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"durable-delivery","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"delivered","within":10000}]}],"id":315,"setup":[{"actor":"customer","do":"signUp","name":"durable-delivery"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link"},{"actor":"customer","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"buy-now"},{"actor":"staff","do":"reload","settleMs":1000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"queue-item"},{"actor":"staff","contains":"Keyboard","do":"expect","testid":"queue-item","within":10000},{"actor":"staff","do":"click","in":{"contains":"Keyboard","testid":"queue-item"},"testid":"ship-submit"},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"durable-delivery","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"shipped","within":10000},{"as":"delivery-start-accepted","do":"recordTime"},{"actor":"customer","do":"wait","ms":20000},{"do":"restartBackend","settleMs":15000},{"actor":"customer","do":"reload","settleMs":3000},{"actor":"customer","do":"ensureSignedIn","name":"durable-delivery","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-durability-specifications","requiresFeatures":["ecommerce.l3.order-delivery-features"],"role":"guarantee","source":"scenarios/03-deferred-durability.json","stablePackId":"ecommerce.l3.deferred-durability"},{"checkGroupId":"restart-survival","feature":{"actors":["customer","watcher"],"criteria":[{"id":"316a","steps":[{"atMost":250000,"do":"expectElapsed","since":"pending-316"},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"durable-cart","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"cart-toggle","unlessVisible":"cart-item"},{"atMost":250000,"do":"expectElapsed","since":"pending-316"},{"actor":"customer","do":"expectNumber","equals":1,"testid":"cart-count","within":1000},{"atMost":250000,"do":"expectElapsed","since":"pending-316"},{"actor":"customer","do":"wait","ms":310000,"since":"pending-316-accepted"},{"actor":"customer","do":"reload","settleMs":3000},{"actor":"customer","do":"ensureSignedIn","name":"durable-cart","readyTestid":"current-user"},{"actor":"customer","do":"click","testid":"cart-toggle"},{"actor":"customer","do":"expectNumber","equals":0,"testid":"cart-count","within":10000},{"actor":"customer","do":"expect","testid":"cart-expired-notice","within":10000},{"actor":"watcher","do":"reload","settleMs":1000},{"actor":"watcher","do":"expectNumber","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"plus":0,"relativeTo":"before","testid":"item-stock","within":10000}]}],"id":316,"setup":[{"actor":"customer","do":"signUp","name":"durable-cart"},{"actor":"watcher","as":"before","do":"recordNumber","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"item-stock"},{"as":"pending-316","do":"recordTime"},{"actor":"customer","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"add-to-cart"},{"actor":"watcher","do":"reload","settleMs":1000},{"actor":"watcher","do":"expectNumber","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"plus":-1,"relativeTo":"before","testid":"item-stock","within":10000},{"as":"pending-316-accepted","do":"recordTime"},{"actor":"customer","do":"wait","ms":120000},{"do":"restartBackend","settleMs":15000},{"actor":"watcher","do":"reload","settleMs":3000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-durability-specifications","requiresFeatures":["ecommerce.l3.cart-expiration-features"],"role":"guarantee","source":"scenarios/03-deferred-durability.json","stablePackId":"ecommerce.l3.deferred-durability"}],"id":"selected-source-052","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/03-deferred-durability.json"},{"checkGroups":[{"checkGroupId":"exactly-once","feature":{"actors":["admin","watcher"],"criteria":[{"id":"311a","steps":[{"actor":"watcher","do":"reload","settleMs":2000},{"do":"dbExpectStock","item":"Air Purifier","plus":5,"relativeTo":"before","within":45000},{"do":"restartBackend","settleMs":15000},{"actor":"watcher","do":"reload","settleMs":3000},{"actor":"watcher","do":"wait","ms":15000},{"do":"dbExpectStock","item":"Air Purifier","plus":5,"relativeTo":"before"}]}],"id":311,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link"},{"as":"before","do":"dbRecordStock","item":"Air Purifier"},{"actor":"admin","do":"fill","testid":"schedule-restock-item","text":"Air Purifier"},{"actor":"admin","do":"fill","testid":"schedule-restock-warehouse","text":"East"},{"actor":"admin","do":"fill","testid":"schedule-restock-qty","text":"5"},{"actor":"admin","do":"fill","testid":"schedule-restock-delay","text":"20"},{"actor":"admin","do":"click","testid":"schedule-restock-submit"},{"actor":"admin","count":1,"do":"expect","testid":"pending-restock-item","within":10000},{"do":"dbExpectStock","item":"Air Purifier","plus":0,"relativeTo":"before"},{"do":"restartBackend","settleMs":15000},{"actor":"watcher","do":"reload","settleMs":3000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-integrity-specifications","requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"role":"guarantee","source":"scenarios/03-deferred-integrity.json","stablePackId":"ecommerce.l3.deferred-integrity"},{"checkGroupId":"exactly-once","feature":{"actors":["customer","staff"],"criteria":[{"id":"312a","steps":[{"actor":"staff","do":"reload","settleMs":2000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"staff-link"},{"actor":"staff","contains":"Desk Lamp","count":1,"do":"expect","testid":"completed-order-item"},{"actor":"staff","do":"expect","ignoreCase":true,"in":{"contains":"Desk Lamp","testid":"completed-order-item"},"testid":"completed-order-status","value":"delivered"}]}],"id":312,"setup":[{"actor":"customer","do":"signUp","name":"once-delivery"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link"},{"actor":"customer","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"buy-now"},{"actor":"staff","do":"reload","settleMs":1000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"queue-item"},{"actor":"staff","contains":"Desk Lamp","do":"expect","testid":"queue-item","within":10000},{"actor":"staff","do":"click","in":{"contains":"Desk Lamp","testid":"queue-item"},"testid":"ship-submit"},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"once-delivery","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-status","value":"shipped","within":10000},{"actor":"customer","do":"wait","ms":70000},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"once-delivery","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-status","value":"delivered","within":15000},{"do":"restartBackend","settleMs":15000},{"actor":"staff","do":"reload","settleMs":3000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"staff-link","within":1000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-integrity-specifications","requiresFeatures":["ecommerce.l3.order-delivery-features"],"role":"guarantee","source":"scenarios/03-deferred-integrity.json","stablePackId":"ecommerce.l3.deferred-integrity"},{"checkGroupId":"stock-conservation","feature":{"actors":["customer","watcher"],"criteria":[{"id":"313a","steps":[{"actor":"watcher","do":"wait","ms":100000},{"actor":"watcher","do":"reload","settleMs":1000},{"actor":"watcher","do":"expectNumber","in":{"contains":"Keyboard","testid":"item-card"},"plus":0,"relativeTo":"before","testid":"item-stock","within":30000}]}],"id":313,"setup":[{"actor":"customer","do":"signUp","name":"conserve-expiry"},{"actor":"watcher","as":"before","do":"recordNumber","in":{"contains":"Keyboard","testid":"item-card"},"testid":"item-stock"},{"actor":"customer","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"actor":"watcher","do":"reload","settleMs":1000},{"actor":"watcher","do":"expectNumber","in":{"contains":"Keyboard","testid":"item-card"},"plus":-1,"relativeTo":"before","testid":"item-stock","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-integrity-specifications","requiresFeatures":["ecommerce.l3.reservations-features"],"role":"guarantee","source":"scenarios/03-deferred-integrity.json","stablePackId":"ecommerce.l3.deferred-integrity"},{"checkGroupId":"stock-conservation","feature":{"actors":["customer","watcher"],"criteria":[{"id":"314a","steps":[{"actor":"customer","do":"click","testid":"checkout-submit"},{"actor":"customer","do":"expectNumber","equals":0,"testid":"cart-count","within":10000},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"conserve-checkout","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle"},{"actor":"customer","contains":"Headphones","do":"expectElementCount","equals":1,"testid":"order-item","within":10000},{"actor":"watcher","do":"reload","settleMs":1000},{"actor":"watcher","do":"expectNumber","in":{"contains":"Headphones","testid":"item-card"},"plus":-1,"relativeTo":"before","testid":"item-stock","within":10000}]}],"id":314,"setup":[{"actor":"customer","do":"signUp","name":"conserve-checkout"},{"actor":"watcher","as":"before","do":"recordNumber","in":{"contains":"Headphones","testid":"item-card"},"testid":"item-stock"},{"actor":"customer","do":"click","in":{"contains":"Headphones","testid":"item-card"},"testid":"add-to-cart"},{"actor":"customer","do":"click","testid":"cart-toggle"},{"actor":"watcher","do":"reload","settleMs":1000},{"actor":"watcher","do":"expectNumber","in":{"contains":"Headphones","testid":"item-card"},"plus":-1,"relativeTo":"before","testid":"item-stock","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-integrity-specifications","requiresFeatures":["ecommerce.l3.reservations-features"],"role":"guarantee","source":"scenarios/03-deferred-integrity.json","stablePackId":"ecommerce.l3.deferred-integrity"}],"id":"selected-source-053","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/03-deferred-integrity.json"},{"checkGroups":[{"checkGroupId":"order-delivery","feature":{"actors":["customer","staff"],"criteria":[{"id":"303a","steps":[{"actor":"customer","do":"wait","ms":70000},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"delivery-live","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Gaming Mouse","testid":"order-item"},"testid":"order-status","value":"delivered","within":15000},{"actor":"staff","do":"reload","settleMs":1000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"completed-order-item"},{"actor":"staff","do":"expect","ignoreCase":true,"in":{"contains":"Gaming Mouse","testid":"completed-order-item"},"testid":"completed-order-status","value":"delivered","within":15000}]}],"id":303,"setup":[{"actor":"customer","do":"signUp","name":"delivery-live"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link"},{"actor":"customer","do":"click","in":{"contains":"Gaming Mouse","testid":"item-card"},"testid":"buy-now"},{"actor":"staff","do":"reload","settleMs":1000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"queue-item"},{"actor":"staff","contains":"Gaming Mouse","do":"expect","testid":"queue-item","within":10000},{"actor":"staff","do":"click","in":{"contains":"Gaming Mouse","testid":"queue-item"},"testid":"ship-submit"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"delivery-live","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Gaming Mouse","testid":"order-item"},"testid":"order-status","value":"shipped","within":10000}]},"packId":"ecommerce.l3.order-delivery-features","role":"feature","source":"scenarios/03-order-delivery.json","stablePackId":"ecommerce.l3.order-delivery"},{"checkGroupId":"order-delivery","feature":{"actors":["customer"],"criteria":[{"id":"305a","steps":[{"actor":"customer","do":"wait","ms":70000},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"delivery-cancel","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"cancelled"}]}],"id":305,"setup":[{"actor":"customer","do":"signUp","name":"delivery-cancel"},{"actor":"customer","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"buy-now"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle"},{"actor":"customer","do":"click","in":{"contains":"Keyboard","testid":"order-item"},"testid":"cancel-order"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"cancelled","within":10000}]},"packId":"ecommerce.l3.order-delivery-features","role":"feature","source":"scenarios/03-order-delivery.json","stablePackId":"ecommerce.l3.order-delivery"}],"id":"selected-source-054","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/03-order-delivery.json"},{"checkGroups":[{"checkGroupId":"reservations","feature":{"actors":["shopper","watcher"],"criteria":[{"id":"301a","steps":[{"actor":"shopper","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"add-to-cart"},{"actor":"watcher","do":"expectNumber","in":{"contains":"Desk Lamp","testid":"item-card"},"plus":-1,"relativeTo":"before","testid":"item-stock","within":10000}]}],"id":301,"setup":[{"actor":"shopper","do":"signUp","name":"reserve-stock"},{"actor":"watcher","as":"before","do":"recordNumber","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock"}]},"packId":"ecommerce.l3.reservations-features","role":"feature","source":"scenarios/03-reservations.json","stablePackId":"ecommerce.l3.reservations"},{"checkGroupId":"reservations","feature":{"actors":["shopper"],"criteria":[{"id":"305a","steps":[{"actor":"shopper","atLeast":1,"atMost":90,"do":"expectNumber","in":{"contains":"Gaming Mouse","testid":"cart-item"},"testid":"cart-reservation-timer"},{"actor":"shopper","as":"initial-countdown","do":"recordNumber","in":{"contains":"Gaming Mouse","testid":"cart-item"},"testid":"cart-reservation-timer"},{"actor":"shopper","do":"wait","ms":1000},{"actor":"shopper","comparison":"atMost","do":"expectNumber","in":{"contains":"Gaming Mouse","testid":"cart-item"},"plus":-1,"relativeTo":"initial-countdown","testid":"cart-reservation-timer","within":10000}]}],"id":305,"setup":[{"actor":"shopper","do":"signUp","name":"reserve-clock"},{"actor":"shopper","do":"click","in":{"contains":"Gaming Mouse","testid":"item-card"},"testid":"add-to-cart"},{"actor":"shopper","do":"click","testid":"cart-toggle"}]},"packId":"ecommerce.l3.reservations-features","role":"feature","source":"scenarios/03-reservations.json","stablePackId":"ecommerce.l3.reservations"},{"checkGroupId":"reservations","feature":{"actors":["shopper"],"criteria":[{"id":"306a","steps":[{"actor":"shopper","do":"click","testid":"checkout-submit"},{"actor":"shopper","do":"expectNumber","equals":0,"testid":"cart-count","within":10000},{"actor":"shopper","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"shopper","do":"click","testid":"orders-toggle"},{"actor":"shopper","contains":"Keyboard","count":1,"do":"expect","testid":"order-item"}]}],"id":306,"setup":[{"actor":"shopper","do":"signUp","name":"reserve-checkout"},{"actor":"shopper","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"actor":"shopper","do":"click","testid":"cart-toggle"}]},"packId":"ecommerce.l3.reservations-features","role":"feature","source":"scenarios/03-reservations.json","stablePackId":"ecommerce.l3.reservations"},{"checkGroupId":"reservations","feature":{"actors":["shopper"],"criteria":[{"id":"307a","steps":[{"actor":"shopper","do":"wait","ms":100000},{"actor":"shopper","do":"expect","in":{"contains":"Bluetooth Speaker","testid":"cart-item"},"testid":"cart-item-expired","within":15000}]}],"id":307,"setup":[{"actor":"shopper","do":"signUp","name":"reserve-expiry"},{"actor":"shopper","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"add-to-cart"},{"actor":"shopper","do":"click","testid":"cart-toggle"}]},"packId":"ecommerce.l3.reservations-features","role":"feature","source":"scenarios/03-reservations.json","stablePackId":"ecommerce.l3.reservations"},{"checkGroupId":"reservations","feature":{"actors":["shopper"],"criteria":[{"id":"308a","steps":[{"actor":"shopper","do":"wait","ms":40000},{"actor":"shopper","atLeast":35,"do":"expectNumber","in":{"contains":"Headphones","testid":"cart-item"},"testid":"cart-reservation-timer"},{"absent":true,"actor":"shopper","do":"expect","in":{"contains":"Headphones","testid":"cart-item"},"testid":"cart-item-expired"}]}],"id":308,"setup":[{"actor":"shopper","do":"signUp","name":"reserve-renew"},{"actor":"shopper","do":"click","in":{"contains":"Headphones","testid":"item-card"},"testid":"add-to-cart"},{"actor":"shopper","do":"wait","ms":60000},{"actor":"shopper","do":"click","in":{"contains":"Headphones","testid":"item-card"},"testid":"add-to-cart"},{"actor":"shopper","do":"click","testid":"cart-toggle"}]},"packId":"ecommerce.l3.reservations-features","role":"feature","source":"scenarios/03-reservations.json","stablePackId":"ecommerce.l3.reservations"}],"id":"selected-source-055","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/03-reservations.json"},{"checkGroups":[{"checkGroupId":"scheduled-restocks","feature":{"actors":["admin","watcher"],"criteria":[{"id":"305a","steps":[{"actor":"admin","as":"ledger-before","count":true,"do":"recordNumber","testid":"stock-ledger-entry"},{"actor":"admin","do":"fill","testid":"schedule-restock-item","text":"Keyboard"},{"actor":"admin","do":"fill","testid":"schedule-restock-warehouse","text":"West"},{"actor":"admin","do":"fill","testid":"schedule-restock-qty","text":"7"},{"actor":"admin","do":"fill","testid":"schedule-restock-delay","text":"15"},{"actor":"admin","do":"click","testid":"schedule-restock-submit"},{"actor":"admin","do":"wait","ms":25000},{"actor":"watcher","do":"reload","settleMs":2000},{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link"},{"actor":"watcher","do":"expectNumber","in":{"contains":"Keyboard","testid":"item-card"},"plus":7,"relativeTo":"before","testid":"item-stock","within":15000},{"absent":true,"actor":"admin","do":"expect","testid":"pending-restock-item"},{"actor":"admin","do":"expectElementCount","plus":1,"relativeTo":"ledger-before","testid":"stock-ledger-entry"}]}],"id":305,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link"},{"actor":"watcher","as":"before","do":"recordNumber","in":{"contains":"Keyboard","testid":"item-card"},"testid":"item-stock"}]},"packId":"ecommerce.l3.scheduled-restocks-features","role":"feature","source":"scenarios/03-scheduled-restock-apply.json","stablePackId":"ecommerce.l3.scheduled-restocks"}],"id":"selected-source-056","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/03-scheduled-restock-apply.json"},{"checkGroups":[{"checkGroupId":"scheduled-restocks","feature":{"actors":["admin","watcher"],"criteria":[{"id":"306a","steps":[{"actor":"admin","do":"fill","testid":"schedule-restock-item","text":"Desk Lamp"},{"actor":"admin","do":"fill","testid":"schedule-restock-warehouse","text":"East"},{"actor":"admin","do":"fill","testid":"schedule-restock-qty","text":"9"},{"actor":"admin","do":"fill","testid":"schedule-restock-delay","text":"15"},{"actor":"admin","do":"click","testid":"schedule-restock-submit"},{"actor":"admin","count":1,"do":"expect","testid":"pending-restock-item"},{"actor":"admin","do":"click","in":{"testid":"pending-restock-item"},"testid":"pending-restock-cancel"},{"actor":"admin","do":"wait","ms":25000},{"actor":"watcher","do":"reload","settleMs":2000},{"actor":"watcher","do":"expectNumber","in":{"contains":"Desk Lamp","testid":"item-card"},"plus":0,"relativeTo":"before","testid":"item-stock"}]}],"id":306,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link"},{"actor":"watcher","as":"before","do":"recordNumber","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock"}]},"packId":"ecommerce.l3.scheduled-restocks-features","role":"feature","source":"scenarios/03-scheduled-restock-cancel.json","stablePackId":"ecommerce.l3.scheduled-restocks"}],"id":"selected-source-057","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/03-scheduled-restock-cancel.json"},{"checkGroups":[{"checkGroupId":"scheduled-restocks","feature":{"actors":["admin"],"criteria":[{"id":"302a","steps":[{"actor":"admin","do":"fill","testid":"schedule-restock-item","text":"Webcam"},{"actor":"admin","do":"fill","testid":"schedule-restock-warehouse","text":"East"},{"actor":"admin","do":"fill","testid":"schedule-restock-qty","text":"7"},{"actor":"admin","do":"fill","testid":"schedule-restock-delay","text":"90"},{"actor":"admin","do":"click","testid":"schedule-restock-submit"},{"actor":"admin","count":1,"do":"expect","testid":"pending-restock-item","within":10000},{"actor":"admin","atLeast":1,"atMost":90,"do":"expectNumber","in":{"testid":"pending-restock-item"},"testid":"pending-restock-remaining"},{"actor":"admin","as":"initial-countdown","do":"recordNumber","in":{"testid":"pending-restock-item"},"testid":"pending-restock-remaining"},{"actor":"admin","do":"wait","ms":1000},{"actor":"admin","comparison":"atMost","do":"expectNumber","in":{"testid":"pending-restock-item"},"plus":-1,"relativeTo":"initial-countdown","testid":"pending-restock-remaining","within":10000},{"actor":"admin","do":"click","in":{"testid":"pending-restock-item"},"testid":"pending-restock-cancel"}]}],"id":302,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link"}]},"packId":"ecommerce.l3.scheduled-restocks-features","role":"feature","source":"scenarios/03-scheduled-restocks.json","stablePackId":"ecommerce.l3.scheduled-restocks"}],"id":"selected-source-058","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/03-scheduled-restocks.json"},{"checkGroups":[{"checkGroupId":"server-time","feature":{"actors":["admin","watcher"],"criteria":[{"id":"312a","steps":[{"actor":"watcher","do":"wait","ms":30000},{"actor":"watcher","do":"reload","settleMs":2000},{"atMost":100000,"do":"expectElapsed","since":"restock-start"},{"do":"dbExpectStock","item":"Espresso Machine","plus":0,"relativeTo":"before"},{"actor":"admin","count":1,"do":"expect","testid":"pending-restock-item"},{"atMost":100000,"do":"expectElapsed","since":"restock-start"},{"actor":"admin","do":"wait","ms":130000,"since":"restock-start-accepted"},{"do":"dbExpectStock","item":"Espresso Machine","plus":4,"relativeTo":"before","within":10000},{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link","within":1000},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link","within":1000},{"actor":"admin","do":"expect","testid":"schedule-restock-submit"},{"absent":true,"actor":"admin","do":"expect","testid":"pending-restock-item"}]}],"id":312,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link","within":1000},{"as":"before","do":"dbRecordStock","item":"Espresso Machine"},{"actor":"admin","do":"fill","testid":"schedule-restock-item","text":"Espresso Machine"},{"actor":"admin","do":"fill","testid":"schedule-restock-warehouse","text":"West"},{"actor":"admin","do":"fill","testid":"schedule-restock-qty","text":"4"},{"actor":"admin","do":"fill","testid":"schedule-restock-delay","text":"120"},{"as":"restock-start","do":"recordTime"},{"actor":"admin","do":"click","testid":"schedule-restock-submit"},{"actor":"admin","count":1,"do":"expect","testid":"pending-restock-item","within":10000},{"as":"restock-start-accepted","do":"recordTime"},{"do":"dbExpectStock","item":"Espresso Machine","plus":0,"relativeTo":"before"},{"do":"restartBackend","settleMs":15000},{"actor":"watcher","do":"reload","settleMs":3000},{"actor":"admin","do":"reload","settleMs":3000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link","within":1000},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link","within":1000},{"actor":"admin","do":"expect","testid":"schedule-restock-submit"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.l3.server-time-specifications","requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"role":"guarantee","source":"scenarios/03-server-time.json","stablePackId":"ecommerce.l3.server-time"},{"checkGroupId":"server-time","feature":{"actors":["customer","watcher"],"criteria":[{"id":"313a","steps":[{"actor":"watcher","do":"wait","ms":100000},{"actor":"watcher","do":"reload","settleMs":1000},{"actor":"watcher","do":"expectNumber","in":{"contains":"Air Purifier","testid":"item-card"},"plus":0,"relativeTo":"before","testid":"item-stock","within":30000}]}],"id":313,"setup":[{"actor":"customer","do":"signUp","name":"server-clock"},{"actor":"watcher","as":"before","do":"recordNumber","in":{"contains":"Air Purifier","testid":"item-card"},"testid":"item-stock"},{"actor":"customer","do":"click","in":{"contains":"Air Purifier","testid":"item-card"},"testid":"add-to-cart"},{"actor":"watcher","do":"reload","settleMs":1000},{"actor":"watcher","do":"expectNumber","in":{"contains":"Air Purifier","testid":"item-card"},"plus":-1,"relativeTo":"before","testid":"item-stock","within":10000},{"actor":"customer","do":"closeClient"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.l3.server-time-specifications","requiresFeatures":["ecommerce.l3.reservations-features"],"role":"guarantee","source":"scenarios/03-server-time.json","stablePackId":"ecommerce.l3.server-time"}],"id":"selected-source-059","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/03-server-time.json"},{"checkGroups":[{"checkGroupId":"account-state-recovery","feature":{"actors":["shopper","peer"],"criteria":[{"id":"105b","steps":[{"actor":"shopper","do":"setOffline","offline":true,"settleMs":3000},{"actor":"peer","do":"click","in":{"contains":"Headphones","testid":"item-card"},"testid":"add-to-cart"},{"actor":"shopper","do":"setOffline","offline":false,"settleMs":6000},{"actor":"shopper","contains":"pat","do":"expect","testid":"current-user"},{"actor":"shopper","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"},{"actor":"shopper","contains":"Keyboard","do":"expect","testid":"cart-item"},{"actor":"shopper","contains":"Headphones","do":"expect","testid":"cart-item"}]}],"id":105,"setup":[{"actor":"shopper","do":"signUp","name":"pat"},{"actor":"peer","do":"signIn","name":"pat"},{"actor":"shopper","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","requiresFeatures":["ecommerce.feature.accounts","ecommerce.feature.checkout"],"role":"guarantee","source":"scenarios/progression-account-state-reconnect.json"}],"id":"selected-source-060","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/progression-account-state-reconnect.json"},{"checkGroups":[{"checkGroupId":"account-state-recovery","feature":{"actors":["shopper"],"criteria":[{"id":"105a","steps":[{"actor":"shopper","do":"click","in":{"contains":"Headphones","testid":"item-card"},"testid":"add-to-cart"},{"actor":"shopper","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"},{"actor":"shopper","do":"click","settleMs":1500,"testid":"checkout-submit"},{"actor":"shopper","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"shopper","do":"click","ifAvailable":true,"settleMs":2500,"testid":"catalog-link"},{"actor":"shopper","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"actor":"shopper","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"},{"actor":"shopper","contains":"Keyboard","do":"expect","testid":"cart-item","within":10000},{"actor":"shopper","do":"reload","settleMs":4000},{"actor":"shopper","contains":"pat","do":"expect","testid":"current-user"},{"actor":"shopper","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"},{"actor":"shopper","contains":"Keyboard","do":"expect","testid":"cart-item"},{"actor":"shopper","do":"reload","settleMs":2500},{"actor":"shopper","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"shopper","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"shopper","contains":"Headphones","count":1,"do":"expect","testid":"order-item"},{"do":"restartBackend","settleMs":1000},{"actor":"shopper","do":"freshClient"},{"actor":"shopper-fresh","do":"signIn","name":"pat"},{"actor":"shopper-fresh","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"},{"actor":"shopper-fresh","contains":"Keyboard","count":1,"do":"expect","testid":"cart-item"},{"actor":"shopper-fresh","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"shopper-fresh","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"shopper-fresh","contains":"Headphones","count":1,"do":"expect","testid":"order-item"}]}],"id":105,"setup":[{"actor":"shopper","do":"signUp","name":"pat"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","requiresFeatures":["ecommerce.feature.accounts","ecommerce.feature.checkout"],"role":"guarantee","source":"scenarios/progression-account-state-reload.json"}],"id":"selected-source-061","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/progression-account-state-reload.json"},{"checkGroups":[{"checkGroupId":"automatic-reorder-access","feature":{"actors":["staff","customer"],"criteria":[{"id":"502c","steps":[{"absent":true,"actor":"customer","do":"expect","testid":"reorder-link"},{"actor":"staff","do":"fill","testid":"reorder-item","text":"Desk Lamp"},{"actor":"staff","do":"fill","testid":"reorder-threshold","text":"1"},{"actor":"staff","do":"fill","testid":"reorder-quantity","text":"9"},{"action":"saveReorderRule","actor":"customer","do":"callAction","from":"staff","input":{"attribute":"data-action-input","testid":"reorder-submit"},"namedAction":{"args":[0,1,9],"id":"saveReorderRule","method":"PUT","params":[{"in":"path","name":"itemId","placeholder":"{itemId}","wireType":"u64"},{"in":"body","name":"threshold"},{"in":"body","name":"quantity"}],"path":"/api/reorders/{itemId}","reducer":"save_reorder_rule"}},{"actor":"customer","do":"expectActionOutcome","outcome":"refused","routeProvenBy":"staff"},{"actor":"staff","do":"reload","settleMs":1000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"staff","do":"click","testid":"reorder-link","unlessVisible":"reorder-submit"},{"actor":"staff","attribute":"data-threshold","contains":"Desk Lamp","count":1,"do":"expect","testid":"reorder-rule-item","value":"2"},{"actor":"staff","attribute":"data-quantity","contains":"Desk Lamp","do":"expect","testid":"reorder-rule-item","value":"5"}]}],"id":502,"setup":[{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"expect","testid":"reorder-link","within":6000},{"actor":"staff","do":"click","testid":"reorder-link"},{"actor":"staff","do":"fill","testid":"reorder-item","text":"Desk Lamp"},{"actor":"staff","do":"fill","testid":"reorder-threshold","text":"2"},{"actor":"staff","do":"fill","testid":"reorder-quantity","text":"5"},{"action":"saveReorderRule","actor":"staff","do":"callAction","input":{"attribute":"data-action-input","testid":"reorder-submit"},"namedAction":{"args":[0,2,5],"id":"saveReorderRule","method":"PUT","params":[{"in":"path","name":"itemId","placeholder":"{itemId}","wireType":"u64"},{"in":"body","name":"threshold"},{"in":"body","name":"quantity"}],"path":"/api/reorders/{itemId}","reducer":"save_reorder_rule"}},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"staff","do":"reload","settleMs":1000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","testid":"reorder-link","unlessVisible":"reorder-submit"},{"actor":"staff","contains":"Desk Lamp","do":"expectElementCount","equals":1,"testid":"reorder-rule-item","within":10000},{"actor":"customer","do":"signUp","name":"reorder-customer"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.automatic-reorder"],"role":"guarantee","source":"scenarios/progression-automatic-reorder-access.json"}],"id":"selected-source-062","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-automatic-reorder-access.json"},{"checkGroups":[{"checkGroupId":"automatic-reorder-deduplication","feature":{"actors":["staff","admin","buyer-a","buyer-b","buyer-c"],"criteria":[{"id":"502b","steps":[{"action":"buy","actor":"buyer-b","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"buyer-b","do":"expectActionOutcome","outcome":"accepted"},{"action":"buy","actor":"buyer-c","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"buyer-c","do":"expectActionOutcome","outcome":"accepted"},{"do":"dbExpectStock","equals":0,"item":"Desk Lamp"},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link","within":1000},{"actor":"admin","do":"expect","testid":"schedule-restock-submit"},{"actor":"admin","do":"expectElementCount","equals":1,"testid":"pending-restock-item","within":10000},{"actor":"admin","attribute":"data-quantity","contains":"Desk Lamp","count":1,"do":"expect","testid":"pending-restock-item","value":"5"}]}],"id":502,"setup":[{"do":"dbSetStock","item":"Desk Lamp","quantity":2,"settleMs":2000,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":1,"settleMs":2000,"warehouse":"West"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"expect","testid":"reorder-link","within":6000},{"actor":"staff","do":"click","testid":"reorder-link"},{"actor":"staff","do":"fill","testid":"reorder-item","text":"Desk Lamp"},{"actor":"staff","do":"fill","testid":"reorder-threshold","text":"2"},{"actor":"staff","do":"fill","testid":"reorder-quantity","text":"5"},{"action":"saveReorderRule","actor":"staff","do":"callAction","input":{"attribute":"data-action-input","testid":"reorder-submit"},"namedAction":{"args":[0,2,5],"id":"saveReorderRule","method":"PUT","params":[{"in":"path","name":"itemId","placeholder":"{itemId}","wireType":"u64"},{"in":"body","name":"threshold"},{"in":"body","name":"quantity"}],"path":"/api/reorders/{itemId}","reducer":"save_reorder_rule"}},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"staff","do":"reload","settleMs":1000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","testid":"reorder-link","unlessVisible":"reorder-submit"},{"actor":"staff","contains":"Desk Lamp","do":"expectElementCount","equals":1,"testid":"reorder-rule-item","within":10000},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link","within":1000},{"actor":"buyer-a","do":"signUp","name":"reorder-a"},{"actor":"buyer-b","do":"signUp","name":"reorder-b"},{"actor":"buyer-c","do":"signUp","name":"reorder-c"},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link","within":1000},{"actor":"admin","do":"expect","testid":"schedule-restock-submit"},{"actor":"admin","do":"expectElementCount","equals":0,"testid":"pending-restock-item"},{"action":"buy","actor":"buyer-a","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"buyer-a","do":"expectActionOutcome","outcome":"accepted"},{"do":"dbExpectStock","equals":2,"item":"Desk Lamp"},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link","within":1000},{"actor":"admin","do":"expect","testid":"schedule-restock-submit"},{"actor":"admin","do":"expectElementCount","equals":1,"testid":"pending-restock-item","within":10000},{"actor":"admin","attribute":"data-quantity","contains":"Desk Lamp","count":1,"do":"expect","testid":"pending-restock-item","value":"5"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.automatic-reorder"],"role":"guarantee","source":"scenarios/progression-automatic-reorder-duplicate.json"}],"id":"selected-source-063","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-automatic-reorder-duplicate.json"},{"checkGroups":[{"checkGroupId":"automatic-reorder","feature":{"actors":["staff","admin","buyer-a"],"criteria":[{"id":"502a","steps":[{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link","within":1000},{"actor":"admin","do":"expect","testid":"schedule-restock-submit"},{"actor":"admin","do":"expectElementCount","equals":0,"testid":"pending-restock-item"},{"action":"buy","actor":"buyer-a","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"buyer-a","do":"expectActionOutcome","outcome":"accepted"},{"do":"dbExpectStock","equals":2,"item":"Desk Lamp"},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link","within":1000},{"actor":"admin","do":"expect","testid":"schedule-restock-submit"},{"actor":"admin","do":"expectElementCount","equals":1,"testid":"pending-restock-item","within":10000},{"actor":"admin","attribute":"data-quantity","contains":"Desk Lamp","count":1,"do":"expect","testid":"pending-restock-item","value":"5"}]}],"id":502,"setup":[{"do":"dbSetStock","item":"Desk Lamp","quantity":2,"settleMs":2000,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":1,"settleMs":2000,"warehouse":"West"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"expect","testid":"reorder-link","within":6000},{"actor":"staff","do":"click","testid":"reorder-link"},{"actor":"staff","do":"fill","testid":"reorder-item","text":"Desk Lamp"},{"actor":"staff","do":"fill","testid":"reorder-threshold","text":"2"},{"actor":"staff","do":"fill","testid":"reorder-quantity","text":"5"},{"action":"saveReorderRule","actor":"staff","do":"callAction","input":{"attribute":"data-action-input","testid":"reorder-submit"},"namedAction":{"args":[0,2,5],"id":"saveReorderRule","method":"PUT","params":[{"in":"path","name":"itemId","placeholder":"{itemId}","wireType":"u64"},{"in":"body","name":"threshold"},{"in":"body","name":"quantity"}],"path":"/api/reorders/{itemId}","reducer":"save_reorder_rule"}},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"staff","do":"reload","settleMs":1000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","testid":"reorder-link","unlessVisible":"reorder-submit"},{"actor":"staff","contains":"Desk Lamp","do":"expectElementCount","equals":1,"testid":"reorder-rule-item","within":10000},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link","within":1000},{"actor":"buyer-a","do":"signUp","name":"reorder-a"}]},"packId":"ecommerce.progression.automatic-reorder","requiresFeatures":["ecommerce.feature.purchasing"],"role":"feature","source":"scenarios/progression-automatic-reorder.json"}],"id":"selected-source-064","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-automatic-reorder.json"},{"checkGroups":[{"checkGroupId":"books-balance","feature":{"actors":["buyer","admin"],"criteria":[{"id":"107a","steps":[{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"expectNumber","plus":58,"relativeTo":"revenue-before","testid":"admin-revenue","within":10000}]},{"id":"107b","steps":[{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link"},{"actor":"admin","do":"expectNumber","in":{"contains":"Laptop Stand","testid":"admin-item-row"},"plus":-2,"relativeTo":"stand-before","testid":"admin-stock"},{"actor":"buyer","do":"freshClient"},{"actor":"buyer-fresh","do":"expectNumber","in":{"contains":"Laptop Stand","testid":"item-card"},"plus":-2,"relativeTo":"stand-before","testid":"item-stock"}]}],"id":107,"setup":[{"actor":"buyer","do":"signUp","name":"sam"},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","as":"revenue-before","do":"recordNumber","testid":"admin-revenue"},{"actor":"admin","as":"stand-before","do":"recordNumber","in":{"contains":"Laptop Stand","testid":"admin-item-row"},"testid":"admin-stock"},{"action":"buy","actor":"buyer","do":"callAction","input":{"attribute":"data-buy-input","contains":"Laptop Stand","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"buyer","do":"expectActionOutcome","outcome":"accepted"},{"action":"buy","actor":"buyer","do":"callAction","input":{"attribute":"data-buy-input","contains":"Laptop Stand","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"buyer","do":"expectActionOutcome","outcome":"accepted"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/progression-books-balance.json"}],"id":"selected-source-065","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/progression-books-balance.json"},{"checkGroups":[{"checkGroupId":"bundle-checkout","feature":{"actors":["admin","buyer"],"criteria":[{"id":"741a","steps":[{"actor":"buyer","do":"click","in":{"contains":"Checkout bundle","testid":"bundle-card"},"settleMs":500,"testid":"bundle-add-to-cart"},{"do":"dbExpectStock","equals":4,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":3,"item":"Desk Lamp","warehouse":"East"},{"actor":"buyer","do":"click","testid":"cart-toggle"},{"actor":"buyer","do":"click","testid":"checkout-submit"},{"actor":"buyer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"buyer","do":"click","testid":"orders-toggle"},{"actor":"buyer","contains":"Checkout bundle","count":1,"do":"expect","testid":"order-item","within":10000},{"actor":"buyer","do":"expectNumber","equals":75,"in":{"contains":"Checkout bundle","testid":"order-item"},"testid":"payment-amount","within":10000},{"do":"dbExpectStock","equals":4,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":3,"item":"Desk Lamp","warehouse":"East"},{"do":"dbExpectStock","equals":0,"item":"Keyboard","warehouse":"West"},{"do":"dbExpectStock","equals":0,"item":"Desk Lamp","warehouse":"West"}]}],"id":741,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"bundles-link","unlessVisible":"bundle-name-input"},{"actor":"buyer","do":"signUp","name":"bundle-checkout"},{"do":"dbSetStock","item":"Keyboard","quantity":6,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Keyboard","quantity":0,"settleMs":0,"warehouse":"West"},{"do":"dbSetStock","item":"Desk Lamp","quantity":4,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":0,"warehouse":"West"},{"actor":"admin","do":"fill","testid":"bundle-name-input","text":"Checkout bundle"},{"actor":"admin","do":"fill","testid":"bundle-price-input","text":"75.00"},{"actor":"admin","do":"fill","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"},{"actor":"admin","do":"click","testid":"bundle-save"},{"actor":"admin","contains":"Checkout bundle","do":"expect","testid":"bundle-card","within":10000},{"actor":"buyer","do":"click","testid":"bundles-link","unlessVisible":"bundle-card"}]},"packId":"ecommerce.feature.bundle-checkout","role":"feature","source":"scenarios/progression-bundle-checkout.json"},{"checkGroupId":"bundle-744","feature":{"actors":["admin","a","b"],"criteria":[{"id":"744a","steps":[{"action":"addBundleToCart","actors":["a","b"],"do":"callConcurrently","input":{"attribute":"data-bundle-input","contains":"Scarce bundle","testid":"bundle-card"},"namedAction":{"args":[0],"id":"addBundleToCart","params":[{"in":"body","name":"bundleId","wireType":"u64"}],"path":"/api/cart/bundles","reducer":"add_bundle_to_cart"},"settleMs":1000},{"accepted":1,"do":"expectCallOutcomes"},{"do":"dbExpectStock","equals":0,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":0,"item":"Desk Lamp","warehouse":"East"},{"actor":"a","do":"reload","settleMs":1000},{"actor":"b","do":"reload","settleMs":1000},{"actor":"a","do":"click","testid":"cart-toggle"},{"actor":"b","do":"click","testid":"cart-toggle"},{"actors":["a","b"],"contains":"Scarce bundle","do":"expectActorsWith","equals":1,"maxEach":1,"testid":"cart-item"},{"actor":"a","do":"click","ifAvailable":true,"in":{"contains":"Scarce bundle","testid":"cart-item"},"settleMs":500,"testid":"bundle-remove"},{"actor":"b","do":"click","ifAvailable":true,"in":{"contains":"Scarce bundle","testid":"cart-item"},"settleMs":500,"testid":"bundle-remove"},{"do":"dbExpectStock","equals":2,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":1,"item":"Desk Lamp","warehouse":"East"}]}],"id":744,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"bundles-link","unlessVisible":"bundle-name-input"},{"actor":"a","do":"signUp","name":"bundle-race-a"},{"actor":"b","do":"signUp","name":"bundle-race-b"},{"do":"dbSetStock","item":"Keyboard","quantity":2,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Keyboard","quantity":0,"settleMs":0,"warehouse":"West"},{"do":"dbSetStock","item":"Desk Lamp","quantity":1,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":0,"warehouse":"West"},{"actor":"admin","do":"fill","testid":"bundle-name-input","text":"Scarce bundle"},{"actor":"admin","do":"fill","testid":"bundle-price-input","text":"75.00"},{"actor":"admin","do":"fill","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"},{"actor":"admin","do":"click","testid":"bundle-save"},{"actor":"admin","contains":"Scarce bundle","do":"expect","testid":"bundle-card","within":10000},{"actor":"a","do":"click","testid":"bundles-link","unlessVisible":"bundle-card"},{"actor":"b","do":"click","testid":"bundles-link","unlessVisible":"bundle-card"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","requiresFeatures":["ecommerce.feature.bundle-checkout"],"role":"guarantee","source":"scenarios/progression-bundle-checkout.json"},{"checkGroupId":"bundle-745","feature":{"actors":["admin","buyer"],"criteria":[{"id":"745a","steps":[{"action":"addBundleToCart","actor":"buyer","do":"callAction","input":{"attribute":"data-bundle-input","contains":"Unavailable bundle","testid":"bundle-card"},"namedAction":{"args":[0],"id":"addBundleToCart","params":[{"in":"body","name":"bundleId","wireType":"u64"}],"path":"/api/cart/bundles","reducer":"add_bundle_to_cart"},"settleMs":1000},{"actor":"buyer","do":"expectActionOutcome","outcome":"application-refused"},{"do":"dbExpectStock","equals":2,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":0,"item":"Desk Lamp","warehouse":"East"},{"actor":"buyer","do":"click","testid":"cart-toggle"},{"absent":true,"actor":"buyer","contains":"Unavailable bundle","do":"expect","testid":"cart-item","within":10000}]}],"id":745,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"bundles-link","unlessVisible":"bundle-name-input"},{"actor":"buyer","do":"signUp","name":"bundle-partial"},{"do":"dbSetStock","item":"Keyboard","quantity":2,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Keyboard","quantity":0,"settleMs":0,"warehouse":"West"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":0,"warehouse":"West"},{"actor":"admin","do":"fill","testid":"bundle-name-input","text":"Unavailable bundle"},{"actor":"admin","do":"fill","testid":"bundle-price-input","text":"75.00"},{"actor":"admin","do":"fill","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"},{"actor":"admin","do":"click","testid":"bundle-save"},{"actor":"admin","contains":"Unavailable bundle","do":"expect","testid":"bundle-card","within":10000},{"actor":"buyer","do":"click","testid":"bundles-link","unlessVisible":"bundle-card"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","requiresFeatures":["ecommerce.feature.bundle-checkout"],"role":"guarantee","source":"scenarios/progression-bundle-checkout.json"},{"checkGroupId":"bundle-746","feature":{"actors":["admin","buyer"],"criteria":[{"id":"746a","steps":[{"actor":"buyer","do":"click","in":{"contains":"Expiring bundle","testid":"bundle-card"},"settleMs":500,"testid":"bundle-add-to-cart"},{"do":"dbExpectStock","equals":4,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":3,"item":"Desk Lamp","warehouse":"East"},{"do":"restartBackend","settleMs":1500},{"actor":"buyer","do":"wait","ms":92000},{"actor":"buyer","do":"reload","settleMs":1000},{"actor":"buyer","do":"ensureSignedIn","name":"bundle-expiry","readyTestid":"current-user"},{"actor":"buyer","do":"click","testid":"cart-toggle"},{"actor":"buyer","do":"expect","in":{"contains":"Expiring bundle","testid":"cart-item"},"testid":"cart-item-expired","within":10000},{"do":"dbExpectStock","equals":6,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":4,"item":"Desk Lamp","warehouse":"East"},{"do":"restartBackend","settleMs":1500},{"do":"dbExpectStock","equals":6,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":4,"item":"Desk Lamp","warehouse":"East"}]}],"id":746,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"bundles-link","unlessVisible":"bundle-name-input"},{"actor":"buyer","do":"signUp","name":"bundle-expiry"},{"do":"dbSetStock","item":"Keyboard","quantity":6,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Keyboard","quantity":0,"settleMs":0,"warehouse":"West"},{"do":"dbSetStock","item":"Desk Lamp","quantity":4,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":0,"warehouse":"West"},{"actor":"admin","do":"fill","testid":"bundle-name-input","text":"Expiring bundle"},{"actor":"admin","do":"fill","testid":"bundle-price-input","text":"75.00"},{"actor":"admin","do":"fill","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"},{"actor":"admin","do":"click","testid":"bundle-save"},{"actor":"admin","contains":"Expiring bundle","do":"expect","testid":"bundle-card","within":10000},{"actor":"buyer","do":"click","testid":"bundles-link","unlessVisible":"bundle-card"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","requiresFeatures":["ecommerce.feature.bundle-checkout"],"role":"guarantee","source":"scenarios/progression-bundle-checkout.json"},{"checkGroupId":"bundle-747","feature":{"actors":["admin","a","b"],"criteria":[{"id":"747a","steps":[{"action":"checkout","actors":["a","b"],"do":"callConcurrently","settleMs":1000},{"accepted":1,"do":"expectCallOutcomes"},{"actor":"a","do":"reload","settleMs":1000},{"actor":"a","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"a","do":"click","testid":"orders-toggle"},{"actor":"a","contains":"Repeated bundle","count":1,"do":"expect","testid":"order-item","within":10000},{"actor":"a","do":"expectNumber","equals":75,"in":{"contains":"Repeated bundle","testid":"order-item"},"testid":"payment-amount","within":10000},{"do":"dbExpectStock","equals":4,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":3,"item":"Desk Lamp","warehouse":"East"}]}],"id":747,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"bundles-link","unlessVisible":"bundle-name-input"},{"actor":"a","do":"signUp","name":"bundle-repeat"},{"actor":"b","do":"signIn","name":"bundle-repeat"},{"do":"dbSetStock","item":"Keyboard","quantity":6,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Keyboard","quantity":0,"settleMs":0,"warehouse":"West"},{"do":"dbSetStock","item":"Desk Lamp","quantity":4,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":0,"warehouse":"West"},{"actor":"admin","do":"fill","testid":"bundle-name-input","text":"Repeated bundle"},{"actor":"admin","do":"fill","testid":"bundle-price-input","text":"75.00"},{"actor":"admin","do":"fill","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"},{"actor":"admin","do":"click","testid":"bundle-save"},{"actor":"admin","contains":"Repeated bundle","do":"expect","testid":"bundle-card","within":10000},{"actor":"a","do":"click","testid":"bundles-link","unlessVisible":"bundle-card"},{"actor":"a","do":"click","in":{"contains":"Repeated bundle","testid":"bundle-card"},"settleMs":500,"testid":"bundle-add-to-cart"},{"do":"dbExpectStock","equals":4,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":3,"item":"Desk Lamp","warehouse":"East"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","requiresFeatures":["ecommerce.feature.bundle-checkout"],"role":"guarantee","source":"scenarios/progression-bundle-checkout.json"}],"id":"selected-source-066","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-bundle-checkout.json"},{"checkGroups":[{"checkGroupId":"bundle-returns","feature":{"actors":["admin","buyer","staff"],"criteria":[{"id":"742a","steps":[{"actor":"admin","do":"fill","testid":"bundle-name-input","text":"Historical bundle"},{"actor":"admin","do":"fill","testid":"bundle-price-input","text":"9.00"},{"actor":"admin","do":"fill","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":1},{\"item\":\"Desk Lamp\",\"quantity\":3}]"},{"actor":"admin","do":"click","testid":"bundle-save"},{"actor":"admin","contains":"Historical bundle","do":"expect","testid":"bundle-card","within":10000},{"actor":"buyer","do":"click","in":{"contains":"Historical bundle","testid":"order-item"},"testid":"return-bundle"},{"actor":"buyer","do":"expect","ignoreCase":true,"in":{"contains":"Historical bundle","testid":"order-item"},"testid":"order-status","value":"returned","within":10000},{"actor":"buyer","do":"expectNumber","equals":75,"in":{"contains":"Historical bundle","testid":"order-item"},"testid":"bundle-refund-amount","within":10000},{"do":"dbExpectStock","equals":6,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":4,"item":"Desk Lamp","warehouse":"East"},{"do":"dbExpectStock","equals":0,"item":"Keyboard","warehouse":"West"},{"do":"dbExpectStock","equals":0,"item":"Desk Lamp","warehouse":"West"}]}],"id":742,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"bundles-link","unlessVisible":"bundle-name-input"},{"actor":"buyer","do":"signUp","name":"Historical bundle"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"dbSetStock","item":"Keyboard","quantity":6,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Keyboard","quantity":0,"settleMs":0,"warehouse":"West"},{"do":"dbSetStock","item":"Desk Lamp","quantity":4,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":0,"warehouse":"West"},{"actor":"admin","do":"fill","testid":"bundle-name-input","text":"Historical bundle"},{"actor":"admin","do":"fill","testid":"bundle-price-input","text":"75.00"},{"actor":"admin","do":"fill","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"},{"actor":"admin","do":"click","testid":"bundle-save"},{"actor":"admin","contains":"Historical bundle","do":"expect","testid":"bundle-card","within":10000},{"actor":"buyer","do":"click","testid":"bundles-link","unlessVisible":"bundle-card"},{"actor":"buyer","do":"click","in":{"contains":"Historical bundle","testid":"bundle-card"},"testid":"bundle-add-to-cart"},{"actor":"buyer","do":"click","testid":"cart-toggle"},{"actor":"buyer","do":"click","testid":"checkout-submit"},{"actor":"buyer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"buyer","do":"click","testid":"orders-toggle"},{"actor":"buyer","contains":"Historical bundle","count":1,"do":"expect","testid":"order-item","within":10000},{"actor":"buyer","do":"expectNumber","equals":75,"in":{"contains":"Historical bundle","testid":"order-item"},"testid":"payment-amount","within":10000},{"do":"dbExpectStock","equals":4,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":3,"item":"Desk Lamp","warehouse":"East"},{"actor":"staff","do":"click","testid":"staff-link"},{"actor":"staff","contains":"Historical bundle","do":"expect","testid":"queue-item","within":10000},{"actor":"staff","do":"click","in":{"contains":"Historical bundle","testid":"queue-item"},"testid":"ship-submit"},{"actor":"buyer","do":"expect","ignoreCase":true,"in":{"contains":"Historical bundle","testid":"order-item"},"testid":"order-status","value":"shipped","within":10000}]},"packId":"ecommerce.feature.bundle-returns","role":"feature","source":"scenarios/progression-bundle-returns.json"},{"checkGroupId":"bundle-742","feature":{"actors":["admin","buyer","staff"],"criteria":[{"id":"742b","steps":[{"do":"restartBackend","settleMs":1500},{"actor":"buyer","do":"reload","settleMs":1000},{"actor":"buyer","do":"ensureSignedIn","name":"Historical bundle","readyTestid":"current-user"},{"actor":"buyer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"buyer","do":"click","testid":"orders-toggle"},{"action":"returnBundle","actor":"buyer","do":"callAction","input":{"attribute":"data-bundle-return-input","contains":"Historical bundle","testid":"order-item"},"namedAction":{"args":[0],"id":"returnBundle","params":[{"in":"path","name":"orderId","placeholder":":orderId","wireType":"u64"}],"path":"/api/bundle-orders/:orderId/return","reducer":"return_bundle"},"settleMs":1000},{"actor":"buyer","do":"reload","settleMs":1000},{"actor":"buyer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"buyer","do":"click","testid":"orders-toggle"},{"actor":"buyer","do":"expectNumber","equals":75,"in":{"contains":"Historical bundle","testid":"order-item"},"testid":"bundle-refund-amount","within":10000},{"do":"dbExpectStock","equals":6,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":4,"item":"Desk Lamp","warehouse":"East"}]}],"id":742,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"bundles-link","unlessVisible":"bundle-name-input"},{"actor":"buyer","do":"signUp","name":"Historical bundle"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"dbSetStock","item":"Keyboard","quantity":6,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Keyboard","quantity":0,"settleMs":0,"warehouse":"West"},{"do":"dbSetStock","item":"Desk Lamp","quantity":4,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":0,"warehouse":"West"},{"actor":"admin","do":"fill","testid":"bundle-name-input","text":"Historical bundle"},{"actor":"admin","do":"fill","testid":"bundle-price-input","text":"75.00"},{"actor":"admin","do":"fill","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"},{"actor":"admin","do":"click","testid":"bundle-save"},{"actor":"admin","contains":"Historical bundle","do":"expect","testid":"bundle-card","within":10000},{"actor":"buyer","do":"click","testid":"bundles-link","unlessVisible":"bundle-card"},{"actor":"buyer","do":"click","in":{"contains":"Historical bundle","testid":"bundle-card"},"testid":"bundle-add-to-cart"},{"actor":"buyer","do":"click","testid":"cart-toggle"},{"actor":"buyer","do":"click","testid":"checkout-submit"},{"actor":"buyer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"buyer","do":"click","testid":"orders-toggle"},{"actor":"buyer","contains":"Historical bundle","count":1,"do":"expect","testid":"order-item","within":10000},{"actor":"buyer","do":"expectNumber","equals":75,"in":{"contains":"Historical bundle","testid":"order-item"},"testid":"payment-amount","within":10000},{"do":"dbExpectStock","equals":4,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":3,"item":"Desk Lamp","warehouse":"East"},{"actor":"staff","do":"click","testid":"staff-link"},{"actor":"staff","contains":"Historical bundle","do":"expect","testid":"queue-item","within":10000},{"actor":"staff","do":"click","in":{"contains":"Historical bundle","testid":"queue-item"},"testid":"ship-submit"},{"actor":"buyer","do":"expect","ignoreCase":true,"in":{"contains":"Historical bundle","testid":"order-item"},"testid":"order-status","value":"shipped","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","requiresFeatures":["ecommerce.feature.bundle-returns"],"role":"guarantee","source":"scenarios/progression-bundle-returns.json"},{"checkGroupId":"bundle-748","feature":{"actors":["admin","buyer","staff","other"],"criteria":[{"id":"748a","steps":[{"action":"returnBundle","actor":"other","do":"callAction","from":"buyer","input":{"attribute":"data-bundle-return-input","contains":"Private bundle","testid":"order-item"},"namedAction":{"args":[0],"id":"returnBundle","params":[{"in":"path","name":"orderId","placeholder":":orderId","wireType":"u64"}],"path":"/api/bundle-orders/:orderId/return","reducer":"return_bundle"},"settleMs":1000},{"actor":"other","do":"expectActionOutcome","outcome":"application-refused"},{"actor":"buyer","do":"reload","settleMs":1000},{"actor":"buyer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"buyer","do":"click","testid":"orders-toggle"},{"actor":"buyer","do":"expect","ignoreCase":true,"in":{"contains":"Private bundle","testid":"order-item"},"testid":"order-status","value":"shipped","within":10000},{"do":"dbExpectStock","equals":4,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":3,"item":"Desk Lamp","warehouse":"East"},{"actor":"buyer","do":"click","in":{"contains":"Private bundle","testid":"order-item"},"testid":"return-bundle"},{"actor":"buyer","do":"expectNumber","equals":75,"in":{"contains":"Private bundle","testid":"order-item"},"testid":"bundle-refund-amount","within":10000},{"do":"dbExpectStock","equals":6,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":4,"item":"Desk Lamp","warehouse":"East"}]}],"id":748,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"bundles-link","unlessVisible":"bundle-name-input"},{"actor":"buyer","do":"signUp","name":"Private bundle"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"dbSetStock","item":"Keyboard","quantity":6,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Keyboard","quantity":0,"settleMs":0,"warehouse":"West"},{"do":"dbSetStock","item":"Desk Lamp","quantity":4,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":0,"warehouse":"West"},{"actor":"admin","do":"fill","testid":"bundle-name-input","text":"Private bundle"},{"actor":"admin","do":"fill","testid":"bundle-price-input","text":"75.00"},{"actor":"admin","do":"fill","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"},{"actor":"admin","do":"click","testid":"bundle-save"},{"actor":"admin","contains":"Private bundle","do":"expect","testid":"bundle-card","within":10000},{"actor":"buyer","do":"click","testid":"bundles-link","unlessVisible":"bundle-card"},{"actor":"buyer","do":"click","in":{"contains":"Private bundle","testid":"bundle-card"},"testid":"bundle-add-to-cart"},{"actor":"buyer","do":"click","testid":"cart-toggle"},{"actor":"buyer","do":"click","testid":"checkout-submit"},{"actor":"buyer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"buyer","do":"click","testid":"orders-toggle"},{"actor":"buyer","contains":"Private bundle","count":1,"do":"expect","testid":"order-item","within":10000},{"actor":"buyer","do":"expectNumber","equals":75,"in":{"contains":"Private bundle","testid":"order-item"},"testid":"payment-amount","within":10000},{"do":"dbExpectStock","equals":4,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":3,"item":"Desk Lamp","warehouse":"East"},{"actor":"staff","do":"click","testid":"staff-link"},{"actor":"staff","contains":"Private bundle","do":"expect","testid":"queue-item","within":10000},{"actor":"staff","do":"click","in":{"contains":"Private bundle","testid":"queue-item"},"testid":"ship-submit"},{"actor":"buyer","do":"expect","ignoreCase":true,"in":{"contains":"Private bundle","testid":"order-item"},"testid":"order-status","value":"shipped","within":10000},{"actor":"other","do":"signUp","name":"bundle-other"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","requiresFeatures":["ecommerce.feature.bundle-returns"],"role":"guarantee","source":"scenarios/progression-bundle-returns.json"}],"id":"selected-source-067","scenario":{"level":6,"writeUrlPattern":null},"source":"scenarios/progression-bundle-returns.json"},{"checkGroups":[{"checkGroupId":"cart","feature":{"actors":["quantity","checkout"],"criteria":[{"id":"4a","steps":[{"actor":"quantity","do":"click","in":{"contains":"Headphones","testid":"item-card"},"testid":"add-to-cart"},{"actor":"quantity","do":"wait","ms":800},{"actor":"quantity","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"quantity","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"quantity","do":"click","in":{"contains":"Headphones","testid":"item-card"},"testid":"add-to-cart"},{"actor":"quantity","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"},{"actor":"quantity","contains":"Headphones","count":1,"do":"expect","testid":"cart-item"},{"actor":"quantity","do":"expectNumber","equals":2,"in":{"contains":"Headphones","testid":"cart-item"},"testid":"cart-quantity"}]}],"id":4,"setup":[{"actor":"quantity","do":"signUp","name":"cart-quantity"},{"actor":"checkout","do":"signUp","name":"cart-checkout"}]},"packId":"ecommerce.feature.cart","role":"feature","source":"scenarios/progression-cart-checkout.json","stablePackId":"ecommerce.feature.cart-checkout"},{"checkGroupId":"cart","feature":{"actors":["quantity","checkout"],"criteria":[{"id":"4d","steps":[{"actor":"checkout","do":"expectNumber","equals":100,"in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock"},{"actor":"checkout","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"add-to-cart"},{"actor":"checkout","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"},{"actor":"checkout","do":"expectNumber","equals":1,"testid":"cart-count"},{"actor":"checkout","do":"click","testid":"checkout-submit"},{"actor":"checkout","do":"wait","ms":1500},{"actor":"checkout","do":"reload","settleMs":2500},{"actor":"checkout","do":"ensureSignedIn","name":"cart-checkout","readyTestid":"current-user"},{"actor":"checkout","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"},{"actor":"checkout","do":"expectNumber","equals":0,"testid":"cart-count"},{"actor":"checkout","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"checkout","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"checkout","do":"expectNumber","equals":99,"in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock"},{"actor":"checkout","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"checkout","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"checkout","contains":"Desk Lamp","count":1,"do":"expect","testid":"order-item"}]}],"id":4,"setup":[{"actor":"quantity","do":"signUp","name":"cart-quantity"},{"actor":"checkout","do":"signUp","name":"cart-checkout"}]},"packId":"ecommerce.feature.checkout","role":"feature","source":"scenarios/progression-cart-checkout.json","stablePackId":"ecommerce.feature.cart-checkout"}],"id":"selected-source-068","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/progression-cart-checkout.json"},{"checkGroups":[{"checkGroupId":"cart-recovery","feature":{"actors":["available-shopper","partial-shopper"],"criteria":[{"id":"503a","steps":[{"actor":"available-shopper","do":"reload","settleMs":1000},{"actor":"available-shopper","do":"ensureSignedIn","name":"cart-recovery-available","readyTestid":"current-user"},{"actor":"available-shopper","do":"expect","testid":"expired-cart"},{"actor":"available-shopper","do":"expectNumber","equals":0,"testid":"cart-count"},{"actor":"available-shopper","as":"restore-available-shopper","do":"recordNumber","in":{"contains":"Keyboard","testid":"item-card"},"testid":"item-stock"},{"actor":"available-shopper","do":"click","in":{"testid":"expired-cart"},"testid":"restore-cart"},{"actor":"available-shopper","do":"click","testid":"cart-toggle"},{"actor":"available-shopper","contains":"Keyboard","do":"expect","testid":"cart-item"},{"absent":true,"actor":"available-shopper","do":"expect","testid":"cart-restore-warning"},{"actor":"available-shopper","do":"expectNumber","equals":1,"in":{"contains":"Keyboard","testid":"cart-item"},"testid":"cart-quantity"},{"actor":"available-shopper","do":"reload","settleMs":1000},{"actor":"available-shopper","do":"ensureSignedIn","name":"cart-recovery-available","readyTestid":"current-user"},{"actor":"available-shopper","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"available-shopper","do":"click","testid":"catalog-link","unlessVisible":"item-card"},{"actor":"available-shopper","do":"expectNumber","in":{"contains":"Keyboard","testid":"item-card"},"plus":-1,"relativeTo":"restore-available-shopper","testid":"item-stock"}]}],"id":503,"setup":[{"actor":"available-shopper","do":"signUp","name":"cart-recovery-available"},{"actor":"available-shopper","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"actor":"partial-shopper","do":"signUp","name":"cart-recovery-partial"},{"actor":"partial-shopper","do":"click","in":{"contains":"Gaming Mouse","testid":"item-card"},"testid":"add-to-cart"},{"actor":"partial-shopper","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"add-to-cart"},{"actor":"available-shopper","do":"wait","ms":310000}]},"packId":"ecommerce.progression.cart-recovery","role":"feature","source":"scenarios/progression-cart-recovery.json"},{"checkGroupId":"cart-recovery","feature":{"actors":["available-shopper","partial-shopper"],"criteria":[{"id":"503b","steps":[{"actor":"partial-shopper","do":"reload","settleMs":1000},{"actor":"partial-shopper","do":"ensureSignedIn","name":"cart-recovery-partial","readyTestid":"current-user"},{"actor":"partial-shopper","do":"expect","testid":"expired-cart"},{"actor":"partial-shopper","do":"expectNumber","equals":0,"testid":"cart-count"},{"actor":"partial-shopper","as":"restore-partial-shopper","do":"recordNumber","in":{"contains":"Gaming Mouse","testid":"item-card"},"testid":"item-stock"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":2000,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":2000,"warehouse":"West"},{"actor":"partial-shopper","do":"click","in":{"testid":"expired-cart"},"testid":"restore-cart"},{"actor":"partial-shopper","do":"click","testid":"cart-toggle"},{"actor":"partial-shopper","contains":"Gaming Mouse","do":"expect","testid":"cart-item"},{"absent":true,"actor":"partial-shopper","contains":"Desk Lamp","do":"expect","testid":"cart-item"},{"actor":"partial-shopper","contains":"Desk Lamp","do":"expect","testid":"cart-restore-warning"},{"actor":"partial-shopper","do":"expectNumber","equals":1,"in":{"contains":"Gaming Mouse","testid":"cart-item"},"testid":"cart-quantity"},{"actor":"partial-shopper","do":"reload","settleMs":1000},{"actor":"partial-shopper","do":"ensureSignedIn","name":"cart-recovery-partial","readyTestid":"current-user"},{"actor":"partial-shopper","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"partial-shopper","do":"click","testid":"catalog-link","unlessVisible":"item-card"},{"actor":"partial-shopper","do":"expectNumber","in":{"contains":"Gaming Mouse","testid":"item-card"},"plus":-1,"relativeTo":"restore-partial-shopper","testid":"item-stock"}]}],"id":503,"setup":[{"actor":"available-shopper","do":"signUp","name":"cart-recovery-available"},{"actor":"available-shopper","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"actor":"partial-shopper","do":"signUp","name":"cart-recovery-partial"},{"actor":"partial-shopper","do":"click","in":{"contains":"Gaming Mouse","testid":"item-card"},"testid":"add-to-cart"},{"actor":"partial-shopper","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"add-to-cart"},{"actor":"available-shopper","do":"wait","ms":310000}]},"packId":"ecommerce.progression.cart-recovery","role":"feature","source":"scenarios/progression-cart-recovery.json"}],"id":"selected-source-069","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-cart-recovery.json"},{"checkGroups":[{"checkGroupId":"catalog-management","feature":{"actors":["admin","visitor"],"criteria":[{"id":"622a","steps":[{"actor":"visitor","contains":"Travel Mug","do":"expect","testid":"item-card","within":10000}]},{"id":"622b","steps":[{"actor":"visitor","do":"openItem","item":"Travel Mug","unlessVisible":"item-variant"},{"actor":"visitor","contains":"Black","do":"expectElementCount","equals":1,"testid":"item-variant","within":10000},{"actor":"visitor","contains":"Silver","do":"expectElementCount","equals":1,"testid":"item-variant","within":10000}]}],"id":622,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"fill","testid":"catalog-name","text":"Travel Mug"},{"actor":"admin","do":"fill","testid":"catalog-category","text":"Kitchen"},{"actor":"admin","do":"fill","testid":"catalog-price","text":"24.00"},{"actor":"admin","do":"fill","testid":"catalog-variants","text":"Black, Silver"},{"actor":"admin","do":"click","testid":"catalog-save"},{"actor":"visitor","do":"reload","settleMs":2500},{"actor":"visitor","do":"fill","enter":true,"testid":"search-input","text":"Travel Mug"}]},"packId":"ecommerce.progression.catalog-management","role":"feature","source":"scenarios/progression-catalog-management.json"}],"id":"selected-source-070","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-catalog-management.json"},{"checkGroups":[{"checkGroupId":"checkout-crash-integrity","feature":{"actors":["a","b","c"],"criteria":[{"id":"910a","steps":[{"do":"expectCrashCheckout","from":"database-crash-observation","verdict":"atomicity"},{"do":"expectCrashCheckout","from":"application-crash-observation","verdict":"atomicity"}]}],"id":910,"setup":[{"actor":"a","do":"signUp","name":"database"},{"account":"{user:database}","as":"database-baseline-before","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"a","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"account":"{user:database}","as":"database-baseline-prepared","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"a","do":"confirmCheckout","namedAction":{"args":[],"id":"checkout","path":"/api/checkout","reducer":"checkout"}},{"before":"database-baseline-before","do":"dbExpectCheckout","prepared":"database-baseline-prepared","quantity":1},{"actor":"a","do":"reload","settleMs":0},{"account":"{user:database}","as":"database-before","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"a","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"account":"{user:database}","as":"database-prepared","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"a","as":"database-crash-observation","before":"database-before","do":"crashCheckout","namedAction":{"args":[],"id":"checkout","path":"/api/checkout","reducer":"checkout"},"offsetMs":0,"prepared":"database-prepared","quantity":1,"requests":16,"target":"database"},{"actor":"b","do":"signUp","name":"application"},{"account":"{user:application}","as":"application-baseline-before","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"b","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"account":"{user:application}","as":"application-baseline-prepared","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"b","do":"confirmCheckout","namedAction":{"args":[],"id":"checkout","path":"/api/checkout","reducer":"checkout"}},{"before":"application-baseline-before","do":"dbExpectCheckout","prepared":"application-baseline-prepared","quantity":1},{"actor":"b","do":"reload","settleMs":0},{"account":"{user:application}","as":"application-before","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"b","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"account":"{user:application}","as":"application-prepared","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"b","as":"application-crash-observation","before":"application-before","do":"crashCheckout","namedAction":{"args":[],"id":"checkout","path":"/api/checkout","reducer":"checkout"},"offsetMs":0,"prepared":"application-prepared","quantity":1,"requests":16,"reuseCombinedFrom":"database-crash-observation","target":"application"},{"actor":"c","do":"signUp","name":"recovered"},{"account":"{user:recovered}","as":"recovered-before","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"c","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"account":"{user:recovered}","as":"recovered-prepared","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"c","do":"click","testid":"cart-toggle"},{"actor":"c","do":"click","testid":"checkout-submit"},{"actor":"c","do":"expectNumber","equals":0,"testid":"cart-count","within":10000},{"actor":"c","do":"click","ifAvailable":true,"testid":"overlay-close"},{"before":"recovered-before","do":"dbExpectCheckout","prepared":"recovered-prepared","quantity":1}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","requiresFeatures":["ecommerce.feature.checkout"],"role":"guarantee","source":"scenarios/progression-checkout-crash.json"},{"checkGroupId":"checkout-crash-durability","feature":{"actors":["a","b","c"],"criteria":[{"id":"910b","steps":[{"do":"expectCrashCheckout","from":"database-crash-observation","verdict":"durability"},{"do":"expectCrashCheckout","from":"application-crash-observation","verdict":"durability"}]}],"id":910,"setup":[{"actor":"a","do":"signUp","name":"database"},{"account":"{user:database}","as":"database-baseline-before","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"a","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"account":"{user:database}","as":"database-baseline-prepared","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"a","do":"confirmCheckout","namedAction":{"args":[],"id":"checkout","path":"/api/checkout","reducer":"checkout"}},{"before":"database-baseline-before","do":"dbExpectCheckout","prepared":"database-baseline-prepared","quantity":1},{"actor":"a","do":"reload","settleMs":0},{"account":"{user:database}","as":"database-before","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"a","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"account":"{user:database}","as":"database-prepared","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"a","as":"database-crash-observation","before":"database-before","do":"crashCheckout","namedAction":{"args":[],"id":"checkout","path":"/api/checkout","reducer":"checkout"},"offsetMs":0,"prepared":"database-prepared","quantity":1,"requests":16,"target":"database"},{"actor":"b","do":"signUp","name":"application"},{"account":"{user:application}","as":"application-baseline-before","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"b","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"account":"{user:application}","as":"application-baseline-prepared","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"b","do":"confirmCheckout","namedAction":{"args":[],"id":"checkout","path":"/api/checkout","reducer":"checkout"}},{"before":"application-baseline-before","do":"dbExpectCheckout","prepared":"application-baseline-prepared","quantity":1},{"actor":"b","do":"reload","settleMs":0},{"account":"{user:application}","as":"application-before","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"b","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"account":"{user:application}","as":"application-prepared","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"b","as":"application-crash-observation","before":"application-before","do":"crashCheckout","namedAction":{"args":[],"id":"checkout","path":"/api/checkout","reducer":"checkout"},"offsetMs":0,"prepared":"application-prepared","quantity":1,"requests":16,"reuseCombinedFrom":"database-crash-observation","target":"application"},{"actor":"c","do":"signUp","name":"recovered"},{"account":"{user:recovered}","as":"recovered-before","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"c","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"account":"{user:recovered}","as":"recovered-prepared","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"c","do":"click","testid":"cart-toggle"},{"actor":"c","do":"click","testid":"checkout-submit"},{"actor":"c","do":"expectNumber","equals":0,"testid":"cart-count","within":10000},{"actor":"c","do":"click","ifAvailable":true,"testid":"overlay-close"},{"before":"recovered-before","do":"dbExpectCheckout","prepared":"recovered-prepared","quantity":1}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","requiresFeatures":["ecommerce.feature.checkout"],"role":"guarantee","source":"scenarios/progression-checkout-crash.json"}],"id":"selected-source-071","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/progression-checkout-crash.json"},{"checkGroups":[{"checkGroupId":"payment-records","feature":{"actors":["tab1","tab2","owner"],"criteria":[{"id":"623a","steps":[{"actor":"owner-fresh","do":"expect","ignoreCase":true,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"payment-status","value":"paid","within":10000},{"actor":"owner-fresh","do":"expectNumber","in":{"contains":"Desk Lamp","testid":"order-item"},"plus":0,"relativeTo":"payment-total","testid":"payment-amount"}]}],"id":623,"setup":[{"actor":"tab1","do":"signUp","name":"payment-record"},{"actor":"tab2","do":"signIn","name":"payment-record"},{"actor":"owner","do":"signIn","name":"payment-record"},{"actor":"owner","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"add-to-cart"},{"actor":"owner","do":"click","testid":"cart-toggle"},{"actor":"owner","as":"payment-total","do":"recordNumber","testid":"cart-total"},{"action":"checkout","actors":["tab1","tab2"],"do":"callConcurrently","settleMs":5000},{"do":"expectCallOutcomes"},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"payment-record"},{"actor":"owner-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner-fresh","do":"click","testid":"orders-toggle"},{"actor":"owner-fresh","contains":"Desk Lamp","do":"expectElementCount","equals":1,"testid":"order-item","within":10000}]},"packId":"ecommerce.progression.payment-records","role":"feature","source":"scenarios/progression-core-business.json"},{"checkGroupId":"payment-deduplication","feature":{"actors":["tab1","tab2","owner"],"criteria":[{"id":"623b","steps":[{"actor":"owner-fresh","do":"expectElementCount","equals":1,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"payment-record","within":10000}]}],"id":623,"setup":[{"actor":"tab1","do":"signUp","name":"payment-record"},{"actor":"tab2","do":"signIn","name":"payment-record"},{"actor":"owner","do":"signIn","name":"payment-record"},{"actor":"owner","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"add-to-cart"},{"actor":"owner","do":"click","testid":"cart-toggle"},{"actor":"owner","as":"payment-total","do":"recordNumber","testid":"cart-total"},{"action":"checkout","actors":["tab1","tab2"],"do":"callConcurrently","settleMs":5000},{"do":"expectCallOutcomes"},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"payment-record"},{"actor":"owner-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner-fresh","do":"click","testid":"orders-toggle"},{"actor":"owner-fresh","contains":"Desk Lamp","do":"expectElementCount","equals":1,"testid":"order-item","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","requiresFeatures":["ecommerce.progression.payment-records"],"role":"guarantee","source":"scenarios/progression-core-business.json"}],"id":"selected-source-072","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-core-business.json"},{"checkGroups":[{"checkGroupId":"customer-profile","feature":{"actors":["owner","privateOwner","other"],"criteria":[{"id":"620c","steps":[{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"owner","do":"click","testid":"profile-link","unlessVisible":"profile-address-summary"},{"actor":"owner","contains":"14 Market Street {user:profilemarker}","do":"expect","testid":"profile-address-summary"}]}],"id":620,"setup":[{"actor":"owner","do":"signUp","name":"profile-owner"},{"actor":"owner","do":"click","testid":"profile-link"},{"actor":"owner","do":"fill","testid":"profile-name","text":"Avery Stone"},{"actor":"owner","do":"fill","testid":"profile-address","text":"14 Market Street {user:profilemarker}"},{"actor":"owner","do":"click","testid":"profile-save"}]},"packId":"ecommerce.progression.customer-profile","role":"feature","source":"scenarios/progression-customer-profile.json"},{"checkGroupId":"customer-profile-reload","feature":{"actors":["owner","privateOwner","other"],"criteria":[{"id":"620a","steps":[{"actor":"owner","do":"reload","settleMs":2500},{"actor":"owner","do":"ensureSignedIn","name":"profile-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"profile-link"},{"actor":"owner","contains":"14 Market Street {user:profilemarker}","do":"expect","testid":"profile-address-summary"},{"do":"restartBackend","settleMs":1000},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"profile-owner"},{"actor":"owner-fresh","do":"click","testid":"profile-link"},{"actor":"owner-fresh","contains":"14 Market Street {user:profilemarker}","do":"expect","testid":"profile-address-summary"}]}],"id":620,"setup":[{"actor":"owner","do":"signUp","name":"profile-owner"},{"actor":"owner","do":"click","testid":"profile-link"},{"actor":"owner","do":"fill","testid":"profile-name","text":"Avery Stone"},{"actor":"owner","do":"fill","testid":"profile-address","text":"14 Market Street {user:profilemarker}"},{"actor":"owner","do":"click","testid":"profile-save"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","requiresFeatures":["ecommerce.progression.customer-profile"],"role":"guarantee","source":"scenarios/progression-customer-profile.json"},{"checkGroupId":"customer-profile-privacy","feature":{"actors":["owner","privateOwner","other"],"criteria":[{"id":"620b","steps":[{"actor":"privateOwner","do":"reload","settleMs":0},{"actor":"privateOwner","do":"signUp","name":"profile-private-owner"},{"actor":"privateOwner","do":"click","testid":"profile-link"},{"actor":"privateOwner","do":"fill","testid":"profile-name","text":"Avery Stone"},{"actor":"privateOwner","do":"fill","testid":"profile-address","text":"14 Market Street {user:profilemarker}"},{"actor":"privateOwner","do":"click","testid":"profile-save"},{"actor":"privateOwner","do":"reload","settleMs":1000},{"actor":"privateOwner","do":"ensureSignedIn","name":"profile-private-owner","readyTestid":"current-user"},{"actor":"privateOwner","do":"click","testid":"profile-link"},{"actor":"privateOwner","contains":"14 Market Street {user:profilemarker}","do":"expect","testid":"profile-address-summary"},{"actor":"privateOwner","contains":"14 Market Street {user:profilemarker}","do":"expectReceived","within":10000},{"actor":"other","do":"reload","settleMs":0},{"actor":"other","do":"signUp","name":"profile-other"},{"actor":"other","do":"click","testid":"profile-link"},{"absent":true,"actor":"other","contains":"14 Market Street {user:profilemarker}","do":"expect","testid":"profile-address-summary"},{"actor":"other","contains":"14 Market Street {user:profilemarker}","do":"expectNotReceived"}]}],"id":620,"setup":[{"actor":"owner","do":"signUp","name":"profile-owner"},{"actor":"owner","do":"click","testid":"profile-link"},{"actor":"owner","do":"fill","testid":"profile-name","text":"Avery Stone"},{"actor":"owner","do":"fill","testid":"profile-address","text":"14 Market Street {user:profilemarker}"},{"actor":"owner","do":"click","testid":"profile-save"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.customer-profile"],"role":"guarantee","source":"scenarios/progression-customer-profile.json"}],"id":"selected-source-073","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/progression-customer-profile.json"},{"checkGroups":[{"checkGroupId":"delivery-notification-delivery","feature":{"actors":["owner","other","staff"],"criteria":[{"id":"501a","steps":[{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"delivery-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"notifications-toggle","unlessVisible":"notifications-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"notifications-panel","value":"false","within":10000},{"actor":"owner","contains":"Desk Lamp","do":"expectElementCount","equals":1,"testid":"notification-item","within":10000}]}],"id":501,"setup":[{"actor":"owner","do":"signUp","name":"delivery-owner"},{"actor":"other","do":"signUp","name":"delivery-other"},{"actor":"owner","do":"click","testid":"notification-settings","unlessVisible":"notification-order"},{"actor":"owner","do":"click","testid":"notification-order"},{"actor":"owner","do":"click","testid":"notification-save"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"catalog-link"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"action":"buy","actor":"owner","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"owner","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"delivery-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"notifications-toggle","unlessVisible":"notifications-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"notifications-panel","value":"false","within":10000},{"actor":"owner","contains":"Desk Lamp","do":"expectElementCount","equals":0,"testid":"notification-item"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"delivery-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"as":"notification-shipped","do":"recordTime"},{"action":"ship","actor":"staff","do":"callAction","from":"owner","input":{"attribute":"data-ship-input","contains":"Desk Lamp","testid":"order-item"},"namedAction":{"args":[0],"id":"ship","params":[{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/fulfilment/ship","reducer":"ship_order"}},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"as":"notification-shipped-accepted","do":"recordTime"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"delivery-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"notifications-toggle","unlessVisible":"notifications-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"notifications-panel","value":"false","within":10000},{"atMost":40000,"do":"expectElapsed","since":"notification-shipped"},{"actor":"owner","contains":"Desk Lamp","do":"expectElementCount","equals":0,"testid":"notification-item"},{"actor":"owner","do":"wait","ms":75000,"since":"notification-shipped-accepted"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"delivery-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner","do":"expect","ignoreCase":true,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-status","value":"delivered"}]},"packId":"ecommerce.progression.delivery-notifications","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"feature","source":"scenarios/progression-delivery-notifications.json"},{"checkGroupId":"delivery-notification-privacy","feature":{"actors":["owner","other","staff"],"criteria":[{"id":"501b","steps":[{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"delivery-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"notifications-toggle","unlessVisible":"notifications-panel"},{"actor":"owner","contains":"Desk Lamp","do":"expectElementCount","equals":1,"testid":"notification-item","within":10000},{"actor":"other","do":"click","testid":"notifications-toggle","unlessVisible":"notifications-panel"},{"actor":"other","attribute":"aria-busy","do":"expect","testid":"notifications-panel","value":"false","within":10000},{"absent":true,"actor":"other","contains":"Desk Lamp","do":"expect","testid":"notification-item"}]}],"id":501,"setup":[{"actor":"owner","do":"signUp","name":"delivery-owner"},{"actor":"other","do":"signUp","name":"delivery-other"},{"actor":"owner","do":"click","testid":"notification-settings","unlessVisible":"notification-order"},{"actor":"owner","do":"click","testid":"notification-order"},{"actor":"owner","do":"click","testid":"notification-save"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"catalog-link"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"action":"buy","actor":"owner","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"owner","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"delivery-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"notifications-toggle","unlessVisible":"notifications-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"notifications-panel","value":"false","within":10000},{"actor":"owner","contains":"Desk Lamp","do":"expectElementCount","equals":0,"testid":"notification-item"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"delivery-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"as":"notification-shipped","do":"recordTime"},{"action":"ship","actor":"staff","do":"callAction","from":"owner","input":{"attribute":"data-ship-input","contains":"Desk Lamp","testid":"order-item"},"namedAction":{"args":[0],"id":"ship","params":[{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/fulfilment/ship","reducer":"ship_order"}},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"as":"notification-shipped-accepted","do":"recordTime"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"delivery-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"notifications-toggle","unlessVisible":"notifications-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"notifications-panel","value":"false","within":10000},{"atMost":40000,"do":"expectElapsed","since":"notification-shipped"},{"actor":"owner","contains":"Desk Lamp","do":"expectElementCount","equals":0,"testid":"notification-item"},{"actor":"owner","do":"wait","ms":75000,"since":"notification-shipped-accepted"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"delivery-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner","do":"expect","ignoreCase":true,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-status","value":"delivered"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.delivery-notifications","ecommerce.progression.fulfilment-queue"],"role":"guarantee","source":"scenarios/progression-delivery-notifications.json"}],"id":"selected-source-074","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-delivery-notifications.json"},{"checkGroups":[{"checkGroupId":"faceted-search","feature":{"actors":["visitor"],"criteria":[{"id":"401a","steps":[{"actor":"visitor","contains":"Coffee Grinder","do":"expectElementCount","equals":1,"in":{"testid":"search-results"},"testid":"item-card","within":10000},{"actor":"visitor","do":"click","testid":"in-stock-filter"},{"actor":"visitor","do":"click","ifAvailable":true,"testid":"filter-apply"},{"actor":"visitor","contains":"Coffee Grinder","do":"waitUntilAbsent","in":{"testid":"search-results"},"testid":"item-card","within":10000},{"actor":"visitor","contains":"Air Purifier","do":"expectElementCount","equals":1,"in":{"testid":"search-results"},"testid":"item-card"},{"absent":true,"actor":"visitor","contains":"USB Cable","do":"expect","in":{"testid":"search-results"},"testid":"item-card"},{"absent":true,"actor":"visitor","contains":"Desk Lamp","do":"expect","in":{"testid":"search-results"},"testid":"item-card"},{"absent":true,"actor":"visitor","contains":"Espresso Machine","do":"expect","in":{"testid":"search-results"},"testid":"item-card"},{"absent":true,"actor":"visitor","contains":"Gaming Mouse","do":"expect","in":{"testid":"search-results"},"testid":"item-card"}]}],"id":401,"setup":[{"do":"dbSetStock","item":"Coffee Grinder","quantity":0,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Coffee Grinder","quantity":0,"settleMs":1000,"warehouse":"West"},{"actor":"visitor","do":"reload","settleMs":1000},{"actor":"visitor","do":"fill","testid":"category-filter","text":"Home"},{"actor":"visitor","do":"fill","testid":"minimum-price","text":"50"},{"actor":"visitor","do":"fill","testid":"maximum-price","text":"200"},{"actor":"visitor","do":"click","ifAvailable":true,"testid":"filter-apply"}]},"packId":"ecommerce.progression.faceted-search","role":"feature","source":"scenarios/progression-faceted-filters.json"}],"id":"selected-source-075","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/progression-faceted-filters.json"},{"checkGroups":[{"checkGroupId":"faceted-search","feature":{"actors":["visitor"],"criteria":[{"id":"402a","steps":[{"actor":"visitor","do":"expectSequence","equals":["Air Purifier","Bluetooth Speaker","Coffee Grinder","Desk Lamp","Espresso Machine","Gaming Mouse","Headphones","Induction Cooktop","Keyboard","Laptop Stand"],"in":{"testid":"search-results"},"testid":"item-name"},{"actor":"visitor","do":"click","testid":"search-next-page"},{"actor":"visitor","do":"expectSequence","equals":["Mirrorless Camera","USB Cable","Webcam"],"in":{"testid":"search-results"},"testid":"item-name"},{"actor":"visitor","do":"click","testid":"search-previous-page"},{"actor":"visitor","do":"expectSequence","equals":["Air Purifier","Bluetooth Speaker","Coffee Grinder","Desk Lamp","Espresso Machine","Gaming Mouse","Headphones","Induction Cooktop","Keyboard","Laptop Stand"],"in":{"testid":"search-results"},"testid":"item-name"}]}],"id":402,"setup":[{"actor":"visitor","do":"fill","testid":"minimum-price","text":"1"},{"actor":"visitor","do":"click","ifAvailable":true,"testid":"filter-apply"}]},"packId":"ecommerce.progression.faceted-search","role":"feature","source":"scenarios/progression-faceted-pagination.json"}],"id":"selected-source-076","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/progression-faceted-pagination.json"},{"checkGroups":[{"checkGroupId":"managed-support-privacy","feature":{"actors":["owner","other","staff"],"criteria":[{"id":"613b","steps":[{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"managed-private-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","contains":"Private managed case {user:casemarker}","do":"expect","testid":"support-ticket"},{"actor":"owner","contains":"Private managed case {user:casemarker}","do":"expectReceived","within":10000},{"absent":true,"actor":"other","contains":"Private managed case {user:casemarker}","do":"expect","testid":"support-ticket"},{"actor":"other","contains":"Private managed case {user:casemarker}","do":"expectNotReceived"},{"actor":"owner","do":"fill","in":{"contains":"Private managed case {user:casemarker}","testid":"support-ticket"},"testid":"support-reply","text":"Owner-only update"},{"actor":"owner","do":"click","in":{"contains":"Private managed case {user:casemarker}","testid":"support-ticket"},"testid":"support-reply-submit"},{"actor":"other","do":"replayAs","from":"owner","match":"Owner-only update","namedAction":{"args":[0,"Owner-only update"],"id":"replySupport","method":"POST","params":[{"in":"path","name":"ticketId","placeholder":":id","wireType":"u64"},{"in":"body","name":"body"}],"path":"/api/support/:id/replies","reducer":"reply_support"},"namedTarget":{"attribute":"data-entity-id","contains":"Private managed case {user:casemarker}","testid":"support-ticket","valueType":"string"},"settleMs":1500},{"actor":"other","allowNotFound":true,"do":"expectReplayRejected"},{"actor":"staff","do":"reload","settleMs":2000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee"},{"actor":"staff","contains":"Private managed case {user:casemarker}","do":"expect","testid":"support-ticket"},{"actor":"staff","contains":"Owner-only update","do":"expectElementCount","equals":1,"testid":"support-reply-item","within":10000},{"actor":"staff","contains":"Owner-only update","do":"expectReceived","within":10000},{"actor":"other","contains":"Owner-only update","do":"expectNotReceived"}]}],"id":613,"setup":[{"actor":"owner","do":"signUp","name":"managed-private-owner"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-email","text":"managed-private@example.com"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Private managed case {user:casemarker}"},{"actor":"owner","do":"fill","testid":"support-message","text":"Private case details."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","do":"expect","nonEmpty":true,"testid":"support-reference"},{"actor":"other","do":"signUp","name":"managed-private-other"},{"actor":"other","do":"click","testid":"support-link"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.managed-support"],"role":"guarantee","source":"scenarios/progression-managed-support-privacy.json"}],"id":"selected-source-077","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/progression-managed-support-privacy.json"},{"checkGroups":[{"checkGroupId":"managed-support","feature":{"actors":["owner","staff"],"criteria":[{"id":"613c","steps":[{"actor":"staff","do":"fill","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-status-input","text":"in progress"},{"actor":"staff","do":"click","in":{"contains":"Shared managed case","testid":"support-ticket"},"settleMs":1500,"testid":"support-update"},{"actor":"staff","do":"fill","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-reply","text":"Case received."},{"actor":"staff","do":"click","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-reply-submit"},{"actor":"owner","do":"reload","settleMs":2000},{"actor":"owner","do":"ensureSignedIn","name":"managed-shared-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","contains":"in progress","do":"expect","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-status"},{"actor":"owner","contains":"Case received.","do":"expect","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-reply-item"},{"actor":"owner","do":"fill","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-reply","text":"Thank you."},{"actor":"owner","do":"click","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-reply-submit"},{"actor":"staff","do":"reload","settleMs":2000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee"},{"actor":"staff","contains":"Thank you.","do":"expect","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-reply-item"}]}],"id":613,"setup":[{"actor":"owner","do":"signUp","name":"managed-shared-owner"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-email","text":"managed-shared@example.com"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Shared managed case"},{"actor":"owner","do":"fill","testid":"support-message","text":"Please investigate this case."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","do":"expect","nonEmpty":true,"testid":"support-reference"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee"}]},"packId":"ecommerce.progression.managed-support","role":"feature","source":"scenarios/progression-managed-support-shared.json"},{"checkGroupId":"managed-support","feature":{"actors":["owner","staff"],"criteria":[{"id":"613a","steps":[{"actor":"staff","do":"fill","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-status-input","text":"open"},{"actor":"staff","do":"click","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-update"},{"actor":"staff","contains":"open","do":"expect","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-status"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"managed-shared-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"support-link","unlessVisible":"support-ticket"},{"actor":"owner","contains":"open","do":"expect","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-status"},{"actor":"staff","do":"fill","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-status-input","text":"in progress"},{"actor":"staff","do":"click","in":{"contains":"Shared managed case","testid":"support-ticket"},"settleMs":1500,"testid":"support-update"},{"actor":"staff","do":"fill","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-reply","text":"We are investigating."},{"actor":"staff","do":"click","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-reply-submit"},{"actor":"owner","contains":"in progress","do":"expect","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-status","within":10000},{"actor":"owner","contains":"We are investigating.","do":"expect","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-reply-item","within":10000},{"actor":"owner","do":"fill","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-reply","text":"Thank you for the update."},{"actor":"owner","do":"click","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-reply-submit"},{"actor":"staff","contains":"Thank you for the update.","do":"expect","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-reply-item","within":10000}]}],"id":613,"setup":[{"actor":"owner","do":"signUp","name":"managed-shared-owner"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-email","text":"managed-shared@example.com"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Shared managed case"},{"actor":"owner","do":"fill","testid":"support-message","text":"Please investigate this case."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","do":"expect","nonEmpty":true,"testid":"support-reference"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","requiresFeatures":["ecommerce.progression.managed-support"],"role":"guarantee","source":"scenarios/progression-managed-support-shared.json"}],"id":"selected-source-078","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/progression-managed-support-shared.json"},{"checkGroups":[{"checkGroupId":"notification-preferences","feature":{"actors":["owner","other"],"criteria":[{"id":"630c","steps":[{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"owner","do":"click","testid":"notification-settings","unlessVisible":"notification-order"},{"actor":"owner","attribute":"data-state","do":"expect","testid":"notification-order","value":"on"}]}],"id":630,"setup":[{"actor":"owner","do":"signUp","name":"notification-owner"},{"actor":"other","do":"signUp","name":"notification-other"},{"actor":"owner","do":"click","testid":"notification-settings","unlessVisible":"notification-order"},{"actor":"owner","do":"click","testid":"notification-order"},{"actor":"owner","do":"click","testid":"notification-save"}]},"packId":"ecommerce.progression.notification-preferences","role":"feature","source":"scenarios/progression-notification-preferences.json"},{"checkGroupId":"notification-preferences-reload","feature":{"actors":["owner","other"],"criteria":[{"id":"630a","steps":[{"actor":"owner","do":"reload","settleMs":3000},{"actor":"owner","do":"ensureSignedIn","name":"notification-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"notification-settings","unlessVisible":"notification-order"},{"actor":"owner","attribute":"data-state","do":"expect","testid":"notification-order","value":"on"},{"do":"restartBackend","settleMs":1000},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"notification-owner"},{"actor":"owner-fresh","do":"click","testid":"notification-settings","unlessVisible":"notification-order"},{"actor":"owner-fresh","attribute":"data-state","do":"expect","testid":"notification-order","value":"on"}]}],"id":630,"setup":[{"actor":"owner","do":"signUp","name":"notification-owner"},{"actor":"other","do":"signUp","name":"notification-other"},{"actor":"owner","do":"click","testid":"notification-settings","unlessVisible":"notification-order"},{"actor":"owner","do":"click","testid":"notification-order"},{"actor":"owner","do":"click","testid":"notification-save"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","requiresFeatures":["ecommerce.progression.notification-preferences"],"role":"guarantee","source":"scenarios/progression-notification-preferences.json"},{"checkGroupId":"notification-preferences-privacy","feature":{"actors":["owner","other"],"criteria":[{"id":"630b","steps":[{"actor":"other","do":"click","testid":"notification-settings","unlessVisible":"notification-order"},{"actor":"other","attribute":"data-state","do":"expect","testid":"notification-order","value":"off"}]}],"id":630,"setup":[{"actor":"owner","do":"signUp","name":"notification-owner"},{"actor":"other","do":"signUp","name":"notification-other"},{"actor":"owner","do":"click","testid":"notification-settings","unlessVisible":"notification-order"},{"actor":"owner","do":"click","testid":"notification-order"},{"actor":"owner","do":"click","testid":"notification-save"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.notification-preferences"],"role":"guarantee","source":"scenarios/progression-notification-preferences.json"}],"id":"selected-source-079","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-notification-preferences.json"},{"checkGroups":[{"checkGroupId":"open-list","feature":{"actors":["reader","reviewer"],"criteria":[{"id":"902a","steps":[{"actor":"reader","do":"openItem","item":"Keyboard"},{"actor":"reader","do":"expect","testid":"item-detail","within":10000},{"actor":"reviewer","do":"click","ifAvailable":true,"testid":"review-toggle","unlessVisible":"review-rating"},{"actor":"reviewer","do":"fill","testid":"review-input","text":"live-review-kbd"},{"actor":"reviewer","do":"click","testid":"review-submit"},{"actor":"reviewer","contains":"live-review-kbd","do":"expectElementCount","equals":1,"testid":"review-item","within":10000},{"actor":"reader","contains":"live-review-kbd","do":"expectElementCount","equals":1,"testid":"review-item","within":10000}]}],"id":902,"setup":[{"actor":"reviewer","do":"signUp","name":"raceR"},{"actor":"reader","do":"signUp","name":"raceL"},{"action":"buy","actor":"reviewer","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"reviewer","do":"expectActionOutcome","outcome":"accepted"},{"actor":"reviewer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"reviewer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"reviewer","contains":"Keyboard","do":"expect","testid":"order-item","within":10000},{"actor":"reviewer","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"reviewer","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"reviewer","do":"openItem","item":"Keyboard"},{"actor":"reviewer","do":"expect","testid":"item-detail","within":10000},{"actor":"reviewer","do":"click","ifAvailable":true,"testid":"review-toggle","unlessVisible":"review-rating"},{"actor":"reviewer","do":"fill","testid":"review-rating","text":"5"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","source":"scenarios/progression-open-list-live.json"}],"id":"selected-source-080","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/progression-open-list-live.json"},{"checkGroups":[{"checkGroupId":"cancellation-and-return","feature":{"actors":["customer"],"criteria":[{"id":"3e","steps":[{"absent":true,"actor":"customer","do":"expect","in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"return-item"}]}],"id":331,"setup":[{"actor":"customer","do":"signUp","name":"return-boundary"},{"action":"buy","actor":"customer","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"customer","do":"expectActionOutcome","outcome":"accepted"},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"return-boundary","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-status","value":"pending"}]},"packId":"ecommerce.l3.order-returns-features","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"feature","source":"scenarios/progression-order-return-boundary.json","stablePackId":"ecommerce.returns-pricing"},{"checkGroupId":"cancellation-and-return","feature":{"actors":["customer","staff","admin"],"criteria":[{"id":"3f","steps":[{"action":"returnItem","actor":"customer","do":"callAction","input":{"attribute":"data-return-input","contains":"Desk Lamp","testid":"order-line"},"namedAction":{"args":[0,0],"id":"returnItem","method":"POST","params":[{"in":"path","name":"orderId","placeholder":"{orderId}","wireType":"u64"},{"in":"path","name":"itemId","placeholder":"{itemId}","wireType":"u64"}],"path":"/api/orders/{orderId}/items/{itemId}/return","reducer":"return_order_item"}},{"actor":"customer","do":"expectActionOutcome","outcome":"validation-refused"},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"return-boundary","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-status","value":"pending"},{"do":"dbExpectStock","item":"Desk Lamp","plus":0,"relativeTo":"pending-East","warehouse":"East"},{"do":"dbExpectStock","item":"Desk Lamp","plus":0,"relativeTo":"pending-West","warehouse":"West"},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"expectNumber","plus":0,"relativeTo":"pending-revenue","testid":"admin-revenue"}]}],"id":332,"setup":[{"actor":"customer","do":"signUp","name":"return-boundary"},{"action":"buy","actor":"customer","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"customer","do":"expectActionOutcome","outcome":"accepted"},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"return-boundary","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-status","value":"pending"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"return-boundary","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"catalog-link","unlessVisible":"item-card"},{"action":"buy","actor":"customer","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"customer","do":"expectActionOutcome","outcome":"accepted"},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"return-boundary","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"action":"ship","actor":"staff","do":"callAction","from":"customer","input":{"attribute":"data-ship-input","contains":"Keyboard","testid":"order-item"},"namedAction":{"args":[0],"id":"ship","params":[{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/fulfilment/ship","reducer":"ship_order"}},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"return-boundary","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"shipped"},{"action":"returnItem","actor":"customer","do":"callAction","input":{"attribute":"data-return-input","contains":"Keyboard","testid":"order-line"},"namedAction":{"args":[0,0],"id":"returnItem","method":"POST","params":[{"in":"path","name":"orderId","placeholder":"{orderId}","wireType":"u64"},{"in":"path","name":"itemId","placeholder":"{itemId}","wireType":"u64"}],"path":"/api/orders/{orderId}/items/{itemId}/return","reducer":"return_order_item"}},{"actor":"customer","do":"expectActionOutcome","outcome":"accepted"},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"return-boundary","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"returned"},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","as":"pending-revenue","do":"recordNumber","testid":"admin-revenue"},{"as":"pending-East","do":"dbRecordStock","item":"Desk Lamp","warehouse":"East"},{"as":"pending-West","do":"dbRecordStock","item":"Desk Lamp","warehouse":"West"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-status","value":"pending"}]},"packId":"ecommerce.l3.order-returns-features","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"feature","source":"scenarios/progression-order-return-boundary.json","stablePackId":"ecommerce.returns-pricing"}],"id":"selected-source-081","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/progression-order-return-boundary.json"},{"checkGroups":[{"checkGroupId":"cancellation-and-return","feature":{"actors":["customer","staff","admin"],"criteria":[{"id":"3c","steps":[{"actor":"customer","do":"click","in":{"contains":"Keyboard","testid":"order-item"},"testid":"return-item"},{"actor":"customer","contains":"Keyboard","containsText":"returned","do":"expect","ignoreCase":true,"testid":"order-item","within":10000},{"actor":"customer","do":"freshClient"},{"actor":"customer-fresh","do":"signIn","name":"return-complete"},{"actor":"customer-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer-fresh","do":"click","testid":"orders-toggle"},{"actor":"customer-fresh","contains":"Keyboard","containsText":"returned","do":"expect","ignoreCase":true,"testid":"order-item","within":10000},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"expectNumber","plus":0,"relativeTo":"return-revenue-before","testid":"admin-revenue","within":10000},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","ifAvailable":true,"testid":"catalog-link","within":1000},{"do":"dbExpectStock","item":"Keyboard","plus":0,"relativeTo":"return-stock-before"}]}],"id":330,"setup":[{"actor":"customer","do":"signUp","name":"return-complete"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"staff","do":"click","testid":"staff-link"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"customer","do":"expectNumber","equals":89,"in":{"contains":"Keyboard","testid":"item-card"},"testid":"item-price"},{"as":"return-stock-before","do":"dbRecordStock","item":"Keyboard"},{"actor":"admin","as":"return-revenue-before","do":"recordNumber","testid":"admin-revenue"},{"actor":"customer","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"buy-now"},{"actor":"staff","do":"reload","settleMs":1000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"queue-item"},{"actor":"staff","contains":"Keyboard","do":"expect","testid":"queue-item","within":10000},{"do":"dbExpectStock","item":"Keyboard","plus":-1,"relativeTo":"return-stock-before"},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"expectNumber","plus":89,"relativeTo":"return-revenue-before","testid":"admin-revenue","within":10000},{"actor":"staff","do":"click","in":{"contains":"Keyboard","testid":"queue-item"},"testid":"ship-submit"},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"return-complete","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"shipped","within":10000}]},"packId":"ecommerce.l3.order-returns-features","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"feature","source":"scenarios/progression-order-return-complete.json","stablePackId":"ecommerce.returns-pricing"}],"id":"selected-source-082","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/progression-order-return-complete.json"},{"checkGroups":[{"checkGroupId":"order-support-ownership","feature":{"actors":["owner","other"],"criteria":[{"id":"614b","steps":[{"absent":true,"actor":"other","contains":"Desk Lamp","do":"expect","testid":"support-order-option"},{"action":"linkSupportOrder","actor":"owner","authentication":"actor","do":"callAction","input":{"attribute":"data-action-input","testid":"support-link-order"},"namedAction":{"args":[0,0],"id":"linkSupportOrder","method":"POST","params":[{"in":"path","name":"caseId","placeholder":"{caseId}","wireType":"u64"},{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/support/cases/{caseId}/order","reducer":"link_support_order"},"settleMs":1500},{"actor":"owner","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","contains":"Desk Lamp","do":"expectElementCount","equals":1,"testid":"support-order","within":10000},{"action":"linkSupportOrder","actor":"other","authentication":"actor","do":"callAction","from":"owner","input":{"attribute":"data-action-input","overrides":{"caseId":{"actor":"other","attribute":"data-entity-id","contains":"Other order case","testid":"support-ticket"}},"testid":"support-link-order"},"namedAction":{"args":[0,0],"id":"linkSupportOrder","method":"POST","params":[{"in":"path","name":"caseId","placeholder":"{caseId}","wireType":"u64"},{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/support/cases/{caseId}/order","reducer":"link_support_order"},"settleMs":1500},{"actor":"other","do":"expectActionOutcome","outcome":"refused"},{"actor":"other","do":"freshClient"},{"actor":"other-fresh","do":"signIn","name":"order-boundary-other"},{"actor":"other-fresh","do":"click","testid":"support-link"},{"absent":true,"actor":"other-fresh","contains":"Desk Lamp","do":"expect","in":{"contains":"Other order case","testid":"support-ticket"},"testid":"support-order"},{"actor":"owner","contains":"Desk Lamp","do":"expectElementCount","equals":1,"testid":"support-order","within":10000}]}],"id":614,"setup":[{"actor":"owner","do":"signUp","name":"order-boundary-owner"},{"actor":"owner","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"buy-now"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Owner order case"},{"actor":"owner","do":"fill","testid":"support-message","text":"This case belongs to the order owner."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","contains":"Desk Lamp","do":"click","in":{"contains":"Owner order case","testid":"support-ticket"},"testid":"support-order-option"},{"actor":"other","do":"signUp","name":"order-boundary-other"},{"actor":"other","do":"click","testid":"support-link"},{"actor":"other","do":"fill","testid":"support-subject","text":"Other order case"},{"actor":"other","do":"fill","testid":"support-message","text":"This is a separate case."},{"actor":"other","do":"click","testid":"support-submit"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.order-support"],"role":"guarantee","source":"scenarios/progression-order-support-boundary.json"}],"id":"selected-source-083","scenario":{"level":4,"writeUrlPattern":null},"source":"scenarios/progression-order-support-boundary.json"},{"checkGroups":[{"checkGroupId":"order-support-owned","feature":{"actors":["owner","staff"],"criteria":[{"id":"614a","steps":[{"actor":"owner","contains":"Desk Lamp","do":"click","in":{"contains":"Owned order case","testid":"support-ticket"},"testid":"support-order-option"},{"actor":"owner","do":"click","in":{"contains":"Owned order case","testid":"support-ticket"},"testid":"support-link-order"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee"},{"actor":"staff","contains":"Desk Lamp","do":"expect","in":{"contains":"Owned order case","testid":"support-ticket"},"testid":"support-order","within":10000}]}],"id":614,"setup":[{"actor":"owner","do":"signUp","name":"order-support-owner"},{"actor":"owner","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"buy-now"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Owned order case"},{"actor":"owner","do":"fill","testid":"support-message","text":"Question about my Desk Lamp order."},{"actor":"owner","do":"click","testid":"support-submit"}]},"packId":"ecommerce.progression.order-support","role":"feature","source":"scenarios/progression-order-support-owned.json"}],"id":"selected-source-084","scenario":{"level":4,"writeUrlPattern":null},"source":"scenarios/progression-order-support-owned.json"},{"checkGroups":[{"checkGroupId":"personalized-recommendations","feature":{"actors":["sales-helper","audio-customer","home-customer","computing-customer"],"criteria":[{"id":"403a","steps":[{"actor":"audio-customer","contains":"Headphones","do":"expect","in":{"testid":"recommendations"},"testid":"recommended-item"},{"actor":"home-customer","do":"expectNumber","equals":1,"in":{"contains":"Desk Lamp","testid":"recommended-item"},"testid":"recommendation-rank","within":10000},{"actor":"computing-customer","do":"expectSequence","equals":["Gaming Mouse","Laptop Stand","Webcam"],"in":{"testid":"recommendations"},"testid":"recommended-item","within":10000}]}],"id":403,"setup":[{"actor":"sales-helper","do":"signUp","name":"recommend-sales"},{"actor":"sales-helper","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"settleMs":500,"testid":"buy-now"},{"actor":"sales-helper","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"settleMs":500,"testid":"buy-now"},{"actor":"sales-helper","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"settleMs":500,"testid":"buy-now"},{"actor":"audio-customer","do":"signUp","name":"recommend-audio"},{"actor":"audio-customer","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"buy-now"},{"actor":"home-customer","do":"signUp","name":"recommend-home"},{"actor":"home-customer","do":"click","in":{"contains":"Coffee Grinder","testid":"item-card"},"testid":"buy-now"},{"actor":"computing-customer","do":"signUp","name":"recommend-computing"},{"actor":"computing-customer","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"buy-now"}]},"packId":"ecommerce.progression.personalized-recommendations","role":"feature","source":"scenarios/progression-personalized-recommendations.json"},{"checkGroupId":"recommendation-profile-isolation","feature":{"actors":["sales-helper","audio-customer","home-customer","computing-customer"],"criteria":[{"id":"403b","steps":[{"actor":"audio-customer","do":"click","in":{"contains":"Headphones","testid":"item-card"},"testid":"buy-now"},{"actor":"audio-customer","contains":"Headphones","do":"waitUntilAbsent","in":{"testid":"recommendations"},"testid":"recommended-item","within":10000},{"actor":"home-customer","do":"expectNumber","equals":1,"in":{"contains":"Desk Lamp","testid":"recommended-item"},"testid":"recommendation-rank","within":10000},{"actor":"computing-customer","do":"expectSequence","equals":["Gaming Mouse","Laptop Stand","Webcam"],"in":{"testid":"recommendations"},"testid":"recommended-item","within":10000}]}],"id":403,"setup":[{"actor":"sales-helper","do":"signUp","name":"recommend-sales"},{"actor":"sales-helper","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"settleMs":500,"testid":"buy-now"},{"actor":"sales-helper","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"settleMs":500,"testid":"buy-now"},{"actor":"sales-helper","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"settleMs":500,"testid":"buy-now"},{"actor":"audio-customer","do":"signUp","name":"recommend-audio"},{"actor":"audio-customer","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"buy-now"},{"actor":"home-customer","do":"signUp","name":"recommend-home"},{"actor":"home-customer","do":"click","in":{"contains":"Coffee Grinder","testid":"item-card"},"testid":"buy-now"},{"actor":"computing-customer","do":"signUp","name":"recommend-computing"},{"actor":"computing-customer","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"buy-now"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.personalized-recommendations"],"role":"guarantee","source":"scenarios/progression-personalized-recommendations.json"}],"id":"selected-source-085","scenario":{"level":4,"writeUrlPattern":null},"source":"scenarios/progression-personalized-recommendations.json"},{"checkGroups":[{"checkGroupId":"price-history","feature":{"actors":["admin","customer"],"criteria":[{"id":"4c","steps":[{"actor":"customer","do":"fill","enter":true,"settleMs":1500,"testid":"search-input","text":"Desk Lamp"},{"actor":"customer","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"add-to-cart"},{"actor":"customer","do":"click","testid":"cart-toggle"},{"actor":"customer","do":"expectNumber","equals":42,"testid":"cart-total"},{"actor":"admin","do":"fill","in":{"contains":"Desk Lamp","testid":"admin-item-row"},"testid":"price-input","text":"52.00"},{"actor":"admin","do":"click","in":{"contains":"Desk Lamp","testid":"admin-item-row"},"testid":"price-submit"},{"actor":"customer","do":"expectNumber","equals":52,"testid":"cart-total","within":10000},{"action":"checkout","actor":"customer","do":"callAction","namedAction":{"args":[],"id":"checkout","method":"POST","path":"/api/checkout","reducer":"checkout"},"settleMs":1500},{"actor":"customer","do":"expectActionOutcome","outcome":"accepted"},{"actor":"customer","do":"freshClient"},{"actor":"customer-fresh","do":"signIn","name":"price-cart"},{"actor":"customer-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer-fresh","do":"click","testid":"orders-toggle"},{"actor":"customer-fresh","contains":"Desk Lamp","count":1,"do":"expect","testid":"order-item"},{"actor":"customer-fresh","do":"expectNumber","equals":52,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-total"}]}],"id":420,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"customer","do":"signUp","name":"price-cart"},{"actor":"admin","do":"click","testid":"admin-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.price-history-specifications","requiresFeatures":["ecommerce.feature.checkout","ecommerce.l2.price-history-features"],"role":"guarantee","source":"scenarios/progression-price-cart-checkout.json","stablePackId":"ecommerce.returns-pricing"}],"id":"selected-source-086","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/progression-price-cart-checkout.json"},{"checkGroups":[{"checkGroupId":"product-bundles","feature":{"actors":["admin","visitor"],"criteria":[{"id":"740a","steps":[{"actor":"admin","do":"fill","testid":"bundle-name-input","text":"Office bundle"},{"actor":"admin","do":"fill","testid":"bundle-price-input","text":"75.00"},{"actor":"admin","do":"fill","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"},{"actor":"admin","do":"click","testid":"bundle-save"},{"actor":"admin","contains":"Office bundle","do":"expect","testid":"bundle-card","within":10000},{"actor":"visitor","do":"reload","settleMs":1000},{"actor":"visitor","do":"click","testid":"bundles-link","unlessVisible":"bundle-card"},{"actor":"visitor","do":"expectNumber","equals":75,"in":{"contains":"Office bundle","testid":"bundle-card"},"testid":"bundle-price","within":10000},{"actor":"visitor","do":"expectElementCount","equals":2,"in":{"contains":"Office bundle","testid":"bundle-card"},"testid":"bundle-component"},{"actor":"visitor","attribute":"data-quantity","contains":"Keyboard","do":"expect","in":{"contains":"Office bundle","testid":"bundle-card"},"testid":"bundle-component","value":"2","within":10000},{"actor":"visitor","attribute":"data-quantity","contains":"Desk Lamp","do":"expect","in":{"contains":"Office bundle","testid":"bundle-card"},"testid":"bundle-component","value":"1","within":10000}]}],"id":740,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"bundles-link","unlessVisible":"bundle-name-input"}]},"packId":"ecommerce.feature.product-bundles","role":"feature","source":"scenarios/progression-product-bundles.json"},{"checkGroupId":"bundle-743","feature":{"actors":["admin","customer"],"criteria":[{"id":"743a","steps":[{"actor":"admin","do":"fill","testid":"bundle-price-input","text":"1.00"},{"action":"saveBundle","actor":"customer","do":"callAction","from":"admin","input":{"attribute":"data-bundle-save-input","testid":"bundle-save"},"namedAction":{"args":["Protected bundle",1,"[{\"item\":\"Keyboard\",\"quantity\":1}]"],"id":"saveBundle","params":[{"in":"body","name":"name"},{"in":"body","name":"price"},{"in":"body","name":"componentsJson"}],"path":"/api/bundles","reducer":"save_bundle"},"settleMs":1000},{"actor":"customer","do":"expectActionOutcome","outcome":"application-refused"},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"click","testid":"bundles-link","unlessVisible":"bundle-card"},{"actor":"admin","do":"expectNumber","equals":75,"in":{"contains":"Protected bundle","testid":"bundle-card"},"testid":"bundle-price","within":10000},{"actor":"admin","attribute":"data-quantity","contains":"Keyboard","do":"expect","in":{"contains":"Protected bundle","testid":"bundle-card"},"testid":"bundle-component","value":"2","within":10000}]}],"id":743,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"bundles-link","unlessVisible":"bundle-name-input"},{"actor":"customer","do":"signUp","name":"bundle-management"},{"actor":"admin","do":"fill","testid":"bundle-name-input","text":"Protected bundle"},{"actor":"admin","do":"fill","testid":"bundle-price-input","text":"75.00"},{"actor":"admin","do":"fill","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"},{"actor":"admin","do":"click","testid":"bundle-save"},{"actor":"admin","contains":"Protected bundle","do":"expect","testid":"bundle-card","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","requiresFeatures":["ecommerce.feature.product-bundles"],"role":"guarantee","source":"scenarios/progression-product-bundles.json"}],"id":"selected-source-087","scenario":{"level":4,"writeUrlPattern":null},"source":"scenarios/progression-product-bundles.json"},{"checkGroups":[{"checkGroupId":"promotion-checkout-active","feature":{"actors":["staff","activeBuyer","expiredBuyer","firstBuyer","secondBuyer"],"criteria":[{"id":"621a","steps":[{"actor":"activeBuyer","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"actor":"activeBuyer","do":"click","testid":"cart-toggle"},{"actor":"activeBuyer","do":"fill","testid":"cart-promotion","text":"LIVE10"},{"actor":"activeBuyer","do":"click","testid":"apply-promotion"},{"actor":"activeBuyer","do":"click","testid":"checkout-submit"},{"actor":"activeBuyer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"activeBuyer","do":"click","testid":"orders-toggle"},{"actor":"activeBuyer","do":"expectNumber","equals":8.9,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-discount"}]}],"id":621,"setup":[{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"promotions-link"},{"actor":"staff","do":"click","testid":"promotions-link","unlessVisible":"promotion-code"},{"actor":"staff","do":"fill","testid":"promotion-code","text":"LIVE10"},{"actor":"staff","do":"fill","testid":"promotion-discount","text":"10"},{"actor":"staff","do":"fill","testid":"promotion-start","text":"2020-01-01"},{"actor":"staff","do":"fill","testid":"promotion-end","text":"2099-12-31"},{"actor":"staff","do":"fill","testid":"promotion-limit","text":"10"},{"actor":"staff","do":"click","testid":"promotion-submit"},{"actor":"staff","do":"fill","testid":"promotion-code","text":"OLD10"},{"actor":"staff","do":"fill","testid":"promotion-discount","text":"10"},{"actor":"staff","do":"fill","testid":"promotion-start","text":"2000-01-01"},{"actor":"staff","do":"fill","testid":"promotion-end","text":"2000-12-31"},{"actor":"staff","do":"fill","testid":"promotion-limit","text":"10"},{"actor":"staff","do":"click","testid":"promotion-submit"},{"actor":"staff","do":"fill","testid":"promotion-code","text":"ONCE10"},{"actor":"staff","do":"fill","testid":"promotion-discount","text":"10"},{"actor":"staff","do":"fill","testid":"promotion-start","text":"2020-01-01"},{"actor":"staff","do":"fill","testid":"promotion-end","text":"2099-12-31"},{"actor":"staff","do":"fill","testid":"promotion-limit","text":"1"},{"actor":"staff","do":"click","testid":"promotion-submit"},{"actor":"activeBuyer","do":"signUp","name":"promotion-active"},{"actor":"expiredBuyer","do":"signUp","name":"promotion-expired"},{"actor":"firstBuyer","do":"signUp","name":"promotion-first"},{"actor":"secondBuyer","do":"signUp","name":"promotion-second"}]},"packId":"ecommerce.progression.promotion-checkout","role":"feature","source":"scenarios/progression-promotion-checkout.json"},{"checkGroupId":"promotion-checkout-expired","feature":{"actors":["staff","activeBuyer","expiredBuyer","firstBuyer","secondBuyer"],"criteria":[{"id":"621b","steps":[{"actor":"expiredBuyer","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"actor":"expiredBuyer","do":"click","testid":"cart-toggle"},{"actor":"expiredBuyer","do":"fill","testid":"cart-promotion","text":"OLD10"},{"actor":"expiredBuyer","do":"click","testid":"apply-promotion"},{"actor":"expiredBuyer","do":"expect","testid":"promotion-error"}]}],"id":621,"setup":[{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"promotions-link"},{"actor":"staff","do":"click","testid":"promotions-link","unlessVisible":"promotion-code"},{"actor":"staff","do":"fill","testid":"promotion-code","text":"LIVE10"},{"actor":"staff","do":"fill","testid":"promotion-discount","text":"10"},{"actor":"staff","do":"fill","testid":"promotion-start","text":"2020-01-01"},{"actor":"staff","do":"fill","testid":"promotion-end","text":"2099-12-31"},{"actor":"staff","do":"fill","testid":"promotion-limit","text":"10"},{"actor":"staff","do":"click","testid":"promotion-submit"},{"actor":"staff","do":"fill","testid":"promotion-code","text":"OLD10"},{"actor":"staff","do":"fill","testid":"promotion-discount","text":"10"},{"actor":"staff","do":"fill","testid":"promotion-start","text":"2000-01-01"},{"actor":"staff","do":"fill","testid":"promotion-end","text":"2000-12-31"},{"actor":"staff","do":"fill","testid":"promotion-limit","text":"10"},{"actor":"staff","do":"click","testid":"promotion-submit"},{"actor":"staff","do":"fill","testid":"promotion-code","text":"ONCE10"},{"actor":"staff","do":"fill","testid":"promotion-discount","text":"10"},{"actor":"staff","do":"fill","testid":"promotion-start","text":"2020-01-01"},{"actor":"staff","do":"fill","testid":"promotion-end","text":"2099-12-31"},{"actor":"staff","do":"fill","testid":"promotion-limit","text":"1"},{"actor":"staff","do":"click","testid":"promotion-submit"},{"actor":"activeBuyer","do":"signUp","name":"promotion-active"},{"actor":"expiredBuyer","do":"signUp","name":"promotion-expired"},{"actor":"firstBuyer","do":"signUp","name":"promotion-first"},{"actor":"secondBuyer","do":"signUp","name":"promotion-second"}]},"packId":"ecommerce.progression.promotion-checkout","role":"feature","source":"scenarios/progression-promotion-checkout.json"},{"checkGroupId":"promotion-checkout-exhausted","feature":{"actors":["staff","activeBuyer","expiredBuyer","firstBuyer","secondBuyer"],"criteria":[{"id":"621c","steps":[{"actor":"firstBuyer","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"actor":"firstBuyer","do":"click","testid":"cart-toggle"},{"actor":"firstBuyer","do":"fill","testid":"cart-promotion","text":"ONCE10"},{"actor":"firstBuyer","do":"click","testid":"apply-promotion"},{"actor":"firstBuyer","do":"click","testid":"checkout-submit"},{"actor":"secondBuyer","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"actor":"secondBuyer","do":"click","testid":"cart-toggle"},{"actor":"secondBuyer","do":"fill","testid":"cart-promotion","text":"ONCE10"},{"actor":"secondBuyer","do":"click","testid":"apply-promotion"},{"actor":"secondBuyer","do":"expect","testid":"promotion-error"}]}],"id":621,"setup":[{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"promotions-link"},{"actor":"staff","do":"click","testid":"promotions-link","unlessVisible":"promotion-code"},{"actor":"staff","do":"fill","testid":"promotion-code","text":"LIVE10"},{"actor":"staff","do":"fill","testid":"promotion-discount","text":"10"},{"actor":"staff","do":"fill","testid":"promotion-start","text":"2020-01-01"},{"actor":"staff","do":"fill","testid":"promotion-end","text":"2099-12-31"},{"actor":"staff","do":"fill","testid":"promotion-limit","text":"10"},{"actor":"staff","do":"click","testid":"promotion-submit"},{"actor":"staff","do":"fill","testid":"promotion-code","text":"OLD10"},{"actor":"staff","do":"fill","testid":"promotion-discount","text":"10"},{"actor":"staff","do":"fill","testid":"promotion-start","text":"2000-01-01"},{"actor":"staff","do":"fill","testid":"promotion-end","text":"2000-12-31"},{"actor":"staff","do":"fill","testid":"promotion-limit","text":"10"},{"actor":"staff","do":"click","testid":"promotion-submit"},{"actor":"staff","do":"fill","testid":"promotion-code","text":"ONCE10"},{"actor":"staff","do":"fill","testid":"promotion-discount","text":"10"},{"actor":"staff","do":"fill","testid":"promotion-start","text":"2020-01-01"},{"actor":"staff","do":"fill","testid":"promotion-end","text":"2099-12-31"},{"actor":"staff","do":"fill","testid":"promotion-limit","text":"1"},{"actor":"staff","do":"click","testid":"promotion-submit"},{"actor":"activeBuyer","do":"signUp","name":"promotion-active"},{"actor":"expiredBuyer","do":"signUp","name":"promotion-expired"},{"actor":"firstBuyer","do":"signUp","name":"promotion-first"},{"actor":"secondBuyer","do":"signUp","name":"promotion-second"}]},"packId":"ecommerce.progression.promotion-checkout","role":"feature","source":"scenarios/progression-promotion-checkout.json"}],"id":"selected-source-088","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-promotion-checkout.json"},{"checkGroups":[{"checkGroupId":"promotion-report-redemptions","feature":{"actors":["staff","buyer"],"criteria":[{"id":"622a","steps":[{"actor":"staff","do":"expectNumber","equals":1,"in":{"contains":"REPORT10","testid":"promotion-report"},"testid":"promotion-redemptions"}]}],"id":622,"setup":[{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"promotions-link"},{"actor":"staff","do":"click","testid":"promotions-link","unlessVisible":"promotion-code"},{"actor":"staff","do":"fill","testid":"promotion-code","text":"REPORT10"},{"actor":"staff","do":"fill","testid":"promotion-discount","text":"10"},{"actor":"staff","do":"fill","testid":"promotion-start","text":"2020-01-01"},{"actor":"staff","do":"fill","testid":"promotion-end","text":"2099-12-31"},{"actor":"staff","do":"fill","testid":"promotion-limit","text":"2"},{"actor":"staff","do":"click","testid":"promotion-submit"},{"actor":"buyer","do":"signUp","name":"promotion-report-buyer"},{"actor":"buyer","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"actor":"buyer","do":"click","testid":"cart-toggle"},{"actor":"buyer","do":"fill","testid":"cart-promotion","text":"REPORT10"},{"actor":"buyer","do":"click","testid":"apply-promotion"},{"actor":"buyer","do":"click","testid":"checkout-submit"}]},"packId":"ecommerce.progression.promotion-reporting","role":"feature","source":"scenarios/progression-promotion-reporting.json"},{"checkGroupId":"promotion-report-revenue","feature":{"actors":["staff","buyer"],"criteria":[{"id":"622b","steps":[{"actor":"staff","do":"expectNumber","equals":80.1,"in":{"contains":"REPORT10","testid":"promotion-report"},"testid":"promotion-revenue"}]}],"id":622,"setup":[{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"promotions-link"},{"actor":"staff","do":"click","testid":"promotions-link","unlessVisible":"promotion-code"},{"actor":"staff","do":"fill","testid":"promotion-code","text":"REPORT10"},{"actor":"staff","do":"fill","testid":"promotion-discount","text":"10"},{"actor":"staff","do":"fill","testid":"promotion-start","text":"2020-01-01"},{"actor":"staff","do":"fill","testid":"promotion-end","text":"2099-12-31"},{"actor":"staff","do":"fill","testid":"promotion-limit","text":"2"},{"actor":"staff","do":"click","testid":"promotion-submit"},{"actor":"buyer","do":"signUp","name":"promotion-report-buyer"},{"actor":"buyer","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"actor":"buyer","do":"click","testid":"cart-toggle"},{"actor":"buyer","do":"fill","testid":"cart-promotion","text":"REPORT10"},{"actor":"buyer","do":"click","testid":"apply-promotion"},{"actor":"buyer","do":"click","testid":"checkout-submit"}]},"packId":"ecommerce.progression.promotion-reporting","role":"feature","source":"scenarios/progression-promotion-reporting.json"}],"id":"selected-source-089","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-promotion-reporting.json"},{"checkGroups":[{"checkGroupId":"promotion-rule-values","feature":{"actors":["staff","customer"],"criteria":[{"id":"620a","steps":[{"actor":"staff","do":"click","testid":"promotions-link","unlessVisible":"promotion-code"},{"actor":"staff","do":"fill","testid":"promotion-code","text":"SAVE10"},{"actor":"staff","do":"fill","testid":"promotion-discount","text":"10"},{"actor":"staff","do":"fill","testid":"promotion-start","text":"2099-01-01"},{"actor":"staff","do":"fill","testid":"promotion-end","text":"2099-12-31"},{"actor":"staff","do":"fill","testid":"promotion-limit","text":"2"},{"actor":"staff","do":"click","testid":"promotion-submit"},{"actor":"staff","do":"expectNumber","equals":10,"in":{"contains":"SAVE10","testid":"promotion-item"},"testid":"promotion-discount"},{"actor":"staff","contains":"2099-01-01","do":"expect","in":{"contains":"SAVE10","testid":"promotion-item"},"testid":"promotion-start"},{"actor":"staff","contains":"2099-12-31","do":"expect","in":{"contains":"SAVE10","testid":"promotion-item"},"testid":"promotion-end"},{"actor":"staff","do":"expectNumber","equals":2,"in":{"contains":"SAVE10","testid":"promotion-item"},"testid":"promotion-limit"}]}],"id":620,"setup":[{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"customer","do":"signUp","name":"promotion-customer"},{"actor":"staff","do":"click","testid":"staff-link"}]},"packId":"ecommerce.progression.promotion-rules","role":"feature","source":"scenarios/progression-promotion-rules.json"},{"checkGroupId":"promotion-management-boundary","feature":{"actors":["staff","customer"],"criteria":[{"id":"620b","steps":[{"actor":"staff","do":"click","testid":"promotions-link","unlessVisible":"promotion-code"},{"actor":"staff","do":"fill","testid":"promotion-code","text":"ACCESS10"},{"actor":"staff","do":"fill","testid":"promotion-discount","text":"10"},{"actor":"staff","do":"fill","testid":"promotion-start","text":"2099-01-01"},{"actor":"staff","do":"fill","testid":"promotion-end","text":"2099-12-31"},{"actor":"staff","do":"fill","testid":"promotion-limit","text":"2"},{"actor":"staff","do":"click","testid":"promotion-submit"},{"actor":"staff","do":"reload","settleMs":1000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"staff-link","unlessVisible":"promotion-code"},{"actor":"staff","do":"click","testid":"promotions-link","unlessVisible":"promotion-code"},{"actor":"staff","contains":"ACCESS10","do":"expect","testid":"promotion-item","within":10000},{"absent":true,"actor":"customer","do":"expect","testid":"promotions-link"},{"actor":"customer","do":"replayAs","from":"staff","match":"ACCESS10","namedAction":{"args":["ACCESS10",10,4070908800000000,4102444740000000,2],"id":"createPromotion","params":[{"in":"body","name":"code"},{"in":"body","name":"discountPercent"},{"in":"body","name":"startMicros"},{"in":"body","name":"endMicros"},{"in":"body","name":"usageLimit"}],"path":"/api/promotions","reducer":"create_promotion"},"settleMs":1500,"swap":{"find":"ACCESS10","with":"HACK10"}},{"actor":"customer","do":"expectReplayRejected"},{"absent":true,"actor":"staff","contains":"HACK10","do":"expect","testid":"promotion-item"}]}],"id":620,"setup":[{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"customer","do":"signUp","name":"promotion-customer"},{"actor":"staff","do":"click","testid":"staff-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.promotion-rules"],"role":"guarantee","source":"scenarios/progression-promotion-rules.json"}],"id":"selected-source-090","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-promotion-rules.json"},{"checkGroups":[{"checkGroupId":"purchase-order","feature":{"actors":["buyer"],"criteria":[{"id":"3c","steps":[{"actor":"buyer","do":"click","in":{"contains":"Coffee Grinder","testid":"item-card"},"testid":"buy-now"},{"actor":"buyer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"buyer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"buyer","contains":"Coffee Grinder","do":"expect","testid":"order-item"},{"actor":"buyer","do":"expectNumber","equals":64,"in":{"contains":"Coffee Grinder","testid":"order-item"},"testid":"order-total"}]}],"id":3,"setup":[{"actor":"buyer","do":"signUp","name":"purchase-buyer"}]},"packId":"ecommerce.feature.purchasing","role":"feature","source":"scenarios/progression-purchasing.json"}],"id":"selected-source-091","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/progression-purchasing.json"},{"checkGroups":[{"checkGroupId":"recommendation-feedback","feature":{"actors":["customer","other"],"criteria":[{"id":"504a","steps":[{"absent":true,"actor":"customer","contains":"Headphones","do":"expect","testid":"recommended-item","within":10000}]}],"id":504,"setup":[{"actor":"customer","do":"signUp","name":"feedback-owner"},{"actor":"customer","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"buy-now"},{"actor":"other","do":"signUp","name":"feedback-other"},{"actor":"other","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"buy-now"},{"actor":"customer","contains":"Headphones","do":"expect","testid":"recommended-item"},{"actor":"other","contains":"Headphones","do":"expect","testid":"recommended-item"},{"actor":"customer","do":"click","in":{"contains":"Headphones","testid":"recommended-item"},"testid":"dismiss-recommendation"}]},"packId":"ecommerce.progression.recommendation-feedback","role":"feature","source":"scenarios/progression-recommendation-feedback.json"},{"checkGroupId":"recommendation-feedback-privacy","feature":{"actors":["customer","other"],"criteria":[{"id":"504b","steps":[{"actor":"customer","contains":"Headphones","do":"waitUntilAbsent","testid":"recommended-item","within":10000},{"actor":"other","contains":"Headphones","do":"expect","testid":"recommended-item"}]}],"id":504,"setup":[{"actor":"customer","do":"signUp","name":"feedback-owner"},{"actor":"customer","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"buy-now"},{"actor":"other","do":"signUp","name":"feedback-other"},{"actor":"other","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"buy-now"},{"actor":"customer","contains":"Headphones","do":"expect","testid":"recommended-item"},{"actor":"other","contains":"Headphones","do":"expect","testid":"recommended-item"},{"actor":"customer","do":"click","in":{"contains":"Headphones","testid":"recommended-item"},"testid":"dismiss-recommendation"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.recommendation-feedback"],"role":"guarantee","source":"scenarios/progression-recommendation-feedback.json"},{"checkGroupId":"recommendation-feedback-restart","feature":{"actors":["customer","other"],"criteria":[{"id":"504c","steps":[{"actor":"customer","do":"reload","settleMs":3000},{"actor":"customer","do":"ensureSignedIn","name":"feedback-owner","readyTestid":"current-user"},{"absent":true,"actor":"customer","contains":"Headphones","do":"expect","testid":"recommended-item"},{"do":"restartBackend","settleMs":1000},{"actor":"customer","do":"freshClient"},{"actor":"customer-fresh","do":"signIn","name":"feedback-owner"},{"actor":"customer-fresh","do":"expect","testid":"recommendations","within":10000},{"absent":true,"actor":"customer-fresh","contains":"Headphones","do":"expect","testid":"recommended-item"}]}],"id":504,"setup":[{"actor":"customer","do":"signUp","name":"feedback-owner"},{"actor":"customer","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"buy-now"},{"actor":"other","do":"signUp","name":"feedback-other"},{"actor":"other","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"buy-now"},{"actor":"customer","contains":"Headphones","do":"expect","testid":"recommended-item"},{"actor":"other","contains":"Headphones","do":"expect","testid":"recommended-item"},{"actor":"customer","do":"click","in":{"contains":"Headphones","testid":"recommended-item"},"testid":"dismiss-recommendation"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","requiresFeatures":["ecommerce.progression.recommendation-feedback"],"role":"guarantee","source":"scenarios/progression-recommendation-feedback.json"}],"id":"selected-source-092","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-recommendation-feedback.json"},{"checkGroups":[{"checkGroupId":"review-eligibility-direct","feature":{"actors":["owner","stranger"],"criteria":[{"id":"618a","steps":[{"action":"submitReview","actor":"owner","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0,5,"Original buyer review"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"}],"path":"/api/items/:id/reviews","reducer":"submit_review"},"settleMs":0},{"actor":"owner","do":"expectActionOutcome","outcome":"accepted"},{"action":"submitReview","actor":"stranger","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0,5,"Nonbuyer review"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"}],"path":"/api/items/:id/reviews","reducer":"submit_review"},"settleMs":0},{"actor":"stranger","do":"expectActionOutcome","outcome":"application-refused","routeProvenBy":"owner"},{"action":"submitReview","actor":"stranger","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0,5,"Forged buyer review","claim-review-owner"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"},{"in":"body","name":"username"}],"path":"/api/items/:id/reviews","reducer":"submit_review"},"settleMs":0},{"actor":"stranger","do":"expectActionOutcome","outcome":"application-refused","routeProvenBy":"owner"},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","exact":true,"name":"claim-review-owner"},{"actor":"owner-fresh","do":"openItem","item":"Keyboard"},{"actor":"owner-fresh","contains":"Original buyer review","do":"expect","testid":"review-item","within":10000},{"absent":true,"actor":"owner-fresh","contains":"Nonbuyer review","do":"expect","testid":"review-item","within":3000},{"absent":true,"actor":"owner-fresh","contains":"Forged buyer review","do":"expect","testid":"review-item","within":3000}]}],"id":618,"setup":[{"actor":"owner","do":"signUp","exact":true,"name":"claim-review-owner"},{"actor":"stranger","do":"signUp","exact":true,"name":"claim-review-stranger"},{"action":"buy","actor":"owner","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"},"settleMs":0},{"actor":"owner","do":"expectActionOutcome","outcome":"accepted"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.review-access-specifications","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","source":"scenarios/progression-review-access.json"}],"id":"selected-source-093","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/progression-review-access.json"},{"checkGroups":[{"checkGroupId":"stored-review-script","feature":{"actors":["owner","writer","reader"],"criteria":[{"id":"9180a","steps":[{"action":"submitReview","actor":"owner","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0,5,"Benign stored review"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"}],"path":"/api/items/:id/reviews","reducer":"submit_review"},"settleMs":0},{"actor":"owner","do":"expectActionOutcome","outcome":"accepted"},{"action":"submitReview","actor":"writer","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0,5,"Stored review marker <img src=\"data:,bad-image\" onerror=\"window.__stackBenchScriptCanary()\">"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"}],"path":"/api/items/:id/reviews","reducer":"submit_review"},"settleMs":0},{"actor":"writer","do":"expectActionOutcome","outcome":"completed","routeProvenBy":"owner"},{"actor":"owner","do":"openItem","item":"Keyboard"},{"actor":"owner","contains":"Benign stored review","do":"expect","testid":"review-item","within":10000},{"actor":"owner","do":"wait","ms":1000},{"actor":"owner","do":"expectNoScriptExecution"},{"actor":"reader","do":"freshClient"},{"actor":"reader-fresh","do":"signIn","name":"script-review-reader"},{"actor":"reader-fresh","do":"armScriptCanary"},{"actor":"reader-fresh","do":"openItem","item":"Keyboard"},{"actor":"reader-fresh","contains":"Benign stored review","do":"expect","testid":"review-item","within":10000},{"actor":"reader-fresh","do":"wait","ms":1000},{"actor":"reader-fresh","do":"expectNoScriptExecution"}]}],"id":9180,"setup":[{"actor":"owner","do":"signUp","name":"script-review-owner"},{"actor":"reader","do":"signUp","name":"script-review-reader"},{"actor":"writer","do":"signUp","name":"script-review-writer"},{"action":"buy","actor":"owner","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"owner","do":"expectActionOutcome","outcome":"accepted"},{"action":"buy","actor":"writer","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"writer","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"armScriptCanary"}]},"observations":["unmentioned"],"packId":"ecommerce.progression.review-access-specifications","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","source":"scenarios/progression-review-script.json"}],"id":"selected-source-094","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/progression-review-script.json"},{"checkGroups":[{"checkGroupId":"search-ordering","feature":{"actors":["visitor","buyer"],"criteria":[{"id":"402b","steps":[{"actor":"visitor","do":"expectSequence","equals":["Air Purifier","Bluetooth Speaker","Coffee Grinder","Desk Lamp","Espresso Machine","Gaming Mouse","Headphones","Induction Cooktop","Keyboard","Laptop Stand"],"in":{"testid":"search-results"},"testid":"item-name"},{"actor":"visitor","do":"fill","enter":true,"testid":"search-input","text":"e"},{"actor":"visitor","do":"fill","testid":"minimum-price","text":""},{"actor":"visitor","do":"click","ifAvailable":true,"testid":"filter-apply"},{"actor":"visitor","do":"expectSequence","equals":["Air Purifier","Bluetooth Speaker","Coffee Grinder","Desk Lamp","Espresso Machine","Gaming Mouse","Headphones","Keyboard","Mirrorless Camera","USB Cable"],"in":{"testid":"search-results"},"testid":"item-name"},{"actor":"visitor","do":"fill","enter":true,"testid":"search-input","text":""},{"actor":"visitor","do":"expectSequence","equals":["Headphones","Air Purifier","Bluetooth Speaker","Coffee Grinder","Desk Lamp","Espresso Machine","Gaming Mouse","Induction Cooktop","Keyboard","Laptop Stand"],"in":{"testid":"item-list"},"testid":"item-name"}]}],"id":402,"setup":[{"actor":"buyer","do":"signUp","name":"search-order-buyer"},{"as":"before-search-purchase","do":"dbRecordStock","item":"Headphones"},{"actor":"buyer","do":"click","in":{"contains":"Headphones","testid":"item-card"},"testid":"buy-now"},{"do":"dbExpectStock","item":"Headphones","plus":-1,"relativeTo":"before-search-purchase"},{"actor":"visitor","do":"reload","settleMs":0},{"actor":"visitor","do":"fill","testid":"minimum-price","text":"1"},{"actor":"visitor","do":"click","ifAvailable":true,"testid":"filter-apply"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.search-ordering","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.faceted-search"],"role":"guarantee","source":"scenarios/progression-search-ordering.json"}],"id":"selected-source-095","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/progression-search-ordering.json"},{"checkGroups":[{"checkGroupId":"shipping-accounting","feature":{"actors":["customer","admin","staff"],"criteria":[{"id":"202e","steps":[{"as":"stock-before-purchase","do":"dbRecordStock","item":"Keyboard"},{"actor":"admin","as":"revenue-before-purchase","do":"recordNumber","testid":"admin-revenue"},{"actor":"customer","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"buy-now"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"pending","within":10000},{"do":"dbExpectStock","item":"Keyboard","plus":-1,"relativeTo":"stock-before-purchase"},{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link"},{"actor":"admin","do":"expectNumber","plus":89,"relativeTo":"revenue-before-purchase","testid":"admin-revenue","within":10000},{"actor":"admin","as":"revenue-before-ship","do":"recordNumber","testid":"admin-revenue"},{"as":"East-before-ship","do":"dbRecordStock","item":"Keyboard","warehouse":"East"},{"as":"West-before-ship","do":"dbRecordStock","item":"Keyboard","warehouse":"West"},{"actor":"staff","do":"reload","settleMs":2000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"staff-link"},{"action":"ship","actor":"staff","do":"callAction","from":"customer","input":{"attribute":"data-ship-input","contains":"Keyboard","testid":"order-item"},"namedAction":{"args":[0],"id":"ship","params":[{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/fulfilment/ship","reducer":"ship_order"},"settleMs":0},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"customer","do":"reload","settleMs":2000},{"actor":"customer","do":"ensureSignedIn","name":"shipping-accounting","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"shipped","within":10000},{"do":"dbExpectStock","item":"Keyboard","plus":0,"relativeTo":"East-before-ship","warehouse":"East"},{"do":"dbExpectStock","item":"Keyboard","plus":0,"relativeTo":"West-before-ship","warehouse":"West"},{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link"},{"actor":"admin","do":"expectNumber","plus":0,"relativeTo":"revenue-before-ship","testid":"admin-revenue","within":10000}]}],"id":202,"setup":[{"actor":"customer","do":"signUp","name":"shipping-accounting"},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin","ecommerce.progression.fulfilment-queue"],"role":"guarantee","source":"scenarios/progression-shipping-accounting.json","stablePackId":"ecommerce.inventory-operations"}],"id":"selected-source-096","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/progression-shipping-accounting.json"},{"checkGroups":[{"checkGroupId":"signed-out-purchase","feature":{"actors":["visitor"],"criteria":[{"id":"3a","steps":[{"actor":"visitor","as":"keyboard-before-guest","do":"recordNumber","in":{"contains":"Keyboard","testid":"item-card"},"testid":"item-stock"},{"actor":"visitor","do":"click","ifAvailable":true,"in":{"contains":"Keyboard","testid":"item-card"},"settleMs":1500,"testid":"buy-now"},{"actor":"visitor","do":"reload","settleMs":2000},{"actor":"visitor","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"visitor","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"visitor","do":"expectNumber","in":{"contains":"Keyboard","testid":"item-card"},"plus":0,"relativeTo":"keyboard-before-guest","testid":"item-stock"}]}],"id":3,"setup":[{"actor":"visitor","contains":"Keyboard","do":"expect","testid":"item-card"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/progression-signed-out-purchase.json"}],"id":"selected-source-097","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/progression-signed-out-purchase.json"},{"checkGroups":[{"checkGroupId":"split-tender-refunds-751","feature":{"actors":["owner","staff"],"criteria":[{"id":"751a","steps":[{"actor":"staff","do":"click","in":{"contains":"Split refund 751","testid":"support-ticket"},"testid":"support-refund"},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"credit-owner-751"},{"actor":"owner-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner-fresh","do":"click","testid":"orders-toggle"},{"actor":"owner-fresh","contains":"Desk Lamp","do":"expectElementCount","equals":1,"testid":"refund-entry","within":10000},{"actor":"owner-fresh","do":"expectNumber","equals":42,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-refund-total"},{"actor":"owner-fresh","do":"expectNumber","equals":10,"in":{"contains":"Desk Lamp","testid":"refund-entry"},"testid":"refund-credit-amount"},{"actor":"owner-fresh","do":"expectNumber","equals":32,"in":{"contains":"Desk Lamp","testid":"refund-entry"},"testid":"refund-external-amount"},{"actor":"owner-fresh","do":"click","testid":"credit-link"},{"actor":"owner-fresh","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"owner-fresh","do":"expectNumber","equals":10,"testid":"credit-balance"}]}],"id":751,"setup":[{"actor":"owner","do":"signUp","name":"credit-owner-751"},{"actor":"owner","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"staff","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"staff","do":"fill","testid":"credit-amount-input","text":"10.00"},{"actor":"staff","do":"fill","testid":"credit-reference-input","text":"credit-751"},{"action":"grantCredit","actor":"staff","do":"callAction","from":"staff","input":{"attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","attribute":"data-account-id","testid":"credit-panel"}},"testid":"credit-grant"},"namedAction":{"args":[0,1000,""],"id":"grantCredit","method":"POST","params":[{"in":"body","name":"accountId","wireType":"u64"},{"in":"body","name":"amountMinor"},{"in":"body","name":"reference"}],"path":"/api/staff/credit","reducer":"grant_credit"},"settleMs":0},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"credit-owner-751","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"credit-link","unlessVisible":"credit-balance"},{"actor":"owner","do":"expectNumber","equals":10,"testid":"credit-balance","within":10000},{"actor":"owner","do":"click","testid":"catalog-link"},{"actor":"owner","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"add-to-cart"},{"actor":"owner","do":"click","testid":"cart-toggle"},{"actor":"owner","do":"expectNumber","equals":42,"testid":"cart-total"},{"actor":"owner","do":"click","testid":"credit-checkout"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner","do":"expectNumber","equals":10,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"payment-credit-amount","within":10000},{"actor":"owner","do":"expectNumber","equals":32,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"payment-external-amount"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Split refund 751"},{"actor":"owner","do":"fill","testid":"support-message","text":"Please refund this order."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","contains":"Desk Lamp","do":"click","in":{"contains":"Split refund 751","testid":"support-ticket"},"testid":"support-order-option"},{"actor":"owner","do":"click","in":{"contains":"Split refund 751","testid":"support-ticket"},"testid":"support-link-order"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee","within":1000}]},"packId":"ecommerce.feature.split-tender-refunds","requiresFeatures":["ecommerce.feature.split-tender-refunds","ecommerce.feature.store-credit","ecommerce.progression.support-refunds"],"role":"feature","source":"scenarios/progression-split-tender-refunds.json"},{"checkGroupId":"production-756","feature":{"actors":["owner","staff","staff2"],"criteria":[{"id":"756a","steps":[{"action":"supportRefund","actors":["staff","staff2"],"do":"callConcurrently","from":"staff","input":{"attribute":"data-refund-input","contains":"Split refund 756","testid":"support-ticket"},"namedAction":{"args":[0],"id":"supportRefund","method":"POST","params":[{"in":"path","name":"caseId","placeholder":"{caseId}","wireType":"u64"}],"path":"/api/support/cases/{caseId}/refund","reducer":"support_refund"},"settleMs":0},{"do":"expectCallOutcomes"},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"credit-owner-756"},{"actor":"owner-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner-fresh","do":"click","testid":"orders-toggle"},{"actor":"owner-fresh","contains":"Desk Lamp","do":"expectElementCount","equals":1,"testid":"refund-entry","within":10000},{"actor":"owner-fresh","do":"expectNumber","equals":42,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-refund-total"},{"actor":"owner-fresh","do":"expectNumber","equals":10,"in":{"contains":"Desk Lamp","testid":"refund-entry"},"testid":"refund-credit-amount"},{"actor":"owner-fresh","do":"expectNumber","equals":32,"in":{"contains":"Desk Lamp","testid":"refund-entry"},"testid":"refund-external-amount"},{"actor":"owner-fresh","do":"click","testid":"credit-link"},{"actor":"owner-fresh","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"owner-fresh","do":"expectNumber","equals":10,"testid":"credit-balance"},{"do":"restartBackend","settleMs":1000},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"credit-owner-756"},{"actor":"owner-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner-fresh","do":"click","testid":"orders-toggle"},{"actor":"owner-fresh","contains":"Desk Lamp","do":"expectElementCount","equals":1,"testid":"refund-entry","within":10000},{"actor":"owner-fresh","do":"expectNumber","equals":42,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-refund-total"},{"actor":"owner-fresh","do":"expectNumber","equals":10,"in":{"contains":"Desk Lamp","testid":"refund-entry"},"testid":"refund-credit-amount"},{"actor":"owner-fresh","do":"expectNumber","equals":32,"in":{"contains":"Desk Lamp","testid":"refund-entry"},"testid":"refund-external-amount"},{"actor":"owner-fresh","do":"click","testid":"credit-link"},{"actor":"owner-fresh","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"owner-fresh","do":"expectNumber","equals":10,"testid":"credit-balance"}]}],"id":756,"setup":[{"actor":"owner","do":"signUp","name":"credit-owner-756"},{"actor":"owner","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"staff","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"staff","do":"fill","testid":"credit-amount-input","text":"10.00"},{"actor":"staff","do":"fill","testid":"credit-reference-input","text":"credit-756"},{"action":"grantCredit","actor":"staff","do":"callAction","from":"staff","input":{"attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","attribute":"data-account-id","testid":"credit-panel"}},"testid":"credit-grant"},"namedAction":{"args":[0,1000,""],"id":"grantCredit","method":"POST","params":[{"in":"body","name":"accountId","wireType":"u64"},{"in":"body","name":"amountMinor"},{"in":"body","name":"reference"}],"path":"/api/staff/credit","reducer":"grant_credit"},"settleMs":0},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"credit-owner-756","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"credit-link","unlessVisible":"credit-balance"},{"actor":"owner","do":"expectNumber","equals":10,"testid":"credit-balance","within":10000},{"actor":"owner","do":"click","testid":"catalog-link"},{"actor":"owner","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"add-to-cart"},{"actor":"owner","do":"click","testid":"cart-toggle"},{"actor":"owner","do":"expectNumber","equals":42,"testid":"cart-total"},{"actor":"owner","do":"click","testid":"credit-checkout"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner","do":"expectNumber","equals":10,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"payment-credit-amount","within":10000},{"actor":"owner","do":"expectNumber","equals":32,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"payment-external-amount"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Split refund 756"},{"actor":"owner","do":"fill","testid":"support-message","text":"Please refund this order."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","contains":"Desk Lamp","do":"click","in":{"contains":"Split refund 756","testid":"support-ticket"},"testid":"support-order-option"},{"actor":"owner","do":"click","in":{"contains":"Split refund 756","testid":"support-ticket"},"testid":"support-link-order"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee","within":1000},{"actor":"staff2","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.split-tender-refunds","requiresFeatures":["ecommerce.feature.split-tender-refunds","ecommerce.feature.store-credit","ecommerce.progression.support-refunds"],"role":"guarantee","source":"scenarios/progression-split-tender-refunds.json"}],"id":"selected-source-098","scenario":{"level":6,"writeUrlPattern":null},"source":"scenarios/progression-split-tender-refunds.json"},{"checkGroups":[{"checkGroupId":"staff-access","feature":{"actors":["customer","staff","admin","authorized"],"criteria":[{"id":"601a","steps":[{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link"},{"actor":"staff","do":"expect","testid":"staff-area","within":6000},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"staff-link"},{"actor":"admin","do":"expect","testid":"staff-area","within":6000}]}],"id":601,"setup":[]},"packId":"ecommerce.progression.staff-access","role":"feature","source":"scenarios/progression-staff-access.json"},{"checkGroupId":"staff-area-boundary","feature":{"actors":["customer","staff","admin","authorized"],"criteria":[{"id":"601b","steps":[{"actor":"authorized","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"authorized","do":"click","testid":"staff-link"},{"actor":"authorized","do":"expect","testid":"staff-area"},{"actor":"customer","do":"signIn","exact":true,"name":"customer","password":"stackbench-customer-2026"},{"actor":"customer","do":"click","ifAvailable":true,"settleMs":1500,"testid":"staff-link"},{"absent":true,"actor":"customer","do":"expect","testid":"staff-area"}]}],"id":601,"setup":[]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.staff-access"],"role":"guarantee","source":"scenarios/progression-staff-access.json"}],"id":"selected-source-099","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/progression-staff-access.json"},{"checkGroups":[{"checkGroupId":"staff-activity","feature":{"actors":["admin","staff","customer"],"criteria":[{"id":"624a","steps":[{"actor":"staff","do":"click","testid":"staff-link"},{"actor":"staff","do":"click","testid":"activity-link"},{"actor":"staff","contains":"Activity Mug","do":"expect","testid":"activity-entry","within":10000},{"actor":"staff","contains":"admin","do":"expect","in":{"contains":"Activity Mug","testid":"activity-entry"},"testid":"activity-actor"},{"actor":"staff","contains":"creat","do":"expect","in":{"contains":"Activity Mug","testid":"activity-entry"},"testid":"activity-action"},{"actor":"staff","contains":"Activity Mug","do":"expect","in":{"contains":"Activity Mug","testid":"activity-entry"},"testid":"activity-subject"},{"actor":"staff","do":"expect","in":{"contains":"Activity Mug","testid":"activity-entry"},"testid":"activity-time"}]}],"id":624,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"customer","do":"signIn","exact":true,"name":"customer","password":"stackbench-customer-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"fill","testid":"catalog-name","text":"Activity Mug"},{"actor":"admin","do":"fill","testid":"catalog-category","text":"Kitchen"},{"actor":"admin","do":"fill","testid":"catalog-price","text":"43.00"},{"actor":"admin","do":"fill","testid":"catalog-variants","text":"Green"},{"actor":"admin","do":"click","testid":"catalog-save"}]},"packId":"ecommerce.progression.staff-activity","role":"feature","source":"scenarios/progression-staff-activity.json"},{"checkGroupId":"staff-activity-privacy","feature":{"actors":["admin","staff","customer"],"criteria":[{"id":"624b","steps":[{"absent":true,"actor":"customer","do":"expect","testid":"activity-link"}]}],"id":624,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"customer","do":"signIn","exact":true,"name":"customer","password":"stackbench-customer-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"fill","testid":"catalog-name","text":"Activity Mug"},{"actor":"admin","do":"fill","testid":"catalog-category","text":"Kitchen"},{"actor":"admin","do":"fill","testid":"catalog-price","text":"43.00"},{"actor":"admin","do":"fill","testid":"catalog-variants","text":"Green"},{"actor":"admin","do":"click","testid":"catalog-save"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.staff-activity"],"role":"guarantee","source":"scenarios/progression-staff-activity.json"}],"id":"selected-source-100","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-staff-activity.json"},{"checkGroups":[{"checkGroupId":"staff-roles","feature":{"actors":["admin","replayAdmin","staff","roleAdmin","promotedStaff"],"criteria":[{"id":"621c","steps":[{"actor":"admin","do":"expect","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-select","value":"inventory","within":10000}]}],"id":621,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"expect","testid":"admin-link","within":6000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"staff-role-account-staff"},{"actor":"admin","do":"fill","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-select","text":"inventory"},{"actor":"admin","do":"click","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-save"}]},"packId":"ecommerce.progression.staff-roles","role":"feature","source":"scenarios/progression-staff-roles.json"},{"checkGroupId":"staff-role-reload","feature":{"actors":["admin","replayAdmin","staff","roleAdmin","promotedStaff"],"criteria":[{"id":"621a","steps":[{"actor":"admin","do":"reload","settleMs":2500},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link","unlessVisible":"staff-role-account-staff"},{"actor":"admin","do":"expect","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-select","value":"inventory","within":10000},{"do":"restartBackend","settleMs":1000},{"actor":"admin","do":"freshClient"},{"actor":"admin-fresh","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin-fresh","do":"click","testid":"admin-link","unlessVisible":"staff-role-account-staff"},{"actor":"admin-fresh","do":"expect","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-select","value":"inventory","within":10000}]}],"id":621,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"expect","testid":"admin-link","within":6000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"staff-role-account-staff"},{"actor":"admin","do":"fill","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-select","text":"inventory"},{"actor":"admin","do":"click","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-save"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","requiresFeatures":["ecommerce.progression.staff-roles"],"role":"guarantee","source":"scenarios/progression-staff-roles.json"},{"checkGroupId":"staff-role-boundary","feature":{"actors":["admin","replayAdmin","staff","roleAdmin","promotedStaff"],"criteria":[{"id":"621b","steps":[{"actor":"replayAdmin","do":"reload","settleMs":0},{"actor":"replayAdmin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"replayAdmin","do":"expect","testid":"admin-link","within":6000},{"actor":"replayAdmin","do":"click","testid":"admin-link","unlessVisible":"staff-role-account-staff"},{"actor":"replayAdmin","do":"fill","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-select","text":"staff"},{"actor":"replayAdmin","do":"click","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-save"},{"actor":"replayAdmin","do":"reload","settleMs":1000},{"actor":"replayAdmin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"replayAdmin","do":"click","ifAvailable":true,"testid":"admin-link","unlessVisible":"staff-role-account-staff"},{"actor":"replayAdmin","do":"expect","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-select","value":"staff","within":10000},{"actor":"staff","do":"reload","settleMs":0},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"staff-link"},{"absent":true,"actor":"staff","do":"expect","testid":"staff-role-save"},{"actor":"staff","do":"replayAs","from":"replayAdmin","match":"role","namedAction":{"args":[0,"inventory"],"id":"assignStaffRole","method":"PUT","params":[{"in":"path","name":"accountId","placeholder":":id","wireType":"u64"},{"in":"body","name":"role"}],"path":"/api/staff/:id/role","reducer":"assign_staff_role"},"namedTarget":{"attribute":"data-account-id","testid":"staff-role-account-staff","valueType":"string"},"settleMs":1500,"swap":{"find":"\"role\":\"staff\"","with":"\"role\":\"inventory\""}},{"actor":"staff","do":"expectReplayRejected"},{"actor":"replayAdmin","do":"reload","settleMs":1000},{"actor":"replayAdmin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"replayAdmin","do":"click","ifAvailable":true,"testid":"admin-link","unlessVisible":"staff-role-account-staff"},{"actor":"replayAdmin","do":"expect","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-select","value":"staff","within":10000}]}],"id":621,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"expect","testid":"admin-link","within":6000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"staff-role-account-staff"},{"actor":"admin","do":"fill","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-select","text":"inventory"},{"actor":"admin","do":"click","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-save"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.staff-roles"],"role":"guarantee","source":"scenarios/progression-staff-roles.json"},{"checkGroupId":"staff-role-revocation","feature":{"actors":["admin","replayAdmin","staff","roleAdmin","promotedStaff"],"criteria":[{"id":"621d","steps":[{"actor":"roleAdmin","do":"reload","settleMs":0},{"actor":"roleAdmin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"roleAdmin","do":"click","testid":"admin-link","unlessVisible":"staff-role-account-staff"},{"actor":"roleAdmin","do":"replayAs","from":"roleAdmin","match":"__role-revocation-named-action__","namedAction":{"args":[0,"admin"],"id":"assignStaffRole","method":"PUT","params":[{"in":"path","name":"accountId","placeholder":":id","wireType":"u64"},{"in":"body","name":"role"}],"path":"/api/staff/:id/role","reducer":"assign_staff_role"},"namedTarget":{"attribute":"data-account-id","testid":"staff-role-account-staff","valueType":"string"},"settleMs":0},{"actor":"roleAdmin","do":"expectReplayCompleted","requireAccepted":true},{"actor":"roleAdmin","do":"reload","settleMs":0},{"actor":"roleAdmin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"roleAdmin","do":"click","ifAvailable":true,"testid":"admin-link","unlessVisible":"staff-role-account-staff"},{"actor":"roleAdmin","do":"expect","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-select","value":"admin","within":10000},{"actor":"promotedStaff","do":"reload","settleMs":0},{"actor":"promotedStaff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"promotedStaff","do":"replayAs","from":"roleAdmin","match":"__role-revocation-named-action__","namedAction":{"args":[0,"admin"],"id":"assignStaffRole","method":"PUT","params":[{"in":"path","name":"accountId","placeholder":":id","wireType":"u64"},{"in":"body","name":"role"}],"path":"/api/staff/:id/role","reducer":"assign_staff_role"},"namedTarget":{"attribute":"data-account-id","testid":"staff-role-account-staff","valueType":"string"},"settleMs":0},{"actor":"promotedStaff","do":"expectReplayCompleted","requireAccepted":true},{"actor":"roleAdmin","do":"replayAs","from":"roleAdmin","match":"__role-revocation-named-action__","namedAction":{"args":[0,"staff"],"id":"assignStaffRole","method":"PUT","params":[{"in":"path","name":"accountId","placeholder":":id","wireType":"u64"},{"in":"body","name":"role"}],"path":"/api/staff/:id/role","reducer":"assign_staff_role"},"namedTarget":{"attribute":"data-account-id","testid":"staff-role-account-staff","valueType":"string"},"settleMs":0},{"actor":"roleAdmin","do":"expectReplayCompleted","requireAccepted":true},{"actor":"roleAdmin","do":"reload","settleMs":0},{"actor":"roleAdmin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"roleAdmin","do":"click","ifAvailable":true,"testid":"admin-link","unlessVisible":"staff-role-account-staff"},{"actor":"roleAdmin","do":"expect","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-select","value":"staff","within":10000},{"actor":"promotedStaff","do":"replayAs","from":"roleAdmin","match":"__role-revocation-named-action__","namedAction":{"args":[0,"admin"],"id":"assignStaffRole","method":"PUT","params":[{"in":"path","name":"accountId","placeholder":":id","wireType":"u64"},{"in":"body","name":"role"}],"path":"/api/staff/:id/role","reducer":"assign_staff_role"},"namedTarget":{"attribute":"data-account-id","testid":"staff-role-account-staff","valueType":"string"},"settleMs":0},{"actor":"promotedStaff","do":"expectReplayRejected"},{"actor":"roleAdmin","do":"reload","settleMs":0},{"actor":"roleAdmin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"roleAdmin","do":"click","ifAvailable":true,"testid":"admin-link","unlessVisible":"staff-role-account-staff"},{"actor":"roleAdmin","do":"expect","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-select","value":"staff","within":10000}]}],"id":621,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"expect","testid":"admin-link","within":6000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"staff-role-account-staff"},{"actor":"admin","do":"fill","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-select","text":"inventory"},{"actor":"admin","do":"click","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-save"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.staff-roles"],"role":"guarantee","source":"scenarios/progression-staff-roles.json"}],"id":"selected-source-101","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/progression-staff-roles.json"},{"checkGroups":[{"checkGroupId":"stock-alert-delivery","feature":{"actors":["subscriber","admin"],"criteria":[{"id":"631c","steps":[{"actor":"subscriber","do":"freshClient"},{"actor":"subscriber-fresh","do":"signIn","name":"stock-subscriber"},{"actor":"subscriber-fresh","do":"click","testid":"notifications-toggle","unlessVisible":"notifications-panel"},{"actor":"subscriber-fresh","attribute":"aria-busy","do":"expect","testid":"notifications-panel","value":"false","within":10000},{"actor":"subscriber-fresh","contains":"Air Purifier","do":"expectElementCount","equals":0,"testid":"stock-alert-delivery"},{"actor":"admin","do":"click","testid":"admin-link"},{"action":"restock","actor":"admin","do":"callAction","input":{"attribute":"data-restock-input","contains":"Air Purifier","testid":"admin-location-row"},"namedAction":{"args":[0,0,1],"id":"restock","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"warehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/restock","reducer":"admin_restock"},"settleMs":0},{"actor":"admin","do":"expectActionOutcome","outcome":"accepted"},{"actor":"admin","do":"wait","ms":10000},{"actor":"subscriber","do":"freshClient"},{"actor":"subscriber-fresh","do":"signIn","name":"stock-subscriber"},{"actor":"subscriber-fresh","do":"click","testid":"notifications-toggle","unlessVisible":"notifications-panel"},{"actor":"subscriber-fresh","attribute":"aria-busy","do":"expect","testid":"notifications-panel","value":"false","within":10000},{"actor":"subscriber-fresh","contains":"Air Purifier","do":"expect","testid":"stock-alert-delivery","within":10000}]}],"id":631,"setup":[{"actor":"subscriber","do":"signUp","name":"stock-subscriber"},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"dbSetStock","item":"Air Purifier","quantity":0,"settleMs":1000,"warehouse":"East"},{"do":"dbSetStock","item":"Air Purifier","quantity":0,"settleMs":1000,"warehouse":"West"},{"actor":"subscriber","do":"reload","settleMs":1000},{"actor":"subscriber","do":"ensureSignedIn","name":"stock-subscriber","readyTestid":"current-user"},{"actor":"subscriber","do":"click","in":{"contains":"Air Purifier","testid":"item-card"},"testid":"stock-alert"},{"actor":"subscriber","attribute":"data-submit-state","contains":"Air Purifier","do":"expect","testid":"item-card","value":"succeeded","within":10000}]},"packId":"ecommerce.progression.stock-alerts","requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"feature","source":"scenarios/progression-stock-alert-delivery.json"}],"id":"selected-source-102","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-stock-alert-delivery.json"},{"checkGroups":[{"checkGroupId":"stock-alert-deduplication","feature":{"actors":["subscriber","other","admin"],"criteria":[{"id":"631a","steps":[{"actor":"subscriber","do":"freshClient"},{"actor":"subscriber-fresh","do":"signIn","name":"stock-subscriber"},{"actor":"subscriber-fresh","do":"click","testid":"notifications-toggle","unlessVisible":"notifications-panel"},{"actor":"subscriber-fresh","attribute":"aria-busy","do":"expect","testid":"notifications-panel","value":"false","within":10000},{"actor":"subscriber-fresh","contains":"Air Purifier","do":"expectElementCount","equals":1,"testid":"stock-alert-delivery","within":10000},{"action":"restock","actor":"admin","do":"callAction","input":{"attribute":"data-restock-input","contains":"Air Purifier","testid":"admin-location-row"},"namedAction":{"args":[0,0,1],"id":"restock","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"warehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/restock","reducer":"admin_restock"},"settleMs":0},{"actor":"admin","do":"expectActionOutcome","outcome":"accepted"},{"actor":"admin","do":"wait","ms":10000},{"actor":"subscriber","do":"freshClient"},{"actor":"subscriber-fresh","do":"signIn","name":"stock-subscriber"},{"actor":"subscriber-fresh","do":"click","testid":"notifications-toggle","unlessVisible":"notifications-panel"},{"actor":"subscriber-fresh","attribute":"aria-busy","do":"expect","testid":"notifications-panel","value":"false","within":10000},{"actor":"subscriber-fresh","contains":"Air Purifier","do":"expectElementCount","equals":1,"testid":"stock-alert-delivery","within":10000}]}],"id":631,"setup":[{"actor":"subscriber","do":"signUp","name":"stock-subscriber"},{"actor":"other","do":"signUp","name":"stock-other"},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"dbSetStock","item":"Air Purifier","quantity":0,"settleMs":1000,"warehouse":"East"},{"do":"dbSetStock","item":"Air Purifier","quantity":0,"settleMs":1000,"warehouse":"West"},{"actor":"subscriber","do":"reload","settleMs":1000},{"actor":"subscriber","do":"ensureSignedIn","name":"stock-subscriber","readyTestid":"current-user"},{"actor":"subscriber","do":"click","in":{"contains":"Air Purifier","testid":"item-card"},"testid":"stock-alert"},{"actor":"subscriber","attribute":"data-submit-state","contains":"Air Purifier","do":"expect","testid":"item-card","value":"succeeded","within":10000},{"actor":"admin","do":"click","testid":"admin-link"},{"action":"restock","actor":"admin","do":"callAction","input":{"attribute":"data-restock-input","contains":"Air Purifier","testid":"admin-location-row"},"namedAction":{"args":[0,0,1],"id":"restock","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"warehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/restock","reducer":"admin_restock"},"settleMs":0},{"actor":"admin","do":"expectActionOutcome","outcome":"accepted"},{"actor":"admin","do":"wait","ms":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","requiresFeatures":["ecommerce.feature.warehouse-admin","ecommerce.progression.stock-alerts"],"role":"guarantee","source":"scenarios/progression-stock-alerts.json"},{"checkGroupId":"stock-alert-privacy","feature":{"actors":["subscriber","other","admin"],"criteria":[{"id":"631b","steps":[{"actor":"subscriber","do":"freshClient"},{"actor":"subscriber-fresh","do":"signIn","name":"stock-subscriber"},{"actor":"subscriber-fresh","do":"click","testid":"notifications-toggle","unlessVisible":"notifications-panel"},{"actor":"subscriber-fresh","attribute":"aria-busy","do":"expect","testid":"notifications-panel","value":"false","within":10000},{"actor":"subscriber-fresh","contains":"Air Purifier","do":"expect","testid":"stock-alert-delivery","within":10000},{"actor":"other","do":"freshClient"},{"actor":"other-fresh","do":"signIn","name":"stock-other"},{"actor":"other-fresh","do":"click","testid":"notifications-toggle","unlessVisible":"notifications-panel"},{"actor":"other-fresh","attribute":"aria-busy","do":"expect","testid":"notifications-panel","value":"false","within":10000},{"absent":true,"actor":"other-fresh","contains":"Air Purifier","do":"expect","testid":"notification-item"}]}],"id":631,"setup":[{"actor":"subscriber","do":"signUp","name":"stock-subscriber"},{"actor":"other","do":"signUp","name":"stock-other"},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"dbSetStock","item":"Air Purifier","quantity":0,"settleMs":1000,"warehouse":"East"},{"do":"dbSetStock","item":"Air Purifier","quantity":0,"settleMs":1000,"warehouse":"West"},{"actor":"subscriber","do":"reload","settleMs":1000},{"actor":"subscriber","do":"ensureSignedIn","name":"stock-subscriber","readyTestid":"current-user"},{"actor":"subscriber","do":"click","in":{"contains":"Air Purifier","testid":"item-card"},"testid":"stock-alert"},{"actor":"subscriber","attribute":"data-submit-state","contains":"Air Purifier","do":"expect","testid":"item-card","value":"succeeded","within":10000},{"actor":"admin","do":"click","testid":"admin-link"},{"action":"restock","actor":"admin","do":"callAction","input":{"attribute":"data-restock-input","contains":"Air Purifier","testid":"admin-location-row"},"namedAction":{"args":[0,0,1],"id":"restock","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"warehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/restock","reducer":"admin_restock"},"settleMs":0},{"actor":"admin","do":"expectActionOutcome","outcome":"accepted"},{"actor":"admin","do":"wait","ms":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.feature.warehouse-admin","ecommerce.progression.stock-alerts"],"role":"guarantee","source":"scenarios/progression-stock-alerts.json"}],"id":"selected-source-103","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-stock-alerts.json"},{"checkGroups":[{"checkGroupId":"stock-limit","feature":{"actors":["buyer","watcher","visitor"],"criteria":[{"id":"3d","steps":[{"actor":"buyer","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"buy-now"},{"actor":"buyer","do":"wait","ms":800},{"actor":"buyer","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"buy-now"},{"actor":"buyer","do":"wait","ms":800},{"actor":"buyer","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"buy-now"},{"actor":"buyer","do":"expectNumber","equals":0,"in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock"},{"actor":"buyer","do":"expect","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"out-of-stock"},{"action":"buy","actor":"buyer","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"},"settleMs":2000},{"actor":"buyer","do":"expectActionOutcome","outcome":"validation-refused"},{"actor":"buyer","do":"reload","settleMs":3000},{"actor":"buyer","do":"ensureSignedIn","name":"eli","readyTestid":"current-user"},{"actor":"buyer","do":"expectNumber","equals":0,"in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock"}]}],"id":3,"setup":[{"do":"dbSetStock","item":"Desk Lamp","quantity":2,"settleMs":2000,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":1,"settleMs":2000,"warehouse":"West"},{"actor":"buyer","do":"signUp","name":"eli"},{"actor":"watcher","do":"signUp","name":"fay"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/progression-stock-limit.json"}],"id":"selected-source-104","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/progression-stock-limit.json"},{"checkGroups":[{"checkGroupId":"store-credit-750","feature":{"actors":["owner","staff","tab"],"criteria":[{"id":"750a","steps":[{"actor":"owner","do":"click","testid":"credit-checkout"},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"credit-owner-750"},{"actor":"owner-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner-fresh","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner-fresh","contains":"Desk Lamp","do":"expectElementCount","equals":1,"testid":"order-item","within":10000},{"actor":"owner-fresh","do":"expectNumber","equals":10,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"payment-credit-amount"},{"actor":"owner-fresh","do":"expectNumber","equals":32,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"payment-external-amount"},{"actor":"owner-fresh","do":"expectNumber","equals":42,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"payment-amount"},{"actor":"owner-fresh","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"owner-fresh","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"owner-fresh","do":"expectNumber","equals":0,"testid":"credit-balance"}]}],"id":750,"setup":[{"actor":"owner","do":"signUp","name":"credit-owner-750"},{"actor":"owner","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"staff","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"staff","do":"fill","testid":"credit-amount-input","text":"10.00"},{"actor":"staff","do":"fill","testid":"credit-reference-input","text":"credit-750"},{"action":"grantCredit","actor":"staff","do":"callAction","from":"staff","input":{"attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","attribute":"data-account-id","testid":"credit-panel"}},"testid":"credit-grant"},"namedAction":{"args":[0,1000,""],"id":"grantCredit","method":"POST","params":[{"in":"body","name":"accountId","wireType":"u64"},{"in":"body","name":"amountMinor"},{"in":"body","name":"reference"}],"path":"/api/staff/credit","reducer":"grant_credit"},"settleMs":0},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"credit-owner-750","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"credit-link","unlessVisible":"credit-balance"},{"actor":"owner","do":"expectNumber","equals":10,"testid":"credit-balance","within":10000},{"actor":"owner","do":"click","testid":"catalog-link"},{"actor":"owner","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"add-to-cart"},{"actor":"owner","do":"click","testid":"cart-toggle"},{"actor":"owner","do":"expectNumber","equals":42,"testid":"cart-total"}]},"packId":"ecommerce.feature.store-credit","role":"feature","source":"scenarios/progression-store-credit.json"},{"checkGroupId":"production-752","feature":{"actors":["owner","staff","tab"],"criteria":[{"id":"752a","steps":[{"action":"grantCredit","actor":"staff","do":"callAction","from":"staff","input":{"attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","attribute":"data-account-id","testid":"credit-panel"}},"testid":"credit-grant"},"namedAction":{"args":[0,1000,""],"id":"grantCredit","method":"POST","params":[{"in":"body","name":"accountId","wireType":"u64"},{"in":"body","name":"amountMinor"},{"in":"body","name":"reference"}],"path":"/api/staff/credit","reducer":"grant_credit"},"settleMs":0},{"actor":"staff","do":"expectActionOutcome","outcome":"completed"},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"credit-owner-752"},{"actor":"owner-fresh","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"owner-fresh","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"owner-fresh","do":"expectNumber","equals":10,"testid":"credit-balance"}]}],"id":752,"setup":[{"actor":"owner","do":"signUp","name":"credit-owner-752"},{"actor":"owner","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"staff","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"staff","do":"fill","testid":"credit-amount-input","text":"10.00"},{"actor":"staff","do":"fill","testid":"credit-reference-input","text":"credit-752"},{"action":"grantCredit","actor":"staff","do":"callAction","from":"staff","input":{"attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","attribute":"data-account-id","testid":"credit-panel"}},"testid":"credit-grant"},"namedAction":{"args":[0,1000,""],"id":"grantCredit","method":"POST","params":[{"in":"body","name":"accountId","wireType":"u64"},{"in":"body","name":"amountMinor"},{"in":"body","name":"reference"}],"path":"/api/staff/credit","reducer":"grant_credit"},"settleMs":0},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"credit-owner-752","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"credit-link","unlessVisible":"credit-balance"},{"actor":"owner","do":"expectNumber","equals":10,"testid":"credit-balance","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.store-credit","requiresFeatures":["ecommerce.feature.store-credit"],"role":"guarantee","source":"scenarios/progression-store-credit.json"},{"checkGroupId":"production-753","feature":{"actors":["owner","staff","tab"],"criteria":[{"id":"753a","steps":[{"actor":"staff","do":"fill","testid":"credit-reference-input","text":"unauthorized-credit-753"},{"action":"grantCredit","actor":"owner","do":"callAction","from":"staff","input":{"attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","attribute":"data-account-id","testid":"credit-panel"}},"testid":"credit-grant"},"namedAction":{"args":[0,1000,""],"id":"grantCredit","method":"POST","params":[{"in":"body","name":"accountId","wireType":"u64"},{"in":"body","name":"amountMinor"},{"in":"body","name":"reference"}],"path":"/api/staff/credit","reducer":"grant_credit"},"settleMs":0},{"actor":"owner","do":"expectActionOutcome","outcome":"refused"},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"credit-owner-753"},{"actor":"owner-fresh","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"owner-fresh","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"owner-fresh","do":"expectNumber","equals":10,"testid":"credit-balance"}]}],"id":753,"setup":[{"actor":"owner","do":"signUp","name":"credit-owner-753"},{"actor":"owner","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"staff","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"staff","do":"fill","testid":"credit-amount-input","text":"10.00"},{"actor":"staff","do":"fill","testid":"credit-reference-input","text":"credit-753"},{"action":"grantCredit","actor":"staff","do":"callAction","from":"staff","input":{"attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","attribute":"data-account-id","testid":"credit-panel"}},"testid":"credit-grant"},"namedAction":{"args":[0,1000,""],"id":"grantCredit","method":"POST","params":[{"in":"body","name":"accountId","wireType":"u64"},{"in":"body","name":"amountMinor"},{"in":"body","name":"reference"}],"path":"/api/staff/credit","reducer":"grant_credit"},"settleMs":0},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"credit-owner-753","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"credit-link","unlessVisible":"credit-balance"},{"actor":"owner","do":"expectNumber","equals":10,"testid":"credit-balance","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.store-credit","requiresFeatures":["ecommerce.feature.store-credit"],"role":"guarantee","source":"scenarios/progression-store-credit.json"},{"checkGroupId":"production-754","feature":{"actors":["owner","staff","tab"],"criteria":[{"id":"754a","steps":[{"action":"checkoutCredit","actors":["owner","tab"],"do":"callConcurrently","namedAction":{"args":[],"id":"checkoutCredit","method":"POST","path":"/api/checkout/credit","reducer":"checkout_credit"},"settleMs":0},{"do":"expectCallOutcomes"},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"credit-owner-754"},{"actor":"owner-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner-fresh","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner-fresh","contains":"Desk Lamp","do":"expectElementCount","equals":1,"testid":"order-item","within":10000},{"actor":"owner-fresh","do":"expectNumber","equals":10,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"payment-credit-amount"},{"actor":"owner-fresh","do":"expectNumber","equals":32,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"payment-external-amount"},{"actor":"owner-fresh","do":"expectNumber","equals":42,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"payment-amount"},{"actor":"owner-fresh","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"owner-fresh","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"owner-fresh","do":"expectNumber","equals":0,"testid":"credit-balance"}]}],"id":754,"setup":[{"actor":"owner","do":"signUp","name":"credit-owner-754"},{"actor":"owner","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"staff","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"staff","do":"fill","testid":"credit-amount-input","text":"10.00"},{"actor":"staff","do":"fill","testid":"credit-reference-input","text":"credit-754"},{"action":"grantCredit","actor":"staff","do":"callAction","from":"staff","input":{"attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","attribute":"data-account-id","testid":"credit-panel"}},"testid":"credit-grant"},"namedAction":{"args":[0,1000,""],"id":"grantCredit","method":"POST","params":[{"in":"body","name":"accountId","wireType":"u64"},{"in":"body","name":"amountMinor"},{"in":"body","name":"reference"}],"path":"/api/staff/credit","reducer":"grant_credit"},"settleMs":0},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"credit-owner-754","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"credit-link","unlessVisible":"credit-balance"},{"actor":"owner","do":"expectNumber","equals":10,"testid":"credit-balance","within":10000},{"actor":"owner","do":"click","testid":"catalog-link"},{"actor":"owner","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"add-to-cart"},{"actor":"owner","do":"click","testid":"cart-toggle"},{"actor":"owner","do":"expectNumber","equals":42,"testid":"cart-total"},{"actor":"tab","do":"signIn","name":"credit-owner-754"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.store-credit","requiresFeatures":["ecommerce.feature.store-credit"],"role":"guarantee","source":"scenarios/progression-store-credit.json"},{"checkGroupId":"production-755","feature":{"actors":["owner","staff","tab"],"criteria":[{"id":"755a","steps":[{"do":"restartBackend","settleMs":1000},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"credit-owner-755"},{"actor":"owner-fresh","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"owner-fresh","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"owner-fresh","do":"expectNumber","equals":10,"testid":"credit-balance"}]}],"id":755,"setup":[{"actor":"owner","do":"signUp","name":"credit-owner-755"},{"actor":"owner","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"staff","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"staff","do":"fill","testid":"credit-amount-input","text":"10.00"},{"actor":"staff","do":"fill","testid":"credit-reference-input","text":"credit-755"},{"action":"grantCredit","actor":"staff","do":"callAction","from":"staff","input":{"attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","attribute":"data-account-id","testid":"credit-panel"}},"testid":"credit-grant"},"namedAction":{"args":[0,1000,""],"id":"grantCredit","method":"POST","params":[{"in":"body","name":"accountId","wireType":"u64"},{"in":"body","name":"amountMinor"},{"in":"body","name":"reference"}],"path":"/api/staff/credit","reducer":"grant_credit"},"settleMs":0},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"credit-owner-755","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"credit-link","unlessVisible":"credit-balance"},{"actor":"owner","do":"expectNumber","equals":10,"testid":"credit-balance","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.store-credit","requiresFeatures":["ecommerce.feature.store-credit"],"role":"guarantee","source":"scenarios/progression-store-credit.json"}],"id":"selected-source-105","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-store-credit.json"},{"checkGroups":[{"checkGroupId":"subscriptions-760","feature":{"actors":["owner","other"],"criteria":[{"id":"760a","steps":[{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"subscription-760"},{"actor":"owner-fresh","do":"click","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"},{"actor":"owner-fresh","attribute":"aria-busy","do":"expect","testid":"subscriptions-panel","value":"false","within":10000},{"actor":"owner-fresh","do":"expect","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-status","value":"complete","within":90000},{"actor":"owner-fresh","do":"expectElementCount","equals":2,"in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-delivery"},{"actor":"owner-fresh","do":"expectSequence","equals":["paid","paid"],"in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-delivery-status"},{"actor":"owner-fresh","do":"expectNumber","equals":84,"in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-total"},{"do":"dbExpectStock","equals":8,"item":"Desk Lamp"},{"actor":"owner-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner-fresh","do":"click","testid":"orders-toggle"},{"actor":"owner-fresh","contains":"Desk Lamp","do":"expectElementCount","equals":2,"testid":"order-item","within":10000},{"actor":"owner-fresh","do":"expectElementCount","equals":2,"testid":"payment-record"}]}],"id":760,"setup":[{"actor":"owner","do":"signUp","name":"subscription-760"},{"do":"dbSetStock","item":"Desk Lamp","quantity":10,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":0,"warehouse":"West"},{"actor":"owner","do":"click","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"subscriptions-panel","value":"false","within":10000},{"actor":"owner","do":"fill","testid":"subscription-item-input","text":"Desk Lamp"},{"actor":"owner","do":"fill","testid":"subscription-quantity-input","text":"1"},{"actor":"owner","do":"fill","testid":"subscription-interval-input","text":"30"},{"actor":"owner","do":"fill","testid":"subscription-deliveries-input","text":"2"},{"actor":"owner","do":"click","testid":"subscription-create"},{"actor":"owner","do":"expect","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-status","value":"active","within":10000}]},"packId":"ecommerce.feature.subscriptions","role":"feature","source":"scenarios/progression-subscriptions.json"},{"checkGroupId":"production-761","feature":{"actors":["owner","other"],"criteria":[{"id":"761a","steps":[{"do":"restartBackend","settleMs":1000},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"subscription-761"},{"actor":"owner-fresh","do":"click","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"},{"actor":"owner-fresh","attribute":"aria-busy","do":"expect","testid":"subscriptions-panel","value":"false","within":10000},{"actor":"owner-fresh","do":"expect","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-status","value":"complete","within":90000},{"actor":"owner-fresh","do":"expectElementCount","equals":2,"in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-delivery"},{"actor":"owner-fresh","do":"expectSequence","equals":["paid","paid"],"in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-delivery-status"},{"actor":"owner-fresh","do":"expectNumber","equals":84,"in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-total"},{"do":"dbExpectStock","equals":8,"item":"Desk Lamp"},{"actor":"owner-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner-fresh","do":"click","testid":"orders-toggle"},{"actor":"owner-fresh","contains":"Desk Lamp","do":"expectElementCount","equals":2,"testid":"order-item","within":10000},{"actor":"owner-fresh","do":"expectElementCount","equals":2,"testid":"payment-record"},{"do":"restartBackend","settleMs":1000},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"subscription-761"},{"actor":"owner-fresh","do":"click","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"},{"actor":"owner-fresh","attribute":"aria-busy","do":"expect","testid":"subscriptions-panel","value":"false","within":10000},{"actor":"owner-fresh","do":"expect","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-status","value":"complete","within":90000},{"actor":"owner-fresh","do":"expectElementCount","equals":2,"in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-delivery"},{"actor":"owner-fresh","do":"expectSequence","equals":["paid","paid"],"in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-delivery-status"},{"actor":"owner-fresh","do":"expectNumber","equals":84,"in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-total"},{"do":"dbExpectStock","equals":8,"item":"Desk Lamp"},{"actor":"owner-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner-fresh","do":"click","testid":"orders-toggle"},{"actor":"owner-fresh","contains":"Desk Lamp","do":"expectElementCount","equals":2,"testid":"order-item","within":10000},{"actor":"owner-fresh","do":"expectElementCount","equals":2,"testid":"payment-record"}]}],"id":761,"setup":[{"actor":"owner","do":"signUp","name":"subscription-761"},{"do":"dbSetStock","item":"Desk Lamp","quantity":10,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":0,"warehouse":"West"},{"actor":"owner","do":"click","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"subscriptions-panel","value":"false","within":10000},{"actor":"owner","do":"fill","testid":"subscription-item-input","text":"Desk Lamp"},{"actor":"owner","do":"fill","testid":"subscription-quantity-input","text":"1"},{"actor":"owner","do":"fill","testid":"subscription-interval-input","text":"30"},{"actor":"owner","do":"fill","testid":"subscription-deliveries-input","text":"2"},{"actor":"owner","do":"click","testid":"subscription-create"},{"actor":"owner","do":"expect","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-status","value":"active","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.subscriptions","requiresFeatures":["ecommerce.feature.subscriptions"],"role":"guarantee","source":"scenarios/progression-subscriptions.json"},{"checkGroupId":"production-762","feature":{"actors":["owner","other"],"criteria":[{"id":"762a","steps":[{"action":"cancelSubscription","actor":"other","do":"callAction","from":"owner","input":{"attribute":"data-action-input","testid":"subscription-cancel"},"namedAction":{"args":[0],"id":"cancelSubscription","method":"POST","params":[{"in":"path","name":"subscriptionId","placeholder":"{subscriptionId}","wireType":"u64"}],"path":"/api/subscriptions/{subscriptionId}/cancel","reducer":"cancel_subscription"},"settleMs":0},{"actor":"other","do":"expectActionOutcome","outcome":"refused"},{"actor":"owner","do":"expect","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-status","value":"active","within":10000},{"actor":"owner","do":"click","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-cancel"},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"subscription-762"},{"actor":"owner-fresh","do":"click","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"},{"actor":"owner-fresh","attribute":"aria-busy","do":"expect","testid":"subscriptions-panel","value":"false","within":10000},{"actor":"owner-fresh","do":"expect","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-status","value":"cancelled","within":10000},{"actor":"owner-fresh","as":"deliveries-at-cancel","count":true,"do":"recordNumber","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-delivery"},{"as":"stock-at-cancel","do":"dbRecordStock","item":"Desk Lamp"},{"actor":"owner-fresh","do":"wait","ms":65000},{"actor":"owner-fresh","do":"expectElementCount","in":{"contains":"Desk Lamp","testid":"subscription-row"},"plus":0,"relativeTo":"deliveries-at-cancel","testid":"subscription-delivery"},{"do":"dbExpectStock","item":"Desk Lamp","plus":0,"relativeTo":"stock-at-cancel"}]}],"id":762,"setup":[{"actor":"other","do":"signUp","name":"subscription-other-762"},{"actor":"owner","do":"signUp","name":"subscription-762"},{"do":"dbSetStock","item":"Desk Lamp","quantity":10,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":0,"warehouse":"West"},{"actor":"owner","do":"click","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"subscriptions-panel","value":"false","within":10000},{"actor":"owner","do":"fill","testid":"subscription-item-input","text":"Desk Lamp"},{"actor":"owner","do":"fill","testid":"subscription-quantity-input","text":"1"},{"actor":"owner","do":"fill","testid":"subscription-interval-input","text":"30"},{"actor":"owner","do":"fill","testid":"subscription-deliveries-input","text":"2"},{"actor":"owner","do":"click","testid":"subscription-create"},{"actor":"owner","do":"expect","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-status","value":"active","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.subscriptions","requiresFeatures":["ecommerce.feature.subscriptions"],"role":"guarantee","source":"scenarios/progression-subscriptions.json"},{"checkGroupId":"production-763","feature":{"actors":["owner","other"],"criteria":[{"id":"763a","steps":[{"actor":"owner","do":"click","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-pause"},{"actor":"owner","do":"expect","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-status","value":"paused","within":10000},{"do":"restartBackend","settleMs":1000},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"subscription-763"},{"actor":"owner-fresh","do":"click","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"},{"actor":"owner-fresh","attribute":"aria-busy","do":"expect","testid":"subscriptions-panel","value":"false","within":10000},{"actor":"owner-fresh","do":"expect","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-status","value":"paused","within":10000},{"actor":"owner-fresh","as":"deliveries-at-pause","count":true,"do":"recordNumber","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-delivery"},{"as":"stock-at-pause","do":"dbRecordStock","item":"Desk Lamp"},{"actor":"owner-fresh","do":"wait","ms":35000},{"actor":"owner-fresh","do":"expectElementCount","in":{"contains":"Desk Lamp","testid":"subscription-row"},"plus":0,"relativeTo":"deliveries-at-pause","testid":"subscription-delivery"},{"do":"dbExpectStock","item":"Desk Lamp","plus":0,"relativeTo":"stock-at-pause"},{"actor":"owner-fresh","do":"click","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-resume"},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"subscription-763"},{"actor":"owner-fresh","do":"click","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"},{"actor":"owner-fresh","attribute":"aria-busy","do":"expect","testid":"subscriptions-panel","value":"false","within":10000},{"actor":"owner-fresh","do":"expect","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-status","value":"complete","within":90000},{"actor":"owner-fresh","do":"expectElementCount","equals":2,"in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-delivery"},{"actor":"owner-fresh","do":"expectSequence","equals":["paid","paid"],"in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-delivery-status"},{"actor":"owner-fresh","do":"expectNumber","equals":84,"in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-total"},{"do":"dbExpectStock","equals":8,"item":"Desk Lamp"},{"actor":"owner-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner-fresh","do":"click","testid":"orders-toggle"},{"actor":"owner-fresh","contains":"Desk Lamp","do":"expectElementCount","equals":2,"testid":"order-item","within":10000},{"actor":"owner-fresh","do":"expectElementCount","equals":2,"testid":"payment-record"}]}],"id":763,"setup":[{"actor":"owner","do":"signUp","name":"subscription-763"},{"do":"dbSetStock","item":"Desk Lamp","quantity":10,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":0,"warehouse":"West"},{"actor":"owner","do":"click","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"subscriptions-panel","value":"false","within":10000},{"actor":"owner","do":"fill","testid":"subscription-item-input","text":"Desk Lamp"},{"actor":"owner","do":"fill","testid":"subscription-quantity-input","text":"1"},{"actor":"owner","do":"fill","testid":"subscription-interval-input","text":"30"},{"actor":"owner","do":"fill","testid":"subscription-deliveries-input","text":"2"},{"actor":"owner","do":"click","testid":"subscription-create"},{"actor":"owner","do":"expect","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-status","value":"active","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.subscriptions","requiresFeatures":["ecommerce.feature.subscriptions"],"role":"guarantee","source":"scenarios/progression-subscriptions.json"}],"id":"selected-source-106","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-subscriptions.json"},{"checkGroups":[{"checkGroupId":"support-history","feature":{"actors":["owner","other"],"criteria":[{"id":"612c","steps":[{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"support-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"support-link","unlessVisible":"support-ticket"},{"actor":"owner","contains":"Owner ticket {user:ticketmarker}","do":"expect","testid":"support-ticket"}]}],"id":612,"setup":[{"actor":"owner","do":"signUp","name":"support-owner"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-email","text":"owner@example.com"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Owner ticket {user:ticketmarker}"},{"actor":"owner","do":"fill","testid":"support-message","text":"Private account issue."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","do":"expect","nonEmpty":true,"testid":"support-reference"},{"actor":"other","do":"signUp","name":"support-other"}]},"packId":"ecommerce.progression.support-history","role":"feature","source":"scenarios/progression-support-history.json"},{"checkGroupId":"support-history-reload","feature":{"actors":["owner","other"],"criteria":[{"id":"612a","steps":[{"actor":"owner","do":"reload","settleMs":3000},{"actor":"owner","do":"ensureSignedIn","name":"support-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","contains":"Owner ticket {user:ticketmarker}","do":"expect","testid":"support-ticket"},{"do":"restartBackend","settleMs":1000},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"support-owner"},{"actor":"owner-fresh","do":"click","testid":"support-link"},{"actor":"owner-fresh","contains":"Owner ticket {user:ticketmarker}","do":"expect","testid":"support-ticket"}]}],"id":612,"setup":[{"actor":"owner","do":"signUp","name":"support-owner"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-email","text":"owner@example.com"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Owner ticket {user:ticketmarker}"},{"actor":"owner","do":"fill","testid":"support-message","text":"Private account issue."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","do":"expect","nonEmpty":true,"testid":"support-reference"},{"actor":"other","do":"signUp","name":"support-other"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","requiresFeatures":["ecommerce.progression.support-history"],"role":"guarantee","source":"scenarios/progression-support-history.json"},{"checkGroupId":"support-history-privacy","feature":{"actors":["owner","other"],"criteria":[{"id":"612b","steps":[{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"support-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-email","text":"owner@example.com"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Private ticket {user:privateticketmarker}"},{"actor":"owner","do":"fill","testid":"support-message","text":"Private account issue."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","do":"expect","nonEmpty":true,"testid":"support-reference"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"support-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","contains":"Private ticket {user:privateticketmarker}","do":"expect","testid":"support-ticket"},{"actor":"owner","contains":"Private ticket {user:privateticketmarker}","do":"expectReceived","within":10000},{"actor":"other","do":"reload","settleMs":0},{"actor":"other","do":"ensureSignedIn","name":"support-other","readyTestid":"current-user"},{"actor":"other","do":"click","testid":"support-link"},{"absent":true,"actor":"other","contains":"Private ticket {user:privateticketmarker}","do":"expect","testid":"support-ticket"},{"actor":"other","contains":"Private ticket {user:privateticketmarker}","do":"expectNotReceived"}]}],"id":612,"setup":[{"actor":"owner","do":"signUp","name":"support-owner"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-email","text":"owner@example.com"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Owner ticket {user:ticketmarker}"},{"actor":"owner","do":"fill","testid":"support-message","text":"Private account issue."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","do":"expect","nonEmpty":true,"testid":"support-reference"},{"actor":"other","do":"signUp","name":"support-other"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.support-history"],"role":"guarantee","source":"scenarios/progression-support-history.json"},{"checkGroupId":"support-history-logout","feature":{"actors":["owner","other"],"criteria":[{"id":"612d","steps":[{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"support-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-email","text":"owner@example.com"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Logout ticket {user:logoutticketmarker}"},{"actor":"owner","do":"fill","testid":"support-message","text":"Private account issue."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","do":"expect","nonEmpty":true,"testid":"support-reference"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"support-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"support-link","unlessVisible":"support-ticket"},{"actor":"owner","contains":"Logout ticket {user:logoutticketmarker}","do":"expect","testid":"support-ticket"},{"actor":"owner","contains":"Logout ticket {user:logoutticketmarker}","do":"expectReceived","within":10000},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"support-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"current-user","unlessVisible":"signout"},{"actor":"owner","do":"click","testid":"signout"},{"actor":"owner","do":"waitUntilAbsent","testid":"current-user","within":6000},{"actor":"owner","do":"freshClient","preserveStorage":true},{"absent":true,"actor":"owner-fresh","do":"expect","testid":"current-user"},{"actor":"owner-fresh","do":"click","testid":"support-link"},{"actor":"owner-fresh","do":"expect","testid":"support-email"},{"absent":true,"actor":"owner-fresh","contains":"Logout ticket {user:logoutticketmarker}","do":"expect","testid":"support-ticket"},{"actor":"owner-fresh","contains":"Logout ticket {user:logoutticketmarker}","do":"expectNotReceived"}]}],"id":612,"setup":[{"actor":"owner","do":"signUp","name":"support-owner"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-email","text":"owner@example.com"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Owner ticket {user:ticketmarker}"},{"actor":"owner","do":"fill","testid":"support-message","text":"Private account issue."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","do":"expect","nonEmpty":true,"testid":"support-reference"},{"actor":"other","do":"signUp","name":"support-other"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.support-history"],"role":"guarantee","source":"scenarios/progression-support-history.json"}],"id":"selected-source-107","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/progression-support-history.json"},{"checkGroups":[{"checkGroupId":"support-intake","feature":{"actors":["visitor"],"criteria":[{"id":"610a","steps":[{"actor":"visitor","do":"click","testid":"support-link"},{"actor":"visitor","do":"fill","testid":"support-email","text":"visitor@example.com"},{"actor":"visitor","do":"fill","testid":"support-subject","text":"Damaged package"},{"actor":"visitor","do":"fill","testid":"support-message","text":"The package arrived damaged."},{"actor":"visitor","do":"click","testid":"support-submit"},{"actor":"visitor","do":"expect","nonEmpty":true,"testid":"support-reference"}]}],"id":610,"setup":[]},"packId":"ecommerce.progression.support-intake","role":"feature","source":"scenarios/progression-support-intake.json"}],"id":"selected-source-108","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/progression-support-intake.json"},{"checkGroups":[{"checkGroupId":"support-refund-access","feature":{"actors":["owner","staff"],"criteria":[{"id":"615c","steps":[{"absent":true,"actor":"owner","do":"expect","testid":"support-refund"},{"action":"supportRefund","actor":"owner","authentication":"actor","do":"callAction","from":"staff","input":{"attribute":"data-action-input","testid":"support-refund"},"namedAction":{"args":[0],"id":"supportRefund","method":"POST","params":[{"in":"path","name":"caseId","placeholder":"{caseId}","wireType":"u64"}],"path":"/api/support/cases/{caseId}/refund","reducer":"support_refund"},"settleMs":1500},{"actor":"owner","do":"expectActionOutcome","outcome":"refused"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"refund-access-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"support-link","unlessVisible":"support-subject"},{"actor":"owner","do":"expect","in":{"contains":"Access refund case","testid":"support-ticket"},"notContains":"resolved","testid":"support-status"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle"},{"actor":"owner","do":"expect","in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"pending"},{"absent":true,"actor":"owner","contains":"Keyboard","do":"expect","testid":"refund-entry"}]}],"id":615,"setup":[{"actor":"owner","do":"signUp","name":"refund-access-owner"},{"actor":"owner","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"buy-now"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Access refund case"},{"actor":"owner","do":"fill","testid":"support-message","text":"This refund requires staff approval."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","contains":"Keyboard","do":"click","in":{"contains":"Access refund case","testid":"support-ticket"},"testid":"support-order-option"},{"actor":"owner","do":"click","in":{"contains":"Access refund case","testid":"support-ticket"},"testid":"support-link-order"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee","within":1000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.support-refunds"],"role":"guarantee","source":"scenarios/progression-support-refunds-access.json"}],"id":"selected-source-109","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-support-refunds-access.json"},{"checkGroups":[{"checkGroupId":"support-refund-accounting","feature":{"actors":["owner","staff"],"criteria":[{"id":"615b","steps":[{"actor":"staff","do":"click","in":{"contains":"Accounting refund case","testid":"support-ticket"},"testid":"support-refund"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"refund-accounting-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"support-link","unlessVisible":"support-subject"},{"actor":"owner","do":"expectNumber","in":{"contains":"Accounting refund case","testid":"support-ticket"},"plus":0,"relativeTo":"paid-total","testid":"support-refund-total","within":10000},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner","do":"expectNumber","in":{"contains":"Keyboard","testid":"order-item"},"plus":0,"relativeTo":"paid-total","testid":"order-refund-total","within":10000},{"actor":"owner","contains":"Keyboard","do":"expectElementCount","equals":1,"testid":"refund-entry","within":10000},{"actor":"staff","do":"replayAs","from":"staff","match":"refund","namedAction":{"args":[0],"id":"supportRefund","method":"POST","params":[{"in":"path","name":"caseId","placeholder":"{caseId}","wireType":"u64"}],"path":"/api/support/cases/{caseId}/refund","reducer":"support_refund"},"namedTarget":{"attribute":"data-entity-id","contains":"Accounting refund case","testid":"support-ticket","valueType":"string"},"settleMs":1500},{"actor":"staff","do":"expectReplayCompleted"},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"refund-accounting-owner"},{"actor":"owner-fresh","do":"click","testid":"support-link"},{"actor":"owner-fresh","do":"expectNumber","in":{"contains":"Accounting refund case","testid":"support-ticket"},"plus":0,"relativeTo":"paid-total","testid":"support-refund-total","within":10000},{"actor":"owner-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner-fresh","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner-fresh","do":"expectNumber","in":{"contains":"Keyboard","testid":"order-item"},"plus":0,"relativeTo":"paid-total","testid":"order-refund-total","within":10000},{"actor":"owner-fresh","contains":"Keyboard","do":"expectElementCount","equals":1,"testid":"refund-entry","within":10000},{"actor":"owner-fresh","do":"expectNumber","equals":0,"in":{"contains":"Mouse","testid":"order-item"},"testid":"order-refund-total","within":10000},{"actor":"owner-fresh","contains":"Mouse","do":"expectElementCount","equals":0,"testid":"refund-entry","within":10000}]}],"id":615,"setup":[{"actor":"owner","do":"signUp","name":"refund-accounting-owner"},{"actor":"owner","do":"click","in":{"contains":"Mouse","testid":"item-card"},"testid":"buy-now"},{"actor":"owner","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"buy-now"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner","as":"paid-total","do":"recordNumber","in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-total"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Accounting refund case"},{"actor":"owner","do":"fill","testid":"support-message","text":"Refund the exact amount paid."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","contains":"Keyboard","do":"click","in":{"contains":"Accounting refund case","testid":"support-ticket"},"testid":"support-order-option"},{"actor":"owner","do":"click","in":{"contains":"Accounting refund case","testid":"support-ticket"},"testid":"support-link-order"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee","within":1000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","requiresFeatures":["ecommerce.progression.support-refunds"],"role":"guarantee","source":"scenarios/progression-support-refunds-accounting.json"}],"id":"selected-source-110","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-support-refunds-accounting.json"},{"checkGroups":[{"checkGroupId":"support-refunds-resolution","feature":{"actors":["owner","staff"],"criteria":[{"id":"615a","steps":[{"actor":"staff","do":"click","in":{"contains":"Resolution refund case","testid":"support-ticket"},"testid":"support-refund"},{"actor":"owner","contains":"resolved","do":"expect","in":{"contains":"Resolution refund case","testid":"support-ticket"},"testid":"support-status","within":10000},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle"},{"actor":"owner","contains":"refunded","do":"expect","in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","within":10000}]}],"id":615,"setup":[{"actor":"owner","do":"signUp","name":"refund-resolution-owner"},{"actor":"owner","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"buy-now"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Resolution refund case"},{"actor":"owner","do":"fill","testid":"support-message","text":"Please refund my Keyboard order."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","contains":"Keyboard","do":"click","in":{"contains":"Resolution refund case","testid":"support-ticket"},"testid":"support-order-option"},{"actor":"owner","do":"click","in":{"contains":"Resolution refund case","testid":"support-ticket"},"testid":"support-link-order"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee"}]},"packId":"ecommerce.progression.support-refunds","role":"feature","source":"scenarios/progression-support-refunds-resolution.json"}],"id":"selected-source-111","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-support-refunds-resolution.json"},{"checkGroups":[{"checkGroupId":"return-refund-interaction","feature":{"actors":["owner","staff","admin"],"criteria":[{"id":"757a","steps":[{"actor":"owner","do":"reload","settleMs":0},{"actor":"owner","do":"ensureSignedIn","name":"return-refund-owner","readyTestid":"current-user","settleMs":0},{"actor":"owner","do":"click","ifAvailable":true,"testid":"catalog-link"},{"as":"757atotal","do":"dbRecordStock","item":"Keyboard"},{"as":"757aEast","do":"dbRecordStock","item":"Keyboard","warehouse":"East"},{"as":"757aWest","do":"dbRecordStock","item":"Keyboard","warehouse":"West"},{"actor":"admin","as":"757arevenue","do":"recordNumber","testid":"admin-revenue"},{"actor":"owner","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"buy-now"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner","do":"expect","ignoreCase":true,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"pending","within":10000},{"actor":"owner","as":"757apaid","do":"recordNumber","in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-total"},{"action":"ship","actor":"staff","do":"callAction","from":"owner","input":{"attribute":"data-ship-input","contains":"Keyboard","testid":"order-item"},"namedAction":{"args":[0],"id":"ship","params":[{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/fulfilment/ship","reducer":"ship_order"},"settleMs":0},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"expect","ignoreCase":true,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"shipped","within":10000},{"do":"dbExpectStock","item":"Keyboard","plus":-1,"relativeTo":"757atotal"},{"actor":"owner","do":"reload","settleMs":0},{"actor":"owner","do":"ensureSignedIn","name":"return-refund-owner","readyTestid":"current-user","settleMs":0},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Return refund 757a"},{"actor":"owner","do":"fill","testid":"support-message","text":"Please refund this order."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","contains":"Return refund 757a","do":"expect","testid":"support-ticket","within":10000},{"actor":"owner","contains":"Keyboard","do":"click","in":{"contains":"Return refund 757a","testid":"support-ticket"},"testid":"support-order-option"},{"actor":"owner","do":"click","in":{"contains":"Return refund 757a","testid":"support-ticket"},"testid":"support-link-order"},{"actor":"owner","do":"expect","in":{"contains":"Return refund 757a","testid":"support-ticket"},"testid":"support-order","within":10000},{"action":"supportRefund","actor":"staff","do":"callAction","from":"owner","input":{"attribute":"data-refund-input","contains":"Return refund 757a","testid":"support-ticket"},"namedAction":{"args":[0],"id":"supportRefund","method":"POST","params":[{"in":"path","name":"caseId","placeholder":"{caseId}","wireType":"u64"}],"path":"/api/support/cases/{caseId}/refund","reducer":"support_refund"},"settleMs":0},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"do":"dbExpectStock","item":"Keyboard","plus":-1,"relativeTo":"757atotal"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner","do":"click","in":{"contains":"Keyboard","testid":"order-item"},"testid":"return-item"},{"actor":"owner","contains":"Keyboard","containsText":"returned","do":"expect","ignoreCase":true,"testid":"order-item","within":10000},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"return-refund-owner"},{"actor":"owner-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner-fresh","do":"click","testid":"orders-toggle"},{"actor":"owner-fresh","contains":"Keyboard","containsText":"returned","do":"expect","ignoreCase":true,"testid":"order-item","within":10000},{"actor":"owner-fresh","do":"expectNumber","in":{"contains":"Keyboard","testid":"order-item"},"plus":0,"relativeTo":"757apaid","testid":"order-refund-total","within":10000},{"do":"dbExpectStock","item":"Keyboard","plus":0,"relativeTo":"757aEast","warehouse":"East"},{"do":"dbExpectStock","item":"Keyboard","plus":0,"relativeTo":"757aWest","warehouse":"West"},{"actor":"admin","do":"reload","settleMs":0},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"expectNumber","plus":0,"relativeTo":"757arevenue","testid":"admin-revenue","within":10000}]},{"id":"757b","steps":[{"actor":"owner","do":"reload","settleMs":0},{"actor":"owner","do":"ensureSignedIn","name":"return-refund-owner","readyTestid":"current-user","settleMs":0},{"actor":"owner","do":"click","ifAvailable":true,"testid":"catalog-link"},{"as":"757btotal","do":"dbRecordStock","item":"Desk Lamp"},{"as":"757bEast","do":"dbRecordStock","item":"Desk Lamp","warehouse":"East"},{"as":"757bWest","do":"dbRecordStock","item":"Desk Lamp","warehouse":"West"},{"actor":"admin","as":"757brevenue","do":"recordNumber","testid":"admin-revenue"},{"actor":"owner","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"buy-now"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner","do":"expect","ignoreCase":true,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-status","value":"pending","within":10000},{"actor":"owner","as":"757bpaid","do":"recordNumber","in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-total"},{"action":"ship","actor":"staff","do":"callAction","from":"owner","input":{"attribute":"data-ship-input","contains":"Desk Lamp","testid":"order-item"},"namedAction":{"args":[0],"id":"ship","params":[{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/fulfilment/ship","reducer":"ship_order"},"settleMs":0},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"expect","ignoreCase":true,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-status","value":"shipped","within":10000},{"do":"dbExpectStock","item":"Desk Lamp","plus":-1,"relativeTo":"757btotal"},{"actor":"owner","do":"reload","settleMs":0},{"actor":"owner","do":"ensureSignedIn","name":"return-refund-owner","readyTestid":"current-user","settleMs":0},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Return refund 757b"},{"actor":"owner","do":"fill","testid":"support-message","text":"Please refund this order."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","contains":"Return refund 757b","do":"expect","testid":"support-ticket","within":10000},{"actor":"owner","contains":"Desk Lamp","do":"click","in":{"contains":"Return refund 757b","testid":"support-ticket"},"testid":"support-order-option"},{"actor":"owner","do":"click","in":{"contains":"Return refund 757b","testid":"support-ticket"},"testid":"support-link-order"},{"actor":"owner","do":"expect","in":{"contains":"Return refund 757b","testid":"support-ticket"},"testid":"support-order","within":10000},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner","do":"click","in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"return-item"},{"actor":"owner","contains":"Desk Lamp","containsText":"returned","do":"expect","ignoreCase":true,"testid":"order-item","within":10000},{"action":"supportRefund","actor":"staff","do":"callAction","from":"owner","input":{"attribute":"data-refund-input","contains":"Return refund 757b","testid":"support-ticket"},"namedAction":{"args":[0],"id":"supportRefund","method":"POST","params":[{"in":"path","name":"caseId","placeholder":"{caseId}","wireType":"u64"}],"path":"/api/support/cases/{caseId}/refund","reducer":"support_refund"},"settleMs":0},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"return-refund-owner"},{"actor":"owner-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner-fresh","do":"click","testid":"orders-toggle"},{"actor":"owner-fresh","contains":"Desk Lamp","containsText":"returned","do":"expect","ignoreCase":true,"testid":"order-item","within":10000},{"actor":"owner-fresh","do":"expectNumber","in":{"contains":"Desk Lamp","testid":"order-item"},"plus":0,"relativeTo":"757bpaid","testid":"order-refund-total","within":10000},{"do":"dbExpectStock","item":"Desk Lamp","plus":0,"relativeTo":"757bEast","warehouse":"East"},{"do":"dbExpectStock","item":"Desk Lamp","plus":0,"relativeTo":"757bWest","warehouse":"West"},{"actor":"admin","do":"reload","settleMs":0},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"expectNumber","plus":0,"relativeTo":"757brevenue","testid":"admin-revenue","within":10000}]}],"id":757,"setup":[{"actor":"owner","do":"signUp","name":"return-refund-owner"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee"},{"actor":"admin","do":"click","testid":"admin-link"}]},"packId":"ecommerce.feature.split-tender-refunds","requiresFeatures":["ecommerce.l3.order-returns-features","ecommerce.progression.fulfilment-queue","ecommerce.progression.support-refunds"],"role":"feature","source":"scenarios/progression-support-return-interaction.json"}],"id":"selected-source-112","scenario":{"level":6,"writeUrlPattern":null},"source":"scenarios/progression-support-return-interaction.json"},{"checkGroups":[{"checkGroupId":"support-assignment","feature":{"actors":["visitor","staff"],"criteria":[{"id":"611a","steps":[{"actor":"staff","do":"fill","in":{"contains":"Missing item","testid":"support-ticket"},"testid":"support-assignee","text":"staff"},{"actor":"staff","do":"click","in":{"contains":"Missing item","testid":"support-ticket"},"settleMs":1500,"testid":"support-update"},{"actor":"staff","do":"reload","settleMs":2500},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"expect","in":{"contains":"Missing item","testid":"support-ticket"},"testid":"support-assignee","value":"staff"}]}],"id":611,"setup":[{"actor":"visitor","do":"click","testid":"support-link"},{"actor":"visitor","do":"fill","testid":"support-email","text":"triage@example.com"},{"actor":"visitor","do":"fill","testid":"support-subject","text":"Missing item"},{"actor":"visitor","do":"fill","testid":"support-message","text":"One item is missing."},{"actor":"visitor","do":"click","testid":"support-submit"},{"actor":"visitor","do":"expect","nonEmpty":true,"testid":"support-reference"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee"}]},"packId":"ecommerce.progression.support-triage","role":"feature","source":"scenarios/progression-support-triage.json"},{"checkGroupId":"support-priority","feature":{"actors":["visitor","staff"],"criteria":[{"id":"611b","steps":[{"actor":"staff","do":"fill","in":{"contains":"Missing item","testid":"support-ticket"},"testid":"support-priority","text":"high"},{"actor":"staff","do":"click","in":{"contains":"Missing item","testid":"support-ticket"},"settleMs":1500,"testid":"support-update"},{"actor":"staff","do":"reload","settleMs":2500},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"expect","in":{"contains":"Missing item","testid":"support-ticket"},"testid":"support-priority","value":"high"}]}],"id":611,"setup":[{"actor":"visitor","do":"click","testid":"support-link"},{"actor":"visitor","do":"fill","testid":"support-email","text":"triage@example.com"},{"actor":"visitor","do":"fill","testid":"support-subject","text":"Missing item"},{"actor":"visitor","do":"fill","testid":"support-message","text":"One item is missing."},{"actor":"visitor","do":"click","testid":"support-submit"},{"actor":"visitor","do":"expect","nonEmpty":true,"testid":"support-reference"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee"}]},"packId":"ecommerce.progression.support-triage","role":"feature","source":"scenarios/progression-support-triage.json"},{"checkGroupId":"support-status","feature":{"actors":["visitor","staff"],"criteria":[{"id":"611c","steps":[{"actor":"staff","do":"fill","in":{"contains":"Missing item","testid":"support-ticket"},"testid":"support-status-input","text":"in progress"},{"actor":"staff","do":"click","in":{"contains":"Missing item","testid":"support-ticket"},"settleMs":1500,"testid":"support-update"},{"actor":"staff","contains":"in progress","do":"expect","in":{"contains":"Missing item","testid":"support-ticket"},"testid":"support-status"}]}],"id":611,"setup":[{"actor":"visitor","do":"click","testid":"support-link"},{"actor":"visitor","do":"fill","testid":"support-email","text":"triage@example.com"},{"actor":"visitor","do":"fill","testid":"support-subject","text":"Missing item"},{"actor":"visitor","do":"fill","testid":"support-message","text":"One item is missing."},{"actor":"visitor","do":"click","testid":"support-submit"},{"actor":"visitor","do":"expect","nonEmpty":true,"testid":"support-reference"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee"}]},"packId":"ecommerce.progression.support-triage","role":"feature","source":"scenarios/progression-support-triage.json"}],"id":"selected-source-113","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/progression-support-triage.json"}],"fixture":{"accounts":[{"password":"stackbench-admin-2026","roles":["admin"],"username":"admin"},{"password":"stackbench-staff-2026","roles":["staff"],"username":"staff"}],"empty":["carts","orders","reviews","returns"],"items":[{"category":"Home","name":"Air Purifier","price":"189.00","stock":{"East":60,"West":40}},{"category":"Audio","name":"Bluetooth Speaker","price":"79.50","stock":{"East":50,"West":50}},{"category":"Home","name":"Coffee Grinder","price":"64.00","stock":{"East":70,"West":30}},{"category":"Home","name":"Desk Lamp","price":"42.00","stock":{"East":55,"West":45}},{"category":"Home","name":"Espresso Machine","price":"449.00","stock":{"East":80,"West":20}},{"category":"Computing","name":"Gaming Mouse","price":"59.00","stock":{"East":50,"West":50}},{"category":"Audio","name":"Headphones","price":"199.00","stock":{"East":60,"West":40}},{"category":"Home","name":"Induction Cooktop","price":"329.00","stock":{"East":50,"West":50}},{"category":"Computing","name":"Keyboard","price":"89.00","stock":{"East":70,"West":30}},{"category":"Computing","name":"Laptop Stand","price":"29.00","stock":{"East":90,"West":10}},{"category":"Photo","name":"Mirrorless Camera","price":"1299.00","stock":{"East":2,"West":1}},{"category":"Home","name":"USB Cable","price":"65.00","stock":{"East":0,"West":0}},{"category":"Computing","name":"Webcam","price":"69.00","stock":{"East":60,"West":40}}],"warehouses":["East","West"]},"packs":[{"actions":["click","expect","signIn","signUp","waitUntilAbsent"],"budget":{"maxRuntimeMs":18000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.feature.accounts","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","dbExpectStock","dbSetStock","expect","expectNumber","fill","signIn","signUp"],"budget":{"maxRuntimeMs":120000,"status":"bounded"},"capabilities":["browser","database-observation","direct-database-write"],"evidence":["browser-observation","database-observation"],"id":"ecommerce.feature.bundle-checkout","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","dbExpectStock","dbSetStock","expect","expectNumber","fill","signIn","signUp"],"budget":{"maxRuntimeMs":120000,"status":"bounded"},"capabilities":["browser","database-observation","direct-database-write"],"evidence":["browser-observation","database-observation"],"id":"ecommerce.feature.bundle-returns","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","expect","expectNumber","signUp","wait"],"budget":{"maxRuntimeMs":42000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.feature.cart","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.feature.cart-checkout"},{"actions":["expect","expectSequence","fill"],"budget":{"maxRuntimeMs":30000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.feature.catalog-discovery","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.feature.catalog"},{"actions":["expect","expectNumber"],"budget":{"maxRuntimeMs":30000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.feature.catalog-items","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.feature.catalog"},{"actions":["click","ensureSignedIn","expect","expectNumber","reload","signUp","wait"],"budget":{"maxRuntimeMs":42000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.feature.checkout","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.feature.cart-checkout"},{"actions":["click","expect","expectElementCount","expectNumber","fill","reload","signIn"],"budget":{"maxRuntimeMs":120000,"status":"bounded"},"capabilities":["browser","database-observation","direct-database-write"],"evidence":["browser-observation","database-observation"],"id":"ecommerce.feature.product-bundles","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","expect","expectNumber","signUp"],"budget":{"maxRuntimeMs":24000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.feature.purchasing","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","expect","fill","openItem","signUp"],"budget":{"maxRuntimeMs":22000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.feature.reviews","includeRoles":["feature"],"moduleType":"feature"},{"actions":["callAction","click","dbExpectStock","dbRecordStock","ensureSignedIn","expect","expectActionOutcome","expectElementCount","expectNumber","fill","freshClient","recordNumber","reload","signIn","signUp"],"budget":{"maxRuntimeMs":120000,"status":"bounded"},"capabilities":["browser","database-observation","direct-server-call"],"evidence":["browser-observation","database-observation","fresh-client-observation"],"id":"ecommerce.feature.split-tender-refunds","includeRoles":["feature"],"moduleType":"feature"},{"actions":["callAction","click","ensureSignedIn","expect","expectActionOutcome","expectElementCount","expectNumber","fill","freshClient","reload","signIn","signUp"],"budget":{"maxRuntimeMs":180000,"status":"bounded"},"capabilities":["browser","direct-server-call"],"evidence":["browser-observation","fresh-client-observation"],"id":"ecommerce.feature.store-credit","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","dbExpectStock","dbSetStock","expect","expectElementCount","expectNumber","expectSequence","fill","freshClient","signIn","signUp"],"budget":{"maxRuntimeMs":180000,"status":"bounded"},"capabilities":["browser","database-observation","direct-server-call"],"evidence":["browser-observation","database-observation","fresh-client-observation"],"id":"ecommerce.feature.subscriptions","includeRoles":["feature"],"moduleType":"feature"},{"actions":["callAction","click","ensureSignedIn","expect","expectActionOutcome","expectNumber","recordNumber","reload","signIn"],"budget":{"maxRuntimeMs":30000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.feature.warehouse-admin","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","dbSetStock","ensureSignedIn","expect","reload","signIn","signUp"],"budget":{"maxRuntimeMs":86000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.l2.inventory-dashboard","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.inventory-operations"},{"actions":["click","ensureSignedIn","expect","expectNumber","recordNumber","reload","signIn","signUp"],"budget":{"maxRuntimeMs":60000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.l2.order-cancellation-features","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.returns-pricing"},{"actions":["click","expectNumber","fill","signIn"],"budget":{"maxRuntimeMs":94000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.l2.price-history-features","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.returns-pricing"},{"actions":["click","expect","signUp","waitUntilAbsent"],"budget":{"maxRuntimeMs":86000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.l2.recommendations","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.inventory-operations"},{"actions":["click","ensureSignedIn","expect","expectNumber","recordNumber","reload","signIn","signUp"],"budget":{"maxRuntimeMs":86000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.l2.sales-dashboard","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.inventory-operations"},{"actions":["click","dbExpectStock","dbRecordStock","expect","expectNumber","fill","recordNumber","signIn"],"budget":{"maxRuntimeMs":86000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.l2.stock-transfers-features","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.inventory-operations"},{"actions":["click","closeClient","expect","expectNumber","openClient","recordNumber","reload","signUp","wait"],"budget":{"maxRuntimeMs":360000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.l3.cart-expiration-features","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.l3.cart-expiration"},{"actions":["callAction","click","expect","expectActionOutcome","expectReplayRejected","fill","replayAs","signIn","signUp"],"budget":{"maxRuntimeMs":60000,"status":"bounded"},"capabilities":["browser","direct-server-call","request-replay"],"evidence":["browser-observation","server-refusal","server-response"],"id":"ecommerce.l3.deferred-access-specifications","includeRoles":["guarantee"],"moduleType":"specification","stableId":"ecommerce.l3.deferred-access"},{"actions":["click","dbExpectStock","dbRecordStock","ensureSignedIn","expect","expectElapsed","expectNumber","fill","recordNumber","recordTime","reload","restartBackend","signIn","signUp","wait"],"budget":{"maxRuntimeMs":720000,"status":"bounded"},"capabilities":["backend-lifecycle","browser"],"evidence":["browser-observation","database-observation"],"id":"ecommerce.l3.deferred-durability-specifications","includeRoles":["guarantee"],"moduleType":"specification","stableId":"ecommerce.l3.deferred-durability"},{"actions":["click","dbExpectStock","dbRecordStock","ensureSignedIn","expect","expectElementCount","expectNumber","fill","recordNumber","reload","restartBackend","signIn","signUp","wait"],"budget":{"maxRuntimeMs":400000,"status":"bounded"},"capabilities":["backend-lifecycle","browser"],"evidence":["browser-observation","database-observation"],"id":"ecommerce.l3.deferred-integrity-specifications","includeRoles":["guarantee"],"moduleType":"specification","stableId":"ecommerce.l3.deferred-integrity"},{"actions":["click","ensureSignedIn","expect","reload","signIn","signUp","wait"],"budget":{"maxRuntimeMs":190000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.l3.order-delivery-features","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.l3.order-delivery"},{"actions":["callAction","click","dbExpectStock","dbRecordStock","ensureSignedIn","expect","expectActionOutcome","expectNumber","freshClient","recordNumber","reload","signIn","signUp"],"budget":{"maxRuntimeMs":120000,"status":"bounded"},"capabilities":["browser","database-observation"],"evidence":["browser-observation","database-observation","fresh-client-observation"],"id":"ecommerce.l3.order-returns-features","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.returns-pricing"},{"actions":["click","expect","expectNumber","recordNumber","signUp","wait"],"budget":{"maxRuntimeMs":300000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.l3.reservations-features","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.l3.reservations"},{"actions":["click","ensureSignedIn","expect","expectElementCount","expectNumber","fill","recordNumber","reload","signIn","wait"],"budget":{"maxRuntimeMs":150000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.l3.scheduled-restocks-features","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.l3.scheduled-restocks"},{"actions":["click","closeClient","dbExpectStock","dbRecordStock","ensureSignedIn","expect","expectElapsed","expectNumber","fill","recordNumber","recordTime","reload","restartBackend","signIn","signUp","wait"],"budget":{"maxRuntimeMs":300000,"status":"bounded"},"capabilities":["backend-lifecycle","browser"],"evidence":["browser-observation","database-observation"],"id":"ecommerce.l3.server-time-specifications","includeRoles":["guarantee"],"moduleType":"specification","stableId":"ecommerce.l3.server-time"},{"actions":["callAction","click","dbExpectStock","dbSetStock","ensureSignedIn","expect","expectActionOutcome","expectElementCount","fill","reload","signIn","signUp"],"budget":{"maxRuntimeMs":90000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.automatic-reorder","includeRoles":["feature"],"moduleType":"feature"},{"actions":["callConcurrently","click","dbExpectCancellation","dbRecordCheckout","ensureSignedIn","expect","expectCallOutcomes","expectNumber","recordNumber","reload","signIn","signUp","wait"],"budget":{"maxRuntimeMs":24000,"status":"bounded"},"capabilities":["browser","concurrent-actors","direct-server-call"],"evidence":["browser-observation","concurrent-outcome","database-observation"],"id":"ecommerce.progression.cancellation-accounting-specifications","includeRoles":["guarantee"],"moduleType":"specification","stableId":"ecommerce.returns-pricing"},{"actions":["click","expect","signIn","signUp","waitUntilAbsent"],"budget":{"maxRuntimeMs":24000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.cancellation-queue-specifications","includeRoles":["guarantee"],"moduleType":"specification","stableId":"ecommerce.returns-pricing"},{"actions":["click","dbSetStock","ensureSignedIn","expect","expectNumber","recordNumber","reload","signUp","wait"],"budget":{"maxRuntimeMs":350000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.cart-recovery","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","expect","expectElementCount","fill","openItem","reload","signIn"],"budget":{"maxRuntimeMs":30000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.catalog-management","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","expect","fill","signUp"],"budget":{"maxRuntimeMs":30000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.customer-profile","includeRoles":["feature"],"moduleType":"feature"},{"actions":["callAction","click","ensureSignedIn","expect","expectActionOutcome","expectElapsed","expectElementCount","recordTime","reload","signIn","signUp","wait"],"budget":{"maxRuntimeMs":110000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.delivery-notifications","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","dbSetStock","expect","expectElementCount","expectSequence","fill","reload","waitUntilAbsent"],"budget":{"maxRuntimeMs":141000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.faceted-search","includeRoles":["feature"],"moduleType":"feature"},{"actions":["callAction","click","dbSetStock","ensureSignedIn","expect","expectActionOutcome","reload","signIn","signUp"],"budget":{"maxRuntimeMs":76000,"status":"bounded"},"capabilities":["browser","direct-server-call"],"evidence":["browser-observation","server-refusal"],"id":"ecommerce.progression.fulfilment-queue","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.operations-access"},{"actions":["callAction","click","dbExpectStock","dbRecordStock","ensureSignedIn","expect","expectActionOutcome","expectNumber","fill","freshClient","race","recordNumber","reload","signIn","signUp"],"budget":{"maxRuntimeMs":138000,"status":"bounded"},"capabilities":["browser","concurrent-actors","direct-server-call"],"evidence":["browser-observation","server-response"],"id":"ecommerce.progression.inventory-conservation-specifications","includeRoles":["guarantee"],"moduleType":"specification","stableId":"ecommerce.inventory-operations"},{"actions":["click","ensureSignedIn","expect","fill","reload","signIn","signUp"],"budget":{"maxRuntimeMs":55000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.managed-support","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","expect","signUp"],"budget":{"maxRuntimeMs":40000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.notification-preferences","includeRoles":["feature"],"moduleType":"feature"},{"actions":["callAction","click","dbExpectStock","dbRecordStock","ensureSignedIn","expect","expectActionOutcome","expectNumber","fill","recordNumber","reload","signIn","signUp"],"budget":{"maxRuntimeMs":98000,"status":"bounded"},"capabilities":["browser","direct-server-call","request-replay"],"evidence":["server-refusal","server-response"],"id":"ecommerce.progression.operations-access-specifications","includeRoles":["guarantee"],"moduleType":"specification","stableId":"ecommerce.operations-access"},{"actions":["click","expect","fill","signIn","signUp"],"budget":{"maxRuntimeMs":65000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.order-support","includeRoles":["feature"],"moduleType":"feature"},{"actions":["callConcurrently","click","expect","expectCallOutcomes","expectElementCount","expectNumber","freshClient","recordNumber","signIn","signUp"],"budget":{"maxRuntimeMs":30000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.payment-records","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","expect","expectNumber","expectSequence","signUp"],"budget":{"maxRuntimeMs":70000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.personalized-recommendations","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","ensureSignedIn","expect","expectNumber","fill","pressKey","recordNumber","reload","signIn","signUp","wait"],"budget":{"maxRuntimeMs":24000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.price-accounting-specifications","includeRoles":["guarantee"],"moduleType":"specification","stableId":"ecommerce.returns-pricing"},{"actions":["callAction","click","ensureSignedIn","expect","expectActionOutcome","expectNumber","fill","freshClient","recordNumber","reload","signIn","signUp"],"budget":{"maxRuntimeMs":94000,"status":"bounded"},"capabilities":["browser","direct-server-call"],"evidence":["browser-observation","fresh-client-observation"],"id":"ecommerce.progression.price-history-specifications","includeRoles":["guarantee"],"moduleType":"specification","stableId":"ecommerce.returns-pricing"},{"actions":["click","expect","expectNumber","fill","signIn","signUp"],"budget":{"maxRuntimeMs":55000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.promotion-checkout","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","expectNumber","fill","signIn","signUp"],"budget":{"maxRuntimeMs":40000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.promotion-reporting","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","expect","expectNumber","fill","signIn","signUp"],"budget":{"maxRuntimeMs":45000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.promotion-rules","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","expect","signUp"],"budget":{"maxRuntimeMs":130000,"status":"bounded"},"capabilities":["backend-lifecycle","browser"],"evidence":["browser-observation","fresh-client-observation"],"id":"ecommerce.progression.recommendation-feedback","includeRoles":["feature"],"moduleType":"feature"},{"actions":["armScriptCanary","callAction","expect","expectActionOutcome","expectNoScriptExecution","freshClient","openItem","signIn","signUp","wait"],"budget":{"maxRuntimeMs":82000,"status":"bounded"},"capabilities":["browser","direct-server-call","request-replay"],"evidence":["browser-observation","fresh-client-observation","server-refusal","server-response"],"id":"ecommerce.progression.review-access-specifications","includeRoles":["guarantee"],"moduleType":"specification"},{"actions":["click","expect","signIn"],"budget":{"maxRuntimeMs":30000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.staff-access","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","expect","fill","signIn"],"budget":{"maxRuntimeMs":30000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.staff-activity","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","expect","fill","signIn"],"budget":{"maxRuntimeMs":30000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.staff-roles","includeRoles":["feature"],"moduleType":"feature"},{"actions":["callAction","click","dbSetStock","ensureSignedIn","expect","expectActionOutcome","expectElementCount","freshClient","reload","signIn","signUp","wait"],"budget":{"maxRuntimeMs":60000,"status":"bounded"},"capabilities":["browser","direct-database-write"],"evidence":["browser-observation"],"id":"ecommerce.progression.stock-alerts","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","ensureSignedIn","expect","fill","reload","signUp"],"budget":{"maxRuntimeMs":50000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.support-history","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","expect","fill"],"budget":{"maxRuntimeMs":30000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.support-intake","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","expect","fill","signIn","signUp"],"budget":{"maxRuntimeMs":70000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.support-refunds","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","ensureSignedIn","expect","fill","reload","signIn"],"budget":{"maxRuntimeMs":45000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.support-triage","includeRoles":["feature"],"moduleType":"feature"},{"actions":["callAction","click","dbExpectStock","dbRecordStock","dbSetStock","ensureSignedIn","expect","expectActionOutcome","expectElapsed","expectElementCount","expectNotReceived","expectNumber","expectReceived","expectReplayCompleted","expectReplayRejected","expectSequence","fill","freshClient","openItem","recordNumber","recordTime","reload","replayAs","signIn","signUp","wait","waitUntilAbsent"],"budget":{"maxRuntimeMs":464000,"status":"bounded"},"capabilities":["backend-lifecycle","browser","direct-server-call","request-replay"],"evidence":["browser-observation","database-observation","fresh-client-observation","server-response"],"id":"ecommerce.spec.access-control","includeRoles":["guarantee"],"moduleType":"specification"},{"actions":["callAction","callConcurrently","click","dbExpectStock","dbSetStock","ensureSignedIn","expect","expectActionOutcome","expectActorsWith","expectCallOutcomes","expectNumber","fill","reload","restartBackend","signIn","signUp","wait"],"budget":{"maxRuntimeMs":480000,"status":"bounded"},"capabilities":["backend-lifecycle","browser","database-observation","direct-database-write","direct-server-call"],"evidence":["browser-observation","database-observation","fresh-client-observation"],"id":"ecommerce.spec.bundle-integrity","includeRoles":["guarantee"],"moduleType":"specification"},{"actions":["callAction","callConcurrently","click","clickConcurrently","dbExpectCheckout","dbExpectPurchases","dbExpectStock","dbRecordCheckout","dbRecordStock","dbSetStock","ensureSignedIn","expect","expectActionOutcome","expectActorsWith","expectAgreement","expectCallOutcomes","expectNumber","fill","race","recordNumber","reload","signIn","signUp","wait"],"budget":{"maxRuntimeMs":125000,"status":"bounded"},"capabilities":["browser","concurrent-actors","direct-server-call"],"evidence":["concurrent-outcome","database-observation"],"id":"ecommerce.spec.concurrency-safety","includeRoles":["guarantee"],"moduleType":"specification"},{"actions":["dbSetStock","expectNumber","reload","setOffline","startAppServer","stopAppServer"],"budget":{"maxRuntimeMs":105000,"status":"bounded"},"capabilities":["backend-lifecycle","browser","direct-database-write"],"evidence":["database-observation","fresh-client-observation"],"id":"ecommerce.spec.external-data-sync","includeRoles":["guarantee"],"moduleType":"specification"},{"actions":["callAction","click","dbExpectStock","dbRecordStock","dbSetStock","ensureSignedIn","expect","expectActionOutcome","expectAgreement","expectElementCount","expectNumber","expectSequence","fill","openItem","recordNumber","reload","signIn","signUp","waitUntilAbsent"],"budget":{"maxRuntimeMs":184000,"status":"bounded"},"capabilities":["browser","concurrent-actors"],"evidence":["browser-observation","fresh-client-observation"],"id":"ecommerce.spec.live-state","includeRoles":["guarantee"],"moduleType":"specification"},{"actions":["click","dbExpectStock","dbRecordStock","expectSequence","fill","reload","signUp"],"budget":{"maxRuntimeMs":60000,"status":"bounded"},"capabilities":["browser","database-observation"],"evidence":["browser-observation","database-observation"],"id":"ecommerce.spec.search-ordering","includeRoles":["guarantee"],"moduleType":"specification"},{"actions":["callAction","callConcurrently","click","ensureSignedIn","expect","expectActionOutcome","expectCallOutcomes","expectElementCount","expectNumber","fill","freshClient","reload","restartBackend","signIn","signUp"],"budget":{"maxRuntimeMs":300000,"status":"bounded"},"capabilities":["backend-lifecycle","browser","database-observation","direct-server-call"],"evidence":["browser-observation","database-observation","fresh-client-observation"],"id":"ecommerce.spec.split-tender-refunds","includeRoles":["guarantee"],"moduleType":"specification"},{"actions":["click","confirmCheckout","crashCheckout","dbExpectCheckout","dbRecordCheckout","ensureSignedIn","expect","expectCrashCheckout","expectNumber","fill","freshClient","reload","restartBackend","setOffline","signIn","signUp"],"budget":{"maxRuntimeMs":768000,"status":"bounded"},"capabilities":["backend-lifecycle","browser","database-read","process-crash"],"evidence":["browser-observation","fresh-client-observation"],"id":"ecommerce.spec.state-durability","includeRoles":["guarantee"],"moduleType":"specification"},{"actions":["callAction","callConcurrently","click","ensureSignedIn","expect","expectActionOutcome","expectCallOutcomes","expectElementCount","expectNumber","fill","freshClient","reload","restartBackend","signIn","signUp"],"budget":{"maxRuntimeMs":300000,"status":"bounded"},"capabilities":["backend-lifecycle","browser","database-observation","direct-server-call"],"evidence":["browser-observation","database-observation","fresh-client-observation"],"id":"ecommerce.spec.store-credit","includeRoles":["guarantee"],"moduleType":"specification"},{"actions":["callAction","click","dbExpectStock","dbRecordStock","dbSetStock","expect","expectActionOutcome","expectElementCount","expectNumber","expectSequence","fill","freshClient","recordNumber","restartBackend","signIn","signUp","wait"],"budget":{"maxRuntimeMs":480000,"status":"bounded"},"capabilities":["backend-lifecycle","browser","database-observation","direct-server-call"],"evidence":["browser-observation","database-observation","fresh-client-observation"],"id":"ecommerce.spec.subscriptions","includeRoles":["guarantee"],"moduleType":"specification"},{"actions":["callAction","callConcurrently","click","dbExpectStock","dbRecordStock","dbSetStock","ensureSignedIn","expect","expectActionOutcome","expectCallOutcomes","expectElementCount","expectNumber","expectReplayCompleted","fill","freshClient","openItem","recordNumber","reload","replayAs","signIn","signUp","wait"],"budget":{"maxRuntimeMs":100000,"status":"bounded"},"capabilities":["browser","database-observation","direct-server-call"],"evidence":["browser-observation","database-observation","server-response"],"id":"ecommerce.spec.transactional-integrity","includeRoles":["guarantee"],"moduleType":"specification"}],"runtime":{"actions":[{"args":[0,1],"id":"addToCart","path":"/api/cart","reducer":"add_to_cart"},{"args":[0],"id":"buy","path":"/api/items/00000000-0000-0000-0000-000000000000/buy","reducer":"buy_now"},{"args":[],"id":"checkout","path":"/api/checkout","reducer":"checkout"},{"args":[0,0,1],"id":"restock","path":"/api/admin/restock","reducer":"admin_restock"},{"args":["",""],"id":"signIn","path":"/api/auth/signin","reducer":"sign_in"},{"args":["",""],"id":"signUp","params":[{"in":"body","name":"username"},{"in":"body","name":"password"}],"path":"/api/auth/signup","reducer":"sign_up"}],"portOffset":300,"reseedOnReset":true,"restartProbe":"/api/items"},"schemaVersion":3,"task":{"baseExecutionSha256":null,"mode":"action"},"track":"ecommerce"}},"calibration":{"calibrationSchemaVersion":2,"controls":[],"equivalenceDecisions":[],"fixture":{"id":"ecommerce.operations","sourceSha256":"d06444b72dc94fe1ef5e08867d875e1f3bbaa5cd82c35a3558f399c1fcb5ceae"},"id":"ecommerce.dependency-l3-calibration","mutations":[{"backend":"mongodb","executionSha256":"7f2d78efab7c52336d532b6df47adc32b99e9e3f5b55f84254217e67ab5adf69","path":"grader/mutations/mongodb-ecommerce.json","referenceId":"ecommerce-reference-mongodb","sha256":"9424f03ba21307933ea4c7a81ee4f06bd80d659c1dfbe02938fa715167992f08","targets":[{"id":"active-search-uses-purchase-ranking","stableKeys":["ecommerce.spec.search-ordering.search-ordering.402b"]},{"id":"authorized-restock-does-not-change-stock","stableKeys":["ecommerce.feature.warehouse-admin.admin-write.103a"]},{"id":"cancel-does-not-restore-stock-feature","stableKeys":["ecommerce.returns-pricing.cancellation-and-return.3a"]},{"id":"cancel-does-not-restore-stock-fresh-client","stableKeys":["ecommerce.inventory-operations.stock-conservation.202b","ecommerce.inventory-operations.stock-conservation.202c"]},{"id":"cancel-restores-stock-but-keeps-pending-status","stableKeys":["ecommerce.returns-pricing.cancellation-and-return.3b"]},{"id":"cancellation-accounting-loses-stock-restoration","stableKeys":["ecommerce.returns-pricing.refund-accounting.203a"]},{"id":"cancelled-order-remains-in-revenue-feature","stableKeys":["ecommerce.returns-pricing.cancellation-and-return.3a"]},{"id":"cancelled-order-remains-in-revenue-invariant","stableKeys":["ecommerce.returns-pricing.refund-accounting.203a"]},{"id":"cancelled-restock-remains-pending","stableKeys":["ecommerce.l3.scheduled-restocks.scheduled-restocks.306a"]},{"id":"cart-add-uses-another-account-cart","stableKeys":["ecommerce.spec.access-control.cart-boundary.109a"]},{"id":"cart-hydration-loses-account-state","stableKeys":["ecommerce.spec.state-durability.account-state-recovery.105a"]},{"id":"cart-repeat-does-not-increment","stableKeys":["ecommerce.feature.cart-checkout.cart.4a"]},{"id":"catalog-initial-ranking-is-reversed","stableKeys":["ecommerce.feature.catalog.catalog-ranking.2b"]},{"id":"catalog-price-is-offset","stableKeys":["ecommerce.feature.catalog.catalog-values.2a"]},{"id":"catalog-product-name-is-not-published","stableKeys":["ecommerce.progression.catalog-management.catalog-management.622a","ecommerce.progression.catalog-management.catalog-management.622b"]},{"id":"catalog-search-requires-exact-name","stableKeys":["ecommerce.feature.catalog.catalog-search.2d"]},{"id":"catalog-variants-are-discarded","stableKeys":["ecommerce.progression.catalog-management.catalog-management.622b"]},{"id":"category-totals-skip-the-newest-order","stableKeys":["ecommerce.inventory-operations.operational-views.5f"]},{"id":"checkout-claim-is-not-atomic","stableKeys":["ecommerce.spec.concurrency-safety.duplicate-checkout.203b"]},{"id":"checkout-crash-durability","stableKeys":["ecommerce.spec.state-durability.checkout-crash-durability.910b"]},{"id":"checkout-crash-integrity","stableKeys":["ecommerce.spec.state-durability.checkout-crash-integrity.910a"]},{"id":"checkout-leaves-cart-claimed","stableKeys":["ecommerce.feature.cart-checkout.cart.4d"]},{"id":"completed-restock-is-replayed","stableKeys":["ecommerce.l3.deferred-integrity.exactly-once.311a"]},{"id":"concurrent-cart-add-does-not-increment","stableKeys":["ecommerce.spec.concurrency-safety.duplicate-checkout.203a"]},{"id":"customer-can-cancel-foreign-order-1-1","stableKeys":["ecommerce.operations-access.order-owner.204a"]},{"id":"customer-can-cancel-scheduled-restock","stableKeys":["ecommerce.l3.deferred-access.scheduled-work-access.317a"]},{"id":"customer-can-create-promotion","stableKeys":["ecommerce.spec.access-control.promotion-management-boundary.620b"]},{"id":"customer-can-ship-order-direct-1-1","stableKeys":["ecommerce.operations-access.operator-authorization.201c"]},{"id":"customer-signin-gains-staff-role","stableKeys":["ecommerce.spec.access-control.staff-area-boundary.601b"]},{"id":"direct-purchase-is-attributed-to-another-account","stableKeys":["ecommerce.spec.access-control.purchase-attribution.102a"]},{"id":"direct-purchase-total-ignores-store-price","stableKeys":["ecommerce.spec.transactional-integrity.server-price.104a"]},{"id":"direct-review-access-is-not-checked","stableKeys":["ecommerce.progression.review-access-specifications.review-eligibility-direct.618a"]},{"id":"due-restock-does-not-change-stock","stableKeys":["ecommerce.l3.scheduled-restocks.scheduled-restocks.305a"]},{"id":"duplicate-signup-reports-success","stableKeys":["ecommerce.feature.accounts.accounts.1b"]},{"id":"espresso-stock-row-ignores-live-updates","stableKeys":["ecommerce.spec.live-state.purchase-stock.3b"]},{"id":"external-stock-polling-disabled","stableKeys":["ecommerce.spec.external-data-sync.external-stock.901a"]},{"id":"faceted-search-ignores-category","stableKeys":["ecommerce.progression.faceted-search.faceted-search.401a"]},{"id":"initial-dashboard-load-omits-low-stock","stableKeys":["ecommerce.inventory-operations.operational-views.5e","ecommerce.spec.live-state.inventory-dashboard.5a"]},{"id":"last-unit-allows-negative-stock","stableKeys":["ecommerce.spec.concurrency-safety.last-unit.201a","ecommerce.spec.concurrency-safety.last-unit.201b","ecommerce.spec.concurrency-safety.last-unit.201c"]},{"id":"live-admin-updates-keep-stale-category-totals","stableKeys":["ecommerce.spec.live-state.sales-dashboard.5b"]},{"id":"live-review-average-uses-an-extra-divisor","stableKeys":["ecommerce.spec.live-state.rating.6c"]},{"id":"low-stock-boundary-excludes-ten-live","stableKeys":["ecommerce.spec.live-state.inventory-dashboard.5a"]},{"id":"managed-support-allows-another-customer","stableKeys":["ecommerce.spec.access-control.managed-support-privacy.613b"]},{"id":"managed-support-live-refresh-keeps-stale-tickets","stableKeys":["ecommerce.spec.live-state.managed-support.613a"]},{"id":"negative-cart-quantity-is-accepted","stableKeys":["ecommerce.spec.access-control.cart-boundary.109b"]},{"id":"notification-preference-is-not-owner-scoped","stableKeys":["ecommerce.spec.access-control.notification-preferences-privacy.630b"]},{"id":"notification-preference-is-not-saved","stableKeys":["ecommerce.progression.notification-preferences.notification-preferences.630c","ecommerce.spec.state-durability.notification-preferences-reload.630a"]},{"id":"notification-toggle-frozen-at-open","stableKeys":["ecommerce.progression.notification-preferences.notification-preferences.630c"]},{"id":"open-review-list-ignores-live-update","stableKeys":["ecommerce.spec.live-state.open-list.902a"]},{"id":"operator-authorization-allows-customer-transfer","stableKeys":["ecommerce.operations-access.operator-authorization.201a"]},{"id":"order-history-is-not-owner-scoped","stableKeys":["ecommerce.spec.access-control.order-ownership.106a"]},{"id":"overdraw-transfer-is-accepted","stableKeys":["ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c"]},{"id":"pagination-repeats-first-page","stableKeys":["ecommerce.progression.faceted-search.faceted-search.402a"]},{"id":"profile-data-is-lost-on-server-restart","stableKeys":["ecommerce.spec.state-durability.customer-profile-reload.620a"]},{"id":"profile-read-is-not-owner-scoped","stableKeys":["ecommerce.spec.access-control.customer-profile-privacy.620b"]},{"id":"profile-summary-frozen-at-open","stableKeys":["ecommerce.progression.customer-profile.customer-profile.620c"]},{"id":"progression-customer-sees-fulfilment-content","stableKeys":["ecommerce.spec.access-control.fulfilment-area-boundary.1d"]},{"id":"promotion-save-drops-bounded-values","stableKeys":["ecommerce.progression.promotion-rules.promotion-rule-values.620a"]},{"id":"purchase-counts-never-affect-ranking","stableKeys":["ecommerce.spec.live-state.ranking.2c"]},{"id":"purchase-does-not-reduce-warehouse-stock","stableKeys":["ecommerce.spec.transactional-integrity.books-balance.107b"]},{"id":"purchase-order-uses-zero-price","stableKeys":["ecommerce.feature.purchasing.purchase-order.3c"]},{"id":"purchase-read-write-loses-concurrent-stock","stableKeys":["ecommerce.spec.concurrency-safety.restock-race.202a"]},{"id":"purchased-review-ui-does-not-submit","stableKeys":["ecommerce.spec.access-control.review-eligibility.108b"]},{"id":"purchases-do-not-affect-best-sellers","stableKeys":["ecommerce.inventory-operations.operational-views.5d"]},{"id":"queue-ignores-live-fulfilment-updates","stableKeys":["ecommerce.spec.live-state.fulfilment-queue.1a"]},{"id":"queue-warehouse-reports-west","stableKeys":["ecommerce.operations-access.fulfilment-queue.1b"]},{"id":"recommendations-ignore-pending-purchases","stableKeys":["ecommerce.inventory-operations.operational-views.5c"]},{"id":"reconnect-generation-ignores-current-catalog","stableKeys":["ecommerce.spec.external-data-sync.external-stock.901d"]},{"id":"reconnect-hydration-loses-account-state","stableKeys":["ecommerce.spec.state-durability.account-state-recovery.105b"]},{"id":"reload-hydrates-an-empty-cart","stableKeys":["ecommerce.spec.state-durability.cart-reload.4b"]},{"id":"repeat-review-uses-a-new-owner-key","stableKeys":["ecommerce.spec.transactional-integrity.unique-review.6b"]},{"id":"restock-adds-the-wrong-quantity","stableKeys":["ecommerce.spec.live-state.warehouse-stock.7c"]},{"id":"restock-does-not-increase-stock","stableKeys":["ecommerce.spec.live-state.warehouse-stock.7c"]},{"id":"restock-race-records-wrong-order-total","stableKeys":["ecommerce.spec.concurrency-safety.restock-race.202a"]},{"id":"revenue-aggregation-ignores-order-totals","stableKeys":["ecommerce.spec.transactional-integrity.books-balance.107a"]},{"id":"review-comment-is-not-persisted","stableKeys":["ecommerce.feature.reviews.reviews.6a"]},{"id":"review-owner-deny-after-write","stableKeys":["ecommerce.progression.review-access-specifications.review-eligibility-direct.618a"]},{"id":"review-owner-reject-all","stableKeys":["ecommerce.progression.review-access-specifications.review-eligibility-direct.618a"]},{"id":"review-owner-trust-username","stableKeys":["ecommerce.progression.review-access-specifications.review-eligibility-direct.618a"]},{"id":"review-script-reject-all","stableKeys":["ecommerce.progression.review-access-specifications.stored-review-script.9180a"]},{"id":"review-script-unsafe-render","stableKeys":["ecommerce.progression.review-access-specifications.stored-review-script.9180a"]},{"id":"role-assignment-drops-role","stableKeys":["ecommerce.spec.state-durability.staff-role-reload.621a"]},{"id":"role-editor-snaps-back-to-stored-role","stableKeys":["ecommerce.progression.staff-roles.staff-roles.621c"]},{"id":"scheduled-restock-countdown-is-fixed","stableKeys":["ecommerce.l3.scheduled-restocks.scheduled-restocks.302a"]},{"id":"scheduled-restock-never-becomes-due-after-restart","stableKeys":["ecommerce.l3.deferred-durability.restart-survival.311a"]},{"id":"server-restart-disables-catalog-recovery","stableKeys":["ecommerce.spec.external-data-sync.external-stock.901c"]},{"id":"server-time-restock-never-completes","stableKeys":["ecommerce.l3.server-time.server-time.312a"]},{"id":"session-token-not-persisted","stableKeys":["ecommerce.spec.state-durability.session-reload.1e"]},{"id":"shared-cart-live-events-ignored","stableKeys":["ecommerce.spec.live-state.shared-cart.4c"]},{"id":"ship-acknowledges-without-changing-status","stableKeys":["ecommerce.operations-access.fulfilment-queue.1c"]},{"id":"shipping-counts-sale-twice","stableKeys":["ecommerce.inventory-operations.shipping-accounting.202e"]},{"id":"signed-out-visitor-purchase-is-accepted","stableKeys":["ecommerce.spec.access-control.signed-out-purchase.3a"]},{"id":"signin-skips-password-verification","stableKeys":["ecommerce.feature.accounts.accounts.1c"]},{"id":"signout-keeps-current-account","stableKeys":["ecommerce.feature.accounts.accounts.1d"]},{"id":"signup-does-not-expose-created-account","stableKeys":["ecommerce.feature.accounts.accounts.1a"]},{"id":"staff-admin-access-survives-role-removal","stableKeys":["ecommerce.spec.access-control.staff-role-revocation.621d"]},{"id":"staff-can-assign-roles","stableKeys":["ecommerce.spec.access-control.staff-role-boundary.621b","ecommerce.spec.access-control.staff-role-revocation.621d"]},{"id":"staff-can-see-admin-navigation","stableKeys":["ecommerce.spec.access-control.warehouse-area-boundary.7a"]},{"id":"staff-can-use-direct-restock","stableKeys":["ecommerce.spec.access-control.warehouse-write-boundary.103b"]},{"id":"staff-role-write-precedes-denial","stableKeys":["ecommerce.spec.access-control.staff-role-boundary.621b","ecommerce.spec.access-control.staff-role-revocation.621d"]},{"id":"staff-signin-loses-staff-role","stableKeys":["ecommerce.progression.staff-access.staff-access.601a"]},{"id":"stock-alert-delivery-is-suppressed","stableKeys":["ecommerce.progression.stock-alerts.stock-alert-delivery.631c"]},{"id":"stock-alert-repeats-while-in-stock","stableKeys":["ecommerce.spec.transactional-integrity.stock-alert-deduplication.631a"]},{"id":"stock-alerts-are-not-owner-scoped","stableKeys":["ecommerce.spec.access-control.stock-alert-privacy.631b"]},{"id":"support-history-is-lost-on-server-restart","stableKeys":["ecommerce.spec.state-durability.support-history-reload.612a"]},{"id":"support-history-is-not-owner-scoped","stableKeys":["ecommerce.spec.access-control.support-history-privacy.612b"]},{"id":"support-history-leaks-to-signed-out-visitors","stableKeys":["ecommerce.spec.access-control.support-history-logout.612d"]},{"id":"support-history-rows-are-hidden","stableKeys":["ecommerce.progression.support-history.support-history.612c","ecommerce.spec.access-control.support-history-logout.612d","ecommerce.spec.access-control.support-history-privacy.612b","ecommerce.spec.state-durability.support-history-reload.612a"]},{"id":"support-intake-returns-no-reference","stableKeys":["ecommerce.progression.support-intake.support-intake.610a"]},{"id":"support-replies-present-at-open-are-hidden","stableKeys":["ecommerce.progression.managed-support.managed-support.613c"]},{"id":"support-triage-discards-updates","stableKeys":["ecommerce.progression.support-triage.support-assignment.611a","ecommerce.progression.support-triage.support-priority.611b","ecommerce.progression.support-triage.support-status.611c"]},{"id":"transfer-creates-stock-during-race","stableKeys":["ecommerce.inventory-operations.stock-conservation.202d"]},{"id":"transfer-debits-source-without-crediting-existing-destination","stableKeys":["ecommerce.inventory-operations.stock-conservation.202a","ecommerce.inventory-operations.warehouse-transfer.2a","ecommerce.operations-access.operator-authorization.201a"]},{"id":"transfer-totals-omit-destination-credit-live","stableKeys":["ecommerce.spec.live-state.stock-transfers.2b"]},{"id":"unauthenticated-purchase-defaults-to-admin","stableKeys":["ecommerce.spec.access-control.purchase-session.101a"]},{"id":"unpurchased-review-is-accepted","stableKeys":["ecommerce.spec.access-control.review-eligibility.108a"]},{"id":"warehouse-view-omits-one-location","stableKeys":["ecommerce.feature.warehouse-admin.warehouse-view.7b"]}]},{"backend":"postgres","executionSha256":"000c81392cdd01dabc87082c7a8b7d669a7eb07beaa28c084f186f35890c2fe2","path":"grader/mutations/postgres-ecommerce.json","referenceId":"ecommerce-reference-postgres","sha256":"13ea2eba20da22e1f1712ccb03715158bf67ce0934672efdc94c31795fb0422a","targets":[{"id":"account-state-reload-discards-session","stableKeys":["ecommerce.spec.state-durability.account-state-recovery.105a"]},{"id":"active-search-uses-purchase-ranking","stableKeys":["ecommerce.spec.search-ordering.search-ordering.402b"]},{"id":"admin-sockets-do-not-join-admin-room","stableKeys":["ecommerce.spec.live-state.sales-dashboard.5b"]},{"id":"admin-state-change-is-not-broadcast","stableKeys":["ecommerce.spec.live-state.inventory-dashboard.5a"]},{"id":"admin-warehouse-view-drops-one-location","stableKeys":["ecommerce.feature.warehouse-admin.warehouse-view.7b"]},{"id":"authorized-restock-does-not-change-stock","stableKeys":["ecommerce.feature.warehouse-admin.admin-write.103a"]},{"id":"cancel-does-not-restore-stock-feature","stableKeys":["ecommerce.returns-pricing.cancellation-and-return.3a"]},{"id":"cancel-does-not-restore-stock-fresh-client","stableKeys":["ecommerce.inventory-operations.stock-conservation.202b","ecommerce.inventory-operations.stock-conservation.202c"]},{"id":"cancel-restores-stock-but-keeps-pending-status","stableKeys":["ecommerce.returns-pricing.cancellation-and-return.3b"]},{"id":"cancellation-accounting-loses-stock-restoration","stableKeys":["ecommerce.returns-pricing.refund-accounting.203a"]},{"id":"cart-update-accepts-negative-quantity","stableKeys":["ecommerce.spec.access-control.cart-boundary.109b"]},{"id":"category-totals-render-as-session-deltas","stableKeys":["ecommerce.inventory-operations.operational-views.5f"]},{"id":"checkout-crash-durability","stableKeys":["ecommerce.spec.state-durability.checkout-crash-durability.910b"]},{"id":"checkout-crash-integrity","stableKeys":["ecommerce.spec.state-durability.checkout-crash-integrity.910a"]},{"id":"correct-signin-is-refused","stableKeys":["ecommerce.feature.accounts.accounts.1d"]},{"id":"customer-can-cancel-foreign-order-1-1","stableKeys":["ecommerce.operations-access.order-owner.204a"]},{"id":"customer-can-ship-order-direct-1-1","stableKeys":["ecommerce.operations-access.operator-authorization.201c"]},{"id":"direct-purchase-is-attributed-to-previous-account","stableKeys":["ecommerce.spec.access-control.purchase-attribution.102a"]},{"id":"direct-purchase-order-total-is-offset","stableKeys":["ecommerce.feature.purchasing.purchase-order.3c"]},{"id":"direct-purchase-uses-constant-price","stableKeys":["ecommerce.spec.transactional-integrity.server-price.104a"]},{"id":"direct-review-access-is-not-checked","stableKeys":["ecommerce.progression.review-access-specifications.review-eligibility-direct.618a"]},{"id":"duplicate-signup-authenticates-existing-account","stableKeys":["ecommerce.feature.accounts.accounts.1b"]},{"id":"external-stock-polling-disabled","stableKeys":["ecommerce.spec.external-data-sync.external-stock.901a"]},{"id":"low-stock-threshold-is-two-units","stableKeys":["ecommerce.inventory-operations.operational-views.5e","ecommerce.spec.live-state.inventory-dashboard.5a"]},{"id":"notification-sync-flips-saved-toggle","stableKeys":["ecommerce.progression.notification-preferences.notification-preferences.630c"]},{"id":"offline-event-clears-account-state","stableKeys":["ecommerce.spec.state-durability.account-state-recovery.105b"]},{"id":"only-shipped-orders-earn-review-eligibility","stableKeys":["ecommerce.spec.access-control.review-eligibility.108a","ecommerce.spec.access-control.review-eligibility.108b"]},{"id":"open-review-list-ignores-live-update","stableKeys":["ecommerce.spec.live-state.open-list.902a"]},{"id":"open-review-list-renders-each-review-twice","stableKeys":["ecommerce.spec.live-state.open-list.902a"]},{"id":"operator-authorization-allows-customer-transfer","stableKeys":["ecommerce.operations-access.operator-authorization.201a"]},{"id":"oversell-no-row-lock","stableKeys":["ecommerce.spec.concurrency-safety.last-unit.201a","ecommerce.spec.concurrency-safety.last-unit.201b","ecommerce.spec.concurrency-safety.last-unit.201c"]},{"id":"password-verification-is-inverted","stableKeys":["ecommerce.feature.accounts.accounts.1c"]},{"id":"profile-summary-ignores-saved-address","stableKeys":["ecommerce.progression.customer-profile.customer-profile.620c"]},{"id":"progression-cancelled-orders-remain-in-revenue","stableKeys":["ecommerce.returns-pricing.refund-accounting.203a"]},{"id":"progression-cancelled-restock-still-runs","stableKeys":["ecommerce.l3.scheduled-restocks.scheduled-restocks.306a"]},{"id":"progression-cart-add-uses-another-account","stableKeys":["ecommerce.spec.access-control.cart-boundary.109a"]},{"id":"progression-cart-line-does-not-increment","stableKeys":["ecommerce.feature.cart-checkout.cart.4a"]},{"id":"progression-cart-update-uses-wrong-room","stableKeys":["ecommerce.spec.live-state.shared-cart.4c"]},{"id":"progression-catalog-price-is-offset","stableKeys":["ecommerce.feature.catalog.catalog-values.2a"]},{"id":"progression-catalog-product-name-is-not-published","stableKeys":["ecommerce.progression.catalog-management.catalog-management.622a","ecommerce.progression.catalog-management.catalog-management.622b"]},{"id":"progression-catalog-ranking-is-reversed","stableKeys":["ecommerce.feature.catalog.catalog-ranking.2b"]},{"id":"progression-catalog-search-requires-exact-name","stableKeys":["ecommerce.feature.catalog.catalog-search.2d"]},{"id":"progression-catalog-variants-are-discarded","stableKeys":["ecommerce.progression.catalog-management.catalog-management.622b"]},{"id":"progression-checkout-leaves-cart-lines","stableKeys":["ecommerce.feature.cart-checkout.cart.4d"]},{"id":"progression-concurrent-cart-line-does-not-increment","stableKeys":["ecommerce.spec.concurrency-safety.duplicate-checkout.203a"]},{"id":"progression-concurrent-checkout-leaves-cart-lines","stableKeys":["ecommerce.spec.concurrency-safety.duplicate-checkout.203b"]},{"id":"progression-customer-can-create-promotions","stableKeys":["ecommerce.spec.access-control.promotion-management-boundary.620b"]},{"id":"progression-customer-sees-fulfilment-content","stableKeys":["ecommerce.spec.access-control.fulfilment-area-boundary.1d"]},{"id":"progression-customer-sees-staff-tools","stableKeys":["ecommerce.spec.access-control.staff-area-boundary.601b"]},{"id":"progression-customers-can-manage-scheduled-work","stableKeys":["ecommerce.l3.deferred-access.scheduled-work-access.317a"]},{"id":"progression-due-restock-does-not-run","stableKeys":["ecommerce.l3.scheduled-restocks.scheduled-restocks.305a"]},{"id":"progression-faceted-filter-ignores-category","stableKeys":["ecommerce.progression.faceted-search.faceted-search.401a"]},{"id":"progression-managed-support-is-not-shared","stableKeys":["ecommerce.spec.live-state.managed-support.613a"]},{"id":"progression-managed-support-leaks","stableKeys":["ecommerce.spec.access-control.managed-support-privacy.613b"]},{"id":"progression-notification-preferences-do-not-save","stableKeys":["ecommerce.spec.state-durability.notification-preferences-reload.630a"]},{"id":"progression-notifications-leak-across-accounts","stableKeys":["ecommerce.spec.access-control.notification-preferences-privacy.630b"]},{"id":"progression-order-history-ignores-owner","stableKeys":["ecommerce.spec.access-control.order-ownership.106a"]},{"id":"progression-pagination-always-shows-first-page","stableKeys":["ecommerce.progression.faceted-search.faceted-search.402a"]},{"id":"progression-profile-address-is-discarded","stableKeys":["ecommerce.spec.state-durability.customer-profile-reload.620a"]},{"id":"progression-profile-reads-another-account","stableKeys":["ecommerce.spec.access-control.customer-profile-privacy.620b"]},{"id":"progression-promotion-discount-is-offset","stableKeys":["ecommerce.progression.promotion-rules.promotion-rule-values.620a"]},{"id":"progression-restart-timer-never-runs","stableKeys":["ecommerce.l3.server-time.server-time.312a"]},{"id":"progression-restock-adds-wrong-quantity","stableKeys":["ecommerce.spec.live-state.warehouse-stock.7c"]},{"id":"progression-restock-can-apply-more-than-once","stableKeys":["ecommerce.l3.deferred-integrity.exactly-once.311a"]},{"id":"progression-restock-countdown-is-fixed","stableKeys":["ecommerce.l3.scheduled-restocks.scheduled-restocks.302a"]},{"id":"progression-restock-does-not-survive-restart","stableKeys":["ecommerce.l3.deferred-durability.restart-survival.311a"]},{"id":"progression-revenue-double-counts-orders","stableKeys":["ecommerce.spec.transactional-integrity.books-balance.107a"]},{"id":"progression-review-conflict-is-not-updated","stableKeys":["ecommerce.spec.transactional-integrity.unique-review.6b"]},{"id":"progression-shipping-keeps-order-pending","stableKeys":["ecommerce.operations-access.fulfilment-queue.1c"]},{"id":"progression-staff-can-assign-roles","stableKeys":["ecommerce.spec.access-control.staff-role-boundary.621b","ecommerce.spec.access-control.staff-role-revocation.621d"]},{"id":"progression-staff-can-restock-directly","stableKeys":["ecommerce.spec.access-control.warehouse-write-boundary.103b"]},{"id":"progression-staff-role-is-lost-on-restart","stableKeys":["ecommerce.spec.state-durability.staff-role-reload.621a"]},{"id":"progression-staff-sees-admin-navigation","stableKeys":["ecommerce.spec.access-control.warehouse-area-boundary.7a"]},{"id":"progression-staff-tools-are-hidden","stableKeys":["ecommerce.progression.staff-access.staff-access.601a"]},{"id":"progression-stock-alerts-leak-across-accounts","stableKeys":["ecommerce.spec.access-control.stock-alert-privacy.631b"]},{"id":"progression-support-history-anonymous-leak","stableKeys":["ecommerce.spec.access-control.support-history-logout.612d"]},{"id":"progression-support-history-is-not-persisted","stableKeys":["ecommerce.spec.state-durability.support-history-reload.612a"]},{"id":"progression-support-history-leaks","stableKeys":["ecommerce.spec.access-control.support-history-privacy.612b"]},{"id":"progression-support-intake-is-disabled","stableKeys":["ecommerce.progression.support-intake.support-intake.610a"]},{"id":"progression-support-triage-update-is-disabled","stableKeys":["ecommerce.progression.support-triage.support-assignment.611a","ecommerce.progression.support-triage.support-priority.611b","ecommerce.progression.support-triage.support-status.611c"]},{"id":"purchase-does-not-broadcast-fulfilment-queue","stableKeys":["ecommerce.spec.live-state.fulfilment-queue.1a"]},{"id":"purchase-does-not-broadcast-ranking","stableKeys":["ecommerce.spec.live-state.ranking.2c"]},{"id":"purchase-does-not-decrement-warehouse-stock","stableKeys":["ecommerce.spec.transactional-integrity.books-balance.107b"]},{"id":"purchase-read-write-loses-concurrent-stock","stableKeys":["ecommerce.spec.concurrency-safety.restock-race.202a"]},{"id":"purchase-stock-change-is-not-broadcast--01-buying","stableKeys":["ecommerce.spec.live-state.purchase-stock.3b"]},{"id":"purchases-do-not-affect-best-sellers","stableKeys":["ecommerce.inventory-operations.operational-views.5d"]},{"id":"queue-warehouse-reports-west","stableKeys":["ecommerce.operations-access.fulfilment-queue.1b"]},{"id":"recommendations-ignore-pending-purchases","stableKeys":["ecommerce.inventory-operations.operational-views.5c"]},{"id":"reconnect-does-not-send-current-catalog","stableKeys":["ecommerce.spec.external-data-sync.external-stock.901d"]},{"id":"reload-discards-session-identity","stableKeys":["ecommerce.spec.state-durability.session-reload.1e"]},{"id":"reload-hydrates-an-empty-cart","stableKeys":["ecommerce.spec.state-durability.cart-reload.4b"]},{"id":"restock-overwrites-instead-of-increments","stableKeys":["ecommerce.spec.live-state.warehouse-stock.7c"]},{"id":"restock-race-records-wrong-order-total","stableKeys":["ecommerce.spec.concurrency-safety.restock-race.202a"]},{"id":"review-average-update-is-not-broadcast","stableKeys":["ecommerce.spec.live-state.rating.6c"]},{"id":"review-owner-deny-after-write","stableKeys":["ecommerce.progression.review-access-specifications.review-eligibility-direct.618a"]},{"id":"review-owner-reject-all","stableKeys":["ecommerce.progression.review-access-specifications.review-eligibility-direct.618a"]},{"id":"review-owner-trust-username","stableKeys":["ecommerce.progression.review-access-specifications.review-eligibility-direct.618a"]},{"id":"review-route-skips-purchase-eligibility","stableKeys":["ecommerce.spec.access-control.review-eligibility.108a"]},{"id":"review-script-reject-all","stableKeys":["ecommerce.progression.review-access-specifications.stored-review-script.9180a"]},{"id":"review-script-unsafe-render","stableKeys":["ecommerce.progression.review-access-specifications.stored-review-script.9180a"]},{"id":"server-restart-does-not-resynchronize-catalog","stableKeys":["ecommerce.spec.external-data-sync.external-stock.901c"]},{"id":"shipping-counts-sale-twice","stableKeys":["ecommerce.inventory-operations.shipping-accounting.202e"]},{"id":"signed-out-purchase-uses-default-account","stableKeys":["ecommerce.spec.access-control.signed-out-purchase.3a"]},{"id":"signed-out-visitors-do-not-see-reviews","stableKeys":["ecommerce.feature.reviews.reviews.6a"]},{"id":"signup-ui-does-not-enter-created-account","stableKeys":["ecommerce.feature.accounts.accounts.1a"]},{"id":"staff-admin-access-survives-role-removal","stableKeys":["ecommerce.spec.access-control.staff-role-revocation.621d"]},{"id":"staff-role-form-reverts-after-save","stableKeys":["ecommerce.progression.staff-roles.staff-roles.621c"]},{"id":"staff-role-write-precedes-denial","stableKeys":["ecommerce.spec.access-control.staff-role-boundary.621b","ecommerce.spec.access-control.staff-role-revocation.621d"]},{"id":"stock-alert-delivery-is-suppressed","stableKeys":["ecommerce.progression.stock-alerts.stock-alert-delivery.631c"]},{"id":"stock-alert-is-sent-after-every-restock","stableKeys":["ecommerce.spec.transactional-integrity.stock-alert-deduplication.631a"]},{"id":"support-first-reply-is-hidden","stableKeys":["ecommerce.progression.managed-support.managed-support.613c"]},{"id":"support-history-rows-are-hidden","stableKeys":["ecommerce.progression.support-history.support-history.612c","ecommerce.spec.access-control.support-history-logout.612d","ecommerce.spec.access-control.support-history-privacy.612b","ecommerce.spec.state-durability.support-history-reload.612a"]},{"id":"transfer-debits-source-without-crediting-existing-destination","stableKeys":["ecommerce.inventory-operations.stock-conservation.202a","ecommerce.inventory-operations.warehouse-transfer.2a","ecommerce.operations-access.operator-authorization.201a"]},{"id":"transfer-does-not-publish-warehouse-totals","stableKeys":["ecommerce.spec.live-state.stock-transfers.2b"]},{"id":"transfer-overdraft-guard-skips-bulk-transfers","stableKeys":["ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c"]},{"id":"transfer-overwrites-concurrent-purchase-with-stale-stock","stableKeys":["ecommerce.inventory-operations.stock-conservation.202d"]},{"id":"unauthenticated-direct-purchase-uses-default-account","stableKeys":["ecommerce.spec.access-control.purchase-session.101a"]}]},{"backend":"spacetime","executionSha256":"188b4c9a4373e4023f9e4a0add511232ac2afedb3283027544c6eff720ba05e8","path":"grader/mutations/spacetime-ecommerce.json","referenceId":"ecommerce-reference-spacetime","sha256":"302f43126a2a09f42a3a24d326f66c0928467b9cd27ccd9ae606908c8a1b0489","targets":[{"id":"account-state-token-is-not-restored-after-reload","stableKeys":["ecommerce.spec.state-durability.account-state-recovery.105a"]},{"id":"active-search-uses-purchase-ranking","stableKeys":["ecommerce.spec.search-ordering.search-ordering.402b"]},{"id":"admin-restock-preserves-existing-stock","stableKeys":["ecommerce.spec.live-state.warehouse-stock.7c"]},{"id":"admin-revenue-double-counts-every-order","stableKeys":["ecommerce.spec.transactional-integrity.books-balance.107a"]},{"id":"admin-total-stock-is-not-rendered","stableKeys":["ecommerce.spec.live-state.warehouse-stock.7c"]},{"id":"administrator-role-assignment-is-discarded","stableKeys":["ecommerce.spec.state-durability.staff-role-reload.621a"]},{"id":"authorized-restock-does-not-change-stock","stableKeys":["ecommerce.feature.warehouse-admin.admin-write.103a"]},{"id":"buy-now-creates-orders-without-reserving-stock--01-buying","stableKeys":["ecommerce.spec.live-state.purchase-stock.3b"]},{"id":"buy-now-records-the-wrong-order-total","stableKeys":["ecommerce.feature.purchasing.purchase-order.3c"]},{"id":"cancel-does-not-restore-stock-feature","stableKeys":["ecommerce.returns-pricing.cancellation-and-return.3a"]},{"id":"cancel-does-not-restore-stock-fresh-client","stableKeys":["ecommerce.inventory-operations.stock-conservation.202b","ecommerce.inventory-operations.stock-conservation.202c"]},{"id":"cancel-restores-stock-but-keeps-pending-status","stableKeys":["ecommerce.returns-pricing.cancellation-and-return.3b"]},{"id":"cancellation-accounting-loses-stock-restoration","stableKeys":["ecommerce.returns-pricing.refund-accounting.203a"]},{"id":"cancelled-order-remains-in-revenue-feature","stableKeys":["ecommerce.returns-pricing.cancellation-and-return.3a"]},{"id":"cancelled-order-remains-in-revenue-invariant","stableKeys":["ecommerce.returns-pricing.refund-accounting.203a"]},{"id":"cancelled-restock-remains-pending","stableKeys":["ecommerce.l3.scheduled-restocks.scheduled-restocks.306a"]},{"id":"cart-is-deleted-when-owner-disconnects","stableKeys":["ecommerce.spec.state-durability.cart-reload.4b"]},{"id":"cart-line-lookup-ignores-cart-ownership","stableKeys":["ecommerce.spec.access-control.cart-boundary.109a"]},{"id":"catalog-product-is-not-published","stableKeys":["ecommerce.progression.catalog-management.catalog-management.622a","ecommerce.progression.catalog-management.catalog-management.622b"]},{"id":"catalog-search-ignores-the-query","stableKeys":["ecommerce.feature.catalog.catalog-search.2d"]},{"id":"catalog-seeds-the-wrong-air-purifier-price","stableKeys":["ecommerce.feature.catalog.catalog-values.2a"]},{"id":"catalog-tie-breaks-in-reverse-alphabetical-order--01-catalog-ranking","stableKeys":["ecommerce.feature.catalog.catalog-ranking.2b"]},{"id":"catalog-tie-breaks-in-reverse-alphabetical-order--01-core","stableKeys":["ecommerce.spec.live-state.ranking.2c"]},{"id":"catalog-variants-are-discarded","stableKeys":["ecommerce.progression.catalog-management.catalog-management.622b"]},{"id":"category-totals-are-frozen-at-page-load","stableKeys":["ecommerce.spec.live-state.sales-dashboard.5b"]},{"id":"category-totals-count-only-since-the-dashboard-opened","stableKeys":["ecommerce.inventory-operations.operational-views.5f"]},{"id":"checkout-crash-durability","stableKeys":["ecommerce.spec.state-durability.checkout-crash-durability.910b"]},{"id":"checkout-crash-integrity","stableKeys":["ecommerce.spec.state-durability.checkout-crash-integrity.910a"]},{"id":"checkout-does-not-empty-cart","stableKeys":["ecommerce.spec.concurrency-safety.duplicate-checkout.203b"]},{"id":"checkout-does-not-empty-the-basic-cart","stableKeys":["ecommerce.feature.cart-checkout.cart.4d"]},{"id":"completed-restock-remains-pending","stableKeys":["ecommerce.l3.deferred-integrity.exactly-once.311a"]},{"id":"customer-can-cancel-foreign-order-1-1","stableKeys":["ecommerce.operations-access.order-owner.204a"]},{"id":"customer-can-ship-order-direct-1-1","stableKeys":["ecommerce.operations-access.operator-authorization.201c"]},{"id":"customer-profile-view-leaks-another-account","stableKeys":["ecommerce.spec.access-control.customer-profile-privacy.620b"]},{"id":"customers-can-create-promotions","stableKeys":["ecommerce.spec.access-control.promotion-management-boundary.620b"]},{"id":"customers-can-open-staff-tools","stableKeys":["ecommerce.spec.access-control.staff-area-boundary.601b"]},{"id":"customers-can-schedule-restocks","stableKeys":["ecommerce.l3.deferred-access.scheduled-work-access.317a"]},{"id":"direct-purchase-ignores-the-stored-price","stableKeys":["ecommerce.spec.transactional-integrity.server-price.104a"]},{"id":"direct-purchases-are-attributed-to-the-system-account","stableKeys":["ecommerce.spec.access-control.purchase-attribution.102a"]},{"id":"direct-restock-does-not-require-an-admin","stableKeys":["ecommerce.spec.access-control.warehouse-write-boundary.103b"]},{"id":"direct-review-access-is-not-checked","stableKeys":["ecommerce.progression.review-access-specifications.review-eligibility-direct.618a"]},{"id":"due-restock-omits-ledger-entry","stableKeys":["ecommerce.l3.scheduled-restocks.scheduled-restocks.305a"]},{"id":"duplicate-signup-is-silently-ignored","stableKeys":["ecommerce.feature.accounts.accounts.1b"]},{"id":"eligible-review-is-accepted-without-being-stored","stableKeys":["ecommerce.spec.access-control.review-eligibility.108a","ecommerce.spec.access-control.review-eligibility.108b"]},{"id":"every-signed-in-customer-is-treated-as-an-admin","stableKeys":["ecommerce.spec.access-control.warehouse-area-boundary.7a"]},{"id":"existing-cart-line-does-not-increment","stableKeys":["ecommerce.spec.concurrency-safety.duplicate-checkout.203a"]},{"id":"existing-cart-line-does-not-increment-basic-cart","stableKeys":["ecommerce.feature.cart-checkout.cart.4a"]},{"id":"faceted-search-ignores-category","stableKeys":["ecommerce.progression.faceted-search.faceted-search.401a"]},{"id":"faceted-search-next-page-does-not-advance","stableKeys":["ecommerce.progression.faceted-search.faceted-search.402a"]},{"id":"fulfilment-queue-is-frozen-at-page-load","stableKeys":["ecommerce.spec.live-state.fulfilment-queue.1a"]},{"id":"guest-purchase-falls-back-to-the-admin-account","stableKeys":["ecommerce.spec.access-control.purchase-session.101a"]},{"id":"low-stock-list-is-frozen-at-page-load","stableKeys":["ecommerce.spec.live-state.inventory-dashboard.5a"]},{"id":"low-stock-threshold-is-two-units","stableKeys":["ecommerce.inventory-operations.operational-views.5e","ecommerce.spec.live-state.inventory-dashboard.5a"]},{"id":"managed-support-leaks-and-accepts-cross-account-replies","stableKeys":["ecommerce.spec.access-control.managed-support-privacy.613b"]},{"id":"managed-support-live-replies-stay-at-initial-snapshot","stableKeys":["ecommerce.spec.live-state.managed-support.613a"]},{"id":"managed-support-replies-are-empty","stableKeys":["ecommerce.progression.managed-support.managed-support.613c","ecommerce.spec.live-state.managed-support.613a"]},{"id":"new-review-is-accepted-without-being-stored","stableKeys":["ecommerce.feature.reviews.reviews.6a"]},{"id":"nonpositive-cart-quantity-is-treated-as-removal","stableKeys":["ecommerce.spec.access-control.cart-boundary.109b"]},{"id":"notification-preferences-are-not-saved","stableKeys":["ecommerce.spec.state-durability.notification-preferences-reload.630a"]},{"id":"notification-preferences-leak-across-accounts","stableKeys":["ecommerce.spec.access-control.notification-preferences-privacy.630b"]},{"id":"open-review-list-renders-each-review-twice","stableKeys":["ecommerce.spec.live-state.open-list.902a"]},{"id":"open-review-list-snapshots-on-selection","stableKeys":["ecommerce.spec.live-state.open-list.902a"]},{"id":"operator-authorization-allows-customer-shipping","stableKeys":["ecommerce.operations-access.operator-authorization.201c"]},{"id":"operator-authorization-allows-customer-transfer","stableKeys":["ecommerce.operations-access.operator-authorization.201a"]},{"id":"order-views-return-every-customers-orders","stableKeys":["ecommerce.spec.access-control.order-ownership.106a"]},{"id":"pending-restock-timer-is-static","stableKeys":["ecommerce.l3.scheduled-restocks.scheduled-restocks.302a"]},{"id":"profile-is-lost-on-fresh-account-login","stableKeys":["ecommerce.spec.state-durability.customer-profile-reload.620a"]},{"id":"profile-summary-ignores-a-profile-saved-this-session","stableKeys":["ecommerce.progression.customer-profile.customer-profile.620c"]},{"id":"progression-customer-sees-fulfilment-content","stableKeys":["ecommerce.spec.access-control.fulfilment-area-boundary.1d"]},{"id":"promotion-rule-stores-the-wrong-discount","stableKeys":["ecommerce.progression.promotion-rules.promotion-rule-values.620a"]},{"id":"purchase-does-not-reserve-stock-last-unit","stableKeys":["ecommerce.spec.concurrency-safety.last-unit.201a","ecommerce.spec.concurrency-safety.last-unit.201b","ecommerce.spec.concurrency-safety.last-unit.201c"]},{"id":"purchase-does-not-update-ranking-count","stableKeys":["ecommerce.spec.live-state.ranking.2c"]},{"id":"purchases-do-not-affect-best-sellers","stableKeys":["ecommerce.inventory-operations.operational-views.5d"]},{"id":"purchases-do-not-leave-the-warehouses","stableKeys":["ecommerce.spec.transactional-integrity.books-balance.107b"]},{"id":"queue-warehouse-reports-west","stableKeys":["ecommerce.operations-access.fulfilment-queue.1b"]},{"id":"recommendations-ignore-pending-purchases","stableKeys":["ecommerce.inventory-operations.operational-views.5c"]},{"id":"reconnect-discards-the-visible-account-state","stableKeys":["ecommerce.spec.state-durability.account-state-recovery.105b"]},{"id":"repeat-review-inserts-a-second-row","stableKeys":["ecommerce.spec.transactional-integrity.unique-review.6b"]},{"id":"restart-restock-runs-early","stableKeys":["ecommerce.l3.server-time.server-time.312a"]},{"id":"restock-client-snapshot-overwrites-concurrent-purchases","stableKeys":["ecommerce.spec.concurrency-safety.restock-race.202a"]},{"id":"restock-race-records-wrong-order-total","stableKeys":["ecommerce.spec.concurrency-safety.restock-race.202a"]},{"id":"review-average-counts-rows-instead-of-ratings","stableKeys":["ecommerce.spec.live-state.rating.6c"]},{"id":"review-owner-reject-all","stableKeys":["ecommerce.progression.review-access-specifications.review-eligibility-direct.618a"]},{"id":"review-purchase-eligibility-is-not-checked","stableKeys":["ecommerce.spec.access-control.review-eligibility.108a"]},{"id":"review-script-reject-all","stableKeys":["ecommerce.progression.review-access-specifications.stored-review-script.9180a"]},{"id":"review-script-unsafe-render","stableKeys":["ecommerce.progression.review-access-specifications.stored-review-script.9180a"]},{"id":"saving-a-staff-role-snaps-the-input-back-to-the-stored-role","stableKeys":["ecommerce.progression.staff-roles.staff-roles.621c"]},{"id":"saving-notification-preferences-resets-the-toggles","stableKeys":["ecommerce.progression.notification-preferences.notification-preferences.630c"]},{"id":"scheduled-restock-execution-queue-is-process-local","stableKeys":["ecommerce.l3.deferred-durability.restart-survival.311a"]},{"id":"session-token-is-not-persisted-for-reload","stableKeys":["ecommerce.spec.state-durability.session-reload.1e"]},{"id":"ship-acknowledges-without-changing-status","stableKeys":["ecommerce.operations-access.fulfilment-queue.1c"]},{"id":"shipping-counts-sale-twice","stableKeys":["ecommerce.inventory-operations.shipping-accounting.202e"]},{"id":"signed-out-purchase-bypasses-account-check","stableKeys":["ecommerce.spec.access-control.signed-out-purchase.3a"]},{"id":"signin-binds-the-second-client-to-a-different-account","stableKeys":["ecommerce.spec.live-state.shared-cart.4c"]},{"id":"signin-does-not-verify-the-password","stableKeys":["ecommerce.feature.accounts.accounts.1c"]},{"id":"signout-keeps-the-account-session","stableKeys":["ecommerce.feature.accounts.accounts.1d"]},{"id":"signup-binds-the-new-account-to-the-admin-session","stableKeys":["ecommerce.feature.accounts.accounts.1a"]},{"id":"staff-admin-access-survives-role-removal","stableKeys":["ecommerce.spec.access-control.staff-role-revocation.621d"]},{"id":"staff-can-assign-roles","stableKeys":["ecommerce.spec.access-control.staff-role-boundary.621b","ecommerce.spec.access-control.staff-role-revocation.621d"]},{"id":"staff-cannot-open-staff-tools","stableKeys":["ecommerce.progression.staff-access.staff-access.601a"]},{"id":"stock-alert-delivery-is-suppressed","stableKeys":["ecommerce.progression.stock-alerts.stock-alert-delivery.631c"]},{"id":"stock-alert-is-sent-after-every-restock","stableKeys":["ecommerce.spec.transactional-integrity.stock-alert-deduplication.631a"]},{"id":"stock-alerts-are-visible-to-other-customers","stableKeys":["ecommerce.spec.access-control.stock-alert-privacy.631b"]},{"id":"stock-subscription-snapshotted-once","stableKeys":["ecommerce.spec.external-data-sync.external-stock.901a"]},{"id":"stock-view-ignores-update-across-app-server-stop","stableKeys":["ecommerce.spec.external-data-sync.external-stock.901c"]},{"id":"stock-view-keeps-pre-reconnect-snapshot","stableKeys":["ecommerce.spec.external-data-sync.external-stock.901d"]},{"id":"stored-support-replies-are-hidden-after-reload","stableKeys":["ecommerce.progression.managed-support.managed-support.613c"]},{"id":"support-assignment-is-discarded","stableKeys":["ecommerce.progression.support-triage.support-assignment.611a"]},{"id":"support-history-is-lost-on-fresh-account-login","stableKeys":["ecommerce.spec.state-durability.support-history-reload.612a"]},{"id":"support-history-leaks-across-customers","stableKeys":["ecommerce.spec.access-control.support-history-logout.612d","ecommerce.spec.access-control.support-history-privacy.612b"]},{"id":"support-history-leaks-to-signed-out-visitors","stableKeys":["ecommerce.spec.access-control.support-history-logout.612d"]},{"id":"support-history-rows-are-hidden","stableKeys":["ecommerce.progression.support-history.support-history.612c","ecommerce.spec.access-control.support-history-logout.612d","ecommerce.spec.access-control.support-history-privacy.612b","ecommerce.spec.state-durability.support-history-reload.612a"]},{"id":"support-priority-is-discarded","stableKeys":["ecommerce.progression.support-triage.support-priority.611b"]},{"id":"support-status-is-discarded","stableKeys":["ecommerce.progression.support-triage.support-status.611c"]},{"id":"transfer-creates-stock-during-race","stableKeys":["ecommerce.inventory-operations.stock-conservation.202d"]},{"id":"transfer-debits-source-without-crediting-existing-destination","stableKeys":["ecommerce.inventory-operations.stock-conservation.202a","ecommerce.inventory-operations.warehouse-transfer.2a","ecommerce.operations-access.operator-authorization.201a"]},{"id":"transfer-skips-the-source-holding-check","stableKeys":["ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c"]},{"id":"visitor-support-reference-is-hidden","stableKeys":["ecommerce.progression.support-intake.support-intake.610a"]},{"id":"warehouse-totals-are-frozen-at-page-load","stableKeys":["ecommerce.spec.live-state.stock-transfers.2b"]},{"id":"warehouse-view-omits-west","stableKeys":["ecommerce.feature.warehouse-admin.warehouse-view.7b"]}]}],"nullControl":{"pointBearing":"must-fail-conclusively","repetitions":1,"zeroPoint":"typed-policy"},"qualification":{"checks":["ecommerce.feature.accounts.accounts.1a","ecommerce.feature.accounts.accounts.1b","ecommerce.feature.accounts.accounts.1c","ecommerce.feature.accounts.accounts.1d","ecommerce.feature.cart-checkout.cart.4a","ecommerce.feature.cart-checkout.cart.4d","ecommerce.feature.catalog.catalog-ranking.2b","ecommerce.feature.catalog.catalog-search.2d","ecommerce.feature.catalog.catalog-values.2a","ecommerce.feature.purchasing.purchase-order.3c","ecommerce.feature.reviews.reviews.6a","ecommerce.feature.warehouse-admin.admin-write.103a","ecommerce.feature.warehouse-admin.warehouse-view.7b","ecommerce.inventory-operations.operational-views.5c","ecommerce.inventory-operations.operational-views.5d","ecommerce.inventory-operations.operational-views.5e","ecommerce.inventory-operations.operational-views.5f","ecommerce.inventory-operations.shipping-accounting.202e","ecommerce.inventory-operations.stock-conservation.202a","ecommerce.inventory-operations.stock-conservation.202b","ecommerce.inventory-operations.stock-conservation.202c","ecommerce.inventory-operations.stock-conservation.202d","ecommerce.inventory-operations.warehouse-transfer.2a","ecommerce.l3.deferred-access.scheduled-work-access.317a","ecommerce.l3.deferred-durability.restart-survival.311a","ecommerce.l3.deferred-integrity.exactly-once.311a","ecommerce.l3.scheduled-restocks.scheduled-restocks.302a","ecommerce.l3.scheduled-restocks.scheduled-restocks.305a","ecommerce.l3.scheduled-restocks.scheduled-restocks.306a","ecommerce.l3.server-time.server-time.312a","ecommerce.operations-access.fulfilment-queue.1b","ecommerce.operations-access.fulfilment-queue.1c","ecommerce.operations-access.operator-authorization.201a","ecommerce.operations-access.operator-authorization.201c","ecommerce.operations-access.order-owner.204a","ecommerce.progression.catalog-management.catalog-management.622a","ecommerce.progression.catalog-management.catalog-management.622b","ecommerce.progression.customer-profile.customer-profile.620c","ecommerce.progression.faceted-search.faceted-search.401a","ecommerce.progression.faceted-search.faceted-search.402a","ecommerce.progression.managed-support.managed-support.613c","ecommerce.progression.notification-preferences.notification-preferences.630c","ecommerce.progression.promotion-rules.promotion-rule-values.620a","ecommerce.progression.review-access-specifications.review-eligibility-direct.618a","ecommerce.progression.review-access-specifications.stored-review-script.9180a","ecommerce.progression.staff-access.staff-access.601a","ecommerce.progression.staff-roles.staff-roles.621c","ecommerce.progression.stock-alerts.stock-alert-delivery.631c","ecommerce.progression.support-history.support-history.612c","ecommerce.progression.support-intake.support-intake.610a","ecommerce.progression.support-triage.support-assignment.611a","ecommerce.progression.support-triage.support-priority.611b","ecommerce.progression.support-triage.support-status.611c","ecommerce.returns-pricing.cancellation-and-return.3a","ecommerce.returns-pricing.cancellation-and-return.3b","ecommerce.returns-pricing.refund-accounting.203a","ecommerce.spec.access-control.cart-boundary.109a","ecommerce.spec.access-control.cart-boundary.109b","ecommerce.spec.access-control.customer-profile-privacy.620b","ecommerce.spec.access-control.fulfilment-area-boundary.1d","ecommerce.spec.access-control.managed-support-privacy.613b","ecommerce.spec.access-control.notification-preferences-privacy.630b","ecommerce.spec.access-control.order-ownership.106a","ecommerce.spec.access-control.promotion-management-boundary.620b","ecommerce.spec.access-control.purchase-attribution.102a","ecommerce.spec.access-control.purchase-session.101a","ecommerce.spec.access-control.review-eligibility.108a","ecommerce.spec.access-control.review-eligibility.108b","ecommerce.spec.access-control.signed-out-purchase.3a","ecommerce.spec.access-control.staff-area-boundary.601b","ecommerce.spec.access-control.staff-role-boundary.621b","ecommerce.spec.access-control.staff-role-revocation.621d","ecommerce.spec.access-control.stock-alert-privacy.631b","ecommerce.spec.access-control.support-history-logout.612d","ecommerce.spec.access-control.support-history-privacy.612b","ecommerce.spec.access-control.warehouse-area-boundary.7a","ecommerce.spec.access-control.warehouse-write-boundary.103b","ecommerce.spec.concurrency-safety.duplicate-checkout.203a","ecommerce.spec.concurrency-safety.duplicate-checkout.203b","ecommerce.spec.concurrency-safety.last-unit.201a","ecommerce.spec.concurrency-safety.last-unit.201b","ecommerce.spec.concurrency-safety.last-unit.201c","ecommerce.spec.concurrency-safety.restock-race.202a","ecommerce.spec.external-data-sync.external-stock.901a","ecommerce.spec.external-data-sync.external-stock.901c","ecommerce.spec.external-data-sync.external-stock.901d","ecommerce.spec.live-state.fulfilment-queue.1a","ecommerce.spec.live-state.inventory-dashboard.5a","ecommerce.spec.live-state.managed-support.613a","ecommerce.spec.live-state.open-list.902a","ecommerce.spec.live-state.purchase-stock.3b","ecommerce.spec.live-state.ranking.2c","ecommerce.spec.live-state.rating.6c","ecommerce.spec.live-state.sales-dashboard.5b","ecommerce.spec.live-state.shared-cart.4c","ecommerce.spec.live-state.stock-transfers.2b","ecommerce.spec.live-state.warehouse-stock.7c","ecommerce.spec.search-ordering.search-ordering.402b","ecommerce.spec.state-durability.account-state-recovery.105a","ecommerce.spec.state-durability.account-state-recovery.105b","ecommerce.spec.state-durability.cart-reload.4b","ecommerce.spec.state-durability.checkout-crash-durability.910b","ecommerce.spec.state-durability.checkout-crash-integrity.910a","ecommerce.spec.state-durability.customer-profile-reload.620a","ecommerce.spec.state-durability.notification-preferences-reload.630a","ecommerce.spec.state-durability.session-reload.1e","ecommerce.spec.state-durability.staff-role-reload.621a","ecommerce.spec.state-durability.support-history-reload.612a","ecommerce.spec.transactional-integrity.books-balance.107a","ecommerce.spec.transactional-integrity.books-balance.107b","ecommerce.spec.transactional-integrity.server-price.104a","ecommerce.spec.transactional-integrity.stock-alert-deduplication.631a","ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c","ecommerce.spec.transactional-integrity.unique-review.6b"],"evidence":[],"exactCombinationRequired":true,"featureCatalog":{"contentSha256":"8671f7883c2e5a24474a546ef1517407cd9334258e1a7cfe91e292a9af1952f2","id":"ecommerce.questlines","path":"progression/ecommerce.json"},"mutationRepetitions":1,"referenceRepetitions":1,"runner":{"architecture":"x64","mode":"appliance","platform":"linux","schemaVersion":1},"stacks":["mongodb","postgres","spacetime"]},"recipe":{"contentSha256":"53fbb8093b2335837d88c246f04da99858c40b57ce24d17fbd2b6c815abdb62d","executionSha256":"558ea7e4033dd1e08c76c2ecc475751f2cabd210f4e18cc0f2c6e3e3531a2498","id":"ecommerce.progression-catalog","meaningSha256":"79dc9a2f8ee5854d3f4edf68862942778d0fa0bd667b35554f065d025726aa76","path":"composition/recipes/progression-catalog.json"},"references":{"entries":[{"backend":"mongodb","id":"ecommerce-reference-mongodb","sourceSha256":"0d21bbc4b2768f4077a81d676f0d4c87a65067dda8f2c5ffeb2fa9208381dc7e","targetPath":"reference-apps/ecommerce/mongodb"},{"backend":"postgres","id":"ecommerce-reference-postgres","sourceSha256":"f3906fa01050f00a0ed76c69c256a6b0013bbf72eb72c03aa92d79e47b71c2b8","targetPath":"reference-apps/ecommerce/postgres"},{"backend":"spacetime","id":"ecommerce-reference-spacetime","sourceSha256":"7ba1f548e89f461b95b5caf843b0fbcf2b81278428d02869d7f47e1a199eaf6e","targetPath":"reference-apps/ecommerce/spacetime"}],"registryPath":"reference-apps/registry.json"},"selection":{"alias":"L3","coveredAliases":["L1","L2","L3"],"path":"composition/dependency.json","sha256":"e73002df8a7a42e612f2bd1ee5de1f9db2d6c13cb9c819e405a5f2512eca3048"},"title":"Ecommerce dependency L3 calibration","track":"ecommerce","qualificationSha256":"253174720de8d80e884e764e4d8018c0a98e1c54b364b33f7679aa189af8d1bd","contentSha256":"11134cdd2ecc17a3843b4808d64b91272abe98682d68922e505508c078af3170","qualificationStaleness":[]},"mutations":{"mongodb":{"schemaVersion":3,"fixtureSha256":"0d21bbc4b2768f4077a81d676f0d4c87a65067dda8f2c5ffeb2fa9208381dc7e","backend":"mongodb","track":"ecommerce","note":"Mutation definitions for the MongoDB ecommerce reference.","mutations":[{"id":"recommendation-dismissal-lost-on-restart","scenario":"tracks/ecommerce/scenarios/progression-recommendation-feedback.json","targets":["ecommerce.spec.state-durability.recommendation-feedback-restart.504c"],"desc":"Erase saved recommendation dismissals when the application starts again.","file":"server/src/index.ts","edits":[{"find":" await mongoose.connect(DATABASE_URL);","replace":" await mongoose.connect(DATABASE_URL);\n await Dismissal.deleteMany({});"}]},{"id":"pending-order-item-return-accepted","scenario":"tracks/ecommerce/scenarios/progression-order-return-boundary.json","targets":["ecommerce.returns-pricing.cancellation-and-return.3f"],"desc":"Accept a pending order return and restore its stock before shipment.","file":"server/src/index.ts","edits":[{"find":"if (!['shipped', 'delivered'].includes(value.status)) throw new Error('No shipped order found');","replace":"if (!['pending', 'shipped', 'delivered'].includes(value.status)) throw new Error('No shipped order found');"}]},{"id":"staff-admin-access-survives-role-removal","scenario":"tracks/ecommerce/scenarios/progression-staff-roles.json","targets":["ecommerce.spec.access-control.staff-role-revocation.621d"],"desc":"Keep administrator access after changing the assigned role back to staff.","file":"server/src/progression.ts","edits":[{"find":" target.isAdmin = role === \"admin\";","replace":" target.isAdmin = target.isAdmin || role === \"admin\";"}]},{"id":"shipping-counts-sale-twice","scenario":"tracks/ecommerce/scenarios/progression-shipping-accounting.json","targets":["ecommerce.inventory-operations.shipping-accounting.202e"],"desc":"Shipping succeeds but doubles the completed sale value in authoritative revenue.","file":"server/src/index.ts","edits":[{"find":" order.status = \"shipped\";\n await order.save();","replace":" order.status = \"shipped\";\n order.total *= 2;\n await order.save();"}]},{"id":"signup-does-not-expose-created-account","scenario":"tracks/ecommerce/scenarios/01-account-create.json","targets":["ecommerce.feature.accounts.accounts.1a"],"desc":"Signup succeeds but the client discards the created account identity from its current session view.","file":"client/src/App.tsx","edits":[{"find":" saveSession(data.token, data.user);\n };\n\n const handleSignIn","replace":" saveSession(data.token, { ...data.user, username: \"\" });\n };\n\n const handleSignIn"}]},{"id":"duplicate-signup-reports-success","scenario":"tracks/ecommerce/scenarios/01-account-duplicate.json","targets":["ecommerce.feature.accounts.accounts.1b"],"desc":"A duplicate username is reported as a successful empty signup response instead of a refusal.","file":"server/src/index.ts","edits":[{"find":" if (existing) return res.status(409).json({ error: \"Username is already taken\" });","replace":" if (existing) return res.json({}); // mutant: duplicate signup is falsely accepted"}]},{"id":"signin-skips-password-verification","scenario":"tracks/ecommerce/scenarios/01-account-password.json","targets":["ecommerce.feature.accounts.accounts.1c"],"desc":"Signin accepts an existing account without requiring its password to match.","file":"server/src/index.ts","edits":[{"find":" if (!valid) return res.status(401).json({ error: \"Invalid username or password\" });","replace":" if (false && !valid) return res.status(401).json({ error: \"Invalid username or password\" });"}]},{"id":"signout-keeps-current-account","scenario":"tracks/ecommerce/scenarios/01-account-signout.json","targets":["ecommerce.feature.accounts.accounts.1d"],"desc":"Signout disconnects the token state but leaves the current account and persisted credential in place.","file":"client/src/App.tsx","edits":[{"find":" const handleSignOut = () => {\n clearSession();\n };","replace":" const handleSignOut = () => {\n setToken(null); // mutant: visible and persisted account state is not cleared\n };"}]},{"id":"session-token-not-persisted","scenario":"tracks/ecommerce/scenarios/01-account-reload.json","targets":["ecommerce.spec.state-durability.session-reload.1e"],"desc":"The active session is kept only in React state and is unavailable after a page reload.","file":"client/src/App.tsx","edits":[{"find":" localStorage.setItem(TOKEN_KEY, tok);\n setToken(tok);","replace":" void tok; // mutant: the session token is never persisted\n setToken(tok);"}]},{"id":"purchase-counts-never-affect-ranking","scenario":"tracks/ecommerce/scenarios/01-core.json","targets":["ecommerce.spec.live-state.ranking.2c"],"desc":"The catalogue ranking ignores recorded purchases and therefore never promotes the bought item.","file":"server/src/index.ts","edits":[{"find":" purchaseCount: purchaseMap.get(id) || 0,","replace":" purchaseCount: 0, // mutant: ranking ignores durable purchase counts"}]},{"id":"signed-out-visitor-purchase-is-accepted","scenario":"tracks/ecommerce/scenarios/progression-signed-out-purchase.json","targets":["ecommerce.spec.access-control.signed-out-purchase.3a"],"desc":"The UI exposes purchase controls to visitors and the buy route assigns unauthenticated requests an unverified identity, allowing an actual stock-debiting order.","file":"client/src/App.tsx","edits":[{"find":" const isCustomer = !!currentUser && !currentUser.isAdmin && !currentUser.isStaff;","replace":" const isCustomer = !currentUser?.isAdmin && !currentUser?.isStaff;"},{"file":"server/src/index.ts","find":"app.post(\"/api/items/:id/buy\", requireAuth, async (req, res) => {","replace":"app.post(\"/api/items/:id/buy\", async (req, _res, next) => {\n (req as any).user = await userFromToken(extractToken(req)) || { _id: new Types.ObjectId() };\n next();\n}, async (req, res) => {"}]},{"id":"espresso-stock-row-ignores-live-updates","scenario":"tracks/ecommerce/scenarios/01-buying.json","targets":["ecommerce.spec.live-state.purchase-stock.3b"],"desc":"The live catalogue handler preserves a stale Espresso Machine stock projection while applying all other item updates.","file":"client/src/App.tsx","edits":[{"find":" socket.on(\"items:update\", (data: ItemT[]) => setItems(data));","replace":" socket.on(\"items:update\", (data: ItemT[]) => setItems((previous) => data.map((item) => item.name === \"Espresso Machine\" ? { ...item, stock: previous.find((old) => old.id === item.id)?.stock ?? item.stock } : item)));"}]},{"id":"restock-race-records-wrong-order-total","scenario":"tracks/ecommerce/scenarios/01-restock-race.json","targets":["ecommerce.spec.concurrency-safety.restock-race.202a"],"desc":"Purchases preserve stock and visible order counts but record the wrong booked total. Native mixed-race reconciliation must reject them.","file":"server/src/index.ts","edits":[{"find":" total: item.price,\n });","replace":" total: 0, // mutant: purchase receipt loses the authoritative price\n });"}]},{"id":"purchase-order-uses-zero-price","scenario":"tracks/ecommerce/scenarios/progression-purchasing.json","targets":["ecommerce.feature.purchasing.purchase-order.3c"],"desc":"A direct purchase records the item but stores a zero order total instead of the price paid.","file":"server/src/index.ts","edits":[{"find":" total: item.price,\n });","replace":" total: 0, // mutant: purchase receipt loses the authoritative price\n });"}]},{"id":"reload-hydrates-an-empty-cart","scenario":"tracks/ecommerce/scenarios/01-cart.json","targets":["ecommerce.spec.state-durability.cart-reload.4b"],"desc":"Cart hydration discards the persisted server response after reload.","file":"client/src/App.tsx","edits":[{"find":" const refreshCart = useCallback(async (tok: string) => {\n const data = await apiFetch(\"/api/cart\", tok);\n setCart(data);\n }, []);","replace":" const refreshCart = useCallback(async (tok: string) => {\n await apiFetch(\"/api/cart\", tok);\n setCart({ items: [], total: 0 }); // mutant: persisted cart response is discarded\n }, []);"}]},{"id":"shared-cart-live-events-ignored","scenario":"tracks/ecommerce/scenarios/01-cart.json","targets":["ecommerce.spec.live-state.shared-cart.4c"],"desc":"An already-open second session ignores committed cart update events.","file":"client/src/App.tsx","edits":[{"find":" socket.on(\"cart:update\", (data: CartT) => setCart(data));","replace":" socket.on(\"cart:update\", (data: CartT) => setCart(current => current.items.length === 0 ? current : data)); // mutant: an empty second-session cart ignores its first remote update"}]},{"id":"review-comment-is-not-persisted","scenario":"tracks/ecommerce/scenarios/01-review-visibility.json","targets":["ecommerce.feature.reviews.reviews.6a"],"desc":"Review submission persists an empty comment rather than the customer's submitted text.","file":"server/src/index.ts","edits":[{"find":" { itemId, userId: user._id, username: user.username, rating: ratingNum,\n comment: typeof comment === \"string\" ? comment : \"\" },","replace":" { itemId, userId: user._id, username: user.username, rating: ratingNum,\n comment: \"\" },"}]},{"id":"repeat-review-uses-a-new-owner-key","scenario":"tracks/ecommerce/scenarios/01-review-uniqueness.json","targets":["ecommerce.spec.transactional-integrity.unique-review.6b"],"desc":"Each review submission is stored under a fresh owner key, bypassing the one-review-per-customer constraint.","file":"server/src/index.ts","edits":[{"find":" { itemId, userId: user._id },\n { itemId, userId: user._id, username: user.username, rating: ratingNum,\n comment: typeof comment === \"string\" ? comment : \"\" },","replace":" { itemId, userId: new Types.ObjectId() },\n { itemId, userId: new Types.ObjectId(), username: user.username, rating: ratingNum,\n comment: typeof comment === \"string\" ? comment : \"\" },"}]},{"id":"live-review-average-uses-an-extra-divisor","scenario":"tracks/ecommerce/scenarios/01-review-rating-live.json","targets":["ecommerce.spec.live-state.rating.6c"],"desc":"The live review event divides the rating sum by one more review than actually exists.","file":"server/src/index.ts","edits":[{"find":"async function broadcastReviews(itemId: string) {\n const reviews = await Review.find({ itemId }).sort({ createdAt: -1 });\n const average = reviews.length ? reviews.reduce((s, r) => s + r.rating, 0) / reviews.length : 0;","replace":"async function broadcastReviews(itemId: string) {\n const reviews = await Review.find({ itemId }).sort({ createdAt: -1 });\n const average = reviews.length ? reviews.reduce((s, r) => s + r.rating, 0) / (reviews.length + 1) : 0;"}]},{"id":"warehouse-view-omits-one-location","scenario":"tracks/ecommerce/scenarios/01-warehouse-admin-staff.json","targets":["ecommerce.feature.warehouse-admin.warehouse-view.7b"],"desc":"The admin warehouse projection truncates the final item-location row.","file":"client/src/App.tsx","edits":[{"find":" {overview.locations.map((loc) => (","replace":" {overview.locations.slice(0, -1).map((loc) => ("}]},{"id":"unauthenticated-purchase-defaults-to-admin","scenario":"tracks/ecommerce/scenarios/01-purchase-session.json","targets":["ecommerce.spec.access-control.purchase-session.101a"],"desc":"The purchase endpoint drops authentication and assigns sessionless purchases to the seeded administrator.","file":"server/src/index.ts","edits":[{"find":"app.post(\"/api/items/:id/buy\", requireAuth, async (req, res) => {","replace":"app.post(\"/api/items/:id/buy\", async (req, res) => {"},{"find":" const user = (req as any).user;\n const order = await Order.create({","replace":" const user = (req as any).user || await User.findOne({ username: \"admin\" });\n const order = await Order.create({"}]},{"id":"direct-purchase-total-ignores-store-price","scenario":"tracks/ecommerce/scenarios/01-server-price.json","targets":["ecommerce.spec.transactional-integrity.server-price.104a"],"desc":"The direct purchase creates one order but records a zero total rather than the store's current price.","file":"server/src/index.ts","edits":[{"find":" total: item.price,\n });","replace":" total: 0, // mutant: direct purchase ignores the authoritative price\n });"}]},{"id":"cart-hydration-loses-account-state","scenario":"tracks/ecommerce/scenarios/progression-account-state-reload.json","targets":["ecommerce.spec.state-durability.account-state-recovery.105a"],"desc":"Reload hydration discards the account's persisted cart response.","file":"client/src/App.tsx","edits":[{"find":" const refreshCart = useCallback(async (tok: string) => {\n const data = await apiFetch(\"/api/cart\", tok);\n setCart(data);\n }, []);","replace":" const refreshCart = useCallback(async (tok: string) => {\n await apiFetch(\"/api/cart\", tok);\n setCart({ items: [], total: 0 }); // mutant: account state is discarded on hydration\n }, []);"}]},{"id":"reconnect-hydration-loses-account-state","scenario":"tracks/ecommerce/scenarios/progression-account-state-reconnect.json","targets":["ecommerce.spec.state-durability.account-state-recovery.105b"],"desc":"The initial account cart loads correctly, but after network restoration the client ignores both refreshed and pushed cart state.","file":"client/src/App.tsx","edits":[{"find":" useEffect(() => {\n const socket = io({ auth: token ? { token } : {} });","replace":" useEffect(() => {\n const clearAccountOffline = () => {\n setCurrentUser(null);\n setCart({ items: [], total: 0 });\n };\n window.addEventListener(\"offline\", clearAccountOffline, { once: true });\n const socket = io({ auth: token ? { token } : {} });"}]},{"id":"order-history-is-not-owner-scoped","scenario":"tracks/ecommerce/scenarios/01-order-ownership.json","targets":["ecommerce.spec.access-control.order-ownership.106a"],"desc":"Order history returns every customer's orders instead of filtering by the authenticated owner.","file":"server/src/index.ts","edits":[{"find":" const orders = await Order.find({ userId }).sort({ createdAt: -1 });","replace":" const orders = await Order.find({}).sort({ createdAt: -1 });"}]},{"id":"revenue-aggregation-ignores-order-totals","scenario":"tracks/ecommerce/scenarios/progression-books-balance.json","targets":["ecommerce.spec.transactional-integrity.books-balance.107a"],"desc":"The admin revenue aggregation counts every order as zero regardless of its stored total.","file":"server/src/index.ts","edits":[{"find":" { $group: { _id: null, total: { $sum: { $subtract: [\"$total\", { $ifNull: [\"$refundTotal\", 0] }] } } } },","replace":" { $group: { _id: null, total: { $sum: 0 } } },"}]},{"id":"unpurchased-review-is-accepted","scenario":"tracks/ecommerce/scenarios/01-review-eligibility.json","targets":["ecommerce.spec.access-control.review-eligibility.108a"],"desc":"The review endpoint bypasses its completed-purchase eligibility check.","file":"server/src/index.ts","edits":[{"find":" if (!hasPurchased) {\n return res.status(403).json({ error: \"You can only review items you have purchased\" });\n }","replace":" if (false && !hasPurchased) {\n return res.status(403).json({ error: \"You can only review items you have purchased\" });\n }"}]},{"id":"purchased-review-ui-does-not-submit","scenario":"tracks/ecommerce/scenarios/01-review-eligibility.json","targets":["ecommerce.spec.access-control.review-eligibility.108b"],"desc":"The review form discards valid customer submissions. Direct authorized and unauthorized review calls retain their normal behavior.","file":"client/src/App.tsx","edits":[{"find":" const handleReviewSubmit = async (itemId: string, rating: number, comment: string) => {\n setReviewError(\"\");","replace":" const handleReviewSubmit = async (itemId: string, rating: number, comment: string) => {\n setReviewError(\"Review submission unavailable\");\n return;"}]},{"id":"external-stock-polling-disabled","scenario":"tracks/ecommerce/scenarios/01-external-live-sync.json","targets":["ecommerce.spec.external-data-sync.external-stock.901a"],"desc":"The server stops reconciling direct database stock writes into live catalogue events.","file":"server/src/index.ts","edits":[{"find":" broadcastItems().catch((err) => console.error(\"broadcastItems poll failed\", err));","replace":" // mutant: direct database stock changes are never reconciled"}]},{"id":"server-restart-disables-catalog-recovery","scenario":"tracks/ecommerce/scenarios/01-external-server-restart-sync.json","targets":["ecommerce.spec.external-data-sync.external-stock.901c"],"desc":"After a socket disconnect, the existing page ignores both reconnect refreshes and later catalogue snapshots.","file":"client/src/App.tsx","edits":[{"find":" const socketRef = useRef<Socket | null>(null);\n\n const saveSession","replace":" const socketRef = useRef<Socket | null>(null);\n const acceptCatalogRecovery = useRef(true);\n\n const saveSession"},{"find":" socket.on(\"connect\", () => {\n refreshItems().catch((err) => console.error(err));","replace":" socket.on(\"disconnect\", () => { acceptCatalogRecovery.current = false; });\n socket.on(\"connect\", () => {\n if (acceptCatalogRecovery.current) refreshItems().catch((err) => console.error(err));"},{"find":" socket.on(\"items:update\", (data: ItemT[]) => setItems(data));","replace":" socket.on(\"items:update\", (data: ItemT[]) => { if (acceptCatalogRecovery.current) setItems(data); });"}]},{"id":"reconnect-generation-ignores-current-catalog","scenario":"tracks/ecommerce/scenarios/01-external-reconnect-sync.json","targets":["ecommerce.spec.external-data-sync.external-stock.901d"],"desc":"After the browser goes offline, the existing page ignores reconnect refreshes and subsequent catalogue events.","file":"client/src/App.tsx","edits":[{"find":" const socketRef = useRef<Socket | null>(null);\n\n const saveSession","replace":" const socketRef = useRef<Socket | null>(null);\n const acceptCatalogUpdates = useRef(true);\n useEffect(() => {\n const stopCatalogRecovery = () => { acceptCatalogUpdates.current = false; };\n window.addEventListener(\"offline\", stopCatalogRecovery);\n return () => window.removeEventListener(\"offline\", stopCatalogRecovery);\n }, []);\n\n const saveSession"},{"find":" setItems(data.items);","replace":" if (acceptCatalogUpdates.current) setItems(data.items);"},{"find":" socket.on(\"items:update\", (data: ItemT[]) => setItems(data));","replace":" socket.on(\"items:update\", (data: ItemT[]) => {\n if (acceptCatalogUpdates.current) setItems(data);\n });"}]},{"id":"open-review-list-ignores-live-update","scenario":"tracks/ecommerce/scenarios/progression-open-list-live.json","targets":["ecommerce.spec.live-state.open-list.902a"],"desc":"The already-open review list ignores a committed review update from another client.","file":"client/src/App.tsx","edits":[{"find":" setItemDetail((prev) => (prev && prev.id === payload.itemId ? { ...prev, reviews: payload.reviews, average: payload.average } : prev));","replace":" void payload; // mutant: the already-open review list ignores committed updates"}]},{"id":"cancel-does-not-restore-stock-feature","scenario":"tracks/ecommerce/scenarios/02-order-cancellation-core.json","targets":["ecommerce.returns-pricing.cancellation-and-return.3a"],"desc":"Cancellation changes order state but skips restoration of its recorded warehouse allocations.","file":"server/src/index.ts","edits":[{"find":" {item_id:line.itemId,warehouse_id:allocation.warehouseId}, {$inc:{quantity:allocation.quantity}}, {session});","replace":" {item_id:line.itemId,warehouse_id:allocation.warehouseId}, {$inc:{quantity:0}}, {session});"}]},{"id":"cancellation-accounting-loses-stock-restoration","scenario":"tracks/ecommerce/scenarios/02-invariants.json","targets":["ecommerce.returns-pricing.refund-accounting.203a"],"desc":"Cancellation removes revenue and changes order status, but loses the original warehouse stock restoration. The native refund-accounting assertion must detect this.","file":"server/src/index.ts","edits":[{"find":" {item_id:line.itemId,warehouse_id:allocation.warehouseId}, {$inc:{quantity:allocation.quantity}}, {session});","replace":" {item_id:line.itemId,warehouse_id:allocation.warehouseId}, {$inc:{quantity:0}}, {session});"}]},{"id":"cancel-does-not-restore-stock-fresh-client","scenario":"tracks/ecommerce/scenarios/02-self-contained.json","targets":["ecommerce.inventory-operations.stock-conservation.202b","ecommerce.inventory-operations.stock-conservation.202c"],"desc":"Cancellation changes order state but skips restoration, so a fresh client reads the persisted shortfall.","file":"server/src/index.ts","edits":[{"find":" {item_id:line.itemId,warehouse_id:allocation.warehouseId}, {$inc:{quantity:allocation.quantity}}, {session});","replace":" {item_id:line.itemId,warehouse_id:allocation.warehouseId}, {$inc:{quantity:0}}, {session});"}]},{"id":"cancel-restores-stock-but-keeps-pending-status","scenario":"tracks/ecommerce/scenarios/02-order-cancellation-history.json","targets":["ecommerce.returns-pricing.cancellation-and-return.3b"],"desc":"Cancellation restores allocations but writes pending back to order history.","file":"server/src/index.ts","edits":[{"find":" value.status = \"cancelled\";\n await value.save({session});","replace":" value.status = \"pending\";\n await value.save({session});"}]},{"id":"cancelled-order-remains-in-revenue-feature","scenario":"tracks/ecommerce/scenarios/02-order-cancellation-core.json","targets":["ecommerce.returns-pricing.cancellation-and-return.3a"],"desc":"Admin revenue includes cancelled orders even though cancellation otherwise succeeds.","file":"server/src/index.ts","edits":[{"find":"async function getRevenue(): Promise<number> {\n const rows = await Order.aggregate([\n { $match: { status: { $ne: \"cancelled\" } } },","replace":"async function getRevenue(): Promise<number> {\n const rows = await Order.aggregate([\n { $match: {} },"}]},{"id":"cancelled-order-remains-in-revenue-invariant","scenario":"tracks/ecommerce/scenarios/02-invariants.json","targets":["ecommerce.returns-pricing.refund-accounting.203a"],"desc":"Admin revenue includes cancelled orders even though cancellation otherwise succeeds.","file":"server/src/index.ts","edits":[{"find":"async function getRevenue(): Promise<number> {\n const rows = await Order.aggregate([\n { $match: { status: { $ne: \"cancelled\" } } },","replace":"async function getRevenue(): Promise<number> {\n const rows = await Order.aggregate([\n { $match: {} },"}]},{"id":"operator-authorization-allows-customer-transfer","scenario":"tracks/ecommerce/scenarios/02-strengthened.json","targets":["ecommerce.operations-access.operator-authorization.201a"],"desc":"The transfer route keeps authentication but drops its administrator role gate.","file":"server/src/index.ts","edits":[{"find":"app.post(\"/api/admin/transfer\", requireAuth, requireAdmin, async (req, res) => {","replace":"app.post(\"/api/admin/transfer\", requireAuth, async (req, res) => {"}]},{"id":"customer-can-ship-order-direct-1-1","scenario":"tracks/ecommerce/scenarios/02-server-actions.json","targets":["ecommerce.operations-access.operator-authorization.201c"],"desc":"The shipping route keeps authentication but drops its staff role gate.","file":"server/src/index.ts","edits":[{"find":"app.post(\"/api/fulfilment/ship\", requireAuth, requireStaff, async (req, res) => {","replace":"app.post(\"/api/fulfilment/ship\", requireAuth, async (req, res) => {"}]},{"id":"customer-can-cancel-foreign-order-1-1","scenario":"tracks/ecommerce/scenarios/02-server-actions.json","targets":["ecommerce.operations-access.order-owner.204a"],"desc":"Cancellation retains authentication and pending-state validation but drops order ownership.","file":"server/src/index.ts","edits":[{"find":" const value = orderId ? await Order.findOne({ _id: orderId, userId: user._id }).session(session) : null;\n if (!value) return null;\n if (value.status !== \"pending\") throw new Error(\"Order cannot be cancelled\");","replace":" const value = orderId ? await Order.findOne({ _id: orderId }).session(session) : null;\n if (!value) return null;\n if (value.status !== \"pending\") throw new Error(\"Order cannot be cancelled\");"}]},{"id":"queue-depth-lags-one-order","scenario":"tracks/ecommerce/scenarios/02-fulfilment-live.json","targets":["ecommerce.operations-access.fulfilment-queue.1a"],"desc":"The queue renders every order but its visible depth remains one behind.","file":"client/src/App.tsx","edits":[{"find":"Orders waiting: <span className=\"value\" data-role=\"queue-depth\">{queue.depth}</span>","replace":"Orders waiting: <span className=\"value\" data-role=\"queue-depth\">{Math.max(0, queue.depth - 1)}</span>"}]},{"id":"ship-acknowledges-without-changing-status","scenario":"tracks/ecommerce/scenarios/02-fulfilment-ship.json","targets":["ecommerce.operations-access.fulfilment-queue.1c"],"desc":"Shipping returns success but writes pending back to the order, leaving both live views unchanged.","file":"server/src/index.ts","edits":[{"find":" order.status = \"shipped\";\n await order.save();","replace":" order.status = \"pending\";\n await order.save();"}]},{"id":"customer-sees-fulfilment-navigation","scenario":"tracks/ecommerce/scenarios/02-features.json","targets":["ecommerce.operations-access.fulfilment-queue.1d"],"desc":"Expose the protected staff area to signed-in customers, including its navigation and content.","file":"client/src/App.tsx","edits":[{"find":" {(currentUser?.isStaff || currentUser?.isAdmin) && (\n <button className=\"btn btn-ghost\" data-role=\"staff-link\"","replace":" {currentUser && (\n <button className=\"btn btn-ghost\" data-role=\"staff-link\""},{"find":"{fulfilmentOpen && (currentUser?.isStaff || currentUser?.isAdmin) && (","replace":"{fulfilmentOpen && currentUser && ("}]},{"id":"progression-customer-sees-fulfilment-content","scenario":"tracks/ecommerce/scenarios/02-fulfilment-access.json","targets":["ecommerce.spec.access-control.fulfilment-area-boundary.1d"],"desc":"Expose the protected staff area to signed-in customers, including its navigation and content.","file":"client/src/App.tsx","edits":[{"find":" {(currentUser?.isStaff || currentUser?.isAdmin) && (\n <button className=\"btn btn-ghost\" data-role=\"staff-link\"","replace":" {currentUser && (\n <button className=\"btn btn-ghost\" data-role=\"staff-link\""},{"find":"{fulfilmentOpen && (currentUser?.isStaff || currentUser?.isAdmin) && (","replace":"{fulfilmentOpen && currentUser && ("}]},{"id":"transfer-debits-source-without-crediting-existing-destination","scenario":"tracks/ecommerce/scenarios/02-strengthened.json","targets":["ecommerce.inventory-operations.stock-conservation.202a","ecommerce.inventory-operations.warehouse-transfer.2a","ecommerce.operations-access.operator-authorization.201a"],"desc":"A transfer debits the source but credits zero to the destination. This breaks both movement/conservation assertions and the authorized-transfer positive observation inside the authorization check; that last failure is not evidence of an unauthorized transfer.","file":"server/src/index.ts","edits":[{"find":" await Stock.findOneAndUpdate(\n { item_id: itemId, warehouse_id: toWarehouseId },\n { $inc: { quantity: qty } },\n { upsert: true }\n );","replace":" await Stock.findOneAndUpdate(\n { item_id: itemId, warehouse_id: toWarehouseId },\n { $inc: { quantity: 0 } },\n { upsert: true }\n );"}]},{"id":"transfer-warehouse-totals-omit-destination-credit","scenario":"tracks/ecommerce/scenarios/02-transfer-totals.json","targets":["ecommerce.inventory-operations.warehouse-transfer.2b"],"desc":"A transfer debits the source but adds zero to the existing destination, so the two live warehouse totals do not move in opposite directions.","file":"server/src/index.ts","edits":[{"find":" await Stock.findOneAndUpdate(\n { item_id: itemId, warehouse_id: toWarehouseId },\n { $inc: { quantity: qty } },\n { upsert: true }\n );","replace":" await Stock.findOneAndUpdate(\n { item_id: itemId, warehouse_id: toWarehouseId },\n { $inc: { quantity: 0 } },\n { upsert: true }\n );"}]},{"id":"transfer-overdraft-guard-removed","scenario":"tracks/ecommerce/scenarios/02-transfer-overdraw.json","targets":["ecommerce.inventory-operations.warehouse-transfer.2c"],"desc":"The atomic source debit no longer requires sufficient quantity, allowing an overdrawn transfer to succeed and change both rows.","file":"server/src/index.ts","edits":[{"find":"{ item_id: itemId, warehouse_id: fromWarehouseId, quantity: { $gte: qty } }","replace":"{ item_id: itemId, warehouse_id: fromWarehouseId }"}]},{"id":"low-stock-excludes-boundary-ten","scenario":"tracks/ecommerce/scenarios/02-low-stock.json","targets":["ecommerce.inventory-operations.operational-views.5a"],"desc":"The low-stock view uses a strict boundary and omits items with exactly ten units.","file":"server/src/index.ts","edits":[{"find":" .filter((it) => it.stock <= 10)","replace":" .filter((it) => it.stock < 10)"}]},{"id":"category-totals-ignore-pending-purchases","scenario":"tracks/ecommerce/scenarios/02-operational-category-totals.json","targets":["ecommerce.inventory-operations.operational-views.5b"],"desc":"Category totals include only shipped orders, so a newly accepted pending purchase is absent from the live totals.","file":"server/src/index.ts","edits":[{"find":"async function getCategoryTotals() {\n const rows = await Order.aggregate([\n { $match: { status: { $ne: \"cancelled\" } } },","replace":"async function getCategoryTotals() {\n const rows = await Order.aggregate([\n { $match: { status: \"shipped\" } },"}]},{"id":"recommendations-ignore-pending-purchases","scenario":"tracks/ecommerce/scenarios/02-operational-recommendations.json","targets":["ecommerce.inventory-operations.operational-views.5c"],"desc":"Personal recommendations derive categories only from shipped orders, so a newly accepted pending purchase has no influence.","file":"server/src/index.ts","edits":[{"find":" { $match: { userId: new Types.ObjectId(userId), status: { $ne: \"cancelled\" } } },","replace":" { $match: { userId: new Types.ObjectId(userId), status: \"shipped\" } },"}]},{"id":"purchases-do-not-affect-best-sellers","scenario":"tracks/ecommerce/scenarios/02-operational-best-sellers.json","targets":["ecommerce.inventory-operations.operational-views.5d"],"desc":"Rank signed-out recommendations without purchase counts.","file":"server/src/index.ts","edits":[{"find":"async function getPurchaseCounts(): Promise<Map<string, number>> {\n const rows = await Order.aggregate([\n { $match: { status: { $ne: \"cancelled\" } } },\n { $unwind: \"$items\" },\n { $match: { \"items.returned\": { $ne: true } } },\n { $group: { _id: \"$items.itemId\", total: { $sum: \"$items.quantity\" } } },\n ]);\n const map = new Map<string, number>();\n for (const row of rows) map.set(row._id.toString(), row.total);\n return map;\n}","replace":"async function getPurchaseCounts(): Promise<Map<string, number>> {\n return new Map<string, number>();\n}"}]},{"id":"queue-warehouse-reports-west","scenario":"tracks/ecommerce/scenarios/02-queue-warehouse.json","targets":["ecommerce.operations-access.fulfilment-queue.1b"],"desc":"The queue renders West for the deterministic Desk Lamp allocation even though the order reserved stock in East.","file":"client/src/App.tsx","edits":[{"find":"{(l.warehouseNames || []).join(\", \") || \"Unknown\"}","replace":"{\"West\"}"}]},{"id":"cart-repeat-does-not-increment","scenario":"tracks/ecommerce/scenarios/progression-cart-checkout.json","targets":["ecommerce.feature.cart-checkout.cart.4a"],"desc":"An existing cart line remains at its old quantity.","file":"server/src/index.ts","edits":[{"find":"{ $inc: { \"items.$.quantity\": qty },","replace":"{ $inc: { \"items.$.quantity\": 0 },"}]},{"id":"checkout-leaves-cart-claimed","scenario":"tracks/ecommerce/scenarios/progression-cart-checkout.json","targets":["ecommerce.feature.cart-checkout.cart.4d"],"desc":"Checkout creates an order without emptying the cart.","file":"server/src/credit.ts","edits":[{"find":" cart.items = cart.items.filter(line => line.reservationExpiresAt && line.reservationExpiresAt <= now) as any;","replace":" // mutant: purchased cart lines remain"}]},{"id":"catalog-initial-ranking-is-reversed","scenario":"tracks/ecommerce/scenarios/01-catalog-ranking.json","targets":["ecommerce.feature.catalog.catalog-ranking.2b"],"desc":"The initial catalog uses reverse alphabetical order.","file":"client/src/App.tsx","edits":[{"find":".sort((a, b) => (filtering ? 0 : b.purchaseCount - a.purchaseCount) || a.name.localeCompare(b.name))","replace":".sort((a, b) => (filtering ? 0 : b.purchaseCount - a.purchaseCount) || b.name.localeCompare(a.name))"}]},{"id":"catalog-search-requires-exact-name","scenario":"tracks/ecommerce/scenarios/01-catalog-search.json","targets":["ecommerce.feature.catalog.catalog-search.2d"],"desc":"Partial catalog search is replaced with exact matching.","file":"client/src/App.tsx","edits":[{"find":"(!q || it.name.toLowerCase().includes(q))","replace":"(!q || it.name.toLowerCase() === q)"}]},{"id":"catalog-price-is-offset","scenario":"tracks/ecommerce/scenarios/01-catalog-values.json","targets":["ecommerce.feature.catalog.catalog-values.2a"],"desc":"Public catalog prices are reported one unit too high.","file":"server/src/index.ts","edits":[{"find":" ...json,\n id,\n stock: stockMap.get(id) || 0,","replace":" ...json,\n id,\n price: json.price + 1,\n stock: stockMap.get(id) || 0,"}]},{"id":"staff-can-see-admin-navigation","scenario":"tracks/ecommerce/scenarios/01-warehouse-admin-staff.json","targets":["ecommerce.spec.access-control.warehouse-area-boundary.7a"],"desc":"Expose administrator identity and protected inventory data to non-admin accounts.","file":"server/src/index.ts","edits":[{"find":" if (!user || !user.isAdmin) return res.status(403).json({ error: \"Admin access required\" });","replace":" if (!user) return res.status(403).json({ error: \"Admin access required\" });"},{"find":"username: user.username, isAdmin: user.isAdmin,","replace":"username: user.username, isAdmin: true,"}]},{"id":"staff-can-use-direct-restock","scenario":"tracks/ecommerce/scenarios/01-admin-write-staff.json","targets":["ecommerce.spec.access-control.warehouse-write-boundary.103b"],"desc":"The direct restock route accepts staff instead of administrators only.","file":"server/src/index.ts","edits":[{"find":"app.post(\"/api/admin/restock\", requireAuth, requireAdmin, async (req, res) => {","replace":"app.post(\"/api/admin/restock\", requireAuth, requireStaff, async (req, res) => {"}]},{"id":"restock-adds-the-wrong-quantity","scenario":"tracks/ecommerce/scenarios/01-warehouse-stock-live-staff.json","targets":["ecommerce.spec.live-state.warehouse-stock.7c"],"desc":"A direct restock stores one fewer unit than requested.","file":"server/src/index.ts","edits":[{"find":" await Stock.findOneAndUpdate(\n { item_id: item._id, warehouse_id: warehouse._id },\n { $inc: { quantity: qty } },\n { upsert: true }\n );","replace":" await Stock.findOneAndUpdate(\n { item_id: item._id, warehouse_id: warehouse._id },\n { $inc: { quantity: qty - 1 } },\n { upsert: true }\n );"}]},{"id":"transfer-creates-stock-during-race","scenario":"tracks/ecommerce/scenarios/02-server-actions.json","targets":["ecommerce.inventory-operations.stock-conservation.202d"],"desc":"A transfer creates one extra unit while a purchase runs.","file":"server/src/index.ts","edits":[{"find":" { $inc: { quantity: qty } },\n { upsert: true }\n );\n\n await Promise.all([broadcastItems(), broadcastAdmin(), broadcastRecommendedForAll()]);","replace":" { $inc: { quantity: qty + 1 } },\n { upsert: true }\n );\n\n await Promise.all([broadcastItems(), broadcastAdmin(), broadcastRecommendedForAll()]);"}]},{"id":"customer-can-cancel-scheduled-restock","scenario":"tracks/ecommerce/scenarios/03-deferred-access.json","targets":["ecommerce.l3.deferred-access.scheduled-work-access.317a"],"desc":"Scheduled-restock cancellation routes omit their staff checks.","file":"server/src/progression.ts","edits":[{"find":" router.delete(\"/scheduled-restocks/:id\", auth, staff, cancelScheduledRestock);","replace":" router.delete(\"/scheduled-restocks/:id\", auth, cancelScheduledRestock);"},{"find":" adminRouter.delete(\"/scheduled-restocks/:id\", auth, staff, cancelScheduledRestock);","replace":" adminRouter.delete(\"/scheduled-restocks/:id\", auth, cancelScheduledRestock);"}]},{"id":"scheduled-restock-never-becomes-due-after-restart","scenario":"tracks/ecommerce/scenarios/03-deferred-durability.json","targets":["ecommerce.l3.deferred-durability.restart-survival.311a"],"desc":"Application startup discards pending restocks. Ordinary timer execution remains intact; completed and cancelled work is retained.","file":"server/src/index.ts","edits":[{"find":"import { Dismissal, Payment, Promotion } from \"./progression-models.js\";","replace":"import { Dismissal, Payment, Promotion, ScheduledRestock } from \"./progression-models.js\";"},{"find":" await mongoose.connect(DATABASE_URL);","replace":" await mongoose.connect(DATABASE_URL);\n await ScheduledRestock.deleteMany({ status: \"pending\" });"}]},{"id":"reservation-expiry-does-not-release-stock-after-restart","scenario":"tracks/ecommerce/scenarios/03-deferred-durability.json","targets":["ecommerce.l3.deferred-durability.restart-survival.314a"],"desc":"Expired reservations clear their record but do not restore stock.","file":"server/src/progression.ts","edits":[{"find":"if (result.modifiedCount) await releaseStock(line.itemId as Types.ObjectId,\n [...line.reservedWarehouseIds] as Types.ObjectId[]);","replace":"if (result.modifiedCount) await releaseStock(line.itemId as Types.ObjectId, []);"}]},{"id":"completed-restock-is-replayed","scenario":"tracks/ecommerce/scenarios/03-deferred-integrity.json","targets":["ecommerce.l3.deferred-integrity.exactly-once.311a"],"desc":"A completed scheduled restock remains pending and applies repeatedly.","file":"server/src/progression.ts","edits":[{"find":"{ $set: { status: \"applied\" } }, { new: true });","replace":"{ $set: { status: \"pending\" } }, { new: true });"}]},{"id":"expired-reservation-is-released-repeatedly","scenario":"tracks/ecommerce/scenarios/03-deferred-integrity.json","targets":["ecommerce.l3.deferred-integrity.stock-conservation.313a"],"desc":"An expired reservation keeps its allocation and restores it repeatedly.","file":"server/src/progression.ts","edits":[{"find":"} } }, { $set: { \"items.$.reservedWarehouseIds\": [] } }, { timestamps: false });","replace":"} } }, { $set: { \"items.$.reservedWarehouseIds\": line.reservedWarehouseIds } }, { timestamps: false });"}]},{"id":"checkout-decrements-reserved-stock-again","scenario":"tracks/ecommerce/scenarios/03-deferred-integrity.json","targets":["ecommerce.l3.deferred-integrity.stock-conservation.314a"],"desc":"Checkout decrements stock after the reservation already did.","file":"server/src/credit.ts","edits":[{"find":" const [order] = await Order.create([{ userId, items: lines, total: totalMinor / 100, discount,","replace":" await mongoose.connection.collection('stocks').updateOne({ item_id: lines[0].itemId }, { $inc: { quantity: -1 } }, { session });\n const [order] = await Order.create([{ userId, items: lines, total: totalMinor / 100, discount,"}]},{"id":"cart-add-does-not-reserve-stock","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.301a"],"desc":"Adding to the cart records an allocation without reserving stock.","file":"server/src/stock-reservations.ts","edits":[{"find":" { $inc: { quantity: -1 } },","replace":" { $inc: { quantity: 0 } },"}]},{"id":"reservation-timer-does-not-decrease","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.305a"],"desc":"The cart always reports a fresh reservation timer.","file":"server/src/index.ts","edits":[{"find":"? Math.max(0, Math.ceil((line.reservationExpiresAt.getTime() - Date.now()) / 1000)) : 0,","replace":"? 90 : 0,"}]},{"id":"reservation-checkout-does-not-clear-cart","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.306a"],"desc":"Reservation checkout leaves the claimed cart lines in place.","file":"server/src/credit.ts","edits":[{"find":" cart.items = cart.items.filter(line => line.reservationExpiresAt && line.reservationExpiresAt <= now) as any;","replace":" // mutant: purchased cart lines remain"}]},{"id":"expired-reservation-is-not-marked","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.307a"],"desc":"Expired cart lines are never marked as expired.","file":"server/src/index.ts","edits":[{"find":"expired: Boolean(line.reservationExpiresAt && line.reservationExpiresAt.getTime() <= Date.now()),","replace":"expired: false,"}]},{"id":"quantity-change-does-not-renew-reservation","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.308a"],"desc":"Raising cart quantity keeps the old reservation deadline.","file":"server/src/index.ts","edits":[{"find":" $set: { \"items.$.reservationExpiresAt\": new Date(Date.now() + 90_000),\n inactiveExpiresAt: new Date(Date.now() + 300_000) },","replace":" $set: { inactiveExpiresAt: new Date(Date.now() + 300_000) },"}]},{"id":"scheduled-restock-countdown-is-fixed","scenario":"tracks/ecommerce/scenarios/03-scheduled-restocks.json","targets":["ecommerce.l3.scheduled-restocks.scheduled-restocks.302a"],"desc":"The pending restock UI shows a fixed countdown.","file":"client/src/ProgressionPanel.tsx","edits":[{"find":"{Math.max(0, Math.ceil((new Date(entry.dueAt).getTime() - Date.now()) / 1000))}","replace":"{90}"}]},{"id":"due-restock-does-not-change-stock","scenario":"tracks/ecommerce/scenarios/03-scheduled-restock-apply.json","targets":["ecommerce.l3.scheduled-restocks.scheduled-restocks.305a"],"desc":"A due restock is recorded without adding stock.","file":"server/src/progression.ts","edits":[{"find":"{ $inc: { quantity: claimed.quantity } });","replace":"{ $inc: { quantity: 0 } });"}]},{"id":"cancelled-restock-remains-pending","scenario":"tracks/ecommerce/scenarios/03-scheduled-restock-cancel.json","targets":["ecommerce.l3.scheduled-restocks.scheduled-restocks.306a"],"desc":"Cancelling a restock leaves it pending so it later applies.","file":"server/src/progression.ts","edits":[{"find":"{ $set: { status: \"cancelled\" } }, { new: true });","replace":"{ $set: { status: \"pending\" } }, { new: true });"}]},{"id":"server-time-restock-never-completes","scenario":"tracks/ecommerce/scenarios/03-server-time.json","targets":["ecommerce.l3.server-time.server-time.312a"],"desc":"Server time never selects a due restock after restart.","file":"server/src/progression.ts","edits":[{"find":"const due = await ScheduledRestock.find({ status: \"pending\", dueAt: { $lte: now } });","replace":"const due = await ScheduledRestock.find({ status: \"pending\", dueAt: { $lte: new Date(0) } });"}]},{"id":"closed-browser-reservation-never-releases","scenario":"tracks/ecommerce/scenarios/03-server-time.json","targets":["ecommerce.l3.server-time.server-time.313a"],"desc":"A reservation that expires without a browser does not restore stock.","file":"server/src/progression.ts","edits":[{"find":"if (result.modifiedCount) await releaseStock(line.itemId as Types.ObjectId,\n [...line.reservedWarehouseIds] as Types.ObjectId[]);","replace":"if (result.modifiedCount) await releaseStock(line.itemId as Types.ObjectId, []);"}]},{"id":"catalog-product-name-is-not-published","scenario":"tracks/ecommerce/scenarios/progression-catalog-management.json","targets":["ecommerce.progression.catalog-management.catalog-management.622a","ecommerce.progression.catalog-management.catalog-management.622b"],"desc":"The public product card omits the submitted name while retaining its product and variants. Both publication and named variant navigation fail because the latter locates the card by its product name; this does not claim the variants were lost.","file":"client/src/App.tsx","edits":[{"find":" <div className=\"item-name\" data-role=\"item-name\">\n {item.name}\n </div>","replace":" <div className=\"item-name\" data-role=\"item-name\">\n {item.name === \"Travel Mug\" ? \"\" : item.name}\n </div>"}]},{"id":"catalog-variants-are-discarded","scenario":"tracks/ecommerce/scenarios/progression-catalog-management.json","targets":["ecommerce.progression.catalog-management.catalog-management.622b"],"desc":"Catalog management stores the submitted product without its named variants.","file":"server/src/progression.ts","edits":[{"find":"const item = await Item.create({ name, price, category, variants });","replace":"const item = await Item.create({ name, price, category, variants: [] });"}]},{"id":"checkout-records-wrong-payment-amount","scenario":"tracks/ecommerce/scenarios/progression-core-business.json","targets":["ecommerce.progression.payment-records.payment-records.623a"],"desc":"Checkout records a payment above the charged total.","file":"server/src/credit.ts","edits":[{"find":" await Payment.create([{ userId, orderId: order!._id, amount: totalMinor / 100, status: 'paid' }], { session });","replace":" await Payment.create([{ userId, orderId: order!._id, amount: totalMinor / 100 + 1, status: 'paid' }], { session });"}]},{"id":"payment-state-repeats-each-record","scenario":"tracks/ecommerce/scenarios/progression-core-business.json","targets":["ecommerce.spec.transactional-integrity.payment-deduplication.623b"],"desc":"The order response repeats every payment record.","file":"server/src/index.ts","edits":[{"find":"payments: (byOrder.get(String(order._id)) ?? []).map(payment => payment.toJSON()),","replace":"payments: [...(byOrder.get(String(order._id)) ?? []), ...(byOrder.get(String(order._id)) ?? [])].map(payment => payment.toJSON()),"}]},{"id":"profile-data-is-lost-on-server-restart","scenario":"tracks/ecommerce/scenarios/progression-customer-profile.json","targets":["ecommerce.spec.state-durability.customer-profile-reload.620a"],"desc":"Application startup discards saved profiles. Profile creation, reads, and browser reload work before runtime restart.","file":"server/src/index.ts","edits":[{"find":"import { Dismissal, Payment, Promotion } from \"./progression-models.js\";","replace":"import { Dismissal, Payment, Promotion, Profile } from \"./progression-models.js\";"},{"find":" await mongoose.connect(DATABASE_URL);","replace":" await mongoose.connect(DATABASE_URL);\n await Profile.deleteMany({});"}]},{"id":"profile-read-is-not-owner-scoped","scenario":"tracks/ecommerce/scenarios/progression-customer-profile.json","targets":["ecommerce.spec.access-control.customer-profile-privacy.620b"],"desc":"Profile reads return another account profile.","file":"server/src/progression.ts","edits":[{"find":"user ? Profile.findOne({ userId: user._id }) : null,","replace":"user ? Profile.findOne({}) : null,"}]},{"id":"faceted-search-ignores-category","scenario":"tracks/ecommerce/scenarios/progression-faceted-filters.json","targets":["ecommerce.progression.faceted-search.faceted-search.401a"],"desc":"Faceted search ignores the selected category.","file":"client/src/App.tsx","edits":[{"find":"&& (!categoryFilter || it.category === categoryFilter)","replace":"&& (!categoryFilter || true)"}]},{"id":"active-search-uses-purchase-ranking","scenario":"tracks/ecommerce/scenarios/progression-search-ordering.json","targets":["ecommerce.spec.search-ordering.search-ordering.402b"],"desc":"Keep purchase ranking when search text or filters are active.","file":"client/src/App.tsx","edits":[{"find":"const filtering = Boolean(q || categoryFilter || minimumPrice || maximumPrice || inStockOnly);","replace":"const filtering = false;"}]},{"id":"pagination-repeats-first-page","scenario":"tracks/ecommerce/scenarios/progression-faceted-pagination.json","targets":["ecommerce.progression.faceted-search.faceted-search.402a"],"desc":"Every catalog page repeats the first page.","file":"client/src/App.tsx","edits":[{"find":"const searchResults = filteredItems.slice(searchPage * CATALOG_PAGE_SIZE,\n searchPage * CATALOG_PAGE_SIZE + CATALOG_PAGE_SIZE);","replace":"const searchResults = filteredItems.slice(0, CATALOG_PAGE_SIZE);"}]},{"id":"managed-support-live-refresh-keeps-stale-tickets","scenario":"tracks/ecommerce/scenarios/progression-managed-support-shared.json","targets":["ecommerce.spec.live-state.managed-support.613a"],"desc":"Periodic refresh keeps the previous support ticket snapshot. Saved replies remain available on initial load and explicit action refresh, but another open client does not receive updates.","file":"client/src/ProgressionPanel.tsx","edits":[{"find":" const refresh = useCallback(async () => {","replace":" const refresh = useCallback(async (live = false) => {"},{"find":" setState({ ...next, loadedToken: token });","replace":" setState((current: any) => ({ ...next, tickets: live ? current.tickets : next.tickets, loadedToken: token }));"},{"find":" const timer = setInterval(refresh, 1000);","replace":" const timer = setInterval(() => refresh(true), 1000);"}]},{"id":"managed-support-allows-another-customer","scenario":"tracks/ecommerce/scenarios/progression-managed-support-privacy.json","targets":["ecommerce.spec.access-control.managed-support-privacy.613b"],"desc":"Managed support ownership is not checked.","file":"server/src/progression.ts","edits":[{"find":" if (user?.isStaff || user?.isAdmin || (ticket.userId && String(ticket.userId) === String(user?._id))) {\n return ticket;\n }\n return null;","replace":" return ticket;"}]},{"id":"notification-preference-is-not-saved","scenario":"tracks/ecommerce/scenarios/progression-notification-preferences.json","targets":["ecommerce.spec.state-durability.notification-preferences-reload.630a","ecommerce.progression.notification-preferences.notification-preferences.630c"],"desc":"Notification preferences are always saved as disabled, so both the saved-choice view and reload show the wrong value.","file":"server/src/progression.ts","edits":[{"find":"userId: req.progressionUser._id, order: Boolean(req.body?.order), stock: Boolean(req.body?.stock),","replace":"userId: req.progressionUser._id, order: false, stock: false,"}]},{"id":"notification-preference-is-not-owner-scoped","scenario":"tracks/ecommerce/scenarios/progression-notification-preferences.json","targets":["ecommerce.spec.access-control.notification-preferences-privacy.630b"],"desc":"Notification settings read another account preference.","file":"server/src/progression.ts","edits":[{"find":"user ? Preference.findOne({ userId: user._id }) : null,","replace":"user ? Preference.findOne({}) : null,"}]},{"id":"active-promotion-has-zero-discount","scenario":"tracks/ecommerce/scenarios/progression-promotion-checkout.json","targets":["ecommerce.progression.promotion-checkout.promotion-checkout-active.621a"],"desc":"An active promotion is recorded without changing checkout total.","file":"server/src/progression.ts","edits":[{"find":"{ $set: { promotionCode: code, discount: promotion.discount } }, { new: true });","replace":"{ $set: { promotionCode: code, discount: 0 } }, { new: true });"}]},{"id":"unavailable-promotions-are-accepted","scenario":"tracks/ecommerce/scenarios/progression-promotion-checkout.json","targets":["ecommerce.progression.promotion-checkout.promotion-checkout-expired.621b","ecommerce.progression.promotion-checkout.promotion-checkout-exhausted.621c"],"desc":"Expired and exhausted promotions are accepted.","file":"server/src/progression.ts","edits":[{"find":" if (!promotion || promotion.start > now || promotion.end < now\n || promotion.redemptions >= promotion.limit) {","replace":" if (!promotion) {"}]},{"id":"promotion-save-drops-bounded-values","scenario":"tracks/ecommerce/scenarios/progression-promotion-rules.json","targets":["ecommerce.progression.promotion-rules.promotion-rule-values.620a"],"desc":"Promotion creation drops its bounded discount and limit.","file":"server/src/progression.ts","edits":[{"find":"{ code, discount, limit, start, end }, { upsert: true, new: true });","replace":"{ code, discount: 1, limit: 1, start, end }, { upsert: true, new: true });"}]},{"id":"customer-can-create-promotion","scenario":"tracks/ecommerce/scenarios/progression-promotion-rules.json","targets":["ecommerce.spec.access-control.promotion-management-boundary.620b"],"desc":"Promotion creation omits its staff authorization check.","file":"server/src/progression.ts","edits":[{"find":"app.post(\"/api/promotions\", auth, staff, async (req: Request, res) => {","replace":"app.post(\"/api/promotions\", auth, async (req: Request, res) => {"}]},{"id":"staff-signin-loses-staff-role","scenario":"tracks/ecommerce/scenarios/progression-staff-access.json","targets":["ecommerce.progression.staff-access.staff-access.601a"],"desc":"Deny administrators entry to staff tools while keeping ordinary staff access working.","file":"client/src/App.tsx","edits":[{"find":"{(currentUser?.isStaff || currentUser?.isAdmin) && (\n <button className=\"btn btn-ghost\" data-role=\"staff-link\"","replace":"{(currentUser?.isStaff && !currentUser?.isAdmin) && (\n <button className=\"btn btn-ghost\" data-role=\"staff-link\""}]},{"id":"customer-signin-gains-staff-role","scenario":"tracks/ecommerce/scenarios/progression-staff-access.json","targets":["ecommerce.spec.access-control.staff-area-boundary.601b"],"desc":"Signin falsely exposes every customer as staff.","file":"server/src/index.ts","edits":[{"find":" isStaff: user.isStaff, roles: user.roles || [] };","replace":" isStaff: true, roles: user.roles || [] };"}]},{"id":"role-assignment-drops-role","scenario":"tracks/ecommerce/scenarios/progression-staff-roles.json","targets":["ecommerce.spec.state-durability.staff-role-reload.621a"],"desc":"Inventory role assignments are not saved; staff and administrator role transitions still work.","file":"server/src/progression.ts","edits":[{"find":" target.roles = roles;","replace":" if (role !== \"inventory\") target.roles = roles;"}]},{"id":"staff-role-write-precedes-denial","scenario":"tracks/ecommerce/scenarios/progression-staff-roles.json","targets":["ecommerce.spec.access-control.staff-role-boundary.621b","ecommerce.spec.access-control.staff-role-revocation.621d"],"desc":"Save an unauthorized role change before returning a forbidden response, including after administrator access is revoked.","file":"server/src/progression.ts","edits":[{"find":"app.put(\"/api/staff/:id/role\", auth, admin, async (req: Request, res) => {","replace":"app.put(\"/api/staff/:id/role\", auth, staff, async (req: Request, res) => {"},{"find":" await recordActivity(req.progressionUser, \"updated roles\", target.username);","replace":" if (!req.progressionUser.isAdmin) { res.status(403).json({ error: \"Admin access required\" }); return; }\n await recordActivity(req.progressionUser, \"updated roles\", target.username);"}]},{"id":"staff-can-assign-roles","scenario":"tracks/ecommerce/scenarios/progression-staff-roles.json","targets":["ecommerce.spec.access-control.staff-role-boundary.621b","ecommerce.spec.access-control.staff-role-revocation.621d"],"desc":"Staff can call the role assignment endpoint, including after administrator access is revoked.","file":"server/src/progression.ts","edits":[{"find":"app.put(\"/api/staff/:id/role\", auth, admin, async (req: Request, res) => {","replace":"app.put(\"/api/staff/:id/role\", auth, staff, async (req: Request, res) => {"}]},{"id":"stock-alert-delivery-is-suppressed","scenario":"tracks/ecommerce/scenarios/progression-stock-alert-delivery.json","targets":["ecommerce.progression.stock-alerts.stock-alert-delivery.631c"],"desc":"A request is accepted but restored stock does not deliver its alert.","file":"server/src/progression.ts","edits":[{"find":"await Notification.updateOne({ userId: alert.userId, key: `stock:${alert.itemId}` },","replace":"if (false) await Notification.updateOne({ userId: alert.userId, key: `stock:${alert.itemId}` },"}]},{"id":"stock-alert-repeats-while-in-stock","scenario":"tracks/ecommerce/scenarios/progression-stock-alerts.json","targets":["ecommerce.spec.transactional-integrity.stock-alert-deduplication.631a"],"desc":"Keep a delivered stock request active and create a notification with a new key on each timer tick while stock remains available, so its owner receives duplicate alerts.","file":"server/src/progression.ts","edits":[{"find":" alert.sent = true;\n await alert.save();","replace":" alert.sent = false;\n await alert.save();"},{"find":" await Notification.updateOne({ userId: alert.userId, key: `stock:${alert.itemId}` },\n { $setOnInsert: { userId: alert.userId, key: `stock:${alert.itemId}`, type: \"stock\",\n message: `${item?.name || \"Item\"} is back in stock` } }, { upsert: true });","replace":" await Notification.create({ userId: alert.userId, key: `stock:${alert.itemId}:${new mongoose.Types.ObjectId()}`, type: \"stock\",\n message: `${item?.name || \"Item\"} is back in stock` });"}]},{"id":"stock-alerts-are-not-owner-scoped","scenario":"tracks/ecommerce/scenarios/progression-stock-alerts.json","targets":["ecommerce.spec.access-control.stock-alert-privacy.631b"],"desc":"Notification reads expose alerts from other accounts.","file":"server/src/progression.ts","edits":[{"find":"user ? Notification.find({ userId: user._id }).sort({ createdAt: -1 }) : [],","replace":"user ? Notification.find({}).sort({ createdAt: -1 }) : [],"}]},{"id":"support-history-is-lost-on-server-restart","scenario":"tracks/ecommerce/scenarios/progression-support-history.json","targets":["ecommerce.spec.state-durability.support-history-reload.612a"],"desc":"Application startup discards support tickets. Ticket creation and owner reads work before runtime restart; privacy uses independently created history.","file":"server/src/index.ts","edits":[{"find":"import { Dismissal, Payment, Promotion } from \"./progression-models.js\";","replace":"import { Dismissal, Payment, Promotion, SupportTicket } from \"./progression-models.js\";"},{"find":" await mongoose.connect(DATABASE_URL);","replace":" await mongoose.connect(DATABASE_URL);\n await SupportTicket.deleteMany({});"}]},{"id":"support-history-is-not-owner-scoped","scenario":"tracks/ecommerce/scenarios/progression-support-history.json","targets":["ecommerce.spec.access-control.support-history-privacy.612b"],"desc":"Signed-in customers can read other customers support history.","file":"server/src/progression.ts","edits":[{"find":"const ticketFilter = isStaff ? {} : user ? { userId: user._id } : { _id: null };","replace":"const ticketFilter = isStaff ? {} : user ? {} : { _id: null };"}]},{"id":"support-intake-returns-no-reference","scenario":"tracks/ecommerce/scenarios/progression-support-intake.json","targets":["ecommerce.progression.support-intake.support-intake.610a"],"desc":"Support intake creates a ticket but omits its reference from the response.","file":"server/src/progression.ts","edits":[{"find":"res.status(201).json({ ticket });","replace":"res.status(201).json({ ticket: { ...ticket.toJSON(), reference: \"\" } });"}]},{"id":"support-triage-discards-updates","scenario":"tracks/ecommerce/scenarios/progression-support-triage.json","targets":["ecommerce.progression.support-triage.support-assignment.611a","ecommerce.progression.support-triage.support-priority.611b","ecommerce.progression.support-triage.support-status.611c"],"desc":"Support triage accepts but discards assignment, priority, and status updates.","file":"server/src/progression.ts","edits":[{"find":"const ticket = ticketId\n ? await SupportTicket.findByIdAndUpdate(ticketId, { $set: update }, { new: true }) : null;","replace":"const ticket = ticketId\n ? await SupportTicket.findByIdAndUpdate(ticketId, { $set: {} }, { new: true }) : null;"}]},{"id":"cart-add-uses-another-account-cart","scenario":"tracks/ecommerce/scenarios/01-cart-boundary.json","targets":["ecommerce.spec.access-control.cart-boundary.109a"],"desc":"Cart add writes to the first account cart instead of the caller cart.","file":"server/src/index.ts","edits":[{"find":" const incResult = await Cart.updateOne(\n { userId, \"items.itemId\": itemId },","replace":" const incResult = await Cart.updateOne(\n { \"items.itemId\": itemId },"}]},{"id":"negative-cart-quantity-is-accepted","scenario":"tracks/ecommerce/scenarios/01-cart-boundary.json","targets":["ecommerce.spec.access-control.cart-boundary.109b"],"desc":"Cart quantity updates accept values below one.","file":"server/src/index.ts","edits":[{"find":" if (!Number.isInteger(qty) || qty < 1) {\n return res.status(400).json({ error: \"Quantity must be at least 1\" });\n }","replace":" if (!Number.isInteger(qty)) {\n return res.status(400).json({ error: \"Quantity must be numeric\" });\n }"}]},{"id":"direct-purchase-is-attributed-to-another-account","scenario":"tracks/ecommerce/scenarios/01-purchase-attribution.json","targets":["ecommerce.spec.access-control.purchase-attribution.102a"],"desc":"Direct purchase records the order against another account.","file":"server/src/index.ts","edits":[{"find":" userId: user._id,\n items: [","replace":" userId: (await User.findOne({ _id: { $ne: user._id } }))!._id,\n items: ["}]},{"id":"concurrent-cart-add-does-not-increment","scenario":"tracks/ecommerce/scenarios/01-duplicate-checkout.json","targets":["ecommerce.spec.concurrency-safety.duplicate-checkout.203a"],"desc":"Concurrent cart adds leave the line quantity and its reservation list unchanged, so the existing cart remains checkable at checkout.","file":"server/src/index.ts","edits":[{"find":"{ $inc: { \"items.$.quantity\": qty },\n $set: { \"items.$.reservationExpiresAt\": new Date(Date.now() + 90_000),\n inactiveExpiresAt: new Date(Date.now() + 300_000) },\n $push: { \"items.$.reservedWarehouseIds\": { $each: reservedWarehouseIds } } }","replace":"{ $inc: { \"items.$.quantity\": 0 },\n $set: { \"items.$.reservationExpiresAt\": new Date(Date.now() + 90_000),\n inactiveExpiresAt: new Date(Date.now() + 300_000) },\n $push: { \"items.$.reservedWarehouseIds\": { $each: [] } } }"}]},{"id":"checkout-claim-is-not-atomic","scenario":"tracks/ecommerce/scenarios/01-duplicate-checkout.json","targets":["ecommerce.spec.concurrency-safety.duplicate-checkout.203b"],"desc":"Concurrent checkout reads the cart without atomically claiming it.","file":"server/src/credit.ts","edits":[{"find":" return mongoose.connection.transaction(async session => {\n const cart = await Cart.findOne({ userId }).session(session);","replace":" const snapshot = await Cart.findOne({ userId });\n if (snapshot) snapshot.set(\"__v\", undefined);\n await new Promise(resolve => setTimeout(resolve, 500));\n return mongoose.connection.transaction(async session => {\n const cart = snapshot ? Cart.hydrate(snapshot.toObject()) : null;"}]},{"id":"last-unit-allows-negative-stock","scenario":"tracks/ecommerce/scenarios/01-last-unit.json","targets":["ecommerce.spec.concurrency-safety.last-unit.201a","ecommerce.spec.concurrency-safety.last-unit.201b","ecommerce.spec.concurrency-safety.last-unit.201c"],"desc":"Stock reservation can decrement a warehouse below zero.","file":"server/src/stock-reservations.ts","edits":[{"find":"{ item_id: itemId, quantity: { $gte: 1 } },","replace":"{ item_id: itemId },"}]},{"id":"purchase-read-write-loses-concurrent-stock","scenario":"tracks/ecommerce/scenarios/01-restock-race.json","targets":["ecommerce.spec.concurrency-safety.restock-race.202a"],"desc":"Replace atomic stock reservation with an unlocked read and absolute write. A fixed pause widens scheduling overlap; serial purchases and restocks retain their stock effects. Concurrent reservations or restocking can lose updates.","file":"server/src/stock-reservations.ts","edits":[{"find":" const stock = await model.findOneAndUpdate(\n { item_id: itemId, quantity: { $gte: 1 } },\n { $inc: { quantity: -1 } },\n { sort: { quantity: -1 }, ...(session ? { session } : {}) },\n );\n if (!stock) {\n await releaseWithModel(model, itemId, warehouseIds, session);\n return null;\n }\n","replace":" const stock = await model.findOneAndUpdate(\n { item_id: itemId, quantity: { $gte: 1 } },\n { $inc: { quantity: 0 } },\n { sort: { quantity: -1 }, ...(session ? { session } : {}) },\n );\n if (!stock) {\n await releaseWithModel(model, itemId, warehouseIds, session);\n return null;\n }\n // Mutant: widen the unlocked read/write window without changing serial behavior.\n await new Promise(resolve => setTimeout(resolve, 500));\n await model.updateOne({ item_id: itemId, warehouse_id: stock.warehouse_id },\n { $set: { quantity: stock.quantity - 1 } });\n"}]},{"id":"stock-limit-allows-negative-stock","scenario":"tracks/ecommerce/scenarios/progression-stock-limit.json","targets":["ecommerce.spec.concurrency-safety.stock-limit.3d"],"desc":"Stock reservation accepts purchases after sellout.","file":"server/src/stock-reservations.ts","edits":[{"find":"{ item_id: itemId, quantity: { $gte: 1 } },","replace":"{ item_id: itemId },"}]},{"id":"purchase-does-not-reduce-warehouse-stock","scenario":"tracks/ecommerce/scenarios/progression-books-balance.json","targets":["ecommerce.spec.transactional-integrity.books-balance.107b"],"desc":"Purchases create orders without reducing warehouse stock.","file":"server/src/stock-reservations.ts","edits":[{"find":" { $inc: { quantity: -1 } },","replace":" { $inc: { quantity: 0 } },"}]},{"id":"restock-does-not-increase-stock","scenario":"tracks/ecommerce/scenarios/01-warehouse-stock-live-staff.json","targets":["ecommerce.spec.live-state.warehouse-stock.7c"],"desc":"The restock endpoint accepts a positive quantity but increments authoritative stock by zero.","file":"server/src/index.ts","edits":[{"find":" await Stock.findOneAndUpdate(\n { item_id: item._id, warehouse_id: warehouse._id },\n { $inc: { quantity: qty } },\n { upsert: true }\n );","replace":" await Stock.findOneAndUpdate(\n { item_id: item._id, warehouse_id: warehouse._id },\n { $inc: { quantity: 0 } },\n { upsert: true }\n );"}]},{"id":"operator-authorization-allows-customer-price-change","scenario":"tracks/ecommerce/scenarios/02-strengthened.json","targets":["ecommerce.operations-access.operator-authorization.201b"],"desc":"The price route keeps authentication but drops its administrator role gate.","file":"server/src/index.ts","edits":[{"find":"app.post(\"/api/admin/price\", requireAuth, requireAdmin, async (req, res) => {","replace":"app.post(\"/api/admin/price\", requireAuth, async (req, res) => {"}]},{"id":"operator-authorization-allows-customer-shipping","scenario":"tracks/ecommerce/scenarios/02-self-contained.json","targets":["ecommerce.operations-access.fulfilment-queue.1e"],"desc":"The shipping route retains authentication but drops the staff role gate.","file":"server/src/index.ts","edits":[{"find":"app.post(\"/api/fulfilment/ship\", requireAuth, requireStaff, async (req, res) => {","replace":"app.post(\"/api/fulfilment/ship\", requireAuth, async (req, res) => {"}]},{"id":"catalog-price-change-not-broadcast-to-visitors","scenario":"tracks/ecommerce/scenarios/02-live-price.json","targets":["ecommerce.returns-pricing.price-history.4b"],"desc":"The signed-out search result caches detached rows and ignores later catalog updates.","file":"client/src/App.tsx","edits":[{"find":" }, [items, searchQuery, categoryFilter, minimumPrice, maximumPrice, inStockOnly]);","replace":" }, [searchQuery, categoryFilter, minimumPrice, maximumPrice, inStockOnly]);"}]},{"id":"catalog-price-change-not-broadcast-to-open-carts","scenario":"tracks/ecommerce/scenarios/progression-price-cart-checkout.json","targets":["ecommerce.returns-pricing.price-history.4c"],"desc":"The price reaches the catalog but the price route omits its targeted open-cart broadcast.","file":"server/src/index.ts","edits":[{"find":" broadcastRecommendedForAll(),\n broadcastCartsContainingItem(item._id),","replace":" broadcastRecommendedForAll(),\n // mutant: carts containing this item are not refreshed"}]},{"id":"catalog-price-rewrites-receipts","scenario":"tracks/ecommerce/scenarios/02-paid-price-history.json","targets":["ecommerce.returns-pricing.price-history.4a"],"desc":"Changing a catalog price cascades into saved order lines and recomputes historical order totals.","file":"server/src/index.ts","edits":[{"find":" const item = await Item.findByIdAndUpdate(itemId, { price: priceNum }, { new: true });\n if (!item) return res.status(404).json({ error: \"Item not found\" });","replace":" const item = await Item.findByIdAndUpdate(itemId, { price: priceNum }, { new: true });\n if (!item) return res.status(404).json({ error: \"Item not found\" });\n const historicalOrders = await Order.find({ \"items.itemId\": item._id });\n for (const historical of historicalOrders) {\n for (const line of historical.items) {\n if (line.itemId.toString() === item._id.toString()) line.price = priceNum;\n }\n historical.total = historical.items.reduce((sum, line) => sum + line.price * line.quantity, 0);\n await historical.save();\n }"}]},{"id":"catalog-price-rewrites-earned-revenue","scenario":"tracks/ecommerce/scenarios/02-invariants.json","targets":["ecommerce.returns-pricing.refund-accounting.203b"],"desc":"Changing a catalog price cascades into saved order lines and recomputes historical order totals and revenue.","file":"server/src/index.ts","edits":[{"find":" const item = await Item.findByIdAndUpdate(itemId, { price: priceNum }, { new: true });\n if (!item) return res.status(404).json({ error: \"Item not found\" });","replace":" const item = await Item.findByIdAndUpdate(itemId, { price: priceNum }, { new: true });\n if (!item) return res.status(404).json({ error: \"Item not found\" });\n const historicalOrders = await Order.find({ \"items.itemId\": item._id });\n for (const historical of historicalOrders) {\n for (const line of historical.items) {\n if (line.itemId.toString() === item._id.toString()) line.price = priceNum;\n }\n historical.total = historical.items.reduce((sum, line) => sum + line.price * line.quantity, 0);\n await historical.save();\n }"}]},{"id":"returned-line-marker-omitted","scenario":"tracks/ecommerce/scenarios/02-strengthened.json","targets":["ecommerce.returns-pricing.cancellation-and-return.3c"],"desc":"A returned order line keeps its persisted returned state, restored stock, and adjusted revenue but omits the visible returned marker.","file":"client/src/App.tsx","edits":[{"find":"l.returned ? \" (returned)\" : \"\"","replace":"\"\""}]},{"id":"direct-review-access-is-not-checked","scenario":"tracks/ecommerce/scenarios/progression-review-access.json","targets":["ecommerce.progression.review-access-specifications.review-eligibility-direct.618a"],"desc":"The direct review action accepts a review from a customer who did not buy the item.","file":"server/src/index.ts","edits":[{"find":" if (!hasPurchased) {\n return res.status(403).json({ error: \"You can only review items you have purchased\" });\n }","replace":" if (false && !hasPurchased) {\n return res.status(403).json({ error: \"You can only review items you have purchased\" });\n }"}]},{"id":"support-history-rows-are-hidden","scenario":"tracks/ecommerce/scenarios/progression-support-history.json","targets":["ecommerce.progression.support-history.support-history.612c","ecommerce.spec.state-durability.support-history-reload.612a","ecommerce.spec.access-control.support-history-privacy.612b","ecommerce.spec.access-control.support-history-logout.612d"],"desc":"Hide submitted support ticket rows while leaving the submission reference available. History, reload, privacy, and logout require the owner to see the ticket first. This breaks those positive observations; it does not create a privacy leak or remove stored data.","file":"client/src/ProgressionPanel.tsx","edits":[{"find":"data-role=\"support-ticket\"","replace":"data-role=\"support-ticket\" style={{ display: \"none\" }}"}]},{"id":"authorized-restock-does-not-change-stock","scenario":"tracks/ecommerce/scenarios/01-admin-write-staff.json","targets":["ecommerce.feature.warehouse-admin.admin-write.103a"],"desc":"Accept an administrator restock without changing warehouse stock.","file":"server/src/index.ts","edits":[{"find":" await Stock.findOneAndUpdate(\n { item_id: item._id, warehouse_id: warehouse._id },\n { $inc: { quantity: qty } },\n { upsert: true }\n );\n\n","replace":" // mutant: accept restock without changing stock\n\n"}]},{"id":"initial-dashboard-load-omits-low-stock","scenario":"tracks/ecommerce/scenarios/02-low-stock.json","targets":["ecommerce.inventory-operations.operational-views.5e","ecommerce.spec.live-state.inventory-dashboard.5a"],"desc":"The dashboard's initial load reports no low-stock items, so an item already at ten units or fewer is missing until stock moves; 5a's opening precondition re-asserts the same listing, so it necessarily regresses with 5e.","file":"server/src/index.ts","edits":[{"find":"app.get(\"/api/admin/overview\", requireAuth, requireAdmin, async (_req, res) => {\n res.json(await getAdminOverview());","replace":"app.get(\"/api/admin/overview\", requireAuth, requireAdmin, async (_req, res) => {\n res.json({ ...(await getAdminOverview()), lowStock: [] }); // mutant: the initial dashboard load omits current low-stock items"}]},{"id":"category-totals-skip-the-newest-order","scenario":"tracks/ecommerce/scenarios/02-operational-category-totals.json","targets":["ecommerce.inventory-operations.operational-views.5f"],"desc":"Category totals leave out the most recent order, so the reloaded dashboard shows a purchase only once a later order pushes it into the aggregate.","file":"server/src/index.ts","edits":[{"find":"async function getCategoryTotals() {\n const rows = await Order.aggregate([\n { $match: { status: { $ne: \"cancelled\" } } },","replace":"async function getCategoryTotals() {\n const rows = await Order.aggregate([\n { $match: { status: { $ne: \"cancelled\" } } },\n { $sort: { createdAt: -1 } },\n { $skip: 1 }, // mutant: the most recent order is left out of the category totals"}]},{"id":"profile-summary-frozen-at-open","scenario":"tracks/ecommerce/scenarios/progression-customer-profile.json","targets":["ecommerce.progression.customer-profile.customer-profile.620c"],"desc":"The profile summary shows the address loaded when the profile card opened, so a freshly saved address is not visible until the page is reloaded.","file":"client/src/ProgressionPanel.tsx","edits":[{"find":" <p data-role=\"profile-address-summary\">{state.profile?.address || \"\"}</p>","replace":" <ProfileAddressSummary address={state.profile?.address || \"\"} />"},{"find":"function SupportTicket({ ticket, user, orders, act }: any) {","replace":"function ProfileAddressSummary({ address }: { address: string }) {\n const [shown] = useState(address); // mutant: the summary keeps the address loaded when the card opened\n return <p data-role=\"profile-address-summary\">{shown}</p>;\n}\n\nfunction SupportTicket({ ticket, user, orders, act }: any) {"}]},{"id":"support-replies-present-at-open-are-hidden","scenario":"tracks/ecommerce/scenarios/progression-managed-support-shared.json","targets":["ecommerce.progression.managed-support.managed-support.613c"],"desc":"A support case renders only replies that arrive while it is open, so after a reload the replies already exchanged are missing while live replies still appear.","file":"client/src/ProgressionPanel.tsx","edits":[{"find":" const [reply, setReply] = useState(\"\");","replace":" const [reply, setReply] = useState(\"\");\n const [repliesAtOpen] = useState(() => new Set((ticket.replies || []).map((entry: any) => entry.id || entry.createdAt))); // mutant: replies already on the case when it opened are never rendered"},{"find":" {(ticket.replies || []).map((entry: any) => <div data-role=\"support-reply-item\" key={entry.id || entry.createdAt}>{entry.username}: {entry.body}</div>)}","replace":" {(ticket.replies || []).filter((entry: any) => !repliesAtOpen.has(entry.id || entry.createdAt)).map((entry: any) => <div data-role=\"support-reply-item\" key={entry.id || entry.createdAt}>{entry.username}: {entry.body}</div>)}"}]},{"id":"notification-toggle-frozen-at-open","scenario":"tracks/ecommerce/scenarios/progression-notification-preferences.json","targets":["ecommerce.progression.notification-preferences.notification-preferences.630c"],"desc":"The order-notification toggle displays the state loaded when the settings card opened, so the saved choice is reported as off until a reload.","file":"client/src/ProgressionPanel.tsx","edits":[{"find":" <button data-role=\"notification-order\" data-state={preference.order ? \"on\" : \"off\"} className=\"btn btn-ghost\"\n onClick={() => setState((value: any) => ({ ...value, preference: { ...preference, order: !preference.order } }))}>\n Order notifications {preference.order ? \"on\" : \"off\"}\n </button>","replace":" <OrderNotificationToggle on={preference.order}\n onToggle={() => setState((value: any) => ({ ...value, preference: { ...preference, order: !preference.order } }))} />"},{"find":"function SupportTicket({ ticket, user, orders, act }: any) {","replace":"function OrderNotificationToggle({ on, onToggle }: { on: boolean; onToggle: () => void }) {\n const [shown] = useState(on); // mutant: the toggle keeps the state loaded when the card opened\n return <button data-role=\"notification-order\" data-state={shown ? \"on\" : \"off\"} className=\"btn btn-ghost\" onClick={onToggle}>\n Order notifications {shown ? \"on\" : \"off\"}\n </button>;\n}\n\nfunction SupportTicket({ ticket, user, orders, act }: any) {"}]},{"id":"role-editor-snaps-back-to-stored-role","scenario":"tracks/ecommerce/scenarios/progression-staff-roles.json","targets":["ecommerce.progression.staff-roles.staff-roles.621c"],"desc":"Pressing Save persists the role but resets the dropdown to the role that was stored before the save, so the assignment is not visible until a reload.","file":"client/src/ProgressionPanel.tsx","edits":[{"find":" <button data-role=\"staff-role-save\" onClick={() => act(`/api/staff/${entry.id}/role`, {\n method: \"PUT\", body: JSON.stringify({ role: roles[entry.username] ?? entry.roles?.[0] ?? \"staff\" }),\n })}>Save</button>","replace":" <button data-role=\"staff-role-save\" onClick={() => {\n setRoles(value => ({ ...value, [entry.username]: entry.roles?.[0] ?? \"staff\" })); // mutant: the editor snaps back to the previously stored role\n return act(`/api/staff/${entry.id}/role`, {\n method: \"PUT\", body: JSON.stringify({ role: roles[entry.username] ?? entry.roles?.[0] ?? \"staff\" }),\n }); }}>Save</button>"}]},{"id":"queue-ignores-live-fulfilment-updates","scenario":"tracks/ecommerce/scenarios/02-fulfilment-live.json","targets":["ecommerce.spec.live-state.fulfilment-queue.1a"],"desc":"The open staff queue ignores live fulfilment events, so a new order appears only after a reload.","file":"client/src/App.tsx","edits":[{"find":" socket.on(\"fulfilment:update\", (data: FulfilmentQueueT) => setFulfilmentQueue(data));","replace":" socket.on(\"fulfilment:update\", (data: FulfilmentQueueT) => { void data; }); // mutant: the open staff queue ignores live fulfilment updates"}]},{"id":"low-stock-boundary-excludes-ten-live","scenario":"tracks/ecommerce/scenarios/02-low-stock.json","targets":["ecommerce.spec.live-state.inventory-dashboard.5a"],"desc":"The low-stock view uses a strict boundary, so an item that falls to exactly ten units never joins the list.","file":"server/src/index.ts","edits":[{"find":" .filter((it) => it.stock <= 10)","replace":" .filter((it) => it.stock < 10)"}]},{"id":"live-admin-updates-keep-stale-category-totals","scenario":"tracks/ecommerce/scenarios/02-operational-category-totals.json","targets":["ecommerce.spec.live-state.sales-dashboard.5b"],"desc":"Live admin updates keep the category totals loaded at page load, so a purchase does not move units or revenue until a reload.","file":"client/src/App.tsx","edits":[{"find":" socket.on(\"admin:update\", (data: AdminOverviewT) => setAdminOverview(data));","replace":" socket.on(\"admin:update\", (data: AdminOverviewT) => setAdminOverview((previous) => previous ? { ...data, categories: previous.categories } : data)); // mutant: live admin updates keep the category totals loaded at page load"}]},{"id":"overdraw-transfer-is-accepted","scenario":"tracks/ecommerce/scenarios/02-transfer-overdraw.json","targets":["ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c"],"desc":"The atomic source debit no longer requires sufficient quantity, so an overdrawn transfer succeeds and moves both warehouse totals.","file":"server/src/index.ts","edits":[{"find":"{ item_id: itemId, warehouse_id: fromWarehouseId, quantity: { $gte: qty } }","replace":"{ item_id: itemId, warehouse_id: fromWarehouseId }"}]},{"id":"transfer-totals-omit-destination-credit-live","scenario":"tracks/ecommerce/scenarios/02-transfer-totals.json","targets":["ecommerce.spec.live-state.stock-transfers.2b"],"desc":"A transfer debits the source but adds zero to the destination, so the two live warehouse totals do not move in opposite directions.","file":"server/src/index.ts","edits":[{"find":" await Stock.findOneAndUpdate(\n { item_id: itemId, warehouse_id: toWarehouseId },\n { $inc: { quantity: qty } },\n { upsert: true }\n );","replace":" await Stock.findOneAndUpdate(\n { item_id: itemId, warehouse_id: toWarehouseId },\n { $inc: { quantity: 0 } },\n { upsert: true }\n );"}]},{"id":"credit-checkout-ignores-wallet","desc":"A credit checkout pays entirely externally despite available credit.","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.feature.store-credit.store-credit-750.750a"],"file":"server/src/credit.ts","edits":[{"find":" const creditMinor = useCredit ? Math.min(user.creditMinor, totalMinor) : 0;","replace":" const creditMinor = 0;"}]},{"id":"credit-grant-replay-increments-balance","desc":"Replaying a grant applies its credit to the wallet again.","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.spec.store-credit.production-752.752a"],"file":"server/src/credit.ts","edits":[{"find":" if (existing.amountMinor !== amountMinor) throw new Error('Reference already identifies another grant');\n return;","replace":" if (existing.amountMinor !== amountMinor) throw new Error('Reference already identifies another grant');\n await User.updateOne({ _id: accountId }, { $inc: { creditMinor: amountMinor } }, { session });\n return;"}]},{"id":"customer-can-grant-credit","desc":"Customer authentication is accepted without staff authorization.","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.spec.store-credit.production-753.753a"],"file":"server/src/credit.ts","edits":[{"find":" app.post('/api/staff/credit', auth, staff, async (req, res) => {","replace":" app.post('/api/staff/credit', auth, async (req, res) => {"}]},{"id":"credit-checkout-retains-purchased-cart","desc":"A second checkout can reuse the purchased cart and create another order.","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.spec.store-credit.production-754.754a"],"file":"server/src/credit.ts","edits":[{"find":" cart.items = cart.items.filter(line => line.reservationExpiresAt && line.reservationExpiresAt <= now) as any;","replace":" // mutant: purchased cart lines remain"}]},{"id":"split-refund-does-not-restore-credit","desc":"The refund is recorded but its original wallet credit is not restored.","scenario":"tracks/ecommerce/scenarios/progression-split-tender-refunds.json","targets":["ecommerce.feature.split-tender-refunds.split-tender-refunds-751.751a"],"file":"server/src/progression.ts","edits":[{"find":" await refundCredit(order, session);","replace":" // mutant: omit wallet restoration"}]},{"id":"split-refund-duplicates-credit","desc":"A refund credits the wallet twice while recording one refund.","scenario":"tracks/ecommerce/scenarios/progression-split-tender-refunds.json","targets":["ecommerce.spec.split-tender-refunds.production-756.756a"],"file":"server/src/credit.ts","edits":[{"find":"{ $inc: { creditMinor: delta } }, { session });","replace":"{ $inc: { creditMinor: delta * 2 } }, { session });"}]},{"id":"subscription-skips-due-purchase","desc":"Due deliveries are recorded as skipped although stock is available.","scenario":"tracks/ecommerce/scenarios/progression-subscriptions.json","targets":["ecommerce.feature.subscriptions.subscriptions-760.760a"],"file":"server/src/subscriptions.ts","edits":[{"find":" const allocation = await reserveStock(row.itemId, row.quantity, session);","replace":" const allocation = row.quantity < 0 ? await reserveStock(row.itemId, row.quantity, session) : null;"}]},{"id":"subscription-allows-foreign-cancellation","desc":"A customer can cancel another customer subscription.","scenario":"tracks/ecommerce/scenarios/progression-subscriptions.json","targets":["ecommerce.spec.subscriptions.production-762.762a"],"file":"server/src/subscriptions.ts","edits":[{"find":" if (!row || String(row.userId) !== String((req as any).user._id)) return false;","replace":" if (!row) return false;"}]},{"id":"subscription-pause-is-not-recorded","desc":"Pause acknowledges the request but the subscription remains active.","scenario":"tracks/ecommerce/scenarios/progression-subscriptions.json","targets":["ecommerce.spec.subscriptions.production-763.763a"],"file":"server/src/subscriptions.ts","edits":[{"find":" row.status = 'paused'; row.pausedAt = new Date();","replace":" row.status = 'active'; row.pausedAt = new Date();"}]},{"id":"credit-balance-is-cleared-at-startup","desc":"Restart clears an issued wallet balance while leaving accounts present.","file":"server/src/index.ts","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.spec.store-credit.production-755.755a"],"edits":[{"find":" await seed();","replace":" await seed();\n await User.updateMany({}, { $set: { creditMinor: 0 } });"}]},{"id":"pending-subscriptions-are-cleared-at-startup","desc":"Restart erases pending subscription work while preserving ordinary timer execution.","file":"server/src/index.ts","scenario":"tracks/ecommerce/scenarios/progression-subscriptions.json","targets":["ecommerce.spec.subscriptions.production-761.761a"],"edits":[{"find":" await seed();","replace":" await seed();\n await mongoose.connection.collection(\"purchasesubscriptions\").deleteMany({ status: \"active\" });"}]},{"id":"bundle-definition-loses-component-quantity","scenario":"tracks/ecommerce/scenarios/progression-product-bundles.json","targets":["ecommerce.feature.product-bundles.product-bundles.740a"],"desc":"definition loses component quantity","file":"server/src/bundles.ts","edits":[{"find":"quantity: component.quantity });","replace":"quantity: 1 });"}]},{"id":"bundle-catalog-write-allows-customers","scenario":"tracks/ecommerce/scenarios/progression-product-bundles.json","targets":["ecommerce.spec.bundle-integrity.bundle-743.743a"],"desc":"catalog write allows customers","file":"server/src/bundles.ts","edits":[{"find":"if (!actor.isAdmin && !actor.roles?.includes('catalog'))","replace":"if (false)"}]},{"id":"bundle-checkout-price-not-snapshot","scenario":"tracks/ecommerce/scenarios/progression-bundle-checkout.json","targets":["ecommerce.feature.bundle-checkout.bundle-checkout.741a"],"desc":"checkout price not snapshot","file":"server/src/bundles.ts","edits":[{"find":"bundlePrice: bundle.price,","replace":"bundlePrice: bundle.price + 1,"}]},{"id":"bundle-expiry-does-not-release-components","scenario":"tracks/ecommerce/scenarios/progression-bundle-checkout.json","targets":["ecommerce.spec.bundle-integrity.bundle-746.746a"],"desc":"expiry does not release components","file":"server/src/bundles.ts","edits":[{"find":"await releaseBundle(line.componentAllocations as Allocation[], session);\n line.componentAllocations = [] as any;","replace":"line.componentAllocations = [] as any; // mutant: component holds leak"}]},{"id":"bundle-return-loses-original-components","scenario":"tracks/ecommerce/scenarios/progression-bundle-returns.json","targets":["ecommerce.feature.bundle-returns.bundle-returns.742a"],"desc":"return loses original components","file":"server/src/bundles.ts","edits":[{"find":"for (const line of bundles) { await releaseBundle(line.componentAllocations as Allocation[], session); line.returned = true; }","replace":"for (const line of bundles) { line.returned = true; }"}]},{"id":"bundle-return-replay-restocks-again","scenario":"tracks/ecommerce/scenarios/progression-bundle-returns.json","targets":["ecommerce.spec.bundle-integrity.bundle-742.742b"],"desc":"return replay restocks again","file":"server/src/bundles.ts","edits":[{"find":"line.isBundle && !line.returned","replace":"line.isBundle"}]},{"id":"bundle-return-crosses-account-boundary","scenario":"tracks/ecommerce/scenarios/progression-bundle-returns.json","targets":["ecommerce.spec.bundle-integrity.bundle-748.748a"],"desc":"return crosses account boundary","file":"server/src/bundles.ts","edits":[{"find":"{ _id: req.params.orderId, userId, status: { $in: ['shipped', 'delivered'] } }","replace":"{ _id: req.params.orderId, status: { $in: ['shipped', 'delivered'] } }"}]},{"id":"bundle-components-can-overdraw","scenario":"tracks/ecommerce/scenarios/progression-bundle-checkout.json","targets":["ecommerce.spec.bundle-integrity.bundle-744.744a","ecommerce.spec.bundle-integrity.bundle-745.745a"],"desc":"components can overdraw","file":"server/src/stock-reservations.ts","edits":[{"find":"{ item_id: itemId, quantity: { $gte: 1 } }","replace":"{ item_id: itemId }"}]},{"id":"bundle-checkout-reuses-reservation","scenario":"tracks/ecommerce/scenarios/progression-bundle-checkout.json","targets":["ecommerce.spec.bundle-integrity.bundle-747.747a"],"desc":"checkout reuses reservation","file":"server/src/credit.ts","edits":[{"find":"cart.items = cart.items.filter(line => line.reservationExpiresAt && line.reservationExpiresAt <= now) as any;","replace":"// mutant: active cart survives checkout"}]},{"id":"return-after-support-refund-is-blocked","scenario":"tracks/ecommerce/scenarios/progression-support-return-interaction.json","targets":["ecommerce.feature.split-tender-refunds.return-refund-interaction.757a"],"desc":"Reject a valid physical return after a financial refund.","file":"server/src/index.ts","edits":[{"find":" if (!line || line.returned) throw new Error('No returnable item found');","replace":" if (!line || line.returned || value.refundTotal > 0) throw new Error('No returnable item found');"}]},{"id":"support-refund-after-return-pays-twice","scenario":"tracks/ecommerce/scenarios/progression-support-return-interaction.json","targets":["ecommerce.feature.split-tender-refunds.return-refund-interaction.757b"],"desc":"Pay the full order again after a physical return.","file":"server/src/progression.ts","edits":[{"find":" order.refundTotal = order.total;","replace":" order.refundTotal += order.total;"}]},{"id":"support-history-leaks-to-signed-out-visitors","scenario":"tracks/ecommerce/scenarios/progression-support-history.json","targets":["ecommerce.spec.access-control.support-history-logout.612d"],"desc":"Disclose support tickets to signed-out visitors while leaving signed-in ownership filtering unchanged.","file":"server/src/progression.ts","edits":[{"find":"const ticketFilter = isStaff ? {} : user ? { userId: user._id } : { _id: null };","replace":"const ticketFilter = isStaff ? {} : user ? { userId: user._id } : {};"}]},{"id":"checkout-crash-integrity","scenario":"tracks/ecommerce/scenarios/progression-checkout-crash.json","targets":["ecommerce.spec.state-durability.checkout-crash-integrity.910a"],"desc":"A lifecycle callback changes a prepared or cleared cart to quantity two, leaving neither a legal rollback nor a complete checkout.","file":"server/src/index.ts","edits":[{"find":" await initializeOrderData(mongoose.connection);","replace":" await initializeOrderData(mongoose.connection);\n for (const cart of await Cart.find()) { const order = await Order.findOne({ userId: cart.userId }); if (order?.items[0]) await Cart.updateOne({ _id: cart._id }, { $set: { items: [{ itemId: order.items[0].itemId, quantity: 2 }] } }); }"}]},{"id":"checkout-crash-durability","scenario":"tracks/ecommerce/scenarios/progression-checkout-crash.json","targets":["ecommerce.spec.state-durability.checkout-crash-durability.910b"],"desc":"A lifecycle callback corrupts earlier order totals. The unchanged prepared snapshots and acknowledged baseline must expose the damage.","file":"server/src/index.ts","edits":[{"find":" await initializeOrderData(mongoose.connection);","replace":" await initializeOrderData(mongoose.connection);\n await Order.updateMany({}, { $set: { total: 0 } });"}]},{"id":"review-script-unsafe-render","scenario":"tracks/ecommerce/scenarios/progression-review-script.json","targets":["ecommerce.progression.review-access-specifications.stored-review-script.9180a"],"desc":"Render stored review text as HTML so it executes in another customer session.","file":"client/src/App.tsx","edits":[{"find":"<div>{r.comment}</div>","replace":"<div dangerouslySetInnerHTML={{ __html: r.comment }} />"}]},{"id":"review-script-reject-all","scenario":"tracks/ecommerce/scenarios/progression-review-script.json","targets":["ecommerce.progression.review-access-specifications.stored-review-script.9180a"],"desc":"Reject all nonempty reviews, including the required benign review; absence of script is insufficient.","file":"server/src/index.ts","edits":[{"find":"const { rating, comment } = req.body || {};","replace":"const { rating, comment } = req.body || {};\n if (String(comment).length > 0) { res.status(400).json({ error: \"Review rejected\" }); return; }"}]},{"id":"review-owner-trust-username","scenario":"tracks/ecommerce/scenarios/progression-review-access.json","targets":["ecommerce.progression.review-access-specifications.review-eligibility-direct.618a"],"desc":"Trust the submitted buyer username instead of the authenticated caller.","file":"server/src/index.ts","edits":[{"find":"const user = (req as any).user;\n const hasPurchased = await Order.exists","replace":"const user = req.body?.username ? await User.findOne({ username: req.body.username }) : (req as any).user;\n const hasPurchased = await Order.exists"}]},{"id":"review-owner-deny-after-write","scenario":"tracks/ecommerce/scenarios/progression-review-access.json","targets":["ecommerce.progression.review-access-specifications.review-eligibility-direct.618a"],"desc":"Replace the buyer review under a forged username, then return a refusal.","file":"server/src/index.ts","edits":[{"find":"const user = (req as any).user;\n const hasPurchased = await Order.exists","replace":"const user = req.body?.username ? await User.findOne({ username: req.body.username }) : (req as any).user;\n const hasPurchased = await Order.exists"},{"find":"await broadcastReviews(String(itemId));","replace":"if (req.body?.username) return res.status(403).json({ error: \"Refused after write\" });\n await broadcastReviews(String(itemId));"}]},{"id":"review-owner-reject-all","scenario":"tracks/ecommerce/scenarios/progression-review-access.json","targets":["ecommerce.progression.review-access-specifications.review-eligibility-direct.618a"],"desc":"Reject the legitimate buyer review as well as unauthorized reviews.","file":"server/src/index.ts","edits":[{"find":"const { rating, comment } = req.body || {};","replace":"const { rating, comment } = req.body || {};\n if (String(comment).length > 0) { res.status(403).json({ error: \"Reject all reviews\" }); return; }"}]}]},"postgres":{"schemaVersion":3,"fixtureSha256":"f3906fa01050f00a0ed76c69c256a6b0013bbf72eb72c03aa92d79e47b71c2b8","backend":"postgres","track":"ecommerce","note":"Mutation definitions for the PostgreSQL ecommerce reference.","mutations":[{"id":"recommendation-dismissal-lost-on-restart","scenario":"tracks/ecommerce/scenarios/progression-recommendation-feedback.json","targets":["ecommerce.spec.state-durability.recommendation-feedback-restart.504c"],"desc":"Erase saved recommendation dismissals when the application starts again.","file":"server/src/index.ts","edits":[{"find":" await initializeProgressionSchema(pool);","replace":" await initializeProgressionSchema(pool);\n await pool.query('DELETE FROM recommendation_dismissal');"}]},{"id":"pending-order-item-return-accepted","scenario":"tracks/ecommerce/scenarios/progression-order-return-boundary.json","targets":["ecommerce.returns-pricing.cancellation-and-return.3f"],"desc":"Accept a pending order return and restore its stock before shipment.","file":"server/src/index.ts","edits":[{"find":"if (!['shipped', 'delivered'].includes(orderRow.rows[0].status)) {","replace":"if (!['pending', 'shipped', 'delivered'].includes(orderRow.rows[0].status)) {"}]},{"id":"staff-admin-access-survives-role-removal","scenario":"tracks/ecommerce/scenarios/progression-staff-roles.json","targets":["ecommerce.spec.access-control.staff-role-revocation.621d"],"desc":"Keep administrator access after changing the assigned role back to staff.","file":"server/src/progression.ts","edits":[{"find":"is_admin = ($1 = 'admin')","replace":"is_admin = (is_admin OR $1 = 'admin')"}]},{"id":"shipping-counts-sale-twice","scenario":"tracks/ecommerce/scenarios/progression-shipping-accounting.json","targets":["ecommerce.inventory-operations.shipping-accounting.202e"],"desc":"Shipping succeeds but doubles the completed sale value in authoritative revenue.","file":"server/src/index.ts","edits":[{"find":"`UPDATE orders SET status = 'shipped', shipped_at = now()\n WHERE id = $1 AND status = 'pending' RETURNING account_id`","replace":"`UPDATE orders SET status = 'shipped', shipped_at = now(), total = total * 2\n WHERE id = $1 AND status = 'pending' RETURNING account_id`"}]},{"id":"signup-ui-does-not-enter-created-account","scenario":"tracks/ecommerce/scenarios/01-account-create.json","targets":["ecommerce.feature.accounts.accounts.1a"],"desc":"Create the account successfully but discard the returned signed-in identity in the client.","file":"client/src/App.tsx","edits":[{"find":" const r = await api<{ account: Account }>(\"/api/auth/signup\", {\n method: \"POST\",\n body: JSON.stringify({ username, password }),\n });\n setAccount(r.account);","replace":" const r = await api<{ account: Account }>(\"/api/auth/signup\", {\n method: \"POST\",\n body: JSON.stringify({ username, password }),\n });\n void r;\n setAccount(null);"}]},{"id":"duplicate-signup-authenticates-existing-account","scenario":"tracks/ecommerce/scenarios/01-account-duplicate.json","targets":["ecommerce.feature.accounts.accounts.1b"],"desc":"Treat a duplicate signup as a successful session for the pre-existing account.","file":"server/src/index.ts","edits":[{"find":" if (existing.length > 0) {\n res.status(409).json({ error: \"username already taken\" });\n return;\n }","replace":" if (existing.length > 0) {\n const token = newToken();\n await db.insert(session).values({ id: token, accountId: existing[0].id });\n res.cookie(\"sid\", token, { httpOnly: true, sameSite: \"lax\", path: \"/\" });\n res.json({ account: { id: existing[0].id, username, isAdmin: false, isStaff: false } });\n return;\n }"}]},{"id":"password-verification-is-inverted","scenario":"tracks/ecommerce/scenarios/01-account-password.json","targets":["ecommerce.feature.accounts.accounts.1c"],"desc":"Accept a wrong password instead of enforcing password verification.","file":"server/src/index.ts","edits":[{"find":" if (rows.length === 0 || !verifyPassword(password, rows[0].passwordHash)) {","replace":" if (rows.length === 0 || verifyPassword(password, rows[0].passwordHash)) {"}]},{"id":"correct-signin-is-refused","scenario":"tracks/ecommerce/scenarios/01-account-signout.json","targets":["ecommerce.feature.accounts.accounts.1d"],"desc":"Preserve wrong-password refusal but reject an otherwise valid sign-in, preventing a signed-out account from returning.","file":"server/src/index.ts","edits":[{"find":" const acc = rows[0];\n const token = newToken();","replace":" const acc = rows[0];\n if (username === acc.username) {\n res.status(401).json({ error: \"sign in is unavailable\" });\n return;\n }\n const token = newToken();"}]},{"id":"reload-discards-session-identity","scenario":"tracks/ecommerce/scenarios/01-account-reload.json","targets":["ecommerce.spec.state-durability.session-reload.1e"],"desc":"Ignore the authenticated identity returned during initial page hydration.","file":"client/src/App.tsx","edits":[{"find":" setAccount(me.account);","replace":" setAccount(null);"}]},{"id":"purchase-does-not-broadcast-ranking","scenario":"tracks/ecommerce/scenarios/01-core.json","targets":["ecommerce.spec.live-state.ranking.2c"],"desc":"Commit the purchase but omit the catalog broadcast that updates already-open rankings.","file":"server/src/index.ts","edits":[{"find":" lastCatalogJson = json;\n io.emit(\"items:update\", { items: catalog });","replace":" lastCatalogJson = json;\n // mutant: changed catalog state is not broadcast"}]},{"id":"signed-out-purchase-uses-default-account","scenario":"tracks/ecommerce/scenarios/progression-signed-out-purchase.json","targets":["ecommerce.spec.access-control.signed-out-purchase.3a"],"desc":"Expose purchase controls to guests and let the purchase route charge the first stored account when no caller is authenticated.","file":"client/src/App.tsx","edits":[{"find":" const canBuy = !!account && !account.isAdmin && !account.isStaff;","replace":" const canBuy = !account || (!account.isAdmin && !account.isStaff);"},{"file":"server/src/index.ts","find":" \"/api/items/:id/buy\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const itemId = Number(req.params.id);\n const accountId = req.account!.id;","replace":" \"/api/items/:id/buy\",\n asyncHandler(async (req, res) => {\n const itemId = Number(req.params.id);\n const accountId = req.account?.id ?? (await pool.query(`SELECT id FROM account ORDER BY id LIMIT 1`)).rows[0].id;"}]},{"id":"purchase-stock-change-is-not-broadcast--01-buying","scenario":"tracks/ecommerce/scenarios/01-buying.json","targets":["ecommerce.spec.live-state.purchase-stock.3b"],"desc":"Commit purchases without broadcasting their stock changes, breaking live stock and sold-out visibility.","file":"server/src/index.ts","edits":[{"find":" lastCatalogJson = json;\n io.emit(\"items:update\", { items: catalog });","replace":" lastCatalogJson = json;\n // mutant: changed stock is not broadcast"}]},{"id":"purchase-stock-change-is-not-broadcast--stock-limit","scenario":"tracks/ecommerce/scenarios/progression-stock-limit.json","targets":["ecommerce.spec.concurrency-safety.stock-limit.3d"],"desc":"Commit purchases without broadcasting their stock changes, breaking live stock and sold-out visibility.","file":"server/src/index.ts","edits":[{"find":" lastCatalogJson = json;\n io.emit(\"items:update\", { items: catalog });","replace":" lastCatalogJson = json;\n // mutant: changed stock is not broadcast"}]},{"id":"restock-race-records-wrong-order-total","scenario":"tracks/ecommerce/scenarios/01-restock-race.json","targets":["ecommerce.spec.concurrency-safety.restock-race.202a"],"desc":"Purchases preserve stock and visible order counts but record the wrong booked total. Native mixed-race reconciliation must reject them.","file":"server/src/index.ts","edits":[{"find":" [accountId, price]\n );","replace":" [accountId, Number(price) + 1]\n );"}]},{"id":"direct-purchase-order-total-is-offset","scenario":"tracks/ecommerce/scenarios/progression-purchasing.json","targets":["ecommerce.feature.purchasing.purchase-order.3c"],"desc":"Record a direct purchase one dollar above the stored price.","file":"server/src/index.ts","edits":[{"find":" [accountId, price]\n );","replace":" [accountId, Number(price) + 1]\n );"}]},{"id":"reload-hydrates-an-empty-cart","scenario":"tracks/ecommerce/scenarios/01-cart.json","targets":["ecommerce.spec.state-durability.cart-reload.4b"],"desc":"Return an empty cart from both reload hydration paths while preserving later live cart broadcasts.","file":"server/src/index.ts","edits":[{"find":" const state = await buildCartState(req.account!.id);\n res.json(state);","replace":" await buildCartState(req.account!.id);\n res.json({ items: [], total: 0 });"},{"find":" const cartState = await buildCartState(acc.id);\n socket.emit(\"cart:update\", cartState);","replace":" await buildCartState(acc.id);\n socket.emit(\"cart:update\", { items: [], total: 0 });"}]},{"id":"signed-out-visitors-do-not-see-reviews","scenario":"tracks/ecommerce/scenarios/01-review-visibility.json","targets":["ecommerce.feature.reviews.reviews.6a"],"desc":"Hide an item's reviews whenever the viewer is signed out.","file":"client/src/App.tsx","edits":[{"find":" {reviews.length === 0 ? (","replace":" {!account || reviews.length === 0 ? ("}]},{"id":"review-average-update-is-not-broadcast","scenario":"tracks/ecommerce/scenarios/01-review-rating-live.json","targets":["ecommerce.spec.live-state.rating.6c"],"desc":"Return the new average to the submitter but omit the live review update to other viewers.","file":"server/src/index.ts","edits":[{"find":" io.emit(\"review:update\", { itemId, reviews, average });","replace":" // mutant: other open review views do not receive the new average"}]},{"id":"admin-warehouse-view-drops-one-location","scenario":"tracks/ecommerce/scenarios/01-warehouse-admin-staff.json","targets":["ecommerce.feature.warehouse-admin.warehouse-view.7b"],"desc":"Render only 23 of the 24 item-by-warehouse stock locations.","file":"client/src/App.tsx","edits":[{"find":" {admin.locations.map((loc) => {","replace":" {admin.locations.slice(0, 23).map((loc) => {"}]},{"id":"unauthenticated-direct-purchase-uses-default-account","scenario":"tracks/ecommerce/scenarios/01-purchase-session.json","targets":["ecommerce.spec.access-control.purchase-session.101a"],"desc":"Remove purchase authentication and attribute unauthenticated requests to a default account.","file":"server/src/index.ts","edits":[{"find":" \"/api/items/:id/buy\",\n requireAuth,","replace":" \"/api/items/:id/buy\","},{"find":" const itemId = Number(req.params.id);\n const accountId = req.account!.id;\n\n const client = await pool.connect();","replace":" const itemId = Number(req.params.id);\n const accountId = req.account?.id ?? 1;\n\n const client = await pool.connect();"}]},{"id":"direct-purchase-is-attributed-to-previous-account","scenario":"tracks/ecommerce/scenarios/01-purchase-attribution.json","targets":["ecommerce.spec.access-control.purchase-attribution.102a"],"desc":"Create the direct-purchase order for the preceding account id rather than the authenticated caller.","file":"server/src/index.ts","edits":[{"find":" const itemId = Number(req.params.id);\n const accountId = req.account!.id;\n\n const client = await pool.connect();","replace":" const itemId = Number(req.params.id);\n const accountId = req.account!.id - 1;\n\n const client = await pool.connect();"}]},{"id":"direct-purchase-uses-constant-price","scenario":"tracks/ecommerce/scenarios/01-server-price.json","targets":["ecommerce.spec.transactional-integrity.server-price.104a"],"desc":"Create a direct-purchase order at a hard-coded price instead of the current stored price.","file":"server/src/index.ts","edits":[{"find":" [accountId, price]\n );","replace":" [accountId, \"1.00\"]\n );"}]},{"id":"account-state-reload-discards-session","scenario":"tracks/ecommerce/scenarios/progression-account-state-reload.json","targets":["ecommerce.spec.state-durability.account-state-recovery.105a"],"desc":"Discard the authenticated account during reload hydration, making its cart and orders unavailable.","file":"client/src/App.tsx","edits":[{"find":" setAccount(me.account);","replace":" setAccount(null);"}]},{"id":"offline-event-clears-account-state","scenario":"tracks/ecommerce/scenarios/progression-account-state-reconnect.json","targets":["ecommerce.spec.state-durability.account-state-recovery.105b"],"desc":"Treat a temporary offline event as a sign-out and clear the account and cart state.","file":"client/src/App.tsx","edits":[{"find":" useEffect(() => {\n const socket = io({ path: \"/socket.io\" });","replace":" useEffect(() => {\n const clearAccountOffline = () => {\n setAccount(null);\n setCart({ items: [], total: 0 });\n };\n window.addEventListener(\"offline\", clearAccountOffline, { once: true });\n const socket = io({ path: \"/socket.io\" });"}]},{"id":"purchase-does-not-decrement-warehouse-stock","scenario":"tracks/ecommerce/scenarios/progression-books-balance.json","targets":["ecommerce.spec.transactional-integrity.books-balance.107b"],"desc":"Create purchase orders without decrementing their selected warehouse stock row.","file":"server/src/index.ts","edits":[{"find":" UPDATE stock s SET quantity = quantity - 1\n FROM target t","replace":" UPDATE stock s SET quantity = quantity\n FROM target t"}]},{"id":"review-route-skips-purchase-eligibility","scenario":"tracks/ecommerce/scenarios/01-review-eligibility.json","targets":["ecommerce.spec.access-control.review-eligibility.108a"],"desc":"Allow review creation even when the caller has never purchased the item.","file":"server/src/index.ts","edits":[{"find":" if (purchased.rowCount === 0) {","replace":" if (false && purchased.rowCount === 0) {"}]},{"id":"only-shipped-orders-earn-review-eligibility","scenario":"tracks/ecommerce/scenarios/01-review-eligibility.json","targets":["ecommerce.spec.access-control.review-eligibility.108b","ecommerce.spec.access-control.review-eligibility.108a"],"desc":"Incorrectly require an order to be shipped before its buyer may review the item. The same restriction also rejects the required successful buyer control in 108a; it does not independently test non-buyer denial.","file":"server/src/index.ts","edits":[{"find":" WHERE o.account_id = $1 AND oi.item_id = $2 LIMIT 1`,","replace":" WHERE o.account_id = $1 AND oi.item_id = $2 AND o.status = 'shipped' LIMIT 1`,"}]},{"id":"cart-update-accepts-negative-quantity","scenario":"tracks/ecommerce/scenarios/01-cart-boundary.json","targets":["ecommerce.spec.access-control.cart-boundary.109b"],"desc":"Accept a negative cart quantity update and persist it instead of refusing the named action.","file":"server/src/index.ts","edits":[{"find":" if (!Number.isInteger(quantity) || quantity < 1) {","replace":" if (!Number.isInteger(quantity)) {"}]},{"id":"oversell-no-row-lock","scenario":"tracks/ecommerce/scenarios/01-last-unit.json","targets":["ecommerce.spec.concurrency-safety.last-unit.201a","ecommerce.spec.concurrency-safety.last-unit.201b","ecommerce.spec.concurrency-safety.last-unit.201c"],"desc":"Drop the row lock so simultaneous buyers can select the same remaining units.","file":"server/src/index.ts","edits":[{"find":" FOR UPDATE\n LIMIT 1","replace":" LIMIT 1"}]},{"id":"purchase-read-write-loses-concurrent-stock","scenario":"tracks/ecommerce/scenarios/01-restock-race.json","targets":["ecommerce.spec.concurrency-safety.restock-race.202a"],"desc":"Replace atomic stock reservation with an unlocked read and absolute write. A fixed pause widens scheduling overlap; serial purchases and restocks retain their stock effects. Concurrent reservations or restocking can lose updates.","file":"server/src/index.ts","edits":[{"find":" const decrement = await client.query(\n `WITH target AS (\n SELECT item_id, warehouse_id FROM stock\n WHERE item_id = $1 AND quantity > 0\n ORDER BY warehouse_id\n FOR UPDATE\n LIMIT 1\n )\n UPDATE stock s SET quantity = quantity - 1\n FROM target t\n WHERE s.item_id = t.item_id AND s.warehouse_id = t.warehouse_id\n RETURNING s.item_id, s.warehouse_id`,\n [itemId]\n );\n","replace":" const snapshot = await client.query(\n `SELECT item_id, warehouse_id, quantity FROM stock\n WHERE item_id = $1 AND quantity > 0 ORDER BY warehouse_id LIMIT 1`, [itemId]\n );\n // Mutant: widen the unlocked read/write window without changing serial behavior.\n await new Promise(resolve => setTimeout(resolve, 500));\n const decrement = snapshot.rowCount === 0 ? snapshot : await client.query(\n `UPDATE stock SET quantity = $3 WHERE item_id = $1 AND warehouse_id = $2\n RETURNING item_id, warehouse_id`,\n [itemId, snapshot.rows[0].warehouse_id, snapshot.rows[0].quantity - 1]\n );\n"}]},{"id":"external-stock-polling-disabled","scenario":"tracks/ecommerce/scenarios/01-external-live-sync.json","targets":["ecommerce.spec.external-data-sync.external-stock.901a"],"desc":"Stop reconciling direct database changes while the server remains online.","file":"server/src/index.ts","edits":[{"find":" broadcastCatalog().catch((err) => console.error(\"poll broadcast failed\", err));","replace":" // mutant: direct database catalog changes are never reconciled"}]},{"id":"server-restart-does-not-resynchronize-catalog","scenario":"tracks/ecommerce/scenarios/01-external-server-restart-sync.json","targets":["ecommerce.spec.external-data-sync.external-stock.901c"],"desc":"After a server restart, omit both connection-time catalog hydration and periodic authoritative reconciliation.","file":"server/src/index.ts","edits":[{"find":" const catalog = await buildCatalog();\n socket.emit(\"items:update\", { items: catalog });","replace":" // mutant: reconnecting clients retain their pre-restart catalog"},{"find":" broadcastCatalog().catch((err) => console.error(\"poll broadcast failed\", err));","replace":" // mutant: restart recovery does not reconcile authoritative catalog state"}]},{"id":"reconnect-does-not-send-current-catalog","scenario":"tracks/ecommerce/scenarios/01-external-reconnect-sync.json","targets":["ecommerce.spec.external-data-sync.external-stock.901d"],"desc":"Follow catalog changes until the browser goes offline, then ignore updates after restoration.","file":"client/src/App.tsx","edits":[{"find":" const cartObservationRef = useRef(0);\n\n function applyCartResponse","replace":" const cartObservationRef = useRef(0);\n const acceptCatalogUpdates = useRef(true);\n useEffect(() => {\n const stopCatalogRecovery = () => { acceptCatalogUpdates.current = false; };\n window.addEventListener(\"offline\", stopCatalogRecovery);\n return () => window.removeEventListener(\"offline\", stopCatalogRecovery);\n }, []);\n\n function applyCartResponse"},{"find":" socket.on(\"items:update\", (payload: { items: Item[] }) => setItems(payload.items));","replace":" socket.on(\"items:update\", (payload: { items: Item[] }) => {\n if (acceptCatalogUpdates.current) setItems(payload.items);\n });"}]},{"id":"open-review-list-ignores-live-update","scenario":"tracks/ecommerce/scenarios/progression-open-list-live.json","targets":["ecommerce.spec.live-state.open-list.902a"],"desc":"Ignore committed review updates in a detail view that is already open.","file":"client/src/App.tsx","edits":[{"find":" setItemDetail({ reviews: payload.reviews, average: payload.average });","replace":" // mutant: the already-open review list ignores committed updates"}]},{"id":"open-review-list-renders-each-review-twice","scenario":"tracks/ecommerce/scenarios/progression-open-list-live.json","targets":["ecommerce.spec.live-state.open-list.902a"],"desc":"Render every committed review twice in the already-open list.","file":"client/src/App.tsx","edits":[{"find":" reviews.map((r) => (","replace":" [...reviews, ...reviews].map((r) => ("}]},{"id":"cancel-does-not-restore-stock-feature","scenario":"tracks/ecommerce/scenarios/02-order-cancellation-core.json","targets":["ecommerce.returns-pricing.cancellation-and-return.3a"],"desc":"Cancellation commits but restores zero units to each recorded warehouse row.","file":"server/src/index.ts","edits":[{"find":" [l.item_id, l.warehouse_id, l.quantity]\n );\n }\n await refundCredit(client, orderRow.rows[0]);\n await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);","replace":" [l.item_id, l.warehouse_id, 0]\n );\n }\n await refundCredit(client, orderRow.rows[0]);\n await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);"}]},{"id":"cancellation-accounting-loses-stock-restoration","scenario":"tracks/ecommerce/scenarios/02-invariants.json","targets":["ecommerce.returns-pricing.refund-accounting.203a"],"desc":"Cancellation removes revenue and changes order status, but loses the original warehouse stock restoration. The native refund-accounting assertion must detect this.","file":"server/src/index.ts","edits":[{"find":" [l.item_id, l.warehouse_id, l.quantity]\n );\n }\n await refundCredit(client, orderRow.rows[0]);\n await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);","replace":" [l.item_id, l.warehouse_id, 0]\n );\n }\n await refundCredit(client, orderRow.rows[0]);\n await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);"}]},{"id":"cancel-does-not-restore-stock-fresh-client","scenario":"tracks/ecommerce/scenarios/02-self-contained.json","targets":["ecommerce.inventory-operations.stock-conservation.202b","ecommerce.inventory-operations.stock-conservation.202c"],"desc":"Cancellation commits but restores zero units, so a fresh client reads the persisted shortfall.","file":"server/src/index.ts","edits":[{"find":" [l.item_id, l.warehouse_id, l.quantity]\n );\n }\n await refundCredit(client, orderRow.rows[0]);\n await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);","replace":" [l.item_id, l.warehouse_id, 0]\n );\n }\n await refundCredit(client, orderRow.rows[0]);\n await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);"}]},{"id":"cancel-restores-stock-but-keeps-pending-status","scenario":"tracks/ecommerce/scenarios/02-order-cancellation-history.json","targets":["ecommerce.returns-pricing.cancellation-and-return.3b"],"desc":"Cancellation restores allocations but writes pending back to order history.","file":"server/src/index.ts","edits":[{"find":"await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);","replace":"await client.query(`UPDATE orders SET status = 'pending' WHERE id = $1`, [orderId]);"}]},{"id":"operator-authorization-allows-customer-transfer","scenario":"tracks/ecommerce/scenarios/02-strengthened.json","targets":["ecommerce.operations-access.operator-authorization.201a"],"desc":"The transfer route replaces its administrator gate with ordinary authentication.","file":"server/src/index.ts","edits":[{"find":"app.post(\n \"/api/admin/transfer\",\n requireAdmin,","replace":"app.post(\n \"/api/admin/transfer\",\n requireAuth,"}]},{"id":"customer-can-ship-order-direct-1-1","scenario":"tracks/ecommerce/scenarios/02-server-actions.json","targets":["ecommerce.operations-access.operator-authorization.201c"],"desc":"The shipping route replaces its staff gate with ordinary authentication.","file":"server/src/index.ts","edits":[{"find":"app.post(\n \"/api/fulfilment/ship\",\n requireStaff,","replace":"app.post(\n \"/api/fulfilment/ship\",\n requireAuth,"}]},{"id":"customer-can-cancel-foreign-order-1-1","scenario":"tracks/ecommerce/scenarios/02-server-actions.json","targets":["ecommerce.operations-access.order-owner.204a"],"desc":"Cancellation retains authentication and pending-state validation but drops order ownership.","file":"server/src/index.ts","edits":[{"find":"app.post(\n \"/api/orders/:id/cancel\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const orderId = Number(req.params.id);\n const accountId = req.account!.id;\n const client = await pool.connect();\n try {\n await client.query(\"BEGIN\");\n const orderRow = await client.query(`SELECT * FROM orders WHERE id = $1 FOR UPDATE`, [orderId]);\n if (orderRow.rowCount === 0 || orderRow.rows[0].account_id !== accountId) {","replace":"app.post(\n \"/api/orders/:id/cancel\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const orderId = Number(req.params.id);\n const accountId = req.account!.id;\n const client = await pool.connect();\n try {\n await client.query(\"BEGIN\");\n const orderRow = await client.query(`SELECT * FROM orders WHERE id = $1 FOR UPDATE`, [orderId]);\n if (orderRow.rowCount === 0) {"}]},{"id":"queue-depth-lags-one-order","scenario":"tracks/ecommerce/scenarios/02-fulfilment-live.json","targets":["ecommerce.operations-access.fulfilment-queue.1a"],"desc":"The queue renders every order but its visible depth remains one behind.","file":"client/src/App.tsx","edits":[{"find":"<strong data-role=\"queue-depth\">{queue.depth}</strong>","replace":"<strong data-role=\"queue-depth\">{Math.max(0, queue.depth - 1)}</strong>"}]},{"id":"customer-sees-fulfilment-navigation","scenario":"tracks/ecommerce/scenarios/02-features.json","targets":["ecommerce.operations-access.fulfilment-queue.1d"],"desc":"Expose the protected staff area to signed-in customers, including its navigation and content.","file":"client/src/App.tsx","edits":[{"find":" {account && (account.isStaff || account.isAdmin) && (\n <button className=\"btn btn-ghost btn-sm\" data-role=\"staff-link\"","replace":" {account && (\n <button className=\"btn btn-ghost btn-sm\" data-role=\"staff-link\""},{"find":"{fulfilmentOpen && account && (account.isAdmin || account.isStaff) && (","replace":"{fulfilmentOpen && account && ("}]},{"id":"progression-customer-sees-fulfilment-content","scenario":"tracks/ecommerce/scenarios/02-fulfilment-access.json","targets":["ecommerce.spec.access-control.fulfilment-area-boundary.1d"],"desc":"Expose the protected staff area to signed-in customers, including its navigation and content.","file":"client/src/App.tsx","edits":[{"find":" {account && (account.isStaff || account.isAdmin) && (\n <button className=\"btn btn-ghost btn-sm\" data-role=\"staff-link\"","replace":" {account && (\n <button className=\"btn btn-ghost btn-sm\" data-role=\"staff-link\""},{"find":"{fulfilmentOpen && account && (account.isAdmin || account.isStaff) && (","replace":"{fulfilmentOpen && account && ("}]},{"id":"transfer-debits-source-without-crediting-existing-destination","scenario":"tracks/ecommerce/scenarios/02-strengthened.json","targets":["ecommerce.inventory-operations.stock-conservation.202a","ecommerce.inventory-operations.warehouse-transfer.2a","ecommerce.operations-access.operator-authorization.201a"],"desc":"A transfer debits the source row but inserts or increments the destination by zero, violating both directional movement and total conservation while still committing successfully. The missing credit also breaks the authorized administrator transfer control in 201a; it does not independently test unauthorized access.","file":"server/src/index.ts","edits":[{"find":" await client.query(\n `INSERT INTO stock (item_id, warehouse_id, quantity) VALUES ($1, $2, $3)\n ON CONFLICT (item_id, warehouse_id) DO UPDATE SET quantity = stock.quantity + excluded.quantity`,\n [itemId, toWarehouseId, qty]\n );","replace":" await client.query(\n `INSERT INTO stock (item_id, warehouse_id, quantity) VALUES ($1, $2, $3)\n ON CONFLICT (item_id, warehouse_id) DO UPDATE SET quantity = stock.quantity + excluded.quantity`,\n [itemId, toWarehouseId, 0]\n );"}]},{"id":"transfer-warehouse-totals-omit-destination-credit","scenario":"tracks/ecommerce/scenarios/02-transfer-totals.json","targets":["ecommerce.inventory-operations.warehouse-transfer.2b"],"desc":"A transfer debits the source but inserts or increments the destination by zero, so the live warehouse totals do not move in opposite directions.","file":"server/src/index.ts","edits":[{"find":" await client.query(\n `INSERT INTO stock (item_id, warehouse_id, quantity) VALUES ($1, $2, $3)\n ON CONFLICT (item_id, warehouse_id) DO UPDATE SET quantity = stock.quantity + excluded.quantity`,\n [itemId, toWarehouseId, qty]\n );","replace":" await client.query(\n `INSERT INTO stock (item_id, warehouse_id, quantity) VALUES ($1, $2, $3)\n ON CONFLICT (item_id, warehouse_id) DO UPDATE SET quantity = stock.quantity + excluded.quantity`,\n [itemId, toWarehouseId, 0]\n );"}]},{"id":"transfer-overdraft-guard-removed","scenario":"tracks/ecommerce/scenarios/02-transfer-overdraw.json","targets":["ecommerce.inventory-operations.warehouse-transfer.2c"],"desc":"The transaction no longer rejects a source warehouse with insufficient quantity, allowing an overdrawn transfer to commit.","file":"server/src/index.ts","edits":[{"find":" if (available < qty) {\n await client.query(\"ROLLBACK\");\n res.status(409).json({ error: \"warehouse does not have enough stock to transfer\" });\n return;\n }","replace":" // mutant: insufficient source stock is not rejected"}]},{"id":"low-stock-excludes-boundary-ten","scenario":"tracks/ecommerce/scenarios/02-low-stock.json","targets":["ecommerce.inventory-operations.operational-views.5a"],"desc":"The low-stock view uses a strict boundary and omits items with exactly ten units.","file":"server/src/index.ts","edits":[{"find":" .filter((it) => it.stock <= 10)","replace":" .filter((it) => it.stock < 10)"}]},{"id":"category-totals-ignore-pending-purchases","scenario":"tracks/ecommerce/scenarios/02-operational-category-totals.json","targets":["ecommerce.inventory-operations.operational-views.5b"],"desc":"Category totals include only shipped orders, so a newly accepted pending purchase is absent from the live totals.","file":"server/src/index.ts","edits":[{"find":" COALESCE(SUM(CASE WHEN oi.returned = false AND o.status != 'cancelled' THEN oi.quantity ELSE 0 END), 0)::int AS units,\n COALESCE(SUM(CASE WHEN oi.returned = false AND o.status != 'cancelled' THEN oi.quantity * oi.price ELSE 0 END), 0) AS revenue","replace":" COALESCE(SUM(CASE WHEN oi.returned = false AND o.status = 'shipped' THEN oi.quantity ELSE 0 END), 0)::int AS units,\n COALESCE(SUM(CASE WHEN oi.returned = false AND o.status = 'shipped' THEN oi.quantity * oi.price ELSE 0 END), 0) AS revenue"}]},{"id":"recommendations-ignore-pending-purchases","scenario":"tracks/ecommerce/scenarios/02-operational-recommendations.json","targets":["ecommerce.inventory-operations.operational-views.5c"],"desc":"Personal recommendations derive categories only from shipped orders, so a newly accepted pending purchase has no influence.","file":"server/src/index.ts","edits":[{"find":" WHERE o.account_id = $1 AND oi.returned = false AND o.status != 'cancelled'`,","replace":" WHERE o.account_id = $1 AND oi.returned = false AND o.status = 'shipped'`,"}]},{"id":"purchases-do-not-affect-best-sellers","scenario":"tracks/ecommerce/scenarios/02-operational-best-sellers.json","targets":["ecommerce.inventory-operations.operational-views.5d"],"desc":"Rank signed-out recommendations without purchase counts.","file":"server/src/index.ts","edits":[{"find":" WHERE oi.returned = false AND o.status != 'cancelled'\n GROUP BY oi.item_id","replace":" WHERE FALSE\n GROUP BY oi.item_id"}]},{"id":"queue-warehouse-reports-west","scenario":"tracks/ecommerce/scenarios/02-queue-warehouse.json","targets":["ecommerce.operations-access.fulfilment-queue.1b"],"desc":"The queue renders West for the deterministic Desk Lamp allocation even though the order reserved stock in East.","file":"client/src/App.tsx","edits":[{"find":"{line.warehouse}","replace":"{\"West\"}"}]},{"id":"transfer-overwrites-concurrent-purchase-with-stale-stock","scenario":"tracks/ecommerce/scenarios/02-server-actions.json","targets":["ecommerce.inventory-operations.stock-conservation.202d"],"desc":"The Headphones transfer reads East stock before taking a row lock, signals the concurrent purchase to proceed, waits for that purchase to commit, then writes its stale source result and credits West. Both actions return accepted, but the stale transfer write loses the purchase's decrement, proving the exact-total check detects broken cross-operation synchronization.","file":"server/src/index.ts","edits":[{"find":"const PORT = Number(process.env.PORT) || 6301;\n\nconst app = express();","replace":"const PORT = Number(process.env.PORT) || 6301;\n\nlet mutationTransferCapturedResolve!: () => void;\nconst mutationTransferCaptured = new Promise<void>((resolve) => {\n mutationTransferCapturedResolve = resolve;\n});\nlet mutationBuyCommittedResolve!: () => void;\nconst mutationBuyCommitted = new Promise<void>((resolve) => {\n mutationBuyCommittedResolve = resolve;\n});\n\nconst app = express();"},{"find":" const itemId = Number(req.params.id);\n const accountId = req.account!.id;\n\n const client = await pool.connect();","replace":" const itemId = Number(req.params.id);\n const accountId = req.account!.id;\n const mutationBuyItem = await pool.query(`SELECT name FROM item WHERE id = $1`, [itemId]);\n if (mutationBuyItem.rows[0]?.name === \"Headphones\") await mutationTransferCaptured;\n\n const client = await pool.connect();"},{"find":" await client.query(\n `INSERT INTO order_item (order_id, item_id, item_name, quantity, price, warehouse_id) VALUES ($1, $2, $3, 1, $4, $5)`,\n [orderId, itemId, name, price, warehouseId]\n );\n await client.query(\"COMMIT\");","replace":" await client.query(\n `INSERT INTO order_item (order_id, item_id, item_name, quantity, price, warehouse_id) VALUES ($1, $2, $3, 1, $4, $5)`,\n [orderId, itemId, name, price, warehouseId]\n );\n await client.query(\"COMMIT\");\n if (name === \"Headphones\") mutationBuyCommittedResolve();"},{"find":" if (\n !Number.isInteger(itemId) ||\n !Number.isInteger(fromWarehouseId) ||\n !Number.isInteger(toWarehouseId) ||\n !Number.isInteger(qty) ||\n qty < 1 ||\n fromWarehouseId === toWarehouseId\n ) {\n res.status(400).json({ error: \"invalid transfer request\" });\n return;\n }\n const client = await pool.connect();","replace":" if (\n !Number.isInteger(itemId) ||\n !Number.isInteger(fromWarehouseId) ||\n !Number.isInteger(toWarehouseId) ||\n !Number.isInteger(qty) ||\n qty < 1 ||\n fromWarehouseId === toWarehouseId\n ) {\n res.status(400).json({ error: \"invalid transfer request\" });\n return;\n }\n const mutationTransferItem = await pool.query(`SELECT name FROM item WHERE id = $1`, [itemId]);\n const mutationStaleSource = mutationTransferItem.rows[0]?.name === \"Headphones\"\n ? await pool.query(\n `SELECT quantity FROM stock WHERE item_id = $1 AND warehouse_id = $2`,\n [itemId, fromWarehouseId]\n )\n : null;\n const mutationStaleSourceQuantity = mutationStaleSource?.rows[0]?.quantity ?? null;\n if (mutationStaleSourceQuantity != null) {\n mutationTransferCapturedResolve();\n await mutationBuyCommitted;\n }\n const client = await pool.connect();"},{"find":" await client.query(`UPDATE stock SET quantity = quantity - $1 WHERE item_id = $2 AND warehouse_id = $3`, [\n qty,\n itemId,\n fromWarehouseId,\n ]);","replace":" if (mutationStaleSourceQuantity != null) {\n await client.query(\n `UPDATE stock SET quantity = $1 WHERE item_id = $2 AND warehouse_id = $3`,\n [mutationStaleSourceQuantity - qty, itemId, fromWarehouseId]\n );\n } else {\n await client.query(`UPDATE stock SET quantity = quantity - $1 WHERE item_id = $2 AND warehouse_id = $3`, [\n qty,\n itemId,\n fromWarehouseId,\n ]);\n }"}]},{"id":"progression-profile-address-is-discarded","scenario":"tracks/ecommerce/scenarios/progression-customer-profile.json","targets":["ecommerce.spec.state-durability.customer-profile-reload.620a"],"desc":"Clear stored profile addresses on server initialization, after ordinary saves work. Reload retains the profile; a runtime restart loses it.","file":"server/src/progression.ts","edits":[{"find":" ALTER TABLE account ADD COLUMN IF NOT EXISTS profile_address text NOT NULL DEFAULT '';","replace":" ALTER TABLE account ADD COLUMN IF NOT EXISTS profile_address text NOT NULL DEFAULT '';\n UPDATE account SET profile_address = '';"}]},{"id":"progression-profile-reads-another-account","scenario":"tracks/ecommerce/scenarios/progression-customer-profile.json","targets":["ecommerce.spec.access-control.customer-profile-privacy.620b"],"desc":"Return another account's saved profile.","file":"server/src/progression.ts","edits":[{"find":"account ? one(`SELECT profile_name, profile_address FROM account WHERE id = $1`, [account.id]) : null,","replace":"account ? one(`SELECT profile_name, profile_address FROM account WHERE profile_address != '' ORDER BY id DESC LIMIT 1`) : null,"}]},{"id":"progression-staff-tools-are-hidden","scenario":"tracks/ecommerce/scenarios/progression-staff-access.json","targets":["ecommerce.progression.staff-access.staff-access.601a"],"desc":"Deny administrators entry to staff tools while keeping ordinary staff access working.","file":"client/src/App.tsx","edits":[{"find":"{account && (account.isStaff || account.isAdmin) && (\n <button className=\"btn btn-ghost btn-sm\" data-role=\"staff-link\"","replace":"{account && (account.isStaff && !account.isAdmin) && (\n <button className=\"btn btn-ghost btn-sm\" data-role=\"staff-link\""}]},{"id":"progression-customer-sees-staff-tools","scenario":"tracks/ecommerce/scenarios/progression-staff-access.json","targets":["ecommerce.spec.access-control.staff-area-boundary.601b"],"desc":"Expose the protected staff area to signed-in customers, including its navigation and content.","file":"client/src/App.tsx","edits":[{"find":"{account && (account.isStaff || account.isAdmin) && (\n <button className=\"btn btn-ghost btn-sm\" data-role=\"staff-link\"","replace":"{account && (\n <button className=\"btn btn-ghost btn-sm\" data-role=\"staff-link\""},{"find":"{fulfilmentOpen && account && (account.isAdmin || account.isStaff) && (","replace":"{fulfilmentOpen && account && ("}]},{"id":"progression-staff-role-is-lost-on-restart","scenario":"tracks/ecommerce/scenarios/progression-staff-roles.json","targets":["ecommerce.spec.state-durability.staff-role-reload.621a"],"desc":"Clear stored staff roles on server initialization while preserving authorized assignment and staff access refusal.","file":"server/src/progression.ts","edits":[{"find":" ALTER TABLE account ADD COLUMN IF NOT EXISTS staff_role text NOT NULL DEFAULT '';","replace":" ALTER TABLE account ADD COLUMN IF NOT EXISTS staff_role text NOT NULL DEFAULT '';\n UPDATE account SET staff_role = '';"}]},{"id":"staff-role-write-precedes-denial","scenario":"tracks/ecommerce/scenarios/progression-staff-roles.json","targets":["ecommerce.spec.access-control.staff-role-boundary.621b","ecommerce.spec.access-control.staff-role-revocation.621d"],"desc":"Save an unauthorized role change before returning a forbidden response, including after administrator access is revoked.","file":"server/src/progression.ts","edits":[{"find":"app.put(\"/api/staff/:id/role\", dependencies.requireAdmin,","replace":"app.put(\"/api/staff/:id/role\", dependencies.requireStaff,"},{"find":" await recordActivity(deps.pool, req.account!.id, \"assign role\", updated.rows[0].username);","replace":" if (!req.account!.isAdmin) { res.status(403).json({ error: \"admin only\" }); return; }\n await recordActivity(deps.pool, req.account!.id, \"assign role\", updated.rows[0].username);"}]},{"id":"progression-staff-can-assign-roles","scenario":"tracks/ecommerce/scenarios/progression-staff-roles.json","targets":["ecommerce.spec.access-control.staff-role-boundary.621b","ecommerce.spec.access-control.staff-role-revocation.621d"],"desc":"Allow staff members to assign roles, including after administrator access is revoked.","file":"server/src/progression.ts","edits":[{"find":"app.put(\"/api/staff/:id/role\", dependencies.requireAdmin,","replace":"app.put(\"/api/staff/:id/role\", dependencies.requireStaff,"}]},{"id":"progression-catalog-product-name-is-not-published","scenario":"tracks/ecommerce/scenarios/progression-catalog-management.json","targets":["ecommerce.progression.catalog-management.catalog-management.622a","ecommerce.progression.catalog-management.catalog-management.622b"],"desc":"The public product card omits the submitted name while retaining the product and its variants. The variants assertion also cannot open that same named product card.","file":"client/src/App.tsx","edits":[{"find":" <span className=\"item-name\" data-role=\"item-name\">{item.name}</span>","replace":" <span className=\"item-name\" data-role=\"item-name\">{item.name === \"Travel Mug\" ? \"\" : item.name}</span>"}]},{"id":"progression-catalog-variants-are-discarded","scenario":"tracks/ecommerce/scenarios/progression-catalog-management.json","targets":["ecommerce.progression.catalog-management.catalog-management.622b"],"desc":"Store the submitted product without its named variants.","file":"server/src/progression.ts","edits":[{"find":" [name, category, price.toFixed(2), variants],","replace":" [name, category, price.toFixed(2), []],"}]},{"id":"progression-payment-amount-is-zero","scenario":"tracks/ecommerce/scenarios/progression-core-business.json","targets":["ecommerce.progression.payment-records.payment-records.623a"],"desc":"Record zero as the paid amount.","file":"server/src/progression.ts","edits":[{"find":"[accountId, total, promotionId, discount],","replace":"[accountId, 0, promotionId, discount],"}]},{"id":"progression-payment-record-is-duplicated","scenario":"tracks/ecommerce/scenarios/progression-core-business.json","targets":["ecommerce.spec.transactional-integrity.payment-deduplication.623b"],"desc":"Render a duplicate payment record.","file":"client/src/App.tsx","edits":[{"find":" <div data-role=\"payment-record\">\n <span data-role=\"payment-status\">{o.paymentStatus}</span>","replace":" <div data-role=\"payment-record\">\n <span data-role=\"payment-status\">{o.paymentStatus}</span>\n <span data-role=\"payment-record\"><span data-role=\"payment-status\">{o.paymentStatus}</span></span>"}]},{"id":"progression-support-intake-is-disabled","scenario":"tracks/ecommerce/scenarios/progression-support-intake.json","targets":["ecommerce.progression.support-intake.support-intake.610a"],"desc":"Do not accept new support cases.","file":"server/src/progression.ts","edits":[{"find":"app.post(\"/api/support/cases\",","replace":"app.post(\"/api/mutation-disabled-support/cases\","}]},{"id":"progression-support-triage-update-is-disabled","scenario":"tracks/ecommerce/scenarios/progression-support-triage.json","targets":["ecommerce.progression.support-triage.support-assignment.611a","ecommerce.progression.support-triage.support-priority.611b","ecommerce.progression.support-triage.support-status.611c"],"desc":"Do not persist support triage updates.","file":"server/src/progression.ts","edits":[{"find":"app.put(\"/api/support/cases/:id\",","replace":"app.put(\"/api/mutation-disabled-support/cases/:id\","}]},{"id":"progression-support-history-is-not-persisted","scenario":"tracks/ecommerce/scenarios/progression-support-history.json","targets":["ecommerce.spec.state-durability.support-history-reload.612a"],"desc":"Delete stored support cases on server initialization, preserving case creation, initial history, and reload before a runtime restart.","file":"server/src/progression.ts","edits":[{"find":" UNIQUE (order_id)\n );\n `);","replace":" UNIQUE (order_id)\n );\n DELETE FROM support_case;\n `);"}]},{"id":"progression-support-history-leaks","scenario":"tracks/ecommerce/scenarios/progression-support-history.json","targets":["ecommerce.spec.access-control.support-history-privacy.612b"],"desc":"Return all support cases to each customer.","file":"server/src/progression.ts","edits":[{"find":"const where = account.isAdmin || account.isStaff ? \"TRUE\" : \"sc.account_id = $1\";\n const values = account.isAdmin || account.isStaff ? [] : [account.id];","replace":"const where = \"TRUE\";\n const values: unknown[] = [];"}]},{"id":"progression-managed-support-is-not-shared","scenario":"tracks/ecommerce/scenarios/progression-managed-support-shared.json","targets":["ecommerce.spec.live-state.managed-support.613a"],"desc":"Retain the previous support snapshot on live socket updates. Explicit refresh still loads committed replies and status.","file":"client/src/App.tsx","edits":[{"find":" socket.on(\"progression:update\", setProgression);","replace":" socket.on(\"progression:update\", (state: any) => setProgression((current: any) => ({ ...state, support: current.support })));"}]},{"id":"progression-managed-support-leaks","scenario":"tracks/ecommerce/scenarios/progression-managed-support-privacy.json","targets":["ecommerce.spec.access-control.managed-support-privacy.613b"],"desc":"Return all managed support cases to each customer.","file":"server/src/progression.ts","edits":[{"find":"const where = account.isAdmin || account.isStaff ? \"TRUE\" : \"sc.account_id = $1\";\n const values = account.isAdmin || account.isStaff ? [] : [account.id];","replace":"const where = \"TRUE\";\n const values: unknown[] = [];"}]},{"id":"progression-promotion-discount-is-offset","scenario":"tracks/ecommerce/scenarios/progression-promotion-rules.json","targets":["ecommerce.progression.promotion-rules.promotion-rule-values.620a"],"desc":"Save a promotion with an incorrect discount while preserving the route, other fields, and staff-only authorization.","file":"server/src/progression.ts","edits":[{"find":" [code, discount, start, end, limit, req.account!.id],","replace":" [code, discount + 1, start, end, limit, req.account!.id],"}]},{"id":"progression-customer-can-create-promotions","scenario":"tracks/ecommerce/scenarios/progression-promotion-rules.json","targets":["ecommerce.spec.access-control.promotion-management-boundary.620b"],"desc":"Allow customers to create promotions.","file":"server/src/progression.ts","edits":[{"find":"app.post(\"/api/promotions\", dependencies.requireStaff,","replace":"app.post(\"/api/promotions\", dependencies.requireAuth,"}]},{"id":"progression-active-promotion-cannot-apply","scenario":"tracks/ecommerce/scenarios/progression-promotion-checkout.json","targets":["ecommerce.progression.promotion-checkout.promotion-checkout-active.621a"],"desc":"Do not apply active promotions.","file":"server/src/progression.ts","edits":[{"find":"app.post(\"/api/cart/promotion\",","replace":"app.post(\"/api/mutation-disabled-cart/promotion\","}]},{"id":"progression-notification-preferences-do-not-save","scenario":"tracks/ecommerce/scenarios/progression-notification-preferences.json","targets":["ecommerce.spec.state-durability.notification-preferences-reload.630a"],"desc":"Do not save notification preferences.","file":"server/src/progression.ts","edits":[{"find":"app.put(\"/api/notifications/preferences\",","replace":"app.put(\"/api/mutation-disabled-notifications/preferences\","}]},{"id":"progression-notifications-leak-across-accounts","scenario":"tracks/ecommerce/scenarios/progression-notification-preferences.json","targets":["ecommerce.spec.access-control.notification-preferences-privacy.630b"],"desc":"Save one customer's notification choice to every other account.","file":"server/src/progression.ts","edits":[{"find":"await deps.pool.query(`UPDATE account SET notify_order = $1, notify_stock = $2 WHERE id = $3`,\n [order, stock, req.account!.id]);","replace":"await deps.pool.query(`UPDATE account SET notify_order = $1, notify_stock = $2 WHERE id = $3 OR id != $3`,\n [order, stock, req.account!.id]);"}]},{"id":"stock-alert-delivery-is-suppressed","scenario":"tracks/ecommerce/scenarios/progression-stock-alert-delivery.json","targets":["ecommerce.progression.stock-alerts.stock-alert-delivery.631c"],"desc":"A request is accepted but restored stock does not deliver its alert.","file":"server/src/progression.ts","edits":[{"find":"for (const request of requests.rows) {","replace":"for (const request of requests.rows.slice(0, 0)) {"}]},{"id":"stock-alert-is-sent-after-every-restock","scenario":"tracks/ecommerce/scenarios/progression-stock-alerts.json","targets":["ecommerce.spec.transactional-integrity.stock-alert-deduplication.631a"],"desc":"Keep a delivered stock request active and use a new notification key on every restock, so its owner receives duplicate alerts.","file":"server/src/progression.ts","edits":[{"find":"`UPDATE stock_alert_request SET fulfilled = true\n WHERE item_id = $1 AND fulfilled = false RETURNING account_id`","replace":"`UPDATE stock_alert_request SET fulfilled = false\n WHERE item_id = $1 AND fulfilled = false RETURNING account_id`"},{"find":"VALUES ($1, 'stock', $2, $3) ON CONFLICT DO NOTHING","replace":"VALUES ($1, 'stock', $2::text || ':' || nextval('notification_id_seq')::text, $3) ON CONFLICT DO NOTHING"}]},{"id":"progression-stock-alerts-leak-across-accounts","scenario":"tracks/ecommerce/scenarios/progression-stock-alerts.json","targets":["ecommerce.spec.access-control.stock-alert-privacy.631b"],"desc":"Return stock alerts for all accounts.","file":"server/src/progression.ts","edits":[{"find":"account ? deps.pool.query(`SELECT * FROM notification WHERE account_id = $1 ORDER BY created_at DESC`, [account.id]) : { rows: [] },","replace":"account ? deps.pool.query(`SELECT * FROM notification ORDER BY created_at DESC`) : { rows: [] },"}]},{"id":"progression-faceted-filter-ignores-category","scenario":"tracks/ecommerce/scenarios/progression-faceted-filters.json","targets":["ecommerce.progression.faceted-search.faceted-search.401a"],"desc":"Ignore the selected catalog category.","file":"client/src/App.tsx","edits":[{"find":" return items.filter((item) => (!q || item.name.toLowerCase().includes(q))\n && (!categoryFilter || item.category === categoryFilter)","replace":" return items.filter((item) => (!q || item.name.toLowerCase().includes(q))\n && (!categoryFilter || true)"}]},{"id":"active-search-uses-purchase-ranking","scenario":"tracks/ecommerce/scenarios/progression-search-ordering.json","targets":["ecommerce.spec.search-ordering.search-ordering.402b"],"desc":"Keep purchase ranking when search text or filters are active.","file":"client/src/App.tsx","edits":[{"find":"const filtering = Boolean(q || categoryFilter || minimumPrice || maximumPrice || inStockOnly);","replace":"const filtering = false;"}]},{"id":"progression-pagination-always-shows-first-page","scenario":"tracks/ecommerce/scenarios/progression-faceted-pagination.json","targets":["ecommerce.progression.faceted-search.faceted-search.402a"],"desc":"Always show the first catalog page.","file":"client/src/App.tsx","edits":[{"find":".slice(searchPage * CATALOG_PAGE_SIZE,\n searchPage * CATALOG_PAGE_SIZE + CATALOG_PAGE_SIZE);","replace":".slice(0, CATALOG_PAGE_SIZE);"}]},{"id":"progression-expired-promotion-is-accepted","scenario":"tracks/ecommerce/scenarios/progression-promotion-checkout.json","targets":["ecommerce.progression.promotion-checkout.promotion-checkout-expired.621b"],"desc":"Accept an expired promotion code.","file":"server/src/progression.ts","edits":[{"find":"`SELECT * FROM promotion WHERE code = $1 AND start_at <= now() AND end_at >= now()\n AND redemptions < redemption_limit`, [code]","replace":"`SELECT * FROM promotion WHERE code = $1\n AND redemptions < redemption_limit`, [code]"}]},{"id":"progression-exhausted-promotion-is-accepted","scenario":"tracks/ecommerce/scenarios/progression-promotion-checkout.json","targets":["ecommerce.progression.promotion-checkout.promotion-checkout-exhausted.621c"],"desc":"Accept a promotion after its redemption limit.","file":"server/src/progression.ts","edits":[{"find":"`SELECT * FROM promotion WHERE code = $1 AND start_at <= now() AND end_at >= now()\n AND redemptions < redemption_limit`, [code]","replace":"`SELECT * FROM promotion WHERE code = $1 AND start_at <= now() AND end_at >= now()`, [code]"}]},{"id":"progression-restock-countdown-is-fixed","scenario":"tracks/ecommerce/scenarios/03-scheduled-restocks.json","targets":["ecommerce.l3.scheduled-restocks.scheduled-restocks.302a"],"desc":"Show a fixed restock countdown.","file":"client/src/ProgressionPanel.tsx","edits":[{"find":"{Math.max(0, Math.ceil((new Date(item.dueAt).valueOf() - Date.now()) / 1000))}","replace":"{90}"}]},{"id":"progression-due-restock-does-not-run","scenario":"tracks/ecommerce/scenarios/03-scheduled-restock-apply.json","targets":["ecommerce.l3.scheduled-restocks.scheduled-restocks.305a"],"desc":"Do not select due scheduled restocks.","file":"server/src/progression.ts","edits":[{"find":"`SELECT * FROM scheduled_restock WHERE due_at <= now() AND cancelled = false AND applied = false FOR UPDATE`,","replace":"`SELECT * FROM scheduled_restock WHERE false FOR UPDATE`,"}]},{"id":"progression-cancelled-restock-still-runs","scenario":"tracks/ecommerce/scenarios/03-scheduled-restock-cancel.json","targets":["ecommerce.l3.scheduled-restocks.scheduled-restocks.306a"],"desc":"Leave a scheduled restock active when it is cancelled.","file":"server/src/progression.ts","edits":[{"find":"`UPDATE scheduled_restock SET cancelled = true WHERE id = $1 AND applied = false RETURNING item_id`,","replace":"`UPDATE scheduled_restock SET cancelled = false WHERE id = $1 AND applied = false RETURNING item_id`,"}]},{"id":"progression-restart-timer-never-runs","scenario":"tracks/ecommerce/scenarios/03-server-time.json","targets":["ecommerce.l3.server-time.server-time.312a"],"desc":"Do not process work that becomes due after restart.","file":"server/src/progression.ts","edits":[{"find":"`SELECT * FROM scheduled_restock WHERE due_at <= now() AND cancelled = false AND applied = false FOR UPDATE`,","replace":"`SELECT * FROM scheduled_restock WHERE false FOR UPDATE`,"}]},{"id":"progression-closed-browser-reservation-never-expires","scenario":"tracks/ecommerce/scenarios/03-server-time.json","targets":["ecommerce.l3.server-time.server-time.313a"],"desc":"Do not select expired reservations for server processing.","file":"server/src/progression.ts","edits":[{"find":"`SELECT ci.id, ci.item_id, c.account_id\n FROM cart_item ci JOIN cart c ON c.id = ci.cart_id\n WHERE ci.expired = false AND ci.reserved_until IS NOT NULL AND ci.reserved_until <= now()\n FOR UPDATE OF ci`,","replace":"`SELECT ci.id, ci.item_id, c.account_id\n FROM cart_item ci JOIN cart c ON c.id = ci.cart_id\n WHERE false\n FOR UPDATE OF ci`,"}]},{"id":"progression-cart-line-does-not-increment","scenario":"tracks/ecommerce/scenarios/progression-cart-checkout.json","targets":["ecommerce.feature.cart-checkout.cart.4a"],"desc":"Keep the old cart quantity when the same item is added again.","file":"server/src/progression.ts","edits":[{"find":"ON CONFLICT (cart_id, item_id) DO UPDATE SET quantity = CASE WHEN cart_item.expired THEN excluded.quantity ELSE cart_item.quantity + excluded.quantity END,","replace":"ON CONFLICT (cart_id, item_id) DO UPDATE SET quantity = cart_item.quantity,"}]},{"id":"progression-checkout-leaves-cart-lines","scenario":"tracks/ecommerce/scenarios/progression-cart-checkout.json","targets":["ecommerce.feature.cart-checkout.cart.4d"],"desc":"Leave cart lines after checkout.","file":"server/src/progression.ts","edits":[{"find":" await client.query(`DELETE FROM cart_item WHERE cart_id = $1`, [cart.rows[0].id]);","replace":" // Mutation: keep checked-out cart lines."}]},{"id":"progression-cart-update-uses-wrong-room","scenario":"tracks/ecommerce/scenarios/01-cart.json","targets":["ecommerce.spec.live-state.shared-cart.4c"],"desc":"Send cart updates to the wrong account room.","file":"server/src/index.ts","edits":[{"find":" try {\n await reserveCartItem(accountId, itemId, qty);\n } catch (error) {\n res.status(409).json({ error: error instanceof Error ? error.message : \"could not reserve stock\" });\n return;\n }\n const state = await buildCartState(accountId);\n io.to(`account:${accountId}`).emit(\"cart:update\", state);","replace":" try {\n await reserveCartItem(accountId, itemId, qty);\n } catch (error) {\n res.status(409).json({ error: error instanceof Error ? error.message : \"could not reserve stock\" });\n return;\n }\n const state = await buildCartState(accountId);\n io.to(`account:${accountId}:mutation`).emit(\"cart:update\", state);"}]},{"id":"progression-order-history-ignores-owner","scenario":"tracks/ecommerce/scenarios/01-order-ownership.json","targets":["ecommerce.spec.access-control.order-ownership.106a"],"desc":"Return orders without filtering by account.","file":"server/src/index.ts","edits":[{"find":"FROM orders WHERE account_id = $1 ORDER BY created_at DESC`,","replace":"FROM orders WHERE $1 IS NOT NULL ORDER BY created_at DESC`,"}]},{"id":"progression-review-conflict-is-not-updated","scenario":"tracks/ecommerce/scenarios/01-review-uniqueness.json","targets":["ecommerce.spec.transactional-integrity.unique-review.6b"],"desc":"Render every stored review twice.","file":"client/src/App.tsx","edits":[{"find":"reviews={itemDetail.reviews}","replace":"reviews={[...itemDetail.reviews, ...itemDetail.reviews]}"}]},{"id":"progression-revenue-double-counts-orders","scenario":"tracks/ecommerce/scenarios/progression-books-balance.json","targets":["ecommerce.spec.transactional-integrity.books-balance.107a"],"desc":"Double the order contribution to revenue.","file":"server/src/index.ts","edits":[{"find":"COALESCE((SELECT SUM(total - refund_total) FROM orders WHERE status != 'cancelled'), 0)","replace":"COALESCE((SELECT SUM(total - refund_total) * 2 FROM orders WHERE status != 'cancelled'), 0)"}]},{"id":"progression-cancelled-orders-remain-in-revenue","scenario":"tracks/ecommerce/scenarios/02-invariants.json","targets":["ecommerce.returns-pricing.refund-accounting.203a"],"desc":"Include cancelled orders in revenue.","file":"server/src/index.ts","edits":[{"find":"COALESCE((SELECT SUM(total - refund_total) FROM orders WHERE status != 'cancelled'), 0)","replace":"COALESCE((SELECT SUM(total - refund_total) FROM orders), 0)"}]},{"id":"progression-shipping-keeps-order-pending","scenario":"tracks/ecommerce/scenarios/02-fulfilment-ship.json","targets":["ecommerce.operations-access.fulfilment-queue.1c"],"desc":"Acknowledge shipping without changing order status.","file":"server/src/index.ts","edits":[{"find":"`UPDATE orders SET status = 'shipped', shipped_at = now()\n WHERE id = $1 AND status = 'pending' RETURNING account_id`","replace":"`UPDATE orders SET status = 'pending', shipped_at = now()\n WHERE id = $1 AND status = 'pending' RETURNING account_id`"}]},{"id":"progression-concurrent-cart-line-does-not-increment","scenario":"tracks/ecommerce/scenarios/01-duplicate-checkout.json","targets":["ecommerce.spec.concurrency-safety.duplicate-checkout.203a"],"desc":"Render the old cart quantity after concurrent adds.","file":"client/src/App.tsx","edits":[{"find":" value={line.quantity}","replace":" value={1}"}]},{"id":"progression-concurrent-checkout-leaves-cart-lines","scenario":"tracks/ecommerce/scenarios/01-duplicate-checkout.json","targets":["ecommerce.spec.concurrency-safety.duplicate-checkout.203b"],"desc":"Leave cart lines after concurrent checkout.","file":"server/src/progression.ts","edits":[{"find":" await client.query(`DELETE FROM cart_item WHERE cart_id = $1`, [cart.rows[0].id]);","replace":" // Mutation: keep checked-out cart lines."}]},{"id":"progression-catalog-ranking-is-reversed","scenario":"tracks/ecommerce/scenarios/01-catalog-ranking.json","targets":["ecommerce.feature.catalog.catalog-ranking.2b"],"desc":"Reverse the catalog tie-break order.","file":"client/src/App.tsx","edits":[{"find":".sort((a, b) => (filtering ? 0 : b.purchaseCount - a.purchaseCount) || a.name.localeCompare(b.name))","replace":".sort((a, b) => (filtering ? 0 : b.purchaseCount - a.purchaseCount) || b.name.localeCompare(a.name))"}]},{"id":"progression-catalog-search-requires-exact-name","scenario":"tracks/ecommerce/scenarios/01-catalog-search.json","targets":["ecommerce.feature.catalog.catalog-search.2d"],"desc":"Require an exact catalog name in search.","file":"client/src/App.tsx","edits":[{"find":"return items.filter((item) => (!q || item.name.toLowerCase().includes(q))","replace":"return items.filter((item) => (!q || item.name.toLowerCase() === q)"}]},{"id":"progression-catalog-price-is-offset","scenario":"tracks/ecommerce/scenarios/01-catalog-values.json","targets":["ecommerce.feature.catalog.catalog-values.2a"],"desc":"Report catalog prices one unit too high.","file":"server/src/index.ts","edits":[{"find":" price: Number(r.price),\n category: r.category,\n variants: r.variants,","replace":" price: Number(r.price) + 1,\n category: r.category,\n variants: r.variants,"}]},{"id":"progression-staff-sees-admin-navigation","scenario":"tracks/ecommerce/scenarios/01-warehouse-admin-staff.json","targets":["ecommerce.spec.access-control.warehouse-area-boundary.7a"],"desc":"Give non-admin accounts administrator identity and access to the protected inventory read.","file":"server/src/index.ts","edits":[{"find":" return rows[0];\n}\n\napp.use(async","replace":" return { ...rows[0], isAdmin: true };\n}\n\napp.use(async"},{"find":"isAdmin: acc.isAdmin, isStaff: acc.isStaff","replace":"isAdmin: true, isStaff: acc.isStaff"}]},{"id":"progression-staff-can-restock-directly","scenario":"tracks/ecommerce/scenarios/01-admin-write-staff.json","targets":["ecommerce.spec.access-control.warehouse-write-boundary.103b"],"desc":"Allow staff members to use the direct restock route.","file":"server/src/index.ts","edits":[{"find":"app.post(\n \"/api/admin/restock\",\n requireAdmin,","replace":"app.post(\n \"/api/admin/restock\",\n requireStaff,"}]},{"id":"progression-restock-adds-wrong-quantity","scenario":"tracks/ecommerce/scenarios/01-warehouse-stock-live-staff.json","targets":["ecommerce.spec.live-state.warehouse-stock.7c"],"desc":"Add one fewer unit during direct restock.","file":"server/src/index.ts","edits":[{"find":" [itemId, warehouseId, qty]\n );\n await processImmediateRestock(itemId);","replace":" [itemId, warehouseId, qty - 1]\n );\n await processImmediateRestock(itemId);"}]},{"id":"progression-cart-add-uses-another-account","scenario":"tracks/ecommerce/scenarios/01-cart-boundary.json","targets":["ecommerce.spec.access-control.cart-boundary.109a"],"desc":"Look up a cart containing the same item without checking its owner, so another customer adds to that cart.","file":"server/src/index.ts","edits":[{"find":" const accountId = req.account!.id;\n try {\n await reserveCartItem(accountId, itemId, qty);","replace":" const other = await pool.query(`SELECT c.account_id FROM cart c JOIN cart_item ci ON ci.cart_id = c.id WHERE ci.item_id = $1 ORDER BY c.id LIMIT 1`, [itemId]);\n const accountId = other.rows[0]?.account_id ?? req.account!.id;\n try {\n await reserveCartItem(accountId, itemId, qty);"}]},{"id":"progression-reservation-does-not-reduce-stock","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.301a"],"desc":"Record a reservation without reducing warehouse stock.","file":"server/src/progression.ts","edits":[{"find":" `UPDATE stock SET quantity = quantity - $1 WHERE item_id = $2 AND warehouse_id = $3`,\n [take, itemId, row.warehouse_id],","replace":" `UPDATE stock SET quantity = quantity + ($1 * 0) WHERE item_id = $2 AND warehouse_id = $3`,\n [take, itemId, row.warehouse_id],"}]},{"id":"progression-reservation-countdown-is-fixed","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.305a"],"desc":"Show a fixed reservation countdown.","file":"client/src/App.tsx","edits":[{"find":"{Math.max(0, (line.reservationSeconds ?? 0) - elapsed)}","replace":"{90}"}]},{"id":"progression-reservation-checkout-keeps-cart","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.306a"],"desc":"Leave reserved cart lines after checkout.","file":"server/src/progression.ts","edits":[{"find":" await client.query(`DELETE FROM cart_item WHERE cart_id = $1`, [cart.rows[0].id]);","replace":" // Mutation: keep reserved cart lines after checkout."}]},{"id":"progression-reservation-never-expires","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.307a"],"desc":"Do not select expired reservations.","file":"server/src/progression.ts","edits":[{"find":"`SELECT ci.id, ci.item_id, c.account_id\n FROM cart_item ci JOIN cart c ON c.id = ci.cart_id\n WHERE ci.expired = false AND ci.reserved_until IS NOT NULL AND ci.reserved_until <= now()\n FOR UPDATE OF ci`,","replace":"`SELECT ci.id, ci.item_id, c.account_id\n FROM cart_item ci JOIN cart c ON c.id = ci.cart_id\n WHERE false\n FOR UPDATE OF ci`,"}]},{"id":"progression-reservation-does-not-renew","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.308a"],"desc":"Keep the old reservation deadline when quantity changes.","file":"server/src/progression.ts","edits":[{"find":" reserved_until = excluded.reserved_until, expired = false RETURNING id`, [cartId, itemId, quantity]);","replace":" reserved_until = cart_item.reserved_until, expired = false RETURNING id`, [cartId, itemId, quantity]);"}]},{"id":"progression-customers-can-manage-scheduled-work","scenario":"tracks/ecommerce/scenarios/03-deferred-access.json","targets":["ecommerce.l3.deferred-access.scheduled-work-access.317a"],"desc":"Allow customers to schedule and cancel restocks.","file":"server/src/progression.ts","edits":[{"find":"app.post(\"/api/admin/scheduled-restocks\", dependencies.requireStaff,","replace":"app.post(\"/api/admin/scheduled-restocks\", dependencies.requireAuth,"},{"find":"app.delete(\"/api/admin/scheduled-restocks/:id\", dependencies.requireStaff,","replace":"app.delete(\"/api/admin/scheduled-restocks/:id\", dependencies.requireAuth,"}]},{"id":"progression-restock-does-not-survive-restart","scenario":"tracks/ecommerce/scenarios/03-deferred-durability.json","targets":["ecommerce.l3.deferred-durability.restart-survival.311a"],"desc":"Application startup discards pending restocks. Ordinary timer execution remains intact; completed and cancelled work is retained.","file":"server/src/index.ts","edits":[{"find":" await initializeProgressionSchema(pool);","replace":" await initializeProgressionSchema(pool);\n await pool.query(`DELETE FROM scheduled_restock WHERE cancelled = false AND applied = false`);"}]},{"id":"progression-reservation-does-not-survive-restart","scenario":"tracks/ecommerce/scenarios/03-deferred-durability.json","targets":["ecommerce.l3.deferred-durability.restart-survival.314a"],"desc":"Do not process expired reservations after restart.","file":"server/src/progression.ts","edits":[{"find":"`SELECT ci.id, ci.item_id, c.account_id\n FROM cart_item ci JOIN cart c ON c.id = ci.cart_id\n WHERE ci.expired = false AND ci.reserved_until IS NOT NULL AND ci.reserved_until <= now()\n FOR UPDATE OF ci`,","replace":"`SELECT ci.id, ci.item_id, c.account_id\n FROM cart_item ci JOIN cart c ON c.id = ci.cart_id\n WHERE false\n FOR UPDATE OF ci`,"}]},{"id":"progression-restock-can-apply-more-than-once","scenario":"tracks/ecommerce/scenarios/03-deferred-integrity.json","targets":["ecommerce.l3.deferred-integrity.exactly-once.311a"],"desc":"Leave an applied restock eligible for later timer passes.","file":"server/src/progression.ts","edits":[{"find":"await client.query(`UPDATE scheduled_restock SET applied = true WHERE id = $1`, [restock.id]);","replace":"await client.query(`UPDATE scheduled_restock SET applied = false WHERE id = $1`, [restock.id]);"}]},{"id":"progression-expiry-returns-double-stock","scenario":"tracks/ecommerce/scenarios/03-deferred-integrity.json","targets":["ecommerce.l3.deferred-integrity.stock-conservation.313a"],"desc":"Return twice the reserved quantity on expiry.","file":"server/src/progression.ts","edits":[{"find":" `UPDATE stock SET quantity = quantity + $1 WHERE item_id = $2 AND warehouse_id = $3`,\n [allocation.quantity, item.rows[0].item_id, allocation.warehouse_id],","replace":" `UPDATE stock SET quantity = quantity + $1 WHERE item_id = $2 AND warehouse_id = $3`,\n [allocation.quantity * 2, item.rows[0].item_id, allocation.warehouse_id],"}]},{"id":"progression-checkout-returns-reserved-stock","scenario":"tracks/ecommerce/scenarios/03-deferred-integrity.json","targets":["ecommerce.l3.deferred-integrity.stock-conservation.314a"],"desc":"Return reserved stock during checkout after creating the order.","file":"server/src/progression.ts","edits":[{"find":" if (promotionId) {\n await client.query(","replace":" for (const line of lines.rows) await releaseReservation(client, line.id);\n if (promotionId) {\n await client.query("}]},{"id":"restock-overwrites-instead-of-increments","scenario":"tracks/ecommerce/scenarios/01-warehouse-stock-live-staff.json","targets":["ecommerce.spec.live-state.warehouse-stock.7c"],"desc":"Overwrite the selected warehouse quantity with the restock amount instead of adding it.","file":"server/src/index.ts","edits":[{"find":" ON CONFLICT (item_id, warehouse_id) DO UPDATE SET quantity = stock.quantity + excluded.quantity`,\n [itemId, warehouseId, qty]","replace":" ON CONFLICT (item_id, warehouse_id) DO UPDATE SET quantity = excluded.quantity`,\n [itemId, warehouseId, qty]"}]},{"id":"operator-authorization-allows-customer-price-change","scenario":"tracks/ecommerce/scenarios/02-strengthened.json","targets":["ecommerce.operations-access.operator-authorization.201b"],"desc":"The price route replaces its administrator gate with ordinary authentication.","file":"server/src/index.ts","edits":[{"find":"app.post(\n \"/api/admin/price\",\n requireAdmin,","replace":"app.post(\n \"/api/admin/price\",\n requireAuth,"}]},{"id":"operator-authorization-allows-customer-shipping","scenario":"tracks/ecommerce/scenarios/02-self-contained.json","targets":["ecommerce.operations-access.fulfilment-queue.1e"],"desc":"The shipping route replaces its staff gate with ordinary authentication.","file":"server/src/index.ts","edits":[{"find":"app.post(\n \"/api/fulfilment/ship\",\n requireStaff,","replace":"app.post(\n \"/api/fulfilment/ship\",\n requireAuth,"}]},{"id":"catalog-price-change-not-broadcast-to-visitors","scenario":"tracks/ecommerce/scenarios/02-live-price.json","targets":["ecommerce.returns-pricing.price-history.4b"],"desc":"The signed-out search result caches detached rows and ignores later catalog updates.","file":"client/src/App.tsx","edits":[{"find":" }, [items, search, categoryFilter, minimumPrice, maximumPrice, inStockOnly, searchPage]);","replace":" }, [search, categoryFilter, minimumPrice, maximumPrice, inStockOnly, searchPage]);"}]},{"id":"catalog-price-change-not-broadcast-to-open-carts","scenario":"tracks/ecommerce/scenarios/progression-price-cart-checkout.json","targets":["ecommerce.returns-pricing.price-history.4c"],"desc":"The price reaches the catalog but the price route omits its targeted open-cart broadcasts.","file":"server/src/index.ts","edits":[{"find":" for (const row of cartAccounts.rows) {\n await broadcastCart(row.account_id);\n }","replace":" void cartAccounts; // mutant: carts containing this item are not refreshed"}]},{"id":"catalog-price-rewrites-receipts","scenario":"tracks/ecommerce/scenarios/02-paid-price-history.json","targets":["ecommerce.returns-pricing.price-history.4a"],"desc":"Changing a catalog price cascades into saved order lines and recomputes historical order totals.","file":"server/src/index.ts","edits":[{"find":" await pool.query(`UPDATE item SET price = $1 WHERE id = $2`, [priceNum.toFixed(2), itemId]);\n await broadcastCatalog();","replace":" await pool.query(`UPDATE item SET price = $1 WHERE id = $2`, [priceNum.toFixed(2), itemId]);\n await pool.query(`UPDATE order_item SET price = $1 WHERE item_id = $2`, [priceNum.toFixed(2), itemId]);\n await pool.query(\n `UPDATE orders o SET total = totals.total FROM (\n SELECT order_id, SUM(quantity * price) AS total FROM order_item GROUP BY order_id\n ) totals WHERE o.id = totals.order_id`\n );\n await broadcastCatalog();"}]},{"id":"catalog-price-rewrites-earned-revenue","scenario":"tracks/ecommerce/scenarios/02-invariants.json","targets":["ecommerce.returns-pricing.refund-accounting.203b"],"desc":"Changing a catalog price cascades into saved order lines and recomputes historical order totals and revenue.","file":"server/src/index.ts","edits":[{"find":" await pool.query(`UPDATE item SET price = $1 WHERE id = $2`, [priceNum.toFixed(2), itemId]);\n await broadcastCatalog();","replace":" await pool.query(`UPDATE item SET price = $1 WHERE id = $2`, [priceNum.toFixed(2), itemId]);\n await pool.query(`UPDATE order_item SET price = $1 WHERE item_id = $2`, [priceNum.toFixed(2), itemId]);\n await pool.query(\n `UPDATE orders o SET total = totals.total FROM (\n SELECT order_id, SUM(quantity * price) AS total FROM order_item GROUP BY order_id\n ) totals WHERE o.id = totals.order_id`\n );\n await broadcastCatalog();"}]},{"id":"returned-line-marker-omitted","scenario":"tracks/ecommerce/scenarios/02-strengthened.json","targets":["ecommerce.returns-pricing.cancellation-and-return.3c"],"desc":"A returned order line keeps its persisted returned state, restored stock, and adjusted revenue but omits the visible returned marker.","file":"client/src/App.tsx","edits":[{"find":"l.returned ? \" (returned)\" : \"\"","replace":"\"\""}]},{"id":"direct-review-access-is-not-checked","scenario":"tracks/ecommerce/scenarios/progression-review-access.json","targets":["ecommerce.progression.review-access-specifications.review-eligibility-direct.618a"],"desc":"The direct review action accepts a review from a customer who did not buy the item.","file":"server/src/index.ts","edits":[{"find":" if (purchased.rowCount === 0) {","replace":" if (false && purchased.rowCount === 0) {"}]},{"id":"support-history-rows-are-hidden","scenario":"tracks/ecommerce/scenarios/progression-support-history.json","targets":["ecommerce.progression.support-history.support-history.612c","ecommerce.spec.state-durability.support-history-reload.612a","ecommerce.spec.access-control.support-history-privacy.612b","ecommerce.spec.access-control.support-history-logout.612d"],"desc":"Hide submitted support ticket rows while leaving the submission reference available. History, reload, privacy, and logout require the owner to see the ticket first. This breaks those positive observations; it does not create a privacy leak or remove stored data.","file":"client/src/ProgressionPanel.tsx","edits":[{"find":"data-role=\"support-ticket\"","replace":"data-role=\"support-ticket\" hidden"}]},{"id":"authorized-restock-does-not-change-stock","scenario":"tracks/ecommerce/scenarios/01-admin-write-staff.json","targets":["ecommerce.feature.warehouse-admin.admin-write.103a"],"desc":"Accept an administrator restock without changing warehouse stock.","file":"server/src/index.ts","edits":[{"find":" await pool.query(\n `INSERT INTO stock (item_id, warehouse_id, quantity) VALUES ($1, $2, $3)\n ON CONFLICT (item_id, warehouse_id) DO UPDATE SET quantity = stock.quantity + excluded.quantity`,\n [itemId, warehouseId, qty]\n );\n await processImmediateRestock(itemId);\n","replace":" // mutant: accept restock without changing stock\n"}]},{"id":"low-stock-threshold-is-two-units","scenario":"tracks/ecommerce/scenarios/02-low-stock.json","targets":["ecommerce.inventory-operations.operational-views.5e","ecommerce.spec.live-state.inventory-dashboard.5a"],"desc":"The low-stock list only includes items at two units or fewer, so the seeded three-unit item is missing when the dashboard is read. The live-state sibling 5a opens with the same observation and fails as necessary collateral.","file":"server/src/index.ts","edits":[{"find":" const lowStock = items\n .filter((it) => it.stock <= 10)","replace":" const lowStock = items\n .filter((it) => it.stock <= 2)"}]},{"id":"category-totals-render-as-session-deltas","scenario":"tracks/ecommerce/scenarios/02-operational-category-totals.json","targets":["ecommerce.inventory-operations.operational-views.5f"],"desc":"Category units and revenue render as the change since the admin panel was opened rather than the stored totals, so a reload resets them to zero while live updates still move by the purchase amount.","file":"client/src/App.tsx","edits":[{"find":" const [namedRestock, setNamedRestock] = useState({ item: \"\", warehouse: \"\", quantity: \"\" });\n","replace":" const [namedRestock, setNamedRestock] = useState({ item: \"\", warehouse: \"\", quantity: \"\" });\n const openedTotals = useRef<CategoryTotal[] | null>(null);\n if (admin && !openedTotals.current) openedTotals.current = admin.categoryTotals;\n const openedTotal = (category: string) => openedTotals.current?.find((c) => c.category === category);\n"},{"find":" Units: <span data-role=\"category-units\">{c.units}</span>","replace":" Units: <span data-role=\"category-units\">{c.units - (openedTotal(c.category)?.units ?? 0)}</span>"},{"find":" Revenue: <span data-role=\"category-revenue\">{c.revenue.toFixed(2)}</span>","replace":" Revenue: <span data-role=\"category-revenue\">{(c.revenue - (openedTotal(c.category)?.revenue ?? 0)).toFixed(2)}</span>"}]},{"id":"profile-summary-ignores-saved-address","scenario":"tracks/ecommerce/scenarios/progression-customer-profile.json","targets":["ecommerce.progression.customer-profile.customer-profile.620c"],"desc":"The profile summary shows only the address loaded with the account and ignores addresses saved afterwards, so the saved address appears only after a reload.","file":"client/src/ProgressionPanel.tsx","edits":[{"find":" useEffect(() => {\n if (state?.profile) setProfile({ name: state.profile.name, address: state.profile.address });\n }, [state?.profile?.name, state?.profile?.address]);\n","replace":" useEffect(() => {\n if (state?.profile) setProfile({ name: state.profile.name, address: state.profile.address });\n }, [state?.profile?.name, state?.profile?.address]);\n const [loadedAddress, setLoadedAddress] = useState<string | null>(null);\n useEffect(() => {\n if (loadedAddress === null && state?.profile) setLoadedAddress(state.profile.address);\n }, [state?.profile?.address]);\n"},{"find":"<span data-role=\"profile-address-summary\">{state?.profile?.address}</span>","replace":"<span data-role=\"profile-address-summary\">{loadedAddress ?? \"\"}</span>"}]},{"id":"support-first-reply-is-hidden","scenario":"tracks/ecommerce/scenarios/progression-managed-support-shared.json","targets":["ecommerce.progression.managed-support.managed-support.613c"],"desc":"The reply list drops the first reply on every support case, so the staff member's first reply is never shown to the customer.","file":"client/src/ProgressionPanel.tsx","edits":[{"find":" {entry.replies.map((item: any) => <div data-role=\"support-reply-item\" key={item.id}>{item.username}: {item.message}</div>)}","replace":" {entry.replies.slice(1).map((item: any) => <div data-role=\"support-reply-item\" key={item.id}>{item.username}: {item.message}</div>)}"}]},{"id":"notification-sync-flips-saved-toggle","scenario":"tracks/ecommerce/scenarios/progression-notification-preferences.json","targets":["ecommerce.progression.notification-preferences.notification-preferences.630c"],"desc":"Merging saved preferences into the form flips each toggle against its current value instead of replacing it, so a choice shows off right after saving but on after a reload.","file":"client/src/ProgressionPanel.tsx","edits":[{"find":" if (state?.preferences) setPreferences(state.preferences);","replace":" if (state?.preferences) setPreferences({ order: state.preferences.order !== preferences.order, stock: state.preferences.stock !== preferences.stock });"}]},{"id":"staff-role-form-reverts-after-save","scenario":"tracks/ecommerce/scenarios/progression-staff-roles.json","targets":["ecommerce.progression.staff-roles.staff-roles.621c"],"desc":"After a successful save the role select snaps back to the previously loaded role, so the assignment is only visible after a reload.","file":"client/src/ProgressionPanel.tsx","edits":[{"find":" <button data-role=\"staff-role-save\" onClick={() => run(() => request(`/api/staff/${role.id}/role`, \"PUT\", {\n role: (document.getElementById(`role-${role.id}`) as HTMLSelectElement).value,\n }))}>Save</button></div>)}","replace":" <button data-role=\"staff-role-save\" onClick={() => run(async () => {\n const select = document.getElementById(`role-${role.id}`) as HTMLSelectElement;\n await request(`/api/staff/${role.id}/role`, \"PUT\", { role: select.value });\n select.value = role.role;\n })}>Save</button></div>)}"}]},{"id":"purchase-does-not-broadcast-fulfilment-queue","scenario":"tracks/ecommerce/scenarios/02-fulfilment-live.json","targets":["ecommerce.spec.live-state.fulfilment-queue.1a"],"desc":"A direct purchase commits its pending order but omits the fulfilment queue broadcast, so an open staff queue never learns about the new order.","file":"server/src/index.ts","edits":[{"find":" await broadcastCatalog();\n await broadcastOrders(accountId);\n await broadcastFulfilment();\n await broadcastRecommended(accountId);\n res.json({ ok: true });\n })\n);\n\n// ---------- cart routes ----------","replace":" await broadcastCatalog();\n await broadcastOrders(accountId);\n // mutant: the new pending order is not pushed to open fulfilment queues\n await broadcastRecommended(accountId);\n res.json({ ok: true });\n })\n);\n\n// ---------- cart routes ----------"}]},{"id":"admin-state-change-is-not-broadcast","scenario":"tracks/ecommerce/scenarios/02-low-stock.json","targets":["ecommerce.spec.live-state.inventory-dashboard.5a"],"desc":"Changed admin dashboard state is never broadcast to open admin views, so a customer purchase that drops an item to ten units does not re-enter the low-stock list live.","file":"server/src/index.ts","edits":[{"find":" lastAdminJson = adminJson;\n io.to(\"admin\").emit(\"admin:update\", adminState);","replace":" lastAdminJson = adminJson;\n // mutant: changed admin state is not broadcast"}]},{"id":"admin-sockets-do-not-join-admin-room","scenario":"tracks/ecommerce/scenarios/02-operational-category-totals.json","targets":["ecommerce.spec.live-state.sales-dashboard.5b"],"desc":"Admin sockets receive their dashboard state on connection but never join the admin room, so a customer purchase does not update the open category totals live.","file":"server/src/index.ts","edits":[{"find":" if (acc.isAdmin) socket.join(\"admin\");","replace":" // mutant: admin sockets never join the admin room"}]},{"id":"transfer-overdraft-guard-skips-bulk-transfers","scenario":"tracks/ecommerce/scenarios/02-transfer-overdraw.json","targets":["ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c"],"desc":"The insufficient-stock guard is only evaluated for transfers under 1000 units, so a bulk transfer that overdraws the source warehouse commits instead of being refused.","file":"server/src/index.ts","edits":[{"find":" if (available < qty) {","replace":" if (available < qty && qty < 1000) {"}]},{"id":"transfer-does-not-publish-warehouse-totals","scenario":"tracks/ecommerce/scenarios/02-transfer-totals.json","targets":["ecommerce.spec.live-state.stock-transfers.2b"],"desc":"A transfer commits but answers with the pre-transfer admin snapshot and admin state is never broadcast, so the open warehouse totals do not move.","file":"server/src/index.ts","edits":[{"find":" const client = await pool.connect();\n try {\n await client.query(\"BEGIN\");\n // Lock both warehouse rows","replace":" const state = await buildAdminState();\n const client = await pool.connect();\n try {\n await client.query(\"BEGIN\");\n // Lock both warehouse rows"},{"find":" await broadcastCatalog();\n const state = await buildAdminState();\n res.json(state);\n })\n);\n\napp.post(\n \"/api/admin/price\",","replace":" await broadcastCatalog();\n res.json(state);\n })\n);\n\napp.post(\n \"/api/admin/price\","},{"find":" lastAdminJson = adminJson;\n io.to(\"admin\").emit(\"admin:update\", adminState);","replace":" lastAdminJson = adminJson;\n // mutant: changed admin state is not broadcast"}]},{"id":"credit-checkout-ignores-wallet","desc":"A credit checkout pays entirely externally despite available credit.","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.feature.store-credit.store-credit-750.750a"],"file":"server/src/credit.ts","edits":[{"find":" const creditMinor = Math.min(Number(account.rows[0].credit_minor), totalMinor);","replace":" const creditMinor = 0;"}]},{"id":"credit-grant-replay-increments-balance","desc":"Replaying a grant applies its credit to the wallet again.","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.spec.store-credit.production-752.752a"],"file":"server/src/credit.ts","edits":[{"find":" if (!existing.rows.length) {","replace":" if (existing.rows.length) await client.query('UPDATE account SET credit_minor=credit_minor+$1 WHERE id=$2', [amountMinor, accountId]);\n if (!existing.rows.length) {"}]},{"id":"customer-can-grant-credit","desc":"Customer authentication is accepted without staff authorization.","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.spec.store-credit.production-753.753a"],"file":"server/src/credit.ts","edits":[{"find":" app.post('/api/staff/credit', auth, staff, async (req, res) => {","replace":" app.post('/api/staff/credit', auth, async (req, res) => {"}]},{"id":"split-refund-does-not-restore-credit","desc":"The refund is recorded but its original wallet credit is not restored.","scenario":"tracks/ecommerce/scenarios/progression-split-tender-refunds.json","targets":["ecommerce.feature.split-tender-refunds.split-tender-refunds-751.751a"],"file":"server/src/progression.ts","edits":[{"find":" await refundCredit(client, order.rows[0]);","replace":" // mutant: omit wallet restoration"}]},{"id":"split-refund-duplicates-credit","desc":"A refund credits the wallet twice while recording one refund.","scenario":"tracks/ecommerce/scenarios/progression-split-tender-refunds.json","targets":["ecommerce.spec.split-tender-refunds.production-756.756a"],"file":"server/src/credit.ts","edits":[{"find":"[delta, order.account_id]);","replace":"[delta * 2, order.account_id]);"}]},{"id":"subscription-skips-due-purchase","desc":"Due deliveries are recorded as skipped although stock is available.","scenario":"tracks/ecommerce/scenarios/progression-subscriptions.json","targets":["ecommerce.feature.subscriptions.subscriptions-760.760a"],"file":"server/src/subscriptions.ts","edits":[{"find":" if (stock.rows.reduce((sum, row) => sum + row.quantity, 0) >= subscription.quantity) {","replace":" if (false && stock.rows.reduce((sum, row) => sum + row.quantity, 0) >= subscription.quantity) {"}]},{"id":"subscription-allows-foreign-cancellation","desc":"A customer can cancel another customer subscription.","scenario":"tracks/ecommerce/scenarios/progression-subscriptions.json","targets":["ecommerce.spec.subscriptions.production-762.762a"],"file":"server/src/subscriptions.ts","edits":[{"find":" if (!row || row.account_id !== req.account!.id) {","replace":" if (!row) {"}]},{"id":"subscription-pause-is-not-recorded","desc":"Pause acknowledges the request but the subscription remains active.","scenario":"tracks/ecommerce/scenarios/progression-subscriptions.json","targets":["ecommerce.spec.subscriptions.production-763.763a"],"file":"server/src/subscriptions.ts","edits":[{"find":"UPDATE purchase_subscription SET status='paused',paused_at=now() WHERE id=$1","replace":"UPDATE purchase_subscription SET status='active',paused_at=now() WHERE id=$1"}]},{"id":"credit-checkout-retains-purchased-cart","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.spec.store-credit.production-754.754a"],"desc":"The purchased cart remains available instead of being consumed by checkout.","file":"server/src/progression.ts","edits":[{"find":" await client.query(`DELETE FROM cart_item WHERE cart_id = $1`, [cart.rows[0].id]);","replace":" // Mutation: keep checked-out cart lines."}]},{"id":"pending-subscriptions-are-cleared-at-startup","desc":"Restart erases pending subscription work while preserving ordinary timer execution.","file":"server/src/subscriptions.ts","scenario":"tracks/ecommerce/scenarios/progression-subscriptions.json","targets":["ecommerce.spec.subscriptions.production-761.761a"],"edits":[{"find":" `);\n}\n\nexport function registerSubscriptions","replace":" `);\n await pool.query(\"UPDATE purchase_subscription SET status='cancelled' WHERE status='active'\");\n}\n\nexport function registerSubscriptions"}]},{"id":"credit-balance-is-cleared-at-startup","desc":"Restart clears an issued wallet balance while leaving accounts present.","file":"server/src/credit.ts","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.spec.store-credit.production-755.755a"],"edits":[{"find":" `);\n}\n\nexport async function spendCredit","replace":" `);\n await pool.query(\"UPDATE account SET credit_minor=0\");\n}\n\nexport async function spendCredit"}]},{"id":"bundle-definition-loses-component-quantity","scenario":"tracks/ecommerce/scenarios/progression-product-bundles.json","targets":["ecommerce.feature.product-bundles.product-bundles.740a"],"desc":"definition loses component quantity","file":"server/src/bundles.ts","edits":[{"find":"values.push({ ...component, itemId: item.rows[0].id });","replace":"values.push({ ...component, quantity: 1, itemId: item.rows[0].id });"}]},{"id":"bundle-catalog-write-allows-customers","scenario":"tracks/ecommerce/scenarios/progression-product-bundles.json","targets":["ecommerce.spec.bundle-integrity.bundle-743.743a"],"desc":"catalog write allows customers","file":"server/src/bundles.ts","edits":[{"find":"if (!actor?.is_admin && actor?.staff_role !== 'catalog')","replace":"if (false)"}]},{"id":"bundle-checkout-price-not-snapshot","scenario":"tracks/ecommerce/scenarios/progression-bundle-checkout.json","targets":["ecommerce.feature.bundle-checkout.bundle-checkout.741a"],"desc":"checkout price not snapshot","file":"server/src/bundles.ts","edits":[{"find":"[cart.rows[0].id, bundleId, bundle.price, JSON.stringify(allocations), JSON.stringify(bundle.bundle_components)]","replace":"[cart.rows[0].id, bundleId, bundle.price + 1, JSON.stringify(allocations), JSON.stringify(bundle.bundle_components)]"}]},{"id":"bundle-expiry-does-not-release-components","scenario":"tracks/ecommerce/scenarios/progression-bundle-checkout.json","targets":["ecommerce.spec.bundle-integrity.bundle-746.746a"],"desc":"expiry does not release components","file":"server/src/progression.ts","edits":[{"find":"await releaseBundle(client, bundle.rows[0].component_allocations);","replace":"/* mutant: component holds leak after expiration */"}]},{"id":"bundle-return-loses-original-components","scenario":"tracks/ecommerce/scenarios/progression-bundle-returns.json","targets":["ecommerce.feature.bundle-returns.bundle-returns.742a"],"desc":"return loses original components","file":"server/src/bundles.ts","edits":[{"find":"await releaseBundle(client, line.component_allocations);","replace":"/* mutant: purchased components are not restored */"}]},{"id":"bundle-return-replay-restocks-again","scenario":"tracks/ecommerce/scenarios/progression-bundle-returns.json","targets":["ecommerce.spec.bundle-integrity.bundle-742.742b"],"desc":"return replay restocks again","file":"server/src/bundles.ts","edits":[{"find":"WHERE order_id=$1 AND is_bundle AND NOT returned FOR UPDATE","replace":"WHERE order_id=$1 AND is_bundle FOR UPDATE"}]},{"id":"bundle-return-crosses-account-boundary","scenario":"tracks/ecommerce/scenarios/progression-bundle-returns.json","targets":["ecommerce.spec.bundle-integrity.bundle-748.748a"],"desc":"return crosses account boundary","file":"server/src/bundles.ts","edits":[{"find":"WHERE id=$1 AND account_id=$2 AND status IN ('shipped','delivered')","replace":"WHERE id=$1 AND $2::integer=$2::integer AND status IN ('shipped','delivered')"}]},{"id":"bundle-components-can-overdraw","scenario":"tracks/ecommerce/scenarios/progression-bundle-checkout.json","targets":["ecommerce.spec.bundle-integrity.bundle-744.744a","ecommerce.spec.bundle-integrity.bundle-745.745a"],"desc":"components can overdraw","file":"server/src/bundles.ts","edits":[{"find":"if (rows.rows.reduce((sum, row) => sum + row.quantity, 0) < component.quantity) throw new Error('A component is unavailable');","replace":"// mutant: incomplete component reservation is accepted"}]},{"id":"bundle-checkout-reuses-reservation","scenario":"tracks/ecommerce/scenarios/progression-bundle-checkout.json","targets":["ecommerce.spec.bundle-integrity.bundle-747.747a"],"desc":"checkout reuses reservation","file":"server/src/progression.ts","edits":[{"find":"await client.query(`DELETE FROM cart_item WHERE cart_id = $1`, [cart.rows[0].id]);","replace":"// mutant: cart survives checkout"}]},{"id":"return-after-support-refund-is-blocked","scenario":"tracks/ecommerce/scenarios/progression-support-return-interaction.json","targets":["ecommerce.feature.split-tender-refunds.return-refund-interaction.757a"],"desc":"Reject a valid physical return after a financial refund.","file":"server/src/index.ts","edits":[{"find":" if (lineRow.rows[0].returned) {","replace":" if (lineRow.rows[0].returned || Number(orderRow.rows[0].refund_total) > 0) {"}]},{"id":"support-refund-after-return-pays-twice","scenario":"tracks/ecommerce/scenarios/progression-support-return-interaction.json","targets":["ecommerce.feature.split-tender-refunds.return-refund-interaction.757b"],"desc":"Pay the full order again after a physical return.","file":"server/src/progression.ts","edits":[{"find":" await client.query(`UPDATE orders SET refund_total = total, status = CASE","replace":" await client.query(`UPDATE orders SET refund_total = refund_total + total, status = CASE"}]},{"id":"progression-support-history-anonymous-leak","scenario":"tracks/ecommerce/scenarios/progression-support-history.json","targets":["ecommerce.spec.access-control.support-history-logout.612d"],"desc":"Return private support subjects to signed-out visitors while preserving authenticated ownership filtering.","file":"server/src/progression.ts","edits":[{"find":" if (!account) return [];\n const where = account.isAdmin || account.isStaff ? \"TRUE\" : \"sc.account_id = $1\";\n const values = account.isAdmin || account.isStaff ? [] : [account.id];","replace":" const where = !account || account.isAdmin || account.isStaff ? \"TRUE\" : \"sc.account_id = $1\";\n const values = !account || account.isAdmin || account.isStaff ? [] : [account.id];"}]},{"id":"checkout-crash-integrity","scenario":"tracks/ecommerce/scenarios/progression-checkout-crash.json","targets":["ecommerce.spec.state-durability.checkout-crash-integrity.910a"],"desc":"A lifecycle callback changes a prepared or cleared cart to quantity two, leaving neither a legal rollback nor a complete checkout.","file":"server/src/index.ts","edits":[{"find":" await initializeProgressionSchema(pool);","replace":" await initializeProgressionSchema(pool);\n await pool.query(`INSERT INTO cart_item (cart_id, item_id, quantity) SELECT c.id, MIN(oi.item_id), 2 FROM cart c JOIN orders o ON o.account_id=c.account_id JOIN order_item oi ON oi.order_id=o.id GROUP BY c.id ON CONFLICT (cart_id,item_id) DO UPDATE SET quantity=2`);"}]},{"id":"checkout-crash-durability","scenario":"tracks/ecommerce/scenarios/progression-checkout-crash.json","targets":["ecommerce.spec.state-durability.checkout-crash-durability.910b"],"desc":"A lifecycle callback corrupts earlier order totals. The unchanged prepared snapshots and acknowledged baseline must expose the damage.","file":"server/src/index.ts","edits":[{"find":" await initializeProgressionSchema(pool);","replace":" await initializeProgressionSchema(pool);\n await pool.query('UPDATE orders SET total=0');"}]},{"id":"review-script-unsafe-render","scenario":"tracks/ecommerce/scenarios/progression-review-script.json","targets":["ecommerce.progression.review-access-specifications.stored-review-script.9180a"],"desc":"Render stored review text as HTML so it executes in another customer session.","file":"client/src/App.tsx","edits":[{"find":"<div>{r.comment}</div>","replace":"<div dangerouslySetInnerHTML={{ __html: r.comment }} />"}]},{"id":"review-script-reject-all","scenario":"tracks/ecommerce/scenarios/progression-review-script.json","targets":["ecommerce.progression.review-access-specifications.stored-review-script.9180a"],"desc":"Reject all nonempty reviews, including the required benign review; absence of script is insufficient.","file":"server/src/index.ts","edits":[{"find":"const { rating, comment } = req.body ?? {};","replace":"const { rating, comment } = req.body ?? {};\n if (String(comment).length > 0) { res.status(400).json({ error: \"Review rejected\" }); return; }"}]},{"id":"review-owner-trust-username","scenario":"tracks/ecommerce/scenarios/progression-review-access.json","targets":["ecommerce.progression.review-access-specifications.review-eligibility-direct.618a"],"desc":"Trust the submitted buyer username instead of the authenticated caller.","file":"server/src/index.ts","edits":[{"find":"\"/api/items/:id/reviews\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const itemId = Number(req.params.id);\n const accountId = req.account!.id;","replace":"\"/api/items/:id/reviews\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const itemId = Number(req.params.id);\n const accountId = req.body?.username ? (await pool.query(\"SELECT id FROM account WHERE username = $1\", [req.body.username])).rows[0]?.id : req.account!.id;"}]},{"id":"review-owner-deny-after-write","scenario":"tracks/ecommerce/scenarios/progression-review-access.json","targets":["ecommerce.progression.review-access-specifications.review-eligibility-direct.618a"],"desc":"Replace the buyer review under a forged username, then return a refusal.","file":"server/src/index.ts","edits":[{"find":"\"/api/items/:id/reviews\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const itemId = Number(req.params.id);\n const accountId = req.account!.id;","replace":"\"/api/items/:id/reviews\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const itemId = Number(req.params.id);\n const accountId = req.body?.username ? (await pool.query(\"SELECT id FROM account WHERE username = $1\", [req.body.username])).rows[0]?.id : req.account!.id;"},{"find":"const { reviews, average } = await buildItemReviews(itemId);\n io.emit(\"review:update\", { itemId, reviews, average });","replace":"if (req.body?.username) { res.status(403).json({ error: \"Refused after write\" }); return; }\n const { reviews, average } = await buildItemReviews(itemId);\n io.emit(\"review:update\", { itemId, reviews, average });"}]},{"id":"review-owner-reject-all","scenario":"tracks/ecommerce/scenarios/progression-review-access.json","targets":["ecommerce.progression.review-access-specifications.review-eligibility-direct.618a"],"desc":"Reject the legitimate buyer review as well as unauthorized reviews.","file":"server/src/index.ts","edits":[{"find":"const { rating, comment } = req.body ?? {};","replace":"const { rating, comment } = req.body ?? {};\n if (String(comment).length > 0) { res.status(403).json({ error: \"Reject all reviews\" }); return; }"}]}]},"spacetime":{"schemaVersion":3,"fixtureSha256":"7ba1f548e89f461b95b5caf843b0fbcf2b81278428d02869d7f47e1a199eaf6e","backend":"spacetime","track":"ecommerce","note":"Mutation definitions for the SpacetimeDB ecommerce reference.","mutations":[{"id":"recommendation-dismissal-lost-on-reconnect","scenario":"tracks/ecommerce/scenarios/progression-recommendation-feedback.json","targets":["ecommerce.spec.state-durability.recommendation-feedback-restart.504c"],"desc":"Erase saved recommendation dismissals when a browser reconnects. This controls reconnect persistence, not backend restart alone.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"export const onConnect = spacetimedb.clientConnected((_ctx) => {});","replace":"export const onConnect = spacetimedb.clientConnected((ctx) => { for (const row of ctx.db.recommendationDismissal.iter()) ctx.db.recommendationDismissal.id.delete(row.id); });"}]},{"id":"pending-order-item-return-accepted","scenario":"tracks/ecommerce/scenarios/progression-order-return-boundary.json","targets":["ecommerce.returns-pricing.cancellation-and-return.3f"],"desc":"Accept a pending order return and restore its stock before shipment.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"if (!['shipped', 'delivered'].includes(order.status)) throw new SenderError('Order has not shipped yet.');","replace":"if (!['pending', 'shipped', 'delivered'].includes(order.status)) throw new SenderError('Order has not shipped yet.');"}]},{"id":"staff-admin-access-survives-role-removal","scenario":"tracks/ecommerce/scenarios/progression-staff-roles.json","targets":["ecommerce.spec.access-control.staff-role-revocation.621d"],"desc":"Keep administrator access after changing the assigned role back to staff.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.account.id.update({ ...target, isAdmin: role === 'admin' });","replace":"ctx.db.account.id.update({ ...target, isAdmin: target.isAdmin || role === 'admin' });"}]},{"id":"shipping-counts-sale-twice","scenario":"tracks/ecommerce/scenarios/progression-shipping-accounting.json","targets":["ecommerce.inventory-operations.shipping-accounting.202e"],"desc":"Shipping succeeds but doubles the completed sale value in authoritative revenue.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" ctx.db.customerOrder.id.update({ ...order, status: 'shipped' });","replace":" ctx.db.customerOrder.id.update({ ...order, status: 'shipped', total: order.total * 2 });"}]},{"id":"restock-client-snapshot-overwrites-concurrent-purchases","scenario":"tracks/ecommerce/scenarios/01-restock-race.json","targets":["ecommerce.spec.concurrency-safety.restock-race.202a"],"desc":"Compute absolute restock quantity when the administrator edits the form and store that captured value in the reducer. A fixed 500 ms submission delay widens the stale-write window. Serial purchases and restocks still work; intervening purchases can be overwritten. This is a stale-form lost update, not a race inside an atomic reducer.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.stock.insert({ ...existing, quantity: existing.quantity + quantity });\n } else {\n ctx.db.stock.insert({ item_id: itemId, warehouse_id: warehouseId, quantity });\n }\n for (const alert of","replace":"ctx.db.stock.insert({ ...existing, quantity });\n } else {\n ctx.db.stock.insert({ item_id: itemId, warehouse_id: warehouseId, quantity });\n }\n for (const alert of"},{"file":"client/src/components/AdminPanel.tsx","find":"onChange={(e) => setRestockInputs((v) => ({ ...v, [k]: e.target.value }))}","replace":"onChange={(e) => setRestockInputs((v) => ({ ...v, [k]: String(stockOf(item.id, wh.id) + Number(e.target.value)) }))}"},{"file":"client/src/App.tsx","find":" await conn?.reducers.adminRestock({ itemId, warehouseId, quantity });","replace":" // Mutant: widen the stale form submission window without changing serial behavior.\n await new Promise(resolve => setTimeout(resolve, 500));\n await conn?.reducers.adminRestock({ itemId, warehouseId, quantity });"}]},{"id":"signup-binds-the-new-account-to-the-admin-session","scenario":"tracks/ecommerce/scenarios/01-account-create.json","targets":["ecommerce.feature.accounts.accounts.1a"],"desc":"Create the requested account but bind the new browser session to the administrator, so account creation no longer signs the visitor in as the account it created.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" const acc = ctx.db.account.insert({\n id: 0n,\n username: uname,\n passwordHash: hashPassword(password),\n isAdmin: false,\n isStaff: false,\n });\n\n const existingSession = ctx.db.session.identity.find(ctx.sender);\n if (existingSession) {\n ctx.db.session.identity.update({ ...existingSession, accountId: acc.id });\n } else {\n ctx.db.session.insert({ identity: ctx.sender, accountId: acc.id });\n }","replace":" const acc = ctx.db.account.insert({\n id: 0n,\n username: uname,\n passwordHash: hashPassword(password),\n isAdmin: false,\n isStaff: false,\n });\n\n const signedInAccount = ctx.db.account.username.find('admin') ?? acc;\n const existingSession = ctx.db.session.identity.find(ctx.sender);\n if (existingSession) {\n ctx.db.session.identity.update({ ...existingSession, accountId: signedInAccount.id });\n } else {\n ctx.db.session.insert({ identity: ctx.sender, accountId: signedInAccount.id });\n }"}]},{"id":"duplicate-signup-is-silently-ignored","scenario":"tracks/ecommerce/scenarios/01-account-duplicate.json","targets":["ecommerce.feature.accounts.accounts.1b"],"desc":"Return success for a taken username without creating a session or surfacing the required refusal.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" if (existing) throw new SenderError('That username is already taken.');","replace":" if (existing) return; // mutant: duplicate signup is silently accepted"}]},{"id":"signin-does-not-verify-the-password","scenario":"tracks/ecommerce/scenarios/01-account-password.json","targets":["ecommerce.feature.accounts.accounts.1c"],"desc":"Accept a known username without comparing the supplied password hash.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" if (!acc || acc.passwordHash !== hashPassword(password)) {","replace":" if (!acc) {"}]},{"id":"signout-keeps-the-account-session","scenario":"tracks/ecommerce/scenarios/01-account-signout.json","targets":["ecommerce.feature.accounts.accounts.1d"],"desc":"Leave the current account session in place when the visitor signs out.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" if (existingSession) ctx.db.session.identity.delete(ctx.sender);","replace":" // mutant: sign out keeps the current account session"}]},{"id":"session-token-is-not-persisted-for-reload","scenario":"tracks/ecommerce/scenarios/01-account-reload.json","targets":["ecommerce.spec.state-durability.session-reload.1e"],"desc":"Discard the connection token instead of persisting it, so a reload receives a new identity with no account session.","file":"client/src/App.tsx","edits":[{"find":" if (token) localStorage.setItem('auth_token', token);","replace":" if (token) localStorage.removeItem('auth_token');"}]},{"id":"catalog-seeds-the-wrong-air-purifier-price","scenario":"tracks/ecommerce/scenarios/01-catalog-values.json","targets":["ecommerce.feature.catalog.catalog-values.2a"],"desc":"Seed Air Purifier with an incorrect stored price while leaving the rest of the catalog intact.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" ['Air Purifier', 189.0, 60, 40, 'Home'],","replace":" ['Air Purifier', 999.0, 60, 40, 'Home'],"}]},{"id":"catalog-tie-breaks-in-reverse-alphabetical-order--01-catalog-ranking","scenario":"tracks/ecommerce/scenarios/01-catalog-ranking.json","targets":["ecommerce.feature.catalog.catalog-ranking.2b"],"desc":"Reverse the specified alphabetical tie-breaker. This necessarily breaks both the initial sequence and the post-purchase sequence in the same scenario.","file":"client/src/App.tsx","edits":[{"find":" return a.name.localeCompare(b.name);","replace":" return b.name.localeCompare(a.name);"}]},{"id":"catalog-tie-breaks-in-reverse-alphabetical-order--01-core","scenario":"tracks/ecommerce/scenarios/01-core.json","targets":["ecommerce.spec.live-state.ranking.2c"],"desc":"Reverse the specified alphabetical tie-breaker. This necessarily breaks both the initial sequence and the post-purchase sequence in the same scenario.","file":"client/src/App.tsx","edits":[{"find":" return a.name.localeCompare(b.name);","replace":" return b.name.localeCompare(a.name);"}]},{"id":"purchase-does-not-update-ranking-count","scenario":"tracks/ecommerce/scenarios/01-core.json","targets":["ecommerce.spec.live-state.ranking.2c"],"desc":"Complete the purchase but leave its popularity count unchanged, so open storefronts cannot rank the purchased item first.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" bumpPurchaseCount(ctx, itemId, quantity);","replace":" // mutant: buy-now never advances the ranking count"}]},{"id":"signed-out-purchase-bypasses-account-check","scenario":"tracks/ecommerce/scenarios/progression-signed-out-purchase.json","targets":["ecommerce.spec.access-control.signed-out-purchase.3a"],"desc":"Expose the guest purchase button and accept its purchase as the existing administrator. The stock observation then exercises the broken account boundary; normal signed-in purchases remain unchanged.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"export const buyNow = spacetimedb.reducer({ itemId: t.u64() }, (ctx, { itemId }) => {\n const acc = requireAccount(ctx);","replace":"export const buyNow = spacetimedb.reducer({ itemId: t.u64() }, (ctx, { itemId }) => {\n const acc = getAccountId(ctx) === null ? ctx.db.account.username.find('admin')! : requireAccount(ctx);"},{"file":"client/src/components/ItemCard.tsx","find":" {isSignedIn && (","replace":" {true && ("}]},{"id":"buy-now-creates-orders-without-reserving-stock--01-buying","scenario":"tracks/ecommerce/scenarios/01-buying.json","targets":["ecommerce.spec.live-state.purchase-stock.3b"],"desc":"Create purchase orders without reserving inventory. The same defect necessarily breaks live purchase stock and the sell-out portion of the stock-limit check.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" const allocations = decrementStockTracked(ctx, itemId, quantity);","replace":" const allocations: Array<{ warehouseId: bigint; quantity: number }> = [];"}]},{"id":"buy-now-creates-orders-without-reserving-stock--stock-limit","scenario":"tracks/ecommerce/scenarios/progression-stock-limit.json","targets":["ecommerce.spec.concurrency-safety.stock-limit.3d"],"desc":"Create purchase orders without reserving inventory. The same defect necessarily breaks live purchase stock and the sell-out portion of the stock-limit check.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" const allocations = decrementStockTracked(ctx, itemId, quantity);","replace":" const allocations: Array<{ warehouseId: bigint; quantity: number }> = [];"}]},{"id":"restock-race-records-wrong-order-total","scenario":"tracks/ecommerce/scenarios/01-restock-race.json","targets":["ecommerce.spec.concurrency-safety.restock-race.202a"],"desc":"Purchases preserve stock and visible order counts but record the wrong booked total. Native mixed-race reconciliation must reject them.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" total: Math.round(price * 100) * quantity / 100,\n status: 'pending',","replace":" total: Math.round(price * 100) * quantity / 100 + 1,\n status: 'pending',"}]},{"id":"buy-now-records-the-wrong-order-total","scenario":"tracks/ecommerce/scenarios/progression-purchasing.json","targets":["ecommerce.feature.purchasing.purchase-order.3c"],"desc":"Record a completed buy-now order one dollar above the stored item price.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" total: Math.round(price * 100) * quantity / 100,\n status: 'pending',","replace":" total: Math.round(price * 100) * quantity / 100 + 1,\n status: 'pending',"}]},{"id":"existing-cart-line-does-not-increment-basic-cart","scenario":"tracks/ecommerce/scenarios/progression-cart-checkout.json","targets":["ecommerce.feature.cart-checkout.cart.4a"],"desc":"Write an existing cart line back without incrementing its quantity.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.cartItem.id.update({ ...existing, quantity: existing.quantity + 1 });","replace":"ctx.db.cartItem.id.update({ ...existing, quantity: existing.quantity });"}]},{"id":"cart-is-deleted-when-owner-disconnects","scenario":"tracks/ecommerce/scenarios/01-cart.json","targets":["ecommerce.spec.state-durability.cart-reload.4b"],"desc":"Delete the account cart on transport disconnect. Reload loses stored cart contents even after the same account signs in again; account and session records remain intact.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"export const onDisconnect = spacetimedb.clientDisconnected((_ctx) => {});","replace":"export const onDisconnect = spacetimedb.clientDisconnected((ctx) => {\n const accountId = getAccountId(ctx);\n if (accountId !== null) for (const row of [...ctx.db.cartItem.byAccountItem.filter(accountId)]) ctx.db.cartItem.id.delete(row.id);\n});"}]},{"id":"signin-binds-the-second-client-to-a-different-account","scenario":"tracks/ecommerce/scenarios/01-cart.json","targets":["ecommerce.spec.live-state.shared-cart.4c"],"desc":"Authenticate valid credentials but bind the second connection to the administrator account, so two sessions for one customer do not share the customer's cart.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" const existingSession = ctx.db.session.identity.find(ctx.sender);\n if (existingSession) {\n ctx.db.session.identity.update({ ...existingSession, accountId: acc.id });\n } else {\n ctx.db.session.insert({ identity: ctx.sender, accountId: acc.id });\n }\n }\n);\n\nexport const signOut","replace":" const wrongAccount = ctx.db.account.username.find('admin') ?? acc;\n const existingSession = ctx.db.session.identity.find(ctx.sender);\n if (existingSession) {\n ctx.db.session.identity.update({ ...existingSession, accountId: wrongAccount.id });\n } else {\n ctx.db.session.insert({ identity: ctx.sender, accountId: wrongAccount.id });\n }\n }\n);\n\nexport const signOut"}]},{"id":"checkout-does-not-empty-the-basic-cart","scenario":"tracks/ecommerce/scenarios/progression-cart-checkout.json","targets":["ecommerce.feature.cart-checkout.cart.4d"],"desc":"Leave completed checkout lines in the durable cart.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" for (const line of lines) ctx.db.cartItem.id.delete(line.id);","replace":" // mutant: checked-out cart lines remain"}]},{"id":"new-review-is-accepted-without-being-stored","scenario":"tracks/ecommerce/scenarios/01-review-visibility.json","targets":["ecommerce.feature.reviews.reviews.6a"],"desc":"Accept an eligible new review but omit its durable insert, so neither author nor visitor can see it.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" ctx.db.review.insert({\n id: 0n,\n itemId,\n accountId: acc.id,\n rating,\n comment,\n createdAt: ctx.timestamp,\n });","replace":" // mutant: accepted review is not persisted"}]},{"id":"repeat-review-inserts-a-second-row","scenario":"tracks/ecommerce/scenarios/01-review-uniqueness.json","targets":["ecommerce.spec.transactional-integrity.unique-review.6b"],"desc":"Insert a second review row instead of updating the customer's existing item review.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" ctx.db.review.id.update({ ...existing, rating, comment, createdAt: ctx.timestamp });","replace":" ctx.db.review.insert({ id: 0n, itemId, accountId: acc.id, rating, comment, createdAt: ctx.timestamp });"}]},{"id":"review-average-counts-rows-instead-of-ratings","scenario":"tracks/ecommerce/scenarios/01-review-rating-live.json","targets":["ecommerce.spec.live-state.rating.6c"],"desc":"Compute the live average from a constant per row rather than each stored rating.","file":"client/src/components/ItemDetail.tsx","edits":[{"find":" : reviews.reduce((sum, r) => sum + r.rating, 0) / reviews.length;","replace":" : reviews.reduce((sum) => sum + 1, 0) / reviews.length;"}]},{"id":"every-signed-in-customer-is-treated-as-an-admin","scenario":"tracks/ecommerce/scenarios/01-warehouse-admin-staff.json","targets":["ecommerce.spec.access-control.warehouse-area-boundary.7a"],"desc":"Use account presence instead of the server-provided administrator flag to expose the admin area.","file":"client/src/App.tsx","edits":[{"find":" const isAdmin = currentUser?.isAdmin ?? false;","replace":" const isAdmin = isSignedIn;"}]},{"id":"warehouse-view-omits-west","scenario":"tracks/ecommerce/scenarios/01-warehouse-admin-staff.json","targets":["ecommerce.feature.warehouse-admin.warehouse-view.7b"],"desc":"Filter one real warehouse out of the administrator's inventory view.","file":"client/src/App.tsx","edits":[{"find":" warehouses={warehouses}","replace":" warehouses={warehouses.filter((warehouse) => warehouse.name !== 'West')}"}]},{"id":"guest-purchase-falls-back-to-the-admin-account","scenario":"tracks/ecommerce/scenarios/01-purchase-session.json","targets":["ecommerce.spec.access-control.purchase-session.101a"],"desc":"Allow an unauthenticated direct purchase by attributing missing sessions to the administrator account, while preserving the authorized control path.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"export const buyNow = spacetimedb.reducer({ itemId: t.u64() }, (ctx, { itemId }) => {\n const acc = requireAccount(ctx);","replace":"export const buyNow = spacetimedb.reducer({ itemId: t.u64() }, (ctx, { itemId }) => {\n const accountId = getAccountId(ctx);\n const acc = (accountId === null ? null : ctx.db.account.id.find(accountId))\n ?? ctx.db.account.username.find('admin')!;"}]},{"id":"direct-purchases-are-attributed-to-the-system-account","scenario":"tracks/ecommerce/scenarios/01-purchase-attribution.json","targets":["ecommerce.spec.access-control.purchase-attribution.102a"],"desc":"Store every buy-now order under the administrator instead of the authenticated caller.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" accountId,\n createdAt: ctx.timestamp,\n total: Math.round(price * 100) * quantity / 100,","replace":" accountId: ctx.db.account.username.find('admin')!.id,\n createdAt: ctx.timestamp,\n total: Math.round(price * 100) * quantity / 100,"}]},{"id":"direct-restock-does-not-require-an-admin","scenario":"tracks/ecommerce/scenarios/01-admin-write-staff.json","targets":["ecommerce.spec.access-control.warehouse-write-boundary.103b"],"desc":"Remove the server-side administrator check from the restock reducer.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"export const adminRestock = spacetimedb.reducer(\n { itemId: t.u64(), warehouseId: t.u64(), quantity: t.u32() },\n (ctx, { itemId, warehouseId, quantity }) => {\n requireAdmin(ctx);","replace":"export const adminRestock = spacetimedb.reducer(\n { itemId: t.u64(), warehouseId: t.u64(), quantity: t.u32() },\n (ctx, { itemId, warehouseId, quantity }) => {\n // mutant: no administrator check"}]},{"id":"direct-purchase-ignores-the-stored-price","scenario":"tracks/ecommerce/scenarios/01-server-price.json","targets":["ecommerce.spec.transactional-integrity.server-price.104a"],"desc":"Create the direct purchase order one dollar above the authoritative stored price.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" total: Math.round(price * 100) * quantity / 100,\n status: 'pending',","replace":" total: 1,\n status: 'pending',"}]},{"id":"account-state-token-is-not-restored-after-reload","scenario":"tracks/ecommerce/scenarios/progression-account-state-reload.json","targets":["ecommerce.spec.state-durability.account-state-recovery.105a"],"desc":"Build a reload connection without the durable identity token, losing the account and its data.","file":"client/src/main.tsx","edits":[{"find":".withToken(localStorage.getItem('auth_token') || undefined)","replace":".withToken(undefined)"}]},{"id":"reconnect-discards-the-visible-account-state","scenario":"tracks/ecommerce/scenarios/progression-account-state-reconnect.json","targets":["ecommerce.spec.state-durability.account-state-recovery.105b"],"desc":"Keep normal reload recovery but discard the client account projection when the browser comes back online.","file":"client/src/App.tsx","edits":[{"find":" const currentUser = currentUserRows[0] ?? null;","replace":" const [discardAccountAfterReconnect, setDiscardAccountAfterReconnect] = useState(false);\n useEffect(() => {\n const discardAccount = () => setDiscardAccountAfterReconnect(true);\n window.addEventListener('online', discardAccount);\n return () => window.removeEventListener('online', discardAccount);\n }, []);\n const currentUser = discardAccountAfterReconnect ? null : currentUserRows[0] ?? null;"}]},{"id":"order-views-return-every-customers-orders","scenario":"tracks/ecommerce/scenarios/01-order-ownership.json","targets":["ecommerce.spec.access-control.order-ownership.106a"],"desc":"Remove account filters from both order views, exposing another customer's order and its line items.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" const rows = [...ctx.db.customerOrder.accountId.filter(accountId)];","replace":" const rows = [...ctx.db.customerOrder.iter()];"},{"find":" for (const o of ctx.db.customerOrder.accountId.filter(accountId)) {","replace":" for (const o of ctx.db.customerOrder.iter()) {"}]},{"id":"admin-revenue-double-counts-every-order","scenario":"tracks/ecommerce/scenarios/progression-books-balance.json","targets":["ecommerce.spec.transactional-integrity.books-balance.107a"],"desc":"Count every completed order twice in the administrator revenue projection.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" total += o.total - o.refundedTotal;","replace":" total += (o.total - o.refundedTotal) * 2;"}]},{"id":"purchases-do-not-leave-the-warehouses","scenario":"tracks/ecommerce/scenarios/progression-books-balance.json","targets":["ecommerce.spec.transactional-integrity.books-balance.107b"],"desc":"Create normal orders and revenue while leaving warehouse stock unchanged.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" const allocations = decrementStockTracked(ctx, itemId, quantity);","replace":" const allocations: Array<{ warehouseId: bigint; quantity: number }> = [];"}]},{"id":"review-purchase-eligibility-is-not-checked","scenario":"tracks/ecommerce/scenarios/01-review-eligibility.json","targets":["ecommerce.spec.access-control.review-eligibility.108a"],"desc":"Allow a signed-in customer to review an item with no matching purchase.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" if (!bought) throw new SenderError('You can only review items you have purchased.');","replace":" // mutant: purchase eligibility is not checked"}]},{"id":"eligible-review-is-accepted-without-being-stored","scenario":"tracks/ecommerce/scenarios/01-review-eligibility.json","targets":["ecommerce.spec.access-control.review-eligibility.108b","ecommerce.spec.access-control.review-eligibility.108a"],"desc":"Keep the non-buyer refusal but omit the insert for a buyer's eligible new review. Both eligibility criteria require a successfully stored eligible review as a positive control; this does not establish a non-buyer authorization defect.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" ctx.db.review.insert({\n id: 0n,\n itemId,\n accountId: acc.id,\n rating,\n comment,\n createdAt: ctx.timestamp,\n });","replace":" // mutant: eligible review is acknowledged but not persisted"}]},{"id":"cart-line-lookup-ignores-cart-ownership","scenario":"tracks/ecommerce/scenarios/01-cart-boundary.json","targets":["ecommerce.spec.access-control.cart-boundary.109a"],"desc":"Find an existing cart line by item alone, so the same named add action from another customer increments the owner's line instead of that customer's cart.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"function findCartLine(ctx: Ctx, accountId: bigint, itemId: bigint) {\n for (const row of ctx.db.cartItem.byAccountItem.filter([accountId, itemId])) {\n return row;\n }\n return null;\n}","replace":"function findCartLine(ctx: Ctx, _accountId: bigint, itemId: bigint) {\n for (const row of ctx.db.cartItem.iter()) {\n if (row.itemId === itemId) return row;\n }\n return null;\n}"}]},{"id":"purchase-does-not-reserve-stock-last-unit","scenario":"tracks/ecommerce/scenarios/01-last-unit.json","targets":["ecommerce.spec.concurrency-safety.last-unit.201a","ecommerce.spec.concurrency-safety.last-unit.201b","ecommerce.spec.concurrency-safety.last-unit.201c"],"desc":"Create purchase orders without reserving stock, proving the focused last-unit stock, order-count, and revenue consequences.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" const allocations = decrementStockTracked(ctx, itemId, quantity);","replace":" const allocations: Array<{ warehouseId: bigint; quantity: number }> = [];"}]},{"id":"existing-cart-line-does-not-increment","scenario":"tracks/ecommerce/scenarios/01-duplicate-checkout.json","targets":["ecommerce.spec.concurrency-safety.duplicate-checkout.203a"],"desc":"Render the old cart quantity after concurrent adds.","file":"client/src/components/CartPanel.tsx","edits":[{"find":" value={line.quantity}","replace":" value={1}"}]},{"id":"checkout-does-not-empty-cart","scenario":"tracks/ecommerce/scenarios/01-duplicate-checkout.json","targets":["ecommerce.spec.concurrency-safety.duplicate-checkout.203b"],"desc":"Keep checked-out cart lines so the next serialized checkout creates a duplicate order.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" for (const line of lines) ctx.db.cartItem.id.delete(line.id);","replace":" // mutant: checked-out lines remain in the cart"}]},{"id":"stock-subscription-snapshotted-once","scenario":"tracks/ecommerce/scenarios/01-external-live-sync.json","targets":["ecommerce.spec.external-data-sync.external-stock.901a"],"desc":"Render the first non-empty stock snapshot forever instead of following committed subscription updates.","file":"client/src/App.tsx","edits":[{"find":"import { useEffect, useMemo, useState } from 'react';","replace":"import { useEffect, useMemo, useRef, useState } from 'react';"},{"find":" const [stocks] = useTable(tables.stock);","replace":" const [liveStocks] = useTable(tables.stock);\n const initialStocks = useRef<typeof liveStocks | null>(null);\n if (initialStocks.current === null && liveStocks.length > 0) {\n initialStocks.current = liveStocks;\n }\n const stocks = initialStocks.current ?? liveStocks;"}]},{"id":"stock-view-ignores-update-across-app-server-stop","scenario":"tracks/ecommerce/scenarios/01-external-server-restart-sync.json","targets":["ecommerce.spec.external-data-sync.external-stock.901c"],"desc":"Persist the first stock quantities in browser session storage and keep rendering them after app-server restart, including any frontend reload. Initial stock remains correct. This validates the stale-view oracle, not SpacetimeDB storage durability.","file":"client/src/App.tsx","edits":[{"find":" const [stocks] = useTable(tables.stock);","replace":" const [liveStocks] = useTable(tables.stock);\n const cacheKey = 'stale-stock-quantities';\n let savedQuantities = sessionStorage.getItem(cacheKey);\n if (!savedQuantities && liveStocks.length > 0) {\n savedQuantities = JSON.stringify(Object.fromEntries(liveStocks.map(row => [`${row.itemId}-${row.warehouseId}`, row.quantity])));\n sessionStorage.setItem(cacheKey, savedQuantities);\n }\n const quantities: Record<string, number> = JSON.parse(savedQuantities ?? '{}');\n const stocks = liveStocks.map(row => ({ ...row, quantity: quantities[`${row.itemId}-${row.warehouseId}`] ?? row.quantity }));"}]},{"id":"stock-view-keeps-pre-reconnect-snapshot","scenario":"tracks/ecommerce/scenarios/01-external-reconnect-sync.json","targets":["ecommerce.spec.external-data-sync.external-stock.901d"],"desc":"Continue following stock until the browser goes offline, then retain the last online snapshot after network restoration.","file":"client/src/App.tsx","edits":[{"find":"import { useEffect, useMemo, useState } from 'react';","replace":"import { useEffect, useMemo, useRef, useState } from 'react';"},{"find":" const [stocks] = useTable(tables.stock);","replace":" const [liveStocks] = useTable(tables.stock);\n const [freezeStockAfterOffline, setFreezeStockAfterOffline] = useState(false);\n const lastOnlineStocks = useRef(liveStocks);\n useEffect(() => {\n const freezeStock = () => setFreezeStockAfterOffline(true);\n window.addEventListener('offline', freezeStock);\n return () => window.removeEventListener('offline', freezeStock);\n }, []);\n if (!freezeStockAfterOffline) {\n lastOnlineStocks.current = liveStocks;\n }\n const stocks = freezeStockAfterOffline ? lastOnlineStocks.current : liveStocks;"}]},{"id":"open-review-list-snapshots-on-selection","scenario":"tracks/ecommerce/scenarios/progression-open-list-live.json","targets":["ecommerce.spec.live-state.open-list.902a"],"desc":"Snapshot the selected item's reviews when the detail opens instead of following later subscription updates.","file":"client/src/App.tsx","edits":[{"find":"import { useEffect, useMemo, useState } from 'react';","replace":"import { useEffect, useMemo, useRef, useState } from 'react';"},{"find":" const selectedItemReviews = selectedItemId !== null ? reviewsByItem.get(selectedItemId) ?? [] : [];","replace":" const openedReviewItem = useRef<bigint | null>(null);\n const openedReviews = useRef<(typeof reviews)[number][]>([]);\n if (selectedItemId !== openedReviewItem.current) {\n openedReviewItem.current = selectedItemId;\n openedReviews.current = selectedItemId !== null ? reviewsByItem.get(selectedItemId) ?? [] : [];\n }\n const selectedItemReviews = openedReviews.current;"}]},{"id":"open-review-list-renders-each-review-twice","scenario":"tracks/ecommerce/scenarios/progression-open-list-live.json","targets":["ecommerce.spec.live-state.open-list.902a"],"desc":"Render every committed review twice in the already-open list.","file":"client/src/components/ItemDetail.tsx","edits":[{"find":" {reviews.map((r) => (","replace":" {[...reviews, ...reviews].map((r) => ("}]},{"id":"cancel-does-not-restore-stock-feature","scenario":"tracks/ecommerce/scenarios/02-order-cancellation-core.json","targets":["ecommerce.returns-pricing.cancellation-and-return.3a"],"desc":"The serialized cancellation reducer changes order state and purchase counts but skips allocation restoration.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" restoreOrderItemStock(ctx, li);\n decrementPurchaseCount(ctx, li.itemId, li.quantity);","replace":" // mutant: cancellation does not restore its reserved stock\n decrementPurchaseCount(ctx, li.itemId, li.quantity);"}]},{"id":"cancellation-accounting-loses-stock-restoration","scenario":"tracks/ecommerce/scenarios/02-invariants.json","targets":["ecommerce.returns-pricing.refund-accounting.203a"],"desc":"Cancellation removes revenue and changes order status, but loses the original warehouse stock restoration. The native refund-accounting assertion must detect this.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" restoreOrderItemStock(ctx, li);\n decrementPurchaseCount(ctx, li.itemId, li.quantity);","replace":" // mutant: cancellation does not restore its reserved stock\n decrementPurchaseCount(ctx, li.itemId, li.quantity);"}]},{"id":"cancel-does-not-restore-stock-fresh-client","scenario":"tracks/ecommerce/scenarios/02-self-contained.json","targets":["ecommerce.inventory-operations.stock-conservation.202b","ecommerce.inventory-operations.stock-conservation.202c"],"desc":"The serialized cancellation reducer skips allocation restoration, so a fresh client reads the persisted shortfall.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" restoreOrderItemStock(ctx, li);\n decrementPurchaseCount(ctx, li.itemId, li.quantity);","replace":" // mutant: cancellation does not restore its reserved stock\n decrementPurchaseCount(ctx, li.itemId, li.quantity);"}]},{"id":"cancel-restores-stock-but-keeps-pending-status","scenario":"tracks/ecommerce/scenarios/02-order-cancellation-history.json","targets":["ecommerce.returns-pricing.cancellation-and-return.3b"],"desc":"The serialized cancellation reducer restores allocations but writes pending back to order history.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" ctx.db.customerOrder.id.update({ ...order, status: 'cancelled' });","replace":" ctx.db.customerOrder.id.update({ ...order, status: 'pending' });"}]},{"id":"cancelled-order-remains-in-revenue-feature","scenario":"tracks/ecommerce/scenarios/02-order-cancellation-core.json","targets":["ecommerce.returns-pricing.cancellation-and-return.3a"],"desc":"The admin revenue view includes cancelled orders even though cancellation otherwise succeeds.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" if (!isOrderCounted(o)) continue;\n total += o.total - o.refundedTotal;","replace":" total += o.total - o.refundedTotal;"}]},{"id":"cancelled-order-remains-in-revenue-invariant","scenario":"tracks/ecommerce/scenarios/02-invariants.json","targets":["ecommerce.returns-pricing.refund-accounting.203a"],"desc":"The admin revenue view includes cancelled orders even though cancellation otherwise succeeds.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" if (!isOrderCounted(o)) continue;\n total += o.total - o.refundedTotal;","replace":" total += o.total - o.refundedTotal;"}]},{"id":"operator-authorization-allows-customer-transfer","scenario":"tracks/ecommerce/scenarios/02-strengthened.json","targets":["ecommerce.operations-access.operator-authorization.201a"],"desc":"The transfer reducer drops its administrator role gate.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" (ctx, { itemId, fromWarehouseId, toWarehouseId, quantity }) => {\n requireAdmin(ctx);","replace":" (ctx, { itemId, fromWarehouseId, toWarehouseId, quantity }) => {\n // mutant: no administrator role check"}]},{"id":"customer-can-ship-order-direct-1-1","scenario":"tracks/ecommerce/scenarios/02-server-actions.json","targets":["ecommerce.operations-access.operator-authorization.201c"],"desc":"The shipping reducer drops its staff role check while retaining pending-state validation.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n requireStaffOrAdmin(ctx);","replace":"export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n // mutant: no staff role check"}]},{"id":"customer-can-cancel-foreign-order-1-1","scenario":"tracks/ecommerce/scenarios/02-server-actions.json","targets":["ecommerce.operations-access.order-owner.204a"],"desc":"Cancellation bypasses the owner helper while retaining missing-order and pending-state validation.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"export const cancelOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n const order = requireOrderOwner(ctx, orderId);","replace":"export const cancelOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n const order = ctx.db.customerOrder.id.find(orderId);\n if (!order) throw new SenderError('Order not found.');"}]},{"id":"queue-depth-lags-one-order","scenario":"tracks/ecommerce/scenarios/02-fulfilment-live.json","targets":["ecommerce.operations-access.fulfilment-queue.1a"],"desc":"The reactive queue renders every order but its visible depth remains one behind.","file":"client/src/components/FulfilmentPanel.tsx","edits":[{"find":"Waiting: <span data-role=\"queue-depth\">{queue.length}</span>","replace":"Waiting: <span data-role=\"queue-depth\">{Math.max(0, queue.length - 1)}</span>"}]},{"id":"ship-acknowledges-without-changing-status","scenario":"tracks/ecommerce/scenarios/02-fulfilment-ship.json","targets":["ecommerce.operations-access.fulfilment-queue.1c"],"desc":"The serialized shipping reducer accepts the call but writes pending back to the order.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" ctx.db.customerOrder.id.update({ ...order, status: 'shipped' });","replace":" ctx.db.customerOrder.id.update({ ...order, status: 'pending' });"}]},{"id":"customer-sees-fulfilment-navigation","scenario":"tracks/ecommerce/scenarios/02-features.json","targets":["ecommerce.operations-access.fulfilment-queue.1d"],"desc":"Expose the protected staff area to signed-in customers, including its navigation and content.","file":"client/src/App.tsx","edits":[{"find":" const isStaff = currentUser?.isStaff ?? false;","replace":" const isStaff = isSignedIn;"}]},{"id":"progression-customer-sees-fulfilment-content","scenario":"tracks/ecommerce/scenarios/02-fulfilment-access.json","targets":["ecommerce.spec.access-control.fulfilment-area-boundary.1d"],"desc":"Expose the protected staff area to signed-in customers, including its navigation and content.","file":"client/src/App.tsx","edits":[{"find":" const isStaff = currentUser?.isStaff ?? false;","replace":" const isStaff = isSignedIn;"}]},{"id":"operator-authorization-allows-customer-shipping","scenario":"tracks/ecommerce/scenarios/02-server-actions.json","targets":["ecommerce.operations-access.operator-authorization.201c"],"desc":"The shipping reducer drops the staff role check while retaining the pending-order guard.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n requireStaffOrAdmin(ctx);","replace":"export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n // mutant: no staff role check"}]},{"id":"transfer-debits-source-without-crediting-existing-destination","scenario":"tracks/ecommerce/scenarios/02-strengthened.json","targets":["ecommerce.inventory-operations.stock-conservation.202a","ecommerce.inventory-operations.warehouse-transfer.2a","ecommerce.operations-access.operator-authorization.201a"],"desc":"A transfer debits the source row but writes the existing destination quantity back unchanged, violating both directional movement and total conservation inside the serialized reducer. It also breaks 201a's authorized-transfer positive control; that coupled failure is not independent evidence of an authorization defect.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" ctx.db.stock.insert({ ...toRow, quantity: toRow.quantity + quantity });","replace":" ctx.db.stock.insert({ ...toRow, quantity: toRow.quantity });"}]},{"id":"transfer-warehouse-totals-omit-destination-credit","scenario":"tracks/ecommerce/scenarios/02-transfer-totals.json","targets":["ecommerce.inventory-operations.warehouse-transfer.2b"],"desc":"The serialized transfer debits the source but writes the existing destination quantity back unchanged.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" ctx.db.stock.insert({ ...toRow, quantity: toRow.quantity + quantity });","replace":" ctx.db.stock.insert({ ...toRow, quantity: toRow.quantity });"}]},{"id":"transfer-overdraft-guard-removed","scenario":"tracks/ecommerce/scenarios/02-transfer-overdraw.json","targets":["ecommerce.inventory-operations.warehouse-transfer.2c"],"desc":"The serialized reducer no longer rejects insufficient source stock and commits negative source quantity.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" if (available < quantity) {\n throw new SenderError(`Not enough stock in source warehouse: only ${available} available.`);\n }","replace":" // mutant: insufficient source stock is not rejected"}]},{"id":"low-stock-excludes-boundary-ten","scenario":"tracks/ecommerce/scenarios/02-low-stock.json","targets":["ecommerce.inventory-operations.operational-views.5a"],"desc":"The reactive low-stock view uses a strict boundary and omits items with exactly ten units.","file":"client/src/App.tsx","edits":[{"find":" .filter((i) => (stockByItem.get(i.id) ?? 0) <= LOW_STOCK_THRESHOLD)","replace":" .filter((i) => (stockByItem.get(i.id) ?? 0) < LOW_STOCK_THRESHOLD)"}]},{"id":"category-totals-ignore-pending-purchases","scenario":"tracks/ecommerce/scenarios/02-operational-category-totals.json","targets":["ecommerce.inventory-operations.operational-views.5b"],"desc":"The category totals view includes only shipped orders, so a newly accepted pending purchase is absent.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" for (const order of ctx.db.customerOrder.iter()) {\n if (!isOrderCounted(order)) continue;","replace":" for (const order of ctx.db.customerOrder.iter()) {\n if (order.status !== 'shipped') continue;"}]},{"id":"recommendations-ignore-pending-purchases","scenario":"tracks/ecommerce/scenarios/02-operational-recommendations.json","targets":["ecommerce.inventory-operations.operational-views.5c"],"desc":"The personal recommendation view derives categories only from shipped orders, so a new pending purchase has no influence.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" for (const order of ctx.db.customerOrder.accountId.filter(accountId)) {\n if (!isOrderCounted(order)) continue;","replace":" for (const order of ctx.db.customerOrder.accountId.filter(accountId)) {\n if (order.status !== 'shipped') continue;"}]},{"id":"purchases-do-not-affect-best-sellers","scenario":"tracks/ecommerce/scenarios/02-operational-best-sellers.json","targets":["ecommerce.inventory-operations.operational-views.5d"],"desc":"Rank signed-out recommendations without purchase counts.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"const purchaseCountOf = (id: bigint) => ctx.db.itemStats.itemId.find(id)?.purchaseCount ?? 0;","replace":"const purchaseCountOf = (_id: bigint) => 0;"}]},{"id":"queue-warehouse-reports-west","scenario":"tracks/ecommerce/scenarios/02-queue-warehouse.json","targets":["ecommerce.operations-access.fulfilment-queue.1b"],"desc":"The queue renders West for the deterministic Desk Lamp allocation even though the order reserved stock in East.","file":"client/src/components/FulfilmentPanel.tsx","edits":[{"find":"{name}: {order.warehouseNames[i]}","replace":"{name}: West"}]},{"id":"transfer-creates-stock-during-race","scenario":"tracks/ecommerce/scenarios/02-server-actions.json","targets":["ecommerce.inventory-operations.stock-conservation.202d"],"desc":"The transfer reducer credits the destination one unit more than it debits from the source, so the item's total after a transfer racing a purchase is the starting total rather than one less. A stored conservation defect; it does not model a lost-update interleaving because SpacetimeDB reducer execution is atomically serialized.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" ctx.db.stock.insert({ ...toRow, quantity: toRow.quantity + quantity });","replace":" ctx.db.stock.insert({ ...toRow, quantity: toRow.quantity + quantity + 1 });"}]},{"id":"catalog-search-ignores-the-query","scenario":"tracks/ecommerce/scenarios/01-catalog-search.json","targets":["ecommerce.feature.catalog.catalog-search.2d"],"desc":"A non-empty catalog query filters out every product instead of matching names.","file":"client/src/App.tsx","edits":[{"find":".filter(item => !q || item.name.toLowerCase().includes(q))","replace":".filter(() => !q) // mutant: non-empty searches return no products"}]},{"id":"admin-total-stock-is-not-rendered","scenario":"tracks/ecommerce/scenarios/01-warehouse-stock-live-staff.json","targets":["ecommerce.spec.live-state.warehouse-stock.7c"],"desc":"The staff stock total always renders zero after a warehouse restock.","file":"client/src/components/AdminPanel.tsx","edits":[{"find":"<td data-role=\"admin-stock\">{totalStockOf(item.id)}</td>","replace":"<td data-role=\"admin-stock\">{0}</td>"}]},{"id":"customers-can-schedule-restocks","scenario":"tracks/ecommerce/scenarios/03-deferred-access.json","targets":["ecommerce.l3.deferred-access.scheduled-work-access.317a"],"desc":"Scheduling a restock no longer checks that the caller is an administrator.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" (ctx, input) => {\n requireAdmin(ctx);\n const itemName = input.item.trim();\n const warehouseName = input.warehouse.trim();\n const item = [...ctx.db.item].find(row => row.name === itemName);\n const warehouse = [...ctx.db.warehouse].find(row => row.name === warehouseName);","replace":" (ctx, input) => {\n // mutant: any signed-in or anonymous caller can schedule work\n const itemName = input.item.trim();\n const warehouseName = input.warehouse.trim();\n const item = [...ctx.db.item].find(row => row.name === itemName);\n const warehouse = [...ctx.db.warehouse].find(row => row.name === warehouseName);"}]},{"id":"scheduled-restock-execution-queue-is-process-local","scenario":"tracks/ecommerce/scenarios/03-deferred-durability.json","targets":["ecommerce.l3.deferred-durability.restart-survival.311a"],"desc":"Keep manual restock execution IDs only in the V8 process. Ordinary timers work, but restart loses the execution queue while pending rows remain. Isolate replacement can also lose this queue.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"export const scheduleRestock = spacetimedb.reducer(","replace":"const pendingRestockExecution = new Set<bigint>();\n\nexport const scheduleRestock = spacetimedb.reducer("},{"find":" ctx.db.scheduledRestock.insert({\n id: 0n,\n itemId: item.id,\n warehouseId: warehouse.id,\n quantity: input.quantity,\n dueMicros: nowMicros(ctx) + BigInt(input.delaySeconds) * SECOND,\n status: 'pending',\n reorderRuleId: undefined,\n });","replace":" const pending = ctx.db.scheduledRestock.insert({\n id: 0n,\n itemId: item.id,\n warehouseId: warehouse.id,\n quantity: input.quantity,\n dueMicros: nowMicros(ctx) + BigInt(input.delaySeconds) * SECOND,\n status: 'pending',\n reorderRuleId: undefined,\n });\n pendingRestockExecution.add(pending.id);"},{"find":" if (pending.status !== 'pending' || pending.dueMicros > now) continue;\n restoreStock(ctx, pending.itemId, pending.warehouseId, pending.quantity);","replace":" if (pending.status !== 'pending' || pending.dueMicros > now) continue;\n if (pending.reorderRuleId === undefined && !pendingRestockExecution.delete(pending.id)) continue;\n restoreStock(ctx, pending.itemId, pending.warehouseId, pending.quantity);"}]},{"id":"reservation-is-delayed-past-the-durability-window","scenario":"tracks/ecommerce/scenarios/03-deferred-durability.json","targets":["ecommerce.l3.deferred-durability.restart-survival.314a"],"desc":"A reservation is persisted with a ten-minute lifetime instead of ninety seconds.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"const expiresMicros = nowMicros(ctx) + 90n * SECOND;","replace":"const expiresMicros = nowMicros(ctx) + 600n * SECOND;"}]},{"id":"completed-restock-remains-pending","scenario":"tracks/ecommerce/scenarios/03-deferred-integrity.json","targets":["ecommerce.l3.deferred-integrity.exactly-once.311a"],"desc":"A completed restock remains pending and is applied again by later maintenance ticks.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.scheduledRestock.id.update({ ...pending, status: 'complete' });","replace":"ctx.db.scheduledRestock.id.update({ ...pending, status: 'pending' });"}]},{"id":"reservation-expiry-restores-stock-twice","scenario":"tracks/ecommerce/scenarios/03-deferred-integrity.json","targets":["ecommerce.l3.deferred-integrity.stock-conservation.313a"],"desc":"Reservation expiry returns twice the quantity that was reserved.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"restoreStock(ctx, row.stockItemId || row.itemId, row.warehouseId, row.quantity);\n ctx.db.reservation.id.update({ ...row, expired: true });","replace":"restoreStock(ctx, row.stockItemId || row.itemId, row.warehouseId, row.quantity * 2);\n ctx.db.reservation.id.update({ ...row, expired: true });"}]},{"id":"checkout-takes-reserved-stock-again","scenario":"tracks/ecommerce/scenarios/03-deferred-integrity.json","targets":["ecommerce.l3.deferred-integrity.stock-conservation.314a"],"desc":"Checkout decrements stock after the cart reservation already took it.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"const allocations = held.map(row => ({ warehouseId: row.warehouseId, quantity: row.quantity, stockItemId: row.stockItemId }));\n const orderItemRow","replace":"const allocations = decrementStockTracked(ctx, p.itemId, p.quantity);\n const orderItemRow"}]},{"id":"reservation-does-not-decrement-stock","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.301a"],"desc":"Creating a reservation leaves the public stock total unchanged.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.stock.insert({ ...row, quantity: row.quantity - allocation.quantity });","replace":"ctx.db.stock.insert({ ...row, quantity: row.quantity });"}]},{"id":"reservation-timer-is-static","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.305a"],"desc":"The cart always renders ninety seconds instead of a decreasing reservation timer.","file":"client/src/components/CartPanel.tsx","edits":[{"find":"const seconds = Math.max(0, Number((reservation.expiresMicros - BigInt(Date.now()) * 1000n) / 1_000_000n));","replace":"const seconds = 90;"}]},{"id":"checkout-leaves-cart-lines","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.306a"],"desc":"Checkout creates an order but leaves the purchased lines in the cart.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"for (const line of lines) ctx.db.cartItem.id.delete(line.id);","replace":"for (const line of lines) void line;"}]},{"id":"expired-reservation-is-still-marked-live","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.307a"],"desc":"Expired reservations keep their live flag, so the cart does not mark them expired.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.reservation.id.update({ ...row, expired: true });","replace":"ctx.db.reservation.id.update({ ...row, expired: false });"}]},{"id":"renewed-reservation-expires-too-soon","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.308a"],"desc":"Renewed quantities receive only a twenty-second reservation window.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"reserveUnits(ctx, accountId, itemId, quantity);","replace":"reserveUnits(ctx, accountId, itemId, quantity);\n for (const renewed of findReservations(ctx, accountId, itemId)) {\n ctx.db.reservation.id.update({ ...renewed, expiresMicros: nowMicros(ctx) + 20n * SECOND });\n }"}]},{"id":"pending-restock-timer-is-static","scenario":"tracks/ecommerce/scenarios/03-scheduled-restocks.json","targets":["ecommerce.l3.scheduled-restocks.scheduled-restocks.302a"],"desc":"The pending restock UI always renders ninety seconds instead of the server due time.","file":"client/src/components/ProgressionWorkbench.tsx","edits":[{"find":"{Math.max(0, Number((row.dueMicros - BigInt(Date.now()) * 1000n) / 1_000_000n))}","replace":"{90}"}]},{"id":"due-restock-omits-ledger-entry","scenario":"tracks/ecommerce/scenarios/03-scheduled-restock-apply.json","targets":["ecommerce.l3.scheduled-restocks.scheduled-restocks.305a"],"desc":"A due restock updates stock but does not create its stock ledger record.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.stockLedger.insert({\n id: 0n,\n itemId: pending.itemId,\n warehouseId: pending.warehouseId,\n quantity: pending.quantity,\n createdMicros: now,\n source: 'scheduled restock',\n });","replace":"// mutant: due restocks are not recorded in the ledger"}]},{"id":"cancelled-restock-remains-pending","scenario":"tracks/ecommerce/scenarios/03-scheduled-restock-cancel.json","targets":["ecommerce.l3.scheduled-restocks.scheduled-restocks.306a"],"desc":"Cancelling a restock leaves it pending, so maintenance later applies it.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.scheduledRestock.id.update({ ...row, status: 'cancelled' });","replace":"ctx.db.scheduledRestock.id.update({ ...row, status: 'pending' });"}]},{"id":"restart-restock-runs-early","scenario":"tracks/ecommerce/scenarios/03-server-time.json","targets":["ecommerce.l3.server-time.server-time.312a"],"desc":"A scheduled restock ignores its requested delay and becomes due after one second.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"dueMicros: nowMicros(ctx) + BigInt(input.delaySeconds) * SECOND,","replace":"dueMicros: nowMicros(ctx) + SECOND,"}]},{"id":"closed-browser-reservation-never-expires","scenario":"tracks/ecommerce/scenarios/03-server-time.json","targets":["ecommerce.l3.server-time.server-time.313a"],"desc":"Server maintenance skips reservation expiry when no customer browser is present.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"for (const row of [...ctx.db.reservation.iter()]) {\n if (row.expired || row.expiresMicros > now) continue;","replace":"for (const row of [...ctx.db.reservation.iter()].filter(() => false)) {\n if (row.expired || row.expiresMicros > now) continue;"}]},{"id":"catalog-product-is-not-published","scenario":"tracks/ecommerce/scenarios/progression-catalog-management.json","targets":["ecommerce.progression.catalog-management.catalog-management.622a","ecommerce.progression.catalog-management.catalog-management.622b"],"desc":"The public product card omits the submitted name. Both create visibility and variant navigation locate the submitted product card by its name, so the hidden name prevents both required observations.","file":"client/src/components/ItemCard.tsx","edits":[{"find":"{item.name}","replace":"{item.name === 'Travel Mug' ? '' : item.name}"}]},{"id":"catalog-variants-are-discarded","scenario":"tracks/ecommerce/scenarios/progression-catalog-management.json","targets":["ecommerce.progression.catalog-management.catalog-management.622b"],"desc":"Catalog management discards every submitted product variant.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"for (const variantName of variants.split(',').map(value => value.trim()).filter(Boolean)) {\n ctx.db.itemVariant.insert({ id: 0n, itemId: product.id, name: variantName });\n }","replace":"void variants; // mutant: submitted variants are discarded"}]},{"id":"profile-is-lost-on-fresh-account-login","scenario":"tracks/ecommerce/scenarios/progression-customer-profile.json","targets":["ecommerce.spec.state-durability.customer-profile-reload.620a"],"desc":"Delete a saved profile when its owner signs in from a new transport identity. Initial save, same-identity reload, and the independent privacy owner remain intact.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" throw new SenderError('Incorrect username or password.');\n }\n\n const existingSession = ctx.db.session.identity.find(ctx.sender);\n if (existingSession) {\n ctx.db.session.identity.update({ ...existingSession, accountId: acc.id });","replace":" throw new SenderError('Incorrect username or password.');\n }\n\n const existingSession = ctx.db.session.identity.find(ctx.sender);\n if (!existingSession) { ctx.db.customerProfile.accountId.delete(acc.id); }\n if (existingSession) {\n ctx.db.session.identity.update({ ...existingSession, accountId: acc.id });"}]},{"id":"customer-profile-view-leaks-another-account","scenario":"tracks/ecommerce/scenarios/progression-customer-profile.json","targets":["ecommerce.spec.access-control.customer-profile-privacy.620b"],"desc":"The customer profile view returns the first stored profile without checking its owner.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"return accountId === null ? undefined : ctx.db.customerProfile.accountId.find(accountId) ?? undefined;","replace":"return accountId === null ? undefined : [...ctx.db.customerProfile.iter()][0];"}]},{"id":"faceted-search-ignores-category","scenario":"tracks/ecommerce/scenarios/progression-faceted-filters.json","targets":["ecommerce.progression.faceted-search.faceted-search.401a"],"desc":"Faceted search applies price and stock filters but ignores the selected category.","file":"client/src/App.tsx","edits":[{"find":".filter(item => !categoryFilter || categoryByItem.get(item.id) === categoryFilter)","replace":".filter(() => true) // mutant: category filter is ignored"}]},{"id":"active-search-uses-purchase-ranking","scenario":"tracks/ecommerce/scenarios/progression-search-ordering.json","targets":["ecommerce.spec.search-ordering.search-ordering.402b"],"desc":"Keep purchase ranking when search text or filters are active.","file":"client/src/App.tsx","edits":[{"find":"const catalogItems = showingSearch ? filteredSearchResults : rankedItems;","replace":"const catalogItems = rankedItems;"}]},{"id":"faceted-search-next-page-does-not-advance","scenario":"tracks/ecommerce/scenarios/progression-faceted-pagination.json","targets":["ecommerce.progression.faceted-search.faceted-search.402a"],"desc":"The next-page control keeps the search on its current page.","file":"client/src/App.tsx","edits":[{"find":"onClick={() => setSearchPage(page => page + 1)}>Next</button>","replace":"onClick={() => setSearchPage(page => page)}>Next</button>"}]},{"id":"managed-support-leaks-and-accepts-cross-account-replies","scenario":"tracks/ecommerce/scenarios/progression-managed-support-privacy.json","targets":["ecommerce.spec.access-control.managed-support-privacy.613b"],"desc":"Managed support tickets are visible across accounts and replayed replies are accepted.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":".filter(row => isGuestTicketCreator(sender, row.creatorIdentity.toHexString(), row.accountId) ||\n !!actor && (actor.isAdmin || actor.isStaff || row.accountId === accountId))","replace":".filter(() => true)"},{"file":"backend/spacetimedb/src/index.ts","find":"if (!actor.isAdmin && !actor.isStaff && ticket.accountId !== actor.id) {\n throw new SenderError('That support ticket is private.');\n }","replace":"// mutant: any signed-in account can access any support ticket"}]},{"id":"managed-support-replies-are-empty","scenario":"tracks/ecommerce/scenarios/progression-managed-support-shared.json","targets":["ecommerce.spec.live-state.managed-support.613a","ecommerce.progression.managed-support.managed-support.613c"],"desc":"Managed support stores replies without their message body. Both the ordinary reply and shared-live reply assertions require the stored message body; neither can pass an empty reply.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"body: body.trim(),\n createdMicros: nowMicros(ctx),","replace":"body: '',\n createdMicros: nowMicros(ctx),"}]},{"id":"managed-support-live-replies-stay-at-initial-snapshot","scenario":"tracks/ecommerce/scenarios/progression-managed-support-shared.json","targets":["ecommerce.spec.live-state.managed-support.613a"],"desc":"Keep the initial subscribed reply snapshot on each page. Reducers still save replies, and a reload shows them, but later replies do not reach the rendered conversation live.","file":"client/src/App.tsx","edits":[{"find":" const [supportReplyRows] = useTable(tables.visibleSupportReplies);","replace":" const [liveSupportReplyRows, supportRepliesReady] = useTable(tables.visibleSupportReplies);\n const supportReplyRows = useMemo(() => [...liveSupportReplyRows], [supportRepliesReady]);"}]},{"id":"notification-preferences-are-not-saved","scenario":"tracks/ecommerce/scenarios/progression-notification-preferences.json","targets":["ecommerce.spec.state-durability.notification-preferences-reload.630a"],"desc":"Saving notification preferences discards the selected values.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"if (existing) ctx.db.notificationPreference.accountId.update(row);\n else ctx.db.notificationPreference.insert(row);","replace":"void existing;\n void row; // mutant: notification preferences are discarded"}]},{"id":"notification-preferences-leak-across-accounts","scenario":"tracks/ecommerce/scenarios/progression-notification-preferences.json","targets":["ecommerce.spec.access-control.notification-preferences-privacy.630b"],"desc":"The preference view returns another account's first stored choice.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"const row = ctx.db.notificationPreference.accountId.find(accountId);\n return row ? { orderEnabled: row.orderEnabled, stockEnabled: row.stockEnabled } : undefined;","replace":"const row = [...ctx.db.notificationPreference.iter()][0];\n return row ? { orderEnabled: row.orderEnabled, stockEnabled: row.stockEnabled } : undefined;"}]},{"id":"checkout-records-zero-payment","scenario":"tracks/ecommerce/scenarios/progression-core-business.json","targets":["ecommerce.progression.payment-records.payment-records.623a"],"desc":"Checkout records a paid payment with a zero amount.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.paymentRecord.insert({ id: 0n, orderId: order.id, amount: order.total, status: 'paid' });\n if (promo) {","replace":"ctx.db.paymentRecord.insert({ id: 0n, orderId: order.id, amount: 0, status: 'paid' });\n if (promo) {"}]},{"id":"checkout-records-duplicate-payments","scenario":"tracks/ecommerce/scenarios/progression-core-business.json","targets":["ecommerce.spec.transactional-integrity.payment-deduplication.623b"],"desc":"Checkout inserts two payment records for one order.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.paymentRecord.insert({ id: 0n, orderId: order.id, amount: order.total, status: 'paid' });\n if (promo) {","replace":"ctx.db.paymentRecord.insert({ id: 0n, orderId: order.id, amount: order.total, status: 'paid' });\n ctx.db.paymentRecord.insert({ id: 0n, orderId: order.id, amount: order.total + 0.01, status: 'paid' });\n if (promo) {"}]},{"id":"active-promotion-does-not-discount-checkout","scenario":"tracks/ecommerce/scenarios/progression-promotion-checkout.json","targets":["ecommerce.progression.promotion-checkout.promotion-checkout-active.621a"],"desc":"Checkout ignores an active promotion when it calculates the discount.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"const discount = promo ? total * (promo.discountPercent / 100) : 0;","replace":"const discount = 0;"}]},{"id":"expired-promotion-is-accepted","scenario":"tracks/ecommerce/scenarios/progression-promotion-checkout.json","targets":["ecommerce.progression.promotion-checkout.promotion-checkout-expired.621b"],"desc":"Promotion application does not reject a promotion after its end time.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"if (!promo || promo.startMicros > now || promo.endMicros < now || promo.redemptions >= promo.usageLimit) {","replace":"if (!promo || promo.startMicros > now || promo.redemptions >= promo.usageLimit) {"}]},{"id":"exhausted-promotion-is-accepted","scenario":"tracks/ecommerce/scenarios/progression-promotion-checkout.json","targets":["ecommerce.progression.promotion-checkout.promotion-checkout-exhausted.621c"],"desc":"Promotion application does not reject a promotion at its usage limit.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"if (!promo || promo.startMicros > now || promo.endMicros < now || promo.redemptions >= promo.usageLimit) {","replace":"if (!promo || promo.startMicros > now || promo.endMicros < now) {"}]},{"id":"customers-can-create-promotions","scenario":"tracks/ecommerce/scenarios/progression-promotion-rules.json","targets":["ecommerce.spec.access-control.promotion-management-boundary.620b"],"desc":"Promotion creation does not require staff access.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" (ctx, input) => {\n requireStaffOrAdmin(ctx);\n if (input.discountPercent <= 0 || input.discountPercent > 100) {","replace":" (ctx, input) => {\n if (input.discountPercent <= 0 || input.discountPercent > 100) {"}]},{"id":"promotion-rule-stores-the-wrong-discount","scenario":"tracks/ecommerce/scenarios/progression-promotion-rules.json","targets":["ecommerce.progression.promotion-rules.promotion-rule-values.620a"],"desc":"Promotion creation stores a one-percent discount instead of the submitted value.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.promotion.insert({ id: 0n, ...input, code: input.code.trim(), redemptions: 0 });","replace":"ctx.db.promotion.insert({ id: 0n, ...input, discountPercent: 1, code: input.code.trim(), redemptions: 0 });"}]},{"id":"staff-cannot-open-staff-tools","scenario":"tracks/ecommerce/scenarios/progression-staff-access.json","targets":["ecommerce.progression.staff-access.staff-access.601a"],"desc":"Deny administrators entry to staff tools while keeping ordinary staff access working.","file":"client/src/App.tsx","edits":[{"find":"{(isStaff || isAdmin) && (\n <button\n type=\"button\"\n className=\"btn btn-ghost btn-sm\"\n data-role=\"staff-link\"","replace":"{(isStaff && !isAdmin) && (\n <button\n type=\"button\"\n className=\"btn btn-ghost btn-sm\"\n data-role=\"staff-link\""}]},{"id":"customers-can-open-staff-tools","scenario":"tracks/ecommerce/scenarios/progression-staff-access.json","targets":["ecommerce.spec.access-control.staff-area-boundary.601b"],"desc":"Expose the protected staff area to signed-in customers, including its navigation and content.","file":"client/src/App.tsx","edits":[{"find":" const isStaff = currentUser?.isStaff ?? false;","replace":" const isStaff = isSignedIn;"}]},{"id":"administrator-role-assignment-is-discarded","scenario":"tracks/ecommerce/scenarios/progression-staff-roles.json","targets":["ecommerce.spec.state-durability.staff-role-reload.621a"],"desc":"Inventory role assignments are not saved; staff and administrator role transitions still work.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"if (existing) ctx.db.staffRole.accountId.update(row);\n else ctx.db.staffRole.insert(row);","replace":"if (role !== 'inventory') {\n if (existing) ctx.db.staffRole.accountId.update(row);\n else ctx.db.staffRole.insert(row);\n }"}]},{"id":"staff-can-assign-roles","scenario":"tracks/ecommerce/scenarios/progression-staff-roles.json","targets":["ecommerce.spec.access-control.staff-role-boundary.621b","ecommerce.spec.access-control.staff-role-revocation.621d"],"desc":"Role assignment accepts staff callers instead of requiring an administrator, including after administrator access is revoked.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"const actor = requireAdmin(ctx);\n const target = ctx.db.account.id.find(accountId);","replace":"const actor = requireStaffOrAdmin(ctx);\n const target = ctx.db.account.id.find(accountId);"}]},{"id":"stock-alert-delivery-is-suppressed","scenario":"tracks/ecommerce/scenarios/progression-stock-alert-delivery.json","targets":["ecommerce.progression.stock-alerts.stock-alert-delivery.631c"],"desc":"A request is accepted but restored stock does not deliver its alert.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"notify(ctx, alert.accountId, 'stock', `${it.name} is available again.`);","replace":"// Suppress stock-alert delivery."}]},{"id":"stock-alert-is-sent-after-every-restock","scenario":"tracks/ecommerce/scenarios/progression-stock-alerts.json","targets":["ecommerce.spec.transactional-integrity.stock-alert-deduplication.631a"],"desc":"A fulfilled stock alert remains active and is sent again after another restock.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.stockAlert.id.update({ ...alert, fulfilled: true });\n }\n }\n);\n\nexport const adminChangePrice","replace":"ctx.db.stockAlert.id.update({ ...alert, fulfilled: false });\n }\n }\n);\n\nexport const adminChangePrice"}]},{"id":"stock-alerts-are-visible-to-other-customers","scenario":"tracks/ecommerce/scenarios/progression-stock-alerts.json","targets":["ecommerce.spec.access-control.stock-alert-privacy.631b"],"desc":"The notifications view returns alerts for every account.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"return [...ctx.db.notification.accountId.filter(accountId)].map(row => ({","replace":"return [...ctx.db.notification.iter()].map(row => ({"}]},{"id":"support-history-is-lost-on-fresh-account-login","scenario":"tracks/ecommerce/scenarios/progression-support-history.json","targets":["ecommerce.spec.state-durability.support-history-reload.612a"],"desc":"Resolve customer support ownership by transport identity only, omitting the account ownership path. Initial submission, same-identity reload, and stored tickets remain intact; fresh account login cannot recover the history.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"!!actor && (actor.isAdmin || actor.isStaff || row.accountId === accountId))","replace":"!!actor && (actor.isAdmin || actor.isStaff || row.creatorIdentity.toHexString() === sender))"}]},{"id":"support-history-leaks-across-customers","scenario":"tracks/ecommerce/scenarios/progression-support-history.json","targets":["ecommerce.spec.access-control.support-history-privacy.612b","ecommerce.spec.access-control.support-history-logout.612d"],"desc":"Return all support tickets, exposing them to other customers and signed-out visitors.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":".filter(row => isGuestTicketCreator(sender, row.creatorIdentity.toHexString(), row.accountId) ||\n !!actor && (actor.isAdmin || actor.isStaff || row.accountId === accountId))","replace":".filter(() => true)"}]},{"id":"visitor-support-reference-is-hidden","scenario":"tracks/ecommerce/scenarios/progression-support-intake.json","targets":["ecommerce.progression.support-intake.support-intake.610a"],"desc":"A visitor can create a support ticket but the returned reference is not rendered.","file":"client/src/components/ProgressionWorkbench.tsx","edits":[{"find":"<div data-role=\"support-reference\">{supportReference}</div>","replace":"<div data-role=\"support-reference\"></div>"}]},{"id":"support-assignment-is-discarded","scenario":"tracks/ecommerce/scenarios/progression-support-triage.json","targets":["ecommerce.progression.support-triage.support-assignment.611a"],"desc":"Support triage saves status and priority but discards the assignee.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.supportTicket.id.update({ ...ticket, assigneeId: nextAssigneeId, priority, status });","replace":"ctx.db.supportTicket.id.update({ ...ticket, assigneeId: undefined, priority, status });"}]},{"id":"support-priority-is-discarded","scenario":"tracks/ecommerce/scenarios/progression-support-triage.json","targets":["ecommerce.progression.support-triage.support-priority.611b"],"desc":"Support triage always saves normal priority instead of the submitted value.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.supportTicket.id.update({ ...ticket, assigneeId: nextAssigneeId, priority, status });","replace":"ctx.db.supportTicket.id.update({ ...ticket, assigneeId: nextAssigneeId, priority: 'normal', status });"}]},{"id":"support-status-is-discarded","scenario":"tracks/ecommerce/scenarios/progression-support-triage.json","targets":["ecommerce.progression.support-triage.support-status.611c"],"desc":"Support triage preserves the old status instead of the submitted value.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.supportTicket.id.update({ ...ticket, assigneeId: nextAssigneeId, priority, status });","replace":"ctx.db.supportTicket.id.update({ ...ticket, assigneeId: nextAssigneeId, priority, status: ticket.status });"}]},{"id":"nonpositive-cart-quantity-is-treated-as-removal","scenario":"tracks/ecommerce/scenarios/01-cart-boundary.json","targets":["ecommerce.spec.access-control.cart-boundary.109b"],"desc":"Accept a negative quantity and remove the cart line instead of refusing the request.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" if (quantity < 1) throw new SenderError('Quantity must be at least 1.');\n const existing = findCartLine(ctx, acc.id, itemId);\n if (!existing) throw new SenderError('That item is not in your cart.');\n if (existing.bundlePrice > 0) throw new SenderError('Remove and re-add a whole bundle.');\n replaceReservation(ctx, acc.id, itemId, quantity);\n ctx.db.cartItem.id.update({ ...existing, quantity });","replace":" const existing = findCartLine(ctx, acc.id, itemId);\n if (!existing) throw new SenderError('That item is not in your cart.');\n if (quantity < 1) {\n ctx.db.cartItem.id.delete(existing.id);\n return;\n }\n if (existing.bundlePrice > 0) throw new SenderError('Remove and re-add a whole bundle.');\n replaceReservation(ctx, acc.id, itemId, quantity);\n ctx.db.cartItem.id.update({ ...existing, quantity });"}]},{"id":"admin-restock-preserves-existing-stock","scenario":"tracks/ecommerce/scenarios/01-warehouse-stock-live-staff.json","targets":["ecommerce.spec.live-state.warehouse-stock.7c"],"desc":"Accept an administrator restock but write the existing quantity back unchanged.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" ctx.db.stock.insert({ ...existing, quantity: existing.quantity + quantity });","replace":" ctx.db.stock.insert({ ...existing, quantity: existing.quantity });"}]},{"id":"operator-authorization-allows-customer-price-change","scenario":"tracks/ecommerce/scenarios/02-strengthened.json","targets":["ecommerce.operations-access.operator-authorization.201b"],"desc":"The price reducer drops its administrator role gate.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" (ctx, { itemId, price }) => {\n requireAdmin(ctx);","replace":" (ctx, { itemId, price }) => {\n // mutant: no administrator role check"}]},{"id":"fulfilment-queue-allows-customer-shipping","scenario":"tracks/ecommerce/scenarios/02-self-contained.json","targets":["ecommerce.operations-access.fulfilment-queue.1e"],"desc":"The shipping reducer drops the staff role check while retaining the pending-order guard.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n requireStaffOrAdmin(ctx);","replace":"export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n // mutant: no staff role check"}]},{"id":"catalog-search-keeps-pre-change-price","scenario":"tracks/ecommerce/scenarios/02-live-price.json","targets":["ecommerce.returns-pricing.price-history.4b"],"desc":"The search result renderer caches each item's first visible price and ignores later live price updates.","file":"client/src/components/ItemCard.tsx","edits":[{"find":"import { ItemRow } from '../types';","replace":"import { useRef } from 'react';\nimport { ItemRow } from '../types';"},{"find":" const outOfStock = stock <= 0;\n const lowStock = !outOfStock && stock <= 5;","replace":" const outOfStock = stock <= 0;\n const lowStock = !outOfStock && stock <= 5;\n // mutant: the card retains the first price it renders\n const firstPrice = useRef(item.price);"},{"find":" {formatMoney(item.price)}","replace":" {formatMoney(firstPrice.current)}"}]},{"id":"open-cart-keeps-pre-change-price","scenario":"tracks/ecommerce/scenarios/progression-price-cart-checkout.json","targets":["ecommerce.returns-pricing.price-history.4c"],"desc":"The open-cart memo ignores reactive item-table price updates while checkout still reads the current server price.","file":"client/src/App.tsx","edits":[{"find":" [cartRows, items, stockByItem]","replace":" [cartRows, stockByItem]"}]},{"id":"catalog-price-rewrites-receipts","scenario":"tracks/ecommerce/scenarios/02-paid-price-history.json","targets":["ecommerce.returns-pricing.price-history.4a"],"desc":"Changing a catalog price cascades into saved order lines and recomputes historical order totals inside the reducer transaction.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" ctx.db.item.id.update({ ...it, price });","replace":" ctx.db.item.id.update({ ...it, price });\n const repricedOrderIds = new Set<bigint>();\n for (const line of ctx.db.orderItem.iter()) {\n if (line.itemId !== itemId) continue;\n ctx.db.orderItem.id.update({ ...line, unitPrice: price });\n repricedOrderIds.add(line.orderId);\n }\n for (const orderId of repricedOrderIds) {\n const historical = ctx.db.customerOrder.id.find(orderId);\n if (!historical) continue;\n let total = 0;\n for (const line of ctx.db.orderItem.orderId.filter(orderId)) total += line.unitPrice * line.quantity;\n ctx.db.customerOrder.id.update({ ...historical, total });\n }"}]},{"id":"catalog-price-rewrites-earned-revenue","scenario":"tracks/ecommerce/scenarios/02-invariants.json","targets":["ecommerce.returns-pricing.refund-accounting.203b"],"desc":"Changing a catalog price cascades into saved order lines and recomputes historical order totals and revenue inside the reducer transaction.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" ctx.db.item.id.update({ ...it, price });","replace":" ctx.db.item.id.update({ ...it, price });\n const repricedOrderIds = new Set<bigint>();\n for (const line of ctx.db.orderItem.iter()) {\n if (line.itemId !== itemId) continue;\n ctx.db.orderItem.id.update({ ...line, unitPrice: price });\n repricedOrderIds.add(line.orderId);\n }\n for (const orderId of repricedOrderIds) {\n const historical = ctx.db.customerOrder.id.find(orderId);\n if (!historical) continue;\n let total = 0;\n for (const line of ctx.db.orderItem.orderId.filter(orderId)) total += line.unitPrice * line.quantity;\n ctx.db.customerOrder.id.update({ ...historical, total });\n }"}]},{"id":"returned-line-marker-omitted","scenario":"tracks/ecommerce/scenarios/02-strengthened.json","targets":["ecommerce.returns-pricing.cancellation-and-return.3c"],"desc":"A returned order line keeps its persisted returned state, restored stock, and adjusted revenue but omits the visible returned marker.","file":"client/src/components/OrdersPanel.tsx","edits":[{"find":"{item.returned && <span className=\"badge badge-muted\" style={{ marginLeft: 6 }}>Returned</span>}","replace":"{false && <span className=\"badge badge-muted\" style={{ marginLeft: 6 }}>Returned</span>}"}]},{"id":"direct-review-access-is-not-checked","scenario":"tracks/ecommerce/scenarios/progression-review-access.json","targets":["ecommerce.progression.review-access-specifications.review-eligibility-direct.618a"],"desc":"The direct review action accepts a review from a customer who did not buy the item.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" if (!bought) throw new SenderError('You can only review items you have purchased.');","replace":" // mutant: purchase eligibility is not checked"}]},{"id":"support-history-rows-are-hidden","scenario":"tracks/ecommerce/scenarios/progression-support-history.json","targets":["ecommerce.progression.support-history.support-history.612c","ecommerce.spec.state-durability.support-history-reload.612a","ecommerce.spec.access-control.support-history-privacy.612b","ecommerce.spec.access-control.support-history-logout.612d"],"desc":"Hide submitted support ticket rows while leaving the submission reference available. History, reload, privacy, and logout require the owner to see the ticket first. This breaks those positive observations; it does not create a privacy leak or remove stored data.","file":"client/src/components/ProgressionWorkbench.tsx","edits":[{"find":"data-role=\"support-ticket\"","replace":"data-role=\"support-ticket\" style={{ display: \"none\" }}"}]},{"id":"authorized-restock-does-not-change-stock","scenario":"tracks/ecommerce/scenarios/01-admin-write-staff.json","targets":["ecommerce.feature.warehouse-admin.admin-write.103a"],"desc":"Accept an administrator restock without changing warehouse stock.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" if (existing) {\n ctx.db.stock.by_item_warehouse.delete([itemId, warehouseId]);\n ctx.db.stock.insert({ ...existing, quantity: existing.quantity + quantity });\n } else {\n ctx.db.stock.insert({ item_id: itemId, warehouse_id: warehouseId, quantity });\n }\n","replace":" // mutant: accept restock without changing stock\n"}]},{"id":"low-stock-threshold-is-two-units","scenario":"tracks/ecommerce/scenarios/02-low-stock.json","targets":["ecommerce.inventory-operations.operational-views.5e","ecommerce.spec.live-state.inventory-dashboard.5a"],"desc":"The dashboard lists only items with two units or fewer, so the seeded three-unit item is missing from the low-stock view. The live check in the same scenario opens with the identical observation and necessarily fails with it.","file":"client/src/App.tsx","edits":[{"find":"const LOW_STOCK_THRESHOLD = 10;","replace":"const LOW_STOCK_THRESHOLD = 2;"}]},{"id":"category-totals-count-only-since-the-dashboard-opened","scenario":"tracks/ecommerce/scenarios/02-operational-category-totals.json","targets":["ecommerce.inventory-operations.operational-views.5f"],"desc":"The category table shows units and revenue accumulated since the dashboard was opened instead of the stored totals, so a reload resets both to zero and the totals recorded before the reload are not reproduced. Live movement within one open dashboard is still correct, so the live check is unaffected.","file":"client/src/components/AdminPanel.tsx","edits":[{"find":"import { useState } from 'react';","replace":"import { useRef, useState } from 'react';"},{"find":" return (\n <div className=\"admin-panel\" data-role=\"admin-panel\">","replace":" const openingTotals = useRef<Map<bigint, { units: number; revenue: number }> | null>(null);\n if (openingTotals.current === null && categoryTotals.length > 0) {\n openingTotals.current = new Map(\n categoryTotals.map((cat): [bigint, { units: number; revenue: number }] => [\n cat.categoryId,\n { units: cat.unitsSold, revenue: cat.revenue },\n ])\n );\n }\n const sessionTotals = categoryTotals.map((cat) => {\n const opening = openingTotals.current?.get(cat.categoryId);\n return {\n ...cat,\n unitsSold: cat.unitsSold - (opening?.units ?? 0),\n revenue: cat.revenue - (opening?.revenue ?? 0),\n };\n });\n\n return (\n <div className=\"admin-panel\" data-role=\"admin-panel\">"},{"find":" {categoryTotals.map((cat) => (","replace":" {sessionTotals.map((cat) => ("}]},{"id":"profile-summary-ignores-a-profile-saved-this-session","scenario":"tracks/ecommerce/scenarios/progression-customer-profile.json","targets":["ecommerce.progression.customer-profile.customer-profile.620c"],"desc":"Hide the profile summary immediately after saving in the current view. Stored profile data and a reopened or reloaded view remain correct, so fresh-login durability and privacy positive controls remain observable.","file":"client/src/components/ProgressionWorkbench.tsx","edits":[{"find":" const [profileName, setProfileName] = useState(profile?.name ?? '');","replace":" const [profileName, setProfileName] = useState(profile?.name ?? '');\n const [profileSavedHere, setProfileSavedHere] = useState(false);"},{"find":"onClick={() => reducers?.saveProfile({ name: profileName, address: profileAddress })}","replace":"onClick={() => { setProfileSavedHere(true); return reducers?.saveProfile({ name: profileName, address: profileAddress }); }}"},{"find":"<div data-role=\"profile-address-summary\">{profile?.name} {profile?.address}</div>","replace":"<div data-role=\"profile-address-summary\">{!profileSavedHere && <>{profile?.name} {profile?.address}</>}</div>"}]},{"id":"stored-support-replies-are-hidden-after-reload","scenario":"tracks/ecommerce/scenarios/progression-managed-support-shared.json","targets":["ecommerce.progression.managed-support.managed-support.613c"],"desc":"Replies already stored when the page loads are hidden and only replies that arrive while the page is open are shown, so a reloaded customer or staff member cannot see the earlier exchange. The live shared-case check exchanges only new replies and is unaffected.","file":"client/src/App.tsx","edits":[{"find":"import { useEffect, useMemo, useState } from 'react';","replace":"import { useEffect, useMemo, useRef, useState } from 'react';"},{"find":" const [supportReplyRows] = useTable(tables.visibleSupportReplies);","replace":" const [liveSupportReplyRows, supportRepliesReady] = useTable(tables.visibleSupportReplies);\n const openedSupportReplies = useRef<Set<string> | null>(null);\n if (openedSupportReplies.current === null && supportRepliesReady) {\n openedSupportReplies.current = new Set(\n liveSupportReplyRows.map((row) => `${row.ticketId}:${row.author}:${row.body}`)\n );\n }\n const supportReplyRows = liveSupportReplyRows.filter(\n (row) => !openedSupportReplies.current?.has(`${row.ticketId}:${row.author}:${row.body}`)\n );"}]},{"id":"saving-notification-preferences-resets-the-toggles","scenario":"tracks/ecommerce/scenarios/progression-notification-preferences.json","targets":["ecommerce.progression.notification-preferences.notification-preferences.630c"],"desc":"Saving sends the chosen preferences but resets both toggles to off and stops the form from following the stored row for the rest of the session, so the saved choice cannot be seen until a reload. The reload and cross-account checks read the stored row on a fresh page and are unaffected.","file":"client/src/components/ProgressionWorkbench.tsx","edits":[{"find":" useEffect(() => {\n setOrderEnabled(preferences?.orderEnabled ?? false);\n setStockEnabled(preferences?.stockEnabled ?? false);\n }, [preferences?.orderEnabled, preferences?.stockEnabled]);","replace":" const [preferencesSubmitted, setPreferencesSubmitted] = useState(false);\n useEffect(() => {\n if (preferencesSubmitted) return;\n setOrderEnabled(preferences?.orderEnabled ?? false);\n setStockEnabled(preferences?.stockEnabled ?? false);\n }, [preferences?.orderEnabled, preferences?.stockEnabled, preferencesSubmitted]);"},{"find":"onClick={() => reducers?.saveNotificationPreferences({ orderEnabled, stockEnabled })}","replace":"onClick={() => { reducers?.saveNotificationPreferences({ orderEnabled, stockEnabled }); setPreferencesSubmitted(true); setOrderEnabled(false); setStockEnabled(false); }}"}]},{"id":"saving-a-staff-role-snaps-the-input-back-to-the-stored-role","scenario":"tracks/ecommerce/scenarios/progression-staff-roles.json","targets":["ecommerce.progression.staff-roles.staff-roles.621c"],"desc":"Saving a role sends the new role to the server but snaps the visible input back to the role stored before the save for the rest of the session, so the administrator cannot see the assignment take. A reload renders the stored role, so the durability and boundary checks are unaffected.","file":"client/src/components/ProgressionWorkbench.tsx","edits":[{"find":" <button className=\"btn btn-ghost btn-sm\" data-role=\"staff-role-save\" onClick={() => reducers?.assignStaffRole({\n accountId: row.accountId,\n role: (document.getElementById(`role-${String(row.accountId)}`) as HTMLSelectElement)?.value ?? 'warehouse',\n })}>Save role</button>","replace":" <button className=\"btn btn-ghost btn-sm\" data-role=\"staff-role-save\" onClick={() => {\n const roleInput = document.getElementById(`role-${String(row.accountId)}`) as HTMLInputElement | null;\n reducers?.assignStaffRole({ accountId: row.accountId, role: roleInput?.value ?? 'warehouse' });\n if (roleInput) roleInput.value = row.role;\n }}>Save role</button>"}]},{"id":"fulfilment-queue-is-frozen-at-page-load","scenario":"tracks/ecommerce/scenarios/02-fulfilment-live.json","targets":["ecommerce.spec.live-state.fulfilment-queue.1a"],"desc":"The fulfilment queue renders the rows delivered with the page's initial subscription and ignores later updates, so an order placed while the queue is open never appears without a reload.","file":"client/src/App.tsx","edits":[{"find":"import { useEffect, useMemo, useState } from 'react';","replace":"import { useEffect, useMemo, useRef, useState } from 'react';"},{"find":" const [queueRows] = useTable(tables.fulfilmentQueue);","replace":" const [liveQueueRows, queueReady] = useTable(tables.fulfilmentQueue);\n const openedQueueRows = useRef<typeof liveQueueRows | null>(null);\n if (openedQueueRows.current === null && queueReady) openedQueueRows.current = liveQueueRows;\n const queueRows = openedQueueRows.current ?? liveQueueRows;"}]},{"id":"low-stock-list-is-frozen-at-page-load","scenario":"tracks/ecommerce/scenarios/02-low-stock.json","targets":["ecommerce.spec.live-state.inventory-dashboard.5a"],"desc":"The low-stock list is computed once from the first complete stock snapshot and never recomputed, so items no longer enter or leave it as stock is restocked or sold. The seeded low item is in that first snapshot, so the static listing check is unaffected.","file":"client/src/App.tsx","edits":[{"find":"import { useEffect, useMemo, useState } from 'react';","replace":"import { useEffect, useMemo, useRef, useState } from 'react';"},{"find":" const lowStockItems = useMemo(\n () =>\n [...items]\n .filter((i) => (stockByItem.get(i.id) ?? 0) <= LOW_STOCK_THRESHOLD)\n .sort((a, b) => (stockByItem.get(a.id) ?? 0) - (stockByItem.get(b.id) ?? 0)),\n [items, stockByItem]\n );","replace":" const liveLowStockItems = useMemo(\n () =>\n [...items]\n .filter((i) => (stockByItem.get(i.id) ?? 0) <= LOW_STOCK_THRESHOLD)\n .sort((a, b) => (stockByItem.get(a.id) ?? 0) - (stockByItem.get(b.id) ?? 0)),\n [items, stockByItem]\n );\n const openedLowStockItems = useRef<typeof liveLowStockItems | null>(null);\n if (openedLowStockItems.current === null && items.length > 0 && stocks.length > 0) {\n openedLowStockItems.current = liveLowStockItems;\n }\n const lowStockItems = openedLowStockItems.current ?? liveLowStockItems;"}]},{"id":"category-totals-are-frozen-at-page-load","scenario":"tracks/ecommerce/scenarios/02-operational-category-totals.json","targets":["ecommerce.spec.live-state.sales-dashboard.5b"],"desc":"The category totals render the first non-empty row set received and ignore later updates, so a purchase does not move units or revenue while the dashboard is open. A reload receives the stored totals, so the reload check is unaffected.","file":"client/src/App.tsx","edits":[{"find":"import { useEffect, useMemo, useState } from 'react';","replace":"import { useEffect, useMemo, useRef, useState } from 'react';"},{"find":" const [categoryTotalRows] = useTable(tables.categoryTotals);","replace":" const [liveCategoryTotalRows] = useTable(tables.categoryTotals);\n const openedCategoryTotalRows = useRef<typeof liveCategoryTotalRows | null>(null);\n if (openedCategoryTotalRows.current === null && liveCategoryTotalRows.length > 0) {\n openedCategoryTotalRows.current = liveCategoryTotalRows;\n }\n const categoryTotalRows = openedCategoryTotalRows.current ?? liveCategoryTotalRows;"}]},{"id":"warehouse-totals-are-frozen-at-page-load","scenario":"tracks/ecommerce/scenarios/02-transfer-totals.json","targets":["ecommerce.spec.live-state.stock-transfers.2b"],"desc":"The per-warehouse totals are computed once from the first stock snapshot and never recomputed, so a transfer moves neither warehouse figure while the dashboard is open.","file":"client/src/App.tsx","edits":[{"find":"import { useEffect, useMemo, useState } from 'react';","replace":"import { useEffect, useMemo, useRef, useState } from 'react';"},{"find":" const stockByWarehouse = useMemo(() => {\n const map = new Map<bigint, number>();\n for (const row of stocks) {\n map.set(row.warehouseId, (map.get(row.warehouseId) ?? 0) + row.quantity);\n }\n return map;\n }, [stocks]);","replace":" const liveStockByWarehouse = useMemo(() => {\n const map = new Map<bigint, number>();\n for (const row of stocks) {\n map.set(row.warehouseId, (map.get(row.warehouseId) ?? 0) + row.quantity);\n }\n return map;\n }, [stocks]);\n const openedStockByWarehouse = useRef<typeof liveStockByWarehouse | null>(null);\n if (openedStockByWarehouse.current === null && liveStockByWarehouse.size > 0) {\n openedStockByWarehouse.current = liveStockByWarehouse;\n }\n const stockByWarehouse = openedStockByWarehouse.current ?? liveStockByWarehouse;"}]},{"id":"transfer-skips-the-source-holding-check","scenario":"tracks/ecommerce/scenarios/02-transfer-overdraw.json","targets":["ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c"],"desc":"The serialized transfer reducer no longer checks that the source warehouse holds the requested quantity, so an overdraw is accepted instead of refused and the source quantity wraps below zero.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" if (available < quantity) {\n throw new SenderError(`Not enough stock in source warehouse: only ${available} available.`);\n }","replace":" // mutant: the source warehouse holding is not checked"}]},{"id":"credit-checkout-ignores-wallet","desc":"A credit checkout pays entirely externally despite available credit.","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.feature.store-credit.store-credit-750.750a"],"file":"backend/spacetimedb/src/index.ts","edits":[{"find":" const creditMinor = useCredit ? Math.min(wallet?.amountMinor ?? 0,totalMinor) : 0;","replace":" const creditMinor = 0;"}]},{"id":"credit-grant-replay-increments-balance","desc":"Replaying a grant applies its credit to the wallet again.","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.spec.store-credit.production-752.752a"],"file":"backend/spacetimedb/src/index.ts","edits":[{"find":" if (existing.amountMinor !== amountMinor) throw new SenderError('Reference identifies another grant.');\n return;","replace":" if (existing.amountMinor !== amountMinor) throw new SenderError('Reference identifies another grant.');\n const wallet = ctx.db.creditWallet.accountId.find(accountId)!;\n ctx.db.creditWallet.accountId.update({ ...wallet, amountMinor: wallet.amountMinor + amountMinor });\n return;"}]},{"id":"customer-can-grant-credit","desc":"Customer authentication is accepted without staff authorization.","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.spec.store-credit.production-753.753a"],"file":"backend/spacetimedb/src/index.ts","edits":[{"find":" (ctx, { accountId, amountMinor, reference }) => {\n requireStaffOrAdmin(ctx);","replace":" (ctx, { accountId, amountMinor, reference }) => {\n requireAccount(ctx);"}]},{"id":"split-refund-does-not-restore-credit","desc":"The refund is recorded but its original wallet credit is not restored.","scenario":"tracks/ecommerce/scenarios/progression-split-tender-refunds.json","targets":["ecommerce.feature.split-tender-refunds.split-tender-refunds-751.751a"],"file":"backend/spacetimedb/src/index.ts","edits":[{"find":" refundOrderCredit(ctx, order, order.total);","replace":" // mutant: omit wallet restoration"}]},{"id":"split-refund-duplicates-credit","desc":"A refund credits the wallet twice while recording one refund.","scenario":"tracks/ecommerce/scenarios/progression-split-tender-refunds.json","targets":["ecommerce.spec.split-tender-refunds.production-756.756a"],"file":"backend/spacetimedb/src/index.ts","edits":[{"find":"amountMinor: wallet.amountMinor + delta","replace":"amountMinor: wallet.amountMinor + delta * 2"}]},{"id":"subscription-skips-due-purchase","desc":"Due deliveries are recorded as skipped although stock is available.","scenario":"tracks/ecommerce/scenarios/progression-subscriptions.json","targets":["ecommerce.feature.subscriptions.subscriptions-760.760a"],"file":"backend/spacetimedb/src/subscriptions.ts","edits":[{"find":" const orderId = purchase(row.accountId, row.itemId, row.quantity, row.price);","replace":" const orderId: bigint | null = null;"}]},{"id":"subscription-allows-foreign-cancellation","desc":"A customer can cancel another customer subscription.","scenario":"tracks/ecommerce/scenarios/progression-subscriptions.json","targets":["ecommerce.spec.subscriptions.production-762.762a"],"file":"backend/spacetimedb/src/subscriptions.ts","edits":[{"find":" if (!row || row.accountId !== accountId) throw new SenderError('Subscription access denied.');","replace":" if (!row) throw new SenderError('Subscription access denied.');"}]},{"id":"subscription-pause-is-not-recorded","desc":"Pause acknowledges the request but the subscription remains active.","scenario":"tracks/ecommerce/scenarios/progression-subscriptions.json","targets":["ecommerce.spec.subscriptions.production-763.763a"],"file":"backend/spacetimedb/src/subscriptions.ts","edits":[{"find":" ctx.db.purchaseSubscription.id.update({ ...row, status: 'paused', pausedMicros: now });","replace":" ctx.db.purchaseSubscription.id.update({ ...row, status: 'active', pausedMicros: now });"}]},{"id":"credit-checkout-retains-purchased-cart","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.spec.store-credit.production-754.754a"],"desc":"The purchased cart remains available instead of being consumed by checkout.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" for (const line of lines) ctx.db.cartItem.id.delete(line.id);","replace":" // mutant: checked-out lines remain in the cart"}]},{"id":"reconnection-erases-stored-credit","desc":"A new connection erases stored wallet credit; the fresh view after restart must catch the data loss.","file":"backend/spacetimedb/src/index.ts","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.spec.store-credit.production-755.755a"],"edits":[{"find":"export const onConnect = spacetimedb.clientConnected((_ctx) => {});","replace":"export const onConnect = spacetimedb.clientConnected(ctx => { for (const row of ctx.db.creditWallet.iter()) ctx.db.creditWallet.accountId.update({ ...row, amountMinor: 0 }); });"}]},{"id":"reconnection-cancels-pending-subscriptions","desc":"A new connection cancels pending subscriptions; restarting and reconnecting must preserve this work.","file":"backend/spacetimedb/src/index.ts","scenario":"tracks/ecommerce/scenarios/progression-subscriptions.json","targets":["ecommerce.spec.subscriptions.production-761.761a"],"edits":[{"find":"export const onConnect = spacetimedb.clientConnected((_ctx) => {});","replace":"export const onConnect = spacetimedb.clientConnected(ctx => { for (const row of ctx.db.purchaseSubscription.iter()) if (row.status === 'active') ctx.db.purchaseSubscription.id.update({ ...row, status: 'cancelled' }); });"}]},{"id":"bundle-definition-loses-component-quantity","scenario":"tracks/ecommerce/scenarios/progression-product-bundles.json","targets":["ecommerce.feature.product-bundles.product-bundles.740a"],"desc":"definition loses component quantity","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"return { ...component, itemId: String(product.id) };","replace":"return { ...component, quantity: 1, itemId: String(product.id) };"}]},{"id":"bundle-catalog-write-allows-customers","scenario":"tracks/ecommerce/scenarios/progression-product-bundles.json","targets":["ecommerce.spec.bundle-integrity.bundle-743.743a"],"desc":"catalog write allows customers","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"const actor = requireStaffOrAdmin(ctx);\n if (!actor.isAdmin && ctx.db.staffRole.accountId.find(actor.id)?.role !== 'catalog') {","replace":"const actor = requireAccount(ctx);\n if (false) {"}]},{"id":"bundle-checkout-price-not-snapshot","scenario":"tracks/ecommerce/scenarios/progression-bundle-checkout.json","targets":["ecommerce.feature.bundle-checkout.bundle-checkout.741a"],"desc":"checkout price not snapshot","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"bundlePrice: product.price, bundleComponentsJson: definition.componentsJson","replace":"bundlePrice: product.price + 1, bundleComponentsJson: definition.componentsJson"}]},{"id":"bundle-return-loses-original-components","scenario":"tracks/ecommerce/scenarios/progression-bundle-returns.json","targets":["ecommerce.feature.bundle-returns.bundle-returns.742a"],"desc":"return loses original components","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"for (const line of bundles) {\n restoreOrderItemStock(ctx, line);","replace":"for (const line of bundles) {\n // mutant: original stock is not restored"}]},{"id":"bundle-return-replay-restocks-again","scenario":"tracks/ecommerce/scenarios/progression-bundle-returns.json","targets":["ecommerce.spec.bundle-integrity.bundle-742.742b"],"desc":"return replay restocks again","file":"backend/spacetimedb/src/index.ts","edits":[{"find":".filter(row => row.isBundle && !row.returned);","replace":".filter(row => row.isBundle);"}]},{"id":"bundle-return-crosses-account-boundary","scenario":"tracks/ecommerce/scenarios/progression-bundle-returns.json","targets":["ecommerce.spec.bundle-integrity.bundle-748.748a"],"desc":"return crosses account boundary","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"if (!order || order.accountId !== account.id || !['shipped', 'delivered'].includes(order.status))","replace":"if (!order || !['shipped', 'delivered'].includes(order.status))"}]},{"id":"bundle-components-can-overdraw","scenario":"tracks/ecommerce/scenarios/progression-bundle-checkout.json","targets":["ecommerce.spec.bundle-integrity.bundle-744.744a","ecommerce.spec.bundle-integrity.bundle-745.745a"],"desc":"components can overdraw","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"if (!allocations) throw new SenderError('Not enough stock to reserve.');","replace":"if (!allocations) { if (bundleId) return; throw new SenderError('Not enough stock to reserve.'); }"}]},{"id":"bundle-checkout-reuses-reservation","scenario":"tracks/ecommerce/scenarios/progression-bundle-checkout.json","targets":["ecommerce.spec.bundle-integrity.bundle-747.747a"],"desc":"checkout reuses reservation","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"for (const row of held) ctx.db.reservation.id.delete(row.id);\n processReorderRules(ctx, p.itemId);","replace":"// mutant: reservations survive checkout\n processReorderRules(ctx, p.itemId);"},{"find":"for (const line of lines) ctx.db.cartItem.id.delete(line.id);","replace":"// mutant: cart survives checkout"}]},{"id":"bundle-expiry-does-not-release-components","scenario":"tracks/ecommerce/scenarios/progression-bundle-checkout.json","targets":["ecommerce.spec.bundle-integrity.bundle-746.746a"],"desc":"Expired bundle reservations retain their component stock after restart.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"if (row.expired || row.expiresMicros > now) continue;\n restoreStock(ctx, row.stockItemId || row.itemId, row.warehouseId, row.quantity);","replace":"if (row.expired || row.expiresMicros > now) continue;\n if (!row.stockItemId) restoreStock(ctx, row.itemId, row.warehouseId, row.quantity);"}]},{"id":"return-after-support-refund-is-blocked","scenario":"tracks/ecommerce/scenarios/progression-support-return-interaction.json","targets":["ecommerce.feature.split-tender-refunds.return-refund-interaction.757a"],"desc":"Reject a valid physical return after a financial refund.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" if (target.returned) throw new SenderError('Item already returned.');","replace":" if (target.returned || order.refundedTotal > 0) throw new SenderError('Item already returned.');"}]},{"id":"support-refund-after-return-pays-twice","scenario":"tracks/ecommerce/scenarios/progression-support-return-interaction.json","targets":["ecommerce.feature.split-tender-refunds.return-refund-interaction.757b"],"desc":"Pay the full order again after the physical return already refunded it.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" const amount = order.total - order.refundedTotal;","replace":" const amount = order.total;"},{"find":"refundedTotal: order.total, status: order.status","replace":"refundedTotal: order.refundedTotal + amount, status: order.status"}]},{"id":"support-history-leaks-to-signed-out-visitors","scenario":"tracks/ecommerce/scenarios/progression-support-history.json","targets":["ecommerce.spec.access-control.support-history-logout.612d"],"desc":"Disclose support tickets to signed-out visitors while leaving signed-in ownership filtering unchanged.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":".filter(row => isGuestTicketCreator(sender, row.creatorIdentity.toHexString(), row.accountId) ||","replace":".filter(row => !actor || isGuestTicketCreator(sender, row.creatorIdentity.toHexString(), row.accountId) ||"}]},{"id":"checkout-crash-integrity","scenario":"tracks/ecommerce/scenarios/progression-checkout-crash.json","targets":["ecommerce.spec.state-durability.checkout-crash-integrity.910a"],"desc":"A lifecycle callback changes a prepared or cleared cart to quantity two, leaving neither a legal rollback nor a complete checkout.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"export const onDisconnect = spacetimedb.clientDisconnected((_ctx) => {});","replace":"export const onDisconnect = spacetimedb.clientDisconnected((ctx) => { for (const row of ctx.db.cartItem.iter()) if ([...ctx.db.customerOrder.iter()].some(order => order.accountId === row.accountId)) ctx.db.cartItem.id.update({ ...row, quantity: 2 }); });"}]},{"id":"checkout-crash-durability","scenario":"tracks/ecommerce/scenarios/progression-checkout-crash.json","targets":["ecommerce.spec.state-durability.checkout-crash-durability.910b"],"desc":"A lifecycle callback corrupts earlier order totals. The unchanged prepared snapshots and acknowledged baseline must expose the damage.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"export const onDisconnect = spacetimedb.clientDisconnected((_ctx) => {});","replace":"export const onDisconnect = spacetimedb.clientDisconnected((ctx) => { for (const order of ctx.db.customerOrder.iter()) if ([...ctx.db.cartItem.iter()].some(row => row.accountId === order.accountId)) ctx.db.customerOrder.id.update({ ...order, total: 0 }); });"}]},{"id":"review-script-unsafe-render","scenario":"tracks/ecommerce/scenarios/progression-review-script.json","targets":["ecommerce.progression.review-access-specifications.stored-review-script.9180a"],"desc":"Render stored review text as HTML so it executes in another customer session.","file":"client/src/components/ItemDetail.tsx","edits":[{"find":"<div>{r.comment}</div>","replace":"<div dangerouslySetInnerHTML={{ __html: r.comment }} />"}]},{"id":"review-script-reject-all","scenario":"tracks/ecommerce/scenarios/progression-review-script.json","targets":["ecommerce.progression.review-access-specifications.stored-review-script.9180a"],"desc":"Reject all nonempty reviews, including the required benign review; absence of script is insufficient.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"(ctx, { itemId, rating, comment }) => {","replace":"(ctx, { itemId, rating, comment }) => {\n if (comment.length > 0) throw new SenderError(\"Review rejected\");"}]},{"id":"review-owner-reject-all","scenario":"tracks/ecommerce/scenarios/progression-review-access.json","targets":["ecommerce.progression.review-access-specifications.review-eligibility-direct.618a"],"desc":"Reject the legitimate buyer review as well as unauthorized reviews.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"(ctx, { itemId, rating, comment }) => {","replace":"(ctx, { itemId, rating, comment }) => {\n if (comment.length > 0) throw new SenderError(\"Reject all reviews\");"}]}]}}} diff --git a/tools/stack-bench/tests/fixtures/qualification-evidence/ecommerce-l3-e804c1302/null-targeted.json b/tools/stack-bench/tests/fixtures/qualification-evidence/ecommerce-l3-e804c1302/null-targeted.json new file mode 100644 index 00000000000..69b6c412f63 --- /dev/null +++ b/tools/stack-bench/tests/fixtures/qualification-evidence/ecommerce-l3-e804c1302/null-targeted.json @@ -0,0 +1,115 @@ +{ + "artifactSchemaVersion": 2, + "kind": "null_control", + "id": "null-control-2026-09-18T01-36-17-108Z", + "attempt": { + "id": "null-control-2026-09-18T01-36-17-108Z", + "parentId": null + }, + "timestamps": { + "startedAt": "2026-09-18T01:36:17.108Z", + "completedAt": "2026-09-18T01:36:39.145Z" + }, + "identities": { + "engine": { + "id": "stack-bench", + "sha256": "c5f7dfe76190f44d716012e3c2e69f5ebf550f2364a3ddb5c2fc0be908771934" + }, + "recipe": { + "id": "ecommerce.progression-catalog", + "sha256": "53fbb8093b2335837d88c246f04da99858c40b57ce24d17fbd2b6c815abdb62d" + }, + "fixture": null, + "calibration": { + "id": "ecommerce.dependency-l3-calibration", + "sha256": "253174720de8d80e884e764e4d8018c0a98e1c54b364b33f7679aa189af8d1bd" + }, + "experiment": null, + "agentAdapter": null, + "stackAdapter": null, + "packs": [] + }, + "payload": { + "durationMs": 22037, + "runner": { + "schemaVersion": 1, + "mode": "appliance", + "platform": "linux", + "architecture": "x64", + "hostname": "docker-desktop", + "dockerEngineVersion": "29.6.2", + "dockerOs": "linux", + "dockerArchitecture": "x86_64", + "kernelVersion": "6.6.87.2-microsoft-standard-WSL2", + "cpuCount": 32, + "memoryBytes": 49232842752, + "containersRunning": 9, + "packageRegistry": "http://127.0.0.1:4873/" + }, + "qualificationScope": { + "checksSha256": "b9d7acf0749c3310b680e9376cc4e931cf59382da420a64a4c5e2bf320e89b88", + "executableSha256": "b5a0becb3c1791c4ec00f0a543a3273cd57cf73b69451e981ef8a2092f5bddf7", + "kind": "null", + "mutationSha256": null, + "recipe": { + "contentSha256": "53fbb8093b2335837d88c246f04da99858c40b57ce24d17fbd2b6c815abdb62d", + "id": "ecommerce.progression-catalog" + }, + "schemaVersion": 3, + "stack": null, + "sha256": "ddef8792f386721bcf2bbe836531d5fff8d8539d6835dda004b8733cdb7cc397" + }, + "tracks": [ + "ecommerce" + ], + "ok": true, + "summary": { + "criteria": 1, + "points": 2, + "expectedFailures": { + "criteria": 1, + "points": 2 + }, + "expectedFailureStages": { + "setup": { + "criteria": 1, + "points": 2 + }, + "assertion": { + "criteria": 0, + "points": 0 + } + }, + "vacuousPasses": { + "criteria": 0, + "points": 0 + }, + "oracleGaps": { + "criteria": 0, + "points": 0 + }, + "unscored": { + "criteria": 0, + "passed": 0, + "failed": 0, + "inconclusive": 0 + } + }, + "criteria": [ + { + "track": "ecommerce", + "level": 3, + "suite": "selected-source-093", + "scenario": "scenarios/progression-review-access.json", + "feature": 618, + "featureName": "Review access", + "criterion": "618a", + "points": 2, + "status": "expected_fail", + "evidenceStatus": "blocked", + "failureStage": "setup", + "detail": "Blocked by a failed prerequisite: none of the signup-username, signup-toggle, signin-toggle controls became visible in time" + } + ] + } +} diff --git a/tools/stack-bench/tests/fixtures/qualification-evidence/ecommerce-l3-e804c1302/postgres-targeted.json b/tools/stack-bench/tests/fixtures/qualification-evidence/ecommerce-l3-e804c1302/postgres-targeted.json new file mode 100644 index 00000000000..2a413ce0178 --- /dev/null +++ b/tools/stack-bench/tests/fixtures/qualification-evidence/ecommerce-l3-e804c1302/postgres-targeted.json @@ -0,0 +1,136 @@ +{ + "artifactSchemaVersion": 2, + "kind": "reference_qualification", + "id": "reference-live-postgres-20260918002256-31", + "attempt": { + "id": "reference-live-postgres-20260918002256-31", + "parentId": null + }, + "timestamps": { + "startedAt": "2026-09-18T00:22:56.721Z", + "completedAt": "2026-09-18T00:24:37.025Z" + }, + "identities": { + "engine": { + "id": "stack-bench", + "sha256": "581451119c180f50fd478aa2c17e1fa766fd38e7648e01d8a354334df1b606d3" + }, + "recipe": { + "id": "ecommerce.progression-catalog", + "sha256": "53fbb8093b2335837d88c246f04da99858c40b57ce24d17fbd2b6c815abdb62d" + }, + "fixture": { + "id": "ecommerce-reference-postgres", + "sha256": "f3906fa01050f00a0ed76c69c256a6b0013bbf72eb72c03aa92d79e47b71c2b8" + }, + "calibration": { + "id": "ecommerce.dependency-l3-calibration", + "sha256": "253174720de8d80e884e764e4d8018c0a98e1c54b364b33f7679aa189af8d1bd" + }, + "experiment": null, + "agentAdapter": null, + "stackAdapter": { + "id": "postgres", + "sha256": null + }, + "packs": [] + }, + "payload": { + "fixture": "ecommerce-reference-postgres", + "fixtureSha256": "f3906fa01050f00a0ed76c69c256a6b0013bbf72eb72c03aa92d79e47b71c2b8", + "requiredRepetitions": 1, + "isolation": "docker", + "runner": { + "schemaVersion": 1, + "mode": "appliance", + "platform": "linux", + "architecture": "x64", + "hostname": "docker-desktop", + "dockerEngineVersion": "29.6.2", + "dockerOs": "linux", + "dockerArchitecture": "x86_64", + "kernelVersion": "6.6.87.2-microsoft-standard-WSL2", + "cpuCount": 32, + "memoryBytes": 49232842752, + "containersRunning": 11, + "packageRegistry": "http://127.0.0.1:4873/" + }, + "qualificationScope": { + "checksSha256": "b9d7acf0749c3310b680e9376cc4e931cf59382da420a64a4c5e2bf320e89b88", + "executableSha256": "53b1dcbe4d602e82f17999dc7e49fad71920e5af156e338e3755119e803477dd", + "kind": "mutation", + "mutationSha256": "01083fb8903afe2390d7e2842f4a3877974296032fc79e06d40eee9212cf8beb", + "recipe": { + "contentSha256": "53fbb8093b2335837d88c246f04da99858c40b57ce24d17fbd2b6c815abdb62d", + "id": "ecommerce.progression-catalog" + }, + "schemaVersion": 3, + "stack": { + "id": "postgres", + "reference": { + "id": "ecommerce-reference-postgres", + "sourceSha256": "f3906fa01050f00a0ed76c69c256a6b0013bbf72eb72c03aa92d79e47b71c2b8" + }, + "version": "1.6.0" + }, + "sha256": "e020cbeac0b10f444a73ad15e2911f6c8f692aefb17eccda1d7022143ad46af3" + }, + "mutationControl": true, + "runs": [ + { + "repetition": 1, + "output": "postgres-targeted.runs/r1", + "durationMs": 100100, + "processError": null, + "harnessSha256Before": "8cf33debdb3c7f4797c0b324facba96dd68ca68736c60170d61f7b76f892b9e4", + "harnessSha256After": "8cf33debdb3c7f4797c0b324facba96dd68ca68736c60170d61f7b76f892b9e4", + "ok": true, + "failures": [], + "runId": "ecommerce-postgres-run0-20260918002257-2eb5c097", + "score": "2/2", + "imageId": "sha256:32ff9a9e519ba9d1af2fe72f24622caa8ca7a9f2baf2338c6b1e0435e6ac5455", + "criteria": 1, + "zeroPointCriteria": 0, + "fingerprint": "f1da4e8dbf9629a97b00bb384d6cc95f9a0005fccc9fde991cd1f69b79f8de8d", + "outcome": "passed", + "packRuntime": { + "schemaVersion": 1, + "metric": "pack-check-wall-clock-sum-v1", + "packs": [ + { + "id": "ecommerce.progression.review-access-specifications", + "checkCount": 1, + "setupRuntimeMs": 704, + "criterionRuntimeMs": 6792, + "measuredRuntimeMs": 7496, + "budget": { + "status": "bounded", + "maxRuntimeMs": 82000 + }, + "exceeded": false + } + ] + }, + "mutations": { + "caught": 4, + "completed": 4, + "total": 4, + "remaining": 0 + } + } + ], + "stable": true, + "sameImage": true, + "sameHarness": true, + "harnessSha256": "8cf33debdb3c7f4797c0b324facba96dd68ca68736c60170d61f7b76f892b9e4", + "qualifiedCheckKeys": [ + "ecommerce.progression.review-access-specifications.review-eligibility-direct.618a" + ], + "featureCatalog": { + "contentSha256": "8671f7883c2e5a24474a546ef1517407cd9334258e1a7cfe91e292a9af1952f2", + "id": "ecommerce.questlines" + }, + "diagnostic": true, + "ok": true + } +} diff --git a/tools/stack-bench/tests/fixtures/qualification-evidence/ecommerce-l3-e804c1302/previous-inputs.json b/tools/stack-bench/tests/fixtures/qualification-evidence/ecommerce-l3-e804c1302/previous-inputs.json new file mode 100644 index 00000000000..c0096720ad7 --- /dev/null +++ b/tools/stack-bench/tests/fixtures/qualification-evidence/ecommerce-l3-e804c1302/previous-inputs.json @@ -0,0 +1 @@ +{"documents":{"release":{"capabilities":["backend-lifecycle","browser","concurrent-actors","database-observation","database-read","direct-database-write","direct-server-call","process-crash","request-replay"],"checkCatalog":[{"category":"feature","checkGroupId":"accounts","criterionId":"1a","description":"a visitor can create an account and is signed in as it","executionId":"selected-source-001","featureId":1,"packId":"ecommerce.feature.accounts","points":1,"role":"feature","source":"scenarios/01-account-create.json","stableKey":"ecommerce.feature.accounts.accounts.1a"},{"category":"production","checkGroupId":"accounts","criterionId":"1b","description":"a taken username is refused and does not sign the visitor in as the existing account","executionId":"selected-source-002","featureId":1,"packId":"ecommerce.feature.accounts","points":1,"role":"feature","source":"scenarios/01-account-duplicate.json","stableKey":"ecommerce.feature.accounts.accounts.1b"},{"category":"production","checkGroupId":"accounts","criterionId":"1c","description":"a wrong password is refused","executionId":"selected-source-003","featureId":1,"packId":"ecommerce.feature.accounts","points":1,"role":"feature","source":"scenarios/01-account-password.json","stableKey":"ecommerce.feature.accounts.accounts.1c"},{"category":"production","checkGroupId":"session-reload","criterionId":"1e","description":"the session survives a reload","executionId":"selected-source-004","featureId":1,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":1,"requiresFeatures":["ecommerce.feature.accounts"],"role":"guarantee","source":"scenarios/01-account-reload.json","stableKey":"ecommerce.spec.state-durability.session-reload.1e"},{"category":"feature","checkGroupId":"accounts","criterionId":"1d","description":"signing out and back in returns the same account","executionId":"selected-source-005","featureId":1,"packId":"ecommerce.feature.accounts","points":1,"role":"feature","source":"scenarios/01-account-signout.json","stableKey":"ecommerce.feature.accounts.accounts.1d"},{"category":"feature","checkGroupId":"admin-write","criterionId":"103a","description":"an administrator can restock a warehouse","executionId":"selected-source-006","featureId":103,"packId":"ecommerce.feature.warehouse-admin","points":1,"role":"feature","source":"scenarios/01-admin-write-staff.json","stableKey":"ecommerce.feature.warehouse-admin.admin-write.103a"},{"category":"production","checkGroupId":"warehouse-write-boundary","criterionId":"103b","description":"the server refuses a warehouse write from staff","executionId":"selected-source-006","featureId":103,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-admin-write-staff.json","stableKey":"ecommerce.spec.access-control.warehouse-write-boundary.103b"},{"category":"production","checkGroupId":"purchase-stock","criterionId":"3b","description":"buying reduces the stock every other client sees, without a reload","executionId":"selected-source-007","featureId":3,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":2,"requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/01-buying.json","stableKey":"ecommerce.spec.live-state.purchase-stock.3b"},{"category":"production","checkGroupId":"cart-boundary","criterionId":"109a","description":"the same cart action run by another customer changes only that customer's cart","executionId":"selected-source-008","featureId":109,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.feature.cart"],"role":"guarantee","source":"scenarios/01-cart-boundary.json","stableKey":"ecommerce.spec.access-control.cart-boundary.109a"},{"category":"production","checkGroupId":"cart-boundary","criterionId":"109b","description":"a negative quantity is refused and leaves the cart unchanged","executionId":"selected-source-008","featureId":109,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"requiresFeatures":["ecommerce.feature.cart"],"role":"guarantee","source":"scenarios/01-cart-boundary.json","stableKey":"ecommerce.spec.access-control.cart-boundary.109b"},{"category":"production","checkGroupId":"cart-reload","criterionId":"4b","description":"the cart survives a reload","executionId":"selected-source-009","featureId":4,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":1,"requiresFeatures":["ecommerce.feature.cart"],"role":"guarantee","source":"scenarios/01-cart.json","stableKey":"ecommerce.spec.state-durability.cart-reload.4b"},{"category":"production","checkGroupId":"shared-cart","criterionId":"4c","description":"the same account signed in elsewhere sees one cart, live","executionId":"selected-source-009","featureId":4,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":2,"requiresFeatures":["ecommerce.feature.cart"],"role":"guarantee","source":"scenarios/01-cart.json","stableKey":"ecommerce.spec.live-state.shared-cart.4c"},{"category":"feature","checkGroupId":"catalog-ranking","criterionId":"2b","description":"the storefront shows the exact alphabetical top ten before any purchase","executionId":"selected-source-010","featureId":2,"packId":"ecommerce.feature.catalog-discovery","points":1,"role":"feature","source":"scenarios/01-catalog-ranking.json","stableKey":"ecommerce.feature.catalog.catalog-ranking.2b","stablePackId":"ecommerce.feature.catalog"},{"category":"feature","checkGroupId":"catalog-search","criterionId":"2d","description":"case-insensitive partial search finds an item outside the storefront top ten","executionId":"selected-source-011","featureId":2,"packId":"ecommerce.feature.catalog-discovery","points":1,"role":"feature","source":"scenarios/01-catalog-search.json","stableKey":"ecommerce.feature.catalog.catalog-search.2d","stablePackId":"ecommerce.feature.catalog"},{"category":"feature","checkGroupId":"catalog-values","criterionId":"2a","description":"a signed-out visitor sees the seeded item name, price, and total stock","executionId":"selected-source-012","featureId":2,"packId":"ecommerce.feature.catalog-items","points":1,"role":"feature","source":"scenarios/01-catalog-values.json","stableKey":"ecommerce.feature.catalog.catalog-values.2a","stablePackId":"ecommerce.feature.catalog"},{"category":"production","checkGroupId":"ranking","criterionId":"2c","description":"a purchase moves the bought item to the front of the ranking, live","executionId":"selected-source-013","featureId":2,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":1,"requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/01-core.json","stableKey":"ecommerce.spec.live-state.ranking.2c"},{"category":"production","checkGroupId":"duplicate-checkout","criterionId":"203a","description":"the same item added from two tabs at once becomes one line of two","executionId":"selected-source-014","featureId":203,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":1,"requiresFeatures":["ecommerce.feature.checkout"],"role":"guarantee","source":"scenarios/01-duplicate-checkout.json","stableKey":"ecommerce.spec.concurrency-safety.duplicate-checkout.203a"},{"category":"production","checkGroupId":"duplicate-checkout","criterionId":"203b","description":"checking the same cart out twice at once produces one order","executionId":"selected-source-014","featureId":203,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":1,"requiresFeatures":["ecommerce.feature.checkout"],"role":"guarantee","source":"scenarios/01-duplicate-checkout.json","stableKey":"ecommerce.spec.concurrency-safety.duplicate-checkout.203b"},{"category":"production","checkGroupId":"external-stock","criterionId":"901a","description":"a direct database write sets Desk Lamp's East stock to 5, and the already-open storefront updates from 100 to 50 without a reload or page action","executionId":"selected-source-015","featureId":901,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.external-data-sync","points":1,"requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-external-live-sync.json","stableKey":"ecommerce.spec.external-data-sync.external-stock.901a"},{"category":"production","checkGroupId":"external-stock","criterionId":"901d","description":"while the storefront is offline, a direct database write sets Desk Lamp's East stock to 7; after reconnecting, the same page catches up from 100 to the authoritative total of 52","executionId":"selected-source-016","featureId":901,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.external-data-sync","points":1,"requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-external-reconnect-sync.json","stableKey":"ecommerce.spec.external-data-sync.external-stock.901d"},{"checkGroupId":"external-stock","criterionId":"901b","description":"after a direct database write sets Desk Lamp's East stock to 5, a reload reads the persisted total of 50","executionId":"selected-source-017","featureId":901,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.external-data-sync","points":0,"requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-external-reload-sync.json","stableKey":"ecommerce.spec.external-data-sync.external-stock.901b"},{"category":"production","checkGroupId":"external-stock","criterionId":"901c","description":"a stock correction lands while the app server is stopped, and the already-open storefront shows the authoritative total of 65 after the server returns without a reload","executionId":"selected-source-018","featureId":901,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.external-data-sync","points":1,"requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-external-server-restart-sync.json","stableKey":"ecommerce.spec.external-data-sync.external-stock.901c"},{"category":"production","checkGroupId":"last-unit","criterionId":"201a","description":"after six customers try to buy the last three units, each warehouse stores zero stock and all observed clients show zero stock","executionId":"selected-source-019","featureId":201,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":1,"requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-last-unit.json","stableKey":"ecommerce.spec.concurrency-safety.last-unit.201a"},{"category":"production","checkGroupId":"last-unit","criterionId":"201c","description":"revenue increases by exactly three sales, not six","executionId":"selected-source-019","featureId":201,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":1,"requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-last-unit.json","stableKey":"ecommerce.spec.concurrency-safety.last-unit.201c"},{"category":"production","checkGroupId":"last-unit","criterionId":"201b","description":"the last three units create complete orders for the successful buyers, and all four affordable purchases succeed when stock is sufficient","executionId":"selected-source-019","featureId":201,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":1,"requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-last-unit.json","stableKey":"ecommerce.spec.concurrency-safety.last-unit.201b"},{"category":"production","checkGroupId":"order-ownership","criterionId":"106a","description":"a working order history contains the customer's own order and not another customer's order","executionId":"selected-source-020","featureId":106,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/01-order-ownership.json","stableKey":"ecommerce.spec.access-control.order-ownership.106a"},{"category":"production","checkGroupId":"purchase-attribution","criterionId":"102a","description":"a direct purchase is attributed to the authenticated caller, not another account","executionId":"selected-source-021","featureId":102,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/01-purchase-attribution.json","stableKey":"ecommerce.spec.access-control.purchase-attribution.102a"},{"category":"production","checkGroupId":"purchase-session","criterionId":"101a","description":"a valid direct purchase works for the buyer but is refused without a session","executionId":"selected-source-022","featureId":101,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/01-purchase-session.json","stableKey":"ecommerce.spec.access-control.purchase-session.101a"},{"category":"feature","checkGroupId":"restock-race","criterionId":"202-control","description":"an uncontended restock of five is stored by the server and shows on the storefront","executionId":"selected-source-023","featureId":202,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":0,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-restock-race.json","stableKey":"ecommerce.spec.concurrency-safety.restock-race.202-control"},{"category":"production","checkGroupId":"restock-race","criterionId":"202a","description":"restocking during purchases preserves stock, complete buyer orders and their warehouse allocations","executionId":"selected-source-023","featureId":202,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":1,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-restock-race.json","stableKey":"ecommerce.spec.concurrency-safety.restock-race.202a"},{"category":"production","checkGroupId":"review-eligibility","criterionId":"108a","description":"someone who never bought the item cannot review it","executionId":"selected-source-024","featureId":108,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","source":"scenarios/01-review-eligibility.json","stableKey":"ecommerce.spec.access-control.review-eligibility.108a"},{"category":"production","checkGroupId":"review-eligibility","criterionId":"108b","description":"buying the item earns the right to review it","executionId":"selected-source-024","featureId":108,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","source":"scenarios/01-review-eligibility.json","stableKey":"ecommerce.spec.access-control.review-eligibility.108b"},{"category":"production","checkGroupId":"rating","criterionId":"6c","description":"the average rating reflects both reviewers and updates live","executionId":"selected-source-025","featureId":6,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":1,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","source":"scenarios/01-review-rating-live.json","stableKey":"ecommerce.spec.live-state.rating.6c"},{"category":"production","checkGroupId":"unique-review","criterionId":"6b","description":"a later review submission does not create a duplicate for the same customer and item","executionId":"selected-source-026","featureId":6,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":1,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","source":"scenarios/01-review-uniqueness.json","stableKey":"ecommerce.spec.transactional-integrity.unique-review.6b"},{"category":"feature","checkGroupId":"reviews","criterionId":"6a","description":"a customer can review an item and everyone sees it, signed out included","executionId":"selected-source-027","featureId":6,"packId":"ecommerce.feature.reviews","points":2,"role":"feature","source":"scenarios/01-review-visibility.json","stableKey":"ecommerce.feature.reviews.reviews.6a"},{"category":"production","checkGroupId":"server-price","criterionId":"104a","description":"direct purchases of two differently priced items persist exactly one correctly priced order each","executionId":"selected-source-028","featureId":104,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":2,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/01-server-price.json","stableKey":"ecommerce.spec.transactional-integrity.server-price.104a"},{"category":"production","checkGroupId":"warehouse-area-boundary","criterionId":"7a","description":"the administrator area stays unavailable to other staff","executionId":"selected-source-029","featureId":7,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-warehouse-admin-staff.json","stableKey":"ecommerce.spec.access-control.warehouse-area-boundary.7a"},{"category":"feature","checkGroupId":"warehouse-view","criterionId":"7b","description":"admin lists every item, every warehouse, and what each warehouse holds","executionId":"selected-source-029","featureId":7,"packId":"ecommerce.feature.warehouse-admin","points":1,"role":"feature","source":"scenarios/01-warehouse-admin-staff.json","stableKey":"ecommerce.feature.warehouse-admin.warehouse-view.7b"},{"category":"production","checkGroupId":"warehouse-stock","criterionId":"7c","description":"the storefront stock is the sum across warehouses, and a restock raises it live","executionId":"selected-source-030","featureId":7,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":2,"requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-warehouse-stock-live-staff.json","stableKey":"ecommerce.spec.live-state.warehouse-stock.7c"},{"category":"production","checkGroupId":"cancellation-and-return","criterionId":"3d","description":"cancelling a pending order removes it from the fulfilment queue","executionId":"selected-source-031","featureId":3,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.cancellation-queue-specifications","points":1,"requiresFeatures":["ecommerce.l2.order-cancellation-features","ecommerce.progression.fulfilment-queue"],"role":"guarantee","source":"scenarios/02-cancellation-queue.json","stableKey":"ecommerce.returns-pricing.cancellation-and-return.3d","stablePackId":"ecommerce.returns-pricing"},{"category":"production","checkGroupId":"fulfilment-area-boundary","criterionId":"1d","description":"staff and administrators can open fulfilment while customers cannot","executionId":"selected-source-032","featureId":1,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"requiresFeatures":["ecommerce.progression.fulfilment-queue"],"role":"guarantee","source":"scenarios/02-fulfilment-access.json","stableKey":"ecommerce.spec.access-control.fulfilment-area-boundary.1d"},{"category":"production","checkGroupId":"fulfilment-queue","criterionId":"1a","description":"an order placed by a customer appears in the staff queue without a reload","executionId":"selected-source-033","featureId":1,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":2,"requiresFeatures":["ecommerce.progression.fulfilment-queue"],"role":"guarantee","source":"scenarios/02-fulfilment-live.json","stableKey":"ecommerce.spec.live-state.fulfilment-queue.1a"},{"category":"feature","checkGroupId":"fulfilment-queue","criterionId":"1c","description":"shipping removes the order from the queue and marks the customer's order shipped","executionId":"selected-source-034","featureId":1,"packId":"ecommerce.progression.fulfilment-queue","points":2,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"feature","source":"scenarios/02-fulfilment-ship.json","stableKey":"ecommerce.operations-access.fulfilment-queue.1c","stablePackId":"ecommerce.operations-access"},{"category":"production","checkGroupId":"refund-accounting","criterionId":"203a","description":"concurrent cancellation restores original stock and the booked amount once, while revenue returns to its prior value","executionId":"selected-source-035","featureId":203,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.cancellation-accounting-specifications","points":3,"requiresFeatures":["ecommerce.l2.order-cancellation-features"],"role":"guarantee","source":"scenarios/02-invariants.json","stableKey":"ecommerce.returns-pricing.refund-accounting.203a","stablePackId":"ecommerce.returns-pricing"},{"category":"production","checkGroupId":"refund-accounting","criterionId":"203b","description":"a price change does not rewrite revenue already earned","executionId":"selected-source-035","featureId":203,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.price-accounting-specifications","points":2,"requiresFeatures":["ecommerce.l2.price-history-features"],"role":"guarantee","source":"scenarios/02-invariants.json","stableKey":"ecommerce.returns-pricing.refund-accounting.203b","stablePackId":"ecommerce.returns-pricing"},{"category":"production","checkGroupId":"price-history","criterionId":"4b","description":"the new price reaches a signed-out visitor without a reload","executionId":"selected-source-036","featureId":4,"packId":"ecommerce.l2.price-history-features","points":2,"role":"feature","source":"scenarios/02-live-price.json","stableKey":"ecommerce.returns-pricing.price-history.4b","stablePackId":"ecommerce.returns-pricing"},{"category":"feature","checkGroupId":"operational-views","criterionId":"5e","description":"the dashboard lists a current low-stock item","executionId":"selected-source-037","featureId":5,"packId":"ecommerce.l2.inventory-dashboard","points":1,"role":"feature","source":"scenarios/02-low-stock.json","stableKey":"ecommerce.inventory-operations.operational-views.5e","stablePackId":"ecommerce.inventory-operations"},{"category":"production","checkGroupId":"inventory-dashboard","criterionId":"5a","description":"an item falling to ten units or fewer joins the low-stock list, live","executionId":"selected-source-037","featureId":5,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":3,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.inventory-dashboard"],"role":"guarantee","source":"scenarios/02-low-stock.json","stableKey":"ecommerce.spec.live-state.inventory-dashboard.5a"},{"category":"feature","checkGroupId":"operational-views","criterionId":"5d","description":"a signed-out visitor sees a best seller in the recommendations list","executionId":"selected-source-038","featureId":5,"packId":"ecommerce.l2.sales-dashboard","points":1,"role":"feature","source":"scenarios/02-operational-best-sellers.json","stableKey":"ecommerce.inventory-operations.operational-views.5d","stablePackId":"ecommerce.inventory-operations"},{"category":"feature","checkGroupId":"operational-views","criterionId":"5f","description":"the dashboard shows category units and revenue","executionId":"selected-source-039","featureId":5,"packId":"ecommerce.l2.sales-dashboard","points":1,"role":"feature","source":"scenarios/02-operational-category-totals.json","stableKey":"ecommerce.inventory-operations.operational-views.5f","stablePackId":"ecommerce.inventory-operations"},{"category":"production","checkGroupId":"sales-dashboard","criterionId":"5b","description":"a purchase updates that category's units and revenue live","executionId":"selected-source-039","featureId":5,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":3,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.sales-dashboard"],"role":"guarantee","source":"scenarios/02-operational-category-totals.json","stableKey":"ecommerce.spec.live-state.sales-dashboard.5b"},{"category":"feature","checkGroupId":"operational-views","criterionId":"5c","description":"a purchase recommends another item from that category and excludes an item in the cart","executionId":"selected-source-040","featureId":5,"packId":"ecommerce.l2.recommendations","points":2,"role":"feature","source":"scenarios/02-operational-recommendations.json","stableKey":"ecommerce.inventory-operations.operational-views.5c","stablePackId":"ecommerce.inventory-operations"},{"category":"production","checkGroupId":"cancellation-and-return","criterionId":"3a","description":"cancelling a pending order restores its stock and revenue","executionId":"selected-source-041","featureId":3,"packId":"ecommerce.l2.order-cancellation-features","points":2,"role":"feature","source":"scenarios/02-order-cancellation-core.json","stableKey":"ecommerce.returns-pricing.cancellation-and-return.3a","stablePackId":"ecommerce.returns-pricing"},{"category":"feature","checkGroupId":"cancellation-and-return","criterionId":"3b","description":"a cancelled order is shown as cancelled in the customer's history","executionId":"selected-source-042","featureId":3,"packId":"ecommerce.l2.order-cancellation-features","points":1,"role":"feature","source":"scenarios/02-order-cancellation-history.json","stableKey":"ecommerce.returns-pricing.cancellation-and-return.3b","stablePackId":"ecommerce.returns-pricing"},{"category":"production","checkGroupId":"price-history","criterionId":"4a","description":"a price change updates the live catalog but leaves the customer's exact paid price unchanged","executionId":"selected-source-043","featureId":4,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.price-history-specifications","points":3,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.price-history-features"],"role":"guarantee","source":"scenarios/02-paid-price-history.json","stableKey":"ecommerce.returns-pricing.price-history.4a","stablePackId":"ecommerce.returns-pricing"},{"category":"feature","checkGroupId":"fulfilment-queue","criterionId":"1b","description":"the queue names the warehouse the order will ship from","executionId":"selected-source-044","featureId":1,"packId":"ecommerce.progression.fulfilment-queue","points":1,"role":"feature","source":"scenarios/02-queue-warehouse.json","stableKey":"ecommerce.operations-access.fulfilment-queue.1b","stablePackId":"ecommerce.operations-access"},{"category":"production","checkGroupId":"stock-conservation","criterionId":"202b","description":"a sale and its cancellation leave the shelf exactly as they found it","executionId":"selected-source-045","featureId":202,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","points":2,"requiresFeatures":["ecommerce.l2.order-cancellation-features"],"role":"guarantee","source":"scenarios/02-self-contained.json","stableKey":"ecommerce.inventory-operations.stock-conservation.202b","stablePackId":"ecommerce.inventory-operations"},{"category":"production","checkGroupId":"stock-conservation","criterionId":"202c","description":"a fresh client sees the restored total after a sale is cancelled","executionId":"selected-source-045","featureId":202,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","points":1,"requiresFeatures":["ecommerce.l2.order-cancellation-features"],"role":"guarantee","source":"scenarios/02-self-contained.json","stableKey":"ecommerce.inventory-operations.stock-conservation.202c","stablePackId":"ecommerce.inventory-operations"},{"category":"production","checkGroupId":"operator-authorization","criterionId":"201c","description":"the server refuses a customer's direct attempt to ship their own pending order","executionId":"selected-source-046","featureId":201,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.operations-access-specifications","points":2,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"guarantee","source":"scenarios/02-server-actions.json","stableKey":"ecommerce.operations-access.operator-authorization.201c","stablePackId":"ecommerce.operations-access"},{"category":"production","checkGroupId":"stock-conservation","criterionId":"202d","description":"a direct transfer racing a direct purchase leaves the exact starting total minus the sold unit","executionId":"selected-source-046","featureId":202,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","points":2,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.stock-transfers-features"],"role":"guarantee","source":"scenarios/02-server-actions.json","stableKey":"ecommerce.inventory-operations.stock-conservation.202d","stablePackId":"ecommerce.inventory-operations"},{"category":"production","checkGroupId":"order-owner","criterionId":"204a","description":"the server refuses one customer trying to cancel another customer's still-pending order","executionId":"selected-source-046","featureId":204,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.operations-access-specifications","points":2,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.order-cancellation-features"],"role":"guarantee","source":"scenarios/02-server-actions.json","stableKey":"ecommerce.operations-access.order-owner.204a","stablePackId":"ecommerce.operations-access"},{"category":"production","checkGroupId":"warehouse-transfer","criterionId":"2a","description":"a transfer decreases the source, increases the destination, and preserves the item's exact total","executionId":"selected-source-047","featureId":2,"packId":"ecommerce.l2.stock-transfers-features","points":3,"role":"feature","source":"scenarios/02-strengthened.json","stableKey":"ecommerce.inventory-operations.warehouse-transfer.2a","stablePackId":"ecommerce.inventory-operations"},{"category":"production","checkGroupId":"operator-authorization","criterionId":"201a","description":"the server refuses a customer's direct transfer and neither warehouse nor the item total changes","executionId":"selected-source-047","featureId":201,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.operations-access-specifications","points":2,"requiresFeatures":["ecommerce.l2.stock-transfers-features"],"role":"guarantee","source":"scenarios/02-strengthened.json","stableKey":"ecommerce.operations-access.operator-authorization.201a","stablePackId":"ecommerce.operations-access"},{"category":"production","checkGroupId":"operator-authorization","criterionId":"201b","description":"the server refuses a customer's direct price change and the last accepted price remains exact","executionId":"selected-source-047","featureId":201,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.operations-access-specifications","points":2,"requiresFeatures":["ecommerce.l2.price-history-features"],"role":"guarantee","source":"scenarios/02-strengthened.json","stableKey":"ecommerce.operations-access.operator-authorization.201b","stablePackId":"ecommerce.operations-access"},{"category":"production","checkGroupId":"stock-conservation","criterionId":"202a","description":"a transfer decreases East, increases West, and leaves the item's exact total unchanged","executionId":"selected-source-047","featureId":202,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","points":2,"requiresFeatures":["ecommerce.l2.stock-transfers-features"],"role":"guarantee","source":"scenarios/02-strengthened.json","stableKey":"ecommerce.inventory-operations.stock-conservation.202a","stablePackId":"ecommerce.inventory-operations"},{"category":"production","checkGroupId":"stock-transfer-overdraw","criterionId":"2c","description":"a transfer that would overdraw a warehouse is refused and changes neither warehouse nor the item total","executionId":"selected-source-048","featureId":2,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":2,"requiresFeatures":["ecommerce.l2.stock-transfers-features"],"role":"guarantee","source":"scenarios/02-transfer-overdraw.json","stableKey":"ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c"},{"category":"production","checkGroupId":"stock-transfers","criterionId":"2b","description":"both warehouse totals move live and in opposite directions as stock is transferred","executionId":"selected-source-049","featureId":2,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":2,"requiresFeatures":["ecommerce.l2.stock-transfers-features"],"role":"guarantee","source":"scenarios/02-transfer-totals.json","stableKey":"ecommerce.spec.live-state.stock-transfers.2b"},{"category":"production","checkGroupId":"cart-expiration","criterionId":"304a","description":"an inactive cart expires without a browser, releases stock, and returns empty","executionId":"selected-source-050","featureId":304,"packId":"ecommerce.l3.cart-expiration-features","points":4,"role":"feature","source":"scenarios/03-cart-expiration.json","stableKey":"ecommerce.l3.cart-expiration.cart-expiration.304a","stablePackId":"ecommerce.l3.cart-expiration"},{"category":"production","checkGroupId":"scheduled-work-access","criterionId":"317a","description":"the server refuses customer scheduling and cancellation of restocks","executionId":"selected-source-051","featureId":317,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-access-specifications","points":3,"requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"role":"guarantee","source":"scenarios/03-deferred-access.json","stableKey":"ecommerce.l3.deferred-access.scheduled-work-access.317a","stablePackId":"ecommerce.l3.deferred-access"},{"category":"production","checkGroupId":"restart-survival","criterionId":"311a","description":"a restock scheduled before restart still applies","executionId":"selected-source-052","featureId":311,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-durability-specifications","points":4,"requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"role":"guarantee","source":"scenarios/03-deferred-durability.json","stableKey":"ecommerce.l3.deferred-durability.restart-survival.311a","stablePackId":"ecommerce.l3.deferred-durability"},{"category":"production","checkGroupId":"restart-survival","criterionId":"314a","description":"a reservation pending before restart still expires and returns stock","executionId":"selected-source-052","featureId":314,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-durability-specifications","points":4,"requiresFeatures":["ecommerce.l3.reservations-features"],"role":"guarantee","source":"scenarios/03-deferred-durability.json","stableKey":"ecommerce.l3.deferred-durability.restart-survival.314a","stablePackId":"ecommerce.l3.deferred-durability"},{"category":"production","checkGroupId":"restart-survival","criterionId":"315a","description":"an order shipped before restart still becomes delivered","executionId":"selected-source-052","featureId":315,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-durability-specifications","points":4,"requiresFeatures":["ecommerce.l3.order-delivery-features"],"role":"guarantee","source":"scenarios/03-deferred-durability.json","stableKey":"ecommerce.l3.deferred-durability.restart-survival.315a","stablePackId":"ecommerce.l3.deferred-durability"},{"category":"production","checkGroupId":"restart-survival","criterionId":"316a","description":"a cart survives restart and expires near its original five-minute deadline","executionId":"selected-source-052","featureId":316,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-durability-specifications","points":4,"requiresFeatures":["ecommerce.l3.cart-expiration-features"],"role":"guarantee","source":"scenarios/03-deferred-durability.json","stableKey":"ecommerce.l3.deferred-durability.restart-survival.316a","stablePackId":"ecommerce.l3.deferred-durability"},{"category":"production","checkGroupId":"exactly-once","criterionId":"311a","description":"restart cannot replay a completed restock","executionId":"selected-source-053","featureId":311,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-integrity-specifications","points":3,"requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"role":"guarantee","source":"scenarios/03-deferred-integrity.json","stableKey":"ecommerce.l3.deferred-integrity.exactly-once.311a","stablePackId":"ecommerce.l3.deferred-integrity"},{"category":"production","checkGroupId":"exactly-once","criterionId":"312a","description":"restart leaves one delivered order record","executionId":"selected-source-053","featureId":312,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-integrity-specifications","points":3,"requiresFeatures":["ecommerce.l3.order-delivery-features"],"role":"guarantee","source":"scenarios/03-deferred-integrity.json","stableKey":"ecommerce.l3.deferred-integrity.exactly-once.312a","stablePackId":"ecommerce.l3.deferred-integrity"},{"category":"production","checkGroupId":"stock-conservation","criterionId":"313a","description":"expiry returns exactly the unit reserved","executionId":"selected-source-053","featureId":313,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-integrity-specifications","points":3,"requiresFeatures":["ecommerce.l3.reservations-features"],"role":"guarantee","source":"scenarios/03-deferred-integrity.json","stableKey":"ecommerce.l3.deferred-integrity.stock-conservation.313a","stablePackId":"ecommerce.l3.deferred-integrity"},{"category":"production","checkGroupId":"stock-conservation","criterionId":"314a","description":"checkout does not decrement stock after the reservation already did","executionId":"selected-source-053","featureId":314,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-integrity-specifications","points":3,"requiresFeatures":["ecommerce.l3.reservations-features"],"role":"guarantee","source":"scenarios/03-deferred-integrity.json","stableKey":"ecommerce.l3.deferred-integrity.stock-conservation.314a","stablePackId":"ecommerce.l3.deferred-integrity"},{"category":"production","checkGroupId":"order-delivery","criterionId":"303a","description":"a shipped order becomes delivered in customer and staff views","executionId":"selected-source-054","featureId":303,"packId":"ecommerce.l3.order-delivery-features","points":3,"role":"feature","source":"scenarios/03-order-delivery.json","stableKey":"ecommerce.l3.order-delivery.order-delivery.303a","stablePackId":"ecommerce.l3.order-delivery"},{"category":"production","checkGroupId":"order-delivery","criterionId":"305a","description":"a cancelled order remains cancelled after the delivery interval","executionId":"selected-source-054","featureId":305,"packId":"ecommerce.l3.order-delivery-features","points":2,"role":"feature","source":"scenarios/03-order-delivery.json","stableKey":"ecommerce.l3.order-delivery.order-delivery.305a","stablePackId":"ecommerce.l3.order-delivery"},{"category":"production","checkGroupId":"reservations","criterionId":"301a","description":"adding an item reserves one unit for every open viewer","executionId":"selected-source-055","featureId":301,"packId":"ecommerce.l3.reservations-features","points":2,"role":"feature","source":"scenarios/03-reservations.json","stableKey":"ecommerce.l3.reservations.reservations.301a","stablePackId":"ecommerce.l3.reservations"},{"category":"interface","checkGroupId":"reservations","criterionId":"305a","description":"the reservation timer decreases","executionId":"selected-source-055","featureId":305,"packId":"ecommerce.l3.reservations-features","points":1,"role":"feature","source":"scenarios/03-reservations.json","stableKey":"ecommerce.l3.reservations.reservations.305a","stablePackId":"ecommerce.l3.reservations"},{"category":"feature","checkGroupId":"reservations","criterionId":"306a","description":"checkout converts the reservation into an order and empties the cart","executionId":"selected-source-055","featureId":306,"packId":"ecommerce.l3.reservations-features","points":2,"role":"feature","source":"scenarios/03-reservations.json","stableKey":"ecommerce.l3.reservations.reservations.306a","stablePackId":"ecommerce.l3.reservations"},{"category":"feature","checkGroupId":"reservations","criterionId":"307a","description":"an expired reservation marks its cart line","executionId":"selected-source-055","featureId":307,"packId":"ecommerce.l3.reservations-features","points":3,"role":"feature","source":"scenarios/03-reservations.json","stableKey":"ecommerce.l3.reservations.reservations.307a","stablePackId":"ecommerce.l3.reservations"},{"category":"feature","checkGroupId":"reservations","criterionId":"308a","description":"raising quantity starts a new reservation window","executionId":"selected-source-055","featureId":308,"packId":"ecommerce.l3.reservations-features","points":2,"role":"feature","source":"scenarios/03-reservations.json","stableKey":"ecommerce.l3.reservations.reservations.308a","stablePackId":"ecommerce.l3.reservations"},{"category":"feature","checkGroupId":"scheduled-restocks","criterionId":"305a","description":"a due restock updates stock and moves to the ledger","executionId":"selected-source-056","featureId":305,"packId":"ecommerce.l3.scheduled-restocks-features","points":3,"role":"feature","source":"scenarios/03-scheduled-restock-apply.json","stableKey":"ecommerce.l3.scheduled-restocks.scheduled-restocks.305a","stablePackId":"ecommerce.l3.scheduled-restocks"},{"category":"production","checkGroupId":"scheduled-restocks","criterionId":"306a","description":"a cancelled restock never applies","executionId":"selected-source-057","featureId":306,"packId":"ecommerce.l3.scheduled-restocks-features","points":2,"role":"feature","source":"scenarios/03-scheduled-restock-cancel.json","stableKey":"ecommerce.l3.scheduled-restocks.scheduled-restocks.306a","stablePackId":"ecommerce.l3.scheduled-restocks"},{"category":"feature","checkGroupId":"scheduled-restocks","criterionId":"302a","description":"a scheduled restock is pending and its remaining time decreases","executionId":"selected-source-058","featureId":302,"packId":"ecommerce.l3.scheduled-restocks-features","points":2,"role":"feature","source":"scenarios/03-scheduled-restocks.json","stableKey":"ecommerce.l3.scheduled-restocks.scheduled-restocks.302a","stablePackId":"ecommerce.l3.scheduled-restocks"},{"category":"production","checkGroupId":"server-time","criterionId":"312a","description":"restart preserves the due time and the work later completes","executionId":"selected-source-059","featureId":312,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.server-time-specifications","points":3,"requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"role":"guarantee","source":"scenarios/03-server-time.json","stableKey":"ecommerce.l3.server-time.server-time.312a","stablePackId":"ecommerce.l3.server-time"},{"category":"production","checkGroupId":"server-time","criterionId":"313a","description":"a reservation expires while its browser is closed","executionId":"selected-source-059","featureId":313,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.server-time-specifications","points":3,"requiresFeatures":["ecommerce.l3.reservations-features"],"role":"guarantee","source":"scenarios/03-server-time.json","stableKey":"ecommerce.l3.server-time.server-time.313a","stablePackId":"ecommerce.l3.server-time"},{"category":"production","checkGroupId":"account-state-recovery","criterionId":"105b","description":"the same account and cart survive the connection dropping and coming back","executionId":"selected-source-060","featureId":105,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":1,"requiresFeatures":["ecommerce.feature.accounts","ecommerce.feature.checkout"],"role":"guarantee","source":"scenarios/progression-account-state-reconnect.json","stableKey":"ecommerce.spec.state-durability.account-state-recovery.105b"},{"category":"production","checkGroupId":"account-state-recovery","criterionId":"105a","description":"cart and order history survive reload and backend restart, including a fresh account login","executionId":"selected-source-061","featureId":105,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":1,"requiresFeatures":["ecommerce.feature.accounts","ecommerce.feature.checkout"],"role":"guarantee","source":"scenarios/progression-account-state-reload.json","stableKey":"ecommerce.spec.state-durability.account-state-recovery.105a"},{"category":"production","checkGroupId":"automatic-reorder-access","criterionId":"502c","description":"a customer cannot see or replay automatic reorder management","executionId":"selected-source-062","featureId":502,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.progression.automatic-reorder"],"role":"guarantee","source":"scenarios/progression-automatic-reorder-access.json","stableKey":"ecommerce.spec.access-control.automatic-reorder-access.502c"},{"category":"production","checkGroupId":"automatic-reorder-deduplication","criterionId":"502b","description":"more sales do not duplicate a pending restock","executionId":"selected-source-063","featureId":502,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":2,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.automatic-reorder"],"role":"guarantee","source":"scenarios/progression-automatic-reorder-duplicate.json","stableKey":"ecommerce.spec.transactional-integrity.automatic-reorder-deduplication.502b"},{"category":"feature","checkGroupId":"automatic-reorder","criterionId":"502a","description":"crossing the threshold creates one pending restock","executionId":"selected-source-064","featureId":502,"packId":"ecommerce.progression.automatic-reorder","points":3,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"feature","source":"scenarios/progression-automatic-reorder.json","stableKey":"ecommerce.progression.automatic-reorder.automatic-reorder.502a"},{"category":"production","checkGroupId":"books-balance","criterionId":"107a","description":"revenue rises by exactly what was bought","executionId":"selected-source-065","featureId":107,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":1,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/progression-books-balance.json","stableKey":"ecommerce.spec.transactional-integrity.books-balance.107a"},{"category":"production","checkGroupId":"books-balance","criterionId":"107b","description":"what the store sold is what left the warehouses, and a fresh client agrees","executionId":"selected-source-065","featureId":107,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":1,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/progression-books-balance.json","stableKey":"ecommerce.spec.transactional-integrity.books-balance.107b"},{"category":"feature","checkGroupId":"bundle-checkout","criterionId":"741a","description":"adding a bundle reserves its components and checkout records the bundle price once","executionId":"selected-source-066","featureId":741,"packId":"ecommerce.feature.bundle-checkout","points":2,"role":"feature","source":"scenarios/progression-bundle-checkout.json","stableKey":"ecommerce.feature.bundle-checkout.bundle-checkout.741a"},{"category":"production","checkGroupId":"bundle-744","criterionId":"744a","description":"two competing reservations accept exactly one whole bundle without consuming extra components","executionId":"selected-source-066","featureId":744,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","points":3,"requiresFeatures":["ecommerce.feature.bundle-checkout"],"role":"guarantee","source":"scenarios/progression-bundle-checkout.json","stableKey":"ecommerce.spec.bundle-integrity.bundle-744.744a"},{"category":"production","checkGroupId":"bundle-745","criterionId":"745a","description":"a missing component refuses the reservation without taking available stock or adding a cart line","executionId":"selected-source-066","featureId":745,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","points":3,"requiresFeatures":["ecommerce.feature.bundle-checkout"],"role":"guarantee","source":"scenarios/progression-bundle-checkout.json","stableKey":"ecommerce.spec.bundle-integrity.bundle-745.745a"},{"category":"production","checkGroupId":"bundle-746","criterionId":"746a","description":"an expired reservation releases each component once across a backend restart","executionId":"selected-source-066","featureId":746,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","points":3,"requiresFeatures":["ecommerce.feature.bundle-checkout"],"role":"guarantee","source":"scenarios/progression-bundle-checkout.json","stableKey":"ecommerce.spec.bundle-integrity.bundle-746.746a"},{"category":"production","checkGroupId":"bundle-747","criterionId":"747a","description":"two checkout requests consume one reservation and create one paid bundle","executionId":"selected-source-066","featureId":747,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","points":3,"requiresFeatures":["ecommerce.feature.bundle-checkout"],"role":"guarantee","source":"scenarios/progression-bundle-checkout.json","stableKey":"ecommerce.spec.bundle-integrity.bundle-747.747a"},{"category":"feature","checkGroupId":"bundle-returns","criterionId":"742a","description":"returning a shipped bundle refunds the paid price and restores original components after its definition changes","executionId":"selected-source-067","featureId":742,"packId":"ecommerce.feature.bundle-returns","points":2,"role":"feature","source":"scenarios/progression-bundle-returns.json","stableKey":"ecommerce.feature.bundle-returns.bundle-returns.742a"},{"category":"production","checkGroupId":"bundle-742","criterionId":"742b","description":"replaying a completed bundle return after restart does not refund or restock it twice","executionId":"selected-source-067","featureId":742,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","points":3,"requiresFeatures":["ecommerce.feature.bundle-returns"],"role":"guarantee","source":"scenarios/progression-bundle-returns.json","stableKey":"ecommerce.spec.bundle-integrity.bundle-742.742b"},{"category":"production","checkGroupId":"bundle-748","criterionId":"748a","description":"another customer cannot return a paid bundle by submitting its order ID","executionId":"selected-source-067","featureId":748,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","points":3,"requiresFeatures":["ecommerce.feature.bundle-returns"],"role":"guarantee","source":"scenarios/progression-bundle-returns.json","stableKey":"ecommerce.spec.bundle-integrity.bundle-748.748a"},{"category":"feature","checkGroupId":"cart","criterionId":"4a","description":"adding the same item twice raises its quantity instead of adding a second line","executionId":"selected-source-068","featureId":4,"packId":"ecommerce.feature.cart","points":1,"role":"feature","source":"scenarios/progression-cart-checkout.json","stableKey":"ecommerce.feature.cart-checkout.cart.4a","stablePackId":"ecommerce.feature.cart-checkout"},{"category":"feature","checkGroupId":"cart","criterionId":"4d","description":"checkout creates one order, reduces stock, and empties the cart","executionId":"selected-source-068","featureId":4,"packId":"ecommerce.feature.checkout","points":2,"role":"feature","source":"scenarios/progression-cart-checkout.json","stableKey":"ecommerce.feature.cart-checkout.cart.4d","stablePackId":"ecommerce.feature.cart-checkout"},{"category":"feature","checkGroupId":"cart-recovery","criterionId":"503a","description":"restoring an expired cart reserves available items again","executionId":"selected-source-069","featureId":503,"packId":"ecommerce.progression.cart-recovery","points":3,"role":"feature","source":"scenarios/progression-cart-recovery.json","stableKey":"ecommerce.progression.cart-recovery.cart-recovery.503a"},{"category":"feature","checkGroupId":"cart-recovery","criterionId":"503b","description":"a partial restore keeps available items and names each unavailable item","executionId":"selected-source-069","featureId":503,"packId":"ecommerce.progression.cart-recovery","points":3,"role":"feature","source":"scenarios/progression-cart-recovery.json","stableKey":"ecommerce.progression.cart-recovery.cart-recovery.503b"},{"category":"feature","checkGroupId":"catalog-management","criterionId":"622a","description":"a new product reaches the public catalog","executionId":"selected-source-070","featureId":622,"packId":"ecommerce.progression.catalog-management","points":2,"role":"feature","source":"scenarios/progression-catalog-management.json","stableKey":"ecommerce.progression.catalog-management.catalog-management.622a"},{"category":"feature","checkGroupId":"catalog-management","criterionId":"622b","description":"the product exposes its named variants","executionId":"selected-source-070","featureId":622,"packId":"ecommerce.progression.catalog-management","points":2,"role":"feature","source":"scenarios/progression-catalog-management.json","stableKey":"ecommerce.progression.catalog-management.catalog-management.622b"},{"category":"production","checkGroupId":"checkout-crash-integrity","criterionId":"910a","description":"interrupted checkout recovers to the prepared cart or one complete order with the cart cleared after each independent process crash","executionId":"selected-source-071","featureId":910,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":1,"requiresFeatures":["ecommerce.feature.checkout"],"role":"guarantee","source":"scenarios/progression-checkout-crash.json","stableKey":"ecommerce.spec.state-durability.checkout-crash-integrity.910a"},{"category":"production","checkGroupId":"checkout-crash-durability","criterionId":"910b","description":"acknowledged checkout is not rolled back and earlier orders remain unchanged after each independent process crash","executionId":"selected-source-071","featureId":910,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":1,"requiresFeatures":["ecommerce.feature.checkout"],"role":"guarantee","source":"scenarios/progression-checkout-crash.json","stableKey":"ecommerce.spec.state-durability.checkout-crash-durability.910b"},{"category":"production","checkGroupId":"payment-records","criterionId":"623a","description":"checkout records the exact paid amount","executionId":"selected-source-072","featureId":623,"packId":"ecommerce.progression.payment-records","points":3,"role":"feature","source":"scenarios/progression-core-business.json","stableKey":"ecommerce.progression.payment-records.payment-records.623a"},{"category":"production","checkGroupId":"payment-deduplication","criterionId":"623b","description":"one checkout has one payment record","executionId":"selected-source-072","featureId":623,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":2,"requiresFeatures":["ecommerce.progression.payment-records"],"role":"guarantee","source":"scenarios/progression-core-business.json","stableKey":"ecommerce.spec.transactional-integrity.payment-deduplication.623b"},{"category":"feature","checkGroupId":"customer-profile","criterionId":"620c","description":"the owner can save and view a customer profile","executionId":"selected-source-073","featureId":620,"packId":"ecommerce.progression.customer-profile","points":1,"role":"feature","source":"scenarios/progression-customer-profile.json","stableKey":"ecommerce.progression.customer-profile.customer-profile.620c"},{"category":"production","checkGroupId":"customer-profile-reload","criterionId":"620a","description":"the saved profile survives reload and backend restart in a fresh browser","executionId":"selected-source-073","featureId":620,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":2,"requiresFeatures":["ecommerce.progression.customer-profile"],"role":"guarantee","source":"scenarios/progression-customer-profile.json","stableKey":"ecommerce.spec.state-durability.customer-profile-reload.620a"},{"category":"production","checkGroupId":"customer-profile-privacy","criterionId":"620b","description":"another customer neither sees nor receives the owner's private address","executionId":"selected-source-073","featureId":620,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.progression.customer-profile"],"role":"guarantee","source":"scenarios/progression-customer-profile.json","stableKey":"ecommerce.spec.access-control.customer-profile-privacy.620b"},{"category":"feature","checkGroupId":"delivery-notification-delivery","criterionId":"501a","description":"the order owner receives one delivery notification","executionId":"selected-source-074","featureId":501,"packId":"ecommerce.progression.delivery-notifications","points":3,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"feature","source":"scenarios/progression-delivery-notifications.json","stableKey":"ecommerce.progression.delivery-notifications.delivery-notification-delivery.501a"},{"category":"production","checkGroupId":"delivery-notification-privacy","criterionId":"501b","description":"another customer cannot see the delivery notification","executionId":"selected-source-074","featureId":501,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.delivery-notifications","ecommerce.progression.fulfilment-queue"],"role":"guarantee","source":"scenarios/progression-delivery-notifications.json","stableKey":"ecommerce.spec.access-control.delivery-notification-privacy.501b"},{"category":"feature","checkGroupId":"faceted-search","criterionId":"401a","description":"category, price, and availability filters apply together","executionId":"selected-source-075","featureId":401,"packId":"ecommerce.progression.faceted-search","points":3,"role":"feature","source":"scenarios/progression-faceted-filters.json","stableKey":"ecommerce.progression.faceted-search.faceted-search.401a"},{"category":"feature","checkGroupId":"faceted-search","criterionId":"402a","description":"moving between pages returns the same ordered items without duplicates","executionId":"selected-source-076","featureId":402,"packId":"ecommerce.progression.faceted-search","points":3,"role":"feature","source":"scenarios/progression-faceted-pagination.json","stableKey":"ecommerce.progression.faceted-search.faceted-search.402a"},{"category":"production","checkGroupId":"managed-support-privacy","criterionId":"613b","description":"another customer cannot read or reply to the managed case","executionId":"selected-source-077","featureId":613,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"requiresFeatures":["ecommerce.progression.managed-support"],"role":"guarantee","source":"scenarios/progression-managed-support-privacy.json","stableKey":"ecommerce.spec.access-control.managed-support-privacy.613b"},{"category":"feature","checkGroupId":"managed-support","criterionId":"613c","description":"staff can update a support case and the customer can reply","executionId":"selected-source-078","featureId":613,"packId":"ecommerce.progression.managed-support","points":1,"role":"feature","source":"scenarios/progression-managed-support-shared.json","stableKey":"ecommerce.progression.managed-support.managed-support.613c"},{"category":"production","checkGroupId":"managed-support","criterionId":"613a","description":"the customer and staff see the same replies and status live","executionId":"selected-source-078","featureId":613,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":2,"requiresFeatures":["ecommerce.progression.managed-support"],"role":"guarantee","source":"scenarios/progression-managed-support-shared.json","stableKey":"ecommerce.spec.live-state.managed-support.613a"},{"category":"feature","checkGroupId":"notification-preferences","criterionId":"630c","description":"the customer can save a notification choice","executionId":"selected-source-079","featureId":630,"packId":"ecommerce.progression.notification-preferences","points":1,"role":"feature","source":"scenarios/progression-notification-preferences.json","stableKey":"ecommerce.progression.notification-preferences.notification-preferences.630c"},{"category":"production","checkGroupId":"notification-preferences-reload","criterionId":"630a","description":"notification choices survive reload and backend restart in a fresh browser","executionId":"selected-source-079","featureId":630,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":2,"requiresFeatures":["ecommerce.progression.notification-preferences"],"role":"guarantee","source":"scenarios/progression-notification-preferences.json","stableKey":"ecommerce.spec.state-durability.notification-preferences-reload.630a"},{"category":"production","checkGroupId":"notification-preferences-privacy","criterionId":"630b","description":"the owner's choice does not change another account","executionId":"selected-source-079","featureId":630,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.progression.notification-preferences"],"role":"guarantee","source":"scenarios/progression-notification-preferences.json","stableKey":"ecommerce.spec.access-control.notification-preferences-privacy.630b"},{"category":"production","checkGroupId":"open-list","criterionId":"902a","description":"one customer has the Keyboard's reviews open before another posts one; the already-open view shows that review exactly once","executionId":"selected-source-080","featureId":902,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":1,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","source":"scenarios/progression-open-list-live.json","stableKey":"ecommerce.spec.live-state.open-list.902a"},{"category":"interface","checkGroupId":"cancellation-and-return","criterionId":"3e","description":"a pending order does not offer a return button","executionId":"selected-source-081","featureId":331,"packId":"ecommerce.l3.order-returns-features","points":1,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"feature","source":"scenarios/progression-order-return-boundary.json","stableKey":"ecommerce.returns-pricing.cancellation-and-return.3e","stablePackId":"ecommerce.returns-pricing"},{"category":"production","checkGroupId":"cancellation-and-return","criterionId":"3f","description":"the server refuses a pending return without changing stock or revenue","executionId":"selected-source-081","featureId":332,"packId":"ecommerce.l3.order-returns-features","points":2,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"feature","source":"scenarios/progression-order-return-boundary.json","stableKey":"ecommerce.returns-pricing.cancellation-and-return.3f","stablePackId":"ecommerce.returns-pricing"},{"category":"production","checkGroupId":"cancellation-and-return","criterionId":"3c","description":"returning a shipped item restores stock and revenue and marks the item returned","executionId":"selected-source-082","featureId":330,"packId":"ecommerce.l3.order-returns-features","points":3,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"feature","source":"scenarios/progression-order-return-complete.json","stableKey":"ecommerce.returns-pricing.cancellation-and-return.3c","stablePackId":"ecommerce.returns-pricing"},{"category":"production","checkGroupId":"order-support-ownership","criterionId":"614b","description":"another customer cannot attach or inspect the owner's order","executionId":"selected-source-083","featureId":614,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.progression.order-support"],"role":"guarantee","source":"scenarios/progression-order-support-boundary.json","stableKey":"ecommerce.spec.access-control.order-support-ownership.614b"},{"category":"feature","checkGroupId":"order-support-owned","criterionId":"614a","description":"the customer can link their order and staff can inspect it","executionId":"selected-source-084","featureId":614,"packId":"ecommerce.progression.order-support","points":3,"role":"feature","source":"scenarios/progression-order-support-owned.json","stableKey":"ecommerce.progression.order-support.order-support-owned.614a"},{"category":"feature","checkGroupId":"personalized-recommendations","criterionId":"403a","description":"recommendations follow the customer's categories, global sales, and name tie-break","executionId":"selected-source-085","featureId":403,"packId":"ecommerce.progression.personalized-recommendations","points":4,"role":"feature","source":"scenarios/progression-personalized-recommendations.json","stableKey":"ecommerce.progression.personalized-recommendations.personalized-recommendations.403a"},{"category":"production","checkGroupId":"recommendation-profile-isolation","criterionId":"403b","description":"one customer's activity does not replace another customer's recommendations","executionId":"selected-source-085","featureId":403,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.progression.personalized-recommendations"],"role":"guarantee","source":"scenarios/progression-personalized-recommendations.json","stableKey":"ecommerce.spec.access-control.recommendation-profile-isolation.403b"},{"category":"production","checkGroupId":"price-history","criterionId":"4c","description":"a price change updates an open cart and direct checkout persists the new total","executionId":"selected-source-086","featureId":420,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.price-history-specifications","points":3,"requiresFeatures":["ecommerce.feature.checkout","ecommerce.l2.price-history-features"],"role":"guarantee","source":"scenarios/progression-price-cart-checkout.json","stableKey":"ecommerce.returns-pricing.price-history.4c","stablePackId":"ecommerce.returns-pricing"},{"category":"feature","checkGroupId":"product-bundles","criterionId":"740a","description":"a saved bundle shows the exact price and component quantities after reopening the application","executionId":"selected-source-087","featureId":740,"packId":"ecommerce.feature.product-bundles","points":2,"role":"feature","source":"scenarios/progression-product-bundles.json","stableKey":"ecommerce.feature.product-bundles.product-bundles.740a"},{"category":"production","checkGroupId":"bundle-743","criterionId":"743a","description":"a customer cannot replace a staff-created bundle through the application write","executionId":"selected-source-087","featureId":743,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","points":3,"requiresFeatures":["ecommerce.feature.product-bundles"],"role":"guarantee","source":"scenarios/progression-product-bundles.json","stableKey":"ecommerce.spec.bundle-integrity.bundle-743.743a"},{"category":"feature","checkGroupId":"promotion-checkout-active","criterionId":"621a","description":"an active promotion changes checkout and is recorded on the order","executionId":"selected-source-088","featureId":621,"packId":"ecommerce.progression.promotion-checkout","points":3,"role":"feature","source":"scenarios/progression-promotion-checkout.json","stableKey":"ecommerce.progression.promotion-checkout.promotion-checkout-active.621a"},{"category":"feature","checkGroupId":"promotion-checkout-expired","criterionId":"621b","description":"an expired promotion is refused","executionId":"selected-source-088","featureId":621,"packId":"ecommerce.progression.promotion-checkout","points":2,"role":"feature","source":"scenarios/progression-promotion-checkout.json","stableKey":"ecommerce.progression.promotion-checkout.promotion-checkout-expired.621b"},{"category":"feature","checkGroupId":"promotion-checkout-exhausted","criterionId":"621c","description":"a fully redeemed promotion is refused","executionId":"selected-source-088","featureId":621,"packId":"ecommerce.progression.promotion-checkout","points":2,"role":"feature","source":"scenarios/progression-promotion-checkout.json","stableKey":"ecommerce.progression.promotion-checkout.promotion-checkout-exhausted.621c"},{"category":"feature","checkGroupId":"promotion-report-redemptions","criterionId":"622a","description":"the promotion report has the exact redemption count","executionId":"selected-source-089","featureId":622,"packId":"ecommerce.progression.promotion-reporting","points":1,"role":"feature","source":"scenarios/progression-promotion-reporting.json","stableKey":"ecommerce.progression.promotion-reporting.promotion-report-redemptions.622a"},{"category":"feature","checkGroupId":"promotion-report-revenue","criterionId":"622b","description":"the promotion report has the exact discounted revenue","executionId":"selected-source-089","featureId":622,"packId":"ecommerce.progression.promotion-reporting","points":2,"role":"feature","source":"scenarios/progression-promotion-reporting.json","stableKey":"ecommerce.progression.promotion-reporting.promotion-report-revenue.622b"},{"category":"feature","checkGroupId":"promotion-rule-values","criterionId":"620a","description":"staff can save every bounded promotion value","executionId":"selected-source-090","featureId":620,"packId":"ecommerce.progression.promotion-rules","points":2,"role":"feature","source":"scenarios/progression-promotion-rules.json","stableKey":"ecommerce.progression.promotion-rules.promotion-rule-values.620a"},{"category":"production","checkGroupId":"promotion-management-boundary","criterionId":"620b","description":"customers cannot open promotion management","executionId":"selected-source-090","featureId":620,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.progression.promotion-rules"],"role":"guarantee","source":"scenarios/progression-promotion-rules.json","stableKey":"ecommerce.spec.access-control.promotion-management-boundary.620b"},{"category":"feature","checkGroupId":"purchase-order","criterionId":"3c","description":"the purchase is recorded in the buyer's order history at the price paid","executionId":"selected-source-091","featureId":3,"packId":"ecommerce.feature.purchasing","points":1,"role":"feature","source":"scenarios/progression-purchasing.json","stableKey":"ecommerce.feature.purchasing.purchase-order.3c"},{"category":"feature","checkGroupId":"recommendation-feedback","criterionId":"504a","description":"dismissing a recommendation removes it from the customer view","executionId":"selected-source-092","featureId":504,"packId":"ecommerce.progression.recommendation-feedback","points":2,"role":"feature","source":"scenarios/progression-recommendation-feedback.json","stableKey":"ecommerce.progression.recommendation-feedback.recommendation-feedback.504a"},{"category":"production","checkGroupId":"recommendation-feedback-privacy","criterionId":"504b","description":"one customer's dismissal does not hide another customer's result","executionId":"selected-source-092","featureId":504,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.progression.recommendation-feedback"],"role":"guarantee","source":"scenarios/progression-recommendation-feedback.json","stableKey":"ecommerce.spec.access-control.recommendation-feedback-privacy.504b"},{"category":"production","checkGroupId":"recommendation-feedback-restart","criterionId":"504c","description":"a dismissed recommendation stays absent after reload and backend restart in a fresh browser","executionId":"selected-source-092","featureId":504,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":2,"requiresFeatures":["ecommerce.progression.recommendation-feedback"],"role":"guarantee","source":"scenarios/progression-recommendation-feedback.json","stableKey":"ecommerce.spec.state-durability.recommendation-feedback-restart.504c"},{"category":"production","checkGroupId":"review-eligibility-direct","criterionId":"618a","description":"the server refuses a review from a customer who did not buy the item","executionId":"selected-source-093","featureId":618,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.review-access-specifications","points":2,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","source":"scenarios/progression-review-access.json","stableKey":"ecommerce.progression.review-access-specifications.review-eligibility-direct.618a"},{"category":"production","checkGroupId":"stored-review-script","criterionId":"9180a","description":"review input is rejected or displayed without executing script in another customer session","executionId":"selected-source-094","featureId":9180,"observations":["unmentioned"],"packId":"ecommerce.progression.review-access-specifications","points":1,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","source":"scenarios/progression-review-script.json","stableKey":"ecommerce.progression.review-access-specifications.stored-review-script.9180a"},{"category":"production","checkGroupId":"search-ordering","criterionId":"402b","description":"active filters and text searches remain alphabetical after purchases; clearing them restores purchase ranking","executionId":"selected-source-095","featureId":402,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.search-ordering","points":1,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.faceted-search"],"role":"guarantee","source":"scenarios/progression-search-ordering.json","stableKey":"ecommerce.spec.search-ordering.search-ordering.402b"},{"category":"production","checkGroupId":"shipping-accounting","criterionId":"202e","description":"shipping a purchased order does not deduct stock or add revenue again","executionId":"selected-source-096","featureId":202,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","points":2,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin","ecommerce.progression.fulfilment-queue"],"role":"guarantee","source":"scenarios/progression-shipping-accounting.json","stableKey":"ecommerce.inventory-operations.shipping-accounting.202e","stablePackId":"ecommerce.inventory-operations"},{"category":"production","checkGroupId":"signed-out-purchase","criterionId":"3a","description":"using the purchase control while signed out does not buy an item","executionId":"selected-source-097","featureId":3,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/progression-signed-out-purchase.json","stableKey":"ecommerce.spec.access-control.signed-out-purchase.3a"},{"category":"feature","checkGroupId":"split-tender-refunds-751","criterionId":"751a","description":"Full refund restores each original payment portion","executionId":"selected-source-098","featureId":751,"packId":"ecommerce.feature.split-tender-refunds","points":2,"requiresFeatures":["ecommerce.feature.split-tender-refunds","ecommerce.feature.store-credit","ecommerce.progression.support-refunds"],"role":"feature","source":"scenarios/progression-split-tender-refunds.json","stableKey":"ecommerce.feature.split-tender-refunds.split-tender-refunds-751.751a"},{"category":"production","checkGroupId":"production-756","criterionId":"756a","description":"Concurrent refunds restore the original credit and external amounts once, including after restart","executionId":"selected-source-098","featureId":756,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.split-tender-refunds","points":2,"requiresFeatures":["ecommerce.feature.split-tender-refunds","ecommerce.feature.store-credit","ecommerce.progression.support-refunds"],"role":"guarantee","source":"scenarios/progression-split-tender-refunds.json","stableKey":"ecommerce.spec.split-tender-refunds.production-756.756a"},{"category":"feature","checkGroupId":"staff-access","criterionId":"601a","description":"staff and administrators can sign in and open staff tools","executionId":"selected-source-099","featureId":601,"packId":"ecommerce.progression.staff-access","points":2,"role":"feature","source":"scenarios/progression-staff-access.json","stableKey":"ecommerce.progression.staff-access.staff-access.601a"},{"category":"production","checkGroupId":"staff-area-boundary","criterionId":"601b","description":"customers cannot open staff tools","executionId":"selected-source-099","featureId":601,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.progression.staff-access"],"role":"guarantee","source":"scenarios/progression-staff-access.json","stableKey":"ecommerce.spec.access-control.staff-area-boundary.601b"},{"category":"feature","checkGroupId":"staff-activity","criterionId":"624a","description":"an administrative change records its actor, action, subject, and time","executionId":"selected-source-100","featureId":624,"packId":"ecommerce.progression.staff-activity","points":3,"role":"feature","source":"scenarios/progression-staff-activity.json","stableKey":"ecommerce.progression.staff-activity.staff-activity.624a"},{"category":"production","checkGroupId":"staff-activity-privacy","criterionId":"624b","description":"customers cannot open staff activity history","executionId":"selected-source-100","featureId":624,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.progression.staff-activity"],"role":"guarantee","source":"scenarios/progression-staff-activity.json","stableKey":"ecommerce.spec.access-control.staff-activity-privacy.624b"},{"category":"feature","checkGroupId":"staff-roles","criterionId":"621c","description":"an administrator can assign a staff role","executionId":"selected-source-101","featureId":621,"packId":"ecommerce.progression.staff-roles","points":1,"role":"feature","source":"scenarios/progression-staff-roles.json","stableKey":"ecommerce.progression.staff-roles.staff-roles.621c"},{"category":"production","checkGroupId":"staff-role-reload","criterionId":"621a","description":"an assigned staff role survives reload and backend restart in a fresh browser","executionId":"selected-source-101","featureId":621,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":2,"requiresFeatures":["ecommerce.progression.staff-roles"],"role":"guarantee","source":"scenarios/progression-staff-roles.json","stableKey":"ecommerce.spec.state-durability.staff-role-reload.621a"},{"category":"production","checkGroupId":"staff-role-boundary","criterionId":"621b","description":"a staff member cannot assign roles through the UI or a replayed request","executionId":"selected-source-101","featureId":621,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.progression.staff-roles"],"role":"guarantee","source":"scenarios/progression-staff-roles.json","stableKey":"ecommerce.spec.access-control.staff-role-boundary.621b"},{"category":"production","checkGroupId":"staff-role-revocation","criterionId":"621d","description":"removing administrator access blocks a previously authorized session without changing the target role","executionId":"selected-source-101","featureId":621,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.progression.staff-roles"],"role":"guarantee","source":"scenarios/progression-staff-roles.json","stableKey":"ecommerce.spec.access-control.staff-role-revocation.621d"},{"category":"feature","checkGroupId":"stock-alert-delivery","criterionId":"631c","description":"restored stock sends the requested alert","executionId":"selected-source-102","featureId":631,"packId":"ecommerce.progression.stock-alerts","points":1,"requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"feature","source":"scenarios/progression-stock-alert-delivery.json","stableKey":"ecommerce.progression.stock-alerts.stock-alert-delivery.631c"},{"category":"production","checkGroupId":"stock-alert-deduplication","criterionId":"631a","description":"restored stock sends one alert and later restocks do not duplicate it","executionId":"selected-source-103","featureId":631,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":2,"requiresFeatures":["ecommerce.feature.warehouse-admin","ecommerce.progression.stock-alerts"],"role":"guarantee","source":"scenarios/progression-stock-alerts.json","stableKey":"ecommerce.spec.transactional-integrity.stock-alert-deduplication.631a"},{"category":"production","checkGroupId":"stock-alert-privacy","criterionId":"631b","description":"a customer who did not request the alert cannot see it","executionId":"selected-source-103","featureId":631,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"requiresFeatures":["ecommerce.feature.warehouse-admin","ecommerce.progression.stock-alerts"],"role":"guarantee","source":"scenarios/progression-stock-alerts.json","stableKey":"ecommerce.spec.access-control.stock-alert-privacy.631b"},{"category":"production","checkGroupId":"stock-limit","criterionId":"3d","description":"an item sells out visibly, and a further purchase is refused without changing stock","executionId":"selected-source-104","featureId":3,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":2,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/progression-stock-limit.json","stableKey":"ecommerce.spec.concurrency-safety.stock-limit.3d"},{"category":"feature","checkGroupId":"store-credit-750","criterionId":"750a","description":"Credit checkout records both payment portions","executionId":"selected-source-105","featureId":750,"packId":"ecommerce.feature.store-credit","points":2,"role":"feature","source":"scenarios/progression-store-credit.json","stableKey":"ecommerce.feature.store-credit.store-credit-750.750a"},{"category":"production","checkGroupId":"production-752","criterionId":"752a","description":"Repeating a grant reference does not increase the balance twice","executionId":"selected-source-105","featureId":752,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.store-credit","points":2,"requiresFeatures":["ecommerce.feature.store-credit"],"role":"guarantee","source":"scenarios/progression-store-credit.json","stableKey":"ecommerce.spec.store-credit.production-752.752a"},{"category":"production","checkGroupId":"production-753","criterionId":"753a","description":"A customer cannot grant credit","executionId":"selected-source-105","featureId":753,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.store-credit","points":2,"requiresFeatures":["ecommerce.feature.store-credit"],"role":"guarantee","source":"scenarios/progression-store-credit.json","stableKey":"ecommerce.spec.store-credit.production-753.753a"},{"category":"production","checkGroupId":"production-754","criterionId":"754a","description":"Concurrent checkout consumes one cart and one credit allocation","executionId":"selected-source-105","featureId":754,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.store-credit","points":2,"requiresFeatures":["ecommerce.feature.store-credit"],"role":"guarantee","source":"scenarios/progression-store-credit.json","stableKey":"ecommerce.spec.store-credit.production-754.754a"},{"category":"production","checkGroupId":"production-755","criterionId":"755a","description":"Issued credit survives a backend restart","executionId":"selected-source-105","featureId":755,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.store-credit","points":2,"requiresFeatures":["ecommerce.feature.store-credit"],"role":"guarantee","source":"scenarios/progression-store-credit.json","stableKey":"ecommerce.spec.store-credit.production-755.755a"},{"category":"feature","checkGroupId":"subscriptions-760","criterionId":"760a","description":"A subscription creates exactly its requested deliveries and payments","executionId":"selected-source-106","featureId":760,"packId":"ecommerce.feature.subscriptions","points":2,"role":"feature","source":"scenarios/progression-subscriptions.json","stableKey":"ecommerce.feature.subscriptions.subscriptions-760.760a"},{"category":"production","checkGroupId":"production-761","criterionId":"761a","description":"A pending subscription continues after backend restart without duplicate deliveries","executionId":"selected-source-106","featureId":761,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.subscriptions","points":2,"requiresFeatures":["ecommerce.feature.subscriptions"],"role":"guarantee","source":"scenarios/progression-subscriptions.json","stableKey":"ecommerce.spec.subscriptions.production-761.761a"},{"category":"production","checkGroupId":"production-762","criterionId":"762a","description":"Another customer cannot cancel an active subscription","executionId":"selected-source-106","featureId":762,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.subscriptions","points":2,"requiresFeatures":["ecommerce.feature.subscriptions"],"role":"guarantee","source":"scenarios/progression-subscriptions.json","stableKey":"ecommerce.spec.subscriptions.production-762.762a"},{"category":"production","checkGroupId":"production-763","criterionId":"763a","description":"Pause survives a restart and resume completes the remaining deliveries","executionId":"selected-source-106","featureId":763,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.subscriptions","points":2,"requiresFeatures":["ecommerce.feature.subscriptions"],"role":"guarantee","source":"scenarios/progression-subscriptions.json","stableKey":"ecommerce.spec.subscriptions.production-763.763a"},{"category":"feature","checkGroupId":"support-history","criterionId":"612c","description":"the customer can view their support ticket history","executionId":"selected-source-107","featureId":612,"packId":"ecommerce.progression.support-history","points":1,"role":"feature","source":"scenarios/progression-support-history.json","stableKey":"ecommerce.progression.support-history.support-history.612c"},{"category":"production","checkGroupId":"support-history-reload","criterionId":"612a","description":"support history survives reload and backend restart in a fresh browser","executionId":"selected-source-107","featureId":612,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":2,"requiresFeatures":["ecommerce.progression.support-history"],"role":"guarantee","source":"scenarios/progression-support-history.json","stableKey":"ecommerce.spec.state-durability.support-history-reload.612a"},{"category":"production","checkGroupId":"support-history-privacy","criterionId":"612b","description":"another customer neither sees nor receives the private ticket","executionId":"selected-source-107","featureId":612,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.progression.support-history"],"role":"guarantee","source":"scenarios/progression-support-history.json","stableKey":"ecommerce.spec.access-control.support-history-privacy.612b"},{"category":"production","checkGroupId":"support-history-logout","criterionId":"612d","description":"after logout the same browser storage no longer grants access to private support history","executionId":"selected-source-107","featureId":612,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"requiresFeatures":["ecommerce.progression.support-history"],"role":"guarantee","source":"scenarios/progression-support-history.json","stableKey":"ecommerce.spec.access-control.support-history-logout.612d"},{"category":"feature","checkGroupId":"support-intake","criterionId":"610a","description":"a visitor can submit a support ticket and receives a reference","executionId":"selected-source-108","featureId":610,"packId":"ecommerce.progression.support-intake","points":2,"role":"feature","source":"scenarios/progression-support-intake.json","stableKey":"ecommerce.progression.support-intake.support-intake.610a"},{"category":"production","checkGroupId":"support-refund-access","criterionId":"615c","description":"a customer cannot issue a refund or change its records","executionId":"selected-source-109","featureId":615,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"requiresFeatures":["ecommerce.progression.support-refunds"],"role":"guarantee","source":"scenarios/progression-support-refunds-access.json","stableKey":"ecommerce.spec.access-control.support-refund-access.615c"},{"category":"production","checkGroupId":"support-refund-accounting","criterionId":"615b","description":"the refund equals the paid total, cannot be applied twice, and leaves another order unrefunded","executionId":"selected-source-110","featureId":615,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":1,"requiresFeatures":["ecommerce.progression.support-refunds"],"role":"guarantee","source":"scenarios/progression-support-refunds-accounting.json","stableKey":"ecommerce.spec.transactional-integrity.support-refund-accounting.615b"},{"category":"feature","checkGroupId":"support-refunds-resolution","criterionId":"615a","description":"an authorized refund resolves the case and updates the order","executionId":"selected-source-111","featureId":615,"packId":"ecommerce.progression.support-refunds","points":2,"role":"feature","source":"scenarios/progression-support-refunds-resolution.json","stableKey":"ecommerce.progression.support-refunds.support-refunds-resolution.615a"},{"category":"production","checkGroupId":"return-refund-interaction","criterionId":"757a","description":"a support refund followed by physical return restores each warehouse once and refunds only the price paid","executionId":"selected-source-112","featureId":757,"packId":"ecommerce.feature.split-tender-refunds","points":2,"requiresFeatures":["ecommerce.l3.order-returns-features","ecommerce.progression.fulfilment-queue","ecommerce.progression.support-refunds"],"role":"feature","source":"scenarios/progression-support-return-interaction.json","stableKey":"ecommerce.feature.split-tender-refunds.return-refund-interaction.757a"},{"category":"production","checkGroupId":"return-refund-interaction","criterionId":"757b","description":"a physical return followed by support refund restores each warehouse once and refunds only the price paid","executionId":"selected-source-112","featureId":757,"packId":"ecommerce.feature.split-tender-refunds","points":2,"requiresFeatures":["ecommerce.l3.order-returns-features","ecommerce.progression.fulfilment-queue","ecommerce.progression.support-refunds"],"role":"feature","source":"scenarios/progression-support-return-interaction.json","stableKey":"ecommerce.feature.split-tender-refunds.return-refund-interaction.757b"},{"category":"feature","checkGroupId":"support-assignment","criterionId":"611a","description":"staff can assign a new ticket","executionId":"selected-source-113","featureId":611,"packId":"ecommerce.progression.support-triage","points":1,"role":"feature","source":"scenarios/progression-support-triage.json","stableKey":"ecommerce.progression.support-triage.support-assignment.611a"},{"category":"feature","checkGroupId":"support-priority","criterionId":"611b","description":"staff can set a ticket priority","executionId":"selected-source-113","featureId":611,"packId":"ecommerce.progression.support-triage","points":1,"role":"feature","source":"scenarios/progression-support-triage.json","stableKey":"ecommerce.progression.support-triage.support-priority.611b"},{"category":"feature","checkGroupId":"support-status","criterionId":"611c","description":"staff can change a ticket status","executionId":"selected-source-113","featureId":611,"packId":"ecommerce.progression.support-triage","points":1,"role":"feature","source":"scenarios/progression-support-triage.json","stableKey":"ecommerce.progression.support-triage.support-status.611c"}],"components":{"fixture":{"id":"ecommerce.operations","path":"composition/fixtures/operations.json","sha256":"d06444b72dc94fe1ef5e08867d875e1f3bbaa5cd82c35a3558f399c1fcb5ceae"},"packs":[{"id":"ecommerce.feature.accounts","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/feature-accounts.json","requiresPacks":[],"sha256":"9eec7949c45ab6af816008b41245dd1e13e9164a6e10d42895eb93ad40664fbf"},{"id":"ecommerce.feature.bundle-checkout","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/feature-bundle-checkout.json","requiresPacks":["ecommerce.feature.product-bundles","ecommerce.l3.reservations-features"],"sha256":"2905f7ee7d8ae6d1086d2194649242b68ef871d10ec60e1bc0382baf5cb6e7df"},{"id":"ecommerce.feature.bundle-returns","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/feature-bundle-returns.json","requiresPacks":["ecommerce.feature.bundle-checkout","ecommerce.l3.order-returns-features"],"sha256":"c0f9424066d11efd12bf2c88ec3607ac1152ba5b1e34d7fe598529c03035e352"},{"id":"ecommerce.feature.cart","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/feature-cart.json","requiresPacks":["ecommerce.feature.accounts","ecommerce.feature.catalog-items"],"sha256":"217d2a7af92bc0551e17c12376b15668431619cc42f919f9d5a0a709f11eba78","stableId":"ecommerce.feature.cart-checkout"},{"id":"ecommerce.feature.catalog-discovery","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/feature-catalog-discovery.json","requiresPacks":["ecommerce.feature.catalog-items"],"sha256":"9aeea0db86cdacc386e5a26fed12b5604d674047c57c792e9aac3a5eabf14b42","stableId":"ecommerce.feature.catalog"},{"id":"ecommerce.feature.catalog-items","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/feature-catalog-items.json","requiresPacks":[],"sha256":"fce1929f7932d508f463793be463896fbb13ad6da2b7c57e0838188579f7a95f","stableId":"ecommerce.feature.catalog"},{"id":"ecommerce.feature.checkout","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/feature-checkout.json","requiresPacks":["ecommerce.feature.cart"],"sha256":"660aaf39db20b4fd6e405948b45f816c6733a2be37b8190b989a4d18c59eb214","stableId":"ecommerce.feature.cart-checkout"},{"id":"ecommerce.feature.product-bundles","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/feature-product-bundles.json","requiresPacks":["ecommerce.progression.catalog-management"],"sha256":"e768568827d4febae15fe2ec1d25410d86f78ada77debf9cb047d70fe3fdb12c"},{"id":"ecommerce.feature.purchasing","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/feature-purchasing.json","requiresPacks":["ecommerce.feature.accounts","ecommerce.feature.catalog-items"],"sha256":"d21746b5e7246b1771ee9f72d0cc66ef2c2cc9573485ffc839f6ca4b1b01ddff"},{"id":"ecommerce.feature.reviews","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/feature-reviews.json","requiresPacks":["ecommerce.feature.accounts","ecommerce.feature.catalog-items","ecommerce.feature.purchasing"],"sha256":"385771e4c68dbc7afbc0d68af897d698a66f744f50e4f580a813db580cf4615c"},{"id":"ecommerce.feature.split-tender-refunds","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/feature-split-tender-refunds.json","requiresPacks":["ecommerce.feature.store-credit","ecommerce.l3.order-returns-features","ecommerce.progression.support-refunds"],"sha256":"8d12162887644dc6bca1fbcccfd06ebd679bd8017b9234cc5632e6a1f508eba8"},{"id":"ecommerce.feature.store-credit","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/feature-store-credit.json","requiresPacks":["ecommerce.progression.payment-records","ecommerce.progression.staff-roles"],"sha256":"f252e74cb95d87a6a7cec629d59bc7a37cac133e6abee255738ff9a1fca50fcd"},{"id":"ecommerce.feature.subscriptions","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/feature-subscriptions.json","requiresPacks":["ecommerce.progression.payment-records"],"sha256":"31bbe7f11ac99af6eeeeadcd62c49bc635bb20384a6b5e3724b8ba9124530799"},{"id":"ecommerce.feature.warehouse-admin","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/feature-warehouse-admin.json","requiresPacks":["ecommerce.feature.catalog-items","ecommerce.progression.staff-access"],"sha256":"9230ca4ddfe048fd3903f3b90aee16a4711c6d53fbe37478917b16a6e550d66d"},{"id":"ecommerce.l2.inventory-dashboard","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/l2-inventory-dashboard.json","requiresPacks":["ecommerce.feature.warehouse-admin"],"sha256":"98d6ac14b32e61e976377df4687a13d6a780a0e96d2e49a05ce2a386e4588778","stableId":"ecommerce.inventory-operations"},{"id":"ecommerce.l2.order-cancellation-features","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/l2-order-cancellation-features.json","requiresPacks":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"sha256":"21f8663b36ea0871493903776ed32fae619183dc5fc0f74a8c1a5e8bc932f561","stableId":"ecommerce.returns-pricing"},{"id":"ecommerce.l2.price-history-features","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/l2-price-history-features.json","requiresPacks":["ecommerce.progression.catalog-management"],"sha256":"786b9f48b9670aa9cb02a439b7eda2feff11c5163a4d49939c892d604206e924","stableId":"ecommerce.returns-pricing"},{"id":"ecommerce.l2.recommendations","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/l2-recommendations.json","requiresPacks":["ecommerce.feature.cart","ecommerce.feature.purchasing"],"sha256":"a21e16ece7b0c4c6e2f0871428af4e4311ae5a91c46174a7ad644b6b93f2c18b","stableId":"ecommerce.inventory-operations"},{"id":"ecommerce.l2.sales-dashboard","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/l2-sales-dashboard.json","requiresPacks":["ecommerce.feature.purchasing"],"sha256":"5e9b1693dcea799edd672e276257a87ad6d7d88099d60be072e952e22e75ca77","stableId":"ecommerce.inventory-operations"},{"id":"ecommerce.l2.stock-transfers-features","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/l2-stock-transfers-features.json","requiresPacks":["ecommerce.feature.warehouse-admin"],"sha256":"fa9514e71b90d38623ede3aa4108cb8a53ca72ed95fc69858733ad38da7235ad","stableId":"ecommerce.inventory-operations"},{"id":"ecommerce.l3.cart-expiration-features","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/l3-cart-expiration-features.json","requiresPacks":["ecommerce.l3.reservations-features"],"sha256":"38029c5ea6d34974b155ef66c311dc500fcece1f82746cffa57ec5f0695f6a5d","stableId":"ecommerce.l3.cart-expiration"},{"id":"ecommerce.l3.deferred-access-specifications","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/l3-deferred-access-specifications.json","requiresPacks":[],"sha256":"cc47732a893439472253b8e209d653959966ca7be20ef0616ab24f8695096814","stableId":"ecommerce.l3.deferred-access"},{"id":"ecommerce.l3.deferred-durability-specifications","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/l3-deferred-durability-specifications.json","requiresPacks":[],"sha256":"9f5109ceb2a8f5e68df317910cea1911dfe79e8c42de51c83b60b760f4075d76","stableId":"ecommerce.l3.deferred-durability"},{"id":"ecommerce.l3.deferred-integrity-specifications","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/l3-deferred-integrity-specifications.json","requiresPacks":[],"sha256":"1a3d27f25be0b41f87afff3412b6f2c93617a68306309f471e0120d1ea359056","stableId":"ecommerce.l3.deferred-integrity"},{"id":"ecommerce.l3.order-delivery-features","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/l3-order-delivery-features.json","requiresPacks":["ecommerce.l2.order-cancellation-features","ecommerce.progression.fulfilment-queue"],"sha256":"7f86ba1b2a2791ba38ba6abeeee6078b061bdf03ea951647d79a74f6fe435b95","stableId":"ecommerce.l3.order-delivery"},{"id":"ecommerce.l3.order-returns-features","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/l3-order-returns-features.json","requiresPacks":["ecommerce.feature.warehouse-admin","ecommerce.l3.order-delivery-features"],"sha256":"cd5dc23d3c7514df91d0a54d92d97e7604e6828b1a287d0c79c714724cdc2206","stableId":"ecommerce.returns-pricing"},{"id":"ecommerce.l3.reservations-features","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/l3-reservations-features.json","requiresPacks":["ecommerce.feature.checkout","ecommerce.feature.warehouse-admin"],"sha256":"972b14948d5dea26cadab3567fc5bbd2c42d4feed5c428c004117b723acecd12","stableId":"ecommerce.l3.reservations"},{"id":"ecommerce.l3.scheduled-restocks-features","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/l3-scheduled-restocks-features.json","requiresPacks":["ecommerce.feature.warehouse-admin"],"sha256":"305c2cf78bc84c63b20cb3e5531760174355128332a365d741dbbd0d64c938cb","stableId":"ecommerce.l3.scheduled-restocks"},{"id":"ecommerce.l3.server-time-specifications","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/l3-server-time-specifications.json","requiresPacks":[],"sha256":"75ad830a0023bf1fe8022695f0776854d75cdbd87f5120a25312c7de33bf57c9","stableId":"ecommerce.l3.server-time"},{"id":"ecommerce.progression.automatic-reorder","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-automatic-reorder.json","requiresPacks":["ecommerce.feature.purchasing","ecommerce.l3.scheduled-restocks-features","ecommerce.progression.staff-roles"],"sha256":"a1d14fcb9cd6c9e96b71d2bacdd4865badaeda913ba9cb3cb5fab8f1eb8497e7"},{"id":"ecommerce.progression.cancellation-accounting-specifications","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/progression-cancellation-accounting-specifications.json","requiresPacks":[],"sha256":"29b290a1356e63a9d2e937c7f92ad540403b27ebcef94df15451d12e59ac86cc","stableId":"ecommerce.returns-pricing"},{"id":"ecommerce.progression.cancellation-queue-specifications","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/progression-cancellation-queue-specifications.json","requiresPacks":[],"sha256":"adf9f45a20d13a400a7812fe825a73de6035f980813c19e3698128e49085b58d","stableId":"ecommerce.returns-pricing"},{"id":"ecommerce.progression.cart-recovery","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-cart-recovery.json","requiresPacks":["ecommerce.l3.cart-expiration-features"],"sha256":"ca6d95040f5b81478546b9afedcc428114e795ced11b8364eb2542c258aec794"},{"id":"ecommerce.progression.catalog-management","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-catalog-management.json","requiresPacks":["ecommerce.feature.catalog-discovery","ecommerce.progression.staff-roles"],"sha256":"f12e7e37d031f470683ff74671b22c1de08eb8f7fe0e9cf03cbcfb4604dd2f06"},{"id":"ecommerce.progression.customer-profile","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-customer-profile.json","requiresPacks":["ecommerce.feature.accounts"],"sha256":"731eeced13c1053432560d4149d2de29fa2cd2b0051759d705cb36e92ad16bb3"},{"id":"ecommerce.progression.delivery-notifications","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-delivery-notifications.json","requiresPacks":["ecommerce.l3.order-delivery-features","ecommerce.progression.notification-preferences"],"sha256":"60b58c4327d74f710f94ffc0deae1b51ef40ca1d2146e45b8e02d1b28e1146bc"},{"id":"ecommerce.progression.faceted-search","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-faceted-search.json","requiresPacks":["ecommerce.feature.catalog-discovery"],"sha256":"5a967859348f967bdc43e0e2490289a859f568e4e8d809c21d5f4404d85e4dd7"},{"id":"ecommerce.progression.fulfilment-queue","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-fulfilment-queue.json","requiresPacks":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"sha256":"811eeb166f261e32b5338e118b0851aa1ffeb06b3107dafb6861057e11004db4","stableId":"ecommerce.operations-access"},{"id":"ecommerce.progression.inventory-conservation-specifications","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/progression-inventory-conservation-specifications.json","requiresPacks":[],"sha256":"b3bff8b33e6069bf3cfd2e68389bbd50f0ec9c981d766fd2d7717dcf892463dc","stableId":"ecommerce.inventory-operations"},{"id":"ecommerce.progression.managed-support","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-managed-support.json","requiresPacks":["ecommerce.progression.support-history","ecommerce.progression.support-triage"],"sha256":"6546f37be296605cad2e06bbe71d0cf014d1025a5a4b3b8997f4c38cf6329841"},{"id":"ecommerce.progression.notification-preferences","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-notification-preferences.json","requiresPacks":["ecommerce.feature.accounts"],"sha256":"e8423fe1a2a615a1f2b5bd7073e9cc7040c8d2e2eadc0ef466d50e5d21aaf134"},{"id":"ecommerce.progression.operations-access-specifications","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/progression-operations-access-specifications.json","requiresPacks":[],"sha256":"796806b8a647fd367f53f6246d51aa998b9aa9e2a8959356d237b6411be96937","stableId":"ecommerce.operations-access"},{"id":"ecommerce.progression.order-support","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-order-support.json","requiresPacks":["ecommerce.feature.purchasing","ecommerce.progression.managed-support"],"sha256":"1e1a1722adc4c72fdbf31aa34261b73975df53ffab27b28134d71b09ada1b27f"},{"id":"ecommerce.progression.payment-records","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-payment-records.json","requiresPacks":["ecommerce.feature.checkout","ecommerce.feature.purchasing"],"sha256":"f9203b2def92ee787077b88ae9e31425c03cff7b43a039823c160bc69303cc3f"},{"id":"ecommerce.progression.personalized-recommendations","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-personalized-recommendations.json","requiresPacks":["ecommerce.l2.recommendations"],"sha256":"fafda848a453566e61af3513659a8884979ae49218e490830774bb4c25f7b383"},{"id":"ecommerce.progression.price-accounting-specifications","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/progression-price-accounting-specifications.json","requiresPacks":[],"sha256":"f49fa571d5e63617537742620e92e95a9671ee87f52fd1abc15c2daaf2a6a195","stableId":"ecommerce.returns-pricing"},{"id":"ecommerce.progression.price-history-specifications","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/progression-price-history-specifications.json","requiresPacks":[],"sha256":"5882d2b0daee49ff6f3b8b6aace9fd63f6dca52b18f4dc72f8329e59ffcb9664","stableId":"ecommerce.returns-pricing"},{"id":"ecommerce.progression.promotion-checkout","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-promotion-checkout.json","requiresPacks":["ecommerce.feature.checkout","ecommerce.progression.promotion-rules"],"sha256":"ead1c8b0ab75a29bc55eaf71ceb216f881655ce20321dc0e36ee8e72271383a9"},{"id":"ecommerce.progression.promotion-reporting","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-promotion-reporting.json","requiresPacks":["ecommerce.progression.promotion-checkout"],"sha256":"a0c28a936e0929584ccf418dbbe5f53ba6b6beccd33a5a8255bbfb334a3ddca8"},{"id":"ecommerce.progression.promotion-rules","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-promotion-rules.json","requiresPacks":["ecommerce.feature.catalog-items","ecommerce.progression.staff-access"],"sha256":"4275d81b85379c0030abd92208b732f616a53a2bad8054e0ee6c7f745488105c"},{"id":"ecommerce.progression.recommendation-feedback","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-recommendation-feedback.json","requiresPacks":["ecommerce.progression.personalized-recommendations"],"sha256":"e7e703471cb30e8caf6acf89e3a0ad089dcde8d9320ceaf4fba7132add4007e5"},{"id":"ecommerce.progression.review-access-specifications","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/progression-review-access-specifications.json","requiresPacks":[],"sha256":"d89b6e559244faed30648f2ebdb462913c0b882c614ffe51821c1eb446e66d55"},{"id":"ecommerce.progression.staff-access","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-staff-access.json","requiresPacks":[],"sha256":"440a036f482be11c4954abbb478693a995aea0fe0843f1959a641f3ab6733539"},{"id":"ecommerce.progression.staff-activity","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-staff-activity.json","requiresPacks":["ecommerce.progression.catalog-management","ecommerce.progression.staff-roles"],"sha256":"129e48d11c4b24ae94d45350ffc93e190265dd69eb7948fc34c24c49b438db2e"},{"id":"ecommerce.progression.staff-roles","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-staff-roles.json","requiresPacks":["ecommerce.progression.staff-access"],"sha256":"341efe45e33b4e2fd94a19036034963393b6f9d3749c63faf0bbff0415d0ead8"},{"id":"ecommerce.progression.stock-alerts","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-stock-alerts.json","requiresPacks":["ecommerce.feature.warehouse-admin","ecommerce.progression.notification-preferences"],"sha256":"837f8fcb2c0c023704372810efdc8532246df2b795b94b8503f9d17dae415db9"},{"id":"ecommerce.progression.support-history","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-support-history.json","requiresPacks":["ecommerce.feature.accounts","ecommerce.progression.support-intake"],"sha256":"23d9dd3e5132a88f3ed0d1c65afac0f6ca769f886b6fb531718831358699b90a"},{"id":"ecommerce.progression.support-intake","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-support-intake.json","requiresPacks":[],"sha256":"6458979c3421deb9916903854a1889580b122813500e3437c475fd3bf73d7721"},{"id":"ecommerce.progression.support-refunds","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-support-refunds.json","requiresPacks":["ecommerce.l2.order-cancellation-features","ecommerce.progression.order-support"],"sha256":"fd05f39442798d4bae9e96c65230268bd80be46f40a57007ede06d02d5a28e7c"},{"id":"ecommerce.progression.support-triage","includeRoles":["feature"],"moduleType":"feature","path":"composition/packs/progression-support-triage.json","requiresPacks":["ecommerce.progression.staff-access","ecommerce.progression.support-intake"],"sha256":"2c90c09870de8922db55a9fd8a1b07553c60fd3aebc679f3e4ff1649c752df85"},{"id":"ecommerce.spec.access-control","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/spec-access-control.json","requiresPacks":[],"sha256":"e8999511e92036535ee412cee4be2739aceedd565bcee9ef54c291bb0dbeaf48"},{"id":"ecommerce.spec.bundle-integrity","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/spec-bundle-integrity.json","requiresPacks":[],"sha256":"e0ca540f434e643c5ddbe7fb204fe1edad39376c719d48d4d544caef559595f9"},{"id":"ecommerce.spec.concurrency-safety","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/spec-concurrency-safety.json","requiresPacks":[],"sha256":"350390ddaf5b65e1af7e734cb69404940ec8ab7a632d7201d334a25eb06e2e4c"},{"id":"ecommerce.spec.external-data-sync","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/spec-external-data-sync.json","requiresPacks":[],"sha256":"f4c03de56a2e98f3d4ceb35063b9b5624f7842a4ba89915fcadd9c8a267faef2"},{"id":"ecommerce.spec.live-state","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/spec-live-state.json","requiresPacks":[],"sha256":"49d210d3c5a7aed24d5b296d3a63fe18e87cb9d33821c1f2cdd872a8e06e92f3"},{"id":"ecommerce.spec.search-ordering","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/spec-search-ordering.json","requiresPacks":[],"sha256":"f33275ca6a9593135c94170bd262203eaf65d980e6f810c393f6d07a28bffa50"},{"id":"ecommerce.spec.split-tender-refunds","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/spec-split-tender-refunds.json","requiresPacks":[],"sha256":"91ca0be0e80ea7d001cdd9a2959c7b2c00be3b87aeb5dc06c7db8b9d179ead8c"},{"id":"ecommerce.spec.state-durability","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/spec-state-durability.json","requiresPacks":[],"sha256":"df7c5808d25201493e6eafd46e37091a5f3b9d8f70f2c6ac708d59395137019d"},{"id":"ecommerce.spec.store-credit","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/spec-store-credit.json","requiresPacks":[],"sha256":"4d858d568aabb314ad118b3e2b5003e43710585fdbf4a85b71b9971ca636843c"},{"id":"ecommerce.spec.subscriptions","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/spec-subscriptions.json","requiresPacks":[],"sha256":"7dac820cfeec043b619d468be0c6b0d7e022093851e7884ad573a80aeeb468f2"},{"id":"ecommerce.spec.transactional-integrity","includeRoles":["guarantee"],"moduleType":"specification","path":"composition/packs/spec-transactional-integrity.json","requiresPacks":[],"sha256":"3894b3d9967c3cbb65ea3a3d5b0f8eb88f2673c06b9122f75acc945c8ca06f5f"}]},"contentSha256":"104589b973c1e7dd1a2410659759989eb0de08b16efeda3131777f82858a0a8b","executionSha256":"2a95cbdeb0deaca8fc6f07167df3c3dbd36541b845133ba34617fc88d54cbd85","id":"ecommerce.progression-catalog","meaningSha256":"9724c6dc245ca2cc7d19c9f17938c6edcf3a0234ffbf84b4a48bc8f627aa9337","recipeReleaseSchemaVersion":3,"scoring":{"checks":190,"mode":"source-points","points":359},"sequence":null,"sourceManifest":[{"kinds":["fixture"],"path":"composition/fixtures/operations.json","sha256":"d06444b72dc94fe1ef5e08867d875e1f3bbaa5cd82c35a3558f399c1fcb5ceae"},{"kinds":["pack"],"path":"composition/packs/feature-accounts.json","sha256":"9eec7949c45ab6af816008b41245dd1e13e9164a6e10d42895eb93ad40664fbf"},{"kinds":["pack"],"path":"composition/packs/feature-bundle-checkout.json","sha256":"2905f7ee7d8ae6d1086d2194649242b68ef871d10ec60e1bc0382baf5cb6e7df"},{"kinds":["pack"],"path":"composition/packs/feature-bundle-returns.json","sha256":"c0f9424066d11efd12bf2c88ec3607ac1152ba5b1e34d7fe598529c03035e352"},{"kinds":["pack"],"path":"composition/packs/feature-cart.json","sha256":"217d2a7af92bc0551e17c12376b15668431619cc42f919f9d5a0a709f11eba78"},{"kinds":["pack"],"path":"composition/packs/feature-catalog-discovery.json","sha256":"9aeea0db86cdacc386e5a26fed12b5604d674047c57c792e9aac3a5eabf14b42"},{"kinds":["pack"],"path":"composition/packs/feature-catalog-items.json","sha256":"fce1929f7932d508f463793be463896fbb13ad6da2b7c57e0838188579f7a95f"},{"kinds":["pack"],"path":"composition/packs/feature-checkout.json","sha256":"660aaf39db20b4fd6e405948b45f816c6733a2be37b8190b989a4d18c59eb214"},{"kinds":["pack"],"path":"composition/packs/feature-product-bundles.json","sha256":"e768568827d4febae15fe2ec1d25410d86f78ada77debf9cb047d70fe3fdb12c"},{"kinds":["pack"],"path":"composition/packs/feature-purchasing.json","sha256":"d21746b5e7246b1771ee9f72d0cc66ef2c2cc9573485ffc839f6ca4b1b01ddff"},{"kinds":["pack"],"path":"composition/packs/feature-reviews.json","sha256":"385771e4c68dbc7afbc0d68af897d698a66f744f50e4f580a813db580cf4615c"},{"kinds":["pack"],"path":"composition/packs/feature-split-tender-refunds.json","sha256":"8d12162887644dc6bca1fbcccfd06ebd679bd8017b9234cc5632e6a1f508eba8"},{"kinds":["pack"],"path":"composition/packs/feature-store-credit.json","sha256":"f252e74cb95d87a6a7cec629d59bc7a37cac133e6abee255738ff9a1fca50fcd"},{"kinds":["pack"],"path":"composition/packs/feature-subscriptions.json","sha256":"31bbe7f11ac99af6eeeeadcd62c49bc635bb20384a6b5e3724b8ba9124530799"},{"kinds":["pack"],"path":"composition/packs/feature-warehouse-admin.json","sha256":"9230ca4ddfe048fd3903f3b90aee16a4711c6d53fbe37478917b16a6e550d66d"},{"kinds":["pack"],"path":"composition/packs/l2-inventory-dashboard.json","sha256":"98d6ac14b32e61e976377df4687a13d6a780a0e96d2e49a05ce2a386e4588778"},{"kinds":["pack"],"path":"composition/packs/l2-order-cancellation-features.json","sha256":"21f8663b36ea0871493903776ed32fae619183dc5fc0f74a8c1a5e8bc932f561"},{"kinds":["pack"],"path":"composition/packs/l2-price-history-features.json","sha256":"786b9f48b9670aa9cb02a439b7eda2feff11c5163a4d49939c892d604206e924"},{"kinds":["pack"],"path":"composition/packs/l2-recommendations.json","sha256":"a21e16ece7b0c4c6e2f0871428af4e4311ae5a91c46174a7ad644b6b93f2c18b"},{"kinds":["pack"],"path":"composition/packs/l2-sales-dashboard.json","sha256":"5e9b1693dcea799edd672e276257a87ad6d7d88099d60be072e952e22e75ca77"},{"kinds":["pack"],"path":"composition/packs/l2-stock-transfers-features.json","sha256":"fa9514e71b90d38623ede3aa4108cb8a53ca72ed95fc69858733ad38da7235ad"},{"kinds":["pack"],"path":"composition/packs/l3-cart-expiration-features.json","sha256":"38029c5ea6d34974b155ef66c311dc500fcece1f82746cffa57ec5f0695f6a5d"},{"kinds":["pack"],"path":"composition/packs/l3-deferred-access-specifications.json","sha256":"cc47732a893439472253b8e209d653959966ca7be20ef0616ab24f8695096814"},{"kinds":["pack"],"path":"composition/packs/l3-deferred-durability-specifications.json","sha256":"9f5109ceb2a8f5e68df317910cea1911dfe79e8c42de51c83b60b760f4075d76"},{"kinds":["pack"],"path":"composition/packs/l3-deferred-integrity-specifications.json","sha256":"1a3d27f25be0b41f87afff3412b6f2c93617a68306309f471e0120d1ea359056"},{"kinds":["pack"],"path":"composition/packs/l3-order-delivery-features.json","sha256":"7f86ba1b2a2791ba38ba6abeeee6078b061bdf03ea951647d79a74f6fe435b95"},{"kinds":["pack"],"path":"composition/packs/l3-order-returns-features.json","sha256":"cd5dc23d3c7514df91d0a54d92d97e7604e6828b1a287d0c79c714724cdc2206"},{"kinds":["pack"],"path":"composition/packs/l3-reservations-features.json","sha256":"972b14948d5dea26cadab3567fc5bbd2c42d4feed5c428c004117b723acecd12"},{"kinds":["pack"],"path":"composition/packs/l3-scheduled-restocks-features.json","sha256":"305c2cf78bc84c63b20cb3e5531760174355128332a365d741dbbd0d64c938cb"},{"kinds":["pack"],"path":"composition/packs/l3-server-time-specifications.json","sha256":"75ad830a0023bf1fe8022695f0776854d75cdbd87f5120a25312c7de33bf57c9"},{"kinds":["pack"],"path":"composition/packs/progression-automatic-reorder.json","sha256":"a1d14fcb9cd6c9e96b71d2bacdd4865badaeda913ba9cb3cb5fab8f1eb8497e7"},{"kinds":["pack"],"path":"composition/packs/progression-cancellation-accounting-specifications.json","sha256":"29b290a1356e63a9d2e937c7f92ad540403b27ebcef94df15451d12e59ac86cc"},{"kinds":["pack"],"path":"composition/packs/progression-cancellation-queue-specifications.json","sha256":"adf9f45a20d13a400a7812fe825a73de6035f980813c19e3698128e49085b58d"},{"kinds":["pack"],"path":"composition/packs/progression-cart-recovery.json","sha256":"ca6d95040f5b81478546b9afedcc428114e795ced11b8364eb2542c258aec794"},{"kinds":["pack"],"path":"composition/packs/progression-catalog-management.json","sha256":"f12e7e37d031f470683ff74671b22c1de08eb8f7fe0e9cf03cbcfb4604dd2f06"},{"kinds":["pack"],"path":"composition/packs/progression-customer-profile.json","sha256":"731eeced13c1053432560d4149d2de29fa2cd2b0051759d705cb36e92ad16bb3"},{"kinds":["pack"],"path":"composition/packs/progression-delivery-notifications.json","sha256":"60b58c4327d74f710f94ffc0deae1b51ef40ca1d2146e45b8e02d1b28e1146bc"},{"kinds":["pack"],"path":"composition/packs/progression-faceted-search.json","sha256":"5a967859348f967bdc43e0e2490289a859f568e4e8d809c21d5f4404d85e4dd7"},{"kinds":["pack"],"path":"composition/packs/progression-fulfilment-queue.json","sha256":"811eeb166f261e32b5338e118b0851aa1ffeb06b3107dafb6861057e11004db4"},{"kinds":["pack"],"path":"composition/packs/progression-inventory-conservation-specifications.json","sha256":"b3bff8b33e6069bf3cfd2e68389bbd50f0ec9c981d766fd2d7717dcf892463dc"},{"kinds":["pack"],"path":"composition/packs/progression-managed-support.json","sha256":"6546f37be296605cad2e06bbe71d0cf014d1025a5a4b3b8997f4c38cf6329841"},{"kinds":["pack"],"path":"composition/packs/progression-notification-preferences.json","sha256":"e8423fe1a2a615a1f2b5bd7073e9cc7040c8d2e2eadc0ef466d50e5d21aaf134"},{"kinds":["pack"],"path":"composition/packs/progression-operations-access-specifications.json","sha256":"796806b8a647fd367f53f6246d51aa998b9aa9e2a8959356d237b6411be96937"},{"kinds":["pack"],"path":"composition/packs/progression-order-support.json","sha256":"1e1a1722adc4c72fdbf31aa34261b73975df53ffab27b28134d71b09ada1b27f"},{"kinds":["pack"],"path":"composition/packs/progression-payment-records.json","sha256":"f9203b2def92ee787077b88ae9e31425c03cff7b43a039823c160bc69303cc3f"},{"kinds":["pack"],"path":"composition/packs/progression-personalized-recommendations.json","sha256":"fafda848a453566e61af3513659a8884979ae49218e490830774bb4c25f7b383"},{"kinds":["pack"],"path":"composition/packs/progression-price-accounting-specifications.json","sha256":"f49fa571d5e63617537742620e92e95a9671ee87f52fd1abc15c2daaf2a6a195"},{"kinds":["pack"],"path":"composition/packs/progression-price-history-specifications.json","sha256":"5882d2b0daee49ff6f3b8b6aace9fd63f6dca52b18f4dc72f8329e59ffcb9664"},{"kinds":["pack"],"path":"composition/packs/progression-promotion-checkout.json","sha256":"ead1c8b0ab75a29bc55eaf71ceb216f881655ce20321dc0e36ee8e72271383a9"},{"kinds":["pack"],"path":"composition/packs/progression-promotion-reporting.json","sha256":"a0c28a936e0929584ccf418dbbe5f53ba6b6beccd33a5a8255bbfb334a3ddca8"},{"kinds":["pack"],"path":"composition/packs/progression-promotion-rules.json","sha256":"4275d81b85379c0030abd92208b732f616a53a2bad8054e0ee6c7f745488105c"},{"kinds":["pack"],"path":"composition/packs/progression-recommendation-feedback.json","sha256":"e7e703471cb30e8caf6acf89e3a0ad089dcde8d9320ceaf4fba7132add4007e5"},{"kinds":["pack"],"path":"composition/packs/progression-review-access-specifications.json","sha256":"d89b6e559244faed30648f2ebdb462913c0b882c614ffe51821c1eb446e66d55"},{"kinds":["pack"],"path":"composition/packs/progression-staff-access.json","sha256":"440a036f482be11c4954abbb478693a995aea0fe0843f1959a641f3ab6733539"},{"kinds":["pack"],"path":"composition/packs/progression-staff-activity.json","sha256":"129e48d11c4b24ae94d45350ffc93e190265dd69eb7948fc34c24c49b438db2e"},{"kinds":["pack"],"path":"composition/packs/progression-staff-roles.json","sha256":"341efe45e33b4e2fd94a19036034963393b6f9d3749c63faf0bbff0415d0ead8"},{"kinds":["pack"],"path":"composition/packs/progression-stock-alerts.json","sha256":"837f8fcb2c0c023704372810efdc8532246df2b795b94b8503f9d17dae415db9"},{"kinds":["pack"],"path":"composition/packs/progression-support-history.json","sha256":"23d9dd3e5132a88f3ed0d1c65afac0f6ca769f886b6fb531718831358699b90a"},{"kinds":["pack"],"path":"composition/packs/progression-support-intake.json","sha256":"6458979c3421deb9916903854a1889580b122813500e3437c475fd3bf73d7721"},{"kinds":["pack"],"path":"composition/packs/progression-support-refunds.json","sha256":"fd05f39442798d4bae9e96c65230268bd80be46f40a57007ede06d02d5a28e7c"},{"kinds":["pack"],"path":"composition/packs/progression-support-triage.json","sha256":"2c90c09870de8922db55a9fd8a1b07553c60fd3aebc679f3e4ff1649c752df85"},{"kinds":["pack"],"path":"composition/packs/spec-access-control.json","sha256":"e8999511e92036535ee412cee4be2739aceedd565bcee9ef54c291bb0dbeaf48"},{"kinds":["pack"],"path":"composition/packs/spec-bundle-integrity.json","sha256":"e0ca540f434e643c5ddbe7fb204fe1edad39376c719d48d4d544caef559595f9"},{"kinds":["pack"],"path":"composition/packs/spec-concurrency-safety.json","sha256":"350390ddaf5b65e1af7e734cb69404940ec8ab7a632d7201d334a25eb06e2e4c"},{"kinds":["pack"],"path":"composition/packs/spec-external-data-sync.json","sha256":"f4c03de56a2e98f3d4ceb35063b9b5624f7842a4ba89915fcadd9c8a267faef2"},{"kinds":["pack"],"path":"composition/packs/spec-live-state.json","sha256":"49d210d3c5a7aed24d5b296d3a63fe18e87cb9d33821c1f2cdd872a8e06e92f3"},{"kinds":["pack"],"path":"composition/packs/spec-search-ordering.json","sha256":"f33275ca6a9593135c94170bd262203eaf65d980e6f810c393f6d07a28bffa50"},{"kinds":["pack"],"path":"composition/packs/spec-split-tender-refunds.json","sha256":"91ca0be0e80ea7d001cdd9a2959c7b2c00be3b87aeb5dc06c7db8b9d179ead8c"},{"kinds":["pack"],"path":"composition/packs/spec-state-durability.json","sha256":"df7c5808d25201493e6eafd46e37091a5f3b9d8f70f2c6ac708d59395137019d"},{"kinds":["pack"],"path":"composition/packs/spec-store-credit.json","sha256":"4d858d568aabb314ad118b3e2b5003e43710585fdbf4a85b71b9971ca636843c"},{"kinds":["pack"],"path":"composition/packs/spec-subscriptions.json","sha256":"7dac820cfeec043b619d468be0c6b0d7e022093851e7884ad573a80aeeb468f2"},{"kinds":["pack"],"path":"composition/packs/spec-transactional-integrity.json","sha256":"3894b3d9967c3cbb65ea3a3d5b0f8eb88f2673c06b9122f75acc945c8ca06f5f"},{"kinds":["recipe"],"path":"composition/recipes/progression-catalog.json","sha256":"a869c95b1d5a222dc679d1529a789d623be3d0a82beff818e6b94702a1791ee9"},{"kinds":["contract-source"],"path":"contracts/accounts.md","sha256":"19b453ded62710d998bce1b169497342209feefccd9c4b1cef3bbb9442f01488"},{"kinds":["contract-source"],"path":"contracts/application-interface.md","sha256":"b770db6c1dbf92c106c7501a17028876a30a63655c8f88b5667b44557f401692"},{"kinds":["contract-source"],"path":"contracts/bundle-checkout.md","sha256":"2c4e70b620934dc927ad4c84c17e54b6e9cef665ba3b098f9f2fc9e79a3c4c3e"},{"kinds":["contract-source"],"path":"contracts/bundle-returns.md","sha256":"69f8bf6afc325a98bab38235e49eaa2875bc674e6a61c7559692593018c9f527"},{"kinds":["contract-source"],"path":"contracts/cart-expiration.md","sha256":"56c0966494cd19b629140f8b8d53e8b9838e34668a5cab6b7cccb41b78734c50"},{"kinds":["contract-source"],"path":"contracts/cart.md","sha256":"bb1b8741b9eb47b5766a330bb1fc23005f7540fdc79d20efdaefb183cf50918d"},{"kinds":["contract-source"],"path":"contracts/catalog-discovery.md","sha256":"d13ea495f78e3827b7a243b97466aefbf0252980c76edb432b6b0506854e8450"},{"kinds":["contract-source"],"path":"contracts/catalog-items.md","sha256":"596ec08b1f4a6b290595570f3bcf2847a1f60feeb20d90f98d25761666bfe9f2"},{"kinds":["contract-source"],"path":"contracts/catalog-management.md","sha256":"5123971185ded331deaef2e323fad31e6133d55fc839289e2c4a0dfc5292ee4e"},{"kinds":["contract-source"],"path":"contracts/checkout.md","sha256":"49416683e06d996a28980ff650a3281f89cae062269b7de761899e71759c731c"},{"kinds":["contract-source"],"path":"contracts/customer-profile.md","sha256":"7451bf14ada950c9b7ad269d63467154c388e9ea18b97f479037cdd397ca519f"},{"kinds":["contract-source"],"path":"contracts/delivery-notifications.md","sha256":"4bf565ea751bb5278bae5910420beeb531a2eb88fb5755e5a06dda68860af490"},{"kinds":["contract-source"],"path":"contracts/faceted-search.md","sha256":"08ace4d2d1947f16ceb4da95e6993086c9bd77a0977c258fef1f0a96c7e7879d"},{"kinds":["contract-source"],"path":"contracts/inventory-dashboard.md","sha256":"8c3de4ff8a958776ca087febbbbf5f950eaef9efacb96b7c4fa49357b2d0a371"},{"kinds":["contract-source"],"path":"contracts/managed-support.md","sha256":"458b89a7f786086a95957c778943660068cd6c00b30635fd1cd786c37df76398"},{"kinds":["contract-source"],"path":"contracts/notification-preferences.md","sha256":"d6b4231f67d5e70a03ec331545b6419acf766b52841c8520b94963c16aa13fec"},{"kinds":["contract-source"],"path":"contracts/operations-access.md","sha256":"fd9c31482b484dc1f9bf3c521d247971334e1f52f51a7548a987bdebc65dbdb5"},{"kinds":["contract-source"],"path":"contracts/order-cancellation.md","sha256":"fc274b1409c50de8edcf797ab83cf8e8c46a54db30ada859f854e1cd40c7dee1"},{"kinds":["contract-source"],"path":"contracts/order-data.md","sha256":"7b7cfc6b102f4836235c7f23344865f60156b3e4ac96c3e5cbb0e298390b3b06"},{"kinds":["contract-source"],"path":"contracts/order-delivery.md","sha256":"74bd21b39c8d5f020ff62cc408992c15788e48eba8e05a76a0cddc4b4920a98e"},{"kinds":["contract-source"],"path":"contracts/order-returns.md","sha256":"fe0668ab3bf00d4c167794dbecb75b26b23cb62dfc0a1b79636a013edb237329"},{"kinds":["contract-source"],"path":"contracts/order-support.md","sha256":"a2901e66af870055b6c3c26f240132ae08f3cb93d0c52a7b2573c9dc153e1e03"},{"kinds":["contract-source"],"path":"contracts/payment-records.md","sha256":"8c752352c7f47a0a65d1823f9ec74009dc95088fc6c2a6cc30d1a1dc7fa11fe0"},{"kinds":["contract-source"],"path":"contracts/price-history-orders.md","sha256":"c3a86a7de94e345c578d6b9ba3527c82c137797f385ca4aca3cf2b7ee39b9d40"},{"kinds":["contract-source"],"path":"contracts/price-history.md","sha256":"3465714f91e0e978607900b19377a134e9beb704848e2462b1438465cc2e98ee"},{"kinds":["contract-source"],"path":"contracts/product-bundles.md","sha256":"b27905002b693e6c63532cd8193c8e3fd1fdeaad85041d84b8240a2092bf1c57"},{"kinds":["contract-source"],"path":"contracts/progression-automatic-reorder.md","sha256":"3e70361965e6ddeb7b05cc7ad5348e363d58d628734de96b8a9792e68e71cc72"},{"kinds":["contract-source"],"path":"contracts/progression-cart-recovery.md","sha256":"daf04aea3082483caa2ebf3a3c89a438329600cb6aa5d74fb47d97bc244e5747"},{"kinds":["contract-source"],"path":"contracts/progression-personalized-recommendations.md","sha256":"42389a6d284dcfdb53a610ce5ce77d40dbc2b9d5e177bbd3f6bccc40079c7d98"},{"kinds":["contract-source"],"path":"contracts/promotion-checkout.md","sha256":"b74e703dc77c9eeec26f5beb41cab102570c8314d817015c858557e9946293aa"},{"kinds":["contract-source"],"path":"contracts/promotion-reporting.md","sha256":"81cded5fd9a8822c8756ea50bedb879971c94f8727d176f22fa53d2a6a1efbb0"},{"kinds":["contract-source"],"path":"contracts/promotion-rules.md","sha256":"f352f8bb938aa874125b10bbb6182643f04b6d1ce82cde41512bd16835a7b90b"},{"kinds":["contract-source"],"path":"contracts/purchasing.md","sha256":"c2d59da67b400d055ab90bf53f2e46a3af9605a1fc68534c2b64d8db2ed3409c"},{"kinds":["contract-source"],"path":"contracts/recommendation-feedback.md","sha256":"00a96f594c8da1ffb4a933c6f8d8d0219216ab6cc2c781cfd17dd9c97c517d58"},{"kinds":["contract-source"],"path":"contracts/recommendations.md","sha256":"ab5417cbf78f43464e82a543e50e9601580bb2b3a57826a9e73da9899f39de16"},{"kinds":["contract-source"],"path":"contracts/reservations.md","sha256":"e1c4af7a67dd200d5f18d550e5b4c30e7a4bbaf60600b434108ebff7bdf90627"},{"kinds":["contract-source"],"path":"contracts/reviews.md","sha256":"6be3567258c0c4bf3331e609f63be26ad0bb43ceca3e7da4accaaad8c9ac6541"},{"kinds":["contract-source"],"path":"contracts/sales-dashboard.md","sha256":"ff9ed3890aa5c8303a0badecf946e7d98da3fbbb0a2890a693099bab1a2056c0"},{"kinds":["contract-source"],"path":"contracts/scheduled-restocks.md","sha256":"a946c0954a641d4fe64b933761e2b92d546ae08ee434d676c6faed3ed8b03c8e"},{"kinds":["contract-source"],"path":"contracts/split-tender-refunds.md","sha256":"62b8b3c233d33858e9b696d06d94c70f6524f8cbed77028acc0d2affdd780841"},{"kinds":["contract-source"],"path":"contracts/staff-access.md","sha256":"dcd7b740d5f65160ddf0b4221d29f84426ee89f04560e953add0856cf0bca87a"},{"kinds":["contract-source"],"path":"contracts/staff-activity.md","sha256":"7e7c1846445f82aa78ad8e32606e6830f9d326ebfea75d13b077c073695c9594"},{"kinds":["contract-source"],"path":"contracts/staff-roles.md","sha256":"cce6145182485fafcd2adc29cec90af20523a2afd9c952800eefb5d15a49f96e"},{"kinds":["contract-source"],"path":"contracts/stock-alerts.md","sha256":"405f106569c84b3062f8159452730f16adadf12d804c554a570e39b958b63421"},{"kinds":["contract-source"],"path":"contracts/stock-transfers.md","sha256":"01f28ebb3487559d4689919f0991e43f74fcc85609fa4c810e92adde30e84096"},{"kinds":["contract-source"],"path":"contracts/store-credit.md","sha256":"505ca48dfc4f9f2d6ea38e53e1bd382910264d0195be8580b5fb01f7e7074713"},{"kinds":["contract-source"],"path":"contracts/subscriptions.md","sha256":"dc533bc7f3b9be03ae7d9e18c2e02ffa996bf934b9ba230826d492b4cefa045e"},{"kinds":["contract-source"],"path":"contracts/support-history.md","sha256":"610022f964c2aa00b409fb2723f91b885e306edde5bf73390e880cffd2a5583d"},{"kinds":["contract-source"],"path":"contracts/support-intake.md","sha256":"fd9f732acf67d6c772f4c05d626e4e9e472818fc5226b7ef3d7223b85a0e0bc7"},{"kinds":["contract-source"],"path":"contracts/support-refunds.md","sha256":"da5d0e69bbf3a028dc79f207b940bbd4a79e7f82c68d89f252eb8c73ea16b8bc"},{"kinds":["contract-source"],"path":"contracts/support-triage.md","sha256":"8079080a299674bb0558a6550243051360be1c0f18252708128c3f71af899a0d"},{"kinds":["contract-source"],"path":"contracts/warehouse-administration.md","sha256":"ea037935ce439d10f4ccea3296bb16aa2ddee4e0c13b2b7ade0638ff7cb9b8d6"},{"kinds":["requirement-source"],"path":"prompts/modular/accounts.md","sha256":"0a8dee0847a02da7777c11c4175577c4533fafa538fc09e5124cf4d825e4c96f"},{"kinds":["requirement-source"],"path":"prompts/modular/bundle-checkout.md","sha256":"d3f21ac08991619b5c9aa6ac99a7554340bae991d5b31a25cdb048cdb314936e"},{"kinds":["requirement-source"],"path":"prompts/modular/bundle-returns.md","sha256":"34b3bee882e6a1f6a1527e67d12f8cbdcfd74d026108f9805159123c41015bf2"},{"kinds":["requirement-source"],"path":"prompts/modular/cancellation-conservation.md","sha256":"f383c20f16d75b77dfe2c0ca10dc1c77c998feb80b7c327616512b486c4714f8"},{"kinds":["requirement-source"],"path":"prompts/modular/cancellation-queue.md","sha256":"7e1adf98391a930017acb5c37d46c9b0d17d89e58c939fc57dc52f22e2f86f31"},{"kinds":["requirement-source"],"path":"prompts/modular/cart-expiration.md","sha256":"7c66696f044249d5f7f31487efb85e83f4c06b862fbaa3dd142ef2aef6afc2b7"},{"kinds":["requirement-source"],"path":"prompts/modular/cart.md","sha256":"da024de4032661a46574c8ac8dbe3d1baf45ef6ea7115ba1976d7d5c1044afbe"},{"kinds":["requirement-source"],"path":"prompts/modular/catalog-discovery.md","sha256":"9a0785c55d17e577b84ef257de482bcd8bc8c961e9d0760c38472936a1a0062b"},{"kinds":["requirement-source"],"path":"prompts/modular/catalog-items.md","sha256":"36a72da77f42d856b691c93dc587fee6d9d7f88b722a9893fc226b1d5bb5d3fe"},{"kinds":["requirement-source"],"path":"prompts/modular/catalog-management.md","sha256":"93fece29e1d68513732398c815fe4ebc83079e705e2b9de40df543281439f390"},{"kinds":["requirement-source"],"path":"prompts/modular/checkout-recovery-specification.md","sha256":"0e851fea800d81cb558f8004f04e628e5ccf9f6e5589a9d2f6fceaf33e50f207"},{"kinds":["requirement-source"],"path":"prompts/modular/checkout.md","sha256":"bb834dec7858349a17a8980eb13459a606cdbd2b1c534f468f666459c1a24ba7"},{"kinds":["requirement-source"],"path":"prompts/modular/customer-profile.md","sha256":"f4ffc5d2a642bbccec9eb5c428a37b2d9bfaa1c9f96648d8544bfcf2fe8187bb"},{"kinds":["requirement-source"],"path":"prompts/modular/delivery-notifications.md","sha256":"12d6db9061749779dcb3db699a492781b34ae955c4c11cbb3e5dc2c8e0300ef3"},{"kinds":["requirement-source"],"path":"prompts/modular/faceted-search.md","sha256":"5dc13b47abcbc85654c82148f44b854979d868578db50db42e060c773edc2207"},{"kinds":["requirement-source"],"path":"prompts/modular/inventory-dashboard.md","sha256":"d2190c809ce459141cfdb716d25d38b01f7aca2ce8b8f74128e731e4e3fede07"},{"kinds":["requirement-source"],"path":"prompts/modular/l1-external-sync-specifications.md","sha256":"7f0486045ef1b18271d1b004515f3a1dbf76d09ed62eb7c0683b099100b7f903"},{"kinds":["requirement-source"],"path":"prompts/modular/l1-specifications.md","sha256":"bc3ab63f33d9bbc59fde776c7fdc83ecfdf3d087be78e2596538b7d6945fdd8a"},{"kinds":["requirement-source"],"path":"prompts/modular/l3-specifications.md","sha256":"e6b9877c0f28ec287e510367a810221d09f9b48bb506f7c3cde4eb4450a3eaf4"},{"kinds":["requirement-source"],"path":"prompts/modular/later-specifications.md","sha256":"097745468f48a0589fd9dd81ad98d3ca0ce9455678156fc74d2331e403a77a39"},{"kinds":["requirement-source"],"path":"prompts/modular/managed-support.md","sha256":"33abfe9e7514c6826df60572557d54f2e1570477afeae14c64bb3c5f3bad5fb6"},{"kinds":["requirement-source"],"path":"prompts/modular/notification-preferences.md","sha256":"496baa70ef300e6874f6dbf58ccd2bac594e53101875fd68740e27779d11871b"},{"kinds":["requirement-source"],"path":"prompts/modular/operations-access.md","sha256":"1699a8d0466e115874e49b8ae73bb0004c3c620a03d3aa04dd83818b6c798003"},{"kinds":["requirement-source"],"path":"prompts/modular/order-accounting.md","sha256":"c83044972a6f53a723dcba9ef9dec157eeacb4d2059b790a426da255134364d0"},{"kinds":["requirement-source"],"path":"prompts/modular/order-cancellation.md","sha256":"7d429d4e8db546961e7ec9a12fb5dcba5efd0562c8059ab354121fd5ed13ea84"},{"kinds":["requirement-source"],"path":"prompts/modular/order-delivery.md","sha256":"40483d0bf91c81efe57a61d62dbda756498792383bed9af595cdd8878a0b0cb8"},{"kinds":["requirement-source"],"path":"prompts/modular/order-ownership.md","sha256":"1f4e5149768d1d0a4917ca9b711f1f40bf2c783ee5d59a42ab307f01babb052e"},{"kinds":["requirement-source"],"path":"prompts/modular/order-returns.md","sha256":"7277266f1e8c7d660ee38ba55ac1359200e56c695862c6d7d755d6ec6a6cb808"},{"kinds":["requirement-source"],"path":"prompts/modular/order-support.md","sha256":"f515fe9827373e110633245df158ce8f78ca040f6c25a059d3885182515abb9a"},{"kinds":["requirement-source"],"path":"prompts/modular/payment-records.md","sha256":"ab5c9c913041e324cad82fe6a135d78906818afa3c9a5c204289142c1d696f09"},{"kinds":["requirement-source"],"path":"prompts/modular/price-authorization.md","sha256":"6e9f949bd45f3bc816336de6e7db6dd18254ceebe6ecacc8e0abfd5f3887b8b5"},{"kinds":["requirement-source"],"path":"prompts/modular/price-history-orders.md","sha256":"005e1db32c0c5e4c4f52c1834f82c3d35c7118912c86580b4295a34e9ff12d3e"},{"kinds":["requirement-source"],"path":"prompts/modular/price-history.md","sha256":"9edb3db98af12f3a08cae7887dd6da91c746be49597f086457fc2ecb70c558b8"},{"kinds":["requirement-source"],"path":"prompts/modular/product-bundles.md","sha256":"a0805178ebbb20ea290994de65b4ded503c37798af59905d47c1cb2fd4e2c467"},{"kinds":["requirement-source"],"path":"prompts/modular/production-specifications.md","sha256":"a49cc5aae32bfb035b878e297ee67c1aa8b83ca90e7c73c3dd0cf9f919ae1cf6"},{"kinds":["requirement-source"],"path":"prompts/modular/progression-automatic-reorder.md","sha256":"64751064e699c709a6550daba6fee8f019f445b817910e1ed0d8831632fa2471"},{"kinds":["requirement-source"],"path":"prompts/modular/progression-cart-recovery.md","sha256":"4dc64aaa87d54a05bd7cce31bdc98254e605251d14f1688b859e9aa908da6a8b"},{"kinds":["requirement-source"],"path":"prompts/modular/progression-framing.md","sha256":"33042f5721843cd3796c8545c2a01a32e6d9b341192036321f6532f5ed0e03d2"},{"kinds":["requirement-source"],"path":"prompts/modular/progression-personalized-recommendations.md","sha256":"6c78b3963318b18da11173f96bed5cb8e5537f49e0008342ff11209fada3e099"},{"kinds":["requirement-source"],"path":"prompts/modular/promotion-checkout.md","sha256":"1979ce8991b38fab30c4ea0d0fb7adba51d79bb82981059b0dafded82fed843a"},{"kinds":["requirement-source"],"path":"prompts/modular/promotion-reporting.md","sha256":"3285eb0a50ceeed40b3415bdab8d6f5fc4138e496d2062a08c5ee2fac70157c6"},{"kinds":["requirement-source"],"path":"prompts/modular/promotion-rules.md","sha256":"7a43b632e8fd43b8060100f9614476e41003e47afa04dd1b7c3b09639f6280b2"},{"kinds":["requirement-source"],"path":"prompts/modular/purchasing.md","sha256":"059073391cf92eef2fb1edb710ec1a62747e82115c2bd0251274111b92f93e46"},{"kinds":["requirement-source"],"path":"prompts/modular/recommendation-feedback.md","sha256":"072a548ae797773041e86cbc62f39011559ca7ab40441bee6b77ec1854cd7f51"},{"kinds":["requirement-source"],"path":"prompts/modular/recommendations.md","sha256":"0eed3feb6fc6acf86260d74eeff557b15837aaa09a857c63cf9dee848cd7201f"},{"kinds":["requirement-source"],"path":"prompts/modular/reservations.md","sha256":"e45b53568066c7200486453f78fdbbe89a116259a4d83d9435bde350316595e0"},{"kinds":["requirement-source"],"path":"prompts/modular/review-access.md","sha256":"430fe60b2f028f67eab9f23b572238d4dacc60f7b89d74f8030618368455f371"},{"kinds":["requirement-source"],"path":"prompts/modular/reviews.md","sha256":"1649822528e1bbcfedc59b3c8c5d7ff1b5f0e42e1e82b81ee7e59361cbefab33"},{"kinds":["requirement-source"],"path":"prompts/modular/sales-dashboard.md","sha256":"c65aaadb1f8da6e2b5f7967cc3bcf07beb0e7a860351e866373667bc33012dbd"},{"kinds":["requirement-source"],"path":"prompts/modular/scheduled-restocks.md","sha256":"ba32547038c681f7214219e5a10df75dbf925ee19d4ba60db5b291570f9c16ec"},{"kinds":["requirement-source"],"path":"prompts/modular/search-ordering-specification.md","sha256":"77d88f2b03c71b6cdc8350020ffd7815cf0bee87e4155c1aa00beb9bb81a83f8"},{"kinds":["requirement-source"],"path":"prompts/modular/shipping-authorization.md","sha256":"f336e9c8e1ea425c3dc44e72fabbc5e9e0e2a3541cab3750dd8e112b6c7997f8"},{"kinds":["requirement-source"],"path":"prompts/modular/split-tender-refunds.md","sha256":"c21e00af3623136de4ae45f089b45b457a34563eb37a756d5b8b3563d1f19734"},{"kinds":["requirement-source"],"path":"prompts/modular/staff-access.md","sha256":"10c6aedbc5c60b440d6d0b5c38bb57363ae021c54d62ee2ffe2294d7f6741550"},{"kinds":["requirement-source"],"path":"prompts/modular/staff-activity.md","sha256":"0cb274acefc0d75cd82415de0eb167133db0412e1de4f0fd59f62983d29a1476"},{"kinds":["requirement-source"],"path":"prompts/modular/staff-roles.md","sha256":"951f617a191161df8b9061b304ad568bb5a942070e72142ddf4b93c3dd099951"},{"kinds":["requirement-source"],"path":"prompts/modular/stock-alerts.md","sha256":"3478629a2831e04baca5eec1d3a2a3695c03f6044c36b43acb7c8ca7e35e9867"},{"kinds":["requirement-source"],"path":"prompts/modular/stock-conservation.md","sha256":"4551438e045bb65b44127d3f92dcb79548c78dc1ecc42d2c7807a599d6c59c8f"},{"kinds":["requirement-source"],"path":"prompts/modular/stock-transfers.md","sha256":"6e2f0735d81fd5376b851893404873af4807b83e807e078cca0b8c9a9920e627"},{"kinds":["requirement-source"],"path":"prompts/modular/store-credit.md","sha256":"d769da6c0ef4eb2fa6ac241aeddb61e8e2d85a0f1fcfd595fbf2c6660f67cd7b"},{"kinds":["requirement-source"],"path":"prompts/modular/subscriptions.md","sha256":"ca93ab01c0d23c2fb3c035981372dc1254b219f1875b9566c0d01123176cc7c1"},{"kinds":["requirement-source"],"path":"prompts/modular/support-history.md","sha256":"df2ca2664901a90814eab17d5f7c029595ef614d76ee29eeb4204d9f4f371429"},{"kinds":["requirement-source"],"path":"prompts/modular/support-intake.md","sha256":"048f11b71b66dadf80c77b5d9aef8bbff2b2030171eeda5012dce74ae16421ed"},{"kinds":["requirement-source"],"path":"prompts/modular/support-refunds.md","sha256":"79090320de03a74660418da9e9adbf8170f8c574be990faeaf3d7df652221aa6"},{"kinds":["requirement-source"],"path":"prompts/modular/support-triage.md","sha256":"bbb35abcbae9b8d3782ee3eb94f3175428edfe3fa564d6b0e3af2e0050e10dee"},{"kinds":["requirement-source"],"path":"prompts/modular/transfer-authorization.md","sha256":"6d1e33c7a6aa209ad3f2705607a839dde5c915b65d78a8bb10b9e0da29172c0b"},{"kinds":["requirement-source"],"path":"prompts/modular/warehouse-administration.md","sha256":"8fa2d7b54b3c3d9601ae69bbfee5bdf389f68fea0d0ef4032201af12d3f424a2"},{"kinds":["scenario"],"path":"scenarios/01-account-create.json","sha256":"f37b83a76ff787e12aae1c86fa91b5a2483bfb1aef9497c291e2a374a4376f45"},{"kinds":["scenario"],"path":"scenarios/01-account-duplicate.json","sha256":"f699c90ca3b1af3eda1b73e3c38a4a7cce19b779afa2f9b208c9ead6fdbf15ba"},{"kinds":["scenario"],"path":"scenarios/01-account-password.json","sha256":"35ea9a3a824444096d9d0a51388296c7c35fc833a532b5252f1df9fc5cfab125"},{"kinds":["scenario"],"path":"scenarios/01-account-reload.json","sha256":"574de181769f0d2fa1a88f1131b2a033385fdd576f9aa5088e95781cbd2266d4"},{"kinds":["scenario"],"path":"scenarios/01-account-signout.json","sha256":"7ce90ffda1338d99e37a87ea5beecaeaaa56adecbeb2f46707d1cf0934e596d1"},{"kinds":["scenario"],"path":"scenarios/01-admin-write-staff.json","sha256":"caf94ac7cbffea4438a343c0883322e090cf83aad743a882fb747c287c89f2a2"},{"kinds":["scenario"],"path":"scenarios/01-buying.json","sha256":"48168c76a51b5e1e6d15573a0891cf90889f99aab78b638a24892f70c55ff257"},{"kinds":["scenario"],"path":"scenarios/01-cart-boundary.json","sha256":"76b9c484f70ac94f0dc1787a1a4e61851bf6586e8d1facc9f970791d47a81777"},{"kinds":["scenario"],"path":"scenarios/01-cart.json","sha256":"88cd43f3f48ae86eeec891677a2817f42f8f6554bad299cfae92889d00262d3c"},{"kinds":["scenario"],"path":"scenarios/01-catalog-ranking.json","sha256":"05436b81ee1fefb5e781f7a68060d41690bcdb7b1b9cb0e7543d183b738f1e14"},{"kinds":["scenario"],"path":"scenarios/01-catalog-search.json","sha256":"a633a4afcdbc5e19cc3189ab5e93c358fad44672a4ec2570fa4569c0e9ee0d02"},{"kinds":["scenario"],"path":"scenarios/01-catalog-values.json","sha256":"6216be4761d0db8ac4480ad1fa1a6d4a628d3a4c022278eb7ebf25105db50ea2"},{"kinds":["scenario"],"path":"scenarios/01-core.json","sha256":"1d7f0cf801d41fb01a4f63b2a414c9d723b0749130d1eefff84730567f9a1614"},{"kinds":["scenario"],"path":"scenarios/01-duplicate-checkout.json","sha256":"b6f470b48e0f504d479376ec25895c1d9c293b1ab52cc9d27a1f626519513290"},{"kinds":["scenario"],"path":"scenarios/01-external-live-sync.json","sha256":"3a3dbe302f7e300ac7e82d680d6fb7c6ee277ec9e175d590f68c1c86569ab752"},{"kinds":["scenario"],"path":"scenarios/01-external-reconnect-sync.json","sha256":"5a9e3822bf5c888709d8fc6e255c29e429ba52be3460d688b8ce0a77ef72e79c"},{"kinds":["scenario"],"path":"scenarios/01-external-reload-sync.json","sha256":"80d9eeb9995c30ae4af156c3d93d42475e8b7de0c1e7dca69d4f882536a5cf86"},{"kinds":["scenario"],"path":"scenarios/01-external-server-restart-sync.json","sha256":"c4dab98e2623e35045fa9dd9f4d251392adf316c7ccacc41982fe16b9a8aa574"},{"kinds":["scenario"],"path":"scenarios/01-last-unit.json","sha256":"499778db939f0392d5576e9ea8feba62cd3f9d7ae07130283cbc3babebc81405"},{"kinds":["scenario"],"path":"scenarios/01-order-ownership.json","sha256":"6bef82c8e28f962d06479e452b2019bf107e4b89cfcda5a11e927f9c518c647b"},{"kinds":["scenario"],"path":"scenarios/01-purchase-attribution.json","sha256":"6e1974179f5bb3c475fd84dc051b3e6f37cf4e02cde17928bc1a68275ce798cc"},{"kinds":["scenario"],"path":"scenarios/01-purchase-session.json","sha256":"219f67e5959209f89ee7179a584bcde6e8ac340bd121cbeb0e25895604d1c614"},{"kinds":["scenario"],"path":"scenarios/01-restock-race.json","sha256":"d37c5071bdb221db025cbe43c6377cc7eca99c62ee6e4720737a32eb60decffa"},{"kinds":["scenario"],"path":"scenarios/01-review-eligibility.json","sha256":"ad387c41d1902fab24915cf9cdeed8a2cd0d7aae26ffc379ae784e61d5c3a277"},{"kinds":["scenario"],"path":"scenarios/01-review-rating-live.json","sha256":"f6d4d0c9ea3bbd1213d130ee1d706e326ffbab33e5b9fd6633fa6ff90fea3e3c"},{"kinds":["scenario"],"path":"scenarios/01-review-uniqueness.json","sha256":"49873dc6cf26204dcebd1038b71bfc5213b091d3d13f5f37c5980998a7f3f8f6"},{"kinds":["scenario"],"path":"scenarios/01-review-visibility.json","sha256":"81360bc6b9735251c7672ee26148912aebcc1426b0be2eca9eecd5e607b927b8"},{"kinds":["scenario"],"path":"scenarios/01-server-price.json","sha256":"24481cb30af8c57ebb26b2adb393b8987642e60ddecdbeff2c0381c122825fdc"},{"kinds":["scenario"],"path":"scenarios/01-warehouse-admin-staff.json","sha256":"5434bf92e5b343862dfcb0e6b26593a71db86c0635222ebb3101f60e00fbea5a"},{"kinds":["scenario"],"path":"scenarios/01-warehouse-stock-live-staff.json","sha256":"708a7bd1df90dba203d6c1236d83a3e6deaaecb6e69b4c2faa4208f773966414"},{"kinds":["scenario"],"path":"scenarios/02-cancellation-queue.json","sha256":"d8758cc074fec24a5d8d1d4395e2edb2a7f376d14d91427df6f70b9a5e8e49ce"},{"kinds":["scenario"],"path":"scenarios/02-fulfilment-access.json","sha256":"ccd33eb7a0fbf80f71c86c963f3fe747581808622a33f842e3aa85d940c20012"},{"kinds":["scenario"],"path":"scenarios/02-fulfilment-live.json","sha256":"54455bebda5a1ef78421b81229d63298fcddae0a8315379c173cca0436e0638f"},{"kinds":["scenario"],"path":"scenarios/02-fulfilment-ship.json","sha256":"3996ecb875ae0ca32eb68053650d0d415246487aac909c5de901789f39d64913"},{"kinds":["scenario"],"path":"scenarios/02-invariants.json","sha256":"23b929b83386165a6fff61ee3cbe6845d5da66e5707cdc1ce8f5ead3207018b4"},{"kinds":["scenario"],"path":"scenarios/02-live-price.json","sha256":"7a905177cae6af15f682ebdb909f695bdee85d3124ee91917cb65e8209a73803"},{"kinds":["scenario"],"path":"scenarios/02-low-stock.json","sha256":"a985505feaa58e6f80f08865ea08658da174a01d08a1e70b832b5dd8579e5c88"},{"kinds":["scenario"],"path":"scenarios/02-operational-best-sellers.json","sha256":"f887d0f4dd660fb85a80f10d6e6e1c9fcf6f2bf25e79fbbd7ba86227968e8ace"},{"kinds":["scenario"],"path":"scenarios/02-operational-category-totals.json","sha256":"263411fec43db53777309f2d64fcbdeef1b551a2d981ee8aa8b4caa0ea2faa16"},{"kinds":["scenario"],"path":"scenarios/02-operational-recommendations.json","sha256":"7ea0fa0f317f963c9d2d37adef945be9b8f19eb8618f387bda0a5cd6b92f30b1"},{"kinds":["scenario"],"path":"scenarios/02-order-cancellation-core.json","sha256":"a490889daee2a92eeb18eb35cb4050a47df179d02e031286d70848a4f2595787"},{"kinds":["scenario"],"path":"scenarios/02-order-cancellation-history.json","sha256":"304c8eb8477bc63fda3a1d90f11e3a03fd6391001d8a12ec405c14aeaced4ccb"},{"kinds":["scenario"],"path":"scenarios/02-paid-price-history.json","sha256":"84896fdea36b4f267443000306a6a349fc7221772278d96c751516b0330dff5d"},{"kinds":["scenario"],"path":"scenarios/02-queue-warehouse.json","sha256":"212e8e907b85aff9e4047e7f6f4258d3078aa78ac6bc1df21abb980759322cf2"},{"kinds":["scenario"],"path":"scenarios/02-self-contained.json","sha256":"aad3ce91dae1a94c0f230bab33b0c7a7942337db51f4f9cde06d1d9e2cc99300"},{"kinds":["scenario"],"path":"scenarios/02-server-actions.json","sha256":"a4f22b10cd76a69e6d761818668c7ec52ad064a0a5b1c391a30fde95849b7211"},{"kinds":["scenario"],"path":"scenarios/02-strengthened.json","sha256":"15fc5e537422dbe8dd875cae0b82ac5b5afbf4b65d45ff4545aaea38dd8a9db8"},{"kinds":["scenario"],"path":"scenarios/02-transfer-overdraw.json","sha256":"df6655ff0bb3c9553384ac37f3659af66d6153fef05d31db5e3d3a1d8165f096"},{"kinds":["scenario"],"path":"scenarios/02-transfer-totals.json","sha256":"81462d3b8d4e78f2dd0e207faf285dca472974cd2cab95f723ce735fc3a5c589"},{"kinds":["scenario"],"path":"scenarios/03-cart-expiration.json","sha256":"22a10c3255506dd1bd50ab7caef5187bd0fc3bad3400ad4af9221c811b570d40"},{"kinds":["scenario"],"path":"scenarios/03-deferred-access.json","sha256":"91ed336253f239ed05cb74bf639be7d9ec0f29bb9b372261295ea7d9babc7098"},{"kinds":["scenario"],"path":"scenarios/03-deferred-durability.json","sha256":"fc425c650e26f4be6134cfdbf590423073fd9e2ebac5ee852b646df7a599a954"},{"kinds":["scenario"],"path":"scenarios/03-deferred-integrity.json","sha256":"5bc1faefe4b3d2547b19afc8a5fafb8dbbfef05d45dcb71b2979e0777eb0bcb0"},{"kinds":["scenario"],"path":"scenarios/03-order-delivery.json","sha256":"880e518338a1615825595b983daf64c6e1d68db924ad5fd8c1b205df7e3ff1aa"},{"kinds":["scenario"],"path":"scenarios/03-reservations.json","sha256":"c2460d2b1f1336d1e4c918b6c27ac962d680f8df897a8ca8fd52bdbd403d17ca"},{"kinds":["scenario"],"path":"scenarios/03-scheduled-restock-apply.json","sha256":"74f49d25eea210b9e0209a57261809a585ca28380edfd50084933beff743d2c1"},{"kinds":["scenario"],"path":"scenarios/03-scheduled-restock-cancel.json","sha256":"462df740d24017bc487d961b1547cafdf0e2bc1cea7805dc669d68ff245261bc"},{"kinds":["scenario"],"path":"scenarios/03-scheduled-restocks.json","sha256":"510a8b147738a249b30dce4900e431ca32880de5ea990600aca063c38535b228"},{"kinds":["scenario"],"path":"scenarios/03-server-time.json","sha256":"667e580173187cb4b0f9538336a7e12108e14b5273c9ac663faa5f2325b006be"},{"kinds":["scenario"],"path":"scenarios/progression-account-state-reconnect.json","sha256":"1ead0e72e9375a95cd03caa09bd1fefd858bef7bc910dd6762a53bcee968ce55"},{"kinds":["scenario"],"path":"scenarios/progression-account-state-reload.json","sha256":"fd464513460c640ddcec6ed7e30f9c434a982ae300d16f5233551c14bff45995"},{"kinds":["scenario"],"path":"scenarios/progression-automatic-reorder-access.json","sha256":"ed57cd01a22e7b7a964108ccf1f971f2e8877921015fdd952cb28109cb7be32d"},{"kinds":["scenario"],"path":"scenarios/progression-automatic-reorder-duplicate.json","sha256":"41fe868171047997d33fe6f424bf771fff8d91fc32a19ef749510fa6f41abb57"},{"kinds":["scenario"],"path":"scenarios/progression-automatic-reorder.json","sha256":"91827fedd89c2e268b2b9102d62a89d39dc8d4e07af619d82073c07595c10d57"},{"kinds":["scenario"],"path":"scenarios/progression-books-balance.json","sha256":"6f8d35c97296d36f26ebe07b9b6abc7d73531e4dab495c370a13e80d43adde46"},{"kinds":["scenario"],"path":"scenarios/progression-bundle-checkout.json","sha256":"b26a777560d208a1e415b58dfd46b3662669c4546208d141f0954033ff575a2f"},{"kinds":["scenario"],"path":"scenarios/progression-bundle-returns.json","sha256":"e8d4c983cb11934ac323abbe0ddf55715d75a832133bfd664197677da4911ec8"},{"kinds":["scenario"],"path":"scenarios/progression-cart-checkout.json","sha256":"b49e07cca26e9461db2467cfce31eb43c7b10fb1481a7252353f08d31bf67abd"},{"kinds":["scenario"],"path":"scenarios/progression-cart-recovery.json","sha256":"8c00fd1d2d10091754577acab63319dd593c137ccb75bcbb45cb43bf000b8298"},{"kinds":["scenario"],"path":"scenarios/progression-catalog-management.json","sha256":"a18d83be7f697b89976b062112acd31755b93a702e12b2bd0f6de993f1c9b0e4"},{"kinds":["scenario"],"path":"scenarios/progression-checkout-crash.json","sha256":"024b80278cdcb30ac53a448db82b45ada850d7c22a04ace3aecf296f03b2b1ef"},{"kinds":["scenario"],"path":"scenarios/progression-core-business.json","sha256":"deac867579938f0e41fb16c02e302416d5dcbfd2d20d11a7374236d58c35b8c1"},{"kinds":["scenario"],"path":"scenarios/progression-customer-profile.json","sha256":"7590ab78f8f475fb16fe7959e625da404d29f1e58f9ceb11f51f557648b91829"},{"kinds":["scenario"],"path":"scenarios/progression-delivery-notifications.json","sha256":"62a30dc8ec24d1758e49f90cdaf60856cc465136595347550ae78fe880d2a8a6"},{"kinds":["scenario"],"path":"scenarios/progression-faceted-filters.json","sha256":"b4610dc373d96ad2eeba7c38fb33173ec74250d8ef4941897bb3c48a98cb5368"},{"kinds":["scenario"],"path":"scenarios/progression-faceted-pagination.json","sha256":"c27b5edbd219e2f440ffa29dbad77dbe2da3485437ec9c0071c2d8a99017aeee"},{"kinds":["scenario"],"path":"scenarios/progression-managed-support-privacy.json","sha256":"74da98e1252bb77f0c3b8bf2a9ac788acb7a0104405d786c9a65c51048af7aad"},{"kinds":["scenario"],"path":"scenarios/progression-managed-support-shared.json","sha256":"712852c3d0289ca15efe348797449fcc3b301df5d3d0a9150b188623fced206d"},{"kinds":["scenario"],"path":"scenarios/progression-notification-preferences.json","sha256":"cb494fbc2543347407abe54f74d9cc3cc698f135690f063ebbbff756aec8c807"},{"kinds":["scenario"],"path":"scenarios/progression-open-list-live.json","sha256":"9abcf41a6919dab036d26869c75de0129b12814e9d49bcf80653fa6e61eff1b0"},{"kinds":["scenario"],"path":"scenarios/progression-order-return-boundary.json","sha256":"c404ce6d68113bd69e290bfcfae4b72b14c0a19f63de349dcf79e47563b1f255"},{"kinds":["scenario"],"path":"scenarios/progression-order-return-complete.json","sha256":"02c41f76b4bc60ba0b0e624df540fcd77e7407c90a2e66487672ed6f9c5ed9f2"},{"kinds":["scenario"],"path":"scenarios/progression-order-support-boundary.json","sha256":"9f19a26a9e4d4fe4385f94150bdef6c4e16913f79c5b0fbaede02984ce098362"},{"kinds":["scenario"],"path":"scenarios/progression-order-support-owned.json","sha256":"dda5208d3dbe3a151e683a8cd238f414118553ea0812584f00a82655255cf1a5"},{"kinds":["scenario"],"path":"scenarios/progression-personalized-recommendations.json","sha256":"8bb96bf7c8a81df356cc0e4bd248184dc13924d78bfe85bf173f840899350e1d"},{"kinds":["scenario"],"path":"scenarios/progression-price-cart-checkout.json","sha256":"0e0f6383af25fa70701c7cc99a4b934da2318a68d168ea897333c8be943c8274"},{"kinds":["scenario"],"path":"scenarios/progression-product-bundles.json","sha256":"7d517f3c1265dae451ed4b0ed0f97186b340717561a10f030edd622f5a50163c"},{"kinds":["scenario"],"path":"scenarios/progression-promotion-checkout.json","sha256":"3e6baa221f1ab203893a2a105c809a3f221d985251a64f115d47e2ab58c180ec"},{"kinds":["scenario"],"path":"scenarios/progression-promotion-reporting.json","sha256":"8c9efe46cdeee1d1dd8d802a222bd1f58b06e997dc53a44fd8e19ff45586673e"},{"kinds":["scenario"],"path":"scenarios/progression-promotion-rules.json","sha256":"e49ea26a881a6d49c485a63d7f1f5cca55fb314522d9d948ed5c2f7eb2a84306"},{"kinds":["scenario"],"path":"scenarios/progression-purchasing.json","sha256":"b2aff32949f55c4819bde7b670c00896aacdbfe59f478fa46df9423da455bd12"},{"kinds":["scenario"],"path":"scenarios/progression-recommendation-feedback.json","sha256":"6a6bae970c29cd443a9e6c66919c909a9f0792a8c37a7d00d2c63f5e4be872c3"},{"kinds":["scenario"],"path":"scenarios/progression-review-access.json","sha256":"7b927506d22d2074af625aaf43e27649e4fc5ec7e5317e04c04dac323661d0cf"},{"kinds":["scenario"],"path":"scenarios/progression-review-script.json","sha256":"267a9134ddb8cd25d3e9d76b5595e97debd2a8071f8ff8e568aa7285b608d30f"},{"kinds":["scenario"],"path":"scenarios/progression-search-ordering.json","sha256":"b96f50c9ec28d68aa1fe4e6d6ea7631f6399f439025c6fe74c1d9a3ce8a55803"},{"kinds":["scenario"],"path":"scenarios/progression-shipping-accounting.json","sha256":"2e310fb4489c154617660b3d247136646c04faa903e958cf2561e9072615b779"},{"kinds":["scenario"],"path":"scenarios/progression-signed-out-purchase.json","sha256":"59de2a84876ae4542d783dca533764eb4f06bc390a67a37b25f0967477ee44fe"},{"kinds":["scenario"],"path":"scenarios/progression-split-tender-refunds.json","sha256":"39771047c0b0a84a6de5c4178e387c95581667cba5741e24d7a3555bc6a19018"},{"kinds":["scenario"],"path":"scenarios/progression-staff-access.json","sha256":"67de6a963d3cdb33dfb119ffacfdd1024d7f9c4d095bf90d553ef1c6efe235c9"},{"kinds":["scenario"],"path":"scenarios/progression-staff-activity.json","sha256":"28600137357ce5620f9f4919b25eb1ff27ba5aff4ed02285349e4d82cd1585a4"},{"kinds":["scenario"],"path":"scenarios/progression-staff-roles.json","sha256":"d87a2f7f0890a84614f538d7656de92082ba1c5843a40d3b2cf0f7f2d9097e11"},{"kinds":["scenario"],"path":"scenarios/progression-stock-alert-delivery.json","sha256":"e4899fb49e87edd9787997e9abb173c890604397fe096186ea0f601fd2e2f67f"},{"kinds":["scenario"],"path":"scenarios/progression-stock-alerts.json","sha256":"582bcb0bea4f109da56e761ad27c809cfb2da38edec41a9e4a528ea603c020f1"},{"kinds":["scenario"],"path":"scenarios/progression-stock-limit.json","sha256":"e07cd6c2b949b392cbb10f0f7cf8b2e8b74a0ef2ed17d563187ad28387eebc4a"},{"kinds":["scenario"],"path":"scenarios/progression-store-credit.json","sha256":"04a1fc19d215155c90a5382e91e15cbf01b400d6b2f8600b5a0e25b42132c820"},{"kinds":["scenario"],"path":"scenarios/progression-subscriptions.json","sha256":"de31264cfc1890ab6c3ebaead98c41964da70d135d34a5f4944ac41108998e94"},{"kinds":["scenario"],"path":"scenarios/progression-support-history.json","sha256":"86928df21aa68fec5c20993ec13e3b1c57291f38a35167996a99ef34f363a2b7"},{"kinds":["scenario"],"path":"scenarios/progression-support-intake.json","sha256":"c8d01303a3316f28de956fb2a72ab289905a46f5d689fe40b8193e12cea1753a"},{"kinds":["scenario"],"path":"scenarios/progression-support-refunds-access.json","sha256":"52f03c8a7ab8b1c7554f7d637bd5d46a19e24f4be5666eac14398ed1f1b2707b"},{"kinds":["scenario"],"path":"scenarios/progression-support-refunds-accounting.json","sha256":"38be5988391c96acfa7eac25248df674e0fe95e2b58f22ea47fa909a02f01576"},{"kinds":["scenario"],"path":"scenarios/progression-support-refunds-resolution.json","sha256":"2627a122fd791ed4ed2c4c08e7fea283e017bfe59dd1a682899207cc89d7ee67"},{"kinds":["scenario"],"path":"scenarios/progression-support-return-interaction.json","sha256":"7c50a90555a616e9e8ce7fefb771e79237ed2890b28545a5951da2fa52902820"},{"kinds":["scenario"],"path":"scenarios/progression-support-triage.json","sha256":"a0ad2546ae46257154a75fa3250353cd443f1c6dc93ff52b4e666e99444101f5"},{"kinds":["track-manifest"],"path":"track.json","sha256":"ae917ab791cfc59a6ece01cc23ab433a235cc5fb20ab37974d8ed36c494a67d9"}],"sourceManifestSha256":"82b2d124ed616d2690d1b8ae082cb1b6e3903cda31500662b334b4fd8cecf49f","task":{"baseRecipe":null,"composedSha256":"d063ef87268f9c5a4bb2e7c0f4570cd2ac6b50e9f45e1ca1c1df76049731f650","contractSha256":"e432e08fc81f67f95f38c44e05fc2adc393f10cd29d67af78d0f1676e73b7578","contracts":[{"from":null,"id":"ecommerce.application-interface","modes":["fresh","upgrade"],"order":0,"owners":["recipe"],"path":"contracts/application-interface.md","sha256":"b770db6c1dbf92c106c7501a17028876a30a63655c8f88b5667b44557f401692","until":null},{"from":null,"id":"ecommerce.progression.staff-access-hooks","modes":["fresh","upgrade"],"order":1900,"owners":["ecommerce.progression.staff-access"],"path":"contracts/staff-access.md","sha256":"dcd7b740d5f65160ddf0b4221d29f84426ee89f04560e953add0856cf0bca87a","until":null},{"from":null,"id":"ecommerce.progression.customer-profile-hooks","modes":["upgrade"],"order":1950,"owners":["ecommerce.progression.customer-profile"],"path":"contracts/customer-profile.md","sha256":"7451bf14ada950c9b7ad269d63467154c388e9ea18b97f479037cdd397ca519f","until":null},{"from":null,"id":"ecommerce.progression.staff-role-hooks","modes":["upgrade"],"order":1960,"owners":["ecommerce.progression.staff-roles"],"path":"contracts/staff-roles.md","sha256":"cce6145182485fafcd2adc29cec90af20523a2afd9c952800eefb5d15a49f96e","until":null},{"from":null,"id":"ecommerce.progression.catalog-management-hooks","modes":["upgrade"],"order":1970,"owners":["ecommerce.progression.catalog-management"],"path":"contracts/catalog-management.md","sha256":"5123971185ded331deaef2e323fad31e6133d55fc839289e2c4a0dfc5292ee4e","until":null},{"from":null,"id":"ecommerce.progression.payment-record-hooks","modes":["upgrade"],"order":1980,"owners":["ecommerce.progression.payment-records"],"path":"contracts/payment-records.md","sha256":"8c752352c7f47a0a65d1823f9ec74009dc95088fc6c2a6cc30d1a1dc7fa11fe0","until":null},{"from":null,"id":"ecommerce.progression.staff-activity-hooks","modes":["upgrade"],"order":1990,"owners":["ecommerce.progression.staff-activity"],"path":"contracts/staff-activity.md","sha256":"7e7c1846445f82aa78ad8e32606e6830f9d326ebfea75d13b077c073695c9594","until":null},{"from":null,"id":"ecommerce.feature.catalog-items.hooks","modes":["fresh","upgrade"],"order":2000,"owners":["ecommerce.feature.catalog-items"],"path":"contracts/catalog-items.md","sha256":"596ec08b1f4a6b290595570f3bcf2847a1f60feeb20d90f98d25761666bfe9f2","until":null},{"from":null,"id":"ecommerce.feature.catalog-discovery.hooks","modes":["fresh","upgrade"],"order":2010,"owners":["ecommerce.feature.catalog-discovery"],"path":"contracts/catalog-discovery.md","sha256":"d13ea495f78e3827b7a243b97466aefbf0252980c76edb432b6b0506854e8450","until":null},{"from":null,"id":"ecommerce.l2.transfer-hooks","modes":["upgrade"],"order":2031,"owners":["ecommerce.l2.stock-transfers-features"],"path":"contracts/stock-transfers.md","sha256":"01f28ebb3487559d4689919f0991e43f74fcc85609fa4c810e92adde30e84096","until":null},{"from":null,"id":"ecommerce.l2.price-hooks","modes":["upgrade"],"order":2041,"owners":["ecommerce.l2.price-history-features"],"path":"contracts/price-history.md","sha256":"3465714f91e0e978607900b19377a134e9beb704848e2462b1438465cc2e98ee","until":null},{"from":"# Price history completed-order interface","id":"ecommerce.progression.price-history-order-hooks","modes":["upgrade"],"order":2042,"owners":["ecommerce.progression.price-history-specifications"],"path":"contracts/price-history-orders.md","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.price-history-features"],"sha256":"45d360052ede67e50e12b5c0e64904dbbdd1dd17a430c38130a2b8a70d274278","until":"# Price history cart and checkout interface"},{"from":"# Price history cart and checkout interface","id":"ecommerce.progression.price-history-cart-hooks","modes":["upgrade"],"order":2043,"owners":["ecommerce.progression.price-history-specifications"],"path":"contracts/price-history-orders.md","requiresFeatures":["ecommerce.feature.checkout","ecommerce.l2.price-history-features"],"sha256":"0d80d8f3bb9ced280a6203d2e8974b1a4db923b2d423205969ca1d44d002fc8b","until":null},{"from":null,"id":"ecommerce.l2.inventory-dashboard-hooks","modes":["upgrade"],"order":2050,"owners":["ecommerce.l2.inventory-dashboard"],"path":"contracts/inventory-dashboard.md","sha256":"8c3de4ff8a958776ca087febbbbf5f950eaef9efacb96b7c4fa49357b2d0a371","until":null},{"from":null,"id":"ecommerce.l2.order-cancellation-hooks","modes":["upgrade"],"order":2060,"owners":["ecommerce.l2.order-cancellation-features"],"path":"contracts/order-cancellation.md","sha256":"fc274b1409c50de8edcf797ab83cf8e8c46a54db30ada859f854e1cd40c7dee1","until":null},{"from":null,"id":"ecommerce.l2.sales-dashboard-hooks","modes":["upgrade"],"order":2060,"owners":["ecommerce.l2.sales-dashboard"],"path":"contracts/sales-dashboard.md","sha256":"ff9ed3890aa5c8303a0badecf946e7d98da3fbbb0a2890a693099bab1a2056c0","until":null},{"from":null,"id":"ecommerce.l2.recommendations-hooks","modes":["upgrade"],"order":2070,"owners":["ecommerce.l2.recommendations"],"path":"contracts/recommendations.md","sha256":"ab5417cbf78f43464e82a543e50e9601580bb2b3a57826a9e73da9899f39de16","until":null},{"from":null,"id":"ecommerce.feature.accounts.hooks","modes":["fresh","upgrade"],"order":2100,"owners":["ecommerce.feature.accounts"],"path":"contracts/accounts.md","sha256":"e281aa0cff56c23117203b13d11f6787a7f8163cb8d1b7bc0d2d86b5390cfff6","until":null},{"from":null,"id":"ecommerce.feature.purchasing.hooks","modes":["fresh","upgrade"],"order":2200,"owners":["ecommerce.feature.purchasing"],"path":"contracts/purchasing.md","sha256":"c2d59da67b400d055ab90bf53f2e46a3af9605a1fc68534c2b64d8db2ed3409c","until":null},{"from":null,"id":"ecommerce.feature.cart.hooks","modes":["fresh","upgrade"],"order":2300,"owners":["ecommerce.feature.cart"],"path":"contracts/cart.md","sha256":"bb1b8741b9eb47b5766a330bb1fc23005f7540fdc79d20efdaefb183cf50918d","until":null},{"from":null,"id":"ecommerce.feature.checkout.hooks","modes":["fresh","upgrade"],"order":2310,"owners":["ecommerce.feature.checkout"],"path":"contracts/checkout.md","sha256":"49416683e06d996a28980ff650a3281f89cae062269b7de761899e71759c731c","until":null},{"from":null,"id":"ecommerce.orders.data","modes":["fresh","upgrade"],"order":2315,"owners":["ecommerce.feature.checkout","ecommerce.feature.purchasing"],"path":"contracts/order-data.md","sha256":"7b7cfc6b102f4836235c7f23344865f60156b3e4ac96c3e5cbb0e298390b3b06","until":null},{"from":null,"id":"ecommerce.feature.reviews.hooks","modes":["fresh","upgrade"],"order":2400,"owners":["ecommerce.feature.reviews"],"path":"contracts/reviews.md","sha256":"6be3567258c0c4bf3331e609f63be26ad0bb43ceca3e7da4accaaad8c9ac6541","until":null},{"from":null,"id":"ecommerce.feature.warehouse-admin.hooks","modes":["fresh","upgrade"],"order":2500,"owners":["ecommerce.feature.warehouse-admin"],"path":"contracts/warehouse-administration.md","sha256":"ea037935ce439d10f4ccea3296bb16aa2ddee4e0c13b2b7ade0638ff7cb9b8d6","until":null},{"from":null,"id":"ecommerce.progression.support-history-hooks","modes":["upgrade"],"order":2900,"owners":["ecommerce.progression.support-history"],"path":"contracts/support-history.md","sha256":"610022f964c2aa00b409fb2723f91b885e306edde5bf73390e880cffd2a5583d","until":null},{"from":null,"id":"ecommerce.progression.support-intake-hooks","modes":["fresh","upgrade"],"order":2900,"owners":["ecommerce.progression.support-intake"],"path":"contracts/support-intake.md","sha256":"fd9f732acf67d6c772f4c05d626e4e9e472818fc5226b7ef3d7223b85a0e0bc7","until":null},{"from":null,"id":"ecommerce.progression.support-triage-hooks","modes":["upgrade"],"order":2900,"owners":["ecommerce.progression.support-triage"],"path":"contracts/support-triage.md","sha256":"8079080a299674bb0558a6550243051360be1c0f18252708128c3f71af899a0d","until":null},{"from":null,"id":"ecommerce.progression.fulfilment-queue-hooks","modes":["upgrade"],"order":2920,"owners":["ecommerce.progression.fulfilment-queue"],"path":"contracts/operations-access.md","sha256":"fd9c31482b484dc1f9bf3c521d247971334e1f52f51a7548a987bdebc65dbdb5","until":null},{"from":null,"id":"ecommerce.progression.promotion-rules-hooks","modes":["upgrade"],"order":2950,"owners":["ecommerce.progression.promotion-rules"],"path":"contracts/promotion-rules.md","sha256":"f352f8bb938aa874125b10bbb6182643f04b6d1ce82cde41512bd16835a7b90b","until":null},{"from":null,"id":"ecommerce.progression.notification-preferences-hooks","modes":["upgrade"],"order":2990,"owners":["ecommerce.progression.notification-preferences"],"path":"contracts/notification-preferences.md","sha256":"d6b4231f67d5e70a03ec331545b6419acf766b52841c8520b94963c16aa13fec","until":null},{"from":null,"id":"ecommerce.l3.reservation-hooks","modes":["upgrade"],"order":3900,"owners":["ecommerce.l3.reservations-features"],"path":"contracts/reservations.md","sha256":"e1c4af7a67dd200d5f18d550e5b4c30e7a4bbaf60600b434108ebff7bdf90627","until":null},{"from":null,"id":"ecommerce.progression.managed-support-hooks","modes":["upgrade"],"order":3900,"owners":["ecommerce.progression.managed-support"],"path":"contracts/managed-support.md","sha256":"458b89a7f786086a95957c778943660068cd6c00b30635fd1cd786c37df76398","until":null},{"from":null,"id":"ecommerce.l3.scheduled-restock-hooks","modes":["upgrade"],"order":3910,"owners":["ecommerce.l3.scheduled-restocks-features"],"path":"contracts/scheduled-restocks.md","sha256":"a946c0954a641d4fe64b933761e2b92d546ae08ee434d676c6faed3ed8b03c8e","until":null},{"from":null,"id":"ecommerce.l3.order-delivery-hooks","modes":["upgrade"],"order":3920,"owners":["ecommerce.l3.order-delivery-features"],"path":"contracts/order-delivery.md","sha256":"74bd21b39c8d5f020ff62cc408992c15788e48eba8e05a76a0cddc4b4920a98e","until":null},{"from":null,"id":"ecommerce.l3.cart-expiration-hooks","modes":["upgrade"],"order":3930,"owners":["ecommerce.l3.cart-expiration-features"],"path":"contracts/cart-expiration.md","sha256":"56c0966494cd19b629140f8b8d53e8b9838e34668a5cab6b7cccb41b78734c50","until":null},{"from":null,"id":"ecommerce.progression.promotion-checkout-hooks","modes":["upgrade"],"order":3950,"owners":["ecommerce.progression.promotion-checkout"],"path":"contracts/promotion-checkout.md","sha256":"b74e703dc77c9eeec26f5beb41cab102570c8314d817015c858557e9946293aa","until":null},{"from":null,"id":"ecommerce.progression.stock-alert-hooks","modes":["upgrade"],"order":3990,"owners":["ecommerce.progression.stock-alerts"],"path":"contracts/stock-alerts.md","sha256":"405f106569c84b3062f8159452730f16adadf12d804c554a570e39b958b63421","until":null},{"from":null,"id":"ecommerce.l3.order-return-hooks","modes":["upgrade"],"order":4041,"owners":["ecommerce.l3.order-returns-features"],"path":"contracts/order-returns.md","sha256":"fe0668ab3bf00d4c167794dbecb75b26b23cb62dfc0a1b79636a013edb237329","until":null},{"from":null,"id":"ecommerce.progression.faceted-search-hooks","modes":["upgrade"],"order":4900,"owners":["ecommerce.progression.faceted-search"],"path":"contracts/faceted-search.md","sha256":"08ace4d2d1947f16ceb4da95e6993086c9bd77a0977c258fef1f0a96c7e7879d","until":null},{"from":null,"id":"ecommerce.progression.order-support-hooks","modes":["upgrade"],"order":4900,"owners":["ecommerce.progression.order-support"],"path":"contracts/order-support.md","sha256":"a2901e66af870055b6c3c26f240132ae08f3cb93d0c52a7b2573c9dc153e1e03","until":null},{"from":null,"id":"ecommerce.progression.personalized-recommendation-hooks","modes":["upgrade"],"order":4910,"owners":["ecommerce.progression.personalized-recommendations"],"path":"contracts/progression-personalized-recommendations.md","sha256":"42389a6d284dcfdb53a610ce5ce77d40dbc2b9d5e177bbd3f6bccc40079c7d98","until":null},{"from":null,"id":"ecommerce.progression.promotion-reporting-hooks","modes":["upgrade"],"order":4950,"owners":["ecommerce.progression.promotion-reporting"],"path":"contracts/promotion-reporting.md","sha256":"81cded5fd9a8822c8756ea50bedb879971c94f8727d176f22fa53d2a6a1efbb0","until":null},{"from":null,"id":"ecommerce.feature.store-credit.interface","modes":["upgrade"],"order":5900,"owners":["ecommerce.feature.store-credit"],"path":"contracts/store-credit.md","sha256":"505ca48dfc4f9f2d6ea38e53e1bd382910264d0195be8580b5fb01f7e7074713","until":null},{"from":null,"id":"ecommerce.feature.subscriptions.interface","modes":["upgrade"],"order":5900,"owners":["ecommerce.feature.subscriptions"],"path":"contracts/subscriptions.md","sha256":"dc533bc7f3b9be03ae7d9e18c2e02ffa996bf934b9ba230826d492b4cefa045e","until":null},{"from":null,"id":"ecommerce.progression.delivery-notification-hooks","modes":["upgrade"],"order":5900,"owners":["ecommerce.progression.delivery-notifications"],"path":"contracts/delivery-notifications.md","sha256":"4bf565ea751bb5278bae5910420beeb531a2eb88fb5755e5a06dda68860af490","until":null},{"from":null,"id":"ecommerce.progression.support-refund-hooks","modes":["upgrade"],"order":5900,"owners":["ecommerce.progression.support-refunds"],"path":"contracts/support-refunds.md","sha256":"da5d0e69bbf3a028dc79f207b940bbd4a79e7f82c68d89f252eb8c73ea16b8bc","until":null},{"from":null,"id":"ecommerce.progression.automatic-reorder-hooks","modes":["upgrade"],"order":5910,"owners":["ecommerce.progression.automatic-reorder"],"path":"contracts/progression-automatic-reorder.md","sha256":"3e70361965e6ddeb7b05cc7ad5348e363d58d628734de96b8a9792e68e71cc72","until":null},{"from":null,"id":"ecommerce.progression.cart-recovery-hooks","modes":["upgrade"],"order":5920,"owners":["ecommerce.progression.cart-recovery"],"path":"contracts/progression-cart-recovery.md","sha256":"daf04aea3082483caa2ebf3a3c89a438329600cb6aa5d74fb47d97bc244e5747","until":null},{"from":null,"id":"ecommerce.progression.recommendation-feedback-hooks","modes":["upgrade"],"order":5930,"owners":["ecommerce.progression.recommendation-feedback"],"path":"contracts/recommendation-feedback.md","sha256":"00a96f594c8da1ffb4a933c6f8d8d0219216ab6cc2c781cfd17dd9c97c517d58","until":null},{"from":null,"id":"ecommerce.feature.split-tender-refunds.interface","modes":["upgrade"],"order":6900,"owners":["ecommerce.feature.split-tender-refunds"],"path":"contracts/split-tender-refunds.md","sha256":"62b8b3c233d33858e9b696d06d94c70f6524f8cbed77028acc0d2affdd780841","until":null},{"from":null,"id":"ecommerce.interface.product-bundles","modes":["upgrade"],"order":8104,"owners":["ecommerce.feature.product-bundles"],"path":"contracts/product-bundles.md","sha256":"b27905002b693e6c63532cd8193c8e3fd1fdeaad85041d84b8240a2092bf1c57","until":null},{"from":null,"id":"ecommerce.interface.bundle-checkout","modes":["upgrade"],"order":8105,"owners":["ecommerce.feature.bundle-checkout"],"path":"contracts/bundle-checkout.md","sha256":"2c4e70b620934dc927ad4c84c17e54b6e9cef665ba3b098f9f2fc9e79a3c4c3e","until":null},{"from":null,"id":"ecommerce.interface.bundle-returns","modes":["upgrade"],"order":8106,"owners":["ecommerce.feature.bundle-returns"],"path":"contracts/bundle-returns.md","sha256":"69f8bf6afc325a98bab38235e49eaa2875bc674e6a61c7559692593018c9f527","until":null}],"mode":"action","requirementSha256":"bd057cbfbcae06db5bdd6a7eb20d03ae00f9c16cff9841199b342a7df5b19279","requirements":[{"from":null,"id":"ecommerce.progression.fresh","modes":["fresh"],"order":0,"owners":["recipe"],"path":"prompts/modular/progression-framing.md","sha256":"7c3267ef4ef6a454ba26217d8a0efc47f1cbe655970fbb1ddfd2ae7637e17df1","until":"## Existing application"},{"from":"## Existing application","id":"ecommerce.progression.upgrade","modes":["upgrade"],"order":0,"owners":["recipe"],"path":"prompts/modular/progression-framing.md","sha256":"ad2dc327dea4410a715b013d9455c37060a193b844c7de7967ce760ef1c2c118","until":null},{"from":null,"id":"ecommerce.l2.inventory-dashboard","modes":["upgrade"],"order":50,"owners":["ecommerce.l2.inventory-dashboard"],"path":"prompts/modular/inventory-dashboard.md","sha256":"d2190c809ce459141cfdb716d25d38b01f7aca2ce8b8f74128e731e4e3fede07","until":null},{"from":null,"id":"ecommerce.l2.sales-dashboard","modes":["upgrade"],"order":60,"owners":["ecommerce.l2.sales-dashboard"],"path":"prompts/modular/sales-dashboard.md","sha256":"c65aaadb1f8da6e2b5f7967cc3bcf07beb0e7a860351e866373667bc33012dbd","until":null},{"from":null,"id":"ecommerce.l2.recommendations","modes":["upgrade"],"order":70,"owners":["ecommerce.l2.recommendations"],"path":"prompts/modular/recommendations.md","sha256":"0eed3feb6fc6acf86260d74eeff557b15837aaa09a857c63cf9dee848cd7201f","until":null},{"from":null,"id":"ecommerce.feature.accounts.requirement","modes":["fresh","upgrade"],"order":100,"owners":["ecommerce.feature.accounts"],"path":"prompts/modular/accounts.md","sha256":"0a8dee0847a02da7777c11c4175577c4533fafa538fc09e5124cf4d825e4c96f","until":null},{"from":null,"id":"ecommerce.feature.catalog-items.requirement","modes":["fresh","upgrade"],"order":200,"owners":["ecommerce.feature.catalog-items"],"path":"prompts/modular/catalog-items.md","sha256":"36a72da77f42d856b691c93dc587fee6d9d7f88b722a9893fc226b1d5bb5d3fe","until":null},{"from":null,"id":"ecommerce.feature.catalog-discovery.requirement","modes":["fresh","upgrade"],"order":210,"owners":["ecommerce.feature.catalog-discovery"],"path":"prompts/modular/catalog-discovery.md","sha256":"9a0785c55d17e577b84ef257de482bcd8bc8c961e9d0760c38472936a1a0062b","until":null},{"from":null,"id":"ecommerce.feature.purchasing.requirement","modes":["fresh","upgrade"],"order":300,"owners":["ecommerce.feature.purchasing"],"path":"prompts/modular/purchasing.md","sha256":"059073391cf92eef2fb1edb710ec1a62747e82115c2bd0251274111b92f93e46","until":null},{"from":null,"id":"ecommerce.feature.cart.requirement","modes":["fresh","upgrade"],"order":400,"owners":["ecommerce.feature.cart"],"path":"prompts/modular/cart.md","sha256":"da024de4032661a46574c8ac8dbe3d1baf45ef6ea7115ba1976d7d5c1044afbe","until":null},{"from":null,"id":"ecommerce.feature.checkout.requirement","modes":["fresh","upgrade"],"order":410,"owners":["ecommerce.feature.checkout"],"path":"prompts/modular/checkout.md","sha256":"bb834dec7858349a17a8980eb13459a606cdbd2b1c534f468f666459c1a24ba7","until":null},{"from":null,"id":"ecommerce.feature.reviews.requirement","modes":["fresh","upgrade"],"order":500,"owners":["ecommerce.feature.reviews"],"path":"prompts/modular/reviews.md","sha256":"1649822528e1bbcfedc59b3c8c5d7ff1b5f0e42e1e82b81ee7e59361cbefab33","until":null},{"from":null,"id":"ecommerce.feature.warehouse-admin.requirement","modes":["fresh","upgrade"],"order":600,"owners":["ecommerce.feature.warehouse-admin"],"path":"prompts/modular/warehouse-administration.md","sha256":"8fa2d7b54b3c3d9601ae69bbfee5bdf389f68fea0d0ef4032201af12d3f424a2","until":null},{"from":"## Access control: purchasing","id":"ecommerce.spec.access-control.purchasing","modes":["fresh"],"order":1000,"owners":["ecommerce.spec.access-control"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.purchasing"],"sha256":"45c2ac9e423af1431c6a9d8b7de707b7d6a8402c004c5472c56d7efe387e61a1","until":"## Access control: warehouse administration"},{"from":"## Access control: warehouse administration","id":"ecommerce.spec.access-control.warehouse-admin","modes":["fresh"],"order":1010,"owners":["ecommerce.spec.access-control"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.warehouse-admin"],"sha256":"02500e3bded8076287f0b4c7264fe48ff3c766b0ab9aa386497c5cc66fc725f4","until":"## Access control: reviews"},{"from":"## Access control: reviews","id":"ecommerce.spec.access-control.reviews","modes":["fresh"],"order":1020,"owners":["ecommerce.spec.access-control"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.reviews"],"sha256":"10d4140e8f649ddf46843100231683f13143c09226195cfe487827b662333f4d","until":"## Access control: cart"},{"from":"## Access control: cart","id":"ecommerce.spec.access-control.cart","modes":["fresh"],"order":1030,"owners":["ecommerce.spec.access-control"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.cart"],"sha256":"353e3d5daa87c988456b392ad53fd93bdf473b5a860049104cd5acdaddf05cee","until":"## State durability: accounts"},{"from":null,"id":"ecommerce.progression.staff-access","modes":["fresh","upgrade"],"order":1100,"owners":["ecommerce.progression.staff-access"],"path":"prompts/modular/staff-access.md","sha256":"10c6aedbc5c60b440d6d0b5c38bb57363ae021c54d62ee2ffe2294d7f6741550","until":null},{"from":"## State durability: accounts","id":"ecommerce.spec.state-durability.accounts","modes":["fresh"],"order":1100,"owners":["ecommerce.spec.state-durability"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.accounts"],"sha256":"3466c8854dd16c38beafe194fb402f8f5c5539defb90a9b856c357347255c3ee","until":"## State durability: account data"},{"from":"## State durability: account data","id":"ecommerce.spec.state-durability.account-data","modes":["fresh"],"order":1110,"owners":["ecommerce.spec.state-durability"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.accounts","ecommerce.feature.cart"],"sha256":"db7e1e0fbf6e6f2956f28ef00dfc7191c90ca05349a605f246776ff3c6f4e2d9","until":"## Live state: catalog and purchasing"},{"from":null,"id":"ecommerce.spec.state-durability.checkout-crash","modes":["fresh","upgrade"],"order":1120,"owners":["ecommerce.spec.state-durability"],"path":"prompts/modular/checkout-recovery-specification.md","requiresFeatures":["ecommerce.feature.checkout"],"sha256":"0e851fea800d81cb558f8004f04e628e5ccf9f6e5589a9d2f6fceaf33e50f207","until":null},{"from":null,"id":"ecommerce.progression.customer-profile","modes":["upgrade"],"order":1200,"owners":["ecommerce.progression.customer-profile"],"path":"prompts/modular/customer-profile.md","sha256":"f4ffc5d2a642bbccec9eb5c428a37b2d9bfaa1c9f96648d8544bfcf2fe8187bb","until":null},{"from":null,"id":"ecommerce.progression.support-intake","modes":["fresh","upgrade"],"order":1200,"owners":["ecommerce.progression.support-intake"],"path":"prompts/modular/support-intake.md","sha256":"048f11b71b66dadf80c77b5d9aef8bbff2b2030171eeda5012dce74ae16421ed","until":null},{"from":"## Live state: catalog and purchasing","id":"ecommerce.spec.live-state.catalog-purchasing","modes":["fresh"],"order":1200,"owners":["ecommerce.spec.live-state"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing"],"sha256":"3bfd8cf26509ffeeb475df0eee040f4743cc31acd9e8292e4144b46055686b55","until":"## Live state: cart"},{"from":"## Live state: cart","id":"ecommerce.spec.live-state.cart","modes":["fresh"],"order":1210,"owners":["ecommerce.spec.live-state"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.cart"],"sha256":"244ee5cf7387c74dd1776e659964e782f042229eff84918de2517781ad095045","until":"## Live state: reviews"},{"from":"## Live state: reviews","id":"ecommerce.spec.live-state.reviews","modes":["fresh"],"order":1220,"owners":["ecommerce.spec.live-state"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.reviews"],"sha256":"aeeab5a0dda0f1df6b1ab3ae78d3c1c6a9ff2f108ac1d09e904c27b7ee71100d","until":"## Live state: warehouse administration"},{"from":"## Live state: warehouse administration","id":"ecommerce.spec.live-state.warehouse-admin","modes":["fresh"],"order":1230,"owners":["ecommerce.spec.live-state"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.warehouse-admin"],"sha256":"362d4ef3422a09d060cdd5b09837b0a4a1f67b8fb108f59dcf7afb4a12d2a4e5","until":"## Concurrency safety: purchasing"},{"from":null,"id":"ecommerce.progression.staff-roles","modes":["upgrade"],"order":1250,"owners":["ecommerce.progression.staff-roles"],"path":"prompts/modular/staff-roles.md","sha256":"175a323c92238509111aa9c366e4ec302d508edc872077ecd26d8acbcbe21ba9","until":null},{"from":null,"id":"ecommerce.progression.catalog-management","modes":["upgrade"],"order":1300,"owners":["ecommerce.progression.catalog-management"],"path":"prompts/modular/catalog-management.md","sha256":"93fece29e1d68513732398c815fe4ebc83079e705e2b9de40df543281439f390","until":null},{"from":"## Concurrency safety: purchasing","id":"ecommerce.spec.concurrency-safety.purchasing","modes":["fresh"],"order":1300,"owners":["ecommerce.spec.concurrency-safety"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.purchasing"],"sha256":"e2995c00f9971e1389bc03b56faf4e03da9c83a84cb205ddfeb84964bd4a905f","until":"## Concurrency safety: restocking"},{"from":"## Concurrency safety: restocking","id":"ecommerce.spec.concurrency-safety.restocking","modes":["fresh"],"order":1310,"owners":["ecommerce.spec.concurrency-safety"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"sha256":"958e12514579d21f92a73806ba6983912f1828f40f51efeb16947e7a6c3d539e","until":"## Concurrency safety: checkout"},{"from":"## Concurrency safety: checkout","id":"ecommerce.spec.concurrency-safety.checkout","modes":["fresh"],"order":1320,"owners":["ecommerce.spec.concurrency-safety"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.checkout"],"sha256":"0bb644068b430d491cfd6c6f5af00cf16cff7ec930a1d8b3150493e4d0bebbe5","until":"## Transactional integrity: reviews"},{"from":null,"id":"ecommerce.progression.payment-records","modes":["upgrade"],"order":1350,"owners":["ecommerce.progression.payment-records"],"path":"prompts/modular/payment-records.md","sha256":"ab5c9c913041e324cad82fe6a135d78906818afa3c9a5c204289142c1d696f09","until":null},{"from":null,"id":"ecommerce.progression.staff-activity","modes":["upgrade"],"order":1400,"owners":["ecommerce.progression.staff-activity"],"path":"prompts/modular/staff-activity.md","sha256":"0cb274acefc0d75cd82415de0eb167133db0412e1de4f0fd59f62983d29a1476","until":null},{"from":"## Transactional integrity: reviews","id":"ecommerce.spec.transactional-integrity.reviews","modes":["fresh"],"order":1400,"owners":["ecommerce.spec.transactional-integrity"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.reviews"],"sha256":"715afabddf7a8ed221f38fdba2b89dad7d0b656c62f5e607153ef39ec52e2fc0","until":"## Transactional integrity: purchasing"},{"from":"## Transactional integrity: purchasing","id":"ecommerce.spec.transactional-integrity.purchasing","modes":["fresh"],"order":1410,"owners":["ecommerce.spec.transactional-integrity"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.purchasing"],"sha256":"1452a3c92e5a35b5bed9273e146de8d1dda3011105ec5d79821462629ef56c8d","until":"## Transactional integrity: warehouse accounting"},{"from":"## Transactional integrity: warehouse accounting","id":"ecommerce.spec.transactional-integrity.warehouse-accounting","modes":["fresh"],"order":1420,"owners":["ecommerce.spec.transactional-integrity"],"path":"prompts/modular/l1-specifications.md","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"sha256":"667b2197f1e9c20e22f03abc7cc756b98b421377adfdb3fd06f35005837c3948","until":"## External data synchronization"},{"from":"## External data synchronization","id":"ecommerce.spec.external-data-sync.requirement","modes":["fresh","upgrade"],"order":1500,"owners":["ecommerce.spec.external-data-sync"],"path":"prompts/modular/l1-external-sync-specifications.md","requiresFeatures":["ecommerce.feature.warehouse-admin"],"sha256":"0e3db28232aedb57b3fb2fbb9afa3ca65fcad28b8c050e1965feee4db32e4239","until":null},{"from":null,"id":"ecommerce.progression.fulfilment-queue","modes":["upgrade"],"order":2020,"owners":["ecommerce.progression.fulfilment-queue"],"path":"prompts/modular/operations-access.md","sha256":"1699a8d0466e115874e49b8ae73bb0004c3c620a03d3aa04dd83818b6c798003","until":null},{"from":null,"id":"ecommerce.l2.stock-transfer","modes":["upgrade"],"order":2030,"owners":["ecommerce.l2.stock-transfers-features"],"path":"prompts/modular/stock-transfers.md","sha256":"6e2f0735d81fd5376b851893404873af4807b83e807e078cca0b8c9a9920e627","until":null},{"from":null,"id":"ecommerce.l2.order-cancellation","modes":["upgrade"],"order":2040,"owners":["ecommerce.l2.order-cancellation-features"],"path":"prompts/modular/order-cancellation.md","sha256":"7d429d4e8db546961e7ec9a12fb5dcba5efd0562c8059ab354121fd5ed13ea84","until":null},{"from":null,"id":"ecommerce.l2.price-history","modes":["upgrade"],"order":2040,"owners":["ecommerce.l2.price-history-features"],"path":"prompts/modular/price-history.md","sha256":"9edb3db98af12f3a08cae7887dd6da91c746be49597f086457fc2ecb70c558b8","until":null},{"from":"## Price history: completed orders","id":"ecommerce.progression.price-history-orders","modes":["upgrade"],"order":2041,"owners":["ecommerce.progression.price-history-specifications"],"path":"prompts/modular/price-history-orders.md","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.price-history-features"],"sha256":"12abf44d08a49b076c721b62c4fc240a82c0fb350b10a3eb052afcf2ee78c6b6","until":"## Price history: cart and checkout"},{"from":"## Price history: cart and checkout","id":"ecommerce.progression.price-history-cart-checkout","modes":["upgrade"],"order":2042,"owners":["ecommerce.progression.price-history-specifications"],"path":"prompts/modular/price-history-orders.md","requiresFeatures":["ecommerce.feature.checkout","ecommerce.l2.price-history-features"],"sha256":"86caeac95b2d3cb1fc0ec0d467ee52303218d087ca30c24eb38c7c7714c8ec2b","until":null},{"from":null,"id":"ecommerce.progression.cancellation-queue","modes":["upgrade"],"order":2045,"owners":["ecommerce.progression.cancellation-queue-specifications"],"path":"prompts/modular/cancellation-queue.md","requiresFeatures":["ecommerce.l2.order-cancellation-features","ecommerce.progression.fulfilment-queue"],"sha256":"7e1adf98391a930017acb5c37d46c9b0d17d89e58c939fc57dc52f22e2f86f31","until":null},{"from":"## Cancellation accounting","id":"ecommerce.progression.cancellation-accounting","modes":["upgrade"],"order":2100,"owners":["ecommerce.progression.cancellation-accounting-specifications"],"path":"prompts/modular/order-accounting.md","requiresFeatures":["ecommerce.l2.order-cancellation-features"],"sha256":"a31e83bef57e79d545be549b9905d1b1f178e40a34e676ec18a62dfddf916d8d","until":"## Price accounting"},{"from":null,"id":"ecommerce.progression.support-triage","modes":["upgrade"],"order":2100,"owners":["ecommerce.progression.support-triage"],"path":"prompts/modular/support-triage.md","sha256":"bbb35abcbae9b8d3782ee3eb94f3175428edfe3fa564d6b0e3af2e0050e10dee","until":null},{"from":"## Price accounting","id":"ecommerce.progression.price-accounting","modes":["upgrade"],"order":2110,"owners":["ecommerce.progression.price-accounting-specifications"],"path":"prompts/modular/order-accounting.md","requiresFeatures":["ecommerce.l2.price-history-features"],"sha256":"64f96277738b0a33453e69bdcb1d79971d979a57ec27b8caacc96b499bc43de5","until":null},{"from":null,"id":"ecommerce.progression.support-history","modes":["upgrade"],"order":2110,"owners":["ecommerce.progression.support-history"],"path":"prompts/modular/support-history.md","sha256":"df2ca2664901a90814eab17d5f7c029595ef614d76ee29eeb4204d9f4f371429","until":null},{"from":null,"id":"ecommerce.progression.promotion-rules","modes":["upgrade"],"order":2200,"owners":["ecommerce.progression.promotion-rules"],"path":"prompts/modular/promotion-rules.md","sha256":"7a43b632e8fd43b8060100f9614476e41003e47afa04dd1b7c3b09639f6280b2","until":null},{"from":null,"id":"ecommerce.progression.notification-preferences","modes":["upgrade"],"order":2300,"owners":["ecommerce.progression.notification-preferences"],"path":"prompts/modular/notification-preferences.md","sha256":"496baa70ef300e6874f6dbf58ccd2bac594e53101875fd68740e27779d11871b","until":null},{"from":null,"id":"ecommerce.progression.transfer-authorization","modes":["upgrade"],"order":2930,"owners":["ecommerce.progression.operations-access-specifications"],"path":"prompts/modular/transfer-authorization.md","requiresFeatures":["ecommerce.l2.stock-transfers-features"],"sha256":"6d1e33c7a6aa209ad3f2705607a839dde5c915b65d78a8bb10b9e0da29172c0b","until":null},{"from":null,"id":"ecommerce.progression.price-authorization","modes":["upgrade"],"order":2940,"owners":["ecommerce.progression.operations-access-specifications"],"path":"prompts/modular/price-authorization.md","requiresFeatures":["ecommerce.l2.price-history-features"],"sha256":"6e9f949bd45f3bc816336de6e7db6dd18254ceebe6ecacc8e0abfd5f3887b8b5","until":null},{"from":null,"id":"ecommerce.progression.shipping-authorization","modes":["upgrade"],"order":2950,"owners":["ecommerce.progression.operations-access-specifications"],"path":"prompts/modular/shipping-authorization.md","requiresFeatures":["ecommerce.progression.fulfilment-queue"],"sha256":"f336e9c8e1ea425c3dc44e72fabbc5e9e0e2a3541cab3750dd8e112b6c7997f8","until":null},{"from":null,"id":"ecommerce.progression.order-ownership","modes":["upgrade"],"order":2960,"owners":["ecommerce.progression.operations-access-specifications"],"path":"prompts/modular/order-ownership.md","requiresFeatures":["ecommerce.l2.order-cancellation-features"],"sha256":"1f4e5149768d1d0a4917ca9b711f1f40bf2c783ee5d59a42ab307f01babb052e","until":null},{"from":null,"id":"ecommerce.progression.review-access","modes":["upgrade"],"order":2970,"owners":["ecommerce.progression.review-access-specifications"],"path":"prompts/modular/review-access.md","requiresFeatures":["ecommerce.feature.reviews"],"sha256":"430fe60b2f028f67eab9f23b572238d4dacc60f7b89d74f8030618368455f371","until":null},{"from":null,"id":"ecommerce.progression.transfer-conservation","modes":["upgrade"],"order":2970,"owners":["ecommerce.progression.inventory-conservation-specifications"],"path":"prompts/modular/stock-conservation.md","requiresFeatures":["ecommerce.l2.stock-transfers-features"],"sha256":"4551438e045bb65b44127d3f92dcb79548c78dc1ecc42d2c7807a599d6c59c8f","until":null},{"from":null,"id":"ecommerce.progression.cancellation-conservation","modes":["upgrade"],"order":2980,"owners":["ecommerce.progression.inventory-conservation-specifications"],"path":"prompts/modular/cancellation-conservation.md","requiresFeatures":["ecommerce.l2.order-cancellation-features"],"sha256":"f383c20f16d75b77dfe2c0ca10dc1c77c998feb80b7c327616512b486c4714f8","until":null},{"from":null,"id":"ecommerce.l3.reservations","modes":["upgrade"],"order":3000,"owners":["ecommerce.l3.reservations-features"],"path":"prompts/modular/reservations.md","sha256":"e45b53568066c7200486453f78fdbbe89a116259a4d83d9435bde350316595e0","until":null},{"from":null,"id":"ecommerce.l3.scheduled-restocks","modes":["upgrade"],"order":3010,"owners":["ecommerce.l3.scheduled-restocks-features"],"path":"prompts/modular/scheduled-restocks.md","sha256":"ba32547038c681f7214219e5a10df75dbf925ee19d4ba60db5b291570f9c16ec","until":null},{"from":null,"id":"ecommerce.l3.order-delivery","modes":["upgrade"],"order":3020,"owners":["ecommerce.l3.order-delivery-features"],"path":"prompts/modular/order-delivery.md","sha256":"40483d0bf91c81efe57a61d62dbda756498792383bed9af595cdd8878a0b0cb8","until":null},{"from":null,"id":"ecommerce.l3.cart-expiration","modes":["upgrade"],"order":3030,"owners":["ecommerce.l3.cart-expiration-features"],"path":"prompts/modular/cart-expiration.md","sha256":"7c66696f044249d5f7f31487efb85e83f4c06b862fbaa3dd142ef2aef6afc2b7","until":null},{"from":"## Durable reservations","id":"ecommerce.l3.durable-reservations","modes":["upgrade"],"order":3100,"owners":["ecommerce.l3.deferred-durability-specifications"],"path":"prompts/modular/l3-specifications.md","requiresFeatures":["ecommerce.l3.reservations-features"],"sha256":"18ec31e622e6d392c68eacef275c6595125aa9bc45296e6064be50353b228bc6","until":"## Durable restocks"},{"from":null,"id":"ecommerce.progression.managed-support","modes":["upgrade"],"order":3100,"owners":["ecommerce.progression.managed-support"],"path":"prompts/modular/managed-support.md","sha256":"33abfe9e7514c6826df60572557d54f2e1570477afeae14c64bb3c5f3bad5fb6","until":null},{"from":"## Durable restocks","id":"ecommerce.l3.durable-restocks","modes":["upgrade"],"order":3101,"owners":["ecommerce.l3.deferred-durability-specifications"],"path":"prompts/modular/l3-specifications.md","requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"sha256":"40c3fac2bde0e10b85416906bcd0080e9748c916820ca44918074f2d342f7deb","until":"## Durable order delivery"},{"from":"## Durable order delivery","id":"ecommerce.l3.durable-order-delivery","modes":["upgrade"],"order":3102,"owners":["ecommerce.l3.deferred-durability-specifications"],"path":"prompts/modular/l3-specifications.md","requiresFeatures":["ecommerce.l3.order-delivery-features"],"sha256":"56a0e9d3113f8c49d563d6a99940e8ee272c32adb841c1dbb5533b17705a1e69","until":"## Durable cart expiration"},{"from":"## Durable cart expiration","id":"ecommerce.l3.durable-cart-expiration","modes":["upgrade"],"order":3103,"owners":["ecommerce.l3.deferred-durability-specifications"],"path":"prompts/modular/l3-specifications.md","requiresFeatures":["ecommerce.l3.cart-expiration-features"],"sha256":"1c81d51182e1a7648dc8e1e34cf1e80aade556a1eaeb63f79ba49b50bf2c2997","until":"## Exactly-once restocks"},{"from":"## Exactly-once restocks","id":"ecommerce.l3.exactly-once-restocks","modes":["upgrade"],"order":3110,"owners":["ecommerce.l3.deferred-integrity-specifications"],"path":"prompts/modular/l3-specifications.md","requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"sha256":"2fe8adca82a8fd5bb0fa1ba02cbb88e9f4ae3c799c49cf0980451d85603b6515","until":"## Exactly-once delivery"},{"from":"## Exactly-once delivery","id":"ecommerce.l3.exactly-once-delivery","modes":["upgrade"],"order":3111,"owners":["ecommerce.l3.deferred-integrity-specifications"],"path":"prompts/modular/l3-specifications.md","requiresFeatures":["ecommerce.l3.order-delivery-features"],"sha256":"70fa7a5b7e6a074950652a574684c4230545cd3b3b21645e568edf45dea5d44f","until":"## Server-timed restocks"},{"from":"## Server-timed restocks","id":"ecommerce.l3.server-timed-restocks","modes":["upgrade"],"order":3120,"owners":["ecommerce.l3.server-time-specifications"],"path":"prompts/modular/l3-specifications.md","requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"sha256":"ffb402340b8e0afa1973da971a48ed9534bf615e36c213dc892f7dc7260a9da3","until":"## Server-timed reservations"},{"from":"## Server-timed reservations","id":"ecommerce.l3.server-timed-reservations","modes":["upgrade"],"order":3121,"owners":["ecommerce.l3.server-time-specifications"],"path":"prompts/modular/l3-specifications.md","requiresFeatures":["ecommerce.l3.reservations-features"],"sha256":"108cb53bdda3f4a651347cec50a2236bb5cbbb01b9f92943fd6d287f151789bc","until":"## Deferred-work access"},{"from":"## Deferred-work access","id":"ecommerce.l3.deferred-access","modes":["upgrade"],"order":3130,"owners":["ecommerce.l3.deferred-access-specifications"],"path":"prompts/modular/l3-specifications.md","requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"sha256":"72874aad78d6da1f467443015f53f82514e45866b09a17c6435b16ec72b4e26f","until":"## Stock conservation"},{"from":"## Stock conservation","id":"ecommerce.l3.stock-conservation","modes":["upgrade"],"order":3140,"owners":["ecommerce.l3.deferred-integrity-specifications"],"path":"prompts/modular/l3-specifications.md","requiresFeatures":["ecommerce.l3.reservations-features"],"sha256":"cf7ef3381b7b3d4beee90d314b4a00d667a89909b67d6fd57f453329a6c48be3","until":null},{"from":null,"id":"ecommerce.progression.promotion-checkout","modes":["upgrade"],"order":3200,"owners":["ecommerce.progression.promotion-checkout"],"path":"prompts/modular/promotion-checkout.md","sha256":"1979ce8991b38fab30c4ea0d0fb7adba51d79bb82981059b0dafded82fed843a","until":null},{"from":null,"id":"ecommerce.progression.stock-alerts","modes":["upgrade"],"order":3300,"owners":["ecommerce.progression.stock-alerts"],"path":"prompts/modular/stock-alerts.md","sha256":"3478629a2831e04baca5eec1d3a2a3695c03f6044c36b43acb7c8ca7e35e9867","until":null},{"from":null,"id":"ecommerce.progression.faceted-search","modes":["upgrade"],"order":4000,"owners":["ecommerce.progression.faceted-search"],"path":"prompts/modular/faceted-search.md","sha256":"5dc13b47abcbc85654c82148f44b854979d868578db50db42e060c773edc2207","until":null},{"from":null,"id":"ecommerce.progression.personalized-recommendations","modes":["upgrade"],"order":4010,"owners":["ecommerce.progression.personalized-recommendations"],"path":"prompts/modular/progression-personalized-recommendations.md","sha256":"6c78b3963318b18da11173f96bed5cb8e5537f49e0008342ff11209fada3e099","until":null},{"from":null,"id":"ecommerce.spec.search-ordering","modes":["fresh","upgrade"],"order":4010,"owners":["ecommerce.spec.search-ordering"],"path":"prompts/modular/search-ordering-specification.md","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.faceted-search"],"sha256":"77d88f2b03c71b6cdc8350020ffd7815cf0bee87e4155c1aa00beb9bb81a83f8","until":null},{"from":null,"id":"ecommerce.l3.order-returns","modes":["upgrade"],"order":4040,"owners":["ecommerce.l3.order-returns-features"],"path":"prompts/modular/order-returns.md","sha256":"7277266f1e8c7d660ee38ba55ac1359200e56c695862c6d7d755d6ec6a6cb808","until":null},{"from":null,"id":"ecommerce.progression.order-support","modes":["upgrade"],"order":4100,"owners":["ecommerce.progression.order-support"],"path":"prompts/modular/order-support.md","sha256":"f515fe9827373e110633245df158ce8f78ca040f6c25a059d3885182515abb9a","until":null},{"from":null,"id":"ecommerce.progression.promotion-reporting","modes":["upgrade"],"order":4200,"owners":["ecommerce.progression.promotion-reporting"],"path":"prompts/modular/promotion-reporting.md","sha256":"3285eb0a50ceeed40b3415bdab8d6f5fc4138e496d2062a08c5ee2fac70157c6","until":null},{"from":null,"id":"ecommerce.progression.delivery-notifications","modes":["upgrade"],"order":5000,"owners":["ecommerce.progression.delivery-notifications"],"path":"prompts/modular/delivery-notifications.md","sha256":"12d6db9061749779dcb3db699a492781b34ae955c4c11cbb3e5dc2c8e0300ef3","until":null},{"from":null,"id":"ecommerce.progression.automatic-reorder","modes":["upgrade"],"order":5010,"owners":["ecommerce.progression.automatic-reorder"],"path":"prompts/modular/progression-automatic-reorder.md","sha256":"64751064e699c709a6550daba6fee8f019f445b817910e1ed0d8831632fa2471","until":null},{"from":null,"id":"ecommerce.progression.cart-recovery","modes":["upgrade"],"order":5020,"owners":["ecommerce.progression.cart-recovery"],"path":"prompts/modular/progression-cart-recovery.md","sha256":"4dc64aaa87d54a05bd7cce31bdc98254e605251d14f1688b859e9aa908da6a8b","until":null},{"from":null,"id":"ecommerce.progression.recommendation-feedback","modes":["upgrade"],"order":5030,"owners":["ecommerce.progression.recommendation-feedback"],"path":"prompts/modular/recommendation-feedback.md","sha256":"072a548ae797773041e86cbc62f39011559ca7ab40441bee6b77ec1854cd7f51","until":null},{"from":null,"id":"ecommerce.feature.store-credit","modes":["upgrade"],"order":5100,"owners":["ecommerce.feature.store-credit"],"path":"prompts/modular/store-credit.md","sha256":"d769da6c0ef4eb2fa6ac241aeddb61e8e2d85a0f1fcfd595fbf2c6660f67cd7b","until":null},{"from":null,"id":"ecommerce.feature.subscriptions","modes":["upgrade"],"order":5100,"owners":["ecommerce.feature.subscriptions"],"path":"prompts/modular/subscriptions.md","sha256":"ca93ab01c0d23c2fb3c035981372dc1254b219f1875b9566c0d01123176cc7c1","until":null},{"from":null,"id":"ecommerce.progression.support-refunds","modes":["upgrade"],"order":5100,"owners":["ecommerce.progression.support-refunds"],"path":"prompts/modular/support-refunds.md","sha256":"79090320de03a74660418da9e9adbf8170f8c574be990faeaf3d7df652221aa6","until":null},{"from":"## automatic-reorder-access","id":"ecommerce.spec.access-control.automatic-reorder-access","modes":["upgrade"],"order":6001,"owners":["ecommerce.spec.access-control"],"path":"prompts/modular/production-specifications.md","requiresFeatures":["ecommerce.progression.automatic-reorder"],"sha256":"f93257ed878a9fc27e9a0b80ceb050ab44e5ced5f6ba3d868939e8dee9ed49b1","until":"## recommendation-profile-isolation"},{"from":"## recommendation-profile-isolation","id":"ecommerce.spec.access-control.recommendation-profile-isolation","modes":["upgrade"],"order":6002,"owners":["ecommerce.spec.access-control"],"path":"prompts/modular/production-specifications.md","requiresFeatures":["ecommerce.progression.personalized-recommendations"],"sha256":"36fe3efec8aa2529e48f5f2ef809256a97957554e2a349e669fddcc3d6942a5a","until":"## staff-activity-privacy"},{"from":"## staff-activity-privacy","id":"ecommerce.spec.access-control.staff-activity-privacy","modes":["upgrade"],"order":6003,"owners":["ecommerce.spec.access-control"],"path":"prompts/modular/production-specifications.md","requiresFeatures":["ecommerce.progression.staff-activity"],"sha256":"3d9a91bd3c10abf3594a2dc690077f5a3b1ed95ddb7257c7c49f8e666448949c","until":"## order-support-ownership"},{"from":"## order-support-ownership","id":"ecommerce.spec.access-control.order-support-ownership","modes":["upgrade"],"order":6004,"owners":["ecommerce.spec.access-control"],"path":"prompts/modular/production-specifications.md","requiresFeatures":["ecommerce.progression.order-support"],"sha256":"5e090901a9bbfe7b0f1869be90a16d8d9df30b735effc5cd55e63784443aad05","until":"## delivery-notification-privacy"},{"from":"## delivery-notification-privacy","id":"ecommerce.spec.access-control.delivery-notification-privacy","modes":["upgrade"],"order":6005,"owners":["ecommerce.spec.access-control"],"path":"prompts/modular/production-specifications.md","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.delivery-notifications","ecommerce.progression.fulfilment-queue"],"sha256":"4fbdbdc513c7cc363198ce55780a421261c057f54ac8a44e3a69908d792d2e22","until":"## support-refund-access"},{"from":"## support-refund-access","id":"ecommerce.spec.access-control.support-refund-access","modes":["upgrade"],"order":6006,"owners":["ecommerce.spec.access-control"],"path":"prompts/modular/production-specifications.md","requiresFeatures":["ecommerce.progression.support-refunds"],"sha256":"a4200b390cf7c85d467425834af2642ceb7a80fa9ee53cccdfaa565d6ff69684","until":"## recommendation-feedback-privacy"},{"from":"## recommendation-feedback-privacy","id":"ecommerce.spec.access-control.recommendation-feedback-privacy","modes":["upgrade"],"order":6007,"owners":["ecommerce.spec.access-control"],"path":"prompts/modular/production-specifications.md","requiresFeatures":["ecommerce.progression.recommendation-feedback"],"sha256":"3df273f53ccd54520a89fedb64952b990ff4f61c36d16a50bd1f28e8523471b2","until":"## recommendation-feedback-restart"},{"from":"## recommendation-feedback-restart","id":"ecommerce.spec.state-durability.recommendation-feedback-restart","modes":["upgrade"],"order":6008,"owners":["ecommerce.spec.state-durability"],"path":"prompts/modular/production-specifications.md","requiresFeatures":["ecommerce.progression.recommendation-feedback"],"sha256":"5531638e5123f8c7c76c6f095ba3dadb52128482acc6d325de997ec280656d7c","until":"## automatic-reorder-deduplication"},{"from":"## automatic-reorder-deduplication","id":"ecommerce.spec.transactional-integrity.automatic-reorder-deduplication","modes":["upgrade"],"order":6009,"owners":["ecommerce.spec.transactional-integrity"],"path":"prompts/modular/production-specifications.md","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.automatic-reorder"],"sha256":"de82b9029ce5ae8c90dc1ea20a024da19d8ea55eac6ab36f6fa1d84b8b241b01","until":"## payment-deduplication"},{"from":"## payment-deduplication","id":"ecommerce.spec.transactional-integrity.payment-deduplication","modes":["upgrade"],"order":6010,"owners":["ecommerce.spec.transactional-integrity"],"path":"prompts/modular/production-specifications.md","requiresFeatures":["ecommerce.progression.payment-records"],"sha256":"f33fb61f31726bbe7e8b617ae12d970a8d876d98d30d49191dd0043a58557760","until":"## support-refund-accounting"},{"from":"## support-refund-accounting","id":"ecommerce.spec.transactional-integrity.support-refund-accounting","modes":["upgrade"],"order":6011,"owners":["ecommerce.spec.transactional-integrity"],"path":"prompts/modular/production-specifications.md","requiresFeatures":["ecommerce.progression.support-refunds"],"sha256":"bd44a5a8c9211493896869df9b086177b022d9840f9a362d0c609e3c6bff6877","until":null},{"from":null,"id":"ecommerce.feature.split-tender-refunds","modes":["upgrade"],"order":6100,"owners":["ecommerce.feature.split-tender-refunds"],"path":"prompts/modular/split-tender-refunds.md","sha256":"c21e00af3623136de4ae45f089b45b457a34563eb37a756d5b8b3563d1f19734","until":null},{"from":"## Product bundles","id":"ecommerce.spec.bundle-integrity.product-bundles","modes":["fresh","upgrade"],"order":6500,"owners":["ecommerce.spec.bundle-integrity"],"path":"prompts/modular/later-specifications.md","requiresFeatures":["ecommerce.feature.product-bundles"],"sha256":"7eb9ee1c794134867f6b27ad3b773bbd99d4c860774618d1286d39b4347470fb","until":"## Bundle checkout"},{"from":"## Bundle checkout","id":"ecommerce.spec.bundle-integrity.bundle-checkout","modes":["fresh","upgrade"],"order":6501,"owners":["ecommerce.spec.bundle-integrity"],"path":"prompts/modular/later-specifications.md","requiresFeatures":["ecommerce.feature.bundle-checkout"],"sha256":"630eead6de0dc2d9672e83e64beb8cb5036eccd9ba4aaf7c9284d91c0e42144b","until":"## Bundle returns"},{"from":"## Bundle returns","id":"ecommerce.spec.bundle-integrity.bundle-returns","modes":["fresh","upgrade"],"order":6502,"owners":["ecommerce.spec.bundle-integrity"],"path":"prompts/modular/later-specifications.md","requiresFeatures":["ecommerce.feature.bundle-returns"],"sha256":"7a2264753c8088c30ecc87cb1f98940198fde663b5a4a1bb8de83c410e01e5b2","until":"## Store credit"},{"from":"## Store credit","id":"ecommerce.spec.store-credit.store-credit","modes":["fresh","upgrade"],"order":6503,"owners":["ecommerce.spec.store-credit"],"path":"prompts/modular/later-specifications.md","requiresFeatures":["ecommerce.feature.store-credit"],"sha256":"ee64f04e0f30fc269021083c4b02c26f9fc87321d41d7a7262e68bf6926b7c83","until":"## Split-tender refunds"},{"from":"## Split-tender refunds","id":"ecommerce.spec.split-tender-refunds.split-tender-refunds","modes":["fresh","upgrade"],"order":6504,"owners":["ecommerce.spec.split-tender-refunds"],"path":"prompts/modular/later-specifications.md","requiresFeatures":["ecommerce.feature.split-tender-refunds"],"sha256":"860929486a2d9e283be8486c85b1ee2086936b22de28db17febf5a1e65bc10ff","until":"## Scheduled purchases"},{"from":"## Scheduled purchases","id":"ecommerce.spec.subscriptions.subscriptions","modes":["fresh","upgrade"],"order":6505,"owners":["ecommerce.spec.subscriptions"],"path":"prompts/modular/later-specifications.md","requiresFeatures":["ecommerce.feature.subscriptions"],"sha256":"69f32aaf688858b89cb8fc56352b1674e13fd18b7b9454da3ba9a025e24c90ab","until":null},{"from":null,"id":"ecommerce.feature.product-bundles","modes":["upgrade"],"order":8004,"owners":["ecommerce.feature.product-bundles"],"path":"prompts/modular/product-bundles.md","sha256":"a0805178ebbb20ea290994de65b4ded503c37798af59905d47c1cb2fd4e2c467","until":null},{"from":null,"id":"ecommerce.feature.bundle-checkout","modes":["upgrade"],"order":8005,"owners":["ecommerce.feature.bundle-checkout"],"path":"prompts/modular/bundle-checkout.md","sha256":"d3f21ac08991619b5c9aa6ac99a7554340bae991d5b31a25cdb048cdb314936e","until":null},{"from":null,"id":"ecommerce.feature.bundle-returns","modes":["upgrade"],"order":8006,"owners":["ecommerce.feature.bundle-returns"],"path":"prompts/modular/bundle-returns.md","sha256":"34b3bee882e6a1f6a1527e67d12f8cbdcfd74d026108f9805159123c41015bf2","until":null}]},"title":"Ecommerce progression catalog","track":"ecommerce"},"meaning":{"checks":[{"category":"feature","checkGroupId":"accounts","criterionId":"1a","description":"a visitor can create an account and is signed in as it","featureId":1,"featureName":"Accounts","note":null,"packId":"ecommerce.feature.accounts","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"signUp","name":"ann"},{"contains":"ann","do":"expect"}],"source":"scenarios/01-account-create.json","stableKey":"ecommerce.feature.accounts.accounts.1a","statedBy":"a visitor can create an account with a username and password","withheld":null},{"category":"production","checkGroupId":"accounts","criterionId":"1b","description":"a taken username is refused and does not sign the visitor in as the existing account","featureId":1,"featureName":"Accounts","note":null,"packId":"ecommerce.feature.accounts","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"signUp","expectFailure":true,"name":"ann","password":"different-pw"},{"do":"expect"},{"absent":true,"do":"expect"}],"source":"scenarios/01-account-duplicate.json","stableKey":"ecommerce.feature.accounts.accounts.1b","statedBy":"signing up with a taken username fails with a visible error and must never sign the visitor in as the existing account","withheld":null},{"category":"production","checkGroupId":"accounts","criterionId":"1c","description":"a wrong password is refused","featureId":1,"featureName":"Accounts","note":null,"packId":"ecommerce.feature.accounts","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"signIn","expectFailure":true,"name":"ann","password":"wrong-pw"},{"do":"expect"},{"absent":true,"do":"expect"}],"source":"scenarios/01-account-password.json","stableKey":"ecommerce.feature.accounts.accounts.1c","statedBy":"signing in with a wrong password fails with a visible error","withheld":null},{"category":"production","checkGroupId":"session-reload","criterionId":"1e","description":"the session survives a reload","featureId":1,"featureName":"Accounts","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.accounts"],"role":"guarantee","semantics":[{"do":"reload"},{"contains":"ann","do":"expect"}],"source":"scenarios/01-account-reload.json","stableKey":"ecommerce.spec.state-durability.session-reload.1e","statedBy":"a signed-in session persists across a page reload","withheld":null},{"category":"feature","checkGroupId":"accounts","criterionId":"1d","description":"signing out and back in returns the same account","featureId":1,"featureName":"Accounts","note":null,"packId":"ecommerce.feature.accounts","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"click","unlessVisible":"signout"},{"do":"click"},{"do":"waitUntilAbsent"},{"do":"signIn","name":"ann"},{"contains":"ann","do":"expect"}],"source":"scenarios/01-account-signout.json","stableKey":"ecommerce.feature.accounts.accounts.1d","statedBy":"a signed-in user can sign out, returning to the signed-out state","withheld":null},{"category":"feature","checkGroupId":"admin-write","criterionId":"103a","description":"an administrator can restock a warehouse","featureId":103,"featureName":"Only an administrator can restock","note":null,"packId":"ecommerce.feature.warehouse-admin","points":1,"provenBy":null,"role":"feature","semantics":[{"as":"purifier-before-control","do":"recordNumber"},{"action":"restock","do":"callAction","input":{"attribute":"data-restock-input","contains":"Air Purifier","testid":"admin-location-row"},"namedAction":{"args":[0,0,1],"id":"restock","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"warehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/restock","reducer":"admin_restock"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"expectNumber","plus":1,"relativeTo":"purifier-before-control"}],"source":"scenarios/01-admin-write-staff.json","stableKey":"ecommerce.feature.warehouse-admin.admin-write.103a","statedBy":"An administrator can add units to a selected item and warehouse.","withheld":null},{"category":"production","checkGroupId":"warehouse-write-boundary","criterionId":"103b","description":"the server refuses a warehouse write from staff","featureId":103,"featureName":"Only an administrator can restock","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","semantics":[{"action":"restock","do":"callAction","input":{"attribute":"data-restock-input","contains":"Air Purifier","testid":"admin-location-row"},"namedAction":{"args":[0,0,1],"id":"restock","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"warehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/restock","reducer":"admin_restock"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"as":"purifier-before-refusal","do":"recordNumber"},{"action":"restock","do":"callAction","from":"admin","input":{"attribute":"data-restock-input","contains":"Air Purifier","testid":"admin-location-row"},"namedAction":{"args":[0,0,1],"id":"restock","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"warehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/restock","reducer":"admin_restock"}},{"do":"expectActionOutcome","outcome":"refused","routeProvenBy":"admin"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"expectNumber","plus":0,"relativeTo":"purifier-before-refusal"}],"source":"scenarios/01-admin-write-staff.json","stableKey":"ecommerce.spec.access-control.warehouse-write-boundary.103b","statedBy":"Only administrators can change warehouse stock.","withheld":null},{"category":"production","checkGroupId":"purchase-stock","criterionId":"3b","description":"buying reduces the stock every other client sees, without a reload","featureId":3,"featureName":"Buying","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing"],"role":"guarantee","semantics":[{"do":"expectNumber","equals":100},{"do":"click"},{"do":"expectNumber","equals":99},{"do":"expectNumber","equals":99}],"source":"scenarios/01-buying.json","stableKey":"ecommerce.spec.live-state.purchase-stock.3b","statedBy":"buying an item reduces its stock by one for everyone","withheld":null},{"category":"production","checkGroupId":"cart-boundary","criterionId":"109a","description":"the same cart action run by another customer changes only that customer's cart","featureId":109,"featureName":"A cart is nobody else's business","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.cart"],"role":"guarantee","semantics":[{"action":"cart-add","do":"callAction","input":{"attribute":"data-buy-input","contains":"Coffee Grinder","testid":"item-card"},"namedAction":{"args":[0],"id":"cart-add","params":[{"in":"body","name":"itemId","wireType":"u64"}],"path":"/api/cart","reducer":"add_to_cart"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"reload"},{"do":"ensureSignedIn","name":"vic"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"cart-total"},{"do":"reload"},{"do":"ensureSignedIn","name":"wes"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"contains":"Coffee Grinder","count":1,"do":"expect"},{"do":"expectNumber","equals":1},{"do":"click","unlessVisible":"cart-total"},{"contains":"Coffee Grinder","count":1,"do":"expect"},{"do":"expectNumber","equals":1}],"source":"scenarios/01-cart-boundary.json","stableKey":"ecommerce.spec.access-control.cart-boundary.109a","statedBy":"one customer cannot read or change another customer's cart","withheld":null},{"category":"production","checkGroupId":"cart-boundary","criterionId":"109b","description":"a negative quantity is refused and leaves the cart unchanged","featureId":109,"featureName":"A cart is nobody else's business","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.cart"],"role":"guarantee","semantics":[{"as":"cart-total-before-invalid","do":"recordNumber"},{"as":"cart-quantity-before-invalid","do":"recordNumber"},{"action":"cart-set-quantity","do":"callAction","input":{"attribute":"data-cart-input","contains":"Coffee Grinder","testid":"cart-item"},"namedAction":{"args":[0,-3],"id":"cart-set-quantity","method":"PATCH","params":[{"in":"path","name":"itemId","placeholder":":itemId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/cart/:itemId","reducer":"update_cart_quantity"}},{"do":"expectActionOutcome","outcome":"validation-refused"},{"do":"reload"},{"do":"ensureSignedIn","name":"vic"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"cart-total"},{"do":"expectNumber","plus":0,"relativeTo":"cart-total-before-invalid"},{"do":"expectNumber","plus":0,"relativeTo":"cart-quantity-before-invalid"}],"source":"scenarios/01-cart-boundary.json","stableKey":"ecommerce.spec.access-control.cart-boundary.109b","statedBy":"a request carrying a negative quantity is refused and changes nothing","withheld":null},{"category":"production","checkGroupId":"cart-reload","criterionId":"4b","description":"the cart survives a reload","featureId":4,"featureName":"Cart belongs to the account","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.cart"],"role":"guarantee","semantics":[{"do":"click"},{"do":"click","unlessVisible":"cart-total"},{"contains":"Laptop Stand","do":"expect"},{"do":"reload"},{"do":"ensureSignedIn","name":"omar"},{"do":"click","unlessVisible":"cart-total"},{"contains":"Laptop Stand","do":"expect"}],"source":"scenarios/01-cart.json","stableKey":"ecommerce.spec.state-durability.cart-reload.4b","statedBy":"the cart survives a reload","withheld":null},{"category":"production","checkGroupId":"shared-cart","criterionId":"4c","description":"the same account signed in elsewhere sees one cart, live","featureId":4,"featureName":"Cart belongs to the account","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.cart"],"role":"guarantee","semantics":[{"do":"signIn","name":"pia"},{"do":"click","unlessVisible":"cart-total"},{"do":"click"},{"contains":"Induction Cooktop","do":"expect"}],"source":"scenarios/01-cart.json","stableKey":"ecommerce.spec.live-state.shared-cart.4c","statedBy":"the same account signed in twice sees one cart, and a change made in one place appears in the other without a reload","withheld":null},{"category":"feature","checkGroupId":"catalog-ranking","criterionId":"2b","description":"the storefront shows the exact alphabetical top ten before any purchase","featureId":2,"featureName":"Public catalog ranking","note":null,"packId":"ecommerce.feature.catalog-discovery","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"expectSequence","equals":["Air Purifier","Bluetooth Speaker","Coffee Grinder","Desk Lamp","Espresso Machine","Gaming Mouse","Headphones","Induction Cooktop","Keyboard","Laptop Stand"]}],"source":"scenarios/01-catalog-ranking.json","stableKey":"ecommerce.feature.catalog.catalog-ranking.2b","statedBy":"Show the ten most-purchased items and break ties by item name","withheld":null},{"category":"feature","checkGroupId":"catalog-search","criterionId":"2d","description":"case-insensitive partial search finds an item outside the storefront top ten","featureId":2,"featureName":"Public catalog search","note":null,"packId":"ecommerce.feature.catalog-discovery","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"fill","enter":true,"text":"mirrorLESS"},{"contains":"Mirrorless Camera","do":"expect"}],"source":"scenarios/01-catalog-search.json","stableKey":"ecommerce.feature.catalog.catalog-search.2d","statedBy":"Search matches any part of an item name without regard to case across the full catalog","withheld":null},{"category":"feature","checkGroupId":"catalog-values","criterionId":"2a","description":"a signed-out visitor sees the seeded item name, price, and total stock","featureId":2,"featureName":"Public catalog values","note":null,"packId":"ecommerce.feature.catalog-items","points":1,"provenBy":null,"role":"feature","semantics":[{"contains":"Air Purifier","do":"expect"},{"do":"expectNumber","equals":189},{"do":"expectNumber","equals":100}],"source":"scenarios/01-catalog-values.json","stableKey":"ecommerce.feature.catalog.catalog-values.2a","statedBy":"Each item shows its name, price, and total stock","withheld":null},{"category":"production","checkGroupId":"ranking","criterionId":"2c","description":"a purchase moves the bought item to the front of the ranking, live","featureId":2,"featureName":"Storefront is public and live","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing"],"role":"guarantee","semantics":[{"do":"click"},{"do":"expectSequence","equals":["Coffee Grinder","Air Purifier","Bluetooth Speaker","Desk Lamp","Espresso Machine","Gaming Mouse","Headphones","Induction Cooktop","Keyboard","Laptop Stand"]},{"do":"expectAgreement","numeric":true}],"source":"scenarios/01-core.json","stableKey":"ecommerce.spec.live-state.ranking.2c","statedBy":"a purchase immediately changes the ranking for every open client","withheld":null},{"category":"production","checkGroupId":"duplicate-checkout","criterionId":"203a","description":"the same item added from two tabs at once becomes one line of two","featureId":203,"featureName":"One cart, two tabs, one checkout","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.checkout"],"role":"guarantee","semantics":[{"contains":"Gaming Mouse","count":1,"do":"expect"},{"do":"expectNumber","equals":2}],"source":"scenarios/01-duplicate-checkout.json","stableKey":"ecommerce.spec.concurrency-safety.duplicate-checkout.203a","statedBy":"raises its quantity rather than adding a second line","withheld":null},{"category":"production","checkGroupId":"duplicate-checkout","criterionId":"203b","description":"checking the same cart out twice at once produces one order","featureId":203,"featureName":"One cart, two tabs, one checkout","note":"callConcurrently resolves the track's named checkout action through the selected backend adapter and issues it at the same time with tab1 and tab2's own session credentials. The filler prepares the shared Keyboard cart. Native order evidence requires one complete order for that account, the booked price and quantity, a cleared cart, and unchanged prior orders. Checkout can precede warehouse administration, so this check reads no warehouse data and retains its visible stock reduction assertion. The second call may succeed idempotently or deliberately refuse; server errors and no progress fail.","observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":1,"provenBy":"The L1 2.3 candidate set binds an exact 203b defect on every reference stack: MongoDB bypasses the atomic cart claim-and-clear, while PostgreSQL and SpacetimeDB leave checked-out cart lines behind. These defects let both named checkout calls reuse the same cart, which the call-outcome and final order assertions are designed to catch. Exact Docker mutation qualification remains the promotion gate.","requiresFeatures":["ecommerce.feature.checkout"],"role":"guarantee","semantics":[{"do":"click"},{"do":"wait"},{"as":"keyboard-before-checkout","do":"recordNumber"},{"account":"{user:twin}","as":"checkout-before","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"do":"click"},{"do":"wait"},{"account":"{user:twin}","as":"checkout-prepared","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"action":"checkout","do":"callConcurrently"},{"do":"expectCallOutcomes"},{"before":"checkout-before","do":"dbExpectCheckout","prepared":"checkout-prepared","quantity":1},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Keyboard","count":1,"do":"expect"},{"do":"reload"},{"do":"ensureSignedIn","name":"twin"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"expectNumber","plus":-1,"relativeTo":"keyboard-before-checkout"}],"source":"scenarios/01-duplicate-checkout.json","stableKey":"ecommerce.spec.concurrency-safety.duplicate-checkout.203b","statedBy":"checking out twice must not produce two orders","withheld":"The named checkout action has an HTTP route for MongoDB and PostgreSQL and a reducer mapping for SpacetimeDB. The stack adapter issues the corresponding credentialed request for each actor, giving all three stacks the same one-cart, two-call expectation."},{"category":"production","checkGroupId":"external-stock","criterionId":"901a","description":"a direct database write sets Desk Lamp's East stock to 5, and the already-open storefront updates from 100 to 50 without a reload or page action","featureId":901,"featureName":"An open storefront follows a direct database write","note":"This proposed score remains draft until the focused pristine and mutation controls are qualified.","observations":["requested","unmentioned"],"packId":"ecommerce.spec.external-data-sync","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","semantics":[{"do":"dbSetStock","item":"Desk Lamp","quantity":5,"warehouse":"East"},{"do":"expectNumber","equals":50}],"source":"scenarios/01-external-live-sync.json","stableKey":"ecommerce.spec.external-data-sync.external-stock.901a","statedBy":"The storefront reflects current database values after changes made outside the application server.","withheld":null},{"category":"production","checkGroupId":"external-stock","criterionId":"901d","description":"while the storefront is offline, a direct database write sets Desk Lamp's East stock to 7; after reconnecting, the same page catches up from 100 to the authoritative total of 52","featureId":901,"featureName":"A reconnecting storefront catches up to an external write","note":"This proposed score remains draft until the focused pristine and mutation controls are qualified.","observations":["requested","unmentioned"],"packId":"ecommerce.spec.external-data-sync","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","semantics":[{"do":"setOffline","offline":true},{"do":"dbSetStock","item":"Desk Lamp","quantity":7,"warehouse":"East"},{"do":"setOffline","offline":false},{"do":"expectNumber","equals":52}],"source":"scenarios/01-external-reconnect-sync.json","stableKey":"ecommerce.spec.external-data-sync.external-stock.901d","statedBy":"The storefront reflects current database values after changes made outside the application server.","withheld":null},{"checkGroupId":"external-stock","criterionId":"901b","description":"after a direct database write sets Desk Lamp's East stock to 5, a reload reads the persisted total of 50","featureId":901,"featureName":"A direct database write survives reload","note":"Supporting evidence for 901a rather than a second score. It rules out a page-only patch but has no distinct calibrated failure mode.","observations":["requested","unmentioned"],"packId":"ecommerce.spec.external-data-sync","points":0,"provenBy":null,"requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","semantics":[{"do":"dbSetStock","item":"Desk Lamp","quantity":5,"warehouse":"East"},{"do":"reload"},{"do":"expectNumber","equals":50}],"source":"scenarios/01-external-reload-sync.json","stableKey":"ecommerce.spec.external-data-sync.external-stock.901b","statedBy":"The storefront reflects current database values after changes made outside the application server.","withheld":null},{"category":"production","checkGroupId":"external-stock","criterionId":"901c","description":"a stock correction lands while the app server is stopped, and the already-open storefront shows the authoritative total of 65 after the server returns without a reload","featureId":901,"featureName":"An open storefront catches up after its server restarts","note":"Preserves the previously promoted score while removing its dependency on 901a changing East stock first.","observations":["requested","unmentioned"],"packId":"ecommerce.spec.external-data-sync","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","semantics":[{"do":"stopAppServer"},{"do":"dbSetStock","item":"Desk Lamp","quantity":10,"warehouse":"West"},{"do":"startAppServer"},{"do":"expectNumber","equals":65}],"source":"scenarios/01-external-server-restart-sync.json","stableKey":"ecommerce.spec.external-data-sync.external-stock.901c","statedBy":"a restart must end up showing the current numbers once the server is back","withheld":null},{"category":"production","checkGroupId":"last-unit","criterionId":"201a","description":"after six customers try to buy the last three units, each warehouse stores zero stock and all observed clients show zero stock","featureId":201,"featureName":"The last unit is sold once","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":1,"provenBy":"Prior qualification covers the client observations only. The added stored-stock observations need matching three-stack reference and defect-control evidence before promotion.","requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","semantics":[{"do":"dbExpectStock","equals":0,"item":"Air Purifier","warehouse":"East"},{"do":"dbExpectStock","equals":0,"item":"Air Purifier","warehouse":"West"},{"do":"expectNumber","equals":0},{"do":"expectAgreement","numeric":true}],"source":"scenarios/01-last-unit.json","stableKey":"ecommerce.spec.concurrency-safety.last-unit.201a","statedBy":"stock may never go negative, and two customers must never both get the last unit","withheld":null},{"category":"production","checkGroupId":"last-unit","criterionId":"201c","description":"revenue increases by exactly three sales, not six","featureId":201,"featureName":"The last unit is sold once","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"admin-revenue"},{"do":"expectNumber","plus":567,"relativeTo":"revenue-before-last-unit"}],"source":"scenarios/01-last-unit.json","stableKey":"ecommerce.spec.concurrency-safety.last-unit.201c","statedBy":"stock must never leave a warehouse without a corresponding order","withheld":null},{"category":"production","checkGroupId":"last-unit","criterionId":"201b","description":"the last three units create complete orders for the successful buyers, and all four affordable purchases succeed when stock is sufficient","featureId":201,"featureName":"The last unit is sold once","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":1,"provenBy":"Draft: native order reconciliation and the sufficient-stock progress case require matching reference and defect controls. This criterion runs after the revenue assertion so its extra purchases cannot alter that assertion's baseline.","requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","semantics":[{"before":{"a":"buy-a","b":"buy-b","c":"buy-c","d":"buy-d","e":"buy-e","f":"buy-f"},"do":"dbExpectPurchases","purchases":3},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Air Purifier","do":"expectActorsWith","equals":3,"maxEach":1},{"do":"dbSetStock","item":"Air Purifier","quantity":2,"warehouse":"East"},{"do":"dbSetStock","item":"Air Purifier","quantity":2,"warehouse":"West"},{"do":"reload"},{"do":"ensureSignedIn","name":"c1"},{"do":"click","ifAvailable":true},{"account":"{user:c1}","as":"ample-a","do":"dbRecordCheckout","item":"Air Purifier","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"account":"{user:c2}","as":"ample-b","do":"dbRecordCheckout","item":"Air Purifier","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"action":"buy","do":"callConcurrently","from":"a","input":{"attribute":"data-buy-input","contains":"Air Purifier","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"},"requests":4},{"do":"expectCallOutcomes"},{"before":{"a":"ample-a","b":"ample-b"},"do":"dbExpectPurchases","purchases":4}],"source":"scenarios/01-last-unit.json","stableKey":"ecommerce.spec.concurrency-safety.last-unit.201b","statedBy":"two customers must never both get the last unit","withheld":null},{"category":"production","checkGroupId":"order-ownership","criterionId":"106a","description":"a working order history contains the customer's own order and not another customer's order","featureId":106,"featureName":"One customer's orders are not another's","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","semantics":[{"do":"click"},{"do":"click"},{"do":"wait"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Desk Lamp","count":1,"do":"expect"},{"absent":true,"contains":"Coffee Grinder","do":"expect"}],"source":"scenarios/01-order-ownership.json","stableKey":"ecommerce.spec.access-control.order-ownership.106a","statedBy":"a customer sees only their own orders","withheld":null},{"category":"production","checkGroupId":"purchase-attribution","criterionId":"102a","description":"a direct purchase is attributed to the authenticated caller, not another account","featureId":102,"featureName":"Purchases are attributed to whoever made them","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","semantics":[{"action":"buy","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Coffee Grinder","count":1,"do":"expect"},{"absent":true,"contains":"Desk Lamp","do":"expect"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Desk Lamp","count":1,"do":"expect"}],"source":"scenarios/01-purchase-attribution.json","stableKey":"ecommerce.spec.access-control.purchase-attribution.102a","statedBy":"the authenticated caller, not a client-supplied identity, owns the order","withheld":null},{"category":"production","checkGroupId":"purchase-session","criterionId":"101a","description":"a valid direct purchase works for the buyer but is refused without a session","featureId":101,"featureName":"Purchase requires an account","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","semantics":[{"action":"buy","authentication":"none","do":"callAction","from":"buyer","input":{"attribute":"data-buy-input","contains":"Bluetooth Speaker","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"do":"expectActionOutcome","outcome":"refused","routeProvenBy":"buyer"},{"do":"dbExpectStock","item":"Bluetooth Speaker","plus":-1,"relativeTo":"speaker-before-control"}],"source":"scenarios/01-purchase-session.json","stableKey":"ecommerce.spec.access-control.purchase-session.101a","statedBy":"the server refuses a purchase without an authenticated customer","withheld":null},{"category":"feature","checkGroupId":"restock-race","criterionId":"202-control","description":"an uncontended restock of five is stored by the server and shows on the storefront","featureId":202,"featureName":"A restock during a rush is not lost","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":0,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","semantics":[{"do":"dbExpectStock","item":"Bluetooth Speaker","plus":5,"relativeTo":"stored-before-control"},{"do":"expectNumber","plus":5,"relativeTo":"storefront-before"}],"source":"scenarios/01-restock-race.json","stableKey":"ecommerce.spec.concurrency-safety.restock-race.202-control","statedBy":null,"withheld":"The ordinary restock is a setup prerequisite retained when only the race is selected. This zero-point control verifies that setup; it does not add product credit."},{"category":"production","checkGroupId":"restock-race","criterionId":"202a","description":"restocking during purchases preserves stock, complete buyer orders and their warehouse allocations","featureId":202,"featureName":"A restock during a rush is not lost","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","semantics":[{"as":"stored-before-rush","do":"dbRecordStock","item":"Bluetooth Speaker"},{"do":"fill","text":"5"},{"as":"rush-before","do":"recordNumber"},{"branches":[[{"do":"clickConcurrently"}],[{"do":"click"}]],"do":"race"},{"do":"dbExpectStock","item":"Bluetooth Speaker","plus":2,"relativeTo":"stored-before-rush"},{"atLeast":0,"do":"dbExpectStock","item":"Bluetooth Speaker","warehouse":"East"},{"atLeast":0,"do":"dbExpectStock","item":"Bluetooth Speaker","warehouse":"West"},{"do":"reload"},{"do":"click"},{"do":"expectNumber","plus":2,"relativeTo":"rush-before"},{"do":"ensureSignedIn","name":"r1"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Bluetooth Speaker","count":2,"do":"expect"},{"do":"reload"},{"do":"ensureSignedIn","name":"r2"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Bluetooth Speaker","count":1,"do":"expect"},{"do":"reload"},{"do":"ensureSignedIn","name":"r3"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Bluetooth Speaker","count":1,"do":"expect"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"expectNumber","plus":2,"relativeTo":"stored-before-rush"},{"do":"reload"},{"do":"click"},{"do":"expectNumber","plus":2,"relativeTo":"stored-before-rush"},{"do":"ensureSignedIn","name":"r1"},{"account":"{user:r1}","as":"mixed-a","do":"dbRecordCheckout","item":"Bluetooth Speaker","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"account":"{user:r2}","as":"mixed-b","do":"dbRecordCheckout","item":"Bluetooth Speaker","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"account":"{user:r3}","as":"mixed-c","do":"dbRecordCheckout","item":"Bluetooth Speaker","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"action":"buy","alongside":[{"action":"restock","actors":["admin"],"input":{"attribute":"data-restock-input","contains":"Bluetooth Speaker","testid":"admin-location-row"},"namedAction":{"args":[0,0,1],"id":"restock","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"warehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/restock","reducer":"admin_restock"},"requests":1}],"do":"callConcurrently","from":"a","input":{"attribute":"data-buy-input","contains":"Bluetooth Speaker","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"},"requests":3},{"accepted":4,"do":"expectCallOutcomes"},{"before":{"a":"mixed-a","b":"mixed-b","c":"mixed-c"},"do":"dbExpectPurchases","purchases":3}],"source":"scenarios/01-restock-race.json","stableKey":"ecommerce.spec.concurrency-safety.restock-race.202a","statedBy":"a restock in one warehouse raises the storefront number live","withheld":null},{"category":"production","checkGroupId":"review-eligibility","criterionId":"108a","description":"someone who never bought the item cannot review it","featureId":108,"featureName":"A review is a claim about a purchase","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","semantics":[{"action":"buy","do":"callAction","input":{"attribute":"data-buy-input","contains":"Air Purifier","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Air Purifier","do":"expect"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"action":"submitReview","do":"callAction","from":"owner","input":{"attribute":"data-buy-input","contains":"Air Purifier","testid":"item-card"},"namedAction":{"args":[0,4,"eligible review control"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"}],"path":"/api/items/:id/reviews","reducer":"submit_review"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"openItem","item":"Air Purifier"},{"contains":"eligible review control","do":"expect"},{"action":"submitReview","do":"callAction","from":"owner","input":{"attribute":"data-buy-input","contains":"Air Purifier","testid":"item-card"},"namedAction":{"args":[0,4,"never bought this"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"}],"path":"/api/items/:id/reviews","reducer":"submit_review"}},{"do":"expectActionOutcome","outcome":"application-refused","routeProvenBy":"owner"},{"do":"reload"},{"do":"ensureSignedIn","name":"uma"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"openItem","item":"Air Purifier"},{"contains":"eligible review control","do":"expect"},{"absent":true,"contains":"never bought this","do":"expect"}],"source":"scenarios/01-review-eligibility.json","stableKey":"ecommerce.spec.access-control.review-eligibility.108a","statedBy":"the server refuses a review from someone who has never ordered the item","withheld":null},{"category":"production","checkGroupId":"review-eligibility","criterionId":"108b","description":"buying the item earns the right to review it","featureId":108,"featureName":"A review is a claim about a purchase","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","semantics":[{"action":"buy","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"openItem","item":"Keyboard"},{"do":"click","ifAvailable":true,"unlessVisible":"review-rating"},{"do":"fill","text":"4"},{"do":"fill","text":"bought and used it"},{"do":"click"},{"contains":"bought and used it","do":"expect"}],"source":"scenarios/01-review-eligibility.json","stableKey":"ecommerce.spec.access-control.review-eligibility.108b","statedBy":"a customer can review an item they bought","withheld":null},{"category":"production","checkGroupId":"rating","criterionId":"6c","description":"the average rating reflects both reviewers and updates live","featureId":6,"featureName":"Reviews","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","semantics":[{"do":"openItem","item":"Gaming Mouse"},{"do":"click","ifAvailable":true,"unlessVisible":"review-rating"},{"do":"fill","text":"4"},{"do":"fill","text":"does the job"},{"do":"click"},{"do":"expectNumber","equals":3},{"do":"expectAgreement","numeric":true}],"source":"scenarios/01-review-rating-live.json","stableKey":"ecommerce.spec.live-state.rating.6c","statedBy":"each item shows its average rating, which updates live as reviews arrive","withheld":null},{"category":"production","checkGroupId":"unique-review","criterionId":"6b","description":"a later review submission does not create a duplicate for the same customer and item","featureId":6,"featureName":"Reviews","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","semantics":[{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"action":"submitReview","do":"callAction","from":"author","input":{"attribute":"data-buy-input","contains":"Air Purifier","testid":"item-card"},"namedAction":{"args":[0,4,"changed my mind"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"}],"path":"/api/items/:id/reviews","reducer":"submit_review"}},{"do":"expectActionOutcome","outcome":"completed"},{"do":"reload"},{"do":"ensureSignedIn","name":"kira"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"openItem","item":"Air Purifier"},{"count":1,"do":"expect"}],"source":"scenarios/01-review-uniqueness.json","stableKey":"ecommerce.spec.transactional-integrity.unique-review.6b","statedBy":"one customer has at most one review per item","withheld":null},{"category":"feature","checkGroupId":"reviews","criterionId":"6a","description":"a customer can review an item and everyone sees it, signed out included","featureId":6,"featureName":"Reviews","note":null,"packId":"ecommerce.feature.reviews","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"openItem","item":"Induction Cooktop"},{"do":"click","ifAvailable":true,"unlessVisible":"review-rating"},{"do":"fill","text":"4"},{"do":"fill","text":"solid mold"},{"do":"click"},{"contains":"solid mold","do":"expect"},{"do":"openItem","item":"Induction Cooktop"},{"contains":"solid mold","do":"expect"}],"source":"scenarios/01-review-visibility.json","stableKey":"ecommerce.feature.reviews.reviews.6a","statedBy":"reviews are visible to everyone, including signed-out visitors","withheld":null},{"category":"production","checkGroupId":"server-price","criterionId":"104a","description":"direct purchases of two differently priced items persist exactly one correctly priced order each","featureId":104,"featureName":"The price is the store's to set","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","semantics":[{"action":"buy","do":"callAction","input":{"attribute":"data-buy-input","contains":"Espresso Machine","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"do":"expectActionOutcome","outcome":"accepted"},{"action":"buy","do":"callAction","input":{"attribute":"data-buy-input","contains":"Coffee Grinder","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"freshClient"},{"do":"signIn","name":"oli"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"count":2,"do":"expect"},{"contains":"Espresso Machine","count":1,"do":"expect"},{"do":"expectNumber","equals":449},{"contains":"Coffee Grinder","count":1,"do":"expect"},{"do":"expectNumber","equals":64}],"source":"scenarios/01-server-price.json","stableKey":"ecommerce.spec.transactional-integrity.server-price.104a","statedBy":"the server uses the current stored price when it creates an order","withheld":null},{"category":"production","checkGroupId":"warehouse-area-boundary","criterionId":"7a","description":"the administrator area stays unavailable to other staff","featureId":7,"featureName":"Admin and warehouses","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","semantics":[{"do":"expect"},{"do":"click","ifAvailable":true},{"absent":true,"do":"expect"}],"source":"scenarios/01-warehouse-admin-staff.json","stableKey":"ecommerce.spec.access-control.warehouse-area-boundary.7a","statedBy":"Administrators can open the administration area; other staff cannot.","withheld":null},{"category":"feature","checkGroupId":"warehouse-view","criterionId":"7b","description":"admin lists every item, every warehouse, and what each warehouse holds","featureId":7,"featureName":"Admin and warehouses","note":null,"packId":"ecommerce.feature.warehouse-admin","points":1,"provenBy":null,"role":"feature","semantics":[{"count":13,"do":"expect"},{"count":26,"do":"expect"},{"contains":"East","do":"expect"},{"contains":"West","do":"expect"},{"do":"expectNumber","equals":100}],"source":"scenarios/01-warehouse-admin-staff.json","stableKey":"ecommerce.feature.warehouse-admin.warehouse-view.7b","statedBy":"admin lists every item with its stock, every warehouse, and the stock of each item in each warehouse","withheld":null},{"category":"production","checkGroupId":"warehouse-stock","criterionId":"7c","description":"the storefront stock is the sum across warehouses, and a restock raises it live","featureId":7,"featureName":"Warehouse stock stays live","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.warehouse-admin"],"role":"guarantee","semantics":[{"do":"fill","text":"25"},{"do":"click"},{"do":"expectNumber","equals":125},{"do":"expectNumber","equals":125}],"source":"scenarios/01-warehouse-stock-live-staff.json","stableKey":"ecommerce.spec.live-state.warehouse-stock.7c","statedBy":"an item's stock on the storefront is the sum of that item's units across all warehouses","withheld":null},{"category":"production","checkGroupId":"cancellation-and-return","criterionId":"3d","description":"cancelling a pending order removes it from the fulfilment queue","featureId":3,"featureName":"Cancellation and fulfilment","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.cancellation-queue-specifications","points":1,"provenBy":null,"requiresFeatures":["ecommerce.l2.order-cancellation-features","ecommerce.progression.fulfilment-queue"],"role":"guarantee","semantics":[{"do":"click"},{"contains":"Coffee Grinder","do":"expect"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"do":"click"},{"do":"waitUntilAbsent"}],"source":"scenarios/02-cancellation-queue.json","stableKey":"ecommerce.returns-pricing.cancellation-and-return.3d","statedBy":"the order leaves the fulfilment queue","withheld":null},{"category":"production","checkGroupId":"fulfilment-area-boundary","criterionId":"1d","description":"staff and administrators can open fulfilment while customers cannot","featureId":1,"featureName":"Fulfilment area access","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"provenBy":null,"requiresFeatures":["ecommerce.progression.fulfilment-queue"],"role":"guarantee","semantics":[{"do":"click"},{"do":"expect"},{"do":"click"},{"do":"expect"},{"do":"click","ifAvailable":true},{"absent":true,"do":"expect"}],"source":"scenarios/02-fulfilment-access.json","stableKey":"ecommerce.spec.access-control.fulfilment-area-boundary.1d","statedBy":"Customers cannot open the fulfilment area","withheld":null},{"category":"production","checkGroupId":"fulfilment-queue","criterionId":"1a","description":"an order placed by a customer appears in the staff queue without a reload","featureId":1,"featureName":"Live fulfilment queue","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.fulfilment-queue"],"role":"guarantee","semantics":[{"as":"depth-before","do":"recordNumber"},{"do":"click"},{"contains":"Desk Lamp","do":"expect"},{"do":"expectNumber","plus":1,"relativeTo":"depth-before"}],"source":"scenarios/02-fulfilment-live.json","stableKey":"ecommerce.spec.live-state.fulfilment-queue.1a","statedBy":"New orders appear in the queue without a reload","withheld":null},{"category":"feature","checkGroupId":"fulfilment-queue","criterionId":"1c","description":"shipping removes the order from the queue and marks the customer's order shipped","featureId":1,"featureName":"Ship a pending order","note":null,"packId":"ecommerce.progression.fulfilment-queue","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"feature","semantics":[{"action":"buy","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"pending"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click","ifAvailable":true},{"contains":"Keyboard","do":"expect"},{"do":"click"},{"attribute":"data-submit-state","do":"expect","value":"succeeded"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click","ifAvailable":true},{"absent":true,"contains":"Keyboard","do":"expect"},{"do":"reload"},{"do":"ensureSignedIn","name":"fq-ship"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"shipped"}],"source":"scenarios/02-fulfilment-ship.json","stableKey":"ecommerce.operations-access.fulfilment-queue.1c","statedBy":"Staff can mark an order as shipped; show the new status in the fulfilment area and the customer's order history.","withheld":null},{"category":"production","checkGroupId":"refund-accounting","criterionId":"203a","description":"concurrent cancellation restores original stock and the booked amount once, while revenue returns to its prior value","featureId":203,"featureName":"The books still balance once money can flow backwards","note":"Draft: native order, refund and original warehouse allocation evidence supplements the retained revenue assertions. Four calls from two sessions of the owner must cancel once; a repeated call may deliberately refuse or succeed without extra effects. Qualification pending.","observations":["requested","unmentioned"],"packId":"ecommerce.progression.cancellation-accounting-specifications","points":3,"provenBy":null,"requiresFeatures":["ecommerce.l2.order-cancellation-features"],"role":"guarantee","semantics":[{"as":"rev-start","do":"recordNumber"},{"do":"click"},{"do":"wait"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"admin-revenue"},{"do":"expectNumber","plus":42,"relativeTo":"rev-start"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"pending"},{"account":"{user:books}","as":"cancel-before","do":"dbRecordCheckout","item":"Desk Lamp","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"action":"cancel","do":"callConcurrently","from":"customer","input":{"attribute":"data-cancel-input","contains":"Desk Lamp","testid":"order-item"},"namedAction":{"args":[0],"id":"cancel","params":[{"in":"path","name":"orderId","placeholder":":id","wireType":"u64"}],"path":"/api/orders/:id/cancel","reducer":"cancel_order"},"requests":4},{"do":"expectCallOutcomes"},{"before":"cancel-before","do":"dbExpectCancellation"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"admin-revenue"},{"do":"expectNumber","plus":0,"relativeTo":"rev-start"}],"source":"scenarios/02-invariants.json","stableKey":"ecommerce.returns-pricing.refund-accounting.203a","statedBy":"revenue always equals the sum of orders that are still standing","withheld":null},{"category":"production","checkGroupId":"refund-accounting","criterionId":"203b","description":"a price change does not rewrite revenue already earned","featureId":203,"featureName":"The books still balance once money can flow backwards","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.price-accounting-specifications","points":2,"provenBy":null,"requiresFeatures":["ecommerce.l2.price-history-features"],"role":"guarantee","semantics":[{"as":"history-revenue-before-sale","do":"recordNumber"},{"do":"pressKey","key":"Escape"},{"do":"click"},{"do":"wait"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"admin-revenue"},{"do":"expectNumber","plus":79.5,"relativeTo":"history-revenue-before-sale"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Bluetooth Speaker","count":1,"do":"expect"},{"as":"rev-after-sale","do":"recordNumber"},{"do":"fill","text":"5.00"},{"do":"click"},{"do":"wait"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"expectNumber","equals":5},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"admin-revenue"},{"do":"expectNumber","plus":0,"relativeTo":"rev-after-sale"}],"source":"scenarios/02-invariants.json","stableKey":"ecommerce.returns-pricing.refund-accounting.203b","statedBy":"changing a price never alters the history, the revenue already recorded","withheld":null},{"category":"production","checkGroupId":"price-history","criterionId":"4b","description":"the new price reaches a signed-out visitor without a reload","featureId":4,"featureName":"Live catalog price","note":null,"packId":"ecommerce.l2.price-history-features","points":2,"provenBy":null,"role":"feature","semantics":[{"atLeast":2,"do":"expectNumber"},{"do":"fill","text":"1.00"},{"do":"click"},{"do":"expectNumber","equals":1}],"source":"scenarios/02-live-price.json","stableKey":"ecommerce.returns-pricing.price-history.4b","statedBy":"the storefront shows the new price immediately, to everyone","withheld":null},{"category":"feature","checkGroupId":"operational-views","criterionId":"5e","description":"the dashboard lists a current low-stock item","featureId":5,"featureName":"The low-stock view","note":null,"packId":"ecommerce.l2.inventory-dashboard","points":1,"provenBy":null,"role":"feature","semantics":[{"contains":"Air Purifier","do":"expect"}],"source":"scenarios/02-low-stock.json","stableKey":"ecommerce.inventory-operations.operational-views.5e","statedBy":"It lists items with 10 units or fewer, most urgent first.","withheld":null},{"category":"production","checkGroupId":"inventory-dashboard","criterionId":"5a","description":"an item falling to ten units or fewer joins the low-stock list, live","featureId":5,"featureName":"The low-stock view","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":3,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.inventory-dashboard"],"role":"guarantee","semantics":[{"contains":"Air Purifier","do":"expect"},{"do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click"},{"do":"fill","text":"8"},{"do":"click"},{"do":"waitUntilAbsent"},{"do":"click"},{"contains":"Air Purifier","do":"expect"}],"source":"scenarios/02-low-stock.json","stableKey":"ecommerce.spec.live-state.inventory-dashboard.5a","statedBy":"items enter and leave this list as stock moves, sells, is restocked, cancelled or returned","withheld":null},{"category":"feature","checkGroupId":"operational-views","criterionId":"5d","description":"a signed-out visitor sees a best seller in the recommendations list","featureId":5,"featureName":"Signed-out best sellers","note":null,"packId":"ecommerce.l2.sales-dashboard","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"reload"},{"do":"expectNumber","equals":1}],"source":"scenarios/02-operational-best-sellers.json","stableKey":"ecommerce.inventory-operations.operational-views.5d","statedBy":"Signed-out visitors see best sellers","withheld":null},{"category":"feature","checkGroupId":"operational-views","criterionId":"5f","description":"the dashboard shows category units and revenue","featureId":5,"featureName":"Category sales totals","note":null,"packId":"ecommerce.l2.sales-dashboard","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"click","ifAvailable":true,"unlessVisible":"category-row"},{"as":"audio-core-units","do":"recordNumber"},{"as":"audio-core-revenue","do":"recordNumber"},{"do":"click"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click"},{"do":"click","ifAvailable":true,"unlessVisible":"category-row"},{"do":"expectNumber","plus":1,"relativeTo":"audio-core-units"},{"do":"expectNumber","plus":79.5,"relativeTo":"audio-core-revenue"}],"source":"scenarios/02-operational-category-totals.json","stableKey":"ecommerce.inventory-operations.operational-views.5f","statedBy":"Category totals show units sold and revenue for each category.","withheld":null},{"category":"production","checkGroupId":"sales-dashboard","criterionId":"5b","description":"a purchase updates that category's units and revenue live","featureId":5,"featureName":"Category sales totals","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":3,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.sales-dashboard"],"role":"guarantee","semantics":[{"do":"click","ifAvailable":true,"unlessVisible":"category-row"},{"as":"audio-units","do":"recordNumber"},{"as":"audio-revenue","do":"recordNumber"},{"do":"click"},{"do":"expectNumber","plus":1,"relativeTo":"audio-units"},{"do":"expectNumber","plus":79.5,"relativeTo":"audio-revenue"}],"source":"scenarios/02-operational-category-totals.json","stableKey":"ecommerce.spec.live-state.sales-dashboard.5b","statedBy":"Category totals show units sold and revenue for each category","withheld":null},{"category":"feature","checkGroupId":"operational-views","criterionId":"5c","description":"a purchase recommends another item from that category and excludes an item in the cart","featureId":5,"featureName":"Customer recommendations","note":null,"packId":"ecommerce.l2.recommendations","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"contains":"Headphones","do":"expect"},{"do":"click"},{"do":"waitUntilAbsent"}],"source":"scenarios/02-operational-recommendations.json","stableKey":"ecommerce.inventory-operations.operational-views.5c","statedBy":"Recommend items from categories the customer bought from and exclude items already in the cart","withheld":null},{"category":"production","checkGroupId":"cancellation-and-return","criterionId":"3a","description":"cancelling a pending order restores its stock and revenue","featureId":3,"featureName":"Cancel a pending order","note":null,"packId":"ecommerce.l2.order-cancellation-features","points":2,"provenBy":null,"role":"feature","semantics":[{"as":"revenue-before","do":"recordNumber"},{"as":"stock-before","do":"recordNumber"},{"do":"click"},{"do":"expectNumber","plus":-1,"relativeTo":"stock-before"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"expectNumber","plus":64,"relativeTo":"revenue-before"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"pending"},{"do":"click"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"expectNumber","plus":0,"relativeTo":"revenue-before"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"expectNumber","plus":0,"relativeTo":"stock-before"}],"source":"scenarios/02-order-cancellation-core.json","stableKey":"ecommerce.returns-pricing.cancellation-and-return.3a","statedBy":"the stock goes back to the warehouse it came from and revenue falls","withheld":null},{"category":"feature","checkGroupId":"cancellation-and-return","criterionId":"3b","description":"a cancelled order is shown as cancelled in the customer's history","featureId":3,"featureName":"Cancellation history","note":null,"packId":"ecommerce.l2.order-cancellation-features","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"click"},{"do":"expect","ignoreCase":true,"value":"cancelled"}],"source":"scenarios/02-order-cancellation-history.json","stableKey":"ecommerce.returns-pricing.cancellation-and-return.3b","statedBy":"a customer can cancel an order that has not shipped","withheld":null},{"category":"production","checkGroupId":"price-history","criterionId":"4a","description":"a price change updates the live catalog but leaves the customer's exact paid price unchanged","featureId":4,"featureName":"Prices change, history does not","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.price-history-specifications","points":3,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.price-history-features"],"role":"guarantee","semantics":[{"do":"fill","enter":true,"text":"Air Purifier"},{"as":"air-purifier-paid","do":"recordNumber"},{"do":"click"},{"do":"fill","enter":true,"text":"Air Purifier"},{"do":"fill","text":"1.00"},{"do":"click"},{"do":"expectNumber","equals":1},{"do":"reload"},{"do":"ensureSignedIn","name":"history-persisted"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"do":"expectNumber","plus":0,"relativeTo":"air-purifier-paid"}],"source":"scenarios/02-paid-price-history.json","stableKey":"ecommerce.returns-pricing.price-history.4a","statedBy":"past orders keep the price that was paid","withheld":null},{"category":"feature","checkGroupId":"fulfilment-queue","criterionId":"1b","description":"the queue names the warehouse the order will ship from","featureId":1,"featureName":"Fulfilment queue","note":null,"packId":"ecommerce.progression.fulfilment-queue","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"warehouse":"West"},{"do":"click"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click","ifAvailable":true},{"contains":"Desk Lamp","do":"expect"},{"contains":"East","do":"expect"}],"source":"scenarios/02-queue-warehouse.json","stableKey":"ecommerce.operations-access.fulfilment-queue.1b","statedBy":"which warehouse each will ship from","withheld":null},{"category":"production","checkGroupId":"stock-conservation","criterionId":"202b","description":"a sale and its cancellation leave the shelf exactly as they found it","featureId":202,"featureName":"Stock recovery is durable across clients","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","points":2,"provenBy":null,"requiresFeatures":["ecommerce.l2.order-cancellation-features"],"role":"guarantee","semantics":[{"as":"east-before-cancel-202b","do":"dbRecordStock","item":"Induction Cooktop","warehouse":"East"},{"as":"west-before-cancel-202b","do":"dbRecordStock","item":"Induction Cooktop","warehouse":"West"},{"as":"stored-before-cancel-202b","do":"dbRecordStock","item":"Induction Cooktop"},{"as":"cancel-stock-before","do":"recordNumber"},{"do":"click"},{"do":"expectNumber","plus":-1,"relativeTo":"cancel-stock-before"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Induction Cooktop","count":1,"do":"expect"},{"do":"dbExpectStock","item":"Induction Cooktop","plus":-1,"relativeTo":"stored-before-cancel-202b"},{"do":"click"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"expectNumber","plus":0,"relativeTo":"cancel-stock-before"},{"do":"dbExpectStock","item":"Induction Cooktop","plus":0,"relativeTo":"stored-before-cancel-202b"},{"do":"dbExpectStock","item":"Induction Cooktop","plus":0,"relativeTo":"east-before-cancel-202b","warehouse":"East"},{"do":"dbExpectStock","item":"Induction Cooktop","plus":0,"relativeTo":"west-before-cancel-202b","warehouse":"West"}],"source":"scenarios/02-self-contained.json","stableKey":"ecommerce.inventory-operations.stock-conservation.202b","statedBy":"the stock goes back to the warehouse it came from","withheld":null},{"category":"production","checkGroupId":"stock-conservation","criterionId":"202c","description":"a fresh client sees the restored total after a sale is cancelled","featureId":202,"featureName":"Stock recovery is durable across clients","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","points":1,"provenBy":null,"requiresFeatures":["ecommerce.l2.order-cancellation-features"],"role":"guarantee","semantics":[{"as":"east-before-cancel-202c","do":"dbRecordStock","item":"Headphones","warehouse":"East"},{"as":"west-before-cancel-202c","do":"dbRecordStock","item":"Headphones","warehouse":"West"},{"as":"stored-before-cancel-202c","do":"dbRecordStock","item":"Headphones"},{"as":"fresh-stock-before","do":"recordNumber"},{"do":"click"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Headphones","count":1,"do":"expect"},{"do":"dbExpectStock","item":"Headphones","plus":-1,"relativeTo":"stored-before-cancel-202c"},{"do":"click"},{"do":"freshClient"},{"do":"expectNumber","plus":0,"relativeTo":"fresh-stock-before"},{"do":"dbExpectStock","item":"Headphones","plus":0,"relativeTo":"stored-before-cancel-202c"},{"do":"dbExpectStock","item":"Headphones","plus":0,"relativeTo":"east-before-cancel-202c","warehouse":"East"},{"do":"dbExpectStock","item":"Headphones","plus":0,"relativeTo":"west-before-cancel-202c","warehouse":"West"}],"source":"scenarios/02-self-contained.json","stableKey":"ecommerce.inventory-operations.stock-conservation.202c","statedBy":"every one of these numbers is the same for every person looking at it","withheld":null},{"category":"production","checkGroupId":"operator-authorization","criterionId":"201c","description":"the server refuses a customer's direct attempt to ship their own pending order","featureId":201,"featureName":"Shipping requires an operator","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.operations-access-specifications","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"guarantee","semantics":[{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"click"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Coffee Grinder","count":1,"do":"expect"},{"action":"ship","do":"callAction","from":"customer","input":{"attribute":"data-ship-input","contains":"Coffee Grinder","testid":"order-item"},"namedAction":{"args":[0],"id":"ship","params":[{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/fulfilment/ship","reducer":"ship_order"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"reload"},{"do":"ensureSignedIn","name":"direct-notstaff"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"shipped"},{"action":"ship","do":"callAction","input":{"attribute":"data-ship-input","contains":"Laptop Stand","testid":"order-item"},"namedAction":{"args":[0],"id":"ship","params":[{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/fulfilment/ship","reducer":"ship_order"}},{"do":"expectActionOutcome","outcome":"refused","routeProvenBy":"staff"},{"do":"reload"},{"do":"ensureSignedIn","name":"direct-notstaff"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"pending"}],"source":"scenarios/02-server-actions.json","stableKey":"ecommerce.operations-access.operator-authorization.201c","statedBy":"Staff mark an order shipped","withheld":null},{"category":"production","checkGroupId":"stock-conservation","criterionId":"202d","description":"a direct transfer racing a direct purchase leaves the exact starting total minus the sold unit","featureId":202,"featureName":"Stock is conserved while operations overlap","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.stock-transfers-features"],"role":"guarantee","semantics":[{"do":"dbExpectStock","equals":60,"item":"Headphones","warehouse":"East"},{"do":"dbExpectStock","equals":40,"item":"Headphones","warehouse":"West"},{"as":"direct-race-stock-before","do":"dbRecordStock","item":"Headphones"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"reload"},{"do":"ensureSignedIn","name":"direct-conserve"},{"do":"fill","text":"East"},{"do":"fill","text":"West"},{"do":"fill","text":"25"},{"branches":[[{"action":"transfer","do":"callAction","input":{"attribute":"data-transfer-input","contains":"Headphones","testid":"admin-item-row"},"namedAction":{"args":[0,0,0,25],"id":"transfer","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"fromWarehouseId","wireType":"u64"},{"in":"body","name":"toWarehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/transfer","reducer":"admin_transfer_stock"}}],[{"action":"buy","do":"callAction","input":{"attribute":"data-buy-input","contains":"Headphones","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}}]],"do":"race"},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"expectNumber","plus":-1,"relativeTo":"direct-race-stock-before"},{"do":"reload"},{"do":"ensureSignedIn","name":"direct-conserve"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"expectNumber","plus":-1,"relativeTo":"direct-race-stock-before"},{"atLeast":34,"atMost":35,"do":"dbExpectStock","item":"Headphones","warehouse":"East"},{"atLeast":64,"atMost":65,"do":"dbExpectStock","item":"Headphones","warehouse":"West"},{"do":"dbExpectStock","item":"Headphones","plus":-1,"relativeTo":"direct-race-stock-before"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Headphones","count":1,"do":"expect"}],"source":"scenarios/02-server-actions.json","stableKey":"ecommerce.inventory-operations.stock-conservation.202d","statedBy":"a transfer moves stock, it does not create or destroy it","withheld":null},{"category":"production","checkGroupId":"order-owner","criterionId":"204a","description":"the server refuses one customer trying to cancel another customer's still-pending order","featureId":204,"featureName":"An order belongs to the person who placed it","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.operations-access-specifications","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.order-cancellation-features"],"role":"guarantee","semantics":[{"action":"cancel","do":"callAction","input":{"attribute":"data-cancel-input","contains":"Desk Lamp","testid":"order-item"},"namedAction":{"args":[0],"id":"cancel","params":[{"in":"path","name":"orderId","placeholder":":id","wireType":"u64"}],"path":"/api/orders/:id/cancel","reducer":"cancel_order"}},{"do":"expectActionOutcome","outcome":"accepted"},{"action":"cancel","do":"callAction","from":"owner","input":{"attribute":"data-cancel-input","contains":"Keyboard","testid":"order-item"},"namedAction":{"args":[0],"id":"cancel","params":[{"in":"path","name":"orderId","placeholder":":id","wireType":"u64"}],"path":"/api/orders/:id/cancel","reducer":"cancel_order"}},{"do":"expectActionOutcome","outcome":"refused","routeProvenBy":"owner"},{"do":"reload"},{"do":"ensureSignedIn","name":"direct-owner"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"pending"}],"source":"scenarios/02-server-actions.json","stableKey":"ecommerce.operations-access.order-owner.204a","statedBy":"A customer can cancel an order that has not shipped","withheld":null},{"category":"production","checkGroupId":"warehouse-transfer","criterionId":"2a","description":"a transfer decreases the source, increases the destination, and preserves the item's exact total","featureId":2,"featureName":"Moving stock between warehouses","note":null,"packId":"ecommerce.l2.stock-transfers-features","points":3,"provenBy":null,"role":"feature","semantics":[{"as":"product-East","do":"dbRecordStock","item":"Headphones","warehouse":"East"},{"as":"product-West","do":"dbRecordStock","item":"Headphones","warehouse":"West"},{"as":"transfer-item-before","do":"recordNumber"},{"contains":"East","count":1,"do":"expect"},{"as":"transfer-east-before","do":"recordNumber"},{"contains":"West","count":1,"do":"expect"},{"as":"transfer-west-before","do":"recordNumber"},{"do":"fill","text":"East"},{"do":"fill","text":"West"},{"do":"fill","text":"10"},{"do":"click"},{"contains":"East","count":1,"do":"expect"},{"do":"dbExpectStock","item":"Headphones","plus":-10,"relativeTo":"product-East","warehouse":"East"},{"do":"dbExpectStock","item":"Headphones","plus":10,"relativeTo":"product-West","warehouse":"West"},{"do":"expectNumber","plus":-10,"relativeTo":"transfer-east-before"},{"contains":"West","count":1,"do":"expect"},{"do":"expectNumber","plus":10,"relativeTo":"transfer-west-before"},{"do":"expectNumber","plus":0,"relativeTo":"transfer-item-before"}],"source":"scenarios/02-strengthened.json","stableKey":"ecommerce.inventory-operations.warehouse-transfer.2a","statedBy":"a transfer moves stock, it does not create or destroy it","withheld":null},{"category":"production","checkGroupId":"operator-authorization","criterionId":"201a","description":"the server refuses a customer's direct transfer and neither warehouse nor the item total changes","featureId":201,"featureName":"Operating the store requires authorization","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.operations-access-specifications","points":2,"provenBy":null,"requiresFeatures":["ecommerce.l2.stock-transfers-features"],"role":"guarantee","semantics":[{"as":"authorized-transfer-east","do":"dbRecordStock","item":"Headphones","warehouse":"East"},{"as":"authorized-transfer-west","do":"dbRecordStock","item":"Headphones","warehouse":"West"},{"do":"fill","text":"East"},{"do":"fill","text":"West"},{"do":"fill","text":"25"},{"action":"transfer","do":"callAction","input":{"attribute":"data-transfer-input","contains":"Headphones","testid":"admin-item-row"},"namedAction":{"args":[0,0,0,25],"id":"transfer","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"fromWarehouseId","wireType":"u64"},{"in":"body","name":"toWarehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/transfer","reducer":"admin_transfer_stock"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"dbExpectStock","item":"Headphones","plus":-25,"relativeTo":"authorized-transfer-east","warehouse":"East"},{"do":"dbExpectStock","item":"Headphones","plus":25,"relativeTo":"authorized-transfer-west","warehouse":"West"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"reload"},{"do":"ensureSignedIn","name":"not-operator"},{"as":"unauthorized-item-before","do":"recordNumber"},{"contains":"East","count":1,"do":"expect"},{"as":"unauthorized-east-before","do":"recordNumber"},{"contains":"West","count":1,"do":"expect"},{"as":"unauthorized-west-before","do":"recordNumber"},{"do":"fill","text":"East"},{"do":"fill","text":"West"},{"as":"refused-East","do":"dbRecordStock","item":"Headphones","warehouse":"East"},{"as":"refused-West","do":"dbRecordStock","item":"Headphones","warehouse":"West"},{"action":"transfer","do":"callAction","from":"admin","input":{"attribute":"data-transfer-input","contains":"Headphones","testid":"admin-item-row"},"namedAction":{"args":[0,0,0,25],"id":"transfer","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"fromWarehouseId","wireType":"u64"},{"in":"body","name":"toWarehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/transfer","reducer":"admin_transfer_stock"}},{"do":"expectActionOutcome","outcome":"refused","routeProvenBy":"admin"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"reload"},{"do":"ensureSignedIn","name":"not-operator"},{"contains":"East","count":1,"do":"expect"},{"do":"dbExpectStock","item":"Headphones","plus":0,"relativeTo":"refused-East","warehouse":"East"},{"do":"dbExpectStock","item":"Headphones","plus":0,"relativeTo":"refused-West","warehouse":"West"},{"do":"expectNumber","plus":0,"relativeTo":"unauthorized-east-before"},{"contains":"West","count":1,"do":"expect"},{"do":"expectNumber","plus":0,"relativeTo":"unauthorized-west-before"},{"do":"expectNumber","plus":0,"relativeTo":"unauthorized-item-before"}],"source":"scenarios/02-strengthened.json","stableKey":"ecommerce.operations-access.operator-authorization.201a","statedBy":"An admin can transfer a number of units of an item from one warehouse to another","withheld":null},{"category":"production","checkGroupId":"operator-authorization","criterionId":"201b","description":"the server refuses a customer's direct price change and the last accepted price remains exact","featureId":201,"featureName":"Operating the store requires authorization","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.operations-access-specifications","points":2,"provenBy":null,"requiresFeatures":["ecommerce.l2.price-history-features"],"role":"guarantee","semantics":[{"do":"fill","text":"77.00"},{"action":"price","do":"callAction","input":{"attribute":"data-price-input","contains":"Gaming Mouse","testid":"admin-item-row"},"namedAction":{"args":[0,1],"id":"price","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"price"}],"path":"/api/admin/price","reducer":"admin_change_price"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"reload"},{"do":"ensureSignedIn","name":"not-operator"},{"do":"expectNumber","equals":77},{"do":"fill","text":"1.00"},{"action":"price","do":"callAction","from":"admin","input":{"attribute":"data-price-input","contains":"Gaming Mouse","testid":"admin-item-row"},"namedAction":{"args":[0,1],"id":"price","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"price"}],"path":"/api/admin/price","reducer":"admin_change_price"}},{"do":"expectActionOutcome","outcome":"refused","routeProvenBy":"admin"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"reload"},{"do":"ensureSignedIn","name":"not-operator"},{"do":"expectNumber","equals":77}],"source":"scenarios/02-strengthened.json","stableKey":"ecommerce.operations-access.operator-authorization.201b","statedBy":"an admin can change an item's price","withheld":null},{"category":"production","checkGroupId":"stock-conservation","criterionId":"202a","description":"a transfer decreases East, increases West, and leaves the item's exact total unchanged","featureId":202,"featureName":"Stock is conserved however it moves","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","points":2,"provenBy":null,"requiresFeatures":["ecommerce.l2.stock-transfers-features"],"role":"guarantee","semantics":[{"as":"product-East","do":"dbRecordStock","item":"Espresso Machine","warehouse":"East"},{"as":"product-West","do":"dbRecordStock","item":"Espresso Machine","warehouse":"West"},{"as":"conservation-item-before","do":"recordNumber"},{"contains":"East","count":1,"do":"expect"},{"as":"conservation-east-before","do":"recordNumber"},{"contains":"West","count":1,"do":"expect"},{"as":"conservation-west-before","do":"recordNumber"},{"do":"fill","text":"East"},{"do":"fill","text":"West"},{"do":"fill","text":"17"},{"do":"click"},{"contains":"East","count":1,"do":"expect"},{"do":"dbExpectStock","item":"Espresso Machine","plus":-17,"relativeTo":"product-East","warehouse":"East"},{"do":"dbExpectStock","item":"Espresso Machine","plus":17,"relativeTo":"product-West","warehouse":"West"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","unlessVisible":"admin-revenue"},{"do":"expectNumber","plus":-17,"relativeTo":"conservation-east-before"},{"contains":"West","count":1,"do":"expect"},{"do":"expectNumber","plus":17,"relativeTo":"conservation-west-before"},{"do":"expectNumber","plus":0,"relativeTo":"conservation-item-before"}],"source":"scenarios/02-strengthened.json","stableKey":"ecommerce.inventory-operations.stock-conservation.202a","statedBy":"a transfer moves stock, it does not create or destroy it","withheld":null},{"category":"production","checkGroupId":"stock-transfer-overdraw","criterionId":"2c","description":"a transfer that would overdraw a warehouse is refused and changes neither warehouse nor the item total","featureId":2,"featureName":"Moving stock between warehouses","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":2,"provenBy":null,"requiresFeatures":["ecommerce.l2.stock-transfers-features"],"role":"guarantee","semantics":[{"as":"product-East","do":"dbRecordStock","item":"Headphones","warehouse":"East"},{"as":"product-West","do":"dbRecordStock","item":"Headphones","warehouse":"West"},{"as":"overdraw-item-before","do":"recordNumber"},{"as":"overdraw-east-before","do":"recordNumber"},{"as":"overdraw-west-before","do":"recordNumber"},{"do":"fill","text":"East"},{"do":"fill","text":"West"},{"do":"fill","text":"99999"},{"do":"click"},{"do":"expect"},{"do":"dbExpectStock","item":"Headphones","plus":0,"relativeTo":"product-East","warehouse":"East"},{"do":"dbExpectStock","item":"Headphones","plus":0,"relativeTo":"product-West","warehouse":"West"},{"do":"expectNumber","plus":0,"relativeTo":"overdraw-east-before"},{"do":"expectNumber","plus":0,"relativeTo":"overdraw-west-before"},{"do":"expectNumber","plus":0,"relativeTo":"overdraw-item-before"}],"source":"scenarios/02-transfer-overdraw.json","stableKey":"ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c","statedBy":"a transfer that would leave a warehouse short is refused and changes nothing","withheld":null},{"category":"production","checkGroupId":"stock-transfers","criterionId":"2b","description":"both warehouse totals move live and in opposite directions as stock is transferred","featureId":2,"featureName":"Warehouse totals","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":2,"provenBy":null,"requiresFeatures":["ecommerce.l2.stock-transfers-features"],"role":"guarantee","semantics":[{"as":"product-East","do":"dbRecordStock","item":"Headphones","warehouse":"East"},{"as":"product-West","do":"dbRecordStock","item":"Headphones","warehouse":"West"},{"contains":"East","count":1,"do":"expect"},{"as":"warehouse-east-before","do":"recordNumber"},{"contains":"West","count":1,"do":"expect"},{"as":"warehouse-west-before","do":"recordNumber"},{"do":"fill","text":"East"},{"do":"fill","text":"West"},{"do":"fill","text":"10"},{"do":"click"},{"contains":"East","count":1,"do":"expect"},{"do":"expectNumber","plus":-10,"relativeTo":"warehouse-east-before"},{"contains":"West","count":1,"do":"expect"},{"do":"expectNumber","plus":10,"relativeTo":"warehouse-west-before"},{"do":"dbExpectStock","item":"Headphones","plus":-10,"relativeTo":"product-East","warehouse":"East"},{"do":"dbExpectStock","item":"Headphones","plus":10,"relativeTo":"product-West","warehouse":"West"}],"source":"scenarios/02-transfer-totals.json","stableKey":"ecommerce.spec.live-state.stock-transfers.2b","statedBy":"the per-warehouse numbers staff and admins see both move at once","withheld":null},{"category":"production","checkGroupId":"cart-expiration","criterionId":"304a","description":"an inactive cart expires without a browser, releases stock, and returns empty","featureId":304,"featureName":"An inactive cart expires","note":null,"packId":"ecommerce.l3.cart-expiration-features","points":4,"provenBy":null,"role":"feature","semantics":[{"do":"wait"},{"do":"reload"},{"do":"expectNumber","plus":0,"relativeTo":"before"},{"do":"openClient"},{"do":"expect"},{"do":"expectNumber","equals":0}],"source":"scenarios/03-cart-expiration.json","stableKey":"ecommerce.l3.cart-expiration.cart-expiration.304a","statedBy":"A cart with no activity for five minutes expires and releases its reservations.","withheld":null},{"category":"production","checkGroupId":"scheduled-work-access","criterionId":"317a","description":"the server refuses customer scheduling and cancellation of restocks","featureId":317,"featureName":"Customers cannot manage scheduled restocks","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-access-specifications","points":3,"provenBy":null,"requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"role":"guarantee","semantics":[{"do":"fill","text":"Webcam"},{"do":"fill","text":"West"},{"do":"fill","text":"3"},{"do":"fill","text":"180"},{"action":"scheduleRestock","authentication":"actor","do":"callAction","from":"admin","input":{"attribute":"data-action-input","testid":"schedule-restock-submit"},"namedAction":{"args":["","",0,0],"id":"scheduleRestock","method":"POST","params":[{"in":"body","name":"item"},{"in":"body","name":"warehouse"},{"in":"body","name":"quantity"},{"in":"body","name":"delaySeconds"}],"path":"/api/admin/scheduled-restocks","reducer":"schedule_restock"}},{"do":"expectActionOutcome","outcome":"refused"},{"do":"replayAs","from":"admin","match":"DELETE","namedAction":{"args":[0],"id":"cancelScheduledRestock","method":"DELETE","params":[{"in":"path","name":"restockId","placeholder":"{restockId}","wireType":"u64"}],"path":"/api/admin/scheduled-restocks/{restockId}","reducer":"cancel_scheduled_restock"},"namedTarget":{"attribute":"data-entity-id","testid":"pending-restock-item","valueType":"string"}},{"do":"expectReplayRejected"},{"count":1,"do":"expect"},{"do":"click"}],"source":"scenarios/03-deferred-access.json","stableKey":"ecommerce.l3.deferred-access.scheduled-work-access.317a","statedBy":"Only an admin can schedule or cancel a restock.","withheld":null},{"category":"production","checkGroupId":"restart-survival","criterionId":"311a","description":"a restock scheduled before restart still applies","featureId":311,"featureName":"A scheduled restock survives restart","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-durability-specifications","points":4,"provenBy":null,"requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"role":"guarantee","semantics":[{"do":"dbExpectStock","item":"Air Purifier","plus":5,"relativeTo":"before"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"expect"},{"absent":true,"do":"expect"}],"source":"scenarios/03-deferred-durability.json","stableKey":"ecommerce.l3.deferred-durability.restart-survival.311a","statedBy":"Pending restocks survive a backend restart.","withheld":null},{"category":"production","checkGroupId":"restart-survival","criterionId":"314a","description":"a reservation pending before restart still expires and returns stock","featureId":314,"featureName":"A reservation survives restart","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-durability-specifications","points":4,"provenBy":null,"requiresFeatures":["ecommerce.l3.reservations-features"],"role":"guarantee","semantics":[{"atMost":70000,"do":"expectElapsed","since":"pending-314"},{"do":"reload"},{"atMost":70000,"do":"expectElapsed","since":"pending-314"},{"do":"expectNumber","plus":-1,"relativeTo":"before"},{"atMost":70000,"do":"expectElapsed","since":"pending-314"},{"do":"wait","since":"pending-314-accepted"},{"do":"reload"},{"do":"expectNumber","plus":0,"relativeTo":"before"},{"do":"reload"},{"do":"ensureSignedIn","name":"durable-reservation"},{"do":"click","unlessVisible":"cart-item"},{"do":"expect"}],"source":"scenarios/03-deferred-durability.json","stableKey":"ecommerce.l3.deferred-durability.restart-survival.314a","statedBy":"Pending reservations survive a backend restart.","withheld":null},{"category":"production","checkGroupId":"restart-survival","criterionId":"315a","description":"an order shipped before restart still becomes delivered","featureId":315,"featureName":"An order transition survives restart","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-durability-specifications","points":4,"provenBy":null,"requiresFeatures":["ecommerce.l3.order-delivery-features"],"role":"guarantee","semantics":[{"do":"wait","since":"delivery-start-accepted"},{"do":"reload"},{"do":"ensureSignedIn","name":"durable-delivery"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"delivered"}],"source":"scenarios/03-deferred-durability.json","stableKey":"ecommerce.l3.deferred-durability.restart-survival.315a","statedBy":"Pending order delivery survives a backend restart.","withheld":null},{"category":"production","checkGroupId":"restart-survival","criterionId":"316a","description":"a cart survives restart and expires near its original five-minute deadline","featureId":316,"featureName":"Cart expiration survives restart","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-durability-specifications","points":4,"provenBy":null,"requiresFeatures":["ecommerce.l3.cart-expiration-features"],"role":"guarantee","semantics":[{"atMost":250000,"do":"expectElapsed","since":"pending-316"},{"do":"reload"},{"do":"ensureSignedIn","name":"durable-cart"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"cart-item"},{"atMost":250000,"do":"expectElapsed","since":"pending-316"},{"do":"expectNumber","equals":1},{"atMost":250000,"do":"expectElapsed","since":"pending-316"},{"do":"wait","since":"pending-316-accepted"},{"do":"reload"},{"do":"ensureSignedIn","name":"durable-cart"},{"do":"click"},{"do":"expectNumber","equals":0},{"do":"expect"},{"do":"reload"},{"do":"expectNumber","plus":0,"relativeTo":"before"}],"source":"scenarios/03-deferred-durability.json","stableKey":"ecommerce.l3.deferred-durability.restart-survival.316a","statedBy":"Pending cart expiration survives a backend restart.","withheld":null},{"category":"production","checkGroupId":"exactly-once","criterionId":"311a","description":"restart cannot replay a completed restock","featureId":311,"featureName":"A restock applies once","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-integrity-specifications","points":3,"provenBy":null,"requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"dbExpectStock","item":"Air Purifier","plus":5,"relativeTo":"before"},{"do":"restartBackend"},{"do":"reload"},{"do":"wait"},{"do":"dbExpectStock","item":"Air Purifier","plus":5,"relativeTo":"before"}],"source":"scenarios/03-deferred-integrity.json","stableKey":"ecommerce.l3.deferred-integrity.exactly-once.311a","statedBy":"Restarting the backend cannot apply a restock more than once.","withheld":null},{"category":"production","checkGroupId":"exactly-once","criterionId":"312a","description":"restart leaves one delivered order record","featureId":312,"featureName":"Delivery applies once","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-integrity-specifications","points":3,"provenBy":null,"requiresFeatures":["ecommerce.l3.order-delivery-features"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click","ifAvailable":true},{"contains":"Desk Lamp","count":1,"do":"expect"},{"do":"expect","ignoreCase":true,"value":"delivered"}],"source":"scenarios/03-deferred-integrity.json","stableKey":"ecommerce.l3.deferred-integrity.exactly-once.312a","statedBy":"A delivered order is not duplicated by a backend restart.","withheld":null},{"category":"production","checkGroupId":"stock-conservation","criterionId":"313a","description":"expiry returns exactly the unit reserved","featureId":313,"featureName":"Reservation expiry conserves stock","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-integrity-specifications","points":3,"provenBy":null,"requiresFeatures":["ecommerce.l3.reservations-features"],"role":"guarantee","semantics":[{"do":"wait"},{"do":"reload"},{"do":"expectNumber","plus":0,"relativeTo":"before"}],"source":"scenarios/03-deferred-integrity.json","stableKey":"ecommerce.l3.deferred-integrity.stock-conservation.313a","statedBy":"Reservation expiry returns exactly the stock that the reservation took.","withheld":null},{"category":"production","checkGroupId":"stock-conservation","criterionId":"314a","description":"checkout does not decrement stock after the reservation already did","featureId":314,"featureName":"Checkout conserves reserved stock","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-integrity-specifications","points":3,"provenBy":null,"requiresFeatures":["ecommerce.l3.reservations-features"],"role":"guarantee","semantics":[{"do":"click"},{"do":"expectNumber","equals":0},{"do":"reload"},{"do":"ensureSignedIn","name":"conserve-checkout"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Headphones","do":"expectElementCount","equals":1},{"do":"reload"},{"do":"expectNumber","plus":-1,"relativeTo":"before"}],"source":"scenarios/03-deferred-integrity.json","stableKey":"ecommerce.l3.deferred-integrity.stock-conservation.314a","statedBy":"Checkout does not take reserved stock twice.","withheld":null},{"category":"production","checkGroupId":"order-delivery","criterionId":"303a","description":"a shipped order becomes delivered in customer and staff views","featureId":303,"featureName":"A shipped order becomes delivered","note":null,"packId":"ecommerce.l3.order-delivery-features","points":3,"provenBy":null,"role":"feature","semantics":[{"do":"wait"},{"do":"reload"},{"do":"ensureSignedIn","name":"delivery-live"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"delivered"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"completed-order-item"},{"do":"expect","ignoreCase":true,"value":"delivered"}],"source":"scenarios/03-order-delivery.json","stableKey":"ecommerce.l3.order-delivery.order-delivery.303a","statedBy":"A shipped order becomes delivered after 60 seconds.","withheld":null},{"category":"production","checkGroupId":"order-delivery","criterionId":"305a","description":"a cancelled order remains cancelled after the delivery interval","featureId":305,"featureName":"Cancellation is final","note":null,"packId":"ecommerce.l3.order-delivery-features","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"wait"},{"do":"reload"},{"do":"ensureSignedIn","name":"delivery-cancel"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"cancelled"}],"source":"scenarios/03-order-delivery.json","stableKey":"ecommerce.l3.order-delivery.order-delivery.305a","statedBy":"A cancelled order never advances.","withheld":null},{"category":"production","checkGroupId":"reservations","criterionId":"301a","description":"adding an item reserves one unit for every open viewer","featureId":301,"featureName":"A cart reserves stock","note":null,"packId":"ecommerce.l3.reservations-features","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"do":"expectNumber","plus":-1,"relativeTo":"before"}],"source":"scenarios/03-reservations.json","stableKey":"ecommerce.l3.reservations.reservations.301a","statedBy":"Adding an item to a cart reserves its stock immediately for 90 seconds.","withheld":null},{"category":"interface","checkGroupId":"reservations","criterionId":"305a","description":"the reservation timer decreases","featureId":305,"featureName":"Reservation time is visible","note":null,"packId":"ecommerce.l3.reservations-features","points":1,"provenBy":null,"role":"feature","semantics":[{"atLeast":1,"atMost":90,"do":"expectNumber"},{"as":"initial-countdown","do":"recordNumber"},{"do":"wait"},{"comparison":"atMost","do":"expectNumber","plus":-1,"relativeTo":"initial-countdown"}],"source":"scenarios/03-reservations.json","stableKey":"ecommerce.l3.reservations.reservations.305a","statedBy":"The cart shows the remaining reservation time.","withheld":null},{"category":"feature","checkGroupId":"reservations","criterionId":"306a","description":"checkout converts the reservation into an order and empties the cart","featureId":306,"featureName":"Checkout consumes a reservation","note":null,"packId":"ecommerce.l3.reservations-features","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"do":"expectNumber","equals":0},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Keyboard","count":1,"do":"expect"}],"source":"scenarios/03-reservations.json","stableKey":"ecommerce.l3.reservations.reservations.306a","statedBy":"Checkout converts a live reservation into a sale.","withheld":null},{"category":"feature","checkGroupId":"reservations","criterionId":"307a","description":"an expired reservation marks its cart line","featureId":307,"featureName":"A reservation expires","note":null,"packId":"ecommerce.l3.reservations-features","points":3,"provenBy":null,"role":"feature","semantics":[{"do":"wait"},{"do":"expect"}],"source":"scenarios/03-reservations.json","stableKey":"ecommerce.l3.reservations.reservations.307a","statedBy":"An expired reservation remains visible as expired.","withheld":null},{"category":"feature","checkGroupId":"reservations","criterionId":"308a","description":"raising quantity starts a new reservation window","featureId":308,"featureName":"Changing quantity renews a reservation","note":null,"packId":"ecommerce.l3.reservations-features","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"wait"},{"atLeast":35,"do":"expectNumber"},{"absent":true,"do":"expect"}],"source":"scenarios/03-reservations.json","stableKey":"ecommerce.l3.reservations.reservations.308a","statedBy":"Adding the item again renews the reservation.","withheld":null},{"category":"feature","checkGroupId":"scheduled-restocks","criterionId":"305a","description":"a due restock updates stock and moves to the ledger","featureId":305,"featureName":"A due restock applies","note":null,"packId":"ecommerce.l3.scheduled-restocks-features","points":3,"provenBy":null,"role":"feature","semantics":[{"as":"ledger-before","count":true,"do":"recordNumber"},{"do":"fill","text":"Keyboard"},{"do":"fill","text":"West"},{"do":"fill","text":"7"},{"do":"fill","text":"15"},{"do":"click"},{"do":"wait"},{"do":"reload"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"expectNumber","plus":7,"relativeTo":"before"},{"absent":true,"do":"expect"},{"do":"expectElementCount","plus":1,"relativeTo":"ledger-before"}],"source":"scenarios/03-scheduled-restock-apply.json","stableKey":"ecommerce.l3.scheduled-restocks.scheduled-restocks.305a","statedBy":"A due restock updates stock, leaves the pending list, and enters the stock ledger.","withheld":null},{"category":"production","checkGroupId":"scheduled-restocks","criterionId":"306a","description":"a cancelled restock never applies","featureId":306,"featureName":"A scheduled restock can be cancelled","note":null,"packId":"ecommerce.l3.scheduled-restocks-features","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"fill","text":"Desk Lamp"},{"do":"fill","text":"East"},{"do":"fill","text":"9"},{"do":"fill","text":"15"},{"do":"click"},{"count":1,"do":"expect"},{"do":"click"},{"do":"wait"},{"do":"reload"},{"do":"expectNumber","plus":0,"relativeTo":"before"}],"source":"scenarios/03-scheduled-restock-cancel.json","stableKey":"ecommerce.l3.scheduled-restocks.scheduled-restocks.306a","statedBy":"An admin can schedule and cancel a restock.","withheld":null},{"category":"feature","checkGroupId":"scheduled-restocks","criterionId":"302a","description":"a scheduled restock is pending and its remaining time decreases","featureId":302,"featureName":"A restock is pending before it is due","note":null,"packId":"ecommerce.l3.scheduled-restocks-features","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"fill","text":"Webcam"},{"do":"fill","text":"East"},{"do":"fill","text":"7"},{"do":"fill","text":"90"},{"do":"click"},{"count":1,"do":"expect"},{"atLeast":1,"atMost":90,"do":"expectNumber"},{"as":"initial-countdown","do":"recordNumber"},{"do":"wait"},{"comparison":"atMost","do":"expectNumber","plus":-1,"relativeTo":"initial-countdown"},{"do":"click"}],"source":"scenarios/03-scheduled-restocks.json","stableKey":"ecommerce.l3.scheduled-restocks.scheduled-restocks.302a","statedBy":"A pending restock shows its remaining time.","withheld":null},{"category":"production","checkGroupId":"server-time","criterionId":"312a","description":"restart preserves the due time and the work later completes","featureId":312,"featureName":"Restart does not run work early","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.server-time-specifications","points":3,"provenBy":null,"requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"role":"guarantee","semantics":[{"do":"wait"},{"do":"reload"},{"atMost":100000,"do":"expectElapsed","since":"restock-start"},{"do":"dbExpectStock","item":"Espresso Machine","plus":0,"relativeTo":"before"},{"count":1,"do":"expect"},{"atMost":100000,"do":"expectElapsed","since":"restock-start"},{"do":"wait","since":"restock-start-accepted"},{"do":"dbExpectStock","item":"Espresso Machine","plus":4,"relativeTo":"before"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"expect"},{"absent":true,"do":"expect"}],"source":"scenarios/03-server-time.json","stableKey":"ecommerce.l3.server-time.server-time.312a","statedBy":"A pending restock does not run early after a restart.","withheld":null},{"category":"production","checkGroupId":"server-time","criterionId":"313a","description":"a reservation expires while its browser is closed","featureId":313,"featureName":"A browser is not the clock","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.l3.server-time-specifications","points":3,"provenBy":null,"requiresFeatures":["ecommerce.l3.reservations-features"],"role":"guarantee","semantics":[{"do":"wait"},{"do":"reload"},{"do":"expectNumber","plus":0,"relativeTo":"before"}],"source":"scenarios/03-server-time.json","stableKey":"ecommerce.l3.server-time.server-time.313a","statedBy":"A reservation expires without an open browser.","withheld":null},{"category":"production","checkGroupId":"account-state-recovery","criterionId":"105b","description":"the same account and cart survive the connection dropping and coming back","featureId":105,"featureName":"An account keeps what belongs to it","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.accounts","ecommerce.feature.checkout"],"role":"guarantee","semantics":[{"do":"setOffline","offline":true},{"do":"click"},{"do":"setOffline","offline":false},{"contains":"pat","do":"expect"},{"do":"click","unlessVisible":"cart-total"},{"contains":"Keyboard","do":"expect"},{"contains":"Headphones","do":"expect"}],"source":"scenarios/progression-account-state-reconnect.json","stableKey":"ecommerce.spec.state-durability.account-state-recovery.105b","statedBy":"the signed-in account and its current data remain available after reconnecting","withheld":null},{"category":"production","checkGroupId":"account-state-recovery","criterionId":"105a","description":"cart and order history survive reload and backend restart, including a fresh account login","featureId":105,"featureName":"An account keeps what belongs to it","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.accounts","ecommerce.feature.checkout"],"role":"guarantee","semantics":[{"do":"click"},{"do":"click","unlessVisible":"cart-total"},{"do":"click"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"click"},{"do":"click","unlessVisible":"cart-total"},{"contains":"Keyboard","do":"expect"},{"do":"reload"},{"contains":"pat","do":"expect"},{"do":"click","unlessVisible":"cart-total"},{"contains":"Keyboard","do":"expect"},{"do":"reload"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Headphones","count":1,"do":"expect"},{"do":"restartBackend"},{"do":"freshClient"},{"do":"signIn","name":"pat"},{"do":"click","unlessVisible":"cart-total"},{"contains":"Keyboard","count":1,"do":"expect"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"order-item"},{"contains":"Headphones","count":1,"do":"expect"}],"source":"scenarios/progression-account-state-reload.json","stableKey":"ecommerce.spec.state-durability.account-state-recovery.105a","statedBy":"the signed-in account, cart, and orders persist across a page reload","withheld":null},{"category":"production","checkGroupId":"automatic-reorder-access","criterionId":"502c","description":"a customer cannot see or replay automatic reorder management","featureId":502,"featureName":"Warehouse staff manage automatic reorder rules","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.automatic-reorder"],"role":"guarantee","semantics":[{"absent":true,"do":"expect"},{"do":"fill","text":"Desk Lamp"},{"do":"fill","text":"1"},{"do":"fill","text":"9"},{"action":"saveReorderRule","do":"callAction","from":"staff","input":{"attribute":"data-action-input","testid":"reorder-submit"},"namedAction":{"args":[0,1,9],"id":"saveReorderRule","method":"PUT","params":[{"in":"path","name":"itemId","placeholder":"{itemId}","wireType":"u64"},{"in":"body","name":"threshold"},{"in":"body","name":"quantity"}],"path":"/api/reorders/{itemId}","reducer":"save_reorder_rule"}},{"do":"expectActionOutcome","outcome":"refused","routeProvenBy":"staff"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"reorder-submit"},{"attribute":"data-threshold","contains":"Desk Lamp","count":1,"do":"expect","value":"2"},{"attribute":"data-quantity","contains":"Desk Lamp","do":"expect","value":"5"}],"source":"scenarios/progression-automatic-reorder-access.json","stableKey":"ecommerce.spec.access-control.automatic-reorder-access.502c","statedBy":"Only warehouse staff can manage automatic reorder rules.","withheld":null},{"category":"production","checkGroupId":"automatic-reorder-deduplication","criterionId":"502b","description":"more sales do not duplicate a pending restock","featureId":502,"featureName":"Warehouse staff manage automatic reorder rules","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.automatic-reorder"],"role":"guarantee","semantics":[{"action":"buy","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"do":"expectActionOutcome","outcome":"accepted"},{"action":"buy","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"dbExpectStock","equals":0,"item":"Desk Lamp"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"admin-revenue"},{"do":"click","ifAvailable":true},{"do":"expect"},{"do":"expectElementCount","equals":1},{"attribute":"data-quantity","contains":"Desk Lamp","count":1,"do":"expect","value":"5"}],"source":"scenarios/progression-automatic-reorder-duplicate.json","stableKey":"ecommerce.spec.transactional-integrity.automatic-reorder-deduplication.502b","statedBy":"A pending automatic restock is not scheduled twice.","withheld":null},{"category":"feature","checkGroupId":"automatic-reorder","criterionId":"502a","description":"crossing the threshold creates one pending restock","featureId":502,"featureName":"Warehouse staff manage automatic reorder rules","note":null,"packId":"ecommerce.progression.automatic-reorder","points":3,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"feature","semantics":[{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"admin-revenue"},{"do":"click","ifAvailable":true},{"do":"expect"},{"do":"expectElementCount","equals":0},{"action":"buy","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"dbExpectStock","equals":2,"item":"Desk Lamp"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"admin-revenue"},{"do":"click","ifAvailable":true},{"do":"expect"},{"do":"expectElementCount","equals":1},{"attribute":"data-quantity","contains":"Desk Lamp","count":1,"do":"expect","value":"5"}],"source":"scenarios/progression-automatic-reorder.json","stableKey":"ecommerce.progression.automatic-reorder.automatic-reorder.502a","statedBy":"Crossing a reorder threshold schedules a restock.","withheld":null},{"category":"production","checkGroupId":"books-balance","criterionId":"107a","description":"revenue rises by exactly what was bought","featureId":107,"featureName":"The books balance","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"admin-revenue"},{"do":"expectNumber","plus":58,"relativeTo":"revenue-before"}],"source":"scenarios/progression-books-balance.json","stableKey":"ecommerce.spec.transactional-integrity.books-balance.107a","statedBy":"total revenue equals the sum of order totals","withheld":null},{"category":"production","checkGroupId":"books-balance","criterionId":"107b","description":"what the store sold is what left the warehouses, and a fresh client agrees","featureId":107,"featureName":"The books balance","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"expectNumber","plus":-2,"relativeTo":"stand-before"},{"do":"freshClient"},{"do":"expectNumber","plus":-2,"relativeTo":"stand-before"}],"source":"scenarios/progression-books-balance.json","stableKey":"ecommerce.spec.transactional-integrity.books-balance.107b","statedBy":"warehouse totals and a fresh storefront agree with completed sales","withheld":null},{"category":"feature","checkGroupId":"bundle-checkout","criterionId":"741a","description":"adding a bundle reserves its components and checkout records the bundle price once","featureId":741,"featureName":"Bundle checkout","note":null,"packId":"ecommerce.feature.bundle-checkout","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"do":"dbExpectStock","equals":4,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":3,"item":"Desk Lamp","warehouse":"East"},{"do":"click"},{"do":"click"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Checkout bundle","count":1,"do":"expect"},{"do":"expectNumber","equals":75},{"do":"dbExpectStock","equals":4,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":3,"item":"Desk Lamp","warehouse":"East"},{"do":"dbExpectStock","equals":0,"item":"Keyboard","warehouse":"West"},{"do":"dbExpectStock","equals":0,"item":"Desk Lamp","warehouse":"West"}],"source":"scenarios/progression-bundle-checkout.json","stableKey":"ecommerce.feature.bundle-checkout.bundle-checkout.741a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"bundle-744","criterionId":"744a","description":"two competing reservations accept exactly one whole bundle without consuming extra components","featureId":744,"featureName":"Competing bundle reservations","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","points":3,"provenBy":null,"requiresFeatures":["ecommerce.feature.bundle-checkout"],"role":"guarantee","semantics":[{"action":"addBundleToCart","do":"callConcurrently","input":{"attribute":"data-bundle-input","contains":"Scarce bundle","testid":"bundle-card"},"namedAction":{"args":[0],"id":"addBundleToCart","params":[{"in":"body","name":"bundleId","wireType":"u64"}],"path":"/api/cart/bundles","reducer":"add_bundle_to_cart"}},{"accepted":1,"do":"expectCallOutcomes"},{"do":"dbExpectStock","equals":0,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":0,"item":"Desk Lamp","warehouse":"East"},{"do":"reload"},{"do":"reload"},{"do":"click"},{"do":"click"},{"contains":"Scarce bundle","do":"expectActorsWith","equals":1,"maxEach":1},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"dbExpectStock","equals":2,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":1,"item":"Desk Lamp","warehouse":"East"}],"source":"scenarios/progression-bundle-checkout.json","stableKey":"ecommerce.spec.bundle-integrity.bundle-744.744a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"bundle-745","criterionId":"745a","description":"a missing component refuses the reservation without taking available stock or adding a cart line","featureId":745,"featureName":"Incomplete bundle stock","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","points":3,"provenBy":null,"requiresFeatures":["ecommerce.feature.bundle-checkout"],"role":"guarantee","semantics":[{"action":"addBundleToCart","do":"callAction","input":{"attribute":"data-bundle-input","contains":"Unavailable bundle","testid":"bundle-card"},"namedAction":{"args":[0],"id":"addBundleToCart","params":[{"in":"body","name":"bundleId","wireType":"u64"}],"path":"/api/cart/bundles","reducer":"add_bundle_to_cart"}},{"do":"expectActionOutcome","outcome":"application-refused"},{"do":"dbExpectStock","equals":2,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":0,"item":"Desk Lamp","warehouse":"East"},{"do":"click"},{"absent":true,"contains":"Unavailable bundle","do":"expect"}],"source":"scenarios/progression-bundle-checkout.json","stableKey":"ecommerce.spec.bundle-integrity.bundle-745.745a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"bundle-746","criterionId":"746a","description":"an expired reservation releases each component once across a backend restart","featureId":746,"featureName":"Bundle reservation restart","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","points":3,"provenBy":null,"requiresFeatures":["ecommerce.feature.bundle-checkout"],"role":"guarantee","semantics":[{"do":"click"},{"do":"dbExpectStock","equals":4,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":3,"item":"Desk Lamp","warehouse":"East"},{"do":"restartBackend"},{"do":"wait"},{"do":"reload"},{"do":"ensureSignedIn","name":"bundle-expiry"},{"do":"click"},{"do":"expect"},{"do":"dbExpectStock","equals":6,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":4,"item":"Desk Lamp","warehouse":"East"},{"do":"restartBackend"},{"do":"dbExpectStock","equals":6,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":4,"item":"Desk Lamp","warehouse":"East"}],"source":"scenarios/progression-bundle-checkout.json","stableKey":"ecommerce.spec.bundle-integrity.bundle-746.746a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"bundle-747","criterionId":"747a","description":"two checkout requests consume one reservation and create one paid bundle","featureId":747,"featureName":"Repeated bundle checkout","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","points":3,"provenBy":null,"requiresFeatures":["ecommerce.feature.bundle-checkout"],"role":"guarantee","semantics":[{"action":"checkout","do":"callConcurrently"},{"accepted":1,"do":"expectCallOutcomes"},{"do":"reload"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Repeated bundle","count":1,"do":"expect"},{"do":"expectNumber","equals":75},{"do":"dbExpectStock","equals":4,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":3,"item":"Desk Lamp","warehouse":"East"}],"source":"scenarios/progression-bundle-checkout.json","stableKey":"ecommerce.spec.bundle-integrity.bundle-747.747a","statedBy":null,"withheld":null},{"category":"feature","checkGroupId":"bundle-returns","criterionId":"742a","description":"returning a shipped bundle refunds the paid price and restores original components after its definition changes","featureId":742,"featureName":"Historical bundle return","note":null,"packId":"ecommerce.feature.bundle-returns","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"fill","text":"Historical bundle"},{"do":"fill","text":"9.00"},{"do":"fill","text":"[{\"item\":\"Keyboard\",\"quantity\":1},{\"item\":\"Desk Lamp\",\"quantity\":3}]"},{"do":"click"},{"contains":"Historical bundle","do":"expect"},{"do":"click"},{"do":"expect","ignoreCase":true,"value":"returned"},{"do":"expectNumber","equals":75},{"do":"dbExpectStock","equals":6,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":4,"item":"Desk Lamp","warehouse":"East"},{"do":"dbExpectStock","equals":0,"item":"Keyboard","warehouse":"West"},{"do":"dbExpectStock","equals":0,"item":"Desk Lamp","warehouse":"West"}],"source":"scenarios/progression-bundle-returns.json","stableKey":"ecommerce.feature.bundle-returns.bundle-returns.742a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"bundle-742","criterionId":"742b","description":"replaying a completed bundle return after restart does not refund or restock it twice","featureId":742,"featureName":"Historical bundle return","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","points":3,"provenBy":null,"requiresFeatures":["ecommerce.feature.bundle-returns"],"role":"guarantee","semantics":[{"do":"restartBackend"},{"do":"reload"},{"do":"ensureSignedIn","name":"Historical bundle"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"action":"returnBundle","do":"callAction","input":{"attribute":"data-bundle-return-input","contains":"Historical bundle","testid":"order-item"},"namedAction":{"args":[0],"id":"returnBundle","params":[{"in":"path","name":"orderId","placeholder":":orderId","wireType":"u64"}],"path":"/api/bundle-orders/:orderId/return","reducer":"return_bundle"}},{"do":"reload"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"do":"expectNumber","equals":75},{"do":"dbExpectStock","equals":6,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":4,"item":"Desk Lamp","warehouse":"East"}],"source":"scenarios/progression-bundle-returns.json","stableKey":"ecommerce.spec.bundle-integrity.bundle-742.742b","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"bundle-748","criterionId":"748a","description":"another customer cannot return a paid bundle by submitting its order ID","featureId":748,"featureName":"Bundle return ownership","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","points":3,"provenBy":null,"requiresFeatures":["ecommerce.feature.bundle-returns"],"role":"guarantee","semantics":[{"action":"returnBundle","do":"callAction","from":"buyer","input":{"attribute":"data-bundle-return-input","contains":"Private bundle","testid":"order-item"},"namedAction":{"args":[0],"id":"returnBundle","params":[{"in":"path","name":"orderId","placeholder":":orderId","wireType":"u64"}],"path":"/api/bundle-orders/:orderId/return","reducer":"return_bundle"}},{"do":"expectActionOutcome","outcome":"application-refused"},{"do":"reload"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"do":"expect","ignoreCase":true,"value":"shipped"},{"do":"dbExpectStock","equals":4,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":3,"item":"Desk Lamp","warehouse":"East"},{"do":"click"},{"do":"expectNumber","equals":75},{"do":"dbExpectStock","equals":6,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":4,"item":"Desk Lamp","warehouse":"East"}],"source":"scenarios/progression-bundle-returns.json","stableKey":"ecommerce.spec.bundle-integrity.bundle-748.748a","statedBy":null,"withheld":null},{"category":"feature","checkGroupId":"cart","criterionId":"4a","description":"adding the same item twice raises its quantity instead of adding a second line","featureId":4,"featureName":"Account cart and checkout","note":null,"packId":"ecommerce.feature.cart","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"do":"wait"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"click"},{"do":"click","unlessVisible":"cart-total"},{"contains":"Headphones","count":1,"do":"expect"},{"do":"expectNumber","equals":2}],"source":"scenarios/progression-cart-checkout.json","stableKey":"ecommerce.feature.cart-checkout.cart.4a","statedBy":"Adding an existing cart item increases its quantity.","withheld":null},{"category":"feature","checkGroupId":"cart","criterionId":"4d","description":"checkout creates one order, reduces stock, and empties the cart","featureId":4,"featureName":"Account cart and checkout","note":null,"packId":"ecommerce.feature.checkout","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"expectNumber","equals":100},{"do":"click"},{"do":"click","unlessVisible":"cart-total"},{"do":"expectNumber","equals":1},{"do":"click"},{"do":"wait"},{"do":"reload"},{"do":"ensureSignedIn","name":"cart-checkout"},{"do":"click","unlessVisible":"cart-total"},{"do":"expectNumber","equals":0},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"expectNumber","equals":99},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Desk Lamp","count":1,"do":"expect"}],"source":"scenarios/progression-cart-checkout.json","stableKey":"ecommerce.feature.cart-checkout.cart.4d","statedBy":"Checkout creates one order, reduces stock, and empties the cart.","withheld":null},{"category":"feature","checkGroupId":"cart-recovery","criterionId":"503a","description":"restoring an expired cart reserves available items again","featureId":503,"featureName":"Expired carts restore only available items","note":null,"packId":"ecommerce.progression.cart-recovery","points":3,"provenBy":null,"role":"feature","semantics":[{"do":"reload"},{"do":"ensureSignedIn","name":"cart-recovery-available"},{"do":"expect"},{"do":"expectNumber","equals":0},{"as":"restore-available-shopper","do":"recordNumber"},{"do":"click"},{"do":"click"},{"contains":"Keyboard","do":"expect"},{"absent":true,"do":"expect"},{"do":"expectNumber","equals":1},{"do":"reload"},{"do":"ensureSignedIn","name":"cart-recovery-available"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"item-card"},{"do":"expectNumber","plus":-1,"relativeTo":"restore-available-shopper"}],"source":"scenarios/progression-cart-recovery.json","stableKey":"ecommerce.progression.cart-recovery.cart-recovery.503a","statedBy":"A customer can restore the available items from an expired cart.","withheld":null},{"category":"feature","checkGroupId":"cart-recovery","criterionId":"503b","description":"a partial restore keeps available items and names each unavailable item","featureId":503,"featureName":"Expired carts restore only available items","note":null,"packId":"ecommerce.progression.cart-recovery","points":3,"provenBy":null,"role":"feature","semantics":[{"do":"reload"},{"do":"ensureSignedIn","name":"cart-recovery-partial"},{"do":"expect"},{"do":"expectNumber","equals":0},{"as":"restore-partial-shopper","do":"recordNumber"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"warehouse":"West"},{"do":"click"},{"do":"click"},{"contains":"Gaming Mouse","do":"expect"},{"absent":true,"contains":"Desk Lamp","do":"expect"},{"contains":"Desk Lamp","do":"expect"},{"do":"expectNumber","equals":1},{"do":"reload"},{"do":"ensureSignedIn","name":"cart-recovery-partial"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"item-card"},{"do":"expectNumber","plus":-1,"relativeTo":"restore-partial-shopper"}],"source":"scenarios/progression-cart-recovery.json","stableKey":"ecommerce.progression.cart-recovery.cart-recovery.503b","statedBy":"Restore the available items and list each item that could not be restored.","withheld":null},{"category":"feature","checkGroupId":"catalog-management","criterionId":"622a","description":"a new product reaches the public catalog","featureId":622,"featureName":"Catalog management","note":null,"packId":"ecommerce.progression.catalog-management","points":2,"provenBy":null,"role":"feature","semantics":[{"contains":"Travel Mug","do":"expect"}],"source":"scenarios/progression-catalog-management.json","stableKey":"ecommerce.progression.catalog-management.catalog-management.622a","statedBy":"Authorized staff can add products.","withheld":null},{"category":"feature","checkGroupId":"catalog-management","criterionId":"622b","description":"the product exposes its named variants","featureId":622,"featureName":"Catalog management","note":null,"packId":"ecommerce.progression.catalog-management","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"openItem","item":"Travel Mug","unlessVisible":"item-variant"},{"contains":"Black","do":"expectElementCount","equals":1},{"contains":"Silver","do":"expectElementCount","equals":1}],"source":"scenarios/progression-catalog-management.json","stableKey":"ecommerce.progression.catalog-management.catalog-management.622b","statedBy":"Products can have named variants.","withheld":null},{"category":"production","checkGroupId":"checkout-crash-integrity","criterionId":"910a","description":"interrupted checkout recovers to the prepared cart or one complete order with the cart cleared after each independent process crash","featureId":910,"featureName":"Checkout crash recovery","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.checkout"],"role":"guarantee","semantics":[{"do":"expectCrashCheckout","from":"database-crash-observation","verdict":"atomicity"},{"do":"expectCrashCheckout","from":"application-crash-observation","verdict":"atomicity"}],"source":"scenarios/progression-checkout-crash.json","stableKey":"ecommerce.spec.state-durability.checkout-crash-integrity.910a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"checkout-crash-durability","criterionId":"910b","description":"acknowledged checkout is not rolled back and earlier orders remain unchanged after each independent process crash","featureId":910,"featureName":"Checkout crash recovery","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.checkout"],"role":"guarantee","semantics":[{"do":"expectCrashCheckout","from":"database-crash-observation","verdict":"durability"},{"do":"expectCrashCheckout","from":"application-crash-observation","verdict":"durability"}],"source":"scenarios/progression-checkout-crash.json","stableKey":"ecommerce.spec.state-durability.checkout-crash-durability.910b","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"payment-records","criterionId":"623a","description":"checkout records the exact paid amount","featureId":623,"featureName":"Payment records","note":null,"packId":"ecommerce.progression.payment-records","points":3,"provenBy":null,"role":"feature","semantics":[{"do":"expect","ignoreCase":true,"value":"paid"},{"do":"expectNumber","plus":0,"relativeTo":"payment-total"}],"source":"scenarios/progression-core-business.json","stableKey":"ecommerce.progression.payment-records.payment-records.623a","statedBy":"Checkout records the amount paid on the order.","withheld":null},{"category":"production","checkGroupId":"payment-deduplication","criterionId":"623b","description":"one checkout has one payment record","featureId":623,"featureName":"Payment records","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.payment-records"],"role":"guarantee","semantics":[{"do":"expectElementCount","equals":1}],"source":"scenarios/progression-core-business.json","stableKey":"ecommerce.spec.transactional-integrity.payment-deduplication.623b","statedBy":"A checkout does not create duplicate payments.","withheld":null},{"category":"feature","checkGroupId":"customer-profile","criterionId":"620c","description":"the owner can save and view a customer profile","featureId":620,"featureName":"Customer profile","note":null,"packId":"ecommerce.progression.customer-profile","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"profile-address-summary"},{"contains":"14 Market Street {user:profilemarker}","do":"expect"}],"source":"scenarios/progression-customer-profile.json","stableKey":"ecommerce.progression.customer-profile.customer-profile.620c","statedBy":"A signed-in customer can save and view their name and shipping address.","withheld":null},{"category":"production","checkGroupId":"customer-profile-reload","criterionId":"620a","description":"the saved profile survives reload and backend restart in a fresh browser","featureId":620,"featureName":"Customer profile","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.customer-profile"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"ensureSignedIn","name":"profile-owner"},{"do":"click"},{"contains":"14 Market Street {user:profilemarker}","do":"expect"},{"do":"restartBackend"},{"do":"freshClient"},{"do":"signIn","name":"profile-owner"},{"do":"click"},{"contains":"14 Market Street {user:profilemarker}","do":"expect"}],"source":"scenarios/progression-customer-profile.json","stableKey":"ecommerce.spec.state-durability.customer-profile-reload.620a","statedBy":"The saved profile remains after a reload.","withheld":null},{"category":"production","checkGroupId":"customer-profile-privacy","criterionId":"620b","description":"another customer neither sees nor receives the owner's private address","featureId":620,"featureName":"Customer profile","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.customer-profile"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"signUp","name":"profile-private-owner"},{"do":"click"},{"do":"fill","text":"Avery Stone"},{"do":"fill","text":"14 Market Street {user:profilemarker}"},{"do":"click"},{"do":"reload"},{"do":"ensureSignedIn","name":"profile-private-owner"},{"do":"click"},{"contains":"14 Market Street {user:profilemarker}","do":"expect"},{"contains":"14 Market Street {user:profilemarker}","do":"expectReceived"},{"do":"reload"},{"do":"signUp","name":"profile-other"},{"do":"click"},{"absent":true,"contains":"14 Market Street {user:profilemarker}","do":"expect"},{"contains":"14 Market Street {user:profilemarker}","do":"expectNotReceived"}],"source":"scenarios/progression-customer-profile.json","stableKey":"ecommerce.spec.access-control.customer-profile-privacy.620b","statedBy":"Each customer can see only their own profile.","withheld":null},{"category":"feature","checkGroupId":"delivery-notification-delivery","criterionId":"501a","description":"the order owner receives one delivery notification","featureId":501,"featureName":"Delivery creates one private notification","note":null,"packId":"ecommerce.progression.delivery-notifications","points":3,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"feature","semantics":[{"do":"reload"},{"do":"ensureSignedIn","name":"delivery-owner"},{"do":"click","unlessVisible":"notifications-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"contains":"Desk Lamp","do":"expectElementCount","equals":1}],"source":"scenarios/progression-delivery-notifications.json","stableKey":"ecommerce.progression.delivery-notifications.delivery-notification-delivery.501a","statedBy":"A delivered order creates one notification for its owner.","withheld":null},{"category":"production","checkGroupId":"delivery-notification-privacy","criterionId":"501b","description":"another customer cannot see the delivery notification","featureId":501,"featureName":"Delivery creates one private notification","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.delivery-notifications","ecommerce.progression.fulfilment-queue"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"ensureSignedIn","name":"delivery-owner"},{"do":"click","unlessVisible":"notifications-panel"},{"contains":"Desk Lamp","do":"expectElementCount","equals":1},{"do":"click","unlessVisible":"notifications-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"absent":true,"contains":"Desk Lamp","do":"expect"}],"source":"scenarios/progression-delivery-notifications.json","stableKey":"ecommerce.spec.access-control.delivery-notification-privacy.501b","statedBy":"Delivery notifications are private to the order owner.","withheld":null},{"category":"feature","checkGroupId":"faceted-search","criterionId":"401a","description":"category, price, and availability filters apply together","featureId":401,"featureName":"Filters compose","note":null,"packId":"ecommerce.progression.faceted-search","points":3,"provenBy":null,"role":"feature","semantics":[{"contains":"Coffee Grinder","do":"expectElementCount","equals":1},{"do":"click"},{"do":"click","ifAvailable":true},{"do":"waitUntilAbsent"},{"contains":"Air Purifier","do":"expectElementCount","equals":1},{"absent":true,"contains":"USB Cable","do":"expect"},{"absent":true,"contains":"Desk Lamp","do":"expect"},{"absent":true,"contains":"Espresso Machine","do":"expect"},{"absent":true,"contains":"Gaming Mouse","do":"expect"}],"source":"scenarios/progression-faceted-filters.json","stableKey":"ecommerce.progression.faceted-search.faceted-search.401a","statedBy":"Apply all selected filters together","withheld":null},{"category":"feature","checkGroupId":"faceted-search","criterionId":"402a","description":"moving between pages returns the same ordered items without duplicates","featureId":402,"featureName":"Pages are stable","note":null,"packId":"ecommerce.progression.faceted-search","points":3,"provenBy":null,"role":"feature","semantics":[{"do":"expectSequence","equals":["Air Purifier","Bluetooth Speaker","Coffee Grinder","Desk Lamp","Espresso Machine","Gaming Mouse","Headphones","Induction Cooktop","Keyboard","Laptop Stand"]},{"do":"click"},{"do":"expectSequence","equals":["Mirrorless Camera","USB Cable","Webcam"]},{"do":"click"},{"do":"expectSequence","equals":["Air Purifier","Bluetooth Speaker","Coffee Grinder","Desk Lamp","Espresso Machine","Gaming Mouse","Headphones","Induction Cooktop","Keyboard","Laptop Stand"]}],"source":"scenarios/progression-faceted-pagination.json","stableKey":"ecommerce.progression.faceted-search.faceted-search.402a","statedBy":"Moving between pages must not omit or repeat an item","withheld":null},{"category":"production","checkGroupId":"managed-support-privacy","criterionId":"613b","description":"another customer cannot read or reply to the managed case","featureId":613,"featureName":"Managed support privacy","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"provenBy":null,"requiresFeatures":["ecommerce.progression.managed-support"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"ensureSignedIn","name":"managed-private-owner"},{"do":"click"},{"contains":"Private managed case {user:casemarker}","do":"expect"},{"contains":"Private managed case {user:casemarker}","do":"expectReceived"},{"absent":true,"contains":"Private managed case {user:casemarker}","do":"expect"},{"contains":"Private managed case {user:casemarker}","do":"expectNotReceived"},{"do":"fill","text":"Owner-only update"},{"do":"click"},{"do":"replayAs","from":"owner","match":"Owner-only update","namedAction":{"args":[0,"Owner-only update"],"id":"replySupport","method":"POST","params":[{"in":"path","name":"ticketId","placeholder":":id","wireType":"u64"},{"in":"body","name":"body"}],"path":"/api/support/:id/replies","reducer":"reply_support"},"namedTarget":{"attribute":"data-entity-id","contains":"Private managed case {user:casemarker}","testid":"support-ticket","valueType":"string"}},{"allowNotFound":true,"do":"expectReplayRejected"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click","ifAvailable":true,"unlessVisible":"support-assignee"},{"do":"click","ifAvailable":true,"unlessVisible":"support-assignee"},{"contains":"Private managed case {user:casemarker}","do":"expect"},{"contains":"Owner-only update","do":"expectElementCount","equals":1},{"contains":"Owner-only update","do":"expectReceived"},{"contains":"Owner-only update","do":"expectNotReceived"}],"source":"scenarios/progression-managed-support-privacy.json","stableKey":"ecommerce.spec.access-control.managed-support-privacy.613b","statedBy":"A customer can access only cases that belong to their account.","withheld":null},{"category":"feature","checkGroupId":"managed-support","criterionId":"613c","description":"staff can update a support case and the customer can reply","featureId":613,"featureName":"Shared managed support case","note":null,"packId":"ecommerce.progression.managed-support","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"fill","text":"in progress"},{"do":"click"},{"do":"fill","text":"Case received."},{"do":"click"},{"do":"reload"},{"do":"ensureSignedIn","name":"managed-shared-owner"},{"do":"click"},{"contains":"in progress","do":"expect"},{"contains":"Case received.","do":"expect"},{"do":"fill","text":"Thank you."},{"do":"click"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click","unlessVisible":"support-assignee"},{"do":"click","ifAvailable":true,"unlessVisible":"support-assignee"},{"contains":"Thank you.","do":"expect"}],"source":"scenarios/progression-managed-support-shared.json","stableKey":"ecommerce.progression.managed-support.managed-support.613c","statedBy":"Customers and staff can exchange replies and update the status of a support case.","withheld":null},{"category":"production","checkGroupId":"managed-support","criterionId":"613a","description":"the customer and staff see the same replies and status live","featureId":613,"featureName":"Shared managed support case","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.managed-support"],"role":"guarantee","semantics":[{"do":"fill","text":"open"},{"do":"click"},{"contains":"open","do":"expect"},{"do":"reload"},{"do":"ensureSignedIn","name":"managed-shared-owner"},{"do":"click","unlessVisible":"support-ticket"},{"contains":"open","do":"expect"},{"do":"fill","text":"in progress"},{"do":"click"},{"do":"fill","text":"We are investigating."},{"do":"click"},{"contains":"in progress","do":"expect"},{"contains":"We are investigating.","do":"expect"},{"do":"fill","text":"Thank you for the update."},{"do":"click"},{"contains":"Thank you for the update.","do":"expect"}],"source":"scenarios/progression-managed-support-shared.json","stableKey":"ecommerce.spec.live-state.managed-support.613a","statedBy":"Customers and authorized staff use one shared support case.","withheld":null},{"category":"feature","checkGroupId":"notification-preferences","criterionId":"630c","description":"the customer can save a notification choice","featureId":630,"featureName":"Account notification preferences","note":null,"packId":"ecommerce.progression.notification-preferences","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"notification-order"},{"attribute":"data-state","do":"expect","value":"on"}],"source":"scenarios/progression-notification-preferences.json","stableKey":"ecommerce.progression.notification-preferences.notification-preferences.630c","statedBy":"Signed-in customers can turn order and stock notifications on or off.","withheld":null},{"category":"production","checkGroupId":"notification-preferences-reload","criterionId":"630a","description":"notification choices survive reload and backend restart in a fresh browser","featureId":630,"featureName":"Account notification preferences","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.notification-preferences"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"ensureSignedIn","name":"notification-owner"},{"do":"click","unlessVisible":"notification-order"},{"attribute":"data-state","do":"expect","value":"on"},{"do":"restartBackend"},{"do":"freshClient"},{"do":"signIn","name":"notification-owner"},{"do":"click","unlessVisible":"notification-order"},{"attribute":"data-state","do":"expect","value":"on"}],"source":"scenarios/progression-notification-preferences.json","stableKey":"ecommerce.spec.state-durability.notification-preferences-reload.630a","statedBy":"Notification choices persist for the account.","withheld":null},{"category":"production","checkGroupId":"notification-preferences-privacy","criterionId":"630b","description":"the owner's choice does not change another account","featureId":630,"featureName":"Account notification preferences","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.notification-preferences"],"role":"guarantee","semantics":[{"do":"click","unlessVisible":"notification-order"},{"attribute":"data-state","do":"expect","value":"off"}],"source":"scenarios/progression-notification-preferences.json","stableKey":"ecommerce.spec.access-control.notification-preferences-privacy.630b","statedBy":"One customer's notification choices do not affect another customer.","withheld":null},{"category":"production","checkGroupId":"open-list","criterionId":"902a","description":"one customer has the Keyboard's reviews open before another posts one; the already-open view shows that review exactly once","featureId":902,"featureName":"An open list stays current","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","semantics":[{"do":"openItem","item":"Keyboard"},{"do":"expect"},{"do":"click","ifAvailable":true,"unlessVisible":"review-rating"},{"do":"fill","text":"live-review-kbd"},{"do":"click"},{"contains":"live-review-kbd","do":"expectElementCount","equals":1},{"contains":"live-review-kbd","do":"expectElementCount","equals":1}],"source":"scenarios/progression-open-list-live.json","stableKey":"ecommerce.spec.live-state.open-list.902a","statedBy":"A view opened while a review is submitted converges to the current review list.","withheld":null},{"category":"interface","checkGroupId":"cancellation-and-return","criterionId":"3e","description":"a pending order does not offer a return button","featureId":331,"featureName":"Pending order return boundary","note":null,"packId":"ecommerce.l3.order-returns-features","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"feature","semantics":[{"absent":true,"do":"expect"}],"source":"scenarios/progression-order-return-boundary.json","stableKey":"ecommerce.returns-pricing.cancellation-and-return.3e","statedBy":"An item from a pending order cannot be returned.","withheld":null},{"category":"production","checkGroupId":"cancellation-and-return","criterionId":"3f","description":"the server refuses a pending return without changing stock or revenue","featureId":332,"featureName":"Pending return server boundary","note":null,"packId":"ecommerce.l3.order-returns-features","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"feature","semantics":[{"action":"returnItem","do":"callAction","input":{"attribute":"data-return-input","contains":"Desk Lamp","testid":"order-line"},"namedAction":{"args":[0,0],"id":"returnItem","method":"POST","params":[{"in":"path","name":"orderId","placeholder":"{orderId}","wireType":"u64"},{"in":"path","name":"itemId","placeholder":"{itemId}","wireType":"u64"}],"path":"/api/orders/{orderId}/items/{itemId}/return","reducer":"return_order_item"}},{"do":"expectActionOutcome","outcome":"validation-refused"},{"do":"reload"},{"do":"ensureSignedIn","name":"return-boundary"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"pending"},{"do":"dbExpectStock","item":"Desk Lamp","plus":0,"relativeTo":"pending-East","warehouse":"East"},{"do":"dbExpectStock","item":"Desk Lamp","plus":0,"relativeTo":"pending-West","warehouse":"West"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"admin-revenue"},{"do":"expectNumber","plus":0,"relativeTo":"pending-revenue"}],"source":"scenarios/progression-order-return-boundary.json","stableKey":"ecommerce.returns-pricing.cancellation-and-return.3f","statedBy":"Only shipped items can be returned.","withheld":null},{"category":"production","checkGroupId":"cancellation-and-return","criterionId":"3c","description":"returning a shipped item restores stock and revenue and marks the item returned","featureId":330,"featureName":"Completed order return","note":null,"packId":"ecommerce.l3.order-returns-features","points":3,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"feature","semantics":[{"do":"click"},{"contains":"Keyboard","containsText":"returned","do":"expect","ignoreCase":true},{"do":"freshClient"},{"do":"signIn","name":"return-complete"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Keyboard","containsText":"returned","do":"expect","ignoreCase":true},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","unlessVisible":"admin-revenue"},{"do":"expectNumber","plus":0,"relativeTo":"return-revenue-before"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"dbExpectStock","item":"Keyboard","plus":0,"relativeTo":"return-stock-before"}],"source":"scenarios/progression-order-return-complete.json","stableKey":"ecommerce.returns-pricing.cancellation-and-return.3c","statedBy":"An accepted return marks the item as returned, restores stock, and reduces revenue by the price paid.","withheld":null},{"category":"production","checkGroupId":"order-support-ownership","criterionId":"614b","description":"another customer cannot attach or inspect the owner's order","featureId":614,"featureName":"Order support ownership boundary","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.order-support"],"role":"guarantee","semantics":[{"absent":true,"contains":"Desk Lamp","do":"expect"},{"action":"linkSupportOrder","authentication":"actor","do":"callAction","input":{"attribute":"data-action-input","testid":"support-link-order"},"namedAction":{"args":[0,0],"id":"linkSupportOrder","method":"POST","params":[{"in":"path","name":"caseId","placeholder":"{caseId}","wireType":"u64"},{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/support/cases/{caseId}/order","reducer":"link_support_order"}},{"do":"expectActionOutcome","outcome":"accepted"},{"contains":"Desk Lamp","do":"expectElementCount","equals":1},{"action":"linkSupportOrder","authentication":"actor","do":"callAction","from":"owner","input":{"attribute":"data-action-input","overrides":{"caseId":{"actor":"other","attribute":"data-entity-id","contains":"Other order case","testid":"support-ticket"}},"testid":"support-link-order"},"namedAction":{"args":[0,0],"id":"linkSupportOrder","method":"POST","params":[{"in":"path","name":"caseId","placeholder":"{caseId}","wireType":"u64"},{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/support/cases/{caseId}/order","reducer":"link_support_order"}},{"do":"expectActionOutcome","outcome":"refused"},{"do":"freshClient"},{"do":"signIn","name":"order-boundary-other"},{"do":"click"},{"absent":true,"contains":"Desk Lamp","do":"expect"},{"contains":"Desk Lamp","do":"expectElementCount","equals":1}],"source":"scenarios/progression-order-support-boundary.json","stableKey":"ecommerce.spec.access-control.order-support-ownership.614b","statedBy":"A customer cannot attach or inspect another customer's order.","withheld":null},{"category":"feature","checkGroupId":"order-support-owned","criterionId":"614a","description":"the customer can link their order and staff can inspect it","featureId":614,"featureName":"Owned order support link","note":null,"packId":"ecommerce.progression.order-support","points":3,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"do":"click"},{"do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click","unlessVisible":"support-assignee"},{"do":"click","ifAvailable":true,"unlessVisible":"support-assignee"},{"contains":"Desk Lamp","do":"expect"}],"source":"scenarios/progression-order-support-owned.json","stableKey":"ecommerce.progression.order-support.order-support-owned.614a","statedBy":"A customer can attach one of their orders to a support case.","withheld":null},{"category":"feature","checkGroupId":"personalized-recommendations","criterionId":"403a","description":"recommendations follow the customer's categories, global sales, and name tie-break","featureId":403,"featureName":"Recommendations use customer activity","note":null,"packId":"ecommerce.progression.personalized-recommendations","points":4,"provenBy":null,"role":"feature","semantics":[{"contains":"Headphones","do":"expect"},{"do":"expectNumber","equals":1},{"do":"expectSequence","equals":["Gaming Mouse","Laptop Stand","Webcam"]}],"source":"scenarios/progression-personalized-recommendations.json","stableKey":"ecommerce.progression.personalized-recommendations.personalized-recommendations.403a","statedBy":"Order the remaining items by global units sold, highest first, then by item name.","withheld":null},{"category":"production","checkGroupId":"recommendation-profile-isolation","criterionId":"403b","description":"one customer's activity does not replace another customer's recommendations","featureId":403,"featureName":"Recommendations use customer activity","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.personalized-recommendations"],"role":"guarantee","semantics":[{"do":"click"},{"do":"waitUntilAbsent"},{"do":"expectNumber","equals":1},{"do":"expectSequence","equals":["Gaming Mouse","Laptop Stand","Webcam"]}],"source":"scenarios/progression-personalized-recommendations.json","stableKey":"ecommerce.spec.access-control.recommendation-profile-isolation.403b","statedBy":"Customer recommendation profiles are isolated.","withheld":null},{"category":"production","checkGroupId":"price-history","criterionId":"4c","description":"a price change updates an open cart and direct checkout persists the new total","featureId":420,"featureName":"Open cart price changes","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.price-history-specifications","points":3,"provenBy":null,"requiresFeatures":["ecommerce.feature.checkout","ecommerce.l2.price-history-features"],"role":"guarantee","semantics":[{"do":"fill","enter":true,"text":"Desk Lamp"},{"do":"click"},{"do":"click"},{"do":"expectNumber","equals":42},{"do":"fill","text":"52.00"},{"do":"click"},{"do":"expectNumber","equals":52},{"action":"checkout","do":"callAction","namedAction":{"args":[],"id":"checkout","method":"POST","path":"/api/checkout","reducer":"checkout"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"freshClient"},{"do":"signIn","name":"price-cart"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Desk Lamp","count":1,"do":"expect"},{"do":"expectNumber","equals":52}],"source":"scenarios/progression-price-cart-checkout.json","stableKey":"ecommerce.returns-pricing.price-history.4c","statedBy":"The public catalog and every open cart show the new price without a reload. Checkout uses the current price.","withheld":null},{"category":"feature","checkGroupId":"product-bundles","criterionId":"740a","description":"a saved bundle shows the exact price and component quantities after reopening the application","featureId":740,"featureName":"Bundle definitions","note":null,"packId":"ecommerce.feature.product-bundles","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"fill","text":"Office bundle"},{"do":"fill","text":"75.00"},{"do":"fill","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"},{"do":"click"},{"contains":"Office bundle","do":"expect"},{"do":"reload"},{"do":"click","unlessVisible":"bundle-card"},{"do":"expectNumber","equals":75},{"do":"expectElementCount","equals":2},{"attribute":"data-quantity","contains":"Keyboard","do":"expect","value":"2"},{"attribute":"data-quantity","contains":"Desk Lamp","do":"expect","value":"1"}],"source":"scenarios/progression-product-bundles.json","stableKey":"ecommerce.feature.product-bundles.product-bundles.740a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"bundle-743","criterionId":"743a","description":"a customer cannot replace a staff-created bundle through the application write","featureId":743,"featureName":"Bundle management authorization","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","points":3,"provenBy":null,"requiresFeatures":["ecommerce.feature.product-bundles"],"role":"guarantee","semantics":[{"do":"fill","text":"1.00"},{"action":"saveBundle","do":"callAction","from":"admin","input":{"attribute":"data-bundle-save-input","testid":"bundle-save"},"namedAction":{"args":["Protected bundle",1,"[{\"item\":\"Keyboard\",\"quantity\":1}]"],"id":"saveBundle","params":[{"in":"body","name":"name"},{"in":"body","name":"price"},{"in":"body","name":"componentsJson"}],"path":"/api/bundles","reducer":"save_bundle"}},{"do":"expectActionOutcome","outcome":"application-refused"},{"do":"reload"},{"do":"click","unlessVisible":"bundle-card"},{"do":"expectNumber","equals":75},{"attribute":"data-quantity","contains":"Keyboard","do":"expect","value":"2"}],"source":"scenarios/progression-product-bundles.json","stableKey":"ecommerce.spec.bundle-integrity.bundle-743.743a","statedBy":null,"withheld":null},{"category":"feature","checkGroupId":"promotion-checkout-active","criterionId":"621a","description":"an active promotion changes checkout and is recorded on the order","featureId":621,"featureName":"Bounded promotions at checkout","note":null,"packId":"ecommerce.progression.promotion-checkout","points":3,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"do":"click"},{"do":"fill","text":"LIVE10"},{"do":"click"},{"do":"click"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"do":"expectNumber","equals":8.9}],"source":"scenarios/progression-promotion-checkout.json","stableKey":"ecommerce.progression.promotion-checkout.promotion-checkout-active.621a","statedBy":"An active promotion applies to the cart and the order records its discount.","withheld":null},{"category":"feature","checkGroupId":"promotion-checkout-expired","criterionId":"621b","description":"an expired promotion is refused","featureId":621,"featureName":"Bounded promotions at checkout","note":null,"packId":"ecommerce.progression.promotion-checkout","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"do":"click"},{"do":"fill","text":"OLD10"},{"do":"click"},{"do":"expect"}],"source":"scenarios/progression-promotion-checkout.json","stableKey":"ecommerce.progression.promotion-checkout.promotion-checkout-expired.621b","statedBy":"Expired promotions cannot be applied.","withheld":null},{"category":"feature","checkGroupId":"promotion-checkout-exhausted","criterionId":"621c","description":"a fully redeemed promotion is refused","featureId":621,"featureName":"Bounded promotions at checkout","note":null,"packId":"ecommerce.progression.promotion-checkout","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"do":"click"},{"do":"fill","text":"ONCE10"},{"do":"click"},{"do":"click"},{"do":"click"},{"do":"click"},{"do":"fill","text":"ONCE10"},{"do":"click"},{"do":"expect"}],"source":"scenarios/progression-promotion-checkout.json","stableKey":"ecommerce.progression.promotion-checkout.promotion-checkout-exhausted.621c","statedBy":"A promotion cannot be used after its redemption limit is reached.","withheld":null},{"category":"feature","checkGroupId":"promotion-report-redemptions","criterionId":"622a","description":"the promotion report has the exact redemption count","featureId":622,"featureName":"Exact promotion totals","note":null,"packId":"ecommerce.progression.promotion-reporting","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"expectNumber","equals":1}],"source":"scenarios/progression-promotion-reporting.json","stableKey":"ecommerce.progression.promotion-reporting.promotion-report-redemptions.622a","statedBy":"Promotion redemption counts match orders that used the promotion.","withheld":null},{"category":"feature","checkGroupId":"promotion-report-revenue","criterionId":"622b","description":"the promotion report has the exact discounted revenue","featureId":622,"featureName":"Exact promotion totals","note":null,"packId":"ecommerce.progression.promotion-reporting","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"expectNumber","equals":80.1}],"source":"scenarios/progression-promotion-reporting.json","stableKey":"ecommerce.progression.promotion-reporting.promotion-report-revenue.622b","statedBy":"Promotion revenue after discounts matches orders that used the promotion.","withheld":null},{"category":"feature","checkGroupId":"promotion-rule-values","criterionId":"620a","description":"staff can save every bounded promotion value","featureId":620,"featureName":"Staff-managed promotion rules","note":null,"packId":"ecommerce.progression.promotion-rules","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"click","unlessVisible":"promotion-code"},{"do":"fill","text":"SAVE10"},{"do":"fill","text":"10"},{"do":"fill","text":"2099-01-01"},{"do":"fill","text":"2099-12-31"},{"do":"fill","text":"2"},{"do":"click"},{"do":"expectNumber","equals":10},{"contains":"2099-01-01","do":"expect"},{"contains":"2099-12-31","do":"expect"},{"do":"expectNumber","equals":2}],"source":"scenarios/progression-promotion-rules.json","stableKey":"ecommerce.progression.promotion-rules.promotion-rule-values.620a","statedBy":"Staff can create a promotion code with a discount, active period, and redemption limit.","withheld":null},{"category":"production","checkGroupId":"promotion-management-boundary","criterionId":"620b","description":"customers cannot open promotion management","featureId":620,"featureName":"Staff-managed promotion rules","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.promotion-rules"],"role":"guarantee","semantics":[{"do":"click","unlessVisible":"promotion-code"},{"do":"fill","text":"ACCESS10"},{"do":"fill","text":"10"},{"do":"fill","text":"2099-01-01"},{"do":"fill","text":"2099-12-31"},{"do":"fill","text":"2"},{"do":"click"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click","ifAvailable":true,"unlessVisible":"promotion-code"},{"do":"click","unlessVisible":"promotion-code"},{"contains":"ACCESS10","do":"expect"},{"absent":true,"do":"expect"},{"do":"replayAs","from":"staff","match":"ACCESS10","namedAction":{"args":["ACCESS10",10,4070908800000000,4102444740000000,2],"id":"createPromotion","params":[{"in":"body","name":"code"},{"in":"body","name":"discountPercent"},{"in":"body","name":"startMicros"},{"in":"body","name":"endMicros"},{"in":"body","name":"usageLimit"}],"path":"/api/promotions","reducer":"create_promotion"},"swap":{"find":"ACCESS10","with":"HACK10"}},{"do":"expectReplayRejected"},{"absent":true,"contains":"HACK10","do":"expect"}],"source":"scenarios/progression-promotion-rules.json","stableKey":"ecommerce.spec.access-control.promotion-management-boundary.620b","statedBy":"Customers cannot manage promotion rules.","withheld":null},{"category":"feature","checkGroupId":"purchase-order","criterionId":"3c","description":"the purchase is recorded in the buyer's order history at the price paid","featureId":3,"featureName":"Purchase order history","note":null,"packId":"ecommerce.feature.purchasing","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Coffee Grinder","do":"expect"},{"do":"expectNumber","equals":64}],"source":"scenarios/progression-purchasing.json","stableKey":"ecommerce.feature.purchasing.purchase-order.3c","statedBy":"A purchase creates an order for the customer at the price paid.","withheld":null},{"category":"feature","checkGroupId":"recommendation-feedback","criterionId":"504a","description":"dismissing a recommendation removes it from the customer view","featureId":504,"featureName":"Recommendation dismissal","note":null,"packId":"ecommerce.progression.recommendation-feedback","points":2,"provenBy":null,"role":"feature","semantics":[{"absent":true,"contains":"Headphones","do":"expect"}],"source":"scenarios/progression-recommendation-feedback.json","stableKey":"ecommerce.progression.recommendation-feedback.recommendation-feedback.504a","statedBy":"Customers can dismiss a recommendation.","withheld":null},{"category":"production","checkGroupId":"recommendation-feedback-privacy","criterionId":"504b","description":"one customer's dismissal does not hide another customer's result","featureId":504,"featureName":"Recommendation dismissal","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.recommendation-feedback"],"role":"guarantee","semantics":[{"do":"waitUntilAbsent"},{"contains":"Headphones","do":"expect"}],"source":"scenarios/progression-recommendation-feedback.json","stableKey":"ecommerce.spec.access-control.recommendation-feedback-privacy.504b","statedBy":"Recommendation feedback belongs to one customer.","withheld":null},{"category":"production","checkGroupId":"recommendation-feedback-restart","criterionId":"504c","description":"a dismissed recommendation stays absent after reload and backend restart in a fresh browser","featureId":504,"featureName":"Recommendation dismissal","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.recommendation-feedback"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"ensureSignedIn","name":"feedback-owner"},{"absent":true,"contains":"Headphones","do":"expect"},{"do":"restartBackend"},{"do":"freshClient"},{"do":"signIn","name":"feedback-owner"},{"do":"expect"},{"absent":true,"contains":"Headphones","do":"expect"}],"source":"scenarios/progression-recommendation-feedback.json","stableKey":"ecommerce.spec.state-durability.recommendation-feedback-restart.504c","statedBy":"A dismissed recommendation stays dismissed for that customer.","withheld":null},{"category":"production","checkGroupId":"review-eligibility-direct","criterionId":"618a","description":"the server refuses a review from a customer who did not buy the item","featureId":618,"featureName":"Review access","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.review-access-specifications","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","semantics":[{"action":"submitReview","do":"callAction","from":"owner","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0,5,"eligible progression review"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"}],"path":"/api/items/:id/reviews","reducer":"submit_review"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"openItem","item":"Keyboard"},{"contains":"eligible progression review","do":"expect"},{"action":"submitReview","do":"callAction","from":"stranger","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0,5,"never bought this"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"}],"path":"/api/items/:id/reviews","reducer":"submit_review"}},{"do":"expectActionOutcome","outcome":"application-refused","routeProvenBy":"owner"},{"do":"freshClient"},{"do":"signIn","name":"review-stranger"},{"do":"openItem","item":"Keyboard"},{"contains":"eligible progression review","do":"expect"},{"absent":true,"contains":"never bought this","do":"expect"}],"source":"scenarios/progression-review-access.json","stableKey":"ecommerce.progression.review-access-specifications.review-eligibility-direct.618a","statedBy":"Only a customer who bought an item can review it.","withheld":null},{"category":"production","checkGroupId":"stored-review-script","criterionId":"9180a","description":"review input is rejected or displayed without executing script in another customer session","featureId":9180,"featureName":"Stored review content","note":null,"observations":["unmentioned"],"packId":"ecommerce.progression.review-access-specifications","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","semantics":[{"action":"submitReview","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0,5,"Benign stored review"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"}],"path":"/api/items/:id/reviews","reducer":"submit_review"}},{"do":"expectActionOutcome","outcome":"accepted"},{"action":"submitReview","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0,5,"Stored review marker <img src=\"data:,bad-image\" onerror=\"window.__stackBenchScriptCanary()\">"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"}],"path":"/api/items/:id/reviews","reducer":"submit_review"}},{"do":"expectActionOutcome","outcome":"completed","routeProvenBy":"owner"},{"do":"openItem","item":"Keyboard"},{"contains":"Benign stored review","do":"expect"},{"do":"wait"},{"do":"expectNoScriptExecution"},{"do":"freshClient"},{"do":"signIn","name":"script-review-reader"},{"do":"armScriptCanary"},{"do":"openItem","item":"Keyboard"},{"contains":"Benign stored review","do":"expect"},{"do":"wait"},{"do":"expectNoScriptExecution"}],"source":"scenarios/progression-review-script.json","stableKey":"ecommerce.progression.review-access-specifications.stored-review-script.9180a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"search-ordering","criterionId":"402b","description":"active filters and text searches remain alphabetical after purchases; clearing them restores purchase ranking","featureId":402,"featureName":"Purchases preserve search ordering","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.search-ordering","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.faceted-search"],"role":"guarantee","semantics":[{"do":"expectSequence","equals":["Air Purifier","Bluetooth Speaker","Coffee Grinder","Desk Lamp","Espresso Machine","Gaming Mouse","Headphones","Induction Cooktop","Keyboard","Laptop Stand"]},{"do":"fill","enter":true,"text":"e"},{"do":"fill","text":""},{"do":"click","ifAvailable":true},{"do":"expectSequence","equals":["Air Purifier","Bluetooth Speaker","Coffee Grinder","Desk Lamp","Espresso Machine","Gaming Mouse","Headphones","Keyboard","Mirrorless Camera","USB Cable"]},{"do":"fill","enter":true,"text":""},{"do":"expectSequence","equals":["Headphones","Air Purifier","Bluetooth Speaker","Coffee Grinder","Desk Lamp","Espresso Machine","Gaming Mouse","Induction Cooktop","Keyboard","Laptop Stand"]}],"source":"scenarios/progression-search-ordering.json","stableKey":"ecommerce.spec.search-ordering.search-ordering.402b","statedBy":"Active search and filter results use item-name order; clearing them restores purchase ranking","withheld":null},{"category":"production","checkGroupId":"shipping-accounting","criterionId":"202e","description":"shipping a purchased order does not deduct stock or add revenue again","featureId":202,"featureName":"Shipping preserves completed purchase accounting","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin","ecommerce.progression.fulfilment-queue"],"role":"guarantee","semantics":[{"as":"stock-before-purchase","do":"dbRecordStock","item":"Keyboard"},{"as":"revenue-before-purchase","do":"recordNumber"},{"do":"click"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"pending"},{"do":"dbExpectStock","item":"Keyboard","plus":-1,"relativeTo":"stock-before-purchase"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"expectNumber","plus":89,"relativeTo":"revenue-before-purchase"},{"as":"revenue-before-ship","do":"recordNumber"},{"as":"East-before-ship","do":"dbRecordStock","item":"Keyboard","warehouse":"East"},{"as":"West-before-ship","do":"dbRecordStock","item":"Keyboard","warehouse":"West"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click","ifAvailable":true},{"action":"ship","do":"callAction","from":"customer","input":{"attribute":"data-ship-input","contains":"Keyboard","testid":"order-item"},"namedAction":{"args":[0],"id":"ship","params":[{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/fulfilment/ship","reducer":"ship_order"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"reload"},{"do":"ensureSignedIn","name":"shipping-accounting"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"shipped"},{"do":"dbExpectStock","item":"Keyboard","plus":0,"relativeTo":"East-before-ship","warehouse":"East"},{"do":"dbExpectStock","item":"Keyboard","plus":0,"relativeTo":"West-before-ship","warehouse":"West"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true},{"do":"expectNumber","plus":0,"relativeTo":"revenue-before-ship"}],"source":"scenarios/progression-shipping-accounting.json","stableKey":"ecommerce.inventory-operations.shipping-accounting.202e","statedBy":"Shipping preserves the stock and revenue recorded for the purchase.","withheld":null},{"category":"production","checkGroupId":"signed-out-purchase","criterionId":"3a","description":"using the purchase control while signed out does not buy an item","featureId":3,"featureName":"Buying","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","semantics":[{"as":"keyboard-before-guest","do":"recordNumber"},{"do":"click","ifAvailable":true},{"do":"reload"},{"do":"click","ifAvailable":true},{"do":"click","ifAvailable":true},{"do":"expectNumber","plus":0,"relativeTo":"keyboard-before-guest"}],"source":"scenarios/progression-signed-out-purchase.json","stableKey":"ecommerce.spec.access-control.signed-out-purchase.3a","statedBy":"Unauthenticated callers cannot purchase.","withheld":null},{"category":"feature","checkGroupId":"split-tender-refunds-751","criterionId":"751a","description":"Full refund restores each original payment portion","featureId":751,"featureName":"Full refund restores each original payment portion","note":null,"packId":"ecommerce.feature.split-tender-refunds","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.split-tender-refunds","ecommerce.feature.store-credit","ecommerce.progression.support-refunds"],"role":"feature","semantics":[{"do":"click"},{"do":"freshClient"},{"do":"signIn","name":"credit-owner-751"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Desk Lamp","do":"expectElementCount","equals":1},{"do":"expectNumber","equals":42},{"do":"expectNumber","equals":10},{"do":"expectNumber","equals":32},{"do":"click"},{"attribute":"aria-busy","do":"expect","value":"false"},{"do":"expectNumber","equals":10}],"source":"scenarios/progression-split-tender-refunds.json","stableKey":"ecommerce.feature.split-tender-refunds.split-tender-refunds-751.751a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"production-756","criterionId":"756a","description":"Concurrent refunds restore the original credit and external amounts once, including after restart","featureId":756,"featureName":"Concurrent refunds do not duplicate credit after restart","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.split-tender-refunds","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.split-tender-refunds","ecommerce.feature.store-credit","ecommerce.progression.support-refunds"],"role":"guarantee","semantics":[{"action":"supportRefund","do":"callConcurrently","from":"staff","input":{"attribute":"data-refund-input","contains":"Split refund 756","testid":"support-ticket"},"namedAction":{"args":[0],"id":"supportRefund","method":"POST","params":[{"in":"path","name":"caseId","placeholder":"{caseId}","wireType":"u64"}],"path":"/api/support/cases/{caseId}/refund","reducer":"support_refund"}},{"do":"expectCallOutcomes"},{"do":"freshClient"},{"do":"signIn","name":"credit-owner-756"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Desk Lamp","do":"expectElementCount","equals":1},{"do":"expectNumber","equals":42},{"do":"expectNumber","equals":10},{"do":"expectNumber","equals":32},{"do":"click"},{"attribute":"aria-busy","do":"expect","value":"false"},{"do":"expectNumber","equals":10},{"do":"restartBackend"},{"do":"freshClient"},{"do":"signIn","name":"credit-owner-756"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Desk Lamp","do":"expectElementCount","equals":1},{"do":"expectNumber","equals":42},{"do":"expectNumber","equals":10},{"do":"expectNumber","equals":32},{"do":"click"},{"attribute":"aria-busy","do":"expect","value":"false"},{"do":"expectNumber","equals":10}],"source":"scenarios/progression-split-tender-refunds.json","stableKey":"ecommerce.spec.split-tender-refunds.production-756.756a","statedBy":null,"withheld":null},{"category":"feature","checkGroupId":"staff-access","criterionId":"601a","description":"staff and administrators can sign in and open staff tools","featureId":601,"featureName":"Staff access","note":null,"packId":"ecommerce.progression.staff-access","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click"},{"do":"expect"},{"do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click"},{"do":"expect"}],"source":"scenarios/progression-staff-access.json","stableKey":"ecommerce.progression.staff-access.staff-access.601a","statedBy":"Staff and administrators can sign in and use staff areas.","withheld":null},{"category":"production","checkGroupId":"staff-area-boundary","criterionId":"601b","description":"customers cannot open staff tools","featureId":601,"featureName":"Staff access","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.staff-access"],"role":"guarantee","semantics":[{"do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click"},{"do":"expect"},{"do":"signIn","exact":true,"name":"customer","password":"stackbench-customer-2026"},{"do":"click","ifAvailable":true},{"absent":true,"do":"expect"}],"source":"scenarios/progression-staff-access.json","stableKey":"ecommerce.spec.access-control.staff-area-boundary.601b","statedBy":"Customers cannot open staff tools.","withheld":null},{"category":"feature","checkGroupId":"staff-activity","criterionId":"624a","description":"an administrative change records its actor, action, subject, and time","featureId":624,"featureName":"Attributable staff activity","note":null,"packId":"ecommerce.progression.staff-activity","points":3,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"do":"click"},{"contains":"Activity Mug","do":"expect"},{"contains":"admin","do":"expect"},{"contains":"creat","do":"expect"},{"contains":"Activity Mug","do":"expect"},{"do":"expect"}],"source":"scenarios/progression-staff-activity.json","stableKey":"ecommerce.progression.staff-activity.staff-activity.624a","statedBy":"Each administrative change identifies its actor, action, subject, and time.","withheld":null},{"category":"production","checkGroupId":"staff-activity-privacy","criterionId":"624b","description":"customers cannot open staff activity history","featureId":624,"featureName":"Attributable staff activity","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.staff-activity"],"role":"guarantee","semantics":[{"absent":true,"do":"expect"}],"source":"scenarios/progression-staff-activity.json","stableKey":"ecommerce.spec.access-control.staff-activity-privacy.624b","statedBy":"Customers cannot open staff activity history.","withheld":null},{"category":"feature","checkGroupId":"staff-roles","criterionId":"621c","description":"an administrator can assign a staff role","featureId":621,"featureName":"Staff roles","note":null,"packId":"ecommerce.progression.staff-roles","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"expect","value":"inventory"}],"source":"scenarios/progression-staff-roles.json","stableKey":"ecommerce.progression.staff-roles.staff-roles.621c","statedBy":"An administrator can assign a role to an existing staff account.","withheld":null},{"category":"production","checkGroupId":"staff-role-reload","criterionId":"621a","description":"an assigned staff role survives reload and backend restart in a fresh browser","featureId":621,"featureName":"Staff roles","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.staff-roles"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true,"unlessVisible":"staff-role-account-staff"},{"do":"expect","value":"inventory"},{"do":"restartBackend"},{"do":"freshClient"},{"do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","unlessVisible":"staff-role-account-staff"},{"do":"expect","value":"inventory"}],"source":"scenarios/progression-staff-roles.json","stableKey":"ecommerce.spec.state-durability.staff-role-reload.621a","statedBy":"Assigned staff roles persist.","withheld":null},{"category":"production","checkGroupId":"staff-role-boundary","criterionId":"621b","description":"a staff member cannot assign roles through the UI or a replayed request","featureId":621,"featureName":"Staff roles","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.staff-roles"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"expect"},{"do":"click","unlessVisible":"staff-role-account-staff"},{"do":"fill","text":"staff"},{"do":"click"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true,"unlessVisible":"staff-role-account-staff"},{"do":"expect","value":"staff"},{"do":"reload"},{"do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click","ifAvailable":true},{"absent":true,"do":"expect"},{"do":"replayAs","from":"replayAdmin","match":"role","namedAction":{"args":[0,"inventory"],"id":"assignStaffRole","method":"PUT","params":[{"in":"path","name":"accountId","placeholder":":id","wireType":"u64"},{"in":"body","name":"role"}],"path":"/api/staff/:id/role","reducer":"assign_staff_role"},"namedTarget":{"attribute":"data-account-id","testid":"staff-role-account-staff","valueType":"string"},"swap":{"find":"\"role\":\"staff\"","with":"\"role\":\"inventory\""}},{"do":"expectReplayRejected"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true,"unlessVisible":"staff-role-account-staff"},{"do":"expect","value":"staff"}],"source":"scenarios/progression-staff-roles.json","stableKey":"ecommerce.spec.access-control.staff-role-boundary.621b","statedBy":"A staff member cannot assign or change roles.","withheld":null},{"category":"production","checkGroupId":"staff-role-revocation","criterionId":"621d","description":"removing administrator access blocks a previously authorized session without changing the target role","featureId":621,"featureName":"Staff roles","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.staff-roles"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","unlessVisible":"staff-role-account-staff"},{"do":"replayAs","from":"roleAdmin","match":"__role-revocation-named-action__","namedAction":{"args":[0,"admin"],"id":"assignStaffRole","method":"PUT","params":[{"in":"path","name":"accountId","placeholder":":id","wireType":"u64"},{"in":"body","name":"role"}],"path":"/api/staff/:id/role","reducer":"assign_staff_role"},"namedTarget":{"attribute":"data-account-id","testid":"staff-role-account-staff","valueType":"string"}},{"do":"expectReplayCompleted","requireAccepted":true},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true,"unlessVisible":"staff-role-account-staff"},{"do":"expect","value":"admin"},{"do":"reload"},{"do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"replayAs","from":"roleAdmin","match":"__role-revocation-named-action__","namedAction":{"args":[0,"admin"],"id":"assignStaffRole","method":"PUT","params":[{"in":"path","name":"accountId","placeholder":":id","wireType":"u64"},{"in":"body","name":"role"}],"path":"/api/staff/:id/role","reducer":"assign_staff_role"},"namedTarget":{"attribute":"data-account-id","testid":"staff-role-account-staff","valueType":"string"}},{"do":"expectReplayCompleted","requireAccepted":true},{"do":"replayAs","from":"roleAdmin","match":"__role-revocation-named-action__","namedAction":{"args":[0,"staff"],"id":"assignStaffRole","method":"PUT","params":[{"in":"path","name":"accountId","placeholder":":id","wireType":"u64"},{"in":"body","name":"role"}],"path":"/api/staff/:id/role","reducer":"assign_staff_role"},"namedTarget":{"attribute":"data-account-id","testid":"staff-role-account-staff","valueType":"string"}},{"do":"expectReplayCompleted","requireAccepted":true},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true,"unlessVisible":"staff-role-account-staff"},{"do":"expect","value":"staff"},{"do":"replayAs","from":"roleAdmin","match":"__role-revocation-named-action__","namedAction":{"args":[0,"admin"],"id":"assignStaffRole","method":"PUT","params":[{"in":"path","name":"accountId","placeholder":":id","wireType":"u64"},{"in":"body","name":"role"}],"path":"/api/staff/:id/role","reducer":"assign_staff_role"},"namedTarget":{"attribute":"data-account-id","testid":"staff-role-account-staff","valueType":"string"}},{"do":"expectReplayRejected"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","ifAvailable":true,"unlessVisible":"staff-role-account-staff"},{"do":"expect","value":"staff"}],"source":"scenarios/progression-staff-roles.json","stableKey":"ecommerce.spec.access-control.staff-role-revocation.621d","statedBy":"Only the admin role grants administrator access.","withheld":null},{"category":"feature","checkGroupId":"stock-alert-delivery","criterionId":"631c","description":"restored stock sends the requested alert","featureId":631,"featureName":"Stock alert delivery","note":null,"packId":"ecommerce.progression.stock-alerts","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"feature","semantics":[{"do":"freshClient"},{"do":"signIn","name":"stock-subscriber"},{"do":"click","unlessVisible":"notifications-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"contains":"Air Purifier","do":"expectElementCount","equals":0},{"do":"click"},{"action":"restock","do":"callAction","input":{"attribute":"data-restock-input","contains":"Air Purifier","testid":"admin-location-row"},"namedAction":{"args":[0,0,1],"id":"restock","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"warehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/restock","reducer":"admin_restock"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"wait"},{"do":"freshClient"},{"do":"signIn","name":"stock-subscriber"},{"do":"click","unlessVisible":"notifications-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"contains":"Air Purifier","do":"expect"}],"source":"scenarios/progression-stock-alert-delivery.json","stableKey":"ecommerce.progression.stock-alerts.stock-alert-delivery.631c","statedBy":"Show an alert when stock returns.","withheld":null},{"category":"production","checkGroupId":"stock-alert-deduplication","criterionId":"631a","description":"restored stock sends one alert and later restocks do not duplicate it","featureId":631,"featureName":"Private one-time stock alerts","note":"Sample a fresh account view 10 seconds after the second accepted restock. This catches persistent duplicates delivered by that sample, not all later or transient duplicates.","observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.warehouse-admin","ecommerce.progression.stock-alerts"],"role":"guarantee","semantics":[{"do":"freshClient"},{"do":"signIn","name":"stock-subscriber"},{"do":"click","unlessVisible":"notifications-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"contains":"Air Purifier","do":"expectElementCount","equals":1},{"action":"restock","do":"callAction","input":{"attribute":"data-restock-input","contains":"Air Purifier","testid":"admin-location-row"},"namedAction":{"args":[0,0,1],"id":"restock","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"warehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/restock","reducer":"admin_restock"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"wait"},{"do":"freshClient"},{"do":"signIn","name":"stock-subscriber"},{"do":"click","unlessVisible":"notifications-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"contains":"Air Purifier","do":"expectElementCount","equals":1}],"source":"scenarios/progression-stock-alerts.json","stableKey":"ecommerce.spec.transactional-integrity.stock-alert-deduplication.631a","statedBy":"A stock request creates one alert when stock returns.","withheld":null},{"category":"production","checkGroupId":"stock-alert-privacy","criterionId":"631b","description":"a customer who did not request the alert cannot see it","featureId":631,"featureName":"Private one-time stock alerts","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"provenBy":null,"requiresFeatures":["ecommerce.feature.warehouse-admin","ecommerce.progression.stock-alerts"],"role":"guarantee","semantics":[{"do":"freshClient"},{"do":"signIn","name":"stock-subscriber"},{"do":"click","unlessVisible":"notifications-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"contains":"Air Purifier","do":"expect"},{"do":"freshClient"},{"do":"signIn","name":"stock-other"},{"do":"click","unlessVisible":"notifications-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"absent":true,"contains":"Air Purifier","do":"expect"}],"source":"scenarios/progression-stock-alerts.json","stableKey":"ecommerce.spec.access-control.stock-alert-privacy.631b","statedBy":"Stock alerts are private to the requesting customer.","withheld":null},{"category":"production","checkGroupId":"stock-limit","criterionId":"3d","description":"an item sells out visibly, and a further purchase is refused without changing stock","featureId":3,"featureName":"Buying","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","semantics":[{"do":"click"},{"do":"wait"},{"do":"click"},{"do":"wait"},{"do":"click"},{"do":"expectNumber","equals":0},{"do":"expect"},{"action":"buy","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"do":"expectActionOutcome","outcome":"validation-refused"},{"do":"reload"},{"do":"ensureSignedIn","name":"eli"},{"do":"expectNumber","equals":0}],"source":"scenarios/progression-stock-limit.json","stableKey":"ecommerce.spec.concurrency-safety.stock-limit.3d","statedBy":"an item with zero stock cannot be bought","withheld":null},{"category":"feature","checkGroupId":"store-credit-750","criterionId":"750a","description":"Credit checkout records both payment portions","featureId":750,"featureName":"Credit checkout records both payment portions","note":null,"packId":"ecommerce.feature.store-credit","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"do":"freshClient"},{"do":"signIn","name":"credit-owner-750"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Desk Lamp","do":"expectElementCount","equals":1},{"do":"expectNumber","equals":10},{"do":"expectNumber","equals":32},{"do":"expectNumber","equals":42},{"do":"click","unlessVisible":"credit-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"do":"expectNumber","equals":0}],"source":"scenarios/progression-store-credit.json","stableKey":"ecommerce.feature.store-credit.store-credit-750.750a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"production-752","criterionId":"752a","description":"Repeating a grant reference does not increase the balance twice","featureId":752,"featureName":"Repeating a grant reference does not increase the balance twice","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.store-credit","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.store-credit"],"role":"guarantee","semantics":[{"action":"grantCredit","do":"callAction","from":"staff","input":{"attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","attribute":"data-account-id","testid":"credit-panel"}},"testid":"credit-grant"},"namedAction":{"args":[0,1000,""],"id":"grantCredit","method":"POST","params":[{"in":"body","name":"accountId","wireType":"u64"},{"in":"body","name":"amountMinor"},{"in":"body","name":"reference"}],"path":"/api/staff/credit","reducer":"grant_credit"}},{"do":"expectActionOutcome","outcome":"completed"},{"do":"freshClient"},{"do":"signIn","name":"credit-owner-752"},{"do":"click","unlessVisible":"credit-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"do":"expectNumber","equals":10}],"source":"scenarios/progression-store-credit.json","stableKey":"ecommerce.spec.store-credit.production-752.752a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"production-753","criterionId":"753a","description":"A customer cannot grant credit","featureId":753,"featureName":"A customer cannot grant credit","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.store-credit","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.store-credit"],"role":"guarantee","semantics":[{"do":"fill","text":"unauthorized-credit-753"},{"action":"grantCredit","do":"callAction","from":"staff","input":{"attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","attribute":"data-account-id","testid":"credit-panel"}},"testid":"credit-grant"},"namedAction":{"args":[0,1000,""],"id":"grantCredit","method":"POST","params":[{"in":"body","name":"accountId","wireType":"u64"},{"in":"body","name":"amountMinor"},{"in":"body","name":"reference"}],"path":"/api/staff/credit","reducer":"grant_credit"}},{"do":"expectActionOutcome","outcome":"refused"},{"do":"freshClient"},{"do":"signIn","name":"credit-owner-753"},{"do":"click","unlessVisible":"credit-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"do":"expectNumber","equals":10}],"source":"scenarios/progression-store-credit.json","stableKey":"ecommerce.spec.store-credit.production-753.753a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"production-754","criterionId":"754a","description":"Concurrent checkout consumes one cart and one credit allocation","featureId":754,"featureName":"Concurrent checkout consumes one cart and one credit allocation","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.store-credit","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.store-credit"],"role":"guarantee","semantics":[{"action":"checkoutCredit","do":"callConcurrently","namedAction":{"args":[],"id":"checkoutCredit","method":"POST","path":"/api/checkout/credit","reducer":"checkout_credit"}},{"do":"expectCallOutcomes"},{"do":"freshClient"},{"do":"signIn","name":"credit-owner-754"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"contains":"Desk Lamp","do":"expectElementCount","equals":1},{"do":"expectNumber","equals":10},{"do":"expectNumber","equals":32},{"do":"expectNumber","equals":42},{"do":"click","unlessVisible":"credit-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"do":"expectNumber","equals":0}],"source":"scenarios/progression-store-credit.json","stableKey":"ecommerce.spec.store-credit.production-754.754a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"production-755","criterionId":"755a","description":"Issued credit survives a backend restart","featureId":755,"featureName":"Issued credit survives a backend restart","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.store-credit","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.store-credit"],"role":"guarantee","semantics":[{"do":"restartBackend"},{"do":"freshClient"},{"do":"signIn","name":"credit-owner-755"},{"do":"click","unlessVisible":"credit-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"do":"expectNumber","equals":10}],"source":"scenarios/progression-store-credit.json","stableKey":"ecommerce.spec.store-credit.production-755.755a","statedBy":null,"withheld":null},{"category":"feature","checkGroupId":"subscriptions-760","criterionId":"760a","description":"A subscription creates exactly its requested deliveries and payments","featureId":760,"featureName":"A subscription creates exactly its requested deliveries and payments","note":null,"packId":"ecommerce.feature.subscriptions","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"freshClient"},{"do":"signIn","name":"subscription-760"},{"do":"click","unlessVisible":"subscriptions-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"do":"expect","value":"complete"},{"do":"expectElementCount","equals":2},{"do":"expectSequence","equals":["paid","paid"]},{"do":"expectNumber","equals":84},{"do":"dbExpectStock","equals":8,"item":"Desk Lamp"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Desk Lamp","do":"expectElementCount","equals":2},{"do":"expectElementCount","equals":2}],"source":"scenarios/progression-subscriptions.json","stableKey":"ecommerce.feature.subscriptions.subscriptions-760.760a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"production-761","criterionId":"761a","description":"A pending subscription continues after backend restart without duplicate deliveries","featureId":761,"featureName":"A pending subscription continues after backend restart without duplicate deliveries","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.subscriptions","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.subscriptions"],"role":"guarantee","semantics":[{"do":"restartBackend"},{"do":"freshClient"},{"do":"signIn","name":"subscription-761"},{"do":"click","unlessVisible":"subscriptions-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"do":"expect","value":"complete"},{"do":"expectElementCount","equals":2},{"do":"expectSequence","equals":["paid","paid"]},{"do":"expectNumber","equals":84},{"do":"dbExpectStock","equals":8,"item":"Desk Lamp"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Desk Lamp","do":"expectElementCount","equals":2},{"do":"expectElementCount","equals":2},{"do":"restartBackend"},{"do":"freshClient"},{"do":"signIn","name":"subscription-761"},{"do":"click","unlessVisible":"subscriptions-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"do":"expect","value":"complete"},{"do":"expectElementCount","equals":2},{"do":"expectSequence","equals":["paid","paid"]},{"do":"expectNumber","equals":84},{"do":"dbExpectStock","equals":8,"item":"Desk Lamp"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Desk Lamp","do":"expectElementCount","equals":2},{"do":"expectElementCount","equals":2}],"source":"scenarios/progression-subscriptions.json","stableKey":"ecommerce.spec.subscriptions.production-761.761a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"production-762","criterionId":"762a","description":"Another customer cannot cancel an active subscription","featureId":762,"featureName":"Another customer cannot cancel an active subscription","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.subscriptions","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.subscriptions"],"role":"guarantee","semantics":[{"action":"cancelSubscription","do":"callAction","from":"owner","input":{"attribute":"data-action-input","testid":"subscription-cancel"},"namedAction":{"args":[0],"id":"cancelSubscription","method":"POST","params":[{"in":"path","name":"subscriptionId","placeholder":"{subscriptionId}","wireType":"u64"}],"path":"/api/subscriptions/{subscriptionId}/cancel","reducer":"cancel_subscription"}},{"do":"expectActionOutcome","outcome":"refused"},{"do":"expect","value":"active"},{"do":"click"},{"do":"freshClient"},{"do":"signIn","name":"subscription-762"},{"do":"click","unlessVisible":"subscriptions-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"do":"expect","value":"cancelled"},{"as":"deliveries-at-cancel","count":true,"do":"recordNumber"},{"as":"stock-at-cancel","do":"dbRecordStock","item":"Desk Lamp"},{"do":"wait"},{"do":"expectElementCount","plus":0,"relativeTo":"deliveries-at-cancel"},{"do":"dbExpectStock","item":"Desk Lamp","plus":0,"relativeTo":"stock-at-cancel"}],"source":"scenarios/progression-subscriptions.json","stableKey":"ecommerce.spec.subscriptions.production-762.762a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"production-763","criterionId":"763a","description":"Pause survives a restart and resume completes the remaining deliveries","featureId":763,"featureName":"Pause survives a restart and resume completes the remaining deliveries","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.subscriptions","points":2,"provenBy":null,"requiresFeatures":["ecommerce.feature.subscriptions"],"role":"guarantee","semantics":[{"do":"click"},{"do":"expect","value":"paused"},{"do":"restartBackend"},{"do":"freshClient"},{"do":"signIn","name":"subscription-763"},{"do":"click","unlessVisible":"subscriptions-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"do":"expect","value":"paused"},{"as":"deliveries-at-pause","count":true,"do":"recordNumber"},{"as":"stock-at-pause","do":"dbRecordStock","item":"Desk Lamp"},{"do":"wait"},{"do":"expectElementCount","plus":0,"relativeTo":"deliveries-at-pause"},{"do":"dbExpectStock","item":"Desk Lamp","plus":0,"relativeTo":"stock-at-pause"},{"do":"click"},{"do":"freshClient"},{"do":"signIn","name":"subscription-763"},{"do":"click","unlessVisible":"subscriptions-panel"},{"attribute":"aria-busy","do":"expect","value":"false"},{"do":"expect","value":"complete"},{"do":"expectElementCount","equals":2},{"do":"expectSequence","equals":["paid","paid"]},{"do":"expectNumber","equals":84},{"do":"dbExpectStock","equals":8,"item":"Desk Lamp"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Desk Lamp","do":"expectElementCount","equals":2},{"do":"expectElementCount","equals":2}],"source":"scenarios/progression-subscriptions.json","stableKey":"ecommerce.spec.subscriptions.production-763.763a","statedBy":null,"withheld":null},{"category":"feature","checkGroupId":"support-history","criterionId":"612c","description":"the customer can view their support ticket history","featureId":612,"featureName":"Customer support history","note":null,"packId":"ecommerce.progression.support-history","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"reload"},{"do":"ensureSignedIn","name":"support-owner"},{"do":"click","unlessVisible":"support-ticket"},{"contains":"Owner ticket {user:ticketmarker}","do":"expect"}],"source":"scenarios/progression-support-history.json","stableKey":"ecommerce.progression.support-history.support-history.612c","statedBy":"Signed-in customers can see their support ticket history.","withheld":null},{"category":"production","checkGroupId":"support-history-reload","criterionId":"612a","description":"support history survives reload and backend restart in a fresh browser","featureId":612,"featureName":"Customer support history","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.support-history"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"ensureSignedIn","name":"support-owner"},{"do":"click"},{"contains":"Owner ticket {user:ticketmarker}","do":"expect"},{"do":"restartBackend"},{"do":"freshClient"},{"do":"signIn","name":"support-owner"},{"do":"click"},{"contains":"Owner ticket {user:ticketmarker}","do":"expect"}],"source":"scenarios/progression-support-history.json","stableKey":"ecommerce.spec.state-durability.support-history-reload.612a","statedBy":"A customer's support history persists.","withheld":null},{"category":"production","checkGroupId":"support-history-privacy","criterionId":"612b","description":"another customer neither sees nor receives the private ticket","featureId":612,"featureName":"Customer support history","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.support-history"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"ensureSignedIn","name":"support-owner"},{"do":"click"},{"do":"fill","text":"owner@example.com"},{"do":"fill","text":"Private ticket {user:privateticketmarker}"},{"do":"fill","text":"Private account issue."},{"do":"click"},{"do":"expect","nonEmpty":true},{"do":"reload"},{"do":"ensureSignedIn","name":"support-owner"},{"do":"click"},{"contains":"Private ticket {user:privateticketmarker}","do":"expect"},{"contains":"Private ticket {user:privateticketmarker}","do":"expectReceived"},{"do":"reload"},{"do":"ensureSignedIn","name":"support-other"},{"do":"click"},{"absent":true,"contains":"Private ticket {user:privateticketmarker}","do":"expect"},{"contains":"Private ticket {user:privateticketmarker}","do":"expectNotReceived"}],"source":"scenarios/progression-support-history.json","stableKey":"ecommerce.spec.access-control.support-history-privacy.612b","statedBy":"Customers cannot see another customer's support tickets.","withheld":null},{"category":"production","checkGroupId":"support-history-logout","criterionId":"612d","description":"after logout the same browser storage no longer grants access to private support history","featureId":612,"featureName":"Customer support history","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":2,"provenBy":null,"requiresFeatures":["ecommerce.progression.support-history"],"role":"guarantee","semantics":[{"do":"reload"},{"do":"ensureSignedIn","name":"support-owner"},{"do":"click"},{"do":"fill","text":"owner@example.com"},{"do":"fill","text":"Logout ticket {user:logoutticketmarker}"},{"do":"fill","text":"Private account issue."},{"do":"click"},{"do":"expect","nonEmpty":true},{"do":"reload"},{"do":"ensureSignedIn","name":"support-owner"},{"do":"click","unlessVisible":"support-ticket"},{"contains":"Logout ticket {user:logoutticketmarker}","do":"expect"},{"contains":"Logout ticket {user:logoutticketmarker}","do":"expectReceived"},{"do":"reload"},{"do":"ensureSignedIn","name":"support-owner"},{"do":"click","unlessVisible":"signout"},{"do":"click"},{"do":"waitUntilAbsent"},{"do":"freshClient","preserveStorage":true},{"absent":true,"do":"expect"},{"do":"click"},{"do":"expect"},{"absent":true,"contains":"Logout ticket {user:logoutticketmarker}","do":"expect"},{"contains":"Logout ticket {user:logoutticketmarker}","do":"expectNotReceived"}],"source":"scenarios/progression-support-history.json","stableKey":"ecommerce.spec.access-control.support-history-logout.612d","statedBy":"Signed-out visitors cannot access private support tickets.","withheld":null},{"category":"feature","checkGroupId":"support-intake","criterionId":"610a","description":"a visitor can submit a support ticket and receives a reference","featureId":610,"featureName":"Support intake","note":null,"packId":"ecommerce.progression.support-intake","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"do":"fill","text":"visitor@example.com"},{"do":"fill","text":"Damaged package"},{"do":"fill","text":"The package arrived damaged."},{"do":"click"},{"do":"expect","nonEmpty":true}],"source":"scenarios/progression-support-intake.json","stableKey":"ecommerce.progression.support-intake.support-intake.610a","statedBy":"Anyone can open a support ticket and receives a reference.","withheld":null},{"category":"production","checkGroupId":"support-refund-access","criterionId":"615c","description":"a customer cannot issue a refund or change its records","featureId":615,"featureName":"Support refund access","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","points":1,"provenBy":null,"requiresFeatures":["ecommerce.progression.support-refunds"],"role":"guarantee","semantics":[{"absent":true,"do":"expect"},{"action":"supportRefund","authentication":"actor","do":"callAction","from":"staff","input":{"attribute":"data-action-input","testid":"support-refund"},"namedAction":{"args":[0],"id":"supportRefund","method":"POST","params":[{"in":"path","name":"caseId","placeholder":"{caseId}","wireType":"u64"}],"path":"/api/support/cases/{caseId}/refund","reducer":"support_refund"}},{"do":"expectActionOutcome","outcome":"refused"},{"do":"reload"},{"do":"ensureSignedIn","name":"refund-access-owner"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"support-subject"},{"do":"expect","notContains":"resolved"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"do":"expect","value":"pending"},{"absent":true,"contains":"Keyboard","do":"expect"}],"source":"scenarios/progression-support-refunds-access.json","stableKey":"ecommerce.spec.access-control.support-refund-access.615c","statedBy":"Only authorized staff can refund an order from a support case.","withheld":null},{"category":"production","checkGroupId":"support-refund-accounting","criterionId":"615b","description":"the refund equals the paid total, cannot be applied twice, and leaves another order unrefunded","featureId":615,"featureName":"Support refund accounting","note":null,"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","points":1,"provenBy":null,"requiresFeatures":["ecommerce.progression.support-refunds"],"role":"guarantee","semantics":[{"do":"click"},{"do":"reload"},{"do":"ensureSignedIn","name":"refund-accounting-owner"},{"do":"click","ifAvailable":true},{"do":"click","unlessVisible":"support-subject"},{"do":"expectNumber","plus":0,"relativeTo":"paid-total"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"expectNumber","plus":0,"relativeTo":"paid-total"},{"contains":"Keyboard","do":"expectElementCount","equals":1},{"do":"replayAs","from":"staff","match":"refund","namedAction":{"args":[0],"id":"supportRefund","method":"POST","params":[{"in":"path","name":"caseId","placeholder":"{caseId}","wireType":"u64"}],"path":"/api/support/cases/{caseId}/refund","reducer":"support_refund"},"namedTarget":{"attribute":"data-entity-id","contains":"Accounting refund case","testid":"support-ticket","valueType":"string"}},{"do":"expectReplayCompleted"},{"do":"freshClient"},{"do":"signIn","name":"refund-accounting-owner"},{"do":"click"},{"do":"expectNumber","plus":0,"relativeTo":"paid-total"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"expectNumber","plus":0,"relativeTo":"paid-total"},{"contains":"Keyboard","do":"expectElementCount","equals":1},{"do":"expectNumber","equals":0},{"contains":"Mouse","do":"expectElementCount","equals":0}],"source":"scenarios/progression-support-refunds-accounting.json","stableKey":"ecommerce.spec.transactional-integrity.support-refund-accounting.615b","statedBy":"The recorded refund equals the amount paid and is applied only once.","withheld":null},{"category":"feature","checkGroupId":"support-refunds-resolution","criterionId":"615a","description":"an authorized refund resolves the case and updates the order","featureId":615,"featureName":"Support refund resolution","note":null,"packId":"ecommerce.progression.support-refunds","points":2,"provenBy":null,"role":"feature","semantics":[{"do":"click"},{"contains":"resolved","do":"expect"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"refunded","do":"expect"}],"source":"scenarios/progression-support-refunds-resolution.json","stableKey":"ecommerce.progression.support-refunds.support-refunds-resolution.615a","statedBy":"Authorized staff can refund an order from its support case.","withheld":null},{"category":"production","checkGroupId":"return-refund-interaction","criterionId":"757a","description":"a support refund followed by physical return restores each warehouse once and refunds only the price paid","featureId":757,"featureName":"Return and support refund accounting","note":null,"packId":"ecommerce.feature.split-tender-refunds","points":2,"provenBy":null,"requiresFeatures":["ecommerce.l3.order-returns-features","ecommerce.progression.fulfilment-queue","ecommerce.progression.support-refunds"],"role":"feature","semantics":[{"do":"reload"},{"do":"ensureSignedIn","name":"return-refund-owner"},{"do":"click","ifAvailable":true},{"as":"757atotal","do":"dbRecordStock","item":"Keyboard"},{"as":"757aEast","do":"dbRecordStock","item":"Keyboard","warehouse":"East"},{"as":"757aWest","do":"dbRecordStock","item":"Keyboard","warehouse":"West"},{"as":"757arevenue","do":"recordNumber"},{"do":"click"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"pending"},{"as":"757apaid","do":"recordNumber"},{"action":"ship","do":"callAction","from":"owner","input":{"attribute":"data-ship-input","contains":"Keyboard","testid":"order-item"},"namedAction":{"args":[0],"id":"ship","params":[{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/fulfilment/ship","reducer":"ship_order"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"expect","ignoreCase":true,"value":"shipped"},{"do":"dbExpectStock","item":"Keyboard","plus":-1,"relativeTo":"757atotal"},{"do":"reload"},{"do":"ensureSignedIn","name":"return-refund-owner"},{"do":"click"},{"do":"fill","text":"Return refund 757a"},{"do":"fill","text":"Please refund this order."},{"do":"click"},{"contains":"Return refund 757a","do":"expect"},{"do":"click"},{"do":"click"},{"do":"expect"},{"action":"supportRefund","do":"callAction","from":"owner","input":{"attribute":"data-refund-input","contains":"Return refund 757a","testid":"support-ticket"},"namedAction":{"args":[0],"id":"supportRefund","method":"POST","params":[{"in":"path","name":"caseId","placeholder":"{caseId}","wireType":"u64"}],"path":"/api/support/cases/{caseId}/refund","reducer":"support_refund"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"dbExpectStock","item":"Keyboard","plus":-1,"relativeTo":"757atotal"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"click"},{"contains":"Keyboard","containsText":"returned","do":"expect","ignoreCase":true},{"do":"freshClient"},{"do":"signIn","name":"return-refund-owner"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Keyboard","containsText":"returned","do":"expect","ignoreCase":true},{"do":"expectNumber","plus":0,"relativeTo":"757apaid"},{"do":"dbExpectStock","item":"Keyboard","plus":0,"relativeTo":"757aEast","warehouse":"East"},{"do":"dbExpectStock","item":"Keyboard","plus":0,"relativeTo":"757aWest","warehouse":"West"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","unlessVisible":"admin-revenue"},{"do":"expectNumber","plus":0,"relativeTo":"757arevenue"}],"source":"scenarios/progression-support-return-interaction.json","stableKey":"ecommerce.feature.split-tender-refunds.return-refund-interaction.757a","statedBy":null,"withheld":null},{"category":"production","checkGroupId":"return-refund-interaction","criterionId":"757b","description":"a physical return followed by support refund restores each warehouse once and refunds only the price paid","featureId":757,"featureName":"Return and support refund accounting","note":null,"packId":"ecommerce.feature.split-tender-refunds","points":2,"provenBy":null,"requiresFeatures":["ecommerce.l3.order-returns-features","ecommerce.progression.fulfilment-queue","ecommerce.progression.support-refunds"],"role":"feature","semantics":[{"do":"reload"},{"do":"ensureSignedIn","name":"return-refund-owner"},{"do":"click","ifAvailable":true},{"as":"757btotal","do":"dbRecordStock","item":"Desk Lamp"},{"as":"757bEast","do":"dbRecordStock","item":"Desk Lamp","warehouse":"East"},{"as":"757bWest","do":"dbRecordStock","item":"Desk Lamp","warehouse":"West"},{"as":"757brevenue","do":"recordNumber"},{"do":"click"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"expect","ignoreCase":true,"value":"pending"},{"as":"757bpaid","do":"recordNumber"},{"action":"ship","do":"callAction","from":"owner","input":{"attribute":"data-ship-input","contains":"Desk Lamp","testid":"order-item"},"namedAction":{"args":[0],"id":"ship","params":[{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/fulfilment/ship","reducer":"ship_order"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"expect","ignoreCase":true,"value":"shipped"},{"do":"dbExpectStock","item":"Desk Lamp","plus":-1,"relativeTo":"757btotal"},{"do":"reload"},{"do":"ensureSignedIn","name":"return-refund-owner"},{"do":"click"},{"do":"fill","text":"Return refund 757b"},{"do":"fill","text":"Please refund this order."},{"do":"click"},{"contains":"Return refund 757b","do":"expect"},{"do":"click"},{"do":"click"},{"do":"expect"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click","unlessVisible":"order-item"},{"do":"click"},{"contains":"Desk Lamp","containsText":"returned","do":"expect","ignoreCase":true},{"action":"supportRefund","do":"callAction","from":"owner","input":{"attribute":"data-refund-input","contains":"Return refund 757b","testid":"support-ticket"},"namedAction":{"args":[0],"id":"supportRefund","method":"POST","params":[{"in":"path","name":"caseId","placeholder":"{caseId}","wireType":"u64"}],"path":"/api/support/cases/{caseId}/refund","reducer":"support_refund"}},{"do":"expectActionOutcome","outcome":"accepted"},{"do":"freshClient"},{"do":"signIn","name":"return-refund-owner"},{"do":"click","ifAvailable":true,"unlessVisible":"order-item"},{"do":"click"},{"contains":"Desk Lamp","containsText":"returned","do":"expect","ignoreCase":true},{"do":"expectNumber","plus":0,"relativeTo":"757bpaid"},{"do":"dbExpectStock","item":"Desk Lamp","plus":0,"relativeTo":"757bEast","warehouse":"East"},{"do":"dbExpectStock","item":"Desk Lamp","plus":0,"relativeTo":"757bWest","warehouse":"West"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"click","unlessVisible":"admin-revenue"},{"do":"expectNumber","plus":0,"relativeTo":"757brevenue"}],"source":"scenarios/progression-support-return-interaction.json","stableKey":"ecommerce.feature.split-tender-refunds.return-refund-interaction.757b","statedBy":null,"withheld":null},{"category":"feature","checkGroupId":"support-assignment","criterionId":"611a","description":"staff can assign a new ticket","featureId":611,"featureName":"Support triage","note":null,"packId":"ecommerce.progression.support-triage","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"fill","text":"staff"},{"do":"click"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click","ifAvailable":true,"unlessVisible":"support-assignee"},{"do":"click","ifAvailable":true,"unlessVisible":"support-assignee"},{"do":"expect","value":"staff"}],"source":"scenarios/progression-support-triage.json","stableKey":"ecommerce.progression.support-triage.support-assignment.611a","statedBy":"Staff can assign support tickets.","withheld":null},{"category":"feature","checkGroupId":"support-priority","criterionId":"611b","description":"staff can set a ticket priority","featureId":611,"featureName":"Support triage","note":null,"packId":"ecommerce.progression.support-triage","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"fill","text":"high"},{"do":"click"},{"do":"reload"},{"do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"click","ifAvailable":true,"unlessVisible":"support-assignee"},{"do":"click","ifAvailable":true,"unlessVisible":"support-assignee"},{"do":"expect","value":"high"}],"source":"scenarios/progression-support-triage.json","stableKey":"ecommerce.progression.support-triage.support-priority.611b","statedBy":"Staff can set support ticket priority.","withheld":null},{"category":"feature","checkGroupId":"support-status","criterionId":"611c","description":"staff can change a ticket status","featureId":611,"featureName":"Support triage","note":null,"packId":"ecommerce.progression.support-triage","points":1,"provenBy":null,"role":"feature","semantics":[{"do":"fill","text":"in progress"},{"do":"click"},{"contains":"in progress","do":"expect"}],"source":"scenarios/progression-support-triage.json","stableKey":"ecommerce.progression.support-triage.support-status.611c","statedBy":"Staff can change support ticket status.","withheld":null}],"schemaVersion":3,"task":{"baseMeaningSha256":null,"contracts":[{"id":"ecommerce.application-interface","owners":["recipe"],"text":"# Application interface\n\nUse each interface name below as an exact HTML attribute value. Use `id` for\na one-off element. Use `data-role` when the same interface can\nappear more than once. These attributes do not prescribe the layout, data\nmodel, libraries, or transport.\n\n<!-- interface:http -->\nFor error responses, use HTTP 401 or 403 for access errors and 400, 409, or 422 for input errors.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nReport reducer errors by failing the call.\n<!-- /interface -->\n\nHuman-readable status text is case-insensitive. Machine identifiers and protocol values keep their specified spelling.\n"},{"id":"ecommerce.progression.staff-access-hooks","owners":["ecommerce.progression.staff-access"],"text":"## Staff access controls\n\nUse the existing account sign-in controls. Use `staff-link` for the staff area and `admin-link`\nfor the administrator area. Use `staff-area` for the staff tools, not for a loading or error message.\n\nUse these provided accounts:\n\n- staff: `staff` / `stackbench-staff-2026`\n- administrator: `admin` / `stackbench-admin-2026`\n- customer: `customer` / `stackbench-customer-2026`\n"},{"id":"ecommerce.progression.customer-profile-hooks","owners":["ecommerce.progression.customer-profile"],"text":"# Customer profile application interface\n\nUse `catalog-link` to return to the catalog. If a profile overlay blocks navigation,\nexpose `overlay-close` to dismiss it. Saving may leave the profile open.\n\nMake `profile-link` available from the catalog while signed in, without first opening another\narea. Use it to open the profile. Use `profile-name` and `profile-address` for the editable\nvalues. Use `profile-save` to save them. Use `profile-address-summary` to display\nthe saved address in the profile view.\n"},{"id":"ecommerce.progression.staff-role-hooks","owners":["ecommerce.progression.staff-roles"],"text":"# Staff role application interface\n\nPut role management in the administrator area opened by `admin-link`.\nUse `staff-role-row` for each staff account and set `data-account-id` to that account's server\nidentifier. Put `staff-role-select` and `staff-role-save` inside the row.\nAlso set the row's HTML `id` to `staff-role-account-` followed by\n`encodeURIComponent(username)`, using the exact account username without changing its case.\nFor example, username `staff` has row ID `staff-role-account-staff`. This identifies the\naccount independently of the role options or other text in the row.\n\nThe staff sign-in and staff-area controls come from the staff access feature.\n\nExpose the same role assignment used by `staff-role-save`.\n\n<!-- interface:http -->\nUse `PUT /api/staff/:id/role`, where `:id` is the account identifier from `data-account-id`.\nThe JSON body is `{ \"role\": \"<selected role>\" }`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nUse the `assign_staff_role` reducer with arguments in this order: `accountId: u64`,\n`role: string`. Render `data-account-id` as the decimal account identifier without precision loss.\n<!-- /interface -->\n\n`staff-role-select` offers the roles `staff`, `inventory`, and `admin` as its option values.\n"},{"id":"ecommerce.progression.catalog-management-hooks","owners":["ecommerce.progression.catalog-management"],"text":"# Catalog management application interface\n\nUse `admin-link` to open the administrator area containing the product controls.\nUse `catalog-name`, `catalog-category`, `catalog-price`, and `catalog-variants` for the product\nvalues; `catalog-category` accepts a new category name as text, and `catalog-variants` accepts\ncomma-separated variant names. Use `catalog-save` to add the product. Use `item-variant` for each named variant shown\nto a visitor.\n"},{"id":"ecommerce.progression.payment-record-hooks","owners":["ecommerce.progression.payment-records"],"text":"# Payment record application interface\n\nUse `payment-record`, `payment-status`, and `payment-amount` inside the matching `order-item`.\n"},{"id":"ecommerce.progression.staff-activity-hooks","owners":["ecommerce.progression.staff-activity"],"text":"# Staff activity application interface\n\nUse `activity-link` to open staff activity history and `activity-entry` for each change. Inside\neach entry, use `activity-actor`, `activity-action`, `activity-subject`, and `activity-time`.\n"},{"id":"ecommerce.feature.catalog-items.hooks","owners":["ecommerce.feature.catalog-items"],"text":"# Catalog item application interface\n\n| Element ID | Required element |\n| --- | --- |\n| `item-list` | Contains the public catalog items. |\n| `item-card` | Shows one catalog item. |\n| `item-name` | Shows the item name inside its `item-card`; activating it opens `item-detail`. |\n| `item-price` | Shows the numeric item price inside its `item-card`. |\n| `item-stock` | Shows total stock inside its `item-card`. |\n| `item-detail` | Contains the selected item's details. |\n"},{"id":"ecommerce.feature.catalog-discovery.hooks","owners":["ecommerce.feature.catalog-discovery"],"text":"# Catalog discovery application interface\n\nNew catalog data starts with zero purchases. Do not seed sample orders or purchase counts.\nWhen adding this feature to an existing app, preserve purchases made through the app.\n\n| Element ID | Required element |\n| --- | --- |\n| `item-list` | Contains exactly the ten ranked storefront items. |\n| `item-card` | Shows one storefront or search result. |\n| `item-name` | Shows the item name inside its `item-card`. |\n| `search-input` | Searches the full catalog as the visitor types or when they press Enter; no separate control runs the search. |\n| `search-results` | Contains matching `item-card` results. |\n"},{"id":"ecommerce.l2.transfer-hooks","owners":["ecommerce.l2.stock-transfers-features"],"text":"# Stock transfer application interface\n\nPut `transfer-from`, `transfer-to`, `transfer-qty`, and `transfer-submit` inside the applicable\n`admin-item-row`. Use `warehouse-total` inside each `admin-warehouse-item` for its numeric stock\ntotal. Show `order-error` when a transfer is refused.\n\nPut `data-transfer-input` on each `admin-item-row`. Its value is a JSON object with exactly\n`itemId`, `fromWarehouseId`, and `toWarehouseId` for the currently selected source and destination.\nIdentifiers can be JSON numbers or strings.\n\n<!-- interface:http -->\nExpose `POST /api/admin/transfer`. The JSON body has `itemId`, `fromWarehouseId`,\n`toWarehouseId`, and `quantity`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nExpose `admin_transfer_stock` with arguments in this order: `itemId: u64`,\n`fromWarehouseId: u64`, `toWarehouseId: u64`, `quantity`.\n<!-- /interface -->\n"},{"id":"ecommerce.l2.price-hooks","owners":["ecommerce.l2.price-history-features"],"text":"# Price history application interface\n\nPut `price-input` and `price-submit` inside the applicable `admin-item-row`.\n\nPut a `data-price-input` attribute on each `admin-item-row`. Its value is a JSON object with\n`itemId` and numeric `price` from the current price input. Identifiers can be\nJSON numbers or strings.\n\n<!-- interface:http -->\nExpose `POST /api/admin/price`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nExpose the `admin_change_price` reducer.\n<!-- /interface -->\n\nUse the same action as the visible price control.\n"},{"id":"ecommerce.progression.price-history-order-hooks","owners":["ecommerce.progression.price-history-specifications"],"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.price-history-features"],"text":"# Price history completed-order interface\n\nUse `orders-toggle`, `order-item`, and `order-total` to inspect the order created at checkout.\nUse `buy-now` inside an `item-card` to create a paid order before a price change.\n\n"},{"id":"ecommerce.progression.price-history-cart-hooks","owners":["ecommerce.progression.price-history-specifications"],"requiresFeatures":["ecommerce.feature.checkout","ecommerce.l2.price-history-features"],"text":"# Price history cart and checkout interface\n\nUse `add-to-cart` inside an `item-card`, `cart-toggle`, `cart-total`, and `checkout-submit`.\n"},{"id":"ecommerce.l2.inventory-dashboard-hooks","owners":["ecommerce.l2.inventory-dashboard"],"text":"# Inventory dashboard application interface\n\nUse `admin-link` to open the administrator area. If `low-stock-list` is on a separate screen\nwithin it, expose `low-stock-link` there to reach it. Use `low-stock-list` for items with 10 units\nor fewer and `low-stock-item` for each item. Use\n`buy-now` inside an `item-card` to create sales that change available stock.\n"},{"id":"ecommerce.l2.order-cancellation-hooks","owners":["ecommerce.l2.order-cancellation-features"],"text":"# Order cancellation application interface\n\nUse `order-status` for an order's state inside its `order-item`. Use `cancel-order` on a pending\norder. Use `catalog-link` to return to the catalog.\n`order-status` reads `pending` until the order ships, `shipped` once it has, and `cancelled`\nafter a cancellation. Later features may add further states after `shipped`.\n\nEach customer `order-item` must have `data-cancel-input` containing a JSON object with\nexactly `orderId`. Use the identifier representation required by the selected stack.\n\n<!-- interface:http -->\nUse `POST /api/orders/:id/cancel`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nUse the `cancel_order` reducer.\n<!-- /interface -->\n"},{"id":"ecommerce.l2.sales-dashboard-hooks","owners":["ecommerce.l2.sales-dashboard"],"text":"# Sales dashboard application interface\n\nOpen the administrator area with `admin-link`. If category totals are on a\nseparate tab or screen, expose `sales-link` there to open them. Omit this control\nwhen the totals are already shown.\n\nUse `category-row` for each product category, including a category with no sales yet, whose\nunits and revenue read 0. Use `category-units` and `category-revenue` inside each row. Use `recommended-list` for signed-out\nbest sellers and `recommended-item` for each item. Use `buy-now` inside an `item-card` to create\nsales activity for the dashboard. Each best seller contains its one-based `recommendation-rank`.\n"},{"id":"ecommerce.l2.recommendations-hooks","owners":["ecommerce.l2.recommendations"],"text":"# Recommendations application interface\n\nUse `recommended-list` for recommendations and `recommended-item` for each item in the list.\nUse `catalog-link` to open the catalog with this list visible.\n"},{"id":"ecommerce.feature.accounts.hooks","owners":["ecommerce.feature.accounts"],"text":"# Account application interface\n\nUse these exact `id` attributes on the corresponding visible controls. They do not prescribe UI\nstructure, data modeling, libraries, or implementation strategy.\n\nFrom the signed-out page, show the sign-up inputs, `signup-toggle`, or\n`signin-toggle`. If sign-up is inside the sign-in dialog, opening `signin-toggle`\nmust reveal the sign-up inputs or `signup-toggle`. That control must reveal the\nsign-up form. No other navigation is required to reach it.\nShow the sign-in inputs or `signin-toggle` on the signed-out page.\nWhile signed in, show `signout` directly or reveal it by clicking `current-user`.\nNo other navigation is required to reach sign-out.\n\n| Element ID | Observable element |\n|---|---|\n| `signup-username` | sign-up username input |\n| `signup-password` | sign-up password input |\n| `signup-submit` | sign-up submit control |\n| `signin-toggle` | control that reveals sign-in |\n| `signup-toggle` | reveals sign-up; available on the signed-out page or after opening `signin-toggle`; omit when sign-up inputs are already visible |\n| `signin-username` | sign-in username input |\n| `signin-password` | sign-in password input |\n| `signin-submit` | sign-in submit control |\n| `current-user` | Active account name, present only while signed in. Do not use this hook for a signed-out message. |\n| `signout` | sign-out control |\n| `auth-error` | visible account error |\n\nAccept any username of up to 48 characters made of letters, digits, and hyphens, and any\npassword of up to 64 characters.\n\nExpose the same account writes used by the UI.\n\nFor bearer-token authentication, expose `window.getSessionToken()` as a synchronous\nfunction that returns the current session's existing token, or `null` when signed out.\nThis hook does not prescribe credential storage. Return the caller's real credential;\ndo not create a separate identity.\n\n<!-- interface:http -->\nUse `POST /api/auth/signup` and `POST /api/auth/signin`. Both accept JSON with\n`username` and `password` fields.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nUse the `signUp` and `signIn` reducers. Both take `username` and `password`, in that order.\n<!-- /interface -->\n"},{"id":"ecommerce.feature.purchasing.hooks","owners":["ecommerce.feature.purchasing"],"text":"# Purchasing application interface\n\nUse `catalog-link` to return to the catalog. Use `buy-now` inside an `item-card` to buy one unit. Use `orders-toggle` to open order history.\nIf an overlay blocks catalog navigation, expose a visible `overlay-close` control\nthat dismisses it before `catalog-link` is used. Screens without a blocking\noverlay need no such control. Dialogs, panels, and ordinary page layouts are all allowed.\nUse `order-item` for each order, containing the names of its purchased items. Inside that\n`order-item`, use `order-total` for its numeric total and `order-status` for its current state. Show `out-of-stock` inside an `item-card` once that item's stock reaches zero.\nUse `buy-error` for a failed purchase.\n\nPut `data-buy-input` on each `item-card`. Its value is a JSON object containing that item's\nserver identifier, for example `{\"itemId\":42}`. The identifier may be a JSON number or string.\nUse the same identifier for the visible buy action.\n\nExpose the same purchase used by `buy-now`.\n\n<!-- interface:http -->\nUse `POST /api/items/:id/buy`, where `:id` is the item identifier.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nUse the `buy_now` reducer with the item identifier.\n<!-- /interface -->\n"},{"id":"ecommerce.feature.cart.hooks","owners":["ecommerce.feature.cart"],"text":"# Cart application interface\n\nUse `catalog-link` to return to the catalog. Use `add-to-cart` inside an `item-card` to add one unit. Use `cart-toggle` to open the cart.\nIf an overlay blocks catalog navigation, expose a visible `overlay-close` control\nthat dismisses it before `catalog-link` is used. Screens without a blocking\noverlay need no such control. Dialogs, panels, and ordinary page layouts are all allowed.\nUse `cart-count` for the total units, `cart-item` for each line, `cart-quantity` for its\nquantity, and `cart-total` for the numeric total. Use `cart-remove` to remove a line and\n`empty-cart` for an empty cart. Keep `cart-count` visible, showing 0, while the cart is empty.\n\nPut `data-buy-input` on each `item-card`. Its value is a JSON object containing that item's\nserver identifier, for example `{\"itemId\":42}`. Put `data-cart-input` on each `cart-item`.\nIts value contains the item identifier, for example `{\"itemId\":42}`. The identifier may be a\nJSON number or string.\n\nExpose the same add and quantity-update operations used by the cart controls.\n\n<!-- interface:http -->\nUse `POST /api/cart`. Put `itemId` in the JSON body.\nUse `PATCH /api/cart/:itemId`. Put `quantity` in the JSON body.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nUse the `add_to_cart` reducer with the item identifier.\nUse the `update_cart_quantity` reducer with the item identifier and quantity.\n<!-- /interface -->\n\nUse `checkout-submit` to check out. Use `orders-toggle` to open order history and `order-item`\nfor each order created by checkout.\n"},{"id":"ecommerce.feature.checkout.hooks","owners":["ecommerce.feature.checkout"],"text":"# Checkout application interface\n\nUse `checkout-submit` to check out and `buy-error` for a failed checkout. Use `orders-toggle`\nto open order history and `order-item` for each order created by checkout.\n\nExpose the same checkout used by `checkout-submit`.\n\n<!-- interface:http -->\nUse `POST /api/checkout`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nUse the `checkout` reducer.\n<!-- /interface -->\n"},{"id":"ecommerce.orders.data","owners":["ecommerce.feature.checkout","ecommerce.feature.purchasing"],"text":"# Order data interface\n\nExpose the following data through the database's native read tools. These names\ndescribe a read interface; tables, collections, or database-native views over the\napplication's current records are allowed. Do not maintain separate copies for\nthis interface. Extra columns are allowed.\n\n- `item(id, name, price)` identifies catalog items and their current prices. These are the same items shown in the catalog.\n- `order_account(id, username)` identifies customer accounts. Do not include passwords or tokens in this interface.\n- `order_header(id, account_id, total, refunded, status)` contains every order from direct purchase or checkout, including cancelled orders. `total` is the amount booked for the order. `refunded` is the amount refunded so far, initially zero.\n- `order_line(id, order_id, item_id, quantity, unit_price)` contains each order's purchased lines and their booked unit prices.\n- When carts are available, `order_cart(account_id, item_id, quantity)` contains their current lines. An empty cart has no lines.\n- When carts and warehouse stock are available, `order_reservation(account_id, item_id, warehouse_id, quantity)` contains any stock held for those carts and already deducted from available `stock.quantity`. If the app does not hold stock for carts, this read interface is empty. This does not require adding stock reservations to the app.\n- When warehouse stock is available, `order_allocation(order_line_id, warehouse_id, quantity)` contains the original warehouse quantities used for each order line. Keep these quantities available after cancellation.\n\nEach `id` is a nonempty string or an exact nonnegative integer. Related identifiers\nrefer to that same `id`; a document may use `_id` when it has no `id`. Item and warehouse\nidentifiers match the existing `item` and `warehouse` data interfaces and the visible\napplication actions. Quantities are whole numbers. Money fields use the same currency\nunits as displayed prices, with at most two decimal places. Status uses the states in\nthe order interface.\n\nThe names and fields above must remain readable by the supplied database credentials\nas features are added. Customer screens and writes must use the same underlying records.\nThis does not require making customer data available to unauthenticated app users.\n"},{"id":"ecommerce.feature.reviews.hooks","owners":["ecommerce.feature.reviews"],"text":"# Review application interface\n\nOn item details, show the review form for eligible customers or expose\n`review-toggle` to open it. Omit this control when the form is already shown.\n\nUse `review-rating` on an input or select with values 1 through 5, `review-input` for the comment, and\n`review-submit` to submit it. Put the item's server identifier in\n`data-review-item-id` on `review-submit`. Use `review-average` for the numeric average,\n`review-item` for each visible review, and `review-error` for a failed submission.\n\nExpose the same review operation used by `review-submit`.\n\n<!-- interface:http -->\nUse `POST /api/items/:id/reviews`, where `:id` is the item identifier. Send `rating` and\n`comment` in the request body.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nUse the `submit_review` reducer with the item identifier, rating, and comment.\n<!-- /interface -->\n"},{"id":"ecommerce.feature.warehouse-admin.hooks","owners":["ecommerce.feature.warehouse-admin"],"text":"# Warehouse administration application interface\n\nUse `admin-link` to open `admin-panel`. Use `admin-item-row` for each item and `admin-stock` for\nits numeric total stock. Use `admin-warehouse-item` for each warehouse. Use `admin-location-row`\nfor every item in every warehouse, including zero quantities; the row shows the item name and\nthe warehouse name, with\n`admin-location-qty` for its quantity. Use\n`restock-input` and `restock-submit` inside that row. Use `admin-revenue` for numeric total\nrevenue.\n\nKeep all item, warehouse, and holding rows available in the open admin panel, without pagination.\n\nPut a `data-restock-input` attribute on each `admin-location-row`. Its value is a JSON object\nwith exactly `itemId`, `warehouseId`, and a valid one-unit `quantity`. Identifiers can be JSON numbers or\nstrings.\n\nUse the same restock action as the visible control.\n\n## Stock data interface\n\nExpose singular tables `item(id, name, price)`, `warehouse(id, name)`, and\n`stock(item_id, warehouse_id, quantity)` for direct database access.\n`stock.item_id` and `stock.warehouse_id` reference `item.id` and `warehouse.id`; in a document\nstore they hold the referenced document's `id` value, or its `_id` when it has no `id`. Keep\nthese tables readable and writable with the database's own tools.\n\n<!-- interface:http -->\nExpose `POST /api/admin/restock`. The JSON body has the same fields as `data-restock-input`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nExpose `admin_restock` with arguments in this order: `itemId: u64`, `warehouseId: u64`,\n`quantity`.\n<!-- /interface -->\n"},{"id":"ecommerce.progression.support-history-hooks","owners":["ecommerce.progression.support-history"],"text":"## Customer support history controls\n\n`support-link` opens support. Use `support-ticket` for each ticket in the history, showing\nits subject. The intake form stays reachable from the same control.\n"},{"id":"ecommerce.progression.support-intake-hooks","owners":["ecommerce.progression.support-intake"],"text":"## Support intake controls\n\nUse `support-link` to open support. Use `support-email`, `support-subject`, and\n`support-message` for the ticket fields. Use `support-submit` to submit the ticket and\n`support-reference` to show its reference.\n"},{"id":"ecommerce.progression.support-triage-hooks","owners":["ecommerce.progression.support-triage"],"text":"## Support triage controls\n\nOpen the staff area with `staff-link`. If its ticket queue is on a separate tab or\nscreen, expose `support-queue-link` there to open it. Omit this control when the\neditable ticket controls are already shown.\n\nUse `support-ticket` for each ticket in the staff view. Within a ticket, use\n`support-assignee`, `support-priority`, and `support-status-input` for the editable fields.\nUse `support-update` to apply the changes. Use `support-status` to show the current status.\n`support-assignee` takes the assignee's username; if it is a select, its option values are the\nusernames. `support-priority` offers `low`, `normal`, and `high` as its option values.\n\n`support-status-input` offers the statuses `open`, `in progress`, and `resolved` as its option\nvalues; `support-status` shows the one chosen.\n"},{"id":"ecommerce.progression.fulfilment-queue-hooks","owners":["ecommerce.progression.fulfilment-queue"],"text":"# Fulfilment application interface\n\n| Element ID | Required element |\n| --- | --- |\n| `staff-link` | Opens the fulfilment area. |\n| `fulfilment-panel` | Contains fulfilment tools, including an empty queue; not a loading or error message. |\n| `queue-depth` | Shows the number of pending orders. |\n| `queue-item` | Shows one pending order and names its items. |\n| `queue-warehouse` | Shows the selected warehouse inside its `queue-item`. |\n| `ship-submit` | Marks the order in its `queue-item` as shipped. |\n\nOn `fulfilment-panel`, expose `data-submit-state` for the latest shipping submission:\n`idle` initially, `pending` immediately when submitted, `succeeded` only after the server\nconfirms success, or `failed` after rejection or transport failure. Keep the terminal state\non the panel when the shipped row disappears. A new submission must replace the old state.\n\n`order-status` reads `pending` until the order ships, `shipped` once it has, and `cancelled`\nafter a cancellation. Later features may add further states after `shipped`.\n\nEach customer `order-item` must have `data-ship-input` containing a JSON object with\nexactly `orderId`.\n\nUse the identifier representation required by the selected stack.\n\n<!-- interface:http -->\nUse `POST /api/fulfilment/ship` with `{ \"orderId\": ... }`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nUse the `ship_order` reducer.\n<!-- /interface -->\n"},{"id":"ecommerce.progression.promotion-rules-hooks","owners":["ecommerce.progression.promotion-rules"],"text":"# Promotion rule application interface\n\nUse `staff-link` to open the staff area. In that area, use `promotions-link` for promotion\nmanagement. Use `promotion-code`, `promotion-discount`,\n`promotion-start`, `promotion-end`, `promotion-limit`, and `promotion-submit` to create a rule.\nList rules as `promotion-item` elements and expose the saved values with the matching field IDs.\n\nExpose the same rule creation used by `promotion-submit`.\n\n<!-- interface:http -->\nUse `POST /api/promotions` with a JSON object containing `code` (string),\n`discountPercent` (number), `startMicros` and `endMicros` (integer numbers of microseconds\nsince the Unix epoch), and `usageLimit` (positive integer).\n<!-- /interface -->\n\n<!-- interface:reducer -->\nUse the `create_promotion` reducer with arguments in this order: `code: string`,\n`discountPercent: f64`, `startMicros: i64`, `endMicros: i64`, `usageLimit: u32`.\nBoth time arguments are microseconds since the Unix epoch.\n<!-- /interface -->\n\nOn a listed rule, `promotion-start` and `promotion-end` show the dates as entered, in ISO\n`YYYY-MM-DD` form.\n"},{"id":"ecommerce.progression.notification-preferences-hooks","owners":["ecommerce.progression.notification-preferences"],"text":"# Notification preference application interface\n\nSaving may leave settings open. Use `catalog-link` to return to the catalog. If\na settings overlay blocks navigation, expose `overlay-close` to dismiss it.\n\nMake `notification-settings` available from the catalog while signed in, without first opening another area. Use it to open the settings. Use `notification-order` and\n`notification-stock` for the choices, and `notification-save` to save them. Each choice exposes\nits current state in `data-state` as `on` or `off`.\nBoth choices start `off` for a new account. Activating `notification-order` or\n`notification-stock` switches it between `on` and `off`.\n"},{"id":"ecommerce.l3.reservation-hooks","owners":["ecommerce.l3.reservations-features"],"text":"# Reservation application interface\n\nUse `cart-reservation-timer` inside a `cart-item` for the remaining reservation time in seconds.\nUse `cart-item-expired` inside an expired cart line and `cart-expired-notice` after cart\nexpiration.\n"},{"id":"ecommerce.progression.managed-support-hooks","owners":["ecommerce.progression.managed-support"],"text":"# Managed support application interface\n\n## Managed support controls\n\nUse `support-ticket` for each case and set `data-entity-id` to that case's server identifier.\nWithin a case, use `support-status` for the current status, `support-reply` for the reply field,\n`support-reply-submit` to send a reply, and `support-reply-item` for each reply.\n\nExpose the same reply operation used by `support-reply-submit`.\n\n<!-- interface:http -->\nUse `POST /api/support/:id/replies`, where `:id` is the case identifier from `data-entity-id`.\nThe JSON body is `{ \"body\": \"<reply text>\" }`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nUse the `reply_support` reducer with arguments in this order: `ticketId: u64`, `body: string`.\nRender `data-entity-id` as the decimal case identifier without precision loss.\n<!-- /interface -->\n"},{"id":"ecommerce.l3.scheduled-restock-hooks","owners":["ecommerce.l3.scheduled-restocks-features"],"text":"# Scheduled restock application interface\n\nUse `admin-link` to open the administrator area. If these controls are on a separate screen within\nit, expose `restocks-link` there to reach them. Use `schedule-restock-item`, `schedule-restock-warehouse`, `schedule-restock-qty`, and\n`schedule-restock-delay` for the inputs. Use `schedule-restock-submit` to schedule the restock.\nSet its `data-action-input` to a JSON object with exactly `item`, `warehouse`, `quantity`, and\n`delaySeconds`. `item` and `warehouse` are their names as strings; `quantity` and\n`delaySeconds` are JSON integers. Use `pending-restock-item` for each pending row and set its\n`data-entity-id` to the restock's server identifier, written as a decimal number.\nEach row contains the item name and sets `data-quantity` to its integer quantity. Use\n`pending-restock-remaining` for its remaining seconds, `pending-restock-cancel` to cancel it,\nand `stock-ledger-entry` for a completed stock movement.\n\n<!-- interface:http -->\nExpose `POST /api/admin/scheduled-restocks` and `DELETE /api/admin/scheduled-restocks/:id`.\nThe POST body has the same fields as `data-action-input`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nExpose `schedule_restock` with arguments in this order: `item: string`, `warehouse: string`,\n`quantity: u32`, `delaySeconds: u32`; and `cancel_scheduled_restock` with `restockId: u64`.\n<!-- /interface -->\n"},{"id":"ecommerce.l3.order-delivery-hooks","owners":["ecommerce.l3.order-delivery-features"],"text":"# Order delivery application interface\n\nUse `completed-order-item` for each completed order in the staff view. Use\n`completed-order-status` inside it for the current state. This extends the order lifecycle:\nafter `shipped`, `order-status` and `completed-order-status` read `delivered`.\n"},{"id":"ecommerce.l3.cart-expiration-hooks","owners":["ecommerce.l3.cart-expiration-features"],"text":"# Cart expiration application interface\n\nUse `cart-reservation-timer` inside a `cart-item` for the remaining reservation time in seconds.\nUse `cart-item-expired` inside an expired cart line and `cart-expired-notice` after cart\nexpiration.\n"},{"id":"ecommerce.progression.promotion-checkout-hooks","owners":["ecommerce.progression.promotion-checkout"],"text":"# Promotion checkout application interface\n\nUse `cart-promotion` for the code and `apply-promotion` to apply it. Use `promotion-error` when a\ncode is refused. Expose the saved discount as `order-discount` inside its `order-item`.\n"},{"id":"ecommerce.progression.stock-alert-hooks","owners":["ecommerce.progression.stock-alerts"],"text":"# Stock alert application interface\n\nUse `stock-alert` inside an unavailable `item-card` to request an alert. Use\n`notifications-toggle` to open notifications and `notification-item` for each alert.\nOn that `item-card`, expose `data-submit-state` for the latest stock-alert request:\n`idle` initially, `pending` immediately when submitted, `succeeded` only after the server\nconfirms success, or `failed` after rejection or transport failure. Keep the terminal state\nif the request button disappears. A new submission must replace the old state.\nUse `notifications-panel` for the opened notification view, including while its contents\nload. Set its `aria-busy` attribute to `false` only when the signed-in account's contents\nhave loaded successfully, including an empty list; keep it `true` while loading or after\na failed read. Its toggle may also close it.\nWithin a delivered stock alert, expose `stock-alert-delivery` containing the item's\ndisplayed name. A pending request must not expose `stock-alert-delivery`.\n\nUse `catalog-link` to return to the catalog. If an overlay blocks navigation,\nexpose a visible `overlay-close` control that dismisses it before navigation.\nScreens without a blocking overlay do not need this control.\n"},{"id":"ecommerce.l3.order-return-hooks","owners":["ecommerce.l3.order-returns-features"],"text":"# Order return application interface\n\nUse `return-item` inside an `order-item` for each item that can be returned. After a return,\nthe same `order-item` contains the word `returned`.\n\nEach ordinary item has an `order-line` containing its name, including while pending.\nSet `data-return-input` on that line to JSON with `orderId` and `itemId`, using the\nidentifiers accepted by the return action.\n\n<!-- interface:http -->\n`returnItem` is `POST /api/orders/{orderId}/items/{itemId}/return`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\n`returnItem` is `return_order_item(orderId, itemId)`.\n<!-- /interface -->\n\nThe existing `orders-toggle`, `order-item`, `item-stock`, `admin-revenue`, and `catalog-link`\ninterfaces expose the order, stock, and accounting results.\n"},{"id":"ecommerce.progression.faceted-search-hooks","owners":["ecommerce.progression.faceted-search"],"text":"# Faceted search application interface\n\n| Element ID | Required element |\n| --- | --- |\n| `category-filter` | Sets the category filter; a text input, or a `select` whose option values are the category names. |\n| `minimum-price` | Sets the inclusive minimum price. |\n| `maximum-price` | Sets the inclusive maximum price. |\n| `in-stock-filter` | Toggles the in-stock-only filter, which starts off. |\n| `search-results` | Contains the filtered page; with no search text and no filter selected, the current page of the full catalog. |\n| `item-card` | Shows one result inside `search-results`. |\n| `search-next-page` | Opens the next page. |\n| `search-previous-page` | Opens the previous page. |\n\nIf a `filter-apply` control exists, activating it applies the filters; otherwise results update\nas each filter changes.\n\nSearch text or any active filter selects alphabetical ordering. With neither, use\npurchase ranking and break ties by item name. Clearing all search text and filters\nrestores purchase ranking. Both modes can use the same rendered list.\n"},{"id":"ecommerce.progression.order-support-hooks","owners":["ecommerce.progression.order-support"],"text":"# Order-linked support application interface\n\n## Order-linked support controls\n\nWithin a `support-ticket`, use `support-order-option` for each order that the customer can attach,\n`support-link-order` to attach the selected order, and `support-order` for the attached order. The\nlink action must expose its input in `data-action-input` for the named\n`linkSupportOrder` application action.\n\n<!-- interface:http -->\n`linkSupportOrder` is `POST /api/support/cases/{caseId}/order`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\n`linkSupportOrder` is the `link_support_order(caseId, orderId)` reducer.\n<!-- /interface -->\n"},{"id":"ecommerce.progression.personalized-recommendation-hooks","owners":["ecommerce.progression.personalized-recommendations"],"text":"# Personalized recommendation application interface\n\nUse `recommendations` for the ordered recommendation list. Use `recommended-item` for each\nitem in that list. Each item contains its item name. Use `recommendation-rank` inside each item\nfor its one-based position in the list.\n"},{"id":"ecommerce.progression.promotion-reporting-hooks","owners":["ecommerce.progression.promotion-reporting"],"text":"# Promotion reporting application interface\n\nUse `promotions-link` to open the staff view. Use `promotion-report` for each promotion and\n`promotion-redemptions` and `promotion-revenue` for its totals.\n"},{"id":"ecommerce.feature.store-credit.interface","owners":["ecommerce.feature.store-credit"],"text":"# Store credit interface\n\nOpen customer credit with `credit-link`. The `credit-panel` exposes the signed-in account's `data-account-id` and `aria-busy=\"false\"` when loaded. Show `credit-balance` in major currency units and one `credit-entry` per movement.\n\nStaff use `credit-customer`, `credit-amount-input` (major units), `credit-reference-input`, and `credit-grant`. The grant control exposes `data-action-input` as JSON with `accountId`, `amountMinor`, and `reference`.\n\nUse `credit-checkout` in the cart. Each `order-item` shows `payment-credit-amount` and `payment-external-amount` in major units. Their sum is `payment-amount`.\n\n<!-- interface:http -->\n`grantCredit` is `POST /api/staff/credit` with `accountId`, `amountMinor`, and `reference`.\n`checkoutCredit` is `POST /api/checkout/credit` with no body fields.\n<!-- /interface -->\n\n<!-- interface:reducer -->\n`grantCredit` is `grant_credit(accountId, amountMinor, reference)`.\n`checkoutCredit` is `checkout_credit()`.\n<!-- /interface -->\n"},{"id":"ecommerce.feature.subscriptions.interface","owners":["ecommerce.feature.subscriptions"],"text":"# Subscription interface\n\nOpen with `subscriptions-link`. Use `subscription-item-input` (item name), `subscription-quantity-input`, `subscription-interval-input` (whole seconds, minimum 30), `subscription-deliveries-input` (1–12), and `subscription-create`.\n\nThe loaded `subscriptions-panel` has `aria-busy=\"false\"`. Each `subscription-row` includes the item name and `subscription-status`: `active`, `paused`, `cancelled`, or `complete`. Use `subscription-pause`, `subscription-resume`, and `subscription-cancel`; each exposes `data-action-input` with `subscriptionId`. Each processed slot has one `subscription-delivery` inside the row, with `subscription-delivery-status` (`paid` or `skipped`). `subscription-total` shows the sum of its recorded payments in major currency units. Ordinary orders and payment records include their item names.\n\n<!-- interface:http -->\n`pauseSubscription` is `POST /api/subscriptions/{subscriptionId}/pause`.\n`resumeSubscription` is `POST /api/subscriptions/{subscriptionId}/resume`.\n`cancelSubscription` is `POST /api/subscriptions/{subscriptionId}/cancel`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\n`pauseSubscription` is `pause_subscription(subscriptionId)`.\n`resumeSubscription` is `resume_subscription(subscriptionId)`.\n`cancelSubscription` is `cancel_subscription(subscriptionId)`.\n<!-- /interface -->\n"},{"id":"ecommerce.progression.delivery-notification-hooks","owners":["ecommerce.progression.delivery-notifications"],"text":"# Delivery notification application interface\n\nUse `notifications-toggle` to open notifications. Use `notification-item` for each notification\nand `notification-unread-count` for the unread total.\n\nThe `notifications-panel` has `aria-busy=\"false\"` only when the signed-in account's\nnotifications have loaded. Keep the panel present when the list is empty.\n"},{"id":"ecommerce.progression.support-refund-hooks","owners":["ecommerce.progression.support-refunds"],"text":"# Support refund application interface\n\n## Support refund controls\n\nWithin a `support-ticket`, use `support-refund` for the refund action and\n`support-refund-total` for the recorded refund amount. The refund action must expose its input in\n`data-action-input` for the named `supportRefund` application action. The `support-ticket` also\nexposes the same JSON `{ \"caseId\": \"...\" }` in `data-refund-input` so the case remains addressable\nafter its refund button is disabled or removed. Within an `order-item`, use\n`order-refund-total` for the refunded amount and `refund-entry` for each refund record. Each\n`refund-entry` includes the order item name.\n\n<!-- interface:http -->\n`supportRefund` is `POST /api/support/cases/{caseId}/refund`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\n`supportRefund` is the `support_refund(caseId)` reducer.\n<!-- /interface -->\n"},{"id":"ecommerce.progression.automatic-reorder-hooks","owners":["ecommerce.progression.automatic-reorder"],"text":"# Automatic reorder application interface\n\nUse these application interface names:\n\n- `reorder-link` opens the automatic reorder rules for warehouse staff.\n- `reorder-item`, `reorder-threshold`, and `reorder-quantity` identify the rule inputs.\n- `reorder-submit` saves the rule.\n- `reorder-rule-item` identifies each saved rule, sets `data-entity-id` to the rule's item\n identifier, and contains its item name, threshold, quantity, and current state.\n Set `data-threshold` and `data-quantity` to their integer values. Set\n `data-action-input` on `reorder-submit` to JSON with `itemId`, `threshold`, and `quantity`\n for the current form values.\n\nSaving a rule is the named `saveReorderRule` application action.\n\n<!-- interface:http -->\n`saveReorderRule` is `PUT /api/reorders/{itemId}` with `threshold` and `quantity` in the body.\n<!-- /interface -->\n\n<!-- interface:reducer -->\n`saveReorderRule` is the `save_reorder_rule(itemId, threshold, quantity)` reducer.\n<!-- /interface -->\n\nUse `buy-now` inside an `item-card` to create stock changes that evaluate a reorder rule.\n"},{"id":"ecommerce.progression.cart-recovery-hooks","owners":["ecommerce.progression.cart-recovery"],"text":"# Cart recovery application interface\n\nUse these application interface names:\n\n- `expired-cart` identifies the expired cart.\n- `restore-cart` restores that cart.\n- `cart-restore-warning` lists the names of items that could not be restored.\n\nThe existing `cart-item` control identifies each item restored to the active cart.\n"},{"id":"ecommerce.progression.recommendation-feedback-hooks","owners":["ecommerce.progression.recommendation-feedback"],"text":"# Recommendation feedback application interface\n\nUse `dismiss-recommendation` inside each `recommended-item`.\n"},{"id":"ecommerce.feature.split-tender-refunds.interface","owners":["ecommerce.feature.split-tender-refunds"],"text":"# Split refund interface\n\nReuse `support-refund` and its existing named action. Each `refund-entry` shows `refund-credit-amount` and `refund-external-amount` in major currency units. Show the restored credit in the existing credit balance and history.\n"},{"id":"ecommerce.interface.product-bundles","owners":["ecommerce.feature.product-bundles"],"text":"# Product bundle interface\n\nThe catalog has a `bundles-link`. The bundle panel contains `bundle-card` rows with\n`bundle-name`, numeric `bundle-price`, and `bundle-component` rows. Each component row has\n`bundle-component-name` and numeric `bundle-component-quantity`. Each component row also\nexposes its quantity in `data-quantity`.\n\nCatalog staff use `bundle-name-input`, `bundle-price-input` (currency units), and\n`bundle-components-input` (JSON array of `{ \"item\": \"product name\", \"quantity\": 1 }`),\nthen `bundle-save`. Saving an existing name edits that bundle. Each `bundle-card` exposes\n`data-bundle-input` as JSON `{ \"bundleId\": \"...\" }`.\nThe save button exposes `data-bundle-save-input` with `{ name, price, componentsJson }`\nfrom the current form values.\n\nUse the same application write for the visible form and this named action:\n\n<!-- interface:http -->\nSave a bundle with `POST /api/bundles` and `{ name, price, componentsJson }`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nSave a bundle with `save_bundle(name: string, price: number, componentsJson: string)`.\n<!-- /interface -->\n\nEach component uses an existing product's exact name. `componentsJson` contains the\ncomponent array as a JSON string.\n"},{"id":"ecommerce.interface.bundle-checkout","owners":["ecommerce.feature.bundle-checkout"],"text":"# Bundle checkout interface\n\nUse `bundle-add-to-cart` inside `bundle-card`. A bundle cart line uses the existing\n`cart-item`, `cart-reservation-timer`, and `cart-item-expired` interfaces, with\n`bundle-remove` to remove it. `checkout-submit` buys the cart through the existing checkout\naction. The existing `order-item` and `payment-amount` show the bundle name and price paid.\n\n<!-- interface:http -->\nAdd one bundle with `POST /api/cart/bundles` and `{ bundleId }`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nAdd one bundle with `add_bundle_to_cart(bundleId: u64)`.\n<!-- /interface -->\n\nUse the same application action as the visible control. The `data-bundle-input` attribute\nsupplies its bundle ID. The cart and order interfaces remain shared with individual products.\n"},{"id":"ecommerce.interface.bundle-returns","owners":["ecommerce.feature.bundle-returns"],"text":"# Bundle return interface\n\nUse `return-bundle` inside the existing `order-item`. Mark each returned bundle line\n`returned`. The order's `order-status` reads `returned` when all its lines have been\nreturned; otherwise keep its current fulfilment status. `bundle-refund-amount` shows\nthe refunded amount in currency units.\nEach bundle `order-item` exposes `data-bundle-return-input` as JSON `{ \"orderId\": \"...\" }`.\n\n<!-- interface:http -->\nReturn a whole bundle with `POST /api/bundle-orders/:orderId/return`.\n<!-- /interface -->\n\n<!-- interface:reducer -->\nReturn a whole bundle with `return_bundle(orderId: u64)`.\n<!-- /interface -->\n\nUse the same application action as the visible control.\n"}],"mode":"action","requirements":[{"id":"ecommerce.progression.fresh","owners":["recipe"],"text":"## New application\n\nBuild an ecommerce application from the product work below. Use `Storefront`\nas the visible page title. Use the application interface names where they are\nprovided. Start with no orders or purchase history.\n\n"},{"id":"ecommerce.progression.upgrade","owners":["recipe"],"text":"## Existing application\n\nAdd the current product work to the existing ecommerce application. Preserve its data;\ndo not add sample orders or purchase history while adding features.\n"},{"id":"ecommerce.l2.inventory-dashboard","owners":["ecommerce.l2.inventory-dashboard"],"text":"## Inventory dashboard\n\nGive administrators a low-stock view. It lists items with 10 units or fewer, most urgent first.\n"},{"id":"ecommerce.l2.sales-dashboard","owners":["ecommerce.l2.sales-dashboard"],"text":"## Sales dashboard\n\nGive customers and administrators sales views. Category totals show units sold and revenue for\neach category. Signed-out visitors see best sellers on the storefront.\n"},{"id":"ecommerce.l2.recommendations","owners":["ecommerce.l2.recommendations"],"text":"## Recommendations\n\nShow customers a recommended-for-you list on the catalog page. It recommends items from categories the customer has\nbought from and excludes items already in the cart.\n"},{"id":"ecommerce.feature.accounts.requirement","owners":["ecommerce.feature.accounts"],"text":"## Accounts\n\nVisitors can create an account with a username and password. Returning users can sign in with\nthose credentials, see which account is active, and sign out. Show a useful error for a taken\nusername or an incorrect password.\n"},{"id":"ecommerce.feature.catalog-items.requirement","owners":["ecommerce.feature.catalog-items"],"text":"## Catalog items\n\nShow public catalog items. Each item shows its name, price, and total stock.\n"},{"id":"ecommerce.feature.catalog-discovery.requirement","owners":["ecommerce.feature.catalog-discovery"],"text":"## Catalog discovery\n\nShow the ten most-purchased items. Break ties by item name. Search matches any part of\nan item name, without regard to case, across the full catalog.\n"},{"id":"ecommerce.feature.purchasing.requirement","owners":["ecommerce.feature.purchasing"],"text":"## Purchasing and orders\n\nA signed-in customer can buy one unit of an available item. The purchase reduces stock and\ncreates an order for that customer at the price paid. Their order history shows the newest\norders first, including items, quantities, prices, and totals.\n"},{"id":"ecommerce.feature.cart.requirement","owners":["ecommerce.feature.cart"],"text":"## Cart\n\nA signed-in customer has one cart. They can add an item, change its quantity, remove it, and\nsee the total. Adding the same item again increases the existing line quantity.\n"},{"id":"ecommerce.feature.checkout.requirement","owners":["ecommerce.feature.checkout"],"text":"## Checkout\n\nCheckout creates one order, reduces stock for every line, and empties the cart. Show an\nexplanation if checkout fails.\n"},{"id":"ecommerce.feature.reviews.requirement","owners":["ecommerce.feature.reviews"],"text":"## Reviews\n\nCustomers can rate items they purchased from one to five and add a comment. Show reviews\nand the average rating on the item detail.\n"},{"id":"ecommerce.feature.warehouse-admin.requirement","owners":["ecommerce.feature.warehouse-admin"],"text":"## Warehouse administration\n\nProvide an administration area that lists every item, both warehouses, and the quantity held in\neach warehouse. An administrator can add units to a selected item and warehouse. The area also\nshows total revenue across orders.\n"},{"id":"ecommerce.spec.access-control.purchasing","owners":["ecommerce.spec.access-control"],"requiresFeatures":["ecommerce.feature.purchasing"],"text":"## Access control: purchasing\n\nTreat identity as server-enforced authority, not UI decoration. Unauthenticated\ncallers cannot purchase, one account cannot place an order for another account,\nand order history is visible only to its owner. Knowing a username never grants\naccess to that account.\n\n"},{"id":"ecommerce.spec.access-control.warehouse-admin","owners":["ecommerce.spec.access-control"],"requiresFeatures":["ecommerce.feature.warehouse-admin"],"text":"## Access control: warehouse administration\n\nCustomer accounts cannot perform warehouse-administration writes. Enforce this\non the server even if the corresponding controls are hidden in the UI.\n\n"},{"id":"ecommerce.spec.access-control.reviews","owners":["ecommerce.spec.access-control"],"requiresFeatures":["ecommerce.feature.reviews"],"text":"## Access control: reviews\n\nOnly a customer who purchased an item may review it. Enforce this on the server\ninstead of relying on whether the review form is visible.\n\n"},{"id":"ecommerce.spec.access-control.cart","owners":["ecommerce.spec.access-control"],"requiresFeatures":["ecommerce.feature.cart"],"text":"## Access control: cart\n\nOne account cannot read or change another account's cart. Refuse a cart request\nwhose quantity is negative without changing state.\n\n"},{"id":"ecommerce.progression.staff-access","owners":["ecommerce.progression.staff-access"],"text":"## Staff access\n\nStaff and administrators can sign in and use staff areas.\n"},{"id":"ecommerce.spec.state-durability.accounts","owners":["ecommerce.spec.state-durability"],"requiresFeatures":["ecommerce.feature.accounts"],"text":"## State durability: accounts\n\nA signed-in session survives a page reload as the same account.\n\n"},{"id":"ecommerce.spec.state-durability.account-data","owners":["ecommerce.spec.state-durability"],"requiresFeatures":["ecommerce.feature.accounts","ecommerce.feature.cart"],"text":"## State durability: account data\n\nThe same account keeps its cart and orders across reload and connection loss.\nAfter reconnect it has current state without another sign-in. Restarting the\napplication must not duplicate starting data or reset state users changed.\n\n"},{"id":"ecommerce.spec.state-durability.checkout-crash","owners":["ecommerce.spec.state-durability"],"requiresFeatures":["ecommerce.feature.checkout"],"text":"After the application or database process restarts, an interrupted checkout leaves\neither its unchanged cart or one complete order with the cart cleared. A checkout\nreported as complete and the account's earlier orders remain recorded correctly.\nThe application can accept new checkouts after recovery.\n"},{"id":"ecommerce.progression.customer-profile","owners":["ecommerce.progression.customer-profile"],"text":"## Customer profile\n\nA signed-in customer can save and view their name and shipping address.\n"},{"id":"ecommerce.progression.support-intake","owners":["ecommerce.progression.support-intake"],"text":"## Support intake\n\nAnyone can open a support ticket with contact details, a subject, and a message. Return a\nreference that the visitor can use to identify the ticket.\n"},{"id":"ecommerce.spec.live-state.catalog-purchasing","owners":["ecommerce.spec.live-state"],"requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing"],"text":"## Live state: catalog and purchasing\n\nStock and best-seller ranking update every affected open storefront without a\nreload, including signed-out storefronts.\n\n"},{"id":"ecommerce.spec.live-state.cart","owners":["ecommerce.spec.live-state"],"requiresFeatures":["ecommerce.feature.cart"],"text":"## Live state: cart\n\nThe same account open in two clients sees one current cart; a change in either\nclient reaches the other without a reload.\n\n"},{"id":"ecommerce.spec.live-state.reviews","owners":["ecommerce.spec.live-state"],"requiresFeatures":["ecommerce.feature.reviews"],"text":"## Live state: reviews\n\nReviews and average ratings update affected open item views without a reload.\nA view opened while a review is submitted converges to the current review list.\n\n"},{"id":"ecommerce.spec.live-state.warehouse-admin","owners":["ecommerce.spec.live-state"],"requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.warehouse-admin"],"text":"## Live state: warehouse administration\n\nRestocking updates warehouse quantities, total item stock, and open storefronts\nwithout a reload.\n\n"},{"id":"ecommerce.progression.staff-roles","owners":["ecommerce.progression.staff-roles"],"text":"## Staff roles\n\nAn administrator can assign a role to an existing staff account. The `admin` role grants\nadministrator access. The `staff` and `inventory` roles grant staff access without\nadministrator access.\n"},{"id":"ecommerce.progression.catalog-management","owners":["ecommerce.progression.catalog-management"],"text":"## Catalog management\n\nAuthorized staff can add a product with named variants. The new product and its variants appear\nin the public catalog.\n"},{"id":"ecommerce.spec.concurrency-safety.purchasing","owners":["ecommerce.spec.concurrency-safety"],"requiresFeatures":["ecommerce.feature.purchasing"],"text":"## Concurrency safety: purchasing\n\nStock never becomes negative and only one customer can receive the last unit.\n\n"},{"id":"ecommerce.spec.concurrency-safety.restocking","owners":["ecommerce.spec.concurrency-safety"],"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"text":"## Concurrency safety: restocking\n\nConcurrent restocks and purchases preserve every accepted stock change.\n\n"},{"id":"ecommerce.spec.concurrency-safety.checkout","owners":["ecommerce.spec.concurrency-safety"],"requiresFeatures":["ecommerce.feature.checkout"],"text":"## Concurrency safety: checkout\n\nRepeating or racing checkout for the same cart creates only one order.\n\n"},{"id":"ecommerce.progression.payment-records","owners":["ecommerce.progression.payment-records"],"text":"## Payment records\n\nShow the payment status and amount paid on each order.\n"},{"id":"ecommerce.progression.staff-activity","owners":["ecommerce.progression.staff-activity"],"text":"## Staff activity history\n\nStaff can inspect a history of administrative changes. Each entry shows the staff member,\naction, subject, and time.\n"},{"id":"ecommerce.spec.transactional-integrity.reviews","owners":["ecommerce.spec.transactional-integrity"],"requiresFeatures":["ecommerce.feature.reviews"],"text":"## Transactional integrity: reviews\n\nA customer has at most one review per item. A later submission must not create\na duplicate; it may update the existing review or be refused.\n\n"},{"id":"ecommerce.spec.transactional-integrity.purchasing","owners":["ecommerce.spec.transactional-integrity"],"requiresFeatures":["ecommerce.feature.purchasing"],"text":"## Transactional integrity: purchasing\n\nThe server controls prices and order attribution rather than trusting client\nvalues. Historical order prices do not change.\n\n"},{"id":"ecommerce.spec.transactional-integrity.warehouse-accounting","owners":["ecommerce.spec.transactional-integrity"],"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"text":"## Transactional integrity: warehouse accounting\n\nEvery stock reduction caused by a sale has its matching order, revenue equals\nthe orders' recorded totals, and fresh clients agree with those results.\n\n"},{"id":"ecommerce.spec.external-data-sync.requirement","owners":["ecommerce.spec.external-data-sync"],"requiresFeatures":["ecommerce.feature.warehouse-admin"],"text":"## External data synchronization\n\nOther systems may write stock directly without calling the application. Open\npages and newly loaded pages must converge to a direct stock correction,\nincluding a correction made while the application server is down.\n"},{"id":"ecommerce.progression.fulfilment-queue","owners":["ecommerce.progression.fulfilment-queue"],"text":"# Fulfilment operations\n\nAdd a fulfilment area for staff and administrators. Show pending orders, their items, and the\nwarehouse that will ship them. Staff and administrators can mark an order as shipped. Show the\nnew status in the fulfilment area and the customer's order history.\n"},{"id":"ecommerce.l2.stock-transfer","owners":["ecommerce.l2.stock-transfers-features"],"text":"## Stock transfers\n\nAn administrator can move units of an item from one warehouse to another.\n"},{"id":"ecommerce.l2.order-cancellation","owners":["ecommerce.l2.order-cancellation-features"],"text":"## Order cancellation\n\nA customer can cancel an order before it ships. Refund the purchase, return its stock to\nthe supplying warehouse, and show its cancelled status in order history.\n"},{"id":"ecommerce.l2.price-history","owners":["ecommerce.l2.price-history-features"],"text":"## Price history\n\nAn administrator can change an item's price. Show the price in the public catalog.\n"},{"id":"ecommerce.progression.price-history-orders","owners":["ecommerce.progression.price-history-specifications"],"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.price-history-features"],"text":"## Price history: completed orders\n\nCompleted orders keep the price paid at checkout. A later price change does not alter a receipt\nor revenue already recorded.\n\n"},{"id":"ecommerce.progression.price-history-cart-checkout","owners":["ecommerce.progression.price-history-specifications"],"requiresFeatures":["ecommerce.feature.checkout","ecommerce.l2.price-history-features"],"text":"## Price history: cart and checkout\n\nEvery open cart shows the new price without a reload. Checkout uses the current price.\n"},{"id":"ecommerce.progression.cancellation-queue","owners":["ecommerce.progression.cancellation-queue-specifications"],"requiresFeatures":["ecommerce.l2.order-cancellation-features","ecommerce.progression.fulfilment-queue"],"text":"## Fulfilment queue integration\n\nWhen fulfilment is available, cancelling a pending order also removes it from the fulfilment\nqueue.\n"},{"id":"ecommerce.progression.cancellation-accounting","owners":["ecommerce.progression.cancellation-accounting-specifications"],"requiresFeatures":["ecommerce.l2.order-cancellation-features"],"text":"## Cancellation accounting\n\nRevenue equals the sum of orders that remain paid. Cancellation removes the cancelled amount.\n\n"},{"id":"ecommerce.progression.support-triage","owners":["ecommerce.progression.support-triage"],"text":"## Support triage\n\nStaff can view new support tickets, assign a ticket, set its priority, and change its status.\n"},{"id":"ecommerce.progression.price-accounting","owners":["ecommerce.progression.price-accounting-specifications"],"requiresFeatures":["ecommerce.l2.price-history-features"],"text":"## Price accounting\n\nLater catalog price changes do not change the amount recorded on an existing order.\n"},{"id":"ecommerce.progression.support-history","owners":["ecommerce.progression.support-history"],"text":"## Customer support history\n\nSigned-in customers can see their support ticket history.\n"},{"id":"ecommerce.progression.promotion-rules","owners":["ecommerce.progression.promotion-rules"],"text":"## Promotion rules\n\nStaff can create promotion codes with a percentage discount, a start date, an end date, and a\nredemption limit.\n"},{"id":"ecommerce.progression.notification-preferences","owners":["ecommerce.progression.notification-preferences"],"text":"## Notification preferences\n\nSigned-in customers can turn order and stock notifications on or off.\n"},{"id":"ecommerce.progression.transfer-authorization","owners":["ecommerce.progression.operations-access-specifications"],"requiresFeatures":["ecommerce.l2.stock-transfers-features"],"text":"## Transfer authorization\n\nThe server allows only staff and administrators to transfer warehouse stock.\n"},{"id":"ecommerce.progression.price-authorization","owners":["ecommerce.progression.operations-access-specifications"],"requiresFeatures":["ecommerce.l2.price-history-features"],"text":"## Price authorization\n\nThe server allows only administrators to change catalog prices.\n"},{"id":"ecommerce.progression.shipping-authorization","owners":["ecommerce.progression.operations-access-specifications"],"requiresFeatures":["ecommerce.progression.fulfilment-queue"],"text":"## Shipping authorization\n\nThe server allows only staff and administrators to mark orders as shipped.\n"},{"id":"ecommerce.progression.order-ownership","owners":["ecommerce.progression.operations-access-specifications"],"requiresFeatures":["ecommerce.l2.order-cancellation-features"],"text":"## Order ownership\n\nThe server allows a customer to act only on that customer's own orders.\n"},{"id":"ecommerce.progression.review-access","owners":["ecommerce.progression.review-access-specifications"],"requiresFeatures":["ecommerce.feature.reviews"],"text":"## Review access\n\nOnly a customer who bought an item can review it.\n"},{"id":"ecommerce.progression.transfer-conservation","owners":["ecommerce.progression.inventory-conservation-specifications"],"requiresFeatures":["ecommerce.l2.stock-transfers-features"],"text":"## Stock conservation\n\nStock moves between warehouses without changing the total. A refused transfer changes nothing.\nWhen a purchase and a transfer overlap, the final total reflects the sold units exactly once.\n"},{"id":"ecommerce.progression.cancellation-conservation","owners":["ecommerce.progression.inventory-conservation-specifications"],"requiresFeatures":["ecommerce.l2.order-cancellation-features"],"text":"## Cancellation conservation\n\nCancelling an order restores its stock to the warehouse that supplied it. The restored total is\nthe same for current and newly opened clients.\n"},{"id":"ecommerce.l3.reservations","owners":["ecommerce.l3.reservations-features"],"text":"## Reservations\n\nAdding an item to a cart reserves its stock for 90 seconds. The cart shows the remaining time.\nCheckout consumes a live reservation. An expired reservation releases its stock and remains\nvisible as expired. Adding the item again renews the reservation.\n"},{"id":"ecommerce.l3.scheduled-restocks","owners":["ecommerce.l3.scheduled-restocks-features"],"text":"## Scheduled restocks\n\nAn admin can schedule and cancel a restock. Show pending restocks and their remaining\ntime. Show completed stock movements in a stock ledger.\n"},{"id":"ecommerce.l3.order-delivery","owners":["ecommerce.l3.order-delivery-features"],"text":"## Order delivery\n\nA shipped order becomes delivered 60 seconds after shipping. Show its status in the\ncustomer's order history and the staff view.\n"},{"id":"ecommerce.l3.cart-expiration","owners":["ecommerce.l3.cart-expiration-features"],"text":"## Cart expiration\n\nA cart with no activity for five minutes expires and releases its reservations. The customer\nsees an empty cart and an expiration notice.\n"},{"id":"ecommerce.l3.durable-reservations","owners":["ecommerce.l3.deferred-durability-specifications"],"requiresFeatures":["ecommerce.l3.reservations-features"],"text":"## Durable reservations\n\nPending reservations survive a backend restart.\n\n"},{"id":"ecommerce.progression.managed-support","owners":["ecommerce.progression.managed-support"],"text":"## Managed support cases\n\nCustomers and staff can exchange replies and update the status of a support case.\n"},{"id":"ecommerce.l3.durable-restocks","owners":["ecommerce.l3.deferred-durability-specifications"],"requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"text":"## Durable restocks\n\nPending restocks survive a backend restart.\n\n"},{"id":"ecommerce.l3.durable-order-delivery","owners":["ecommerce.l3.deferred-durability-specifications"],"requiresFeatures":["ecommerce.l3.order-delivery-features"],"text":"## Durable order delivery\n\nPending order delivery survives a backend restart.\n\n"},{"id":"ecommerce.l3.durable-cart-expiration","owners":["ecommerce.l3.deferred-durability-specifications"],"requiresFeatures":["ecommerce.l3.cart-expiration-features"],"text":"## Durable cart expiration\n\nPending cart expiration survives a backend restart.\n\n"},{"id":"ecommerce.l3.exactly-once-restocks","owners":["ecommerce.l3.deferred-integrity-specifications"],"requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"text":"## Exactly-once restocks\n\nRestarting the backend cannot apply a restock more than once.\n\n"},{"id":"ecommerce.l3.exactly-once-delivery","owners":["ecommerce.l3.deferred-integrity-specifications"],"requiresFeatures":["ecommerce.l3.order-delivery-features"],"text":"## Exactly-once delivery\n\nRestarting the backend cannot apply an order transition more than once.\n\n"},{"id":"ecommerce.l3.server-timed-restocks","owners":["ecommerce.l3.server-time-specifications"],"requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"text":"## Server-timed restocks\n\nA pending restock does not run early after a restart.\n\n"},{"id":"ecommerce.l3.server-timed-reservations","owners":["ecommerce.l3.server-time-specifications"],"requiresFeatures":["ecommerce.l3.reservations-features"],"text":"## Server-timed reservations\n\nA reservation expires without an open browser.\n\n"},{"id":"ecommerce.l3.deferred-access","owners":["ecommerce.l3.deferred-access-specifications"],"requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"text":"## Deferred-work access\n\nOnly an admin can schedule or cancel a restock.\n\n"},{"id":"ecommerce.l3.stock-conservation","owners":["ecommerce.l3.deferred-integrity-specifications"],"requiresFeatures":["ecommerce.l3.reservations-features"],"text":"## Stock conservation\n\nReservation expiry returns exactly the stock that the reservation took. Checkout does not take reserved stock twice.\n"},{"id":"ecommerce.progression.promotion-checkout","owners":["ecommerce.progression.promotion-checkout"],"text":"## Promotion checkout\n\nCustomers can apply an active promotion code to a cart. The final order records the applied\ndiscount. Expired and fully redeemed promotions are refused.\n"},{"id":"ecommerce.progression.stock-alerts","owners":["ecommerce.progression.stock-alerts"],"text":"## Stock alerts\n\nA signed-in customer can request an alert for an unavailable item. Show an alert when stock\nreturns.\n"},{"id":"ecommerce.progression.faceted-search","owners":["ecommerce.progression.faceted-search"],"text":"# Faceted search\n\nLet visitors filter the catalog by category, minimum price, maximum price, and availability.\nCombine the selected filters. While search text or any filter is active, order matching\nitems by name, with ten results per page. With no search text or active filter, use\nthe storefront's purchase ranking, breaking ties by item name. Clearing all search\ntext and filters returns to that ranking.\n"},{"id":"ecommerce.progression.personalized-recommendations","owners":["ecommerce.progression.personalized-recommendations"],"text":"# Personalized recommendations\n\nRecommend items from categories in the signed-in customer's purchase history. Exclude\nitems they already purchased. Order by global units sold, highest first, then by item name.\n"},{"id":"ecommerce.spec.search-ordering","owners":["ecommerce.spec.search-ordering"],"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.faceted-search"],"text":"# Search ordering after purchases\n\nPurchases must not change the alphabetical order of active search or filter results.\nClearing all search text and filters restores the current purchase ranking.\n"},{"id":"ecommerce.l3.order-returns","owners":["ecommerce.l3.order-returns-features"],"text":"## Order returns\n\nA customer can return an item after the order ships for a refund of its purchase price.\nRestock the item and mark it returned in order history.\n"},{"id":"ecommerce.progression.order-support","owners":["ecommerce.progression.order-support"],"text":"## Order-linked support\n\nA customer can attach an order to a support case. Staff can inspect the linked order\nfrom the case.\n"},{"id":"ecommerce.progression.promotion-reporting","owners":["ecommerce.progression.promotion-reporting"],"text":"## Promotion reporting\n\nStaff can see each promotion's redemption count and revenue after discounts.\n"},{"id":"ecommerce.progression.delivery-notifications","owners":["ecommerce.progression.delivery-notifications"],"text":"## Delivery notifications\n\nNotify customers when their orders are delivered, using their notification preferences.\n"},{"id":"ecommerce.progression.automatic-reorder","owners":["ecommerce.progression.automatic-reorder"],"text":"# Automatic reorder rules\n\nWarehouse staff can manage automatic reorder rules. A rule names an item, a stock\nthreshold, and a restock quantity. Schedule a restock when stock falls from above the\nthreshold to the threshold or below.\nThe automatic restock is due 60 seconds after it is scheduled.\n"},{"id":"ecommerce.progression.cart-recovery","owners":["ecommerce.progression.cart-recovery"],"text":"# Cart recovery\n\nLet a signed-in customer restore an expired cart. Reserve each item again only when its full\nquantity is available. Restore the available items and list each item that could not be restored.\n"},{"id":"ecommerce.progression.recommendation-feedback","owners":["ecommerce.progression.recommendation-feedback"],"text":"## Recommendation feedback\n\nA customer can dismiss a recommendation from their list.\n"},{"id":"ecommerce.feature.store-credit","owners":["ecommerce.feature.store-credit"],"text":"## Store credit\n\nStaff can issue customer credit with a reference. Customers can see their balance and history and choose credit at cart checkout. Use credit first, up to the order total; record any remainder as the existing payment. Amounts use the shop currency and whole minor units. A reference identifies one grant.\n"},{"id":"ecommerce.feature.subscriptions","owners":["ecommerce.feature.subscriptions"],"text":"## Scheduled purchases\n\nCustomers can subscribe to an individual catalog item (not a bundle) and quantity for a chosen number of deliveries at a chosen interval. Use the item price at subscription creation. The first delivery is due after one interval. Each delivery creates an ordinary order and payment. Skip an unavailable delivery without charging; it still uses one delivery slot. Customers can pause, resume, or cancel future deliveries; a pause moves future due times by the pause duration.\n"},{"id":"ecommerce.progression.support-refunds","owners":["ecommerce.progression.support-refunds"],"text":"## Support refunds\n\nStaff can refund an entire order from its support case. A successful refund resolves\nthe case. Show the refund amount and case status.\n\nA refund does not prevent a later physical return of shipped goods. Refund only\nthe amount not already refunded, and restock goods when they are returned.\n"},{"id":"ecommerce.spec.access-control.automatic-reorder-access","owners":["ecommerce.spec.access-control"],"requiresFeatures":["ecommerce.progression.automatic-reorder"],"text":"## automatic-reorder-access\n\nOnly warehouse staff can manage automatic reorder rules.\n\n"},{"id":"ecommerce.spec.access-control.recommendation-profile-isolation","owners":["ecommerce.spec.access-control"],"requiresFeatures":["ecommerce.progression.personalized-recommendations"],"text":"## recommendation-profile-isolation\n\nCustomer recommendation profiles are isolated.\n\n"},{"id":"ecommerce.spec.access-control.staff-activity-privacy","owners":["ecommerce.spec.access-control"],"requiresFeatures":["ecommerce.progression.staff-activity"],"text":"## staff-activity-privacy\n\nCustomers cannot open staff activity history.\n\n"},{"id":"ecommerce.spec.access-control.order-support-ownership","owners":["ecommerce.spec.access-control"],"requiresFeatures":["ecommerce.progression.order-support"],"text":"## order-support-ownership\n\nA customer cannot attach or inspect another customer's order.\n\n"},{"id":"ecommerce.spec.access-control.delivery-notification-privacy","owners":["ecommerce.spec.access-control"],"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.delivery-notifications","ecommerce.progression.fulfilment-queue"],"text":"## delivery-notification-privacy\n\nDelivery notifications are private to the order owner.\n\n"},{"id":"ecommerce.spec.access-control.support-refund-access","owners":["ecommerce.spec.access-control"],"requiresFeatures":["ecommerce.progression.support-refunds"],"text":"## support-refund-access\n\nOnly authorized staff can refund an order from a support case.\n\n"},{"id":"ecommerce.spec.access-control.recommendation-feedback-privacy","owners":["ecommerce.spec.access-control"],"requiresFeatures":["ecommerce.progression.recommendation-feedback"],"text":"## recommendation-feedback-privacy\n\nRecommendation feedback belongs to one customer.\n\n"},{"id":"ecommerce.spec.state-durability.recommendation-feedback-restart","owners":["ecommerce.spec.state-durability"],"requiresFeatures":["ecommerce.progression.recommendation-feedback"],"text":"## recommendation-feedback-restart\n\nDismissed recommendations stay dismissed after reconnecting or restarting the server.\n\n"},{"id":"ecommerce.spec.transactional-integrity.automatic-reorder-deduplication","owners":["ecommerce.spec.transactional-integrity"],"requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.automatic-reorder"],"text":"## automatic-reorder-deduplication\n\nA pending automatic restock is not scheduled twice.\n\n"},{"id":"ecommerce.spec.transactional-integrity.payment-deduplication","owners":["ecommerce.spec.transactional-integrity"],"requiresFeatures":["ecommerce.progression.payment-records"],"text":"## payment-deduplication\n\nA checkout does not create duplicate payments.\n\n"},{"id":"ecommerce.spec.transactional-integrity.support-refund-accounting","owners":["ecommerce.spec.transactional-integrity"],"requiresFeatures":["ecommerce.progression.support-refunds"],"text":"## support-refund-accounting\n\nThe recorded refund equals the amount paid and is applied only once.\n"},{"id":"ecommerce.feature.split-tender-refunds","owners":["ecommerce.feature.split-tender-refunds"],"text":"## Refunds with store credit\n\nExtend support refunds and item returns to orders paid with credit and another payment. Return each part to its original source. A full refund restores the original credit portion to the customer balance and refunds the original external portion.\n"},{"id":"ecommerce.spec.bundle-integrity.product-bundles","owners":["ecommerce.spec.bundle-integrity"],"requiresFeatures":["ecommerce.feature.product-bundles"],"text":"## Product bundles\n\nOnly authorized staff can change bundles.\n\n"},{"id":"ecommerce.spec.bundle-integrity.bundle-checkout","owners":["ecommerce.spec.bundle-integrity"],"requiresFeatures":["ecommerce.feature.bundle-checkout"],"text":"## Bundle checkout\n\nReserve all components or none. Competing purchases share the same stock. Release component reservations when the cart expires or the bundle is removed.\n\n"},{"id":"ecommerce.spec.bundle-integrity.bundle-returns","owners":["ecommerce.spec.bundle-integrity"],"requiresFeatures":["ecommerce.feature.bundle-returns"],"text":"## Bundle returns\n\nReturn the original component allocations and price paid. Repeating a return must not change stock or refunds again. Customers cannot return another account's order.\n\n"},{"id":"ecommerce.spec.store-credit.store-credit","owners":["ecommerce.spec.store-credit"],"requiresFeatures":["ecommerce.feature.store-credit"],"text":"## Store credit\n\nOnly authorized staff can grant credit. Repeating a reference must not issue credit twice. Concurrent checkout must not duplicate the order or credit use. Accepted credit survives a backend restart.\n\n"},{"id":"ecommerce.spec.split-tender-refunds.split-tender-refunds","owners":["ecommerce.spec.split-tender-refunds"],"requiresFeatures":["ecommerce.feature.split-tender-refunds"],"text":"## Split-tender refunds\n\nConcurrent or repeated refunds must restore each original payment portion only once. The resulting balance and refund records survive a backend restart.\n\n"},{"id":"ecommerce.spec.subscriptions.subscriptions","owners":["ecommerce.spec.subscriptions"],"requiresFeatures":["ecommerce.feature.subscriptions"],"text":"## Scheduled purchases\n\nPending deliveries and pauses survive a backend restart. Process elapsed pending slots after recovery. Completed delivery slots must not run again. Customers cannot change another account's subscription.\n"},{"id":"ecommerce.feature.product-bundles","owners":["ecommerce.feature.product-bundles"],"text":"## Product bundles\n\nCatalog staff can create and edit a named bundle of existing products, with a quantity\nfor each component and one bundle price. Show bundles and their components in the catalog.\nBundles contain products only, not other bundles.\n"},{"id":"ecommerce.feature.bundle-checkout","owners":["ecommerce.feature.bundle-checkout"],"text":"## Bundle checkout\n\nCustomers can add a whole bundle to their cart and buy it through checkout. Its components\nuse the existing stock reservations and reservation lifetime. Removing a bundle releases\nits reservation. Show the bundle as one order line at its bundle price.\n"},{"id":"ecommerce.feature.bundle-returns","owners":["ecommerce.feature.bundle-returns"],"text":"## Bundle returns\n\nCustomers can return a shipped bundle as a whole. Restore the purchased component quantities\nto their original warehouses and refund the price paid. Show the return and refund on the\norder. Partial bundle returns are not supported.\n"}]},"track":"ecommerce"},"execution":{"capabilities":["backend-lifecycle","browser","concurrent-actors","database-observation","database-read","direct-database-write","direct-server-call","process-crash","request-replay"],"execution":[{"checkGroups":[{"checkGroupId":"accounts","feature":{"actors":["shopper"],"criteria":[{"id":"1a","steps":[{"actor":"shopper","do":"signUp","name":"ann"},{"actor":"shopper","contains":"ann","do":"expect","testid":"current-user","within":6000}]}],"id":1,"setup":[]},"packId":"ecommerce.feature.accounts","role":"feature","source":"scenarios/01-account-create.json"}],"id":"selected-source-001","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-account-create.json"},{"checkGroups":[{"checkGroupId":"accounts","feature":{"actors":["owner","impostor"],"criteria":[{"id":"1b","steps":[{"actor":"impostor","do":"signUp","expectFailure":true,"name":"ann","password":"different-pw"},{"actor":"impostor","do":"expect","testid":"auth-error","within":6000},{"absent":true,"actor":"impostor","do":"expect","testid":"current-user"}]}],"id":1,"setup":[{"actor":"owner","do":"signUp","name":"ann"}]},"packId":"ecommerce.feature.accounts","role":"feature","source":"scenarios/01-account-duplicate.json"}],"id":"selected-source-002","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-account-duplicate.json"},{"checkGroups":[{"checkGroupId":"accounts","feature":{"actors":["owner","impostor"],"criteria":[{"id":"1c","steps":[{"actor":"impostor","do":"signIn","expectFailure":true,"name":"ann","password":"wrong-pw"},{"actor":"impostor","do":"expect","testid":"auth-error","within":6000},{"absent":true,"actor":"impostor","do":"expect","testid":"current-user"}]}],"id":1,"setup":[{"actor":"owner","do":"signUp","name":"ann"}]},"packId":"ecommerce.feature.accounts","role":"feature","source":"scenarios/01-account-password.json"}],"id":"selected-source-003","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-account-password.json"},{"checkGroups":[{"checkGroupId":"session-reload","feature":{"actors":["shopper"],"criteria":[{"id":"1e","steps":[{"actor":"shopper","do":"reload","settleMs":4000},{"actor":"shopper","contains":"ann","do":"expect","testid":"current-user","within":6000}]}],"id":1,"setup":[{"actor":"shopper","do":"signUp","name":"ann"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","requiresFeatures":["ecommerce.feature.accounts"],"role":"guarantee","source":"scenarios/01-account-reload.json"}],"id":"selected-source-004","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-account-reload.json"},{"checkGroups":[{"checkGroupId":"accounts","feature":{"actors":["shopper"],"criteria":[{"id":"1d","steps":[{"actor":"shopper","do":"click","testid":"current-user","unlessVisible":"signout"},{"actor":"shopper","do":"click","testid":"signout"},{"actor":"shopper","do":"waitUntilAbsent","testid":"current-user","within":6000},{"actor":"shopper","do":"signIn","name":"ann"},{"actor":"shopper","contains":"ann","do":"expect","testid":"current-user","within":6000}]}],"id":1,"setup":[{"actor":"shopper","do":"signUp","name":"ann"}]},"packId":"ecommerce.feature.accounts","role":"feature","source":"scenarios/01-account-signout.json"}],"id":"selected-source-005","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-account-signout.json"},{"checkGroups":[{"checkGroupId":"admin-write","feature":{"actors":["admin","staff"],"criteria":[{"id":"103a","steps":[{"actor":"staff","as":"purifier-before-control","do":"recordNumber","in":{"contains":"Air Purifier","testid":"item-card"},"testid":"item-stock"},{"action":"restock","actor":"admin","do":"callAction","input":{"attribute":"data-restock-input","contains":"Air Purifier","testid":"admin-location-row"},"namedAction":{"args":[0,0,1],"id":"restock","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"warehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/restock","reducer":"admin_restock"},"settleMs":2000},{"actor":"admin","do":"expectActionOutcome","outcome":"accepted"},{"actor":"staff","do":"reload","settleMs":2000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"expectNumber","in":{"contains":"Air Purifier","testid":"item-card"},"plus":1,"relativeTo":"purifier-before-control","testid":"item-stock"}]}],"id":103,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"admin","do":"click","testid":"admin-link"}]},"packId":"ecommerce.feature.warehouse-admin","role":"feature","source":"scenarios/01-admin-write-staff.json"},{"checkGroupId":"warehouse-write-boundary","feature":{"actors":["admin","staff"],"criteria":[{"id":"103b","steps":[{"action":"restock","actor":"admin","do":"callAction","input":{"attribute":"data-restock-input","contains":"Air Purifier","testid":"admin-location-row"},"namedAction":{"args":[0,0,1],"id":"restock","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"warehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/restock","reducer":"admin_restock"},"settleMs":2000},{"actor":"admin","do":"expectActionOutcome","outcome":"accepted"},{"actor":"staff","do":"reload","settleMs":2000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","as":"purifier-before-refusal","do":"recordNumber","in":{"contains":"Air Purifier","testid":"item-card"},"testid":"item-stock"},{"action":"restock","actor":"staff","do":"callAction","from":"admin","input":{"attribute":"data-restock-input","contains":"Air Purifier","testid":"admin-location-row"},"namedAction":{"args":[0,0,1],"id":"restock","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"warehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/restock","reducer":"admin_restock"},"settleMs":2000},{"actor":"staff","do":"expectActionOutcome","outcome":"refused","routeProvenBy":"admin"},{"actor":"staff","do":"reload","settleMs":2000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"expectNumber","in":{"contains":"Air Purifier","testid":"item-card"},"plus":0,"relativeTo":"purifier-before-refusal","testid":"item-stock"}]}],"id":103,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"admin","do":"click","testid":"admin-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-admin-write-staff.json"}],"id":"selected-source-006","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-admin-write-staff.json"},{"checkGroups":[{"checkGroupId":"purchase-stock","feature":{"actors":["buyer","watcher","visitor"],"criteria":[{"id":"3b","steps":[{"actor":"watcher","do":"expectNumber","equals":100,"in":{"contains":"Espresso Machine","testid":"item-card"},"testid":"item-stock"},{"actor":"buyer","do":"click","in":{"contains":"Espresso Machine","testid":"item-card"},"testid":"buy-now"},{"actor":"watcher","do":"expectNumber","equals":99,"in":{"contains":"Espresso Machine","testid":"item-card"},"testid":"item-stock"},{"actor":"visitor","do":"expectNumber","equals":99,"in":{"contains":"Espresso Machine","testid":"item-card"},"testid":"item-stock"}]}],"id":3,"setup":[{"actor":"buyer","do":"signUp","name":"eli"},{"actor":"watcher","do":"signUp","name":"fay"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/01-buying.json"}],"id":"selected-source-007","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-buying.json"},{"checkGroups":[{"checkGroupId":"cart-boundary","feature":{"actors":["owner","stranger"],"criteria":[{"id":"109a","steps":[{"action":"cart-add","actor":"stranger","do":"callAction","input":{"attribute":"data-buy-input","contains":"Coffee Grinder","testid":"item-card"},"namedAction":{"args":[0],"id":"cart-add","params":[{"in":"body","name":"itemId","wireType":"u64"}],"path":"/api/cart","reducer":"add_to_cart"},"settleMs":2000},{"actor":"stranger","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"reload","settleMs":1500},{"actor":"owner","do":"ensureSignedIn","name":"vic","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"owner","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"},{"actor":"stranger","do":"reload","settleMs":1500},{"actor":"stranger","do":"ensureSignedIn","name":"wes","readyTestid":"current-user"},{"actor":"stranger","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"stranger","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"owner","contains":"Coffee Grinder","count":1,"do":"expect","testid":"cart-item"},{"actor":"owner","do":"expectNumber","equals":1,"in":{"contains":"Coffee Grinder","testid":"cart-item"},"testid":"cart-quantity"},{"actor":"stranger","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"},{"actor":"stranger","contains":"Coffee Grinder","count":1,"do":"expect","testid":"cart-item"},{"actor":"stranger","do":"expectNumber","equals":1,"in":{"contains":"Coffee Grinder","testid":"cart-item"},"testid":"cart-quantity"}]},{"id":"109b","steps":[{"actor":"owner","as":"cart-total-before-invalid","do":"recordNumber","testid":"cart-total"},{"actor":"owner","as":"cart-quantity-before-invalid","do":"recordNumber","in":{"contains":"Coffee Grinder","testid":"cart-item"},"testid":"cart-quantity"},{"action":"cart-set-quantity","actor":"owner","do":"callAction","input":{"attribute":"data-cart-input","contains":"Coffee Grinder","testid":"cart-item"},"namedAction":{"args":[0,-3],"id":"cart-set-quantity","method":"PATCH","params":[{"in":"path","name":"itemId","placeholder":":itemId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/cart/:itemId","reducer":"update_cart_quantity"},"settleMs":2000},{"actor":"owner","do":"expectActionOutcome","outcome":"validation-refused"},{"actor":"owner","do":"reload","settleMs":1500},{"actor":"owner","do":"ensureSignedIn","name":"vic","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"owner","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"},{"actor":"owner","do":"expectNumber","plus":0,"relativeTo":"cart-total-before-invalid","testid":"cart-total"},{"actor":"owner","do":"expectNumber","in":{"contains":"Coffee Grinder","testid":"cart-item"},"plus":0,"relativeTo":"cart-quantity-before-invalid","testid":"cart-quantity"}]}],"id":109,"setup":[{"actor":"owner","do":"signUp","name":"vic"},{"actor":"stranger","do":"signUp","name":"wes"},{"actor":"owner","do":"click","in":{"contains":"Coffee Grinder","testid":"item-card"},"testid":"add-to-cart"},{"actor":"owner","do":"wait","ms":1500},{"actor":"owner","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.feature.cart"],"role":"guarantee","source":"scenarios/01-cart-boundary.json"}],"id":"selected-source-008","scenario":{"level":1,"writeUrlPattern":"\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses"},"source":"scenarios/01-cart-boundary.json"},{"checkGroups":[{"checkGroupId":"cart-reload","feature":{"actors":["quantity","reload","live1","live2","checkout"],"criteria":[{"id":"4b","steps":[{"actor":"reload","do":"click","in":{"contains":"Laptop Stand","testid":"item-card"},"testid":"add-to-cart"},{"actor":"reload","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"},{"actor":"reload","contains":"Laptop Stand","do":"expect","testid":"cart-item","within":10000},{"actor":"reload","do":"reload","settleMs":3000},{"actor":"reload","do":"ensureSignedIn","name":"omar","readyTestid":"current-user"},{"actor":"reload","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"},{"actor":"reload","contains":"Laptop Stand","do":"expect","testid":"cart-item"}]}],"id":4,"setup":[{"actor":"quantity","do":"signUp","name":"nora"},{"actor":"reload","do":"signUp","name":"omar"},{"actor":"live1","do":"signUp","name":"pia"},{"actor":"checkout","do":"signUp","name":"quinn"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","requiresFeatures":["ecommerce.feature.cart"],"role":"guarantee","source":"scenarios/01-cart.json"},{"checkGroupId":"shared-cart","feature":{"actors":["quantity","reload","live1","live2","checkout"],"criteria":[{"id":"4c","steps":[{"actor":"live2","do":"signIn","name":"pia"},{"actor":"live2","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"},{"actor":"live1","do":"click","in":{"contains":"Induction Cooktop","testid":"item-card"},"testid":"add-to-cart"},{"actor":"live2","contains":"Induction Cooktop","do":"expect","testid":"cart-item","within":10000}]}],"id":4,"setup":[{"actor":"quantity","do":"signUp","name":"nora"},{"actor":"reload","do":"signUp","name":"omar"},{"actor":"live1","do":"signUp","name":"pia"},{"actor":"checkout","do":"signUp","name":"quinn"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","requiresFeatures":["ecommerce.feature.cart"],"role":"guarantee","source":"scenarios/01-cart.json"}],"id":"selected-source-009","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-cart.json"},{"checkGroups":[{"checkGroupId":"catalog-ranking","feature":{"actors":["visitor"],"criteria":[{"id":"2b","steps":[{"actor":"visitor","do":"expectSequence","equals":["Air Purifier","Bluetooth Speaker","Coffee Grinder","Desk Lamp","Espresso Machine","Gaming Mouse","Headphones","Induction Cooktop","Keyboard","Laptop Stand"],"in":{"testid":"item-list"},"testid":"item-name"}]}],"id":2,"setup":[]},"packId":"ecommerce.feature.catalog-discovery","role":"feature","source":"scenarios/01-catalog-ranking.json","stablePackId":"ecommerce.feature.catalog"}],"id":"selected-source-010","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-catalog-ranking.json"},{"checkGroups":[{"checkGroupId":"catalog-search","feature":{"actors":["visitor"],"criteria":[{"id":"2d","steps":[{"actor":"visitor","do":"fill","enter":true,"settleMs":1500,"testid":"search-input","text":"mirrorLESS"},{"actor":"visitor","contains":"Mirrorless Camera","do":"expect","in":{"testid":"search-results"},"testid":"item-card"}]}],"id":2,"setup":[]},"packId":"ecommerce.feature.catalog-discovery","role":"feature","source":"scenarios/01-catalog-search.json","stablePackId":"ecommerce.feature.catalog"}],"id":"selected-source-011","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-catalog-search.json"},{"checkGroups":[{"checkGroupId":"catalog-values","feature":{"actors":["visitor"],"criteria":[{"id":"2a","steps":[{"actor":"visitor","contains":"Air Purifier","do":"expect","testid":"item-card"},{"actor":"visitor","do":"expectNumber","equals":189,"in":{"contains":"Air Purifier","testid":"item-card"},"testid":"item-price"},{"actor":"visitor","do":"expectNumber","equals":100,"in":{"contains":"Air Purifier","testid":"item-card"},"testid":"item-stock"}]}],"id":2,"setup":[]},"packId":"ecommerce.feature.catalog-items","role":"feature","source":"scenarios/01-catalog-values.json","stablePackId":"ecommerce.feature.catalog"}],"id":"selected-source-012","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-catalog-values.json"},{"checkGroups":[{"checkGroupId":"ranking","feature":{"actors":["buyer","visitor","inspector"],"criteria":[{"id":"2c","steps":[{"actor":"buyer","do":"click","in":{"contains":"Coffee Grinder","testid":"item-card"},"testid":"buy-now"},{"actor":"visitor","do":"expectSequence","equals":["Coffee Grinder","Air Purifier","Bluetooth Speaker","Desk Lamp","Espresso Machine","Gaming Mouse","Headphones","Induction Cooktop","Keyboard","Laptop Stand"],"in":{"testid":"item-list"},"testid":"item-name","within":10000},{"actors":["buyer","visitor"],"do":"expectAgreement","in":{"contains":"Coffee Grinder","testid":"item-card"},"numeric":true,"testid":"item-stock"}]}],"id":2,"setup":[{"actor":"buyer","do":"signUp","name":"dov"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/01-core.json"}],"id":"selected-source-013","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-core.json"},{"checkGroups":[{"checkGroupId":"duplicate-checkout","feature":{"actors":["tab1","tab2","filler"],"criteria":[{"id":"203a","steps":[{"actor":"tab1","contains":"Gaming Mouse","count":1,"do":"expect","testid":"cart-item"},{"actor":"tab1","do":"expectNumber","equals":2,"in":{"contains":"Gaming Mouse","testid":"cart-item"},"testid":"cart-quantity"}]},{"id":"203b","steps":[{"actor":"tab1","do":"click","testid":"checkout-submit"},{"actor":"tab1","do":"wait","ms":2500},{"actor":"filler","as":"keyboard-before-checkout","do":"recordNumber","in":{"contains":"Keyboard","testid":"item-card"},"testid":"item-stock"},{"account":"{user:twin}","as":"checkout-before","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"filler","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"actor":"filler","do":"wait","ms":2000},{"account":"{user:twin}","as":"checkout-prepared","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"action":"checkout","actors":["tab1","tab2"],"do":"callConcurrently","settleMs":5000},{"do":"expectCallOutcomes"},{"before":"checkout-before","do":"dbExpectCheckout","prepared":"checkout-prepared","quantity":1},{"actor":"filler","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"filler","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"filler","contains":"Keyboard","count":1,"do":"expect","testid":"order-item"},{"actor":"filler","do":"reload","settleMs":2000},{"actor":"filler","do":"ensureSignedIn","name":"twin","readyTestid":"current-user"},{"actor":"filler","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"filler","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"filler","do":"expectNumber","in":{"contains":"Keyboard","testid":"item-card"},"plus":-1,"relativeTo":"keyboard-before-checkout","testid":"item-stock"}]}],"id":203,"setup":[{"actor":"tab1","do":"signUp","name":"twin"},{"actor":"tab2","do":"signIn","name":"twin"},{"actor":"filler","do":"signIn","name":"twin"},{"actors":["tab1","tab2"],"do":"clickConcurrently","in":{"contains":"Gaming Mouse","testid":"item-card"},"settleMs":4000,"testid":"add-to-cart"},{"actor":"tab1","do":"reload","settleMs":2500},{"actor":"tab1","do":"ensureSignedIn","name":"twin","readyTestid":"current-user"},{"actor":"tab1","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","requiresFeatures":["ecommerce.feature.checkout"],"role":"guarantee","source":"scenarios/01-duplicate-checkout.json"}],"id":"selected-source-014","scenario":{"level":1,"writeUrlPattern":"\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses"},"source":"scenarios/01-duplicate-checkout.json"},{"checkGroups":[{"checkGroupId":"external-stock","feature":{"actors":["viewer"],"criteria":[{"id":"901a","steps":[{"do":"dbSetStock","item":"Desk Lamp","quantity":5,"settleMs":4000,"warehouse":"East"},{"actor":"viewer","do":"expectNumber","equals":50,"in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock","within":10000}]}],"id":901,"setup":[{"actor":"viewer","do":"expectNumber","equals":100,"in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.external-data-sync","requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-external-live-sync.json"}],"id":"selected-source-015","scenario":{"level":1,"writeUrlPattern":"\\/api\\/|\\/items|\\/stock|\\/warehouses"},"source":"scenarios/01-external-live-sync.json"},{"checkGroups":[{"checkGroupId":"external-stock","feature":{"actors":["viewer"],"criteria":[{"id":"901d","steps":[{"actor":"viewer","do":"setOffline","offline":true,"settleMs":1000},{"do":"dbSetStock","item":"Desk Lamp","quantity":7,"settleMs":4000,"warehouse":"East"},{"actor":"viewer","do":"setOffline","offline":false,"settleMs":1000},{"actor":"viewer","do":"expectNumber","equals":52,"in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock","within":20000}]}],"id":901,"setup":[{"actor":"viewer","do":"expectNumber","equals":100,"in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.external-data-sync","requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-external-reconnect-sync.json"}],"id":"selected-source-016","scenario":{"level":1,"writeUrlPattern":"\\/api\\/|\\/items|\\/stock|\\/warehouses"},"source":"scenarios/01-external-reconnect-sync.json"},{"checkGroups":[{"checkGroupId":"external-stock","feature":{"actors":["viewer"],"criteria":[{"id":"901b","steps":[{"do":"dbSetStock","item":"Desk Lamp","quantity":5,"settleMs":1000,"warehouse":"East"},{"actor":"viewer","do":"reload","settleMs":3000},{"actor":"viewer","do":"expectNumber","equals":50,"in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock","within":10000}]}],"id":901,"setup":[{"actor":"viewer","do":"expectNumber","equals":100,"in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.external-data-sync","requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-external-reload-sync.json"}],"id":"selected-source-017","scenario":{"level":1,"writeUrlPattern":"\\/api\\/|\\/items|\\/stock|\\/warehouses"},"source":"scenarios/01-external-reload-sync.json"},{"checkGroups":[{"checkGroupId":"external-stock","feature":{"actors":["viewer"],"criteria":[{"id":"901c","steps":[{"do":"stopAppServer"},{"do":"dbSetStock","item":"Desk Lamp","quantity":10,"settleMs":4000,"warehouse":"West"},{"do":"startAppServer"},{"actor":"viewer","do":"expectNumber","equals":65,"in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock","within":20000}]}],"id":901,"setup":[{"actor":"viewer","do":"expectNumber","equals":100,"in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.external-data-sync","requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-external-server-restart-sync.json"}],"id":"selected-source-018","scenario":{"level":1,"writeUrlPattern":"\\/api\\/|\\/items|\\/stock|\\/warehouses"},"source":"scenarios/01-external-server-restart-sync.json"},{"checkGroups":[{"checkGroupId":"last-unit","feature":{"actors":["admin","a","b","c","d","e","f"],"criteria":[{"id":"201a","steps":[{"do":"dbExpectStock","equals":0,"item":"Air Purifier","warehouse":"East"},{"do":"dbExpectStock","equals":0,"item":"Air Purifier","warehouse":"West"},{"actor":"a","do":"expectNumber","equals":0,"in":{"contains":"Air Purifier","testid":"item-card"},"testid":"item-stock"},{"actors":["a","b","c","d","e","f"],"do":"expectAgreement","in":{"contains":"Air Purifier","testid":"item-card"},"numeric":true,"testid":"item-stock"}]},{"id":"201c","steps":[{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"expectNumber","plus":567,"relativeTo":"revenue-before-last-unit","testid":"admin-revenue","within":10000}]},{"id":"201b","steps":[{"before":{"a":"buy-a","b":"buy-b","c":"buy-c","d":"buy-d","e":"buy-e","f":"buy-f"},"do":"dbExpectPurchases","purchases":3},{"actor":"a","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"a","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"b","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"b","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"c","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"c","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"d","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"d","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"e","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"e","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"f","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"f","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actors":["a","b","c","d","e","f"],"contains":"Air Purifier","do":"expectActorsWith","equals":3,"maxEach":1,"testid":"order-item"},{"do":"dbSetStock","item":"Air Purifier","quantity":2,"settleMs":250,"warehouse":"East"},{"do":"dbSetStock","item":"Air Purifier","quantity":2,"settleMs":250,"warehouse":"West"},{"actor":"a","do":"reload","settleMs":1000},{"actor":"a","do":"ensureSignedIn","name":"c1","readyTestid":"current-user"},{"actor":"a","do":"click","ifAvailable":true,"testid":"catalog-link"},{"account":"{user:c1}","as":"ample-a","do":"dbRecordCheckout","item":"Air Purifier","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"account":"{user:c2}","as":"ample-b","do":"dbRecordCheckout","item":"Air Purifier","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"action":"buy","actors":["a","b"],"do":"callConcurrently","from":"a","input":{"attribute":"data-buy-input","contains":"Air Purifier","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"},"requests":4,"settleMs":3000},{"do":"expectCallOutcomes"},{"before":{"a":"ample-a","b":"ample-b"},"do":"dbExpectPurchases","purchases":4}]}],"id":201,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","as":"revenue-before-last-unit","do":"recordNumber","testid":"admin-revenue"},{"do":"dbSetStock","item":"Air Purifier","quantity":2,"settleMs":250,"warehouse":"East"},{"do":"dbSetStock","item":"Air Purifier","quantity":1,"settleMs":250,"warehouse":"West"},{"actor":"a","do":"signUp","name":"c1"},{"actor":"b","do":"signUp","name":"c2"},{"actor":"c","do":"signUp","name":"c3"},{"actor":"d","do":"signUp","name":"c4"},{"actor":"e","do":"signUp","name":"c5"},{"actor":"f","do":"signUp","name":"c6"},{"account":"{user:c1}","as":"buy-a","do":"dbRecordCheckout","item":"Air Purifier","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"account":"{user:c2}","as":"buy-b","do":"dbRecordCheckout","item":"Air Purifier","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"account":"{user:c3}","as":"buy-c","do":"dbRecordCheckout","item":"Air Purifier","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"account":"{user:c4}","as":"buy-d","do":"dbRecordCheckout","item":"Air Purifier","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"account":"{user:c5}","as":"buy-e","do":"dbRecordCheckout","item":"Air Purifier","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"account":"{user:c6}","as":"buy-f","do":"dbRecordCheckout","item":"Air Purifier","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"action":"buy","actors":["a","b","c","d","e","f"],"do":"callConcurrently","input":{"attribute":"data-buy-input","contains":"Air Purifier","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"},"settleMs":6000},{"do":"expectCallOutcomes"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-last-unit.json"}],"id":"selected-source-019","scenario":{"level":1,"writeUrlPattern":"\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses"},"source":"scenarios/01-last-unit.json"},{"checkGroups":[{"checkGroupId":"order-ownership","feature":{"actors":["one","two"],"criteria":[{"id":"106a","steps":[{"actor":"one","do":"click","in":{"contains":"Coffee Grinder","testid":"item-card"},"testid":"buy-now"},{"actor":"two","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"buy-now"},{"actor":"two","do":"wait","ms":2000},{"actor":"two","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"two","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"two","contains":"Desk Lamp","count":1,"do":"expect","testid":"order-item"},{"absent":true,"actor":"two","contains":"Coffee Grinder","do":"expect","testid":"order-item"}]}],"id":106,"setup":[{"actor":"one","do":"signUp","name":"quin"},{"actor":"two","do":"signUp","name":"ros"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/01-order-ownership.json"}],"id":"selected-source-020","scenario":{"level":1,"writeUrlPattern":"\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses"},"source":"scenarios/01-order-ownership.json"},{"checkGroups":[{"checkGroupId":"purchase-attribution","feature":{"actors":["victim","attacker"],"criteria":[{"id":"102a","steps":[{"action":"buy","actor":"attacker","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"},"settleMs":2000},{"actor":"attacker","do":"expectActionOutcome","outcome":"accepted"},{"actor":"victim","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"victim","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"victim","contains":"Coffee Grinder","count":1,"do":"expect","testid":"order-item"},{"absent":true,"actor":"victim","contains":"Desk Lamp","do":"expect","testid":"order-item"},{"actor":"attacker","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"attacker","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"attacker","contains":"Desk Lamp","count":1,"do":"expect","testid":"order-item"}]}],"id":102,"setup":[{"actor":"victim","do":"signUp","name":"lee"},{"actor":"attacker","do":"signUp","name":"mel"},{"actor":"victim","do":"click","in":{"contains":"Coffee Grinder","testid":"item-card"},"testid":"buy-now"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/01-purchase-attribution.json"}],"id":"selected-source-021","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-purchase-attribution.json"},{"checkGroups":[{"checkGroupId":"purchase-session","feature":{"actors":["buyer","guest"],"criteria":[{"id":"101a","steps":[{"action":"buy","actor":"guest","authentication":"none","do":"callAction","from":"buyer","input":{"attribute":"data-buy-input","contains":"Bluetooth Speaker","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"},"settleMs":2000},{"actor":"guest","do":"expectActionOutcome","outcome":"refused","routeProvenBy":"buyer"},{"do":"dbExpectStock","item":"Bluetooth Speaker","plus":-1,"relativeTo":"speaker-before-control"}]}],"id":101,"setup":[{"actor":"buyer","do":"signUp","name":"kim"},{"as":"speaker-before-control","do":"dbRecordStock","item":"Bluetooth Speaker"},{"action":"buy","actor":"buyer","do":"callAction","input":{"attribute":"data-buy-input","contains":"Bluetooth Speaker","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"},"settleMs":2000},{"actor":"buyer","do":"expectActionOutcome","outcome":"accepted"},{"do":"dbExpectStock","item":"Bluetooth Speaker","plus":-1,"relativeTo":"speaker-before-control"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/01-purchase-session.json"}],"id":"selected-source-022","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-purchase-session.json"},{"checkGroups":[{"checkGroupId":"restock-race","feature":{"actors":["admin","a","b","c"],"criteria":[{"id":"202-control","steps":[{"do":"dbExpectStock","item":"Bluetooth Speaker","plus":5,"relativeTo":"stored-before-control"},{"actor":"a","do":"expectNumber","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"plus":5,"relativeTo":"storefront-before","testid":"item-stock","within":8000}]},{"id":"202a","steps":[{"as":"stored-before-rush","do":"dbRecordStock","item":"Bluetooth Speaker"},{"actor":"admin","do":"fill","in":{"contains":"Bluetooth Speaker","testid":"admin-location-row"},"testid":"restock-input","text":"5"},{"actor":"a","as":"rush-before","do":"recordNumber","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"item-stock"},{"branches":[[{"actors":["a","b","c"],"do":"clickConcurrently","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"settleMs":500,"testid":"buy-now"}],[{"actor":"admin","do":"click","in":{"contains":"Bluetooth Speaker","testid":"admin-location-row"},"testid":"restock-submit"}]],"do":"race","settleMs":6000},{"do":"dbExpectStock","item":"Bluetooth Speaker","plus":2,"relativeTo":"stored-before-rush"},{"atLeast":0,"do":"dbExpectStock","item":"Bluetooth Speaker","warehouse":"East"},{"atLeast":0,"do":"dbExpectStock","item":"Bluetooth Speaker","warehouse":"West"},{"actor":"a","do":"reload","settleMs":1000},{"actor":"a","do":"click","testid":"catalog-link"},{"actor":"a","do":"expectNumber","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"plus":2,"relativeTo":"rush-before","testid":"item-stock","within":10000},{"actor":"a","do":"ensureSignedIn","name":"r1","readyTestid":"current-user"},{"actor":"a","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"a","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"a","contains":"Bluetooth Speaker","count":2,"do":"expect","testid":"order-item","within":10000},{"actor":"b","do":"reload","settleMs":1000},{"actor":"b","do":"ensureSignedIn","name":"r2","readyTestid":"current-user"},{"actor":"b","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"b","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"b","contains":"Bluetooth Speaker","count":1,"do":"expect","testid":"order-item","within":10000},{"actor":"c","do":"reload","settleMs":1000},{"actor":"c","do":"ensureSignedIn","name":"r3","readyTestid":"current-user"},{"actor":"c","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"c","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"c","contains":"Bluetooth Speaker","count":1,"do":"expect","testid":"order-item","within":10000},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link"},{"actor":"admin","do":"expectNumber","in":{"contains":"Bluetooth Speaker","testid":"admin-item-row"},"plus":2,"relativeTo":"stored-before-rush","testid":"admin-stock","within":10000},{"actor":"a","do":"reload","settleMs":1000},{"actor":"a","do":"click","testid":"catalog-link"},{"actor":"a","do":"expectNumber","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"plus":2,"relativeTo":"stored-before-rush","testid":"item-stock","within":10000},{"actor":"a","do":"ensureSignedIn","name":"r1","readyTestid":"current-user"},{"account":"{user:r1}","as":"mixed-a","do":"dbRecordCheckout","item":"Bluetooth Speaker","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"account":"{user:r2}","as":"mixed-b","do":"dbRecordCheckout","item":"Bluetooth Speaker","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"account":"{user:r3}","as":"mixed-c","do":"dbRecordCheckout","item":"Bluetooth Speaker","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"action":"buy","actors":["a","b","c"],"alongside":[{"action":"restock","actors":["admin"],"input":{"attribute":"data-restock-input","contains":"Bluetooth Speaker","testid":"admin-location-row"},"namedAction":{"args":[0,0,1],"id":"restock","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"warehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/restock","reducer":"admin_restock"},"requests":1}],"do":"callConcurrently","from":"a","input":{"attribute":"data-buy-input","contains":"Bluetooth Speaker","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"},"requests":3,"settleMs":3000},{"accepted":4,"do":"expectCallOutcomes"},{"before":{"a":"mixed-a","b":"mixed-b","c":"mixed-c"},"do":"dbExpectPurchases","purchases":3}]}],"id":202,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"a","do":"signUp","name":"r1"},{"actor":"b","do":"signUp","name":"r2"},{"actor":"c","do":"signUp","name":"r3"},{"as":"stored-before-serial-purchase","do":"dbRecordStock","item":"Bluetooth Speaker"},{"actor":"a","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"buy-now"},{"actor":"a","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"a","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"a","contains":"Bluetooth Speaker","count":1,"do":"expect","testid":"order-item","within":10000},{"do":"dbExpectStock","item":"Bluetooth Speaker","plus":-1,"relativeTo":"stored-before-serial-purchase"},{"actor":"a","do":"reload","settleMs":1000},{"actor":"a","do":"ensureSignedIn","name":"r1","readyTestid":"current-user"},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"expectNumber","in":{"contains":"Bluetooth Speaker","testid":"admin-item-row"},"plus":-1,"relativeTo":"stored-before-serial-purchase","testid":"admin-stock","within":10000},{"as":"stored-before-control","do":"dbRecordStock","item":"Bluetooth Speaker"},{"actor":"admin","as":"warehouse-before","do":"recordNumber","in":{"contains":"Bluetooth Speaker","testid":"admin-location-row"},"testid":"admin-location-qty"},{"actor":"a","do":"click","testid":"catalog-link"},{"actor":"a","as":"storefront-before","do":"recordNumber","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"item-stock"},{"actor":"admin","do":"fill","in":{"contains":"Bluetooth Speaker","testid":"admin-location-row"},"testid":"restock-input","text":"5"},{"actor":"admin","do":"click","in":{"contains":"Bluetooth Speaker","testid":"admin-location-row"},"testid":"restock-submit"},{"do":"dbExpectStock","item":"Bluetooth Speaker","plus":5,"relativeTo":"stored-before-control","within":8000},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"expectNumber","in":{"contains":"Bluetooth Speaker","testid":"admin-location-row"},"plus":5,"relativeTo":"warehouse-before","testid":"admin-location-qty","within":8000},{"actor":"a","do":"reload","settleMs":1000},{"actor":"a","do":"click","testid":"catalog-link"},{"actor":"a","do":"expectNumber","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"plus":5,"relativeTo":"storefront-before","testid":"item-stock","within":8000},{"do":"dbExpectStock","item":"Bluetooth Speaker","plus":5,"relativeTo":"stored-before-control"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-restock-race.json"}],"id":"selected-source-023","scenario":{"level":1,"writeUrlPattern":"\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses"},"source":"scenarios/01-restock-race.json"},{"checkGroups":[{"checkGroupId":"review-eligibility","feature":{"actors":["owner","stranger"],"criteria":[{"id":"108a","steps":[{"action":"buy","actor":"owner","do":"callAction","input":{"attribute":"data-buy-input","contains":"Air Purifier","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"owner","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner","contains":"Air Purifier","do":"expect","testid":"order-item"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"catalog-link"},{"action":"submitReview","actor":"owner","do":"callAction","from":"owner","input":{"attribute":"data-buy-input","contains":"Air Purifier","testid":"item-card"},"namedAction":{"args":[0,4,"eligible review control"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"}],"path":"/api/items/:id/reviews","reducer":"submit_review"},"settleMs":1500},{"actor":"owner","do":"expectActionOutcome","outcome":"accepted"},{"actor":"stranger","do":"openItem","item":"Air Purifier"},{"actor":"stranger","contains":"eligible review control","do":"expect","testid":"review-item"},{"action":"submitReview","actor":"stranger","do":"callAction","from":"owner","input":{"attribute":"data-buy-input","contains":"Air Purifier","testid":"item-card"},"namedAction":{"args":[0,4,"never bought this"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"}],"path":"/api/items/:id/reviews","reducer":"submit_review"},"settleMs":1500},{"actor":"stranger","do":"expectActionOutcome","outcome":"application-refused","routeProvenBy":"owner"},{"actor":"stranger","do":"reload","settleMs":1500},{"actor":"stranger","do":"ensureSignedIn","name":"uma","readyTestid":"current-user"},{"actor":"stranger","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"stranger","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"stranger","do":"openItem","item":"Air Purifier"},{"actor":"stranger","contains":"eligible review control","do":"expect","testid":"review-item"},{"absent":true,"actor":"stranger","contains":"never bought this","do":"expect","testid":"review-item"}]},{"id":"108b","steps":[{"action":"buy","actor":"owner","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"owner","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"openItem","item":"Keyboard"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"review-toggle","unlessVisible":"review-rating"},{"actor":"owner","do":"fill","testid":"review-rating","text":"4"},{"actor":"owner","do":"fill","testid":"review-input","text":"bought and used it"},{"actor":"owner","do":"click","testid":"review-submit"},{"actor":"owner","contains":"bought and used it","do":"expect","testid":"review-item","within":8000}]}],"id":108,"setup":[{"actor":"owner","do":"signUp","name":"tam"},{"actor":"stranger","do":"signUp","name":"uma"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","source":"scenarios/01-review-eligibility.json"}],"id":"selected-source-024","scenario":{"level":1,"writeUrlPattern":"\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses"},"source":"scenarios/01-review-eligibility.json"},{"checkGroups":[{"checkGroupId":"rating","feature":{"actors":["author","other"],"criteria":[{"id":"6c","steps":[{"actor":"other","do":"openItem","item":"Gaming Mouse"},{"actor":"other","do":"click","ifAvailable":true,"testid":"review-toggle","unlessVisible":"review-rating"},{"actor":"other","do":"fill","testid":"review-rating","text":"4"},{"actor":"other","do":"fill","testid":"review-input","text":"does the job"},{"actor":"other","do":"click","testid":"review-submit"},{"actor":"other","do":"expectNumber","equals":3,"testid":"review-average","within":10000},{"actors":["author","other"],"do":"expectAgreement","numeric":true,"testid":"review-average","within":10000}]}],"id":6,"setup":[{"actor":"author","do":"signUp","name":"leon"},{"actor":"other","do":"signUp","name":"maya"},{"action":"buy","actor":"author","do":"callAction","input":{"attribute":"data-buy-input","contains":"Gaming Mouse","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"author","do":"expectActionOutcome","outcome":"accepted"},{"actor":"author","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"author","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"author","contains":"Gaming Mouse","do":"expect","testid":"order-item","within":10000},{"actor":"author","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"author","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"author","do":"openItem","item":"Gaming Mouse"},{"actor":"author","do":"click","ifAvailable":true,"testid":"review-toggle","unlessVisible":"review-rating"},{"actor":"author","do":"fill","testid":"review-rating","text":"2"},{"actor":"author","do":"fill","testid":"review-input","text":"works for travel"},{"actor":"author","do":"click","testid":"review-submit"},{"actor":"author","contains":"works for travel","do":"expect","testid":"review-item"},{"action":"buy","actor":"other","do":"callAction","input":{"attribute":"data-buy-input","contains":"Gaming Mouse","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"other","do":"expectActionOutcome","outcome":"accepted"},{"actor":"other","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"other","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"other","contains":"Gaming Mouse","do":"expect","testid":"order-item","within":10000},{"actor":"other","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"other","do":"click","ifAvailable":true,"testid":"catalog-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","source":"scenarios/01-review-rating-live.json"}],"id":"selected-source-025","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-review-rating-live.json"},{"checkGroups":[{"checkGroupId":"unique-review","feature":{"actors":["author"],"criteria":[{"id":"6b","steps":[{"actor":"author","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"author","do":"click","ifAvailable":true,"testid":"catalog-link"},{"action":"submitReview","actor":"author","do":"callAction","from":"author","input":{"attribute":"data-buy-input","contains":"Air Purifier","testid":"item-card"},"namedAction":{"args":[0,4,"changed my mind"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"}],"path":"/api/items/:id/reviews","reducer":"submit_review"},"settleMs":1500},{"actor":"author","do":"expectActionOutcome","outcome":"completed"},{"actor":"author","do":"reload","settleMs":2500},{"actor":"author","do":"ensureSignedIn","name":"kira","readyTestid":"current-user"},{"actor":"author","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"author","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"author","do":"openItem","item":"Air Purifier"},{"actor":"author","count":1,"do":"expect","testid":"review-item","within":10000}]}],"id":6,"setup":[{"actor":"author","do":"signUp","name":"kira"},{"action":"buy","actor":"author","do":"callAction","input":{"attribute":"data-buy-input","contains":"Air Purifier","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"author","do":"expectActionOutcome","outcome":"accepted"},{"actor":"author","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"author","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"author","contains":"Air Purifier","do":"expect","testid":"order-item","within":10000},{"actor":"author","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"author","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"author","do":"openItem","item":"Air Purifier"},{"actor":"author","do":"click","ifAvailable":true,"testid":"review-toggle","unlessVisible":"review-rating"},{"actor":"author","do":"fill","testid":"review-rating","text":"4"},{"actor":"author","do":"fill","testid":"review-input","text":"quiet and effective"},{"actor":"author","do":"click","testid":"review-submit"},{"actor":"author","contains":"quiet and effective","do":"expect","testid":"review-item"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","source":"scenarios/01-review-uniqueness.json"}],"id":"selected-source-026","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-review-uniqueness.json"},{"checkGroups":[{"checkGroupId":"reviews","feature":{"actors":["author","visitor"],"criteria":[{"id":"6a","steps":[{"actor":"author","do":"openItem","item":"Induction Cooktop"},{"actor":"author","do":"click","ifAvailable":true,"testid":"review-toggle","unlessVisible":"review-rating"},{"actor":"author","do":"fill","testid":"review-rating","text":"4"},{"actor":"author","do":"fill","testid":"review-input","text":"solid mold"},{"actor":"author","do":"click","testid":"review-submit"},{"actor":"author","contains":"solid mold","do":"expect","testid":"review-item"},{"actor":"visitor","do":"openItem","item":"Induction Cooktop"},{"actor":"visitor","contains":"solid mold","do":"expect","testid":"review-item","within":10000}]}],"id":6,"setup":[{"actor":"author","do":"signUp","name":"hal"},{"actor":"author","do":"click","in":{"contains":"Induction Cooktop","testid":"item-card"},"testid":"buy-now"},{"actor":"author","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"author","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"author","contains":"Induction Cooktop","do":"expect","testid":"order-item","within":10000},{"actor":"author","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"author","do":"click","ifAvailable":true,"testid":"catalog-link"}]},"packId":"ecommerce.feature.reviews","role":"feature","source":"scenarios/01-review-visibility.json"}],"id":"selected-source-027","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-review-visibility.json"},{"checkGroups":[{"checkGroupId":"server-price","feature":{"actors":["buyer"],"criteria":[{"id":"104a","steps":[{"action":"buy","actor":"buyer","do":"callAction","input":{"attribute":"data-buy-input","contains":"Espresso Machine","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"},"settleMs":2000},{"actor":"buyer","do":"expectActionOutcome","outcome":"accepted"},{"action":"buy","actor":"buyer","do":"callAction","input":{"attribute":"data-buy-input","contains":"Coffee Grinder","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"},"settleMs":2000},{"actor":"buyer","do":"expectActionOutcome","outcome":"accepted"},{"actor":"buyer","do":"freshClient"},{"actor":"buyer-fresh","do":"signIn","name":"oli"},{"actor":"buyer-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"buyer-fresh","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"buyer-fresh","count":2,"do":"expect","testid":"order-item"},{"actor":"buyer-fresh","contains":"Espresso Machine","count":1,"do":"expect","testid":"order-item"},{"actor":"buyer-fresh","do":"expectNumber","equals":449,"in":{"contains":"Espresso Machine","testid":"order-item"},"testid":"order-total"},{"actor":"buyer-fresh","contains":"Coffee Grinder","count":1,"do":"expect","testid":"order-item"},{"actor":"buyer-fresh","do":"expectNumber","equals":64,"in":{"contains":"Coffee Grinder","testid":"order-item"},"testid":"order-total"}]}],"id":104,"setup":[{"actor":"buyer","do":"signUp","name":"oli"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/01-server-price.json"}],"id":"selected-source-028","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-server-price.json"},{"checkGroups":[{"checkGroupId":"warehouse-area-boundary","feature":{"actors":["admin","staff"],"criteria":[{"id":"7a","steps":[{"actor":"admin","do":"expect","testid":"admin-item-row"},{"actor":"staff","do":"click","ifAvailable":true,"settleMs":1500,"testid":"admin-link"},{"absent":true,"actor":"staff","do":"expect","testid":"admin-item-row"}]}],"id":7,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"admin","do":"click","testid":"admin-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-warehouse-admin-staff.json"},{"checkGroupId":"warehouse-view","feature":{"actors":["admin","staff"],"criteria":[{"id":"7b","steps":[{"actor":"admin","count":13,"do":"expect","testid":"admin-item-row"},{"actor":"admin","count":26,"do":"expect","testid":"admin-location-row"},{"actor":"admin","contains":"East","do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","contains":"West","do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","do":"expectNumber","equals":100,"in":{"contains":"Gaming Mouse","testid":"admin-item-row"},"testid":"admin-stock"}]}],"id":7,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"admin","do":"click","testid":"admin-link"}]},"packId":"ecommerce.feature.warehouse-admin","role":"feature","source":"scenarios/01-warehouse-admin-staff.json"}],"id":"selected-source-029","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-warehouse-admin-staff.json"},{"checkGroups":[{"checkGroupId":"warehouse-stock","feature":{"actors":["admin","visitor"],"criteria":[{"id":"7c","steps":[{"actor":"admin","do":"fill","in":{"contains":"Gaming Mouse","testid":"admin-location-row"},"testid":"restock-input","text":"25"},{"actor":"admin","do":"click","in":{"contains":"Gaming Mouse","testid":"admin-location-row"},"testid":"restock-submit"},{"actor":"visitor","do":"expectNumber","equals":125,"in":{"contains":"Gaming Mouse","testid":"item-card"},"testid":"item-stock","within":10000},{"actor":"admin","do":"expectNumber","equals":125,"in":{"contains":"Gaming Mouse","testid":"admin-item-row"},"testid":"admin-stock"}]}],"id":7,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"visitor","do":"expectNumber","equals":100,"in":{"contains":"Gaming Mouse","testid":"item-card"},"testid":"item-stock"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","requiresFeatures":["ecommerce.feature.catalog-items","ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/01-warehouse-stock-live-staff.json"}],"id":"selected-source-030","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/01-warehouse-stock-live-staff.json"},{"checkGroups":[{"checkGroupId":"cancellation-and-return","feature":{"actors":["customer","staff"],"criteria":[{"id":"3d","steps":[{"actor":"customer","do":"click","in":{"contains":"Coffee Grinder","testid":"item-card"},"testid":"buy-now"},{"actor":"staff","contains":"Coffee Grinder","do":"expect","testid":"queue-item","within":10000},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle"},{"actor":"customer","do":"click","in":{"contains":"Coffee Grinder","testid":"order-item"},"testid":"cancel-order"},{"actor":"staff","contains":"Coffee Grinder","do":"waitUntilAbsent","testid":"queue-item","within":10000}]}],"id":3,"setup":[{"actor":"customer","do":"signUp","name":"cancel-queue"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.cancellation-queue-specifications","requiresFeatures":["ecommerce.l2.order-cancellation-features","ecommerce.progression.fulfilment-queue"],"role":"guarantee","source":"scenarios/02-cancellation-queue.json","stablePackId":"ecommerce.returns-pricing"}],"id":"selected-source-031","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-cancellation-queue.json"},{"checkGroups":[{"checkGroupId":"fulfilment-area-boundary","feature":{"actors":["customer","staff","admin"],"criteria":[{"id":"1d","steps":[{"actor":"staff","do":"click","testid":"staff-link"},{"actor":"staff","do":"expect","testid":"fulfilment-panel"},{"actor":"admin","do":"click","testid":"staff-link"},{"actor":"admin","do":"expect","testid":"fulfilment-panel"},{"actor":"customer","do":"click","ifAvailable":true,"settleMs":1500,"testid":"staff-link"},{"absent":true,"actor":"customer","do":"expect","testid":"fulfilment-panel"}]}],"id":1,"setup":[{"actor":"customer","do":"signUp","name":"fq-customer"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.fulfilment-queue"],"role":"guarantee","source":"scenarios/02-fulfilment-access.json"}],"id":"selected-source-032","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-fulfilment-access.json"},{"checkGroups":[{"checkGroupId":"fulfilment-queue","feature":{"actors":["customer","staff"],"criteria":[{"id":"1a","steps":[{"actor":"staff","as":"depth-before","do":"recordNumber","testid":"queue-depth"},{"actor":"customer","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"buy-now"},{"actor":"staff","contains":"Desk Lamp","do":"expect","testid":"queue-item","within":10000},{"actor":"staff","do":"expectNumber","plus":1,"relativeTo":"depth-before","testid":"queue-depth"}]}],"id":1,"setup":[{"actor":"customer","do":"signUp","name":"fq-live"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","requiresFeatures":["ecommerce.progression.fulfilment-queue"],"role":"guarantee","source":"scenarios/02-fulfilment-live.json"}],"id":"selected-source-033","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-fulfilment-live.json"},{"checkGroups":[{"checkGroupId":"fulfilment-queue","feature":{"actors":["customer","staff"],"criteria":[{"id":"1c","steps":[{"action":"buy","actor":"customer","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"customer","do":"expectActionOutcome","outcome":"accepted"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"pending","within":10000},{"actor":"staff","do":"reload","settleMs":2000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"staff-link"},{"actor":"staff","contains":"Keyboard","do":"expect","testid":"queue-item","within":10000},{"actor":"staff","do":"click","in":{"contains":"Keyboard","testid":"queue-item"},"testid":"ship-submit"},{"actor":"staff","attribute":"data-submit-state","do":"expect","testid":"fulfilment-panel","value":"succeeded","within":10000},{"actor":"staff","do":"reload","settleMs":2000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"staff-link"},{"absent":true,"actor":"staff","contains":"Keyboard","do":"expect","testid":"queue-item","within":10000},{"actor":"customer","do":"reload","settleMs":2000},{"actor":"customer","do":"ensureSignedIn","name":"fq-ship","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"shipped","within":10000}]}],"id":1,"setup":[{"actor":"customer","do":"signUp","name":"fq-ship"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link"}]},"packId":"ecommerce.progression.fulfilment-queue","requiresFeatures":["ecommerce.feature.purchasing"],"role":"feature","source":"scenarios/02-fulfilment-ship.json","stablePackId":"ecommerce.operations-access"}],"id":"selected-source-034","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-fulfilment-ship.json"},{"checkGroups":[{"checkGroupId":"refund-accounting","feature":{"actors":["admin","customer","customer2"],"criteria":[{"id":"203a","steps":[{"actor":"admin","as":"rev-start","do":"recordNumber","testid":"admin-revenue"},{"actor":"customer","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"buy-now"},{"actor":"customer","do":"wait","ms":2500},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"expectNumber","plus":42,"relativeTo":"rev-start","testid":"admin-revenue","within":10000},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-status","value":"pending","within":10000},{"account":"{user:books}","as":"cancel-before","do":"dbRecordCheckout","item":"Desk Lamp","storage":{"cart":false,"kind":"order-data","warehouses":true}},{"action":"cancel","actors":["customer","customer2"],"do":"callConcurrently","from":"customer","input":{"attribute":"data-cancel-input","contains":"Desk Lamp","testid":"order-item"},"namedAction":{"args":[0],"id":"cancel","params":[{"in":"path","name":"orderId","placeholder":":id","wireType":"u64"}],"path":"/api/orders/:id/cancel","reducer":"cancel_order"},"requests":4,"settleMs":3000},{"do":"expectCallOutcomes"},{"before":"cancel-before","do":"dbExpectCancellation"},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"expectNumber","plus":0,"relativeTo":"rev-start","testid":"admin-revenue","within":10000}]}],"id":203,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"customer","do":"signUp","name":"books"},{"actor":"customer2","do":"signIn","name":"books"},{"actor":"admin","do":"click","testid":"admin-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.cancellation-accounting-specifications","requiresFeatures":["ecommerce.l2.order-cancellation-features"],"role":"guarantee","source":"scenarios/02-invariants.json","stablePackId":"ecommerce.returns-pricing"},{"checkGroupId":"refund-accounting","feature":{"actors":["admin","customer","customer2"],"criteria":[{"id":"203b","steps":[{"actor":"admin","as":"history-revenue-before-sale","do":"recordNumber","testid":"admin-revenue"},{"actor":"customer","do":"pressKey","key":"Escape"},{"actor":"customer","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"buy-now"},{"actor":"customer","do":"wait","ms":2500},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"expectNumber","plus":79.5,"relativeTo":"history-revenue-before-sale","testid":"admin-revenue","within":10000},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","contains":"Bluetooth Speaker","count":1,"do":"expect","testid":"order-item","within":10000},{"actor":"admin","as":"rev-after-sale","do":"recordNumber","testid":"admin-revenue"},{"actor":"admin","do":"fill","in":{"contains":"Bluetooth Speaker","testid":"admin-item-row"},"testid":"price-input","text":"5.00"},{"actor":"admin","do":"click","in":{"contains":"Bluetooth Speaker","testid":"admin-item-row"},"testid":"price-submit"},{"actor":"admin","do":"wait","ms":3000},{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link","within":1000},{"actor":"admin","do":"expectNumber","equals":5,"in":{"contains":"Bluetooth Speaker","testid":"admin-item-row"},"testid":"price-input","within":10000},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"expectNumber","plus":0,"relativeTo":"rev-after-sale","testid":"admin-revenue"}]}],"id":203,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"customer","do":"signUp","name":"books"},{"actor":"customer2","do":"signIn","name":"books"},{"actor":"admin","do":"click","testid":"admin-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.price-accounting-specifications","requiresFeatures":["ecommerce.l2.price-history-features"],"role":"guarantee","source":"scenarios/02-invariants.json","stablePackId":"ecommerce.returns-pricing"}],"id":"selected-source-035","scenario":{"level":2,"writeUrlPattern":"\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses|\\/transfer|\\/cancel|\\/return|\\/ship|\\/price|\\/fulfil"},"source":"scenarios/02-invariants.json"},{"checkGroups":[{"checkGroupId":"price-history","feature":{"actors":["admin","visitor"],"criteria":[{"id":"4b","steps":[{"actor":"visitor","atLeast":2,"do":"expectNumber","in":{"contains":"Air Purifier","testid":"item-card"},"testid":"item-price"},{"actor":"admin","do":"fill","in":{"contains":"Air Purifier","testid":"admin-item-row"},"testid":"price-input","text":"1.00"},{"actor":"admin","do":"click","in":{"contains":"Air Purifier","testid":"admin-item-row"},"testid":"price-submit"},{"actor":"visitor","do":"expectNumber","equals":1,"in":{"contains":"Air Purifier","testid":"item-card"},"testid":"item-price","within":10000}]}],"id":4,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"visitor","do":"fill","enter":true,"settleMs":1500,"testid":"search-input","text":"Air Purifier"}]},"packId":"ecommerce.l2.price-history-features","role":"feature","source":"scenarios/02-live-price.json","stablePackId":"ecommerce.returns-pricing"}],"id":"selected-source-036","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-live-price.json"},{"checkGroups":[{"checkGroupId":"operational-views","feature":{"actors":["admin","customer","restocker"],"criteria":[{"id":"5e","steps":[{"actor":"admin","contains":"Air Purifier","do":"expect","testid":"low-stock-item","within":8000}]}],"id":5,"setup":[{"do":"dbSetStock","item":"Air Purifier","quantity":2,"settleMs":250,"warehouse":"East"},{"do":"dbSetStock","item":"Air Purifier","quantity":1,"settleMs":250,"warehouse":"West"},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"customer","do":"signUp","name":"low-stock-focused"},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"low-stock-focused","readyTestid":"current-user"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"low-stock-link","unlessVisible":"low-stock-item","within":10000}]},"packId":"ecommerce.l2.inventory-dashboard","role":"feature","source":"scenarios/02-low-stock.json","stablePackId":"ecommerce.inventory-operations"},{"checkGroupId":"inventory-dashboard","feature":{"actors":["admin","customer","restocker"],"criteria":[{"id":"5a","steps":[{"actor":"admin","contains":"Air Purifier","do":"expect","testid":"low-stock-item","within":8000},{"actor":"restocker","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"restocker","do":"click","testid":"admin-link"},{"actor":"restocker","do":"fill","in":{"contains":"Air Purifier","testid":"admin-location-row"},"testid":"restock-input","text":"8"},{"actor":"restocker","do":"click","in":{"contains":"Air Purifier","testid":"admin-location-row"},"testid":"restock-submit"},{"actor":"admin","contains":"Air Purifier","do":"waitUntilAbsent","testid":"low-stock-item","within":10000},{"actor":"customer","do":"click","in":{"contains":"Air Purifier","testid":"item-card"},"testid":"buy-now"},{"actor":"admin","contains":"Air Purifier","do":"expect","testid":"low-stock-item","within":10000}]}],"id":5,"setup":[{"do":"dbSetStock","item":"Air Purifier","quantity":2,"settleMs":250,"warehouse":"East"},{"do":"dbSetStock","item":"Air Purifier","quantity":1,"settleMs":250,"warehouse":"West"},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"customer","do":"signUp","name":"low-stock-focused"},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"low-stock-focused","readyTestid":"current-user"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"low-stock-link","unlessVisible":"low-stock-item","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.inventory-dashboard"],"role":"guarantee","source":"scenarios/02-low-stock.json"}],"id":"selected-source-037","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-low-stock.json"},{"checkGroups":[{"checkGroupId":"operational-views","feature":{"actors":["buyer","visitor"],"criteria":[{"id":"5d","steps":[{"actor":"visitor","do":"reload","settleMs":2500},{"actor":"visitor","do":"expectNumber","equals":1,"in":{"contains":"Gaming Mouse","testid":"recommended-item"},"testid":"recommendation-rank","within":10000}]}],"id":5,"setup":[{"actor":"buyer","do":"signUp","name":"best-seller-buyer"},{"actor":"buyer","do":"click","in":{"contains":"Gaming Mouse","testid":"item-card"},"testid":"buy-now"},{"actor":"buyer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"buyer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"buyer","contains":"Gaming Mouse","do":"expect","testid":"order-item","within":10000}]},"packId":"ecommerce.l2.sales-dashboard","role":"feature","source":"scenarios/02-operational-best-sellers.json","stablePackId":"ecommerce.inventory-operations"}],"id":"selected-source-038","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-operational-best-sellers.json"},{"checkGroups":[{"checkGroupId":"operational-views","feature":{"actors":["admin","customer"],"criteria":[{"id":"5f","steps":[{"actor":"admin","do":"click","ifAvailable":true,"testid":"sales-link","unlessVisible":"category-row"},{"actor":"admin","as":"audio-core-units","do":"recordNumber","in":{"contains":"Audio","testid":"category-row"},"testid":"category-units"},{"actor":"admin","as":"audio-core-revenue","do":"recordNumber","in":{"contains":"Audio","testid":"category-row"},"testid":"category-revenue"},{"actor":"customer","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"buy-now"},{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"sales-link","unlessVisible":"category-row"},{"actor":"admin","do":"expectNumber","in":{"contains":"Audio","testid":"category-row"},"plus":1,"relativeTo":"audio-core-units","testid":"category-units"},{"actor":"admin","do":"expectNumber","in":{"contains":"Audio","testid":"category-row"},"plus":79.5,"relativeTo":"audio-core-revenue","testid":"category-revenue"}]}],"id":5,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"customer","do":"signUp","name":"category-totals"},{"actor":"admin","do":"click","testid":"admin-link"}]},"packId":"ecommerce.l2.sales-dashboard","role":"feature","source":"scenarios/02-operational-category-totals.json","stablePackId":"ecommerce.inventory-operations"},{"checkGroupId":"sales-dashboard","feature":{"actors":["admin","customer"],"criteria":[{"id":"5b","steps":[{"actor":"admin","do":"click","ifAvailable":true,"testid":"sales-link","unlessVisible":"category-row"},{"actor":"admin","as":"audio-units","do":"recordNumber","in":{"contains":"Audio","testid":"category-row"},"testid":"category-units"},{"actor":"admin","as":"audio-revenue","do":"recordNumber","in":{"contains":"Audio","testid":"category-row"},"testid":"category-revenue"},{"actor":"customer","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"buy-now"},{"actor":"admin","do":"expectNumber","in":{"contains":"Audio","testid":"category-row"},"plus":1,"relativeTo":"audio-units","testid":"category-units","within":10000},{"actor":"admin","do":"expectNumber","in":{"contains":"Audio","testid":"category-row"},"plus":79.5,"relativeTo":"audio-revenue","testid":"category-revenue","within":10000}]}],"id":5,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"customer","do":"signUp","name":"category-totals"},{"actor":"admin","do":"click","testid":"admin-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.sales-dashboard"],"role":"guarantee","source":"scenarios/02-operational-category-totals.json"}],"id":"selected-source-039","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-operational-category-totals.json"},{"checkGroups":[{"checkGroupId":"operational-views","feature":{"actors":["customer"],"criteria":[{"id":"5c","steps":[{"actor":"customer","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"buy-now"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"customer","contains":"Headphones","do":"expect","in":{"testid":"recommended-list"},"testid":"recommended-item","within":10000},{"actor":"customer","do":"click","in":{"contains":"Headphones","testid":"item-card"},"testid":"add-to-cart"},{"actor":"customer","contains":"Headphones","do":"waitUntilAbsent","in":{"testid":"recommended-list"},"testid":"recommended-item","within":10000}]}],"id":5,"setup":[{"actor":"customer","do":"signUp","name":"recommendations"}]},"packId":"ecommerce.l2.recommendations","role":"feature","source":"scenarios/02-operational-recommendations.json","stablePackId":"ecommerce.inventory-operations"}],"id":"selected-source-040","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-operational-recommendations.json"},{"checkGroups":[{"checkGroupId":"cancellation-and-return","feature":{"actors":["customer","admin"],"criteria":[{"id":"3a","steps":[{"actor":"admin","as":"revenue-before","do":"recordNumber","testid":"admin-revenue"},{"actor":"customer","as":"stock-before","do":"recordNumber","in":{"contains":"Coffee Grinder","testid":"item-card"},"testid":"item-stock"},{"actor":"customer","do":"click","in":{"contains":"Coffee Grinder","testid":"item-card"},"testid":"buy-now"},{"actor":"customer","do":"expectNumber","in":{"contains":"Coffee Grinder","testid":"item-card"},"plus":-1,"relativeTo":"stock-before","testid":"item-stock","within":10000},{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link"},{"actor":"admin","do":"expectNumber","plus":64,"relativeTo":"revenue-before","testid":"admin-revenue","within":10000},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","settleMs":2000,"testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Coffee Grinder","testid":"order-item"},"testid":"order-status","value":"pending"},{"actor":"customer","do":"click","in":{"contains":"Coffee Grinder","testid":"order-item"},"testid":"cancel-order"},{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link"},{"actor":"admin","do":"expectNumber","plus":0,"relativeTo":"revenue-before","testid":"admin-revenue","within":10000},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"customer","do":"expectNumber","in":{"contains":"Coffee Grinder","testid":"item-card"},"plus":0,"relativeTo":"stock-before","testid":"item-stock","within":10000}]}],"id":3,"setup":[{"actor":"customer","do":"signUp","name":"cancel-core"},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"}]},"packId":"ecommerce.l2.order-cancellation-features","role":"feature","source":"scenarios/02-order-cancellation-core.json","stablePackId":"ecommerce.returns-pricing"}],"id":"selected-source-041","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-order-cancellation-core.json"},{"checkGroups":[{"checkGroupId":"cancellation-and-return","feature":{"actors":["customer"],"criteria":[{"id":"3b","steps":[{"actor":"customer","do":"click","in":{"contains":"Coffee Grinder","testid":"item-card"},"settleMs":2000,"testid":"buy-now"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"click","in":{"contains":"Coffee Grinder","testid":"order-item"},"testid":"cancel-order"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Coffee Grinder","testid":"order-item"},"testid":"order-status","value":"cancelled","within":10000}]}],"id":3,"setup":[{"actor":"customer","do":"signUp","name":"cancel-focused"}]},"packId":"ecommerce.l2.order-cancellation-features","role":"feature","source":"scenarios/02-order-cancellation-history.json","stablePackId":"ecommerce.returns-pricing"}],"id":"selected-source-042","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-order-cancellation-history.json"},{"checkGroups":[{"checkGroupId":"price-history","feature":{"actors":["admin","customer","visitor"],"criteria":[{"id":"4a","steps":[{"actor":"customer","do":"fill","enter":true,"settleMs":1500,"testid":"search-input","text":"Air Purifier"},{"actor":"customer","as":"air-purifier-paid","do":"recordNumber","in":{"contains":"Air Purifier","testid":"item-card"},"testid":"item-price"},{"actor":"customer","do":"click","in":{"contains":"Air Purifier","testid":"item-card"},"settleMs":2000,"testid":"buy-now"},{"actor":"visitor","do":"fill","enter":true,"settleMs":1500,"testid":"search-input","text":"Air Purifier"},{"actor":"admin","do":"fill","in":{"contains":"Air Purifier","testid":"admin-item-row"},"testid":"price-input","text":"1.00"},{"actor":"admin","do":"click","in":{"contains":"Air Purifier","testid":"admin-item-row"},"testid":"price-submit"},{"actor":"visitor","do":"expectNumber","equals":1,"in":{"contains":"Air Purifier","testid":"item-card"},"testid":"item-price","within":10000},{"actor":"customer","do":"reload","settleMs":2000},{"actor":"customer","do":"ensureSignedIn","name":"history-persisted","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle"},{"actor":"customer","do":"expectNumber","in":{"contains":"Air Purifier","testid":"order-item"},"plus":0,"relativeTo":"air-purifier-paid","testid":"order-total"}]}],"id":4,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"customer","do":"signUp","name":"history-persisted"},{"actor":"admin","do":"click","testid":"admin-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.price-history-specifications","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.price-history-features"],"role":"guarantee","source":"scenarios/02-paid-price-history.json","stablePackId":"ecommerce.returns-pricing"}],"id":"selected-source-043","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-paid-price-history.json"},{"checkGroups":[{"checkGroupId":"fulfilment-queue","feature":{"actors":["customer","staff"],"criteria":[{"id":"1b","steps":[{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":2000,"warehouse":"West"},{"actor":"customer","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"settleMs":1500,"testid":"buy-now"},{"actor":"staff","do":"reload","settleMs":2000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"staff-link"},{"actor":"staff","contains":"Desk Lamp","do":"expect","testid":"queue-item","within":10000},{"actor":"staff","contains":"East","do":"expect","in":{"contains":"Desk Lamp","testid":"queue-item"},"testid":"queue-warehouse"}]}],"id":1,"setup":[{"actor":"customer","do":"signUp","name":"fq-warehouse"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link"}]},"packId":"ecommerce.progression.fulfilment-queue","role":"feature","source":"scenarios/02-queue-warehouse.json","stablePackId":"ecommerce.operations-access"}],"id":"selected-source-044","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-queue-warehouse.json"},{"checkGroups":[{"checkGroupId":"stock-conservation","feature":{"actors":["customer"],"criteria":[{"id":"202b","steps":[{"as":"east-before-cancel-202b","do":"dbRecordStock","item":"Induction Cooktop","warehouse":"East"},{"as":"west-before-cancel-202b","do":"dbRecordStock","item":"Induction Cooktop","warehouse":"West"},{"as":"stored-before-cancel-202b","do":"dbRecordStock","item":"Induction Cooktop"},{"actor":"customer","as":"cancel-stock-before","do":"recordNumber","in":{"contains":"Induction Cooktop","testid":"item-card"},"testid":"item-stock"},{"actor":"customer","do":"click","in":{"contains":"Induction Cooktop","testid":"item-card"},"settleMs":2000,"testid":"buy-now"},{"actor":"customer","do":"expectNumber","in":{"contains":"Induction Cooktop","testid":"item-card"},"plus":-1,"relativeTo":"cancel-stock-before","testid":"item-stock"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","contains":"Induction Cooktop","count":1,"do":"expect","testid":"order-item","within":10000},{"do":"dbExpectStock","item":"Induction Cooktop","plus":-1,"relativeTo":"stored-before-cancel-202b"},{"actor":"customer","do":"click","in":{"contains":"Induction Cooktop","testid":"order-item"},"settleMs":2000,"testid":"cancel-order"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"customer","do":"expectNumber","in":{"contains":"Induction Cooktop","testid":"item-card"},"plus":0,"relativeTo":"cancel-stock-before","testid":"item-stock"},{"do":"dbExpectStock","item":"Induction Cooktop","plus":0,"relativeTo":"stored-before-cancel-202b"},{"do":"dbExpectStock","item":"Induction Cooktop","plus":0,"relativeTo":"east-before-cancel-202b","warehouse":"East"},{"do":"dbExpectStock","item":"Induction Cooktop","plus":0,"relativeTo":"west-before-cancel-202b","warehouse":"West"}]}],"id":202,"setup":[{"actor":"customer","do":"signUp","name":"fresh-stock"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","requiresFeatures":["ecommerce.l2.order-cancellation-features"],"role":"guarantee","source":"scenarios/02-self-contained.json","stablePackId":"ecommerce.inventory-operations"},{"checkGroupId":"stock-conservation","feature":{"actors":["customer"],"criteria":[{"id":"202c","steps":[{"as":"east-before-cancel-202c","do":"dbRecordStock","item":"Headphones","warehouse":"East"},{"as":"west-before-cancel-202c","do":"dbRecordStock","item":"Headphones","warehouse":"West"},{"as":"stored-before-cancel-202c","do":"dbRecordStock","item":"Headphones"},{"actor":"customer","as":"fresh-stock-before","do":"recordNumber","in":{"contains":"Headphones","testid":"item-card"},"testid":"item-stock"},{"actor":"customer","do":"click","in":{"contains":"Headphones","testid":"item-card"},"settleMs":2000,"testid":"buy-now"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","contains":"Headphones","count":1,"do":"expect","testid":"order-item","within":10000},{"do":"dbExpectStock","item":"Headphones","plus":-1,"relativeTo":"stored-before-cancel-202c"},{"actor":"customer","do":"click","in":{"contains":"Headphones","testid":"order-item"},"settleMs":2000,"testid":"cancel-order"},{"actor":"customer","do":"freshClient"},{"actor":"customer-fresh","do":"expectNumber","in":{"contains":"Headphones","testid":"item-card"},"plus":0,"relativeTo":"fresh-stock-before","testid":"item-stock"},{"do":"dbExpectStock","item":"Headphones","plus":0,"relativeTo":"stored-before-cancel-202c"},{"do":"dbExpectStock","item":"Headphones","plus":0,"relativeTo":"east-before-cancel-202c","warehouse":"East"},{"do":"dbExpectStock","item":"Headphones","plus":0,"relativeTo":"west-before-cancel-202c","warehouse":"West"}]}],"id":202,"setup":[{"actor":"customer","do":"signUp","name":"fresh-stock"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","requiresFeatures":["ecommerce.l2.order-cancellation-features"],"role":"guarantee","source":"scenarios/02-self-contained.json","stablePackId":"ecommerce.inventory-operations"}],"id":"selected-source-045","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-self-contained.json"},{"checkGroups":[{"checkGroupId":"operator-authorization","feature":{"actors":["customer","staff"],"criteria":[{"id":"201c","steps":[{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","ifAvailable":true,"testid":"catalog-link","within":1000},{"actor":"customer","do":"click","in":{"contains":"Coffee Grinder","testid":"item-card"},"settleMs":2000,"testid":"buy-now"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","contains":"Coffee Grinder","count":1,"do":"expect","testid":"order-item","within":10000},{"action":"ship","actor":"staff","do":"callAction","from":"customer","input":{"attribute":"data-ship-input","contains":"Coffee Grinder","testid":"order-item"},"namedAction":{"args":[0],"id":"ship","params":[{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/fulfilment/ship","reducer":"ship_order"},"settleMs":2000},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"customer","do":"reload","settleMs":2000},{"actor":"customer","do":"ensureSignedIn","name":"direct-notstaff","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Coffee Grinder","testid":"order-item"},"testid":"order-status","value":"shipped","within":10000},{"action":"ship","actor":"customer","do":"callAction","input":{"attribute":"data-ship-input","contains":"Laptop Stand","testid":"order-item"},"namedAction":{"args":[0],"id":"ship","params":[{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/fulfilment/ship","reducer":"ship_order"},"settleMs":2000},{"actor":"customer","do":"expectActionOutcome","outcome":"refused","routeProvenBy":"staff"},{"actor":"customer","do":"reload","settleMs":2000},{"actor":"customer","do":"ensureSignedIn","name":"direct-notstaff","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Laptop Stand","testid":"order-item"},"testid":"order-status","value":"pending"}]}],"id":201,"setup":[{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"customer","do":"signUp","name":"direct-notstaff"},{"actor":"customer","do":"click","in":{"contains":"Laptop Stand","testid":"item-card"},"settleMs":2000,"testid":"buy-now"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.operations-access-specifications","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"guarantee","source":"scenarios/02-server-actions.json","stablePackId":"ecommerce.operations-access"},{"checkGroupId":"stock-conservation","feature":{"actors":["admin","customer"],"criteria":[{"id":"202d","steps":[{"do":"dbExpectStock","equals":60,"item":"Headphones","warehouse":"East"},{"do":"dbExpectStock","equals":40,"item":"Headphones","warehouse":"West"},{"as":"direct-race-stock-before","do":"dbRecordStock","item":"Headphones"},{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link","within":1000},{"actor":"customer","do":"reload","settleMs":2000},{"actor":"customer","do":"ensureSignedIn","name":"direct-conserve","readyTestid":"current-user"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-from","text":"East"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-to","text":"West"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-qty","text":"25"},{"branches":[[{"action":"transfer","actor":"admin","do":"callAction","input":{"attribute":"data-transfer-input","contains":"Headphones","testid":"admin-item-row"},"namedAction":{"args":[0,0,0,25],"id":"transfer","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"fromWarehouseId","wireType":"u64"},{"in":"body","name":"toWarehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/transfer","reducer":"admin_transfer_stock"}}],[{"action":"buy","actor":"customer","do":"callAction","input":{"attribute":"data-buy-input","contains":"Headphones","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}}]],"do":"race","settleMs":5000},{"actor":"admin","do":"expectActionOutcome","outcome":"accepted"},{"actor":"customer","do":"expectActionOutcome","outcome":"accepted"},{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link","within":1000},{"actor":"admin","do":"expectNumber","in":{"contains":"Headphones","testid":"admin-item-row"},"plus":-1,"relativeTo":"direct-race-stock-before","testid":"admin-stock","within":12000},{"actor":"customer","do":"reload","settleMs":2000},{"actor":"customer","do":"ensureSignedIn","name":"direct-conserve","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","ifAvailable":true,"testid":"catalog-link","within":1000},{"actor":"customer","do":"expectNumber","in":{"contains":"Headphones","testid":"item-card"},"plus":-1,"relativeTo":"direct-race-stock-before","testid":"item-stock","within":12000},{"atLeast":34,"atMost":35,"do":"dbExpectStock","item":"Headphones","warehouse":"East"},{"atLeast":64,"atMost":65,"do":"dbExpectStock","item":"Headphones","warehouse":"West"},{"do":"dbExpectStock","item":"Headphones","plus":-1,"relativeTo":"direct-race-stock-before"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","contains":"Headphones","count":1,"do":"expect","testid":"order-item","within":10000}]}],"id":202,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"customer","do":"signUp","name":"direct-conserve"},{"actor":"admin","do":"click","testid":"admin-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.stock-transfers-features"],"role":"guarantee","source":"scenarios/02-server-actions.json","stablePackId":"ecommerce.inventory-operations"},{"checkGroupId":"order-owner","feature":{"actors":["owner","other"],"criteria":[{"id":"204a","steps":[{"action":"cancel","actor":"owner","do":"callAction","input":{"attribute":"data-cancel-input","contains":"Desk Lamp","testid":"order-item"},"namedAction":{"args":[0],"id":"cancel","params":[{"in":"path","name":"orderId","placeholder":":id","wireType":"u64"}],"path":"/api/orders/:id/cancel","reducer":"cancel_order"},"settleMs":2000},{"actor":"owner","do":"expectActionOutcome","outcome":"accepted"},{"action":"cancel","actor":"other","do":"callAction","from":"owner","input":{"attribute":"data-cancel-input","contains":"Keyboard","testid":"order-item"},"namedAction":{"args":[0],"id":"cancel","params":[{"in":"path","name":"orderId","placeholder":":id","wireType":"u64"}],"path":"/api/orders/:id/cancel","reducer":"cancel_order"},"settleMs":2000},{"actor":"other","do":"expectActionOutcome","outcome":"refused","routeProvenBy":"owner"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"direct-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner","do":"expect","ignoreCase":true,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"pending"}]}],"id":204,"setup":[{"actor":"owner","do":"signUp","name":"direct-owner"},{"actor":"other","do":"signUp","name":"direct-other"},{"action":"buy","actor":"owner","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"owner","do":"expectActionOutcome","outcome":"accepted"},{"action":"buy","actor":"owner","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"owner","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.operations-access-specifications","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.l2.order-cancellation-features"],"role":"guarantee","source":"scenarios/02-server-actions.json","stablePackId":"ecommerce.operations-access"}],"id":"selected-source-046","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-server-actions.json"},{"checkGroups":[{"checkGroupId":"warehouse-transfer","feature":{"actors":["admin","visitor"],"criteria":[{"id":"2a","steps":[{"as":"product-East","do":"dbRecordStock","item":"Headphones","warehouse":"East"},{"as":"product-West","do":"dbRecordStock","item":"Headphones","warehouse":"West"},{"actor":"visitor","as":"transfer-item-before","do":"recordNumber","in":{"contains":"Headphones","testid":"item-card"},"testid":"item-stock"},{"actor":"admin","contains":"East","count":1,"do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","as":"transfer-east-before","do":"recordNumber","in":{"contains":"East","testid":"admin-warehouse-item"},"testid":"warehouse-total"},{"actor":"admin","contains":"West","count":1,"do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","as":"transfer-west-before","do":"recordNumber","in":{"contains":"West","testid":"admin-warehouse-item"},"testid":"warehouse-total"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-from","text":"East"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-to","text":"West"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-qty","text":"10"},{"actor":"admin","do":"click","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-submit"},{"actor":"admin","contains":"East","count":1,"do":"expect","testid":"admin-warehouse-item"},{"do":"dbExpectStock","item":"Headphones","plus":-10,"relativeTo":"product-East","warehouse":"East","within":10000},{"do":"dbExpectStock","item":"Headphones","plus":10,"relativeTo":"product-West","warehouse":"West","within":10000},{"actor":"admin","do":"expectNumber","in":{"contains":"East","testid":"admin-warehouse-item"},"plus":-10,"relativeTo":"transfer-east-before","testid":"warehouse-total","within":10000},{"actor":"admin","contains":"West","count":1,"do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","do":"expectNumber","in":{"contains":"West","testid":"admin-warehouse-item"},"plus":10,"relativeTo":"transfer-west-before","testid":"warehouse-total","within":10000},{"actor":"visitor","do":"expectNumber","in":{"contains":"Headphones","testid":"item-card"},"plus":0,"relativeTo":"transfer-item-before","testid":"item-stock","within":10000}]}],"id":2,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"}]},"packId":"ecommerce.l2.stock-transfers-features","role":"feature","source":"scenarios/02-strengthened.json","stablePackId":"ecommerce.inventory-operations"},{"checkGroupId":"operator-authorization","feature":{"actors":["admin","customer"],"criteria":[{"id":"201a","steps":[{"as":"authorized-transfer-east","do":"dbRecordStock","item":"Headphones","warehouse":"East"},{"as":"authorized-transfer-west","do":"dbRecordStock","item":"Headphones","warehouse":"West"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-from","text":"East"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-to","text":"West"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-qty","text":"25"},{"action":"transfer","actor":"admin","do":"callAction","input":{"attribute":"data-transfer-input","contains":"Headphones","testid":"admin-item-row"},"namedAction":{"args":[0,0,0,25],"id":"transfer","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"fromWarehouseId","wireType":"u64"},{"in":"body","name":"toWarehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/transfer","reducer":"admin_transfer_stock"},"settleMs":2000},{"actor":"admin","do":"expectActionOutcome","outcome":"accepted"},{"do":"dbExpectStock","item":"Headphones","plus":-25,"relativeTo":"authorized-transfer-east","warehouse":"East"},{"do":"dbExpectStock","item":"Headphones","plus":25,"relativeTo":"authorized-transfer-west","warehouse":"West"},{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link","within":1000},{"actor":"customer","do":"reload","settleMs":2000},{"actor":"customer","do":"ensureSignedIn","name":"not-operator","readyTestid":"current-user"},{"actor":"customer","as":"unauthorized-item-before","do":"recordNumber","in":{"contains":"Headphones","testid":"item-card"},"testid":"item-stock"},{"actor":"admin","contains":"East","count":1,"do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","as":"unauthorized-east-before","do":"recordNumber","in":{"contains":"East","testid":"admin-warehouse-item"},"testid":"warehouse-total"},{"actor":"admin","contains":"West","count":1,"do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","as":"unauthorized-west-before","do":"recordNumber","in":{"contains":"West","testid":"admin-warehouse-item"},"testid":"warehouse-total"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-from","text":"East"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-to","text":"West"},{"as":"refused-East","do":"dbRecordStock","item":"Headphones","warehouse":"East"},{"as":"refused-West","do":"dbRecordStock","item":"Headphones","warehouse":"West"},{"action":"transfer","actor":"customer","do":"callAction","from":"admin","input":{"attribute":"data-transfer-input","contains":"Headphones","testid":"admin-item-row"},"namedAction":{"args":[0,0,0,25],"id":"transfer","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"fromWarehouseId","wireType":"u64"},{"in":"body","name":"toWarehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/transfer","reducer":"admin_transfer_stock"},"settleMs":2000},{"actor":"customer","do":"expectActionOutcome","outcome":"refused","routeProvenBy":"admin"},{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link","within":1000},{"actor":"customer","do":"reload","settleMs":2000},{"actor":"customer","do":"ensureSignedIn","name":"not-operator","readyTestid":"current-user"},{"actor":"admin","contains":"East","count":1,"do":"expect","testid":"admin-warehouse-item"},{"do":"dbExpectStock","item":"Headphones","plus":0,"relativeTo":"refused-East","warehouse":"East"},{"do":"dbExpectStock","item":"Headphones","plus":0,"relativeTo":"refused-West","warehouse":"West"},{"actor":"admin","do":"expectNumber","in":{"contains":"East","testid":"admin-warehouse-item"},"plus":0,"relativeTo":"unauthorized-east-before","testid":"warehouse-total"},{"actor":"admin","contains":"West","count":1,"do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","do":"expectNumber","in":{"contains":"West","testid":"admin-warehouse-item"},"plus":0,"relativeTo":"unauthorized-west-before","testid":"warehouse-total"},{"actor":"customer","do":"expectNumber","in":{"contains":"Headphones","testid":"item-card"},"plus":0,"relativeTo":"unauthorized-item-before","testid":"item-stock"}]}],"id":201,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"customer","do":"signUp","name":"not-operator"},{"actor":"admin","do":"click","testid":"admin-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.operations-access-specifications","requiresFeatures":["ecommerce.l2.stock-transfers-features"],"role":"guarantee","source":"scenarios/02-strengthened.json","stablePackId":"ecommerce.operations-access"},{"checkGroupId":"operator-authorization","feature":{"actors":["admin","customer"],"criteria":[{"id":"201b","steps":[{"actor":"admin","do":"fill","in":{"contains":"Gaming Mouse","testid":"admin-item-row"},"testid":"price-input","text":"77.00"},{"action":"price","actor":"admin","do":"callAction","input":{"attribute":"data-price-input","contains":"Gaming Mouse","testid":"admin-item-row"},"namedAction":{"args":[0,1],"id":"price","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"price"}],"path":"/api/admin/price","reducer":"admin_change_price"},"settleMs":2000},{"actor":"admin","do":"expectActionOutcome","outcome":"accepted"},{"actor":"customer","do":"reload","settleMs":2000},{"actor":"customer","do":"ensureSignedIn","name":"not-operator","readyTestid":"current-user"},{"actor":"customer","do":"expectNumber","equals":77,"in":{"contains":"Gaming Mouse","testid":"item-card"},"testid":"item-price","within":10000},{"actor":"admin","do":"fill","in":{"contains":"Gaming Mouse","testid":"admin-item-row"},"testid":"price-input","text":"1.00"},{"action":"price","actor":"customer","do":"callAction","from":"admin","input":{"attribute":"data-price-input","contains":"Gaming Mouse","testid":"admin-item-row"},"namedAction":{"args":[0,1],"id":"price","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"price"}],"path":"/api/admin/price","reducer":"admin_change_price"},"settleMs":2000},{"actor":"customer","do":"expectActionOutcome","outcome":"refused","routeProvenBy":"admin"},{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link","within":1000},{"actor":"customer","do":"reload","settleMs":2000},{"actor":"customer","do":"ensureSignedIn","name":"not-operator","readyTestid":"current-user"},{"actor":"customer","do":"expectNumber","equals":77,"in":{"contains":"Gaming Mouse","testid":"item-card"},"testid":"item-price","within":10000}]}],"id":201,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"customer","do":"signUp","name":"not-operator"},{"actor":"admin","do":"click","testid":"admin-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.operations-access-specifications","requiresFeatures":["ecommerce.l2.price-history-features"],"role":"guarantee","source":"scenarios/02-strengthened.json","stablePackId":"ecommerce.operations-access"},{"checkGroupId":"stock-conservation","feature":{"actors":["admin","customer"],"criteria":[{"id":"202a","steps":[{"as":"product-East","do":"dbRecordStock","item":"Espresso Machine","warehouse":"East"},{"as":"product-West","do":"dbRecordStock","item":"Espresso Machine","warehouse":"West"},{"actor":"customer","as":"conservation-item-before","do":"recordNumber","in":{"contains":"Espresso Machine","testid":"item-card"},"testid":"item-stock"},{"actor":"admin","contains":"East","count":1,"do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","as":"conservation-east-before","do":"recordNumber","in":{"contains":"East","testid":"admin-warehouse-item"},"testid":"warehouse-total"},{"actor":"admin","contains":"West","count":1,"do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","as":"conservation-west-before","do":"recordNumber","in":{"contains":"West","testid":"admin-warehouse-item"},"testid":"warehouse-total"},{"actor":"admin","do":"fill","in":{"contains":"Espresso Machine","testid":"admin-item-row"},"testid":"transfer-from","text":"East"},{"actor":"admin","do":"fill","in":{"contains":"Espresso Machine","testid":"admin-item-row"},"testid":"transfer-to","text":"West"},{"actor":"admin","do":"fill","in":{"contains":"Espresso Machine","testid":"admin-item-row"},"testid":"transfer-qty","text":"17"},{"actor":"admin","do":"click","in":{"contains":"Espresso Machine","testid":"admin-item-row"},"testid":"transfer-submit"},{"actor":"admin","contains":"East","count":1,"do":"expect","testid":"admin-warehouse-item"},{"do":"dbExpectStock","item":"Espresso Machine","plus":-17,"relativeTo":"product-East","warehouse":"East","within":10000},{"do":"dbExpectStock","item":"Espresso Machine","plus":17,"relativeTo":"product-West","warehouse":"West","within":10000},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"expectNumber","in":{"contains":"East","testid":"admin-warehouse-item"},"plus":-17,"relativeTo":"conservation-east-before","testid":"warehouse-total","within":10000},{"actor":"admin","contains":"West","count":1,"do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","do":"expectNumber","in":{"contains":"West","testid":"admin-warehouse-item"},"plus":17,"relativeTo":"conservation-west-before","testid":"warehouse-total","within":10000},{"actor":"customer","do":"expectNumber","in":{"contains":"Espresso Machine","testid":"item-card"},"plus":0,"relativeTo":"conservation-item-before","testid":"item-stock","within":10000}]}],"id":202,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"customer","do":"signUp","name":"directional-stock"},{"actor":"admin","do":"click","testid":"admin-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","requiresFeatures":["ecommerce.l2.stock-transfers-features"],"role":"guarantee","source":"scenarios/02-strengthened.json","stablePackId":"ecommerce.inventory-operations"}],"id":"selected-source-047","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-strengthened.json"},{"checkGroups":[{"checkGroupId":"stock-transfer-overdraw","feature":{"actors":["admin","visitor"],"criteria":[{"id":"2c","steps":[{"as":"product-East","do":"dbRecordStock","item":"Headphones","warehouse":"East"},{"as":"product-West","do":"dbRecordStock","item":"Headphones","warehouse":"West"},{"actor":"visitor","as":"overdraw-item-before","do":"recordNumber","in":{"contains":"Headphones","testid":"item-card"},"testid":"item-stock"},{"actor":"admin","as":"overdraw-east-before","do":"recordNumber","in":{"contains":"East","testid":"admin-warehouse-item"},"testid":"warehouse-total"},{"actor":"admin","as":"overdraw-west-before","do":"recordNumber","in":{"contains":"West","testid":"admin-warehouse-item"},"testid":"warehouse-total"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-from","text":"East"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-to","text":"West"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-qty","text":"99999"},{"actor":"admin","do":"click","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-submit"},{"actor":"admin","do":"expect","testid":"order-error","within":6000},{"do":"dbExpectStock","item":"Headphones","plus":0,"relativeTo":"product-East","warehouse":"East"},{"do":"dbExpectStock","item":"Headphones","plus":0,"relativeTo":"product-West","warehouse":"West"},{"actor":"admin","do":"expectNumber","in":{"contains":"East","testid":"admin-warehouse-item"},"plus":0,"relativeTo":"overdraw-east-before","testid":"warehouse-total"},{"actor":"admin","do":"expectNumber","in":{"contains":"West","testid":"admin-warehouse-item"},"plus":0,"relativeTo":"overdraw-west-before","testid":"warehouse-total"},{"actor":"visitor","do":"expectNumber","in":{"contains":"Headphones","testid":"item-card"},"plus":0,"relativeTo":"overdraw-item-before","testid":"item-stock"}]}],"id":2,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","requiresFeatures":["ecommerce.l2.stock-transfers-features"],"role":"guarantee","source":"scenarios/02-transfer-overdraw.json"}],"id":"selected-source-048","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-transfer-overdraw.json"},{"checkGroups":[{"checkGroupId":"stock-transfers","feature":{"actors":["admin"],"criteria":[{"id":"2b","steps":[{"as":"product-East","do":"dbRecordStock","item":"Headphones","warehouse":"East"},{"as":"product-West","do":"dbRecordStock","item":"Headphones","warehouse":"West"},{"actor":"admin","contains":"East","count":1,"do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","as":"warehouse-east-before","do":"recordNumber","in":{"contains":"East","testid":"admin-warehouse-item"},"testid":"warehouse-total"},{"actor":"admin","contains":"West","count":1,"do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","as":"warehouse-west-before","do":"recordNumber","in":{"contains":"West","testid":"admin-warehouse-item"},"testid":"warehouse-total"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-from","text":"East"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-to","text":"West"},{"actor":"admin","do":"fill","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-qty","text":"10"},{"actor":"admin","do":"click","in":{"contains":"Headphones","testid":"admin-item-row"},"testid":"transfer-submit"},{"actor":"admin","contains":"East","count":1,"do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","do":"expectNumber","in":{"contains":"East","testid":"admin-warehouse-item"},"plus":-10,"relativeTo":"warehouse-east-before","testid":"warehouse-total","within":10000},{"actor":"admin","contains":"West","count":1,"do":"expect","testid":"admin-warehouse-item"},{"actor":"admin","do":"expectNumber","in":{"contains":"West","testid":"admin-warehouse-item"},"plus":10,"relativeTo":"warehouse-west-before","testid":"warehouse-total","within":10000},{"do":"dbExpectStock","item":"Headphones","plus":-10,"relativeTo":"product-East","warehouse":"East","within":10000},{"do":"dbExpectStock","item":"Headphones","plus":10,"relativeTo":"product-West","warehouse":"West","within":10000}]}],"id":2,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","requiresFeatures":["ecommerce.l2.stock-transfers-features"],"role":"guarantee","source":"scenarios/02-transfer-totals.json"}],"id":"selected-source-049","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/02-transfer-totals.json"},{"checkGroups":[{"checkGroupId":"cart-expiration","feature":{"actors":["customer","watcher"],"criteria":[{"id":"304a","steps":[{"actor":"watcher","do":"wait","ms":310000},{"actor":"watcher","do":"reload","settleMs":1000},{"actor":"watcher","do":"expectNumber","in":{"contains":"Air Purifier","testid":"item-card"},"plus":0,"relativeTo":"before","testid":"item-stock","within":30000},{"actor":"customer","do":"openClient","settleMs":3000},{"actor":"customer","do":"expect","testid":"cart-expired-notice","within":10000},{"actor":"customer","do":"expectNumber","equals":0,"testid":"cart-count"}]}],"id":304,"setup":[{"actor":"customer","do":"signUp","name":"cart-expiry"},{"actor":"watcher","as":"before","do":"recordNumber","in":{"contains":"Air Purifier","testid":"item-card"},"testid":"item-stock"},{"actor":"customer","do":"click","in":{"contains":"Air Purifier","testid":"item-card"},"testid":"add-to-cart"},{"actor":"watcher","do":"reload","settleMs":1000},{"actor":"watcher","do":"expectNumber","in":{"contains":"Air Purifier","testid":"item-card"},"plus":-1,"relativeTo":"before","testid":"item-stock","within":10000},{"actor":"customer","do":"closeClient"}]},"packId":"ecommerce.l3.cart-expiration-features","role":"feature","source":"scenarios/03-cart-expiration.json","stablePackId":"ecommerce.l3.cart-expiration"}],"id":"selected-source-050","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/03-cart-expiration.json"},{"checkGroups":[{"checkGroupId":"scheduled-work-access","feature":{"actors":["admin","customer"],"criteria":[{"id":"317a","steps":[{"actor":"admin","do":"fill","testid":"schedule-restock-item","text":"Webcam"},{"actor":"admin","do":"fill","testid":"schedule-restock-warehouse","text":"West"},{"actor":"admin","do":"fill","testid":"schedule-restock-qty","text":"3"},{"actor":"admin","do":"fill","testid":"schedule-restock-delay","text":"180"},{"action":"scheduleRestock","actor":"customer","authentication":"actor","do":"callAction","from":"admin","input":{"attribute":"data-action-input","testid":"schedule-restock-submit"},"namedAction":{"args":["","",0,0],"id":"scheduleRestock","method":"POST","params":[{"in":"body","name":"item"},{"in":"body","name":"warehouse"},{"in":"body","name":"quantity"},{"in":"body","name":"delaySeconds"}],"path":"/api/admin/scheduled-restocks","reducer":"schedule_restock"},"settleMs":2000},{"actor":"customer","do":"expectActionOutcome","outcome":"refused"},{"actor":"customer","do":"replayAs","from":"admin","match":"DELETE","namedAction":{"args":[0],"id":"cancelScheduledRestock","method":"DELETE","params":[{"in":"path","name":"restockId","placeholder":"{restockId}","wireType":"u64"}],"path":"/api/admin/scheduled-restocks/{restockId}","reducer":"cancel_scheduled_restock"},"namedTarget":{"attribute":"data-entity-id","testid":"pending-restock-item","valueType":"string"},"settleMs":2000},{"actor":"customer","do":"expectReplayRejected"},{"actor":"admin","count":1,"do":"expect","testid":"pending-restock-item"},{"actor":"admin","do":"click","in":{"testid":"pending-restock-item"},"testid":"pending-restock-cancel"}]}],"id":317,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link"},{"actor":"customer","do":"signUp","name":"restock-outsider"},{"actor":"admin","do":"fill","testid":"schedule-restock-item","text":"Webcam"},{"actor":"admin","do":"fill","testid":"schedule-restock-warehouse","text":"West"},{"actor":"admin","do":"fill","testid":"schedule-restock-qty","text":"3"},{"actor":"admin","do":"fill","testid":"schedule-restock-delay","text":"180"},{"actor":"admin","do":"click","testid":"schedule-restock-submit"},{"actor":"admin","count":1,"do":"expect","testid":"pending-restock-item"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-access-specifications","requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"role":"guarantee","source":"scenarios/03-deferred-access.json","stablePackId":"ecommerce.l3.deferred-access"}],"id":"selected-source-051","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/03-deferred-access.json"},{"checkGroups":[{"checkGroupId":"restart-survival","feature":{"actors":["admin"],"criteria":[{"id":"311a","steps":[{"do":"dbExpectStock","item":"Air Purifier","plus":5,"relativeTo":"before","within":70000},{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link"},{"actor":"admin","do":"expect","testid":"schedule-restock-submit"},{"absent":true,"actor":"admin","do":"expect","testid":"pending-restock-item"}]}],"id":311,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link"},{"as":"ordinaryBefore","do":"dbRecordStock","item":"Air Purifier"},{"actor":"admin","do":"fill","testid":"schedule-restock-item","text":"Air Purifier"},{"actor":"admin","do":"fill","testid":"schedule-restock-warehouse","text":"East"},{"actor":"admin","do":"fill","testid":"schedule-restock-qty","text":"5"},{"actor":"admin","do":"fill","testid":"schedule-restock-delay","text":"45"},{"actor":"admin","do":"click","testid":"schedule-restock-submit"},{"actor":"admin","count":1,"do":"expect","testid":"pending-restock-item","within":10000},{"do":"dbExpectStock","item":"Air Purifier","plus":5,"relativeTo":"ordinaryBefore","within":70000},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link"},{"actor":"admin","do":"expect","testid":"schedule-restock-submit"},{"absent":true,"actor":"admin","do":"expect","testid":"pending-restock-item"},{"as":"before","do":"dbRecordStock","item":"Air Purifier"},{"actor":"admin","do":"fill","testid":"schedule-restock-item","text":"Air Purifier"},{"actor":"admin","do":"fill","testid":"schedule-restock-warehouse","text":"East"},{"actor":"admin","do":"fill","testid":"schedule-restock-qty","text":"5"},{"actor":"admin","do":"fill","testid":"schedule-restock-delay","text":"45"},{"actor":"admin","do":"click","testid":"schedule-restock-submit"},{"actor":"admin","count":1,"do":"expect","testid":"pending-restock-item","within":10000},{"do":"dbExpectStock","item":"Air Purifier","plus":0,"relativeTo":"before"},{"do":"restartBackend","settleMs":15000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-durability-specifications","requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"role":"guarantee","source":"scenarios/03-deferred-durability.json","stablePackId":"ecommerce.l3.deferred-durability"},{"checkGroupId":"restart-survival","feature":{"actors":["customer","watcher"],"criteria":[{"id":"314a","steps":[{"atMost":70000,"do":"expectElapsed","since":"pending-314"},{"actor":"watcher","do":"reload","settleMs":1000},{"atMost":70000,"do":"expectElapsed","since":"pending-314"},{"actor":"watcher","do":"expectNumber","in":{"contains":"Desk Lamp","testid":"item-card"},"plus":-1,"relativeTo":"before","testid":"item-stock","within":1000},{"atMost":70000,"do":"expectElapsed","since":"pending-314"},{"actor":"customer","do":"wait","ms":110000,"since":"pending-314-accepted"},{"actor":"watcher","do":"reload","settleMs":2000},{"actor":"watcher","do":"expectNumber","in":{"contains":"Desk Lamp","testid":"item-card"},"plus":0,"relativeTo":"before","testid":"item-stock","within":10000},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"durable-reservation","readyTestid":"current-user"},{"actor":"customer","do":"click","testid":"cart-toggle","unlessVisible":"cart-item"},{"actor":"customer","do":"expect","in":{"contains":"Desk Lamp","testid":"cart-item"},"testid":"cart-item-expired","within":10000}]}],"id":314,"setup":[{"actor":"customer","do":"signUp","name":"durable-reservation"},{"actor":"watcher","as":"before","do":"recordNumber","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock"},{"as":"pending-314","do":"recordTime"},{"actor":"customer","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"add-to-cart"},{"actor":"watcher","do":"reload","settleMs":1000},{"actor":"watcher","do":"expectNumber","in":{"contains":"Desk Lamp","testid":"item-card"},"plus":-1,"relativeTo":"before","testid":"item-stock","within":10000},{"as":"pending-314-accepted","do":"recordTime"},{"actor":"customer","do":"wait","ms":30000},{"do":"restartBackend","settleMs":15000},{"actor":"watcher","do":"reload","settleMs":3000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-durability-specifications","requiresFeatures":["ecommerce.l3.reservations-features"],"role":"guarantee","source":"scenarios/03-deferred-durability.json","stablePackId":"ecommerce.l3.deferred-durability"},{"checkGroupId":"restart-survival","feature":{"actors":["customer","staff"],"criteria":[{"id":"315a","steps":[{"actor":"customer","do":"wait","ms":75000,"since":"delivery-start-accepted"},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"durable-delivery","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"delivered","within":10000}]}],"id":315,"setup":[{"actor":"customer","do":"signUp","name":"durable-delivery"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link"},{"actor":"customer","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"buy-now"},{"actor":"staff","do":"reload","settleMs":1000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"queue-item"},{"actor":"staff","contains":"Keyboard","do":"expect","testid":"queue-item","within":10000},{"actor":"staff","do":"click","in":{"contains":"Keyboard","testid":"queue-item"},"testid":"ship-submit"},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"durable-delivery","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"shipped","within":10000},{"as":"delivery-start-accepted","do":"recordTime"},{"actor":"customer","do":"wait","ms":20000},{"do":"restartBackend","settleMs":15000},{"actor":"customer","do":"reload","settleMs":3000},{"actor":"customer","do":"ensureSignedIn","name":"durable-delivery","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-durability-specifications","requiresFeatures":["ecommerce.l3.order-delivery-features"],"role":"guarantee","source":"scenarios/03-deferred-durability.json","stablePackId":"ecommerce.l3.deferred-durability"},{"checkGroupId":"restart-survival","feature":{"actors":["customer","watcher"],"criteria":[{"id":"316a","steps":[{"atMost":250000,"do":"expectElapsed","since":"pending-316"},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"durable-cart","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"cart-toggle","unlessVisible":"cart-item"},{"atMost":250000,"do":"expectElapsed","since":"pending-316"},{"actor":"customer","do":"expectNumber","equals":1,"testid":"cart-count","within":1000},{"atMost":250000,"do":"expectElapsed","since":"pending-316"},{"actor":"customer","do":"wait","ms":310000,"since":"pending-316-accepted"},{"actor":"customer","do":"reload","settleMs":3000},{"actor":"customer","do":"ensureSignedIn","name":"durable-cart","readyTestid":"current-user"},{"actor":"customer","do":"click","testid":"cart-toggle"},{"actor":"customer","do":"expectNumber","equals":0,"testid":"cart-count","within":10000},{"actor":"customer","do":"expect","testid":"cart-expired-notice","within":10000},{"actor":"watcher","do":"reload","settleMs":1000},{"actor":"watcher","do":"expectNumber","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"plus":0,"relativeTo":"before","testid":"item-stock","within":10000}]}],"id":316,"setup":[{"actor":"customer","do":"signUp","name":"durable-cart"},{"actor":"watcher","as":"before","do":"recordNumber","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"item-stock"},{"as":"pending-316","do":"recordTime"},{"actor":"customer","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"add-to-cart"},{"actor":"watcher","do":"reload","settleMs":1000},{"actor":"watcher","do":"expectNumber","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"plus":-1,"relativeTo":"before","testid":"item-stock","within":10000},{"as":"pending-316-accepted","do":"recordTime"},{"actor":"customer","do":"wait","ms":120000},{"do":"restartBackend","settleMs":15000},{"actor":"watcher","do":"reload","settleMs":3000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-durability-specifications","requiresFeatures":["ecommerce.l3.cart-expiration-features"],"role":"guarantee","source":"scenarios/03-deferred-durability.json","stablePackId":"ecommerce.l3.deferred-durability"}],"id":"selected-source-052","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/03-deferred-durability.json"},{"checkGroups":[{"checkGroupId":"exactly-once","feature":{"actors":["admin","watcher"],"criteria":[{"id":"311a","steps":[{"actor":"watcher","do":"reload","settleMs":2000},{"do":"dbExpectStock","item":"Air Purifier","plus":5,"relativeTo":"before","within":45000},{"do":"restartBackend","settleMs":15000},{"actor":"watcher","do":"reload","settleMs":3000},{"actor":"watcher","do":"wait","ms":15000},{"do":"dbExpectStock","item":"Air Purifier","plus":5,"relativeTo":"before"}]}],"id":311,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link"},{"as":"before","do":"dbRecordStock","item":"Air Purifier"},{"actor":"admin","do":"fill","testid":"schedule-restock-item","text":"Air Purifier"},{"actor":"admin","do":"fill","testid":"schedule-restock-warehouse","text":"East"},{"actor":"admin","do":"fill","testid":"schedule-restock-qty","text":"5"},{"actor":"admin","do":"fill","testid":"schedule-restock-delay","text":"20"},{"actor":"admin","do":"click","testid":"schedule-restock-submit"},{"actor":"admin","count":1,"do":"expect","testid":"pending-restock-item","within":10000},{"do":"dbExpectStock","item":"Air Purifier","plus":0,"relativeTo":"before"},{"do":"restartBackend","settleMs":15000},{"actor":"watcher","do":"reload","settleMs":3000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-integrity-specifications","requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"role":"guarantee","source":"scenarios/03-deferred-integrity.json","stablePackId":"ecommerce.l3.deferred-integrity"},{"checkGroupId":"exactly-once","feature":{"actors":["customer","staff"],"criteria":[{"id":"312a","steps":[{"actor":"staff","do":"reload","settleMs":2000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"staff-link"},{"actor":"staff","contains":"Desk Lamp","count":1,"do":"expect","testid":"completed-order-item"},{"actor":"staff","do":"expect","ignoreCase":true,"in":{"contains":"Desk Lamp","testid":"completed-order-item"},"testid":"completed-order-status","value":"delivered"}]}],"id":312,"setup":[{"actor":"customer","do":"signUp","name":"once-delivery"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link"},{"actor":"customer","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"buy-now"},{"actor":"staff","do":"reload","settleMs":1000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"queue-item"},{"actor":"staff","contains":"Desk Lamp","do":"expect","testid":"queue-item","within":10000},{"actor":"staff","do":"click","in":{"contains":"Desk Lamp","testid":"queue-item"},"testid":"ship-submit"},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"once-delivery","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-status","value":"shipped","within":10000},{"actor":"customer","do":"wait","ms":70000},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"once-delivery","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-status","value":"delivered","within":15000},{"do":"restartBackend","settleMs":15000},{"actor":"staff","do":"reload","settleMs":3000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"staff-link","within":1000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-integrity-specifications","requiresFeatures":["ecommerce.l3.order-delivery-features"],"role":"guarantee","source":"scenarios/03-deferred-integrity.json","stablePackId":"ecommerce.l3.deferred-integrity"},{"checkGroupId":"stock-conservation","feature":{"actors":["customer","watcher"],"criteria":[{"id":"313a","steps":[{"actor":"watcher","do":"wait","ms":100000},{"actor":"watcher","do":"reload","settleMs":1000},{"actor":"watcher","do":"expectNumber","in":{"contains":"Keyboard","testid":"item-card"},"plus":0,"relativeTo":"before","testid":"item-stock","within":30000}]}],"id":313,"setup":[{"actor":"customer","do":"signUp","name":"conserve-expiry"},{"actor":"watcher","as":"before","do":"recordNumber","in":{"contains":"Keyboard","testid":"item-card"},"testid":"item-stock"},{"actor":"customer","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"actor":"watcher","do":"reload","settleMs":1000},{"actor":"watcher","do":"expectNumber","in":{"contains":"Keyboard","testid":"item-card"},"plus":-1,"relativeTo":"before","testid":"item-stock","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-integrity-specifications","requiresFeatures":["ecommerce.l3.reservations-features"],"role":"guarantee","source":"scenarios/03-deferred-integrity.json","stablePackId":"ecommerce.l3.deferred-integrity"},{"checkGroupId":"stock-conservation","feature":{"actors":["customer","watcher"],"criteria":[{"id":"314a","steps":[{"actor":"customer","do":"click","testid":"checkout-submit"},{"actor":"customer","do":"expectNumber","equals":0,"testid":"cart-count","within":10000},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"conserve-checkout","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle"},{"actor":"customer","contains":"Headphones","do":"expectElementCount","equals":1,"testid":"order-item","within":10000},{"actor":"watcher","do":"reload","settleMs":1000},{"actor":"watcher","do":"expectNumber","in":{"contains":"Headphones","testid":"item-card"},"plus":-1,"relativeTo":"before","testid":"item-stock","within":10000}]}],"id":314,"setup":[{"actor":"customer","do":"signUp","name":"conserve-checkout"},{"actor":"watcher","as":"before","do":"recordNumber","in":{"contains":"Headphones","testid":"item-card"},"testid":"item-stock"},{"actor":"customer","do":"click","in":{"contains":"Headphones","testid":"item-card"},"testid":"add-to-cart"},{"actor":"customer","do":"click","testid":"cart-toggle"},{"actor":"watcher","do":"reload","settleMs":1000},{"actor":"watcher","do":"expectNumber","in":{"contains":"Headphones","testid":"item-card"},"plus":-1,"relativeTo":"before","testid":"item-stock","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.l3.deferred-integrity-specifications","requiresFeatures":["ecommerce.l3.reservations-features"],"role":"guarantee","source":"scenarios/03-deferred-integrity.json","stablePackId":"ecommerce.l3.deferred-integrity"}],"id":"selected-source-053","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/03-deferred-integrity.json"},{"checkGroups":[{"checkGroupId":"order-delivery","feature":{"actors":["customer","staff"],"criteria":[{"id":"303a","steps":[{"actor":"customer","do":"wait","ms":70000},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"delivery-live","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Gaming Mouse","testid":"order-item"},"testid":"order-status","value":"delivered","within":15000},{"actor":"staff","do":"reload","settleMs":1000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"completed-order-item"},{"actor":"staff","do":"expect","ignoreCase":true,"in":{"contains":"Gaming Mouse","testid":"completed-order-item"},"testid":"completed-order-status","value":"delivered","within":15000}]}],"id":303,"setup":[{"actor":"customer","do":"signUp","name":"delivery-live"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link"},{"actor":"customer","do":"click","in":{"contains":"Gaming Mouse","testid":"item-card"},"testid":"buy-now"},{"actor":"staff","do":"reload","settleMs":1000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"queue-item"},{"actor":"staff","contains":"Gaming Mouse","do":"expect","testid":"queue-item","within":10000},{"actor":"staff","do":"click","in":{"contains":"Gaming Mouse","testid":"queue-item"},"testid":"ship-submit"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"delivery-live","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Gaming Mouse","testid":"order-item"},"testid":"order-status","value":"shipped","within":10000}]},"packId":"ecommerce.l3.order-delivery-features","role":"feature","source":"scenarios/03-order-delivery.json","stablePackId":"ecommerce.l3.order-delivery"},{"checkGroupId":"order-delivery","feature":{"actors":["customer"],"criteria":[{"id":"305a","steps":[{"actor":"customer","do":"wait","ms":70000},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"delivery-cancel","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"cancelled"}]}],"id":305,"setup":[{"actor":"customer","do":"signUp","name":"delivery-cancel"},{"actor":"customer","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"buy-now"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle"},{"actor":"customer","do":"click","in":{"contains":"Keyboard","testid":"order-item"},"testid":"cancel-order"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"cancelled","within":10000}]},"packId":"ecommerce.l3.order-delivery-features","role":"feature","source":"scenarios/03-order-delivery.json","stablePackId":"ecommerce.l3.order-delivery"}],"id":"selected-source-054","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/03-order-delivery.json"},{"checkGroups":[{"checkGroupId":"reservations","feature":{"actors":["shopper","watcher"],"criteria":[{"id":"301a","steps":[{"actor":"shopper","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"add-to-cart"},{"actor":"watcher","do":"expectNumber","in":{"contains":"Desk Lamp","testid":"item-card"},"plus":-1,"relativeTo":"before","testid":"item-stock","within":10000}]}],"id":301,"setup":[{"actor":"shopper","do":"signUp","name":"reserve-stock"},{"actor":"watcher","as":"before","do":"recordNumber","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock"}]},"packId":"ecommerce.l3.reservations-features","role":"feature","source":"scenarios/03-reservations.json","stablePackId":"ecommerce.l3.reservations"},{"checkGroupId":"reservations","feature":{"actors":["shopper"],"criteria":[{"id":"305a","steps":[{"actor":"shopper","atLeast":1,"atMost":90,"do":"expectNumber","in":{"contains":"Gaming Mouse","testid":"cart-item"},"testid":"cart-reservation-timer"},{"actor":"shopper","as":"initial-countdown","do":"recordNumber","in":{"contains":"Gaming Mouse","testid":"cart-item"},"testid":"cart-reservation-timer"},{"actor":"shopper","do":"wait","ms":1000},{"actor":"shopper","comparison":"atMost","do":"expectNumber","in":{"contains":"Gaming Mouse","testid":"cart-item"},"plus":-1,"relativeTo":"initial-countdown","testid":"cart-reservation-timer","within":10000}]}],"id":305,"setup":[{"actor":"shopper","do":"signUp","name":"reserve-clock"},{"actor":"shopper","do":"click","in":{"contains":"Gaming Mouse","testid":"item-card"},"testid":"add-to-cart"},{"actor":"shopper","do":"click","testid":"cart-toggle"}]},"packId":"ecommerce.l3.reservations-features","role":"feature","source":"scenarios/03-reservations.json","stablePackId":"ecommerce.l3.reservations"},{"checkGroupId":"reservations","feature":{"actors":["shopper"],"criteria":[{"id":"306a","steps":[{"actor":"shopper","do":"click","testid":"checkout-submit"},{"actor":"shopper","do":"expectNumber","equals":0,"testid":"cart-count","within":10000},{"actor":"shopper","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"shopper","do":"click","testid":"orders-toggle"},{"actor":"shopper","contains":"Keyboard","count":1,"do":"expect","testid":"order-item"}]}],"id":306,"setup":[{"actor":"shopper","do":"signUp","name":"reserve-checkout"},{"actor":"shopper","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"actor":"shopper","do":"click","testid":"cart-toggle"}]},"packId":"ecommerce.l3.reservations-features","role":"feature","source":"scenarios/03-reservations.json","stablePackId":"ecommerce.l3.reservations"},{"checkGroupId":"reservations","feature":{"actors":["shopper"],"criteria":[{"id":"307a","steps":[{"actor":"shopper","do":"wait","ms":100000},{"actor":"shopper","do":"expect","in":{"contains":"Bluetooth Speaker","testid":"cart-item"},"testid":"cart-item-expired","within":15000}]}],"id":307,"setup":[{"actor":"shopper","do":"signUp","name":"reserve-expiry"},{"actor":"shopper","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"add-to-cart"},{"actor":"shopper","do":"click","testid":"cart-toggle"}]},"packId":"ecommerce.l3.reservations-features","role":"feature","source":"scenarios/03-reservations.json","stablePackId":"ecommerce.l3.reservations"},{"checkGroupId":"reservations","feature":{"actors":["shopper"],"criteria":[{"id":"308a","steps":[{"actor":"shopper","do":"wait","ms":40000},{"actor":"shopper","atLeast":35,"do":"expectNumber","in":{"contains":"Headphones","testid":"cart-item"},"testid":"cart-reservation-timer"},{"absent":true,"actor":"shopper","do":"expect","in":{"contains":"Headphones","testid":"cart-item"},"testid":"cart-item-expired"}]}],"id":308,"setup":[{"actor":"shopper","do":"signUp","name":"reserve-renew"},{"actor":"shopper","do":"click","in":{"contains":"Headphones","testid":"item-card"},"testid":"add-to-cart"},{"actor":"shopper","do":"wait","ms":60000},{"actor":"shopper","do":"click","in":{"contains":"Headphones","testid":"item-card"},"testid":"add-to-cart"},{"actor":"shopper","do":"click","testid":"cart-toggle"}]},"packId":"ecommerce.l3.reservations-features","role":"feature","source":"scenarios/03-reservations.json","stablePackId":"ecommerce.l3.reservations"}],"id":"selected-source-055","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/03-reservations.json"},{"checkGroups":[{"checkGroupId":"scheduled-restocks","feature":{"actors":["admin","watcher"],"criteria":[{"id":"305a","steps":[{"actor":"admin","as":"ledger-before","count":true,"do":"recordNumber","testid":"stock-ledger-entry"},{"actor":"admin","do":"fill","testid":"schedule-restock-item","text":"Keyboard"},{"actor":"admin","do":"fill","testid":"schedule-restock-warehouse","text":"West"},{"actor":"admin","do":"fill","testid":"schedule-restock-qty","text":"7"},{"actor":"admin","do":"fill","testid":"schedule-restock-delay","text":"15"},{"actor":"admin","do":"click","testid":"schedule-restock-submit"},{"actor":"admin","do":"wait","ms":25000},{"actor":"watcher","do":"reload","settleMs":2000},{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link"},{"actor":"watcher","do":"expectNumber","in":{"contains":"Keyboard","testid":"item-card"},"plus":7,"relativeTo":"before","testid":"item-stock","within":15000},{"absent":true,"actor":"admin","do":"expect","testid":"pending-restock-item"},{"actor":"admin","do":"expectElementCount","plus":1,"relativeTo":"ledger-before","testid":"stock-ledger-entry"}]}],"id":305,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link"},{"actor":"watcher","as":"before","do":"recordNumber","in":{"contains":"Keyboard","testid":"item-card"},"testid":"item-stock"}]},"packId":"ecommerce.l3.scheduled-restocks-features","role":"feature","source":"scenarios/03-scheduled-restock-apply.json","stablePackId":"ecommerce.l3.scheduled-restocks"}],"id":"selected-source-056","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/03-scheduled-restock-apply.json"},{"checkGroups":[{"checkGroupId":"scheduled-restocks","feature":{"actors":["admin","watcher"],"criteria":[{"id":"306a","steps":[{"actor":"admin","do":"fill","testid":"schedule-restock-item","text":"Desk Lamp"},{"actor":"admin","do":"fill","testid":"schedule-restock-warehouse","text":"East"},{"actor":"admin","do":"fill","testid":"schedule-restock-qty","text":"9"},{"actor":"admin","do":"fill","testid":"schedule-restock-delay","text":"15"},{"actor":"admin","do":"click","testid":"schedule-restock-submit"},{"actor":"admin","count":1,"do":"expect","testid":"pending-restock-item"},{"actor":"admin","do":"click","in":{"testid":"pending-restock-item"},"testid":"pending-restock-cancel"},{"actor":"admin","do":"wait","ms":25000},{"actor":"watcher","do":"reload","settleMs":2000},{"actor":"watcher","do":"expectNumber","in":{"contains":"Desk Lamp","testid":"item-card"},"plus":0,"relativeTo":"before","testid":"item-stock"}]}],"id":306,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link"},{"actor":"watcher","as":"before","do":"recordNumber","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock"}]},"packId":"ecommerce.l3.scheduled-restocks-features","role":"feature","source":"scenarios/03-scheduled-restock-cancel.json","stablePackId":"ecommerce.l3.scheduled-restocks"}],"id":"selected-source-057","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/03-scheduled-restock-cancel.json"},{"checkGroups":[{"checkGroupId":"scheduled-restocks","feature":{"actors":["admin"],"criteria":[{"id":"302a","steps":[{"actor":"admin","do":"fill","testid":"schedule-restock-item","text":"Webcam"},{"actor":"admin","do":"fill","testid":"schedule-restock-warehouse","text":"East"},{"actor":"admin","do":"fill","testid":"schedule-restock-qty","text":"7"},{"actor":"admin","do":"fill","testid":"schedule-restock-delay","text":"90"},{"actor":"admin","do":"click","testid":"schedule-restock-submit"},{"actor":"admin","count":1,"do":"expect","testid":"pending-restock-item","within":10000},{"actor":"admin","atLeast":1,"atMost":90,"do":"expectNumber","in":{"testid":"pending-restock-item"},"testid":"pending-restock-remaining"},{"actor":"admin","as":"initial-countdown","do":"recordNumber","in":{"testid":"pending-restock-item"},"testid":"pending-restock-remaining"},{"actor":"admin","do":"wait","ms":1000},{"actor":"admin","comparison":"atMost","do":"expectNumber","in":{"testid":"pending-restock-item"},"plus":-1,"relativeTo":"initial-countdown","testid":"pending-restock-remaining","within":10000},{"actor":"admin","do":"click","in":{"testid":"pending-restock-item"},"testid":"pending-restock-cancel"}]}],"id":302,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link"}]},"packId":"ecommerce.l3.scheduled-restocks-features","role":"feature","source":"scenarios/03-scheduled-restocks.json","stablePackId":"ecommerce.l3.scheduled-restocks"}],"id":"selected-source-058","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/03-scheduled-restocks.json"},{"checkGroups":[{"checkGroupId":"server-time","feature":{"actors":["admin","watcher"],"criteria":[{"id":"312a","steps":[{"actor":"watcher","do":"wait","ms":30000},{"actor":"watcher","do":"reload","settleMs":2000},{"atMost":100000,"do":"expectElapsed","since":"restock-start"},{"do":"dbExpectStock","item":"Espresso Machine","plus":0,"relativeTo":"before"},{"actor":"admin","count":1,"do":"expect","testid":"pending-restock-item"},{"atMost":100000,"do":"expectElapsed","since":"restock-start"},{"actor":"admin","do":"wait","ms":130000,"since":"restock-start-accepted"},{"do":"dbExpectStock","item":"Espresso Machine","plus":4,"relativeTo":"before","within":10000},{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link","within":1000},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link","within":1000},{"actor":"admin","do":"expect","testid":"schedule-restock-submit"},{"absent":true,"actor":"admin","do":"expect","testid":"pending-restock-item"}]}],"id":312,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link","within":1000},{"as":"before","do":"dbRecordStock","item":"Espresso Machine"},{"actor":"admin","do":"fill","testid":"schedule-restock-item","text":"Espresso Machine"},{"actor":"admin","do":"fill","testid":"schedule-restock-warehouse","text":"West"},{"actor":"admin","do":"fill","testid":"schedule-restock-qty","text":"4"},{"actor":"admin","do":"fill","testid":"schedule-restock-delay","text":"120"},{"as":"restock-start","do":"recordTime"},{"actor":"admin","do":"click","testid":"schedule-restock-submit"},{"actor":"admin","count":1,"do":"expect","testid":"pending-restock-item","within":10000},{"as":"restock-start-accepted","do":"recordTime"},{"do":"dbExpectStock","item":"Espresso Machine","plus":0,"relativeTo":"before"},{"do":"restartBackend","settleMs":15000},{"actor":"watcher","do":"reload","settleMs":3000},{"actor":"admin","do":"reload","settleMs":3000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link","within":1000},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link","within":1000},{"actor":"admin","do":"expect","testid":"schedule-restock-submit"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.l3.server-time-specifications","requiresFeatures":["ecommerce.l3.scheduled-restocks-features"],"role":"guarantee","source":"scenarios/03-server-time.json","stablePackId":"ecommerce.l3.server-time"},{"checkGroupId":"server-time","feature":{"actors":["customer","watcher"],"criteria":[{"id":"313a","steps":[{"actor":"watcher","do":"wait","ms":100000},{"actor":"watcher","do":"reload","settleMs":1000},{"actor":"watcher","do":"expectNumber","in":{"contains":"Air Purifier","testid":"item-card"},"plus":0,"relativeTo":"before","testid":"item-stock","within":30000}]}],"id":313,"setup":[{"actor":"customer","do":"signUp","name":"server-clock"},{"actor":"watcher","as":"before","do":"recordNumber","in":{"contains":"Air Purifier","testid":"item-card"},"testid":"item-stock"},{"actor":"customer","do":"click","in":{"contains":"Air Purifier","testid":"item-card"},"testid":"add-to-cart"},{"actor":"watcher","do":"reload","settleMs":1000},{"actor":"watcher","do":"expectNumber","in":{"contains":"Air Purifier","testid":"item-card"},"plus":-1,"relativeTo":"before","testid":"item-stock","within":10000},{"actor":"customer","do":"closeClient"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.l3.server-time-specifications","requiresFeatures":["ecommerce.l3.reservations-features"],"role":"guarantee","source":"scenarios/03-server-time.json","stablePackId":"ecommerce.l3.server-time"}],"id":"selected-source-059","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/03-server-time.json"},{"checkGroups":[{"checkGroupId":"account-state-recovery","feature":{"actors":["shopper","peer"],"criteria":[{"id":"105b","steps":[{"actor":"shopper","do":"setOffline","offline":true,"settleMs":3000},{"actor":"peer","do":"click","in":{"contains":"Headphones","testid":"item-card"},"testid":"add-to-cart"},{"actor":"shopper","do":"setOffline","offline":false,"settleMs":6000},{"actor":"shopper","contains":"pat","do":"expect","testid":"current-user"},{"actor":"shopper","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"},{"actor":"shopper","contains":"Keyboard","do":"expect","testid":"cart-item"},{"actor":"shopper","contains":"Headphones","do":"expect","testid":"cart-item"}]}],"id":105,"setup":[{"actor":"shopper","do":"signUp","name":"pat"},{"actor":"peer","do":"signIn","name":"pat"},{"actor":"shopper","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","requiresFeatures":["ecommerce.feature.accounts","ecommerce.feature.checkout"],"role":"guarantee","source":"scenarios/progression-account-state-reconnect.json"}],"id":"selected-source-060","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/progression-account-state-reconnect.json"},{"checkGroups":[{"checkGroupId":"account-state-recovery","feature":{"actors":["shopper"],"criteria":[{"id":"105a","steps":[{"actor":"shopper","do":"click","in":{"contains":"Headphones","testid":"item-card"},"testid":"add-to-cart"},{"actor":"shopper","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"},{"actor":"shopper","do":"click","settleMs":1500,"testid":"checkout-submit"},{"actor":"shopper","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"shopper","do":"click","ifAvailable":true,"settleMs":2500,"testid":"catalog-link"},{"actor":"shopper","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"actor":"shopper","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"},{"actor":"shopper","contains":"Keyboard","do":"expect","testid":"cart-item","within":10000},{"actor":"shopper","do":"reload","settleMs":4000},{"actor":"shopper","contains":"pat","do":"expect","testid":"current-user"},{"actor":"shopper","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"},{"actor":"shopper","contains":"Keyboard","do":"expect","testid":"cart-item"},{"actor":"shopper","do":"reload","settleMs":2500},{"actor":"shopper","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"shopper","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"shopper","contains":"Headphones","count":1,"do":"expect","testid":"order-item"},{"do":"restartBackend","settleMs":1000},{"actor":"shopper","do":"freshClient"},{"actor":"shopper-fresh","do":"signIn","name":"pat"},{"actor":"shopper-fresh","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"},{"actor":"shopper-fresh","contains":"Keyboard","count":1,"do":"expect","testid":"cart-item"},{"actor":"shopper-fresh","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"shopper-fresh","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"shopper-fresh","contains":"Headphones","count":1,"do":"expect","testid":"order-item"}]}],"id":105,"setup":[{"actor":"shopper","do":"signUp","name":"pat"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","requiresFeatures":["ecommerce.feature.accounts","ecommerce.feature.checkout"],"role":"guarantee","source":"scenarios/progression-account-state-reload.json"}],"id":"selected-source-061","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/progression-account-state-reload.json"},{"checkGroups":[{"checkGroupId":"automatic-reorder-access","feature":{"actors":["staff","customer"],"criteria":[{"id":"502c","steps":[{"absent":true,"actor":"customer","do":"expect","testid":"reorder-link"},{"actor":"staff","do":"fill","testid":"reorder-item","text":"Desk Lamp"},{"actor":"staff","do":"fill","testid":"reorder-threshold","text":"1"},{"actor":"staff","do":"fill","testid":"reorder-quantity","text":"9"},{"action":"saveReorderRule","actor":"customer","do":"callAction","from":"staff","input":{"attribute":"data-action-input","testid":"reorder-submit"},"namedAction":{"args":[0,1,9],"id":"saveReorderRule","method":"PUT","params":[{"in":"path","name":"itemId","placeholder":"{itemId}","wireType":"u64"},{"in":"body","name":"threshold"},{"in":"body","name":"quantity"}],"path":"/api/reorders/{itemId}","reducer":"save_reorder_rule"}},{"actor":"customer","do":"expectActionOutcome","outcome":"refused","routeProvenBy":"staff"},{"actor":"staff","do":"reload","settleMs":1000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"staff","do":"click","testid":"reorder-link","unlessVisible":"reorder-submit"},{"actor":"staff","attribute":"data-threshold","contains":"Desk Lamp","count":1,"do":"expect","testid":"reorder-rule-item","value":"2"},{"actor":"staff","attribute":"data-quantity","contains":"Desk Lamp","do":"expect","testid":"reorder-rule-item","value":"5"}]}],"id":502,"setup":[{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"expect","testid":"reorder-link","within":6000},{"actor":"staff","do":"click","testid":"reorder-link"},{"actor":"staff","do":"fill","testid":"reorder-item","text":"Desk Lamp"},{"actor":"staff","do":"fill","testid":"reorder-threshold","text":"2"},{"actor":"staff","do":"fill","testid":"reorder-quantity","text":"5"},{"action":"saveReorderRule","actor":"staff","do":"callAction","input":{"attribute":"data-action-input","testid":"reorder-submit"},"namedAction":{"args":[0,2,5],"id":"saveReorderRule","method":"PUT","params":[{"in":"path","name":"itemId","placeholder":"{itemId}","wireType":"u64"},{"in":"body","name":"threshold"},{"in":"body","name":"quantity"}],"path":"/api/reorders/{itemId}","reducer":"save_reorder_rule"}},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"staff","do":"reload","settleMs":1000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","testid":"reorder-link","unlessVisible":"reorder-submit"},{"actor":"staff","contains":"Desk Lamp","do":"expectElementCount","equals":1,"testid":"reorder-rule-item","within":10000},{"actor":"customer","do":"signUp","name":"reorder-customer"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.automatic-reorder"],"role":"guarantee","source":"scenarios/progression-automatic-reorder-access.json"}],"id":"selected-source-062","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-automatic-reorder-access.json"},{"checkGroups":[{"checkGroupId":"automatic-reorder-deduplication","feature":{"actors":["staff","admin","buyer-a","buyer-b","buyer-c"],"criteria":[{"id":"502b","steps":[{"action":"buy","actor":"buyer-b","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"buyer-b","do":"expectActionOutcome","outcome":"accepted"},{"action":"buy","actor":"buyer-c","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"buyer-c","do":"expectActionOutcome","outcome":"accepted"},{"do":"dbExpectStock","equals":0,"item":"Desk Lamp"},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link","within":1000},{"actor":"admin","do":"expect","testid":"schedule-restock-submit"},{"actor":"admin","do":"expectElementCount","equals":1,"testid":"pending-restock-item","within":10000},{"actor":"admin","attribute":"data-quantity","contains":"Desk Lamp","count":1,"do":"expect","testid":"pending-restock-item","value":"5"}]}],"id":502,"setup":[{"do":"dbSetStock","item":"Desk Lamp","quantity":2,"settleMs":2000,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":1,"settleMs":2000,"warehouse":"West"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"expect","testid":"reorder-link","within":6000},{"actor":"staff","do":"click","testid":"reorder-link"},{"actor":"staff","do":"fill","testid":"reorder-item","text":"Desk Lamp"},{"actor":"staff","do":"fill","testid":"reorder-threshold","text":"2"},{"actor":"staff","do":"fill","testid":"reorder-quantity","text":"5"},{"action":"saveReorderRule","actor":"staff","do":"callAction","input":{"attribute":"data-action-input","testid":"reorder-submit"},"namedAction":{"args":[0,2,5],"id":"saveReorderRule","method":"PUT","params":[{"in":"path","name":"itemId","placeholder":"{itemId}","wireType":"u64"},{"in":"body","name":"threshold"},{"in":"body","name":"quantity"}],"path":"/api/reorders/{itemId}","reducer":"save_reorder_rule"}},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"staff","do":"reload","settleMs":1000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","testid":"reorder-link","unlessVisible":"reorder-submit"},{"actor":"staff","contains":"Desk Lamp","do":"expectElementCount","equals":1,"testid":"reorder-rule-item","within":10000},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link","within":1000},{"actor":"buyer-a","do":"signUp","name":"reorder-a"},{"actor":"buyer-b","do":"signUp","name":"reorder-b"},{"actor":"buyer-c","do":"signUp","name":"reorder-c"},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link","within":1000},{"actor":"admin","do":"expect","testid":"schedule-restock-submit"},{"actor":"admin","do":"expectElementCount","equals":0,"testid":"pending-restock-item"},{"action":"buy","actor":"buyer-a","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"buyer-a","do":"expectActionOutcome","outcome":"accepted"},{"do":"dbExpectStock","equals":2,"item":"Desk Lamp"},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link","within":1000},{"actor":"admin","do":"expect","testid":"schedule-restock-submit"},{"actor":"admin","do":"expectElementCount","equals":1,"testid":"pending-restock-item","within":10000},{"actor":"admin","attribute":"data-quantity","contains":"Desk Lamp","count":1,"do":"expect","testid":"pending-restock-item","value":"5"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.automatic-reorder"],"role":"guarantee","source":"scenarios/progression-automatic-reorder-duplicate.json"}],"id":"selected-source-063","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-automatic-reorder-duplicate.json"},{"checkGroups":[{"checkGroupId":"automatic-reorder","feature":{"actors":["staff","admin","buyer-a"],"criteria":[{"id":"502a","steps":[{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link","within":1000},{"actor":"admin","do":"expect","testid":"schedule-restock-submit"},{"actor":"admin","do":"expectElementCount","equals":0,"testid":"pending-restock-item"},{"action":"buy","actor":"buyer-a","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"buyer-a","do":"expectActionOutcome","outcome":"accepted"},{"do":"dbExpectStock","equals":2,"item":"Desk Lamp"},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link","within":1000},{"actor":"admin","do":"expect","testid":"schedule-restock-submit"},{"actor":"admin","do":"expectElementCount","equals":1,"testid":"pending-restock-item","within":10000},{"actor":"admin","attribute":"data-quantity","contains":"Desk Lamp","count":1,"do":"expect","testid":"pending-restock-item","value":"5"}]}],"id":502,"setup":[{"do":"dbSetStock","item":"Desk Lamp","quantity":2,"settleMs":2000,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":1,"settleMs":2000,"warehouse":"West"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"expect","testid":"reorder-link","within":6000},{"actor":"staff","do":"click","testid":"reorder-link"},{"actor":"staff","do":"fill","testid":"reorder-item","text":"Desk Lamp"},{"actor":"staff","do":"fill","testid":"reorder-threshold","text":"2"},{"actor":"staff","do":"fill","testid":"reorder-quantity","text":"5"},{"action":"saveReorderRule","actor":"staff","do":"callAction","input":{"attribute":"data-action-input","testid":"reorder-submit"},"namedAction":{"args":[0,2,5],"id":"saveReorderRule","method":"PUT","params":[{"in":"path","name":"itemId","placeholder":"{itemId}","wireType":"u64"},{"in":"body","name":"threshold"},{"in":"body","name":"quantity"}],"path":"/api/reorders/{itemId}","reducer":"save_reorder_rule"}},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"staff","do":"reload","settleMs":1000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","testid":"reorder-link","unlessVisible":"reorder-submit"},{"actor":"staff","contains":"Desk Lamp","do":"expectElementCount","equals":1,"testid":"reorder-rule-item","within":10000},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"restocks-link","within":1000},{"actor":"buyer-a","do":"signUp","name":"reorder-a"}]},"packId":"ecommerce.progression.automatic-reorder","requiresFeatures":["ecommerce.feature.purchasing"],"role":"feature","source":"scenarios/progression-automatic-reorder.json"}],"id":"selected-source-064","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-automatic-reorder.json"},{"checkGroups":[{"checkGroupId":"books-balance","feature":{"actors":["buyer","admin"],"criteria":[{"id":"107a","steps":[{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"expectNumber","plus":58,"relativeTo":"revenue-before","testid":"admin-revenue","within":10000}]},{"id":"107b","steps":[{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link"},{"actor":"admin","do":"expectNumber","in":{"contains":"Laptop Stand","testid":"admin-item-row"},"plus":-2,"relativeTo":"stand-before","testid":"admin-stock"},{"actor":"buyer","do":"freshClient"},{"actor":"buyer-fresh","do":"expectNumber","in":{"contains":"Laptop Stand","testid":"item-card"},"plus":-2,"relativeTo":"stand-before","testid":"item-stock"}]}],"id":107,"setup":[{"actor":"buyer","do":"signUp","name":"sam"},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","as":"revenue-before","do":"recordNumber","testid":"admin-revenue"},{"actor":"admin","as":"stand-before","do":"recordNumber","in":{"contains":"Laptop Stand","testid":"admin-item-row"},"testid":"admin-stock"},{"action":"buy","actor":"buyer","do":"callAction","input":{"attribute":"data-buy-input","contains":"Laptop Stand","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"buyer","do":"expectActionOutcome","outcome":"accepted"},{"action":"buy","actor":"buyer","do":"callAction","input":{"attribute":"data-buy-input","contains":"Laptop Stand","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"buyer","do":"expectActionOutcome","outcome":"accepted"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin"],"role":"guarantee","source":"scenarios/progression-books-balance.json"}],"id":"selected-source-065","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/progression-books-balance.json"},{"checkGroups":[{"checkGroupId":"bundle-checkout","feature":{"actors":["admin","buyer"],"criteria":[{"id":"741a","steps":[{"actor":"buyer","do":"click","in":{"contains":"Checkout bundle","testid":"bundle-card"},"settleMs":500,"testid":"bundle-add-to-cart"},{"do":"dbExpectStock","equals":4,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":3,"item":"Desk Lamp","warehouse":"East"},{"actor":"buyer","do":"click","testid":"cart-toggle"},{"actor":"buyer","do":"click","testid":"checkout-submit"},{"actor":"buyer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"buyer","do":"click","testid":"orders-toggle"},{"actor":"buyer","contains":"Checkout bundle","count":1,"do":"expect","testid":"order-item","within":10000},{"actor":"buyer","do":"expectNumber","equals":75,"in":{"contains":"Checkout bundle","testid":"order-item"},"testid":"payment-amount","within":10000},{"do":"dbExpectStock","equals":4,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":3,"item":"Desk Lamp","warehouse":"East"},{"do":"dbExpectStock","equals":0,"item":"Keyboard","warehouse":"West"},{"do":"dbExpectStock","equals":0,"item":"Desk Lamp","warehouse":"West"}]}],"id":741,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"bundles-link","unlessVisible":"bundle-name-input"},{"actor":"buyer","do":"signUp","name":"bundle-checkout"},{"do":"dbSetStock","item":"Keyboard","quantity":6,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Keyboard","quantity":0,"settleMs":0,"warehouse":"West"},{"do":"dbSetStock","item":"Desk Lamp","quantity":4,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":0,"warehouse":"West"},{"actor":"admin","do":"fill","testid":"bundle-name-input","text":"Checkout bundle"},{"actor":"admin","do":"fill","testid":"bundle-price-input","text":"75.00"},{"actor":"admin","do":"fill","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"},{"actor":"admin","do":"click","testid":"bundle-save"},{"actor":"admin","contains":"Checkout bundle","do":"expect","testid":"bundle-card","within":10000},{"actor":"buyer","do":"click","testid":"bundles-link","unlessVisible":"bundle-card"}]},"packId":"ecommerce.feature.bundle-checkout","role":"feature","source":"scenarios/progression-bundle-checkout.json"},{"checkGroupId":"bundle-744","feature":{"actors":["admin","a","b"],"criteria":[{"id":"744a","steps":[{"action":"addBundleToCart","actors":["a","b"],"do":"callConcurrently","input":{"attribute":"data-bundle-input","contains":"Scarce bundle","testid":"bundle-card"},"namedAction":{"args":[0],"id":"addBundleToCart","params":[{"in":"body","name":"bundleId","wireType":"u64"}],"path":"/api/cart/bundles","reducer":"add_bundle_to_cart"},"settleMs":1000},{"accepted":1,"do":"expectCallOutcomes"},{"do":"dbExpectStock","equals":0,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":0,"item":"Desk Lamp","warehouse":"East"},{"actor":"a","do":"reload","settleMs":1000},{"actor":"b","do":"reload","settleMs":1000},{"actor":"a","do":"click","testid":"cart-toggle"},{"actor":"b","do":"click","testid":"cart-toggle"},{"actors":["a","b"],"contains":"Scarce bundle","do":"expectActorsWith","equals":1,"maxEach":1,"testid":"cart-item"},{"actor":"a","do":"click","ifAvailable":true,"in":{"contains":"Scarce bundle","testid":"cart-item"},"settleMs":500,"testid":"bundle-remove"},{"actor":"b","do":"click","ifAvailable":true,"in":{"contains":"Scarce bundle","testid":"cart-item"},"settleMs":500,"testid":"bundle-remove"},{"do":"dbExpectStock","equals":2,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":1,"item":"Desk Lamp","warehouse":"East"}]}],"id":744,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"bundles-link","unlessVisible":"bundle-name-input"},{"actor":"a","do":"signUp","name":"bundle-race-a"},{"actor":"b","do":"signUp","name":"bundle-race-b"},{"do":"dbSetStock","item":"Keyboard","quantity":2,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Keyboard","quantity":0,"settleMs":0,"warehouse":"West"},{"do":"dbSetStock","item":"Desk Lamp","quantity":1,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":0,"warehouse":"West"},{"actor":"admin","do":"fill","testid":"bundle-name-input","text":"Scarce bundle"},{"actor":"admin","do":"fill","testid":"bundle-price-input","text":"75.00"},{"actor":"admin","do":"fill","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"},{"actor":"admin","do":"click","testid":"bundle-save"},{"actor":"admin","contains":"Scarce bundle","do":"expect","testid":"bundle-card","within":10000},{"actor":"a","do":"click","testid":"bundles-link","unlessVisible":"bundle-card"},{"actor":"b","do":"click","testid":"bundles-link","unlessVisible":"bundle-card"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","requiresFeatures":["ecommerce.feature.bundle-checkout"],"role":"guarantee","source":"scenarios/progression-bundle-checkout.json"},{"checkGroupId":"bundle-745","feature":{"actors":["admin","buyer"],"criteria":[{"id":"745a","steps":[{"action":"addBundleToCart","actor":"buyer","do":"callAction","input":{"attribute":"data-bundle-input","contains":"Unavailable bundle","testid":"bundle-card"},"namedAction":{"args":[0],"id":"addBundleToCart","params":[{"in":"body","name":"bundleId","wireType":"u64"}],"path":"/api/cart/bundles","reducer":"add_bundle_to_cart"},"settleMs":1000},{"actor":"buyer","do":"expectActionOutcome","outcome":"application-refused"},{"do":"dbExpectStock","equals":2,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":0,"item":"Desk Lamp","warehouse":"East"},{"actor":"buyer","do":"click","testid":"cart-toggle"},{"absent":true,"actor":"buyer","contains":"Unavailable bundle","do":"expect","testid":"cart-item","within":10000}]}],"id":745,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"bundles-link","unlessVisible":"bundle-name-input"},{"actor":"buyer","do":"signUp","name":"bundle-partial"},{"do":"dbSetStock","item":"Keyboard","quantity":2,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Keyboard","quantity":0,"settleMs":0,"warehouse":"West"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":0,"warehouse":"West"},{"actor":"admin","do":"fill","testid":"bundle-name-input","text":"Unavailable bundle"},{"actor":"admin","do":"fill","testid":"bundle-price-input","text":"75.00"},{"actor":"admin","do":"fill","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"},{"actor":"admin","do":"click","testid":"bundle-save"},{"actor":"admin","contains":"Unavailable bundle","do":"expect","testid":"bundle-card","within":10000},{"actor":"buyer","do":"click","testid":"bundles-link","unlessVisible":"bundle-card"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","requiresFeatures":["ecommerce.feature.bundle-checkout"],"role":"guarantee","source":"scenarios/progression-bundle-checkout.json"},{"checkGroupId":"bundle-746","feature":{"actors":["admin","buyer"],"criteria":[{"id":"746a","steps":[{"actor":"buyer","do":"click","in":{"contains":"Expiring bundle","testid":"bundle-card"},"settleMs":500,"testid":"bundle-add-to-cart"},{"do":"dbExpectStock","equals":4,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":3,"item":"Desk Lamp","warehouse":"East"},{"do":"restartBackend","settleMs":1500},{"actor":"buyer","do":"wait","ms":92000},{"actor":"buyer","do":"reload","settleMs":1000},{"actor":"buyer","do":"ensureSignedIn","name":"bundle-expiry","readyTestid":"current-user"},{"actor":"buyer","do":"click","testid":"cart-toggle"},{"actor":"buyer","do":"expect","in":{"contains":"Expiring bundle","testid":"cart-item"},"testid":"cart-item-expired","within":10000},{"do":"dbExpectStock","equals":6,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":4,"item":"Desk Lamp","warehouse":"East"},{"do":"restartBackend","settleMs":1500},{"do":"dbExpectStock","equals":6,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":4,"item":"Desk Lamp","warehouse":"East"}]}],"id":746,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"bundles-link","unlessVisible":"bundle-name-input"},{"actor":"buyer","do":"signUp","name":"bundle-expiry"},{"do":"dbSetStock","item":"Keyboard","quantity":6,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Keyboard","quantity":0,"settleMs":0,"warehouse":"West"},{"do":"dbSetStock","item":"Desk Lamp","quantity":4,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":0,"warehouse":"West"},{"actor":"admin","do":"fill","testid":"bundle-name-input","text":"Expiring bundle"},{"actor":"admin","do":"fill","testid":"bundle-price-input","text":"75.00"},{"actor":"admin","do":"fill","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"},{"actor":"admin","do":"click","testid":"bundle-save"},{"actor":"admin","contains":"Expiring bundle","do":"expect","testid":"bundle-card","within":10000},{"actor":"buyer","do":"click","testid":"bundles-link","unlessVisible":"bundle-card"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","requiresFeatures":["ecommerce.feature.bundle-checkout"],"role":"guarantee","source":"scenarios/progression-bundle-checkout.json"},{"checkGroupId":"bundle-747","feature":{"actors":["admin","a","b"],"criteria":[{"id":"747a","steps":[{"action":"checkout","actors":["a","b"],"do":"callConcurrently","settleMs":1000},{"accepted":1,"do":"expectCallOutcomes"},{"actor":"a","do":"reload","settleMs":1000},{"actor":"a","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"a","do":"click","testid":"orders-toggle"},{"actor":"a","contains":"Repeated bundle","count":1,"do":"expect","testid":"order-item","within":10000},{"actor":"a","do":"expectNumber","equals":75,"in":{"contains":"Repeated bundle","testid":"order-item"},"testid":"payment-amount","within":10000},{"do":"dbExpectStock","equals":4,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":3,"item":"Desk Lamp","warehouse":"East"}]}],"id":747,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"bundles-link","unlessVisible":"bundle-name-input"},{"actor":"a","do":"signUp","name":"bundle-repeat"},{"actor":"b","do":"signIn","name":"bundle-repeat"},{"do":"dbSetStock","item":"Keyboard","quantity":6,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Keyboard","quantity":0,"settleMs":0,"warehouse":"West"},{"do":"dbSetStock","item":"Desk Lamp","quantity":4,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":0,"warehouse":"West"},{"actor":"admin","do":"fill","testid":"bundle-name-input","text":"Repeated bundle"},{"actor":"admin","do":"fill","testid":"bundle-price-input","text":"75.00"},{"actor":"admin","do":"fill","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"},{"actor":"admin","do":"click","testid":"bundle-save"},{"actor":"admin","contains":"Repeated bundle","do":"expect","testid":"bundle-card","within":10000},{"actor":"a","do":"click","testid":"bundles-link","unlessVisible":"bundle-card"},{"actor":"a","do":"click","in":{"contains":"Repeated bundle","testid":"bundle-card"},"settleMs":500,"testid":"bundle-add-to-cart"},{"do":"dbExpectStock","equals":4,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":3,"item":"Desk Lamp","warehouse":"East"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","requiresFeatures":["ecommerce.feature.bundle-checkout"],"role":"guarantee","source":"scenarios/progression-bundle-checkout.json"}],"id":"selected-source-066","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-bundle-checkout.json"},{"checkGroups":[{"checkGroupId":"bundle-returns","feature":{"actors":["admin","buyer","staff"],"criteria":[{"id":"742a","steps":[{"actor":"admin","do":"fill","testid":"bundle-name-input","text":"Historical bundle"},{"actor":"admin","do":"fill","testid":"bundle-price-input","text":"9.00"},{"actor":"admin","do":"fill","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":1},{\"item\":\"Desk Lamp\",\"quantity\":3}]"},{"actor":"admin","do":"click","testid":"bundle-save"},{"actor":"admin","contains":"Historical bundle","do":"expect","testid":"bundle-card","within":10000},{"actor":"buyer","do":"click","in":{"contains":"Historical bundle","testid":"order-item"},"testid":"return-bundle"},{"actor":"buyer","do":"expect","ignoreCase":true,"in":{"contains":"Historical bundle","testid":"order-item"},"testid":"order-status","value":"returned","within":10000},{"actor":"buyer","do":"expectNumber","equals":75,"in":{"contains":"Historical bundle","testid":"order-item"},"testid":"bundle-refund-amount","within":10000},{"do":"dbExpectStock","equals":6,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":4,"item":"Desk Lamp","warehouse":"East"},{"do":"dbExpectStock","equals":0,"item":"Keyboard","warehouse":"West"},{"do":"dbExpectStock","equals":0,"item":"Desk Lamp","warehouse":"West"}]}],"id":742,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"bundles-link","unlessVisible":"bundle-name-input"},{"actor":"buyer","do":"signUp","name":"Historical bundle"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"dbSetStock","item":"Keyboard","quantity":6,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Keyboard","quantity":0,"settleMs":0,"warehouse":"West"},{"do":"dbSetStock","item":"Desk Lamp","quantity":4,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":0,"warehouse":"West"},{"actor":"admin","do":"fill","testid":"bundle-name-input","text":"Historical bundle"},{"actor":"admin","do":"fill","testid":"bundle-price-input","text":"75.00"},{"actor":"admin","do":"fill","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"},{"actor":"admin","do":"click","testid":"bundle-save"},{"actor":"admin","contains":"Historical bundle","do":"expect","testid":"bundle-card","within":10000},{"actor":"buyer","do":"click","testid":"bundles-link","unlessVisible":"bundle-card"},{"actor":"buyer","do":"click","in":{"contains":"Historical bundle","testid":"bundle-card"},"testid":"bundle-add-to-cart"},{"actor":"buyer","do":"click","testid":"cart-toggle"},{"actor":"buyer","do":"click","testid":"checkout-submit"},{"actor":"buyer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"buyer","do":"click","testid":"orders-toggle"},{"actor":"buyer","contains":"Historical bundle","count":1,"do":"expect","testid":"order-item","within":10000},{"actor":"buyer","do":"expectNumber","equals":75,"in":{"contains":"Historical bundle","testid":"order-item"},"testid":"payment-amount","within":10000},{"do":"dbExpectStock","equals":4,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":3,"item":"Desk Lamp","warehouse":"East"},{"actor":"staff","do":"click","testid":"staff-link"},{"actor":"staff","contains":"Historical bundle","do":"expect","testid":"queue-item","within":10000},{"actor":"staff","do":"click","in":{"contains":"Historical bundle","testid":"queue-item"},"testid":"ship-submit"},{"actor":"buyer","do":"expect","ignoreCase":true,"in":{"contains":"Historical bundle","testid":"order-item"},"testid":"order-status","value":"shipped","within":10000}]},"packId":"ecommerce.feature.bundle-returns","role":"feature","source":"scenarios/progression-bundle-returns.json"},{"checkGroupId":"bundle-742","feature":{"actors":["admin","buyer","staff"],"criteria":[{"id":"742b","steps":[{"do":"restartBackend","settleMs":1500},{"actor":"buyer","do":"reload","settleMs":1000},{"actor":"buyer","do":"ensureSignedIn","name":"Historical bundle","readyTestid":"current-user"},{"actor":"buyer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"buyer","do":"click","testid":"orders-toggle"},{"action":"returnBundle","actor":"buyer","do":"callAction","input":{"attribute":"data-bundle-return-input","contains":"Historical bundle","testid":"order-item"},"namedAction":{"args":[0],"id":"returnBundle","params":[{"in":"path","name":"orderId","placeholder":":orderId","wireType":"u64"}],"path":"/api/bundle-orders/:orderId/return","reducer":"return_bundle"},"settleMs":1000},{"actor":"buyer","do":"reload","settleMs":1000},{"actor":"buyer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"buyer","do":"click","testid":"orders-toggle"},{"actor":"buyer","do":"expectNumber","equals":75,"in":{"contains":"Historical bundle","testid":"order-item"},"testid":"bundle-refund-amount","within":10000},{"do":"dbExpectStock","equals":6,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":4,"item":"Desk Lamp","warehouse":"East"}]}],"id":742,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"bundles-link","unlessVisible":"bundle-name-input"},{"actor":"buyer","do":"signUp","name":"Historical bundle"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"dbSetStock","item":"Keyboard","quantity":6,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Keyboard","quantity":0,"settleMs":0,"warehouse":"West"},{"do":"dbSetStock","item":"Desk Lamp","quantity":4,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":0,"warehouse":"West"},{"actor":"admin","do":"fill","testid":"bundle-name-input","text":"Historical bundle"},{"actor":"admin","do":"fill","testid":"bundle-price-input","text":"75.00"},{"actor":"admin","do":"fill","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"},{"actor":"admin","do":"click","testid":"bundle-save"},{"actor":"admin","contains":"Historical bundle","do":"expect","testid":"bundle-card","within":10000},{"actor":"buyer","do":"click","testid":"bundles-link","unlessVisible":"bundle-card"},{"actor":"buyer","do":"click","in":{"contains":"Historical bundle","testid":"bundle-card"},"testid":"bundle-add-to-cart"},{"actor":"buyer","do":"click","testid":"cart-toggle"},{"actor":"buyer","do":"click","testid":"checkout-submit"},{"actor":"buyer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"buyer","do":"click","testid":"orders-toggle"},{"actor":"buyer","contains":"Historical bundle","count":1,"do":"expect","testid":"order-item","within":10000},{"actor":"buyer","do":"expectNumber","equals":75,"in":{"contains":"Historical bundle","testid":"order-item"},"testid":"payment-amount","within":10000},{"do":"dbExpectStock","equals":4,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":3,"item":"Desk Lamp","warehouse":"East"},{"actor":"staff","do":"click","testid":"staff-link"},{"actor":"staff","contains":"Historical bundle","do":"expect","testid":"queue-item","within":10000},{"actor":"staff","do":"click","in":{"contains":"Historical bundle","testid":"queue-item"},"testid":"ship-submit"},{"actor":"buyer","do":"expect","ignoreCase":true,"in":{"contains":"Historical bundle","testid":"order-item"},"testid":"order-status","value":"shipped","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","requiresFeatures":["ecommerce.feature.bundle-returns"],"role":"guarantee","source":"scenarios/progression-bundle-returns.json"},{"checkGroupId":"bundle-748","feature":{"actors":["admin","buyer","staff","other"],"criteria":[{"id":"748a","steps":[{"action":"returnBundle","actor":"other","do":"callAction","from":"buyer","input":{"attribute":"data-bundle-return-input","contains":"Private bundle","testid":"order-item"},"namedAction":{"args":[0],"id":"returnBundle","params":[{"in":"path","name":"orderId","placeholder":":orderId","wireType":"u64"}],"path":"/api/bundle-orders/:orderId/return","reducer":"return_bundle"},"settleMs":1000},{"actor":"other","do":"expectActionOutcome","outcome":"application-refused"},{"actor":"buyer","do":"reload","settleMs":1000},{"actor":"buyer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"buyer","do":"click","testid":"orders-toggle"},{"actor":"buyer","do":"expect","ignoreCase":true,"in":{"contains":"Private bundle","testid":"order-item"},"testid":"order-status","value":"shipped","within":10000},{"do":"dbExpectStock","equals":4,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":3,"item":"Desk Lamp","warehouse":"East"},{"actor":"buyer","do":"click","in":{"contains":"Private bundle","testid":"order-item"},"testid":"return-bundle"},{"actor":"buyer","do":"expectNumber","equals":75,"in":{"contains":"Private bundle","testid":"order-item"},"testid":"bundle-refund-amount","within":10000},{"do":"dbExpectStock","equals":6,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":4,"item":"Desk Lamp","warehouse":"East"}]}],"id":748,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"bundles-link","unlessVisible":"bundle-name-input"},{"actor":"buyer","do":"signUp","name":"Private bundle"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"do":"dbSetStock","item":"Keyboard","quantity":6,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Keyboard","quantity":0,"settleMs":0,"warehouse":"West"},{"do":"dbSetStock","item":"Desk Lamp","quantity":4,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":0,"warehouse":"West"},{"actor":"admin","do":"fill","testid":"bundle-name-input","text":"Private bundle"},{"actor":"admin","do":"fill","testid":"bundle-price-input","text":"75.00"},{"actor":"admin","do":"fill","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"},{"actor":"admin","do":"click","testid":"bundle-save"},{"actor":"admin","contains":"Private bundle","do":"expect","testid":"bundle-card","within":10000},{"actor":"buyer","do":"click","testid":"bundles-link","unlessVisible":"bundle-card"},{"actor":"buyer","do":"click","in":{"contains":"Private bundle","testid":"bundle-card"},"testid":"bundle-add-to-cart"},{"actor":"buyer","do":"click","testid":"cart-toggle"},{"actor":"buyer","do":"click","testid":"checkout-submit"},{"actor":"buyer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"buyer","do":"click","testid":"orders-toggle"},{"actor":"buyer","contains":"Private bundle","count":1,"do":"expect","testid":"order-item","within":10000},{"actor":"buyer","do":"expectNumber","equals":75,"in":{"contains":"Private bundle","testid":"order-item"},"testid":"payment-amount","within":10000},{"do":"dbExpectStock","equals":4,"item":"Keyboard","warehouse":"East"},{"do":"dbExpectStock","equals":3,"item":"Desk Lamp","warehouse":"East"},{"actor":"staff","do":"click","testid":"staff-link"},{"actor":"staff","contains":"Private bundle","do":"expect","testid":"queue-item","within":10000},{"actor":"staff","do":"click","in":{"contains":"Private bundle","testid":"queue-item"},"testid":"ship-submit"},{"actor":"buyer","do":"expect","ignoreCase":true,"in":{"contains":"Private bundle","testid":"order-item"},"testid":"order-status","value":"shipped","within":10000},{"actor":"other","do":"signUp","name":"bundle-other"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","requiresFeatures":["ecommerce.feature.bundle-returns"],"role":"guarantee","source":"scenarios/progression-bundle-returns.json"}],"id":"selected-source-067","scenario":{"level":6,"writeUrlPattern":null},"source":"scenarios/progression-bundle-returns.json"},{"checkGroups":[{"checkGroupId":"cart","feature":{"actors":["quantity","checkout"],"criteria":[{"id":"4a","steps":[{"actor":"quantity","do":"click","in":{"contains":"Headphones","testid":"item-card"},"testid":"add-to-cart"},{"actor":"quantity","do":"wait","ms":800},{"actor":"quantity","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"quantity","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"quantity","do":"click","in":{"contains":"Headphones","testid":"item-card"},"testid":"add-to-cart"},{"actor":"quantity","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"},{"actor":"quantity","contains":"Headphones","count":1,"do":"expect","testid":"cart-item"},{"actor":"quantity","do":"expectNumber","equals":2,"in":{"contains":"Headphones","testid":"cart-item"},"testid":"cart-quantity"}]}],"id":4,"setup":[{"actor":"quantity","do":"signUp","name":"cart-quantity"},{"actor":"checkout","do":"signUp","name":"cart-checkout"}]},"packId":"ecommerce.feature.cart","role":"feature","source":"scenarios/progression-cart-checkout.json","stablePackId":"ecommerce.feature.cart-checkout"},{"checkGroupId":"cart","feature":{"actors":["quantity","checkout"],"criteria":[{"id":"4d","steps":[{"actor":"checkout","do":"expectNumber","equals":100,"in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock"},{"actor":"checkout","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"add-to-cart"},{"actor":"checkout","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"},{"actor":"checkout","do":"expectNumber","equals":1,"testid":"cart-count"},{"actor":"checkout","do":"click","testid":"checkout-submit"},{"actor":"checkout","do":"wait","ms":1500},{"actor":"checkout","do":"reload","settleMs":2500},{"actor":"checkout","do":"ensureSignedIn","name":"cart-checkout","readyTestid":"current-user"},{"actor":"checkout","do":"click","testid":"cart-toggle","unlessVisible":"cart-total"},{"actor":"checkout","do":"expectNumber","equals":0,"testid":"cart-count"},{"actor":"checkout","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"checkout","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"checkout","do":"expectNumber","equals":99,"in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock"},{"actor":"checkout","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"checkout","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"checkout","contains":"Desk Lamp","count":1,"do":"expect","testid":"order-item"}]}],"id":4,"setup":[{"actor":"quantity","do":"signUp","name":"cart-quantity"},{"actor":"checkout","do":"signUp","name":"cart-checkout"}]},"packId":"ecommerce.feature.checkout","role":"feature","source":"scenarios/progression-cart-checkout.json","stablePackId":"ecommerce.feature.cart-checkout"}],"id":"selected-source-068","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/progression-cart-checkout.json"},{"checkGroups":[{"checkGroupId":"cart-recovery","feature":{"actors":["available-shopper","partial-shopper"],"criteria":[{"id":"503a","steps":[{"actor":"available-shopper","do":"reload","settleMs":1000},{"actor":"available-shopper","do":"ensureSignedIn","name":"cart-recovery-available","readyTestid":"current-user"},{"actor":"available-shopper","do":"expect","testid":"expired-cart"},{"actor":"available-shopper","do":"expectNumber","equals":0,"testid":"cart-count"},{"actor":"available-shopper","as":"restore-available-shopper","do":"recordNumber","in":{"contains":"Keyboard","testid":"item-card"},"testid":"item-stock"},{"actor":"available-shopper","do":"click","in":{"testid":"expired-cart"},"testid":"restore-cart"},{"actor":"available-shopper","do":"click","testid":"cart-toggle"},{"actor":"available-shopper","contains":"Keyboard","do":"expect","testid":"cart-item"},{"absent":true,"actor":"available-shopper","do":"expect","testid":"cart-restore-warning"},{"actor":"available-shopper","do":"expectNumber","equals":1,"in":{"contains":"Keyboard","testid":"cart-item"},"testid":"cart-quantity"},{"actor":"available-shopper","do":"reload","settleMs":1000},{"actor":"available-shopper","do":"ensureSignedIn","name":"cart-recovery-available","readyTestid":"current-user"},{"actor":"available-shopper","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"available-shopper","do":"click","testid":"catalog-link","unlessVisible":"item-card"},{"actor":"available-shopper","do":"expectNumber","in":{"contains":"Keyboard","testid":"item-card"},"plus":-1,"relativeTo":"restore-available-shopper","testid":"item-stock"}]}],"id":503,"setup":[{"actor":"available-shopper","do":"signUp","name":"cart-recovery-available"},{"actor":"available-shopper","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"actor":"partial-shopper","do":"signUp","name":"cart-recovery-partial"},{"actor":"partial-shopper","do":"click","in":{"contains":"Gaming Mouse","testid":"item-card"},"testid":"add-to-cart"},{"actor":"partial-shopper","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"add-to-cart"},{"actor":"available-shopper","do":"wait","ms":310000}]},"packId":"ecommerce.progression.cart-recovery","role":"feature","source":"scenarios/progression-cart-recovery.json"},{"checkGroupId":"cart-recovery","feature":{"actors":["available-shopper","partial-shopper"],"criteria":[{"id":"503b","steps":[{"actor":"partial-shopper","do":"reload","settleMs":1000},{"actor":"partial-shopper","do":"ensureSignedIn","name":"cart-recovery-partial","readyTestid":"current-user"},{"actor":"partial-shopper","do":"expect","testid":"expired-cart"},{"actor":"partial-shopper","do":"expectNumber","equals":0,"testid":"cart-count"},{"actor":"partial-shopper","as":"restore-partial-shopper","do":"recordNumber","in":{"contains":"Gaming Mouse","testid":"item-card"},"testid":"item-stock"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":2000,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":2000,"warehouse":"West"},{"actor":"partial-shopper","do":"click","in":{"testid":"expired-cart"},"testid":"restore-cart"},{"actor":"partial-shopper","do":"click","testid":"cart-toggle"},{"actor":"partial-shopper","contains":"Gaming Mouse","do":"expect","testid":"cart-item"},{"absent":true,"actor":"partial-shopper","contains":"Desk Lamp","do":"expect","testid":"cart-item"},{"actor":"partial-shopper","contains":"Desk Lamp","do":"expect","testid":"cart-restore-warning"},{"actor":"partial-shopper","do":"expectNumber","equals":1,"in":{"contains":"Gaming Mouse","testid":"cart-item"},"testid":"cart-quantity"},{"actor":"partial-shopper","do":"reload","settleMs":1000},{"actor":"partial-shopper","do":"ensureSignedIn","name":"cart-recovery-partial","readyTestid":"current-user"},{"actor":"partial-shopper","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"partial-shopper","do":"click","testid":"catalog-link","unlessVisible":"item-card"},{"actor":"partial-shopper","do":"expectNumber","in":{"contains":"Gaming Mouse","testid":"item-card"},"plus":-1,"relativeTo":"restore-partial-shopper","testid":"item-stock"}]}],"id":503,"setup":[{"actor":"available-shopper","do":"signUp","name":"cart-recovery-available"},{"actor":"available-shopper","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"actor":"partial-shopper","do":"signUp","name":"cart-recovery-partial"},{"actor":"partial-shopper","do":"click","in":{"contains":"Gaming Mouse","testid":"item-card"},"testid":"add-to-cart"},{"actor":"partial-shopper","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"add-to-cart"},{"actor":"available-shopper","do":"wait","ms":310000}]},"packId":"ecommerce.progression.cart-recovery","role":"feature","source":"scenarios/progression-cart-recovery.json"}],"id":"selected-source-069","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-cart-recovery.json"},{"checkGroups":[{"checkGroupId":"catalog-management","feature":{"actors":["admin","visitor"],"criteria":[{"id":"622a","steps":[{"actor":"visitor","contains":"Travel Mug","do":"expect","testid":"item-card","within":10000}]},{"id":"622b","steps":[{"actor":"visitor","do":"openItem","item":"Travel Mug","unlessVisible":"item-variant"},{"actor":"visitor","contains":"Black","do":"expectElementCount","equals":1,"testid":"item-variant","within":10000},{"actor":"visitor","contains":"Silver","do":"expectElementCount","equals":1,"testid":"item-variant","within":10000}]}],"id":622,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"fill","testid":"catalog-name","text":"Travel Mug"},{"actor":"admin","do":"fill","testid":"catalog-category","text":"Kitchen"},{"actor":"admin","do":"fill","testid":"catalog-price","text":"24.00"},{"actor":"admin","do":"fill","testid":"catalog-variants","text":"Black, Silver"},{"actor":"admin","do":"click","testid":"catalog-save"},{"actor":"visitor","do":"reload","settleMs":2500},{"actor":"visitor","do":"fill","enter":true,"testid":"search-input","text":"Travel Mug"}]},"packId":"ecommerce.progression.catalog-management","role":"feature","source":"scenarios/progression-catalog-management.json"}],"id":"selected-source-070","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-catalog-management.json"},{"checkGroups":[{"checkGroupId":"checkout-crash-integrity","feature":{"actors":["a","b","c"],"criteria":[{"id":"910a","steps":[{"do":"expectCrashCheckout","from":"database-crash-observation","verdict":"atomicity"},{"do":"expectCrashCheckout","from":"application-crash-observation","verdict":"atomicity"}]}],"id":910,"setup":[{"actor":"a","do":"signUp","name":"database"},{"account":"{user:database}","as":"database-baseline-before","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"a","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"account":"{user:database}","as":"database-baseline-prepared","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"a","do":"confirmCheckout","namedAction":{"args":[],"id":"checkout","path":"/api/checkout","reducer":"checkout"}},{"before":"database-baseline-before","do":"dbExpectCheckout","prepared":"database-baseline-prepared","quantity":1},{"actor":"a","do":"reload","settleMs":0},{"account":"{user:database}","as":"database-before","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"a","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"account":"{user:database}","as":"database-prepared","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"a","as":"database-crash-observation","before":"database-before","do":"crashCheckout","namedAction":{"args":[],"id":"checkout","path":"/api/checkout","reducer":"checkout"},"offsetMs":0,"prepared":"database-prepared","quantity":1,"requests":16,"target":"database"},{"actor":"b","do":"signUp","name":"application"},{"account":"{user:application}","as":"application-baseline-before","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"b","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"account":"{user:application}","as":"application-baseline-prepared","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"b","do":"confirmCheckout","namedAction":{"args":[],"id":"checkout","path":"/api/checkout","reducer":"checkout"}},{"before":"application-baseline-before","do":"dbExpectCheckout","prepared":"application-baseline-prepared","quantity":1},{"actor":"b","do":"reload","settleMs":0},{"account":"{user:application}","as":"application-before","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"b","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"account":"{user:application}","as":"application-prepared","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"b","as":"application-crash-observation","before":"application-before","do":"crashCheckout","namedAction":{"args":[],"id":"checkout","path":"/api/checkout","reducer":"checkout"},"offsetMs":0,"prepared":"application-prepared","quantity":1,"requests":16,"reuseCombinedFrom":"database-crash-observation","target":"application"},{"actor":"c","do":"signUp","name":"recovered"},{"account":"{user:recovered}","as":"recovered-before","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"c","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"account":"{user:recovered}","as":"recovered-prepared","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"c","do":"click","testid":"cart-toggle"},{"actor":"c","do":"click","testid":"checkout-submit"},{"actor":"c","do":"expectNumber","equals":0,"testid":"cart-count","within":10000},{"actor":"c","do":"click","ifAvailable":true,"testid":"overlay-close"},{"before":"recovered-before","do":"dbExpectCheckout","prepared":"recovered-prepared","quantity":1}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","requiresFeatures":["ecommerce.feature.checkout"],"role":"guarantee","source":"scenarios/progression-checkout-crash.json"},{"checkGroupId":"checkout-crash-durability","feature":{"actors":["a","b","c"],"criteria":[{"id":"910b","steps":[{"do":"expectCrashCheckout","from":"database-crash-observation","verdict":"durability"},{"do":"expectCrashCheckout","from":"application-crash-observation","verdict":"durability"}]}],"id":910,"setup":[{"actor":"a","do":"signUp","name":"database"},{"account":"{user:database}","as":"database-baseline-before","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"a","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"account":"{user:database}","as":"database-baseline-prepared","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"a","do":"confirmCheckout","namedAction":{"args":[],"id":"checkout","path":"/api/checkout","reducer":"checkout"}},{"before":"database-baseline-before","do":"dbExpectCheckout","prepared":"database-baseline-prepared","quantity":1},{"actor":"a","do":"reload","settleMs":0},{"account":"{user:database}","as":"database-before","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"a","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"account":"{user:database}","as":"database-prepared","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"a","as":"database-crash-observation","before":"database-before","do":"crashCheckout","namedAction":{"args":[],"id":"checkout","path":"/api/checkout","reducer":"checkout"},"offsetMs":0,"prepared":"database-prepared","quantity":1,"requests":16,"target":"database"},{"actor":"b","do":"signUp","name":"application"},{"account":"{user:application}","as":"application-baseline-before","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"b","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"account":"{user:application}","as":"application-baseline-prepared","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"b","do":"confirmCheckout","namedAction":{"args":[],"id":"checkout","path":"/api/checkout","reducer":"checkout"}},{"before":"application-baseline-before","do":"dbExpectCheckout","prepared":"application-baseline-prepared","quantity":1},{"actor":"b","do":"reload","settleMs":0},{"account":"{user:application}","as":"application-before","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"b","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"account":"{user:application}","as":"application-prepared","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"b","as":"application-crash-observation","before":"application-before","do":"crashCheckout","namedAction":{"args":[],"id":"checkout","path":"/api/checkout","reducer":"checkout"},"offsetMs":0,"prepared":"application-prepared","quantity":1,"requests":16,"reuseCombinedFrom":"database-crash-observation","target":"application"},{"actor":"c","do":"signUp","name":"recovered"},{"account":"{user:recovered}","as":"recovered-before","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"c","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"account":"{user:recovered}","as":"recovered-prepared","do":"dbRecordCheckout","item":"Keyboard","storage":{"cart":true,"kind":"order-data","warehouses":false}},{"actor":"c","do":"click","testid":"cart-toggle"},{"actor":"c","do":"click","testid":"checkout-submit"},{"actor":"c","do":"expectNumber","equals":0,"testid":"cart-count","within":10000},{"actor":"c","do":"click","ifAvailable":true,"testid":"overlay-close"},{"before":"recovered-before","do":"dbExpectCheckout","prepared":"recovered-prepared","quantity":1}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","requiresFeatures":["ecommerce.feature.checkout"],"role":"guarantee","source":"scenarios/progression-checkout-crash.json"}],"id":"selected-source-071","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/progression-checkout-crash.json"},{"checkGroups":[{"checkGroupId":"payment-records","feature":{"actors":["tab1","tab2","owner"],"criteria":[{"id":"623a","steps":[{"actor":"owner-fresh","do":"expect","ignoreCase":true,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"payment-status","value":"paid","within":10000},{"actor":"owner-fresh","do":"expectNumber","in":{"contains":"Desk Lamp","testid":"order-item"},"plus":0,"relativeTo":"payment-total","testid":"payment-amount"}]}],"id":623,"setup":[{"actor":"tab1","do":"signUp","name":"payment-record"},{"actor":"tab2","do":"signIn","name":"payment-record"},{"actor":"owner","do":"signIn","name":"payment-record"},{"actor":"owner","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"add-to-cart"},{"actor":"owner","do":"click","testid":"cart-toggle"},{"actor":"owner","as":"payment-total","do":"recordNumber","testid":"cart-total"},{"action":"checkout","actors":["tab1","tab2"],"do":"callConcurrently","settleMs":5000},{"do":"expectCallOutcomes"},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"payment-record"},{"actor":"owner-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner-fresh","do":"click","testid":"orders-toggle"},{"actor":"owner-fresh","contains":"Desk Lamp","do":"expectElementCount","equals":1,"testid":"order-item","within":10000}]},"packId":"ecommerce.progression.payment-records","role":"feature","source":"scenarios/progression-core-business.json"},{"checkGroupId":"payment-deduplication","feature":{"actors":["tab1","tab2","owner"],"criteria":[{"id":"623b","steps":[{"actor":"owner-fresh","do":"expectElementCount","equals":1,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"payment-record","within":10000}]}],"id":623,"setup":[{"actor":"tab1","do":"signUp","name":"payment-record"},{"actor":"tab2","do":"signIn","name":"payment-record"},{"actor":"owner","do":"signIn","name":"payment-record"},{"actor":"owner","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"add-to-cart"},{"actor":"owner","do":"click","testid":"cart-toggle"},{"actor":"owner","as":"payment-total","do":"recordNumber","testid":"cart-total"},{"action":"checkout","actors":["tab1","tab2"],"do":"callConcurrently","settleMs":5000},{"do":"expectCallOutcomes"},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"payment-record"},{"actor":"owner-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner-fresh","do":"click","testid":"orders-toggle"},{"actor":"owner-fresh","contains":"Desk Lamp","do":"expectElementCount","equals":1,"testid":"order-item","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","requiresFeatures":["ecommerce.progression.payment-records"],"role":"guarantee","source":"scenarios/progression-core-business.json"}],"id":"selected-source-072","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-core-business.json"},{"checkGroups":[{"checkGroupId":"customer-profile","feature":{"actors":["owner","privateOwner","other"],"criteria":[{"id":"620c","steps":[{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"owner","do":"click","testid":"profile-link","unlessVisible":"profile-address-summary"},{"actor":"owner","contains":"14 Market Street {user:profilemarker}","do":"expect","testid":"profile-address-summary"}]}],"id":620,"setup":[{"actor":"owner","do":"signUp","name":"profile-owner"},{"actor":"owner","do":"click","testid":"profile-link"},{"actor":"owner","do":"fill","testid":"profile-name","text":"Avery Stone"},{"actor":"owner","do":"fill","testid":"profile-address","text":"14 Market Street {user:profilemarker}"},{"actor":"owner","do":"click","testid":"profile-save"}]},"packId":"ecommerce.progression.customer-profile","role":"feature","source":"scenarios/progression-customer-profile.json"},{"checkGroupId":"customer-profile-reload","feature":{"actors":["owner","privateOwner","other"],"criteria":[{"id":"620a","steps":[{"actor":"owner","do":"reload","settleMs":2500},{"actor":"owner","do":"ensureSignedIn","name":"profile-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"profile-link"},{"actor":"owner","contains":"14 Market Street {user:profilemarker}","do":"expect","testid":"profile-address-summary"},{"do":"restartBackend","settleMs":1000},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"profile-owner"},{"actor":"owner-fresh","do":"click","testid":"profile-link"},{"actor":"owner-fresh","contains":"14 Market Street {user:profilemarker}","do":"expect","testid":"profile-address-summary"}]}],"id":620,"setup":[{"actor":"owner","do":"signUp","name":"profile-owner"},{"actor":"owner","do":"click","testid":"profile-link"},{"actor":"owner","do":"fill","testid":"profile-name","text":"Avery Stone"},{"actor":"owner","do":"fill","testid":"profile-address","text":"14 Market Street {user:profilemarker}"},{"actor":"owner","do":"click","testid":"profile-save"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","requiresFeatures":["ecommerce.progression.customer-profile"],"role":"guarantee","source":"scenarios/progression-customer-profile.json"},{"checkGroupId":"customer-profile-privacy","feature":{"actors":["owner","privateOwner","other"],"criteria":[{"id":"620b","steps":[{"actor":"privateOwner","do":"reload","settleMs":0},{"actor":"privateOwner","do":"signUp","name":"profile-private-owner"},{"actor":"privateOwner","do":"click","testid":"profile-link"},{"actor":"privateOwner","do":"fill","testid":"profile-name","text":"Avery Stone"},{"actor":"privateOwner","do":"fill","testid":"profile-address","text":"14 Market Street {user:profilemarker}"},{"actor":"privateOwner","do":"click","testid":"profile-save"},{"actor":"privateOwner","do":"reload","settleMs":1000},{"actor":"privateOwner","do":"ensureSignedIn","name":"profile-private-owner","readyTestid":"current-user"},{"actor":"privateOwner","do":"click","testid":"profile-link"},{"actor":"privateOwner","contains":"14 Market Street {user:profilemarker}","do":"expect","testid":"profile-address-summary"},{"actor":"privateOwner","contains":"14 Market Street {user:profilemarker}","do":"expectReceived","within":10000},{"actor":"other","do":"reload","settleMs":0},{"actor":"other","do":"signUp","name":"profile-other"},{"actor":"other","do":"click","testid":"profile-link"},{"absent":true,"actor":"other","contains":"14 Market Street {user:profilemarker}","do":"expect","testid":"profile-address-summary"},{"actor":"other","contains":"14 Market Street {user:profilemarker}","do":"expectNotReceived"}]}],"id":620,"setup":[{"actor":"owner","do":"signUp","name":"profile-owner"},{"actor":"owner","do":"click","testid":"profile-link"},{"actor":"owner","do":"fill","testid":"profile-name","text":"Avery Stone"},{"actor":"owner","do":"fill","testid":"profile-address","text":"14 Market Street {user:profilemarker}"},{"actor":"owner","do":"click","testid":"profile-save"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.customer-profile"],"role":"guarantee","source":"scenarios/progression-customer-profile.json"}],"id":"selected-source-073","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/progression-customer-profile.json"},{"checkGroups":[{"checkGroupId":"delivery-notification-delivery","feature":{"actors":["owner","other","staff"],"criteria":[{"id":"501a","steps":[{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"delivery-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"notifications-toggle","unlessVisible":"notifications-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"notifications-panel","value":"false","within":10000},{"actor":"owner","contains":"Desk Lamp","do":"expectElementCount","equals":1,"testid":"notification-item","within":10000}]}],"id":501,"setup":[{"actor":"owner","do":"signUp","name":"delivery-owner"},{"actor":"other","do":"signUp","name":"delivery-other"},{"actor":"owner","do":"click","testid":"notification-settings","unlessVisible":"notification-order"},{"actor":"owner","do":"click","testid":"notification-order"},{"actor":"owner","do":"click","testid":"notification-save"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"catalog-link"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"action":"buy","actor":"owner","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"owner","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"delivery-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"notifications-toggle","unlessVisible":"notifications-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"notifications-panel","value":"false","within":10000},{"actor":"owner","contains":"Desk Lamp","do":"expectElementCount","equals":0,"testid":"notification-item"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"delivery-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"as":"notification-shipped","do":"recordTime"},{"action":"ship","actor":"staff","do":"callAction","from":"owner","input":{"attribute":"data-ship-input","contains":"Desk Lamp","testid":"order-item"},"namedAction":{"args":[0],"id":"ship","params":[{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/fulfilment/ship","reducer":"ship_order"}},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"as":"notification-shipped-accepted","do":"recordTime"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"delivery-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"notifications-toggle","unlessVisible":"notifications-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"notifications-panel","value":"false","within":10000},{"atMost":40000,"do":"expectElapsed","since":"notification-shipped"},{"actor":"owner","contains":"Desk Lamp","do":"expectElementCount","equals":0,"testid":"notification-item"},{"actor":"owner","do":"wait","ms":75000,"since":"notification-shipped-accepted"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"delivery-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner","do":"expect","ignoreCase":true,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-status","value":"delivered"}]},"packId":"ecommerce.progression.delivery-notifications","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"feature","source":"scenarios/progression-delivery-notifications.json"},{"checkGroupId":"delivery-notification-privacy","feature":{"actors":["owner","other","staff"],"criteria":[{"id":"501b","steps":[{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"delivery-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"notifications-toggle","unlessVisible":"notifications-panel"},{"actor":"owner","contains":"Desk Lamp","do":"expectElementCount","equals":1,"testid":"notification-item","within":10000},{"actor":"other","do":"click","testid":"notifications-toggle","unlessVisible":"notifications-panel"},{"actor":"other","attribute":"aria-busy","do":"expect","testid":"notifications-panel","value":"false","within":10000},{"absent":true,"actor":"other","contains":"Desk Lamp","do":"expect","testid":"notification-item"}]}],"id":501,"setup":[{"actor":"owner","do":"signUp","name":"delivery-owner"},{"actor":"other","do":"signUp","name":"delivery-other"},{"actor":"owner","do":"click","testid":"notification-settings","unlessVisible":"notification-order"},{"actor":"owner","do":"click","testid":"notification-order"},{"actor":"owner","do":"click","testid":"notification-save"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"catalog-link"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"action":"buy","actor":"owner","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"owner","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"delivery-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"notifications-toggle","unlessVisible":"notifications-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"notifications-panel","value":"false","within":10000},{"actor":"owner","contains":"Desk Lamp","do":"expectElementCount","equals":0,"testid":"notification-item"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"delivery-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"as":"notification-shipped","do":"recordTime"},{"action":"ship","actor":"staff","do":"callAction","from":"owner","input":{"attribute":"data-ship-input","contains":"Desk Lamp","testid":"order-item"},"namedAction":{"args":[0],"id":"ship","params":[{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/fulfilment/ship","reducer":"ship_order"}},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"as":"notification-shipped-accepted","do":"recordTime"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"delivery-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"notifications-toggle","unlessVisible":"notifications-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"notifications-panel","value":"false","within":10000},{"atMost":40000,"do":"expectElapsed","since":"notification-shipped"},{"actor":"owner","contains":"Desk Lamp","do":"expectElementCount","equals":0,"testid":"notification-item"},{"actor":"owner","do":"wait","ms":75000,"since":"notification-shipped-accepted"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"delivery-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner","do":"expect","ignoreCase":true,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-status","value":"delivered"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.delivery-notifications","ecommerce.progression.fulfilment-queue"],"role":"guarantee","source":"scenarios/progression-delivery-notifications.json"}],"id":"selected-source-074","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-delivery-notifications.json"},{"checkGroups":[{"checkGroupId":"faceted-search","feature":{"actors":["visitor"],"criteria":[{"id":"401a","steps":[{"actor":"visitor","contains":"Coffee Grinder","do":"expectElementCount","equals":1,"in":{"testid":"search-results"},"testid":"item-card","within":10000},{"actor":"visitor","do":"click","testid":"in-stock-filter"},{"actor":"visitor","do":"click","ifAvailable":true,"testid":"filter-apply"},{"actor":"visitor","contains":"Coffee Grinder","do":"waitUntilAbsent","in":{"testid":"search-results"},"testid":"item-card","within":10000},{"actor":"visitor","contains":"Air Purifier","do":"expectElementCount","equals":1,"in":{"testid":"search-results"},"testid":"item-card"},{"absent":true,"actor":"visitor","contains":"USB Cable","do":"expect","in":{"testid":"search-results"},"testid":"item-card"},{"absent":true,"actor":"visitor","contains":"Desk Lamp","do":"expect","in":{"testid":"search-results"},"testid":"item-card"},{"absent":true,"actor":"visitor","contains":"Espresso Machine","do":"expect","in":{"testid":"search-results"},"testid":"item-card"},{"absent":true,"actor":"visitor","contains":"Gaming Mouse","do":"expect","in":{"testid":"search-results"},"testid":"item-card"}]}],"id":401,"setup":[{"do":"dbSetStock","item":"Coffee Grinder","quantity":0,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Coffee Grinder","quantity":0,"settleMs":1000,"warehouse":"West"},{"actor":"visitor","do":"reload","settleMs":1000},{"actor":"visitor","do":"fill","testid":"category-filter","text":"Home"},{"actor":"visitor","do":"fill","testid":"minimum-price","text":"50"},{"actor":"visitor","do":"fill","testid":"maximum-price","text":"200"},{"actor":"visitor","do":"click","ifAvailable":true,"testid":"filter-apply"}]},"packId":"ecommerce.progression.faceted-search","role":"feature","source":"scenarios/progression-faceted-filters.json"}],"id":"selected-source-075","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/progression-faceted-filters.json"},{"checkGroups":[{"checkGroupId":"faceted-search","feature":{"actors":["visitor"],"criteria":[{"id":"402a","steps":[{"actor":"visitor","do":"expectSequence","equals":["Air Purifier","Bluetooth Speaker","Coffee Grinder","Desk Lamp","Espresso Machine","Gaming Mouse","Headphones","Induction Cooktop","Keyboard","Laptop Stand"],"in":{"testid":"search-results"},"testid":"item-name"},{"actor":"visitor","do":"click","testid":"search-next-page"},{"actor":"visitor","do":"expectSequence","equals":["Mirrorless Camera","USB Cable","Webcam"],"in":{"testid":"search-results"},"testid":"item-name"},{"actor":"visitor","do":"click","testid":"search-previous-page"},{"actor":"visitor","do":"expectSequence","equals":["Air Purifier","Bluetooth Speaker","Coffee Grinder","Desk Lamp","Espresso Machine","Gaming Mouse","Headphones","Induction Cooktop","Keyboard","Laptop Stand"],"in":{"testid":"search-results"},"testid":"item-name"}]}],"id":402,"setup":[{"actor":"visitor","do":"fill","testid":"minimum-price","text":"1"},{"actor":"visitor","do":"click","ifAvailable":true,"testid":"filter-apply"}]},"packId":"ecommerce.progression.faceted-search","role":"feature","source":"scenarios/progression-faceted-pagination.json"}],"id":"selected-source-076","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/progression-faceted-pagination.json"},{"checkGroups":[{"checkGroupId":"managed-support-privacy","feature":{"actors":["owner","other","staff"],"criteria":[{"id":"613b","steps":[{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"managed-private-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","contains":"Private managed case {user:casemarker}","do":"expect","testid":"support-ticket"},{"actor":"owner","contains":"Private managed case {user:casemarker}","do":"expectReceived","within":10000},{"absent":true,"actor":"other","contains":"Private managed case {user:casemarker}","do":"expect","testid":"support-ticket"},{"actor":"other","contains":"Private managed case {user:casemarker}","do":"expectNotReceived"},{"actor":"owner","do":"fill","in":{"contains":"Private managed case {user:casemarker}","testid":"support-ticket"},"testid":"support-reply","text":"Owner-only update"},{"actor":"owner","do":"click","in":{"contains":"Private managed case {user:casemarker}","testid":"support-ticket"},"testid":"support-reply-submit"},{"actor":"other","do":"replayAs","from":"owner","match":"Owner-only update","namedAction":{"args":[0,"Owner-only update"],"id":"replySupport","method":"POST","params":[{"in":"path","name":"ticketId","placeholder":":id","wireType":"u64"},{"in":"body","name":"body"}],"path":"/api/support/:id/replies","reducer":"reply_support"},"namedTarget":{"attribute":"data-entity-id","contains":"Private managed case {user:casemarker}","testid":"support-ticket","valueType":"string"},"settleMs":1500},{"actor":"other","allowNotFound":true,"do":"expectReplayRejected"},{"actor":"staff","do":"reload","settleMs":2000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee"},{"actor":"staff","contains":"Private managed case {user:casemarker}","do":"expect","testid":"support-ticket"},{"actor":"staff","contains":"Owner-only update","do":"expectElementCount","equals":1,"testid":"support-reply-item","within":10000},{"actor":"staff","contains":"Owner-only update","do":"expectReceived","within":10000},{"actor":"other","contains":"Owner-only update","do":"expectNotReceived"}]}],"id":613,"setup":[{"actor":"owner","do":"signUp","name":"managed-private-owner"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-email","text":"managed-private@example.com"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Private managed case {user:casemarker}"},{"actor":"owner","do":"fill","testid":"support-message","text":"Private case details."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","do":"expect","nonEmpty":true,"testid":"support-reference"},{"actor":"other","do":"signUp","name":"managed-private-other"},{"actor":"other","do":"click","testid":"support-link"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.managed-support"],"role":"guarantee","source":"scenarios/progression-managed-support-privacy.json"}],"id":"selected-source-077","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/progression-managed-support-privacy.json"},{"checkGroups":[{"checkGroupId":"managed-support","feature":{"actors":["owner","staff"],"criteria":[{"id":"613c","steps":[{"actor":"staff","do":"fill","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-status-input","text":"in progress"},{"actor":"staff","do":"click","in":{"contains":"Shared managed case","testid":"support-ticket"},"settleMs":1500,"testid":"support-update"},{"actor":"staff","do":"fill","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-reply","text":"Case received."},{"actor":"staff","do":"click","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-reply-submit"},{"actor":"owner","do":"reload","settleMs":2000},{"actor":"owner","do":"ensureSignedIn","name":"managed-shared-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","contains":"in progress","do":"expect","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-status"},{"actor":"owner","contains":"Case received.","do":"expect","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-reply-item"},{"actor":"owner","do":"fill","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-reply","text":"Thank you."},{"actor":"owner","do":"click","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-reply-submit"},{"actor":"staff","do":"reload","settleMs":2000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee"},{"actor":"staff","contains":"Thank you.","do":"expect","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-reply-item"}]}],"id":613,"setup":[{"actor":"owner","do":"signUp","name":"managed-shared-owner"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-email","text":"managed-shared@example.com"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Shared managed case"},{"actor":"owner","do":"fill","testid":"support-message","text":"Please investigate this case."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","do":"expect","nonEmpty":true,"testid":"support-reference"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee"}]},"packId":"ecommerce.progression.managed-support","role":"feature","source":"scenarios/progression-managed-support-shared.json"},{"checkGroupId":"managed-support","feature":{"actors":["owner","staff"],"criteria":[{"id":"613a","steps":[{"actor":"staff","do":"fill","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-status-input","text":"open"},{"actor":"staff","do":"click","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-update"},{"actor":"staff","contains":"open","do":"expect","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-status"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"managed-shared-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"support-link","unlessVisible":"support-ticket"},{"actor":"owner","contains":"open","do":"expect","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-status"},{"actor":"staff","do":"fill","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-status-input","text":"in progress"},{"actor":"staff","do":"click","in":{"contains":"Shared managed case","testid":"support-ticket"},"settleMs":1500,"testid":"support-update"},{"actor":"staff","do":"fill","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-reply","text":"We are investigating."},{"actor":"staff","do":"click","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-reply-submit"},{"actor":"owner","contains":"in progress","do":"expect","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-status","within":10000},{"actor":"owner","contains":"We are investigating.","do":"expect","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-reply-item","within":10000},{"actor":"owner","do":"fill","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-reply","text":"Thank you for the update."},{"actor":"owner","do":"click","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-reply-submit"},{"actor":"staff","contains":"Thank you for the update.","do":"expect","in":{"contains":"Shared managed case","testid":"support-ticket"},"testid":"support-reply-item","within":10000}]}],"id":613,"setup":[{"actor":"owner","do":"signUp","name":"managed-shared-owner"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-email","text":"managed-shared@example.com"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Shared managed case"},{"actor":"owner","do":"fill","testid":"support-message","text":"Please investigate this case."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","do":"expect","nonEmpty":true,"testid":"support-reference"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","requiresFeatures":["ecommerce.progression.managed-support"],"role":"guarantee","source":"scenarios/progression-managed-support-shared.json"}],"id":"selected-source-078","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/progression-managed-support-shared.json"},{"checkGroups":[{"checkGroupId":"notification-preferences","feature":{"actors":["owner","other"],"criteria":[{"id":"630c","steps":[{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"owner","do":"click","testid":"notification-settings","unlessVisible":"notification-order"},{"actor":"owner","attribute":"data-state","do":"expect","testid":"notification-order","value":"on"}]}],"id":630,"setup":[{"actor":"owner","do":"signUp","name":"notification-owner"},{"actor":"other","do":"signUp","name":"notification-other"},{"actor":"owner","do":"click","testid":"notification-settings","unlessVisible":"notification-order"},{"actor":"owner","do":"click","testid":"notification-order"},{"actor":"owner","do":"click","testid":"notification-save"}]},"packId":"ecommerce.progression.notification-preferences","role":"feature","source":"scenarios/progression-notification-preferences.json"},{"checkGroupId":"notification-preferences-reload","feature":{"actors":["owner","other"],"criteria":[{"id":"630a","steps":[{"actor":"owner","do":"reload","settleMs":3000},{"actor":"owner","do":"ensureSignedIn","name":"notification-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"notification-settings","unlessVisible":"notification-order"},{"actor":"owner","attribute":"data-state","do":"expect","testid":"notification-order","value":"on"},{"do":"restartBackend","settleMs":1000},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"notification-owner"},{"actor":"owner-fresh","do":"click","testid":"notification-settings","unlessVisible":"notification-order"},{"actor":"owner-fresh","attribute":"data-state","do":"expect","testid":"notification-order","value":"on"}]}],"id":630,"setup":[{"actor":"owner","do":"signUp","name":"notification-owner"},{"actor":"other","do":"signUp","name":"notification-other"},{"actor":"owner","do":"click","testid":"notification-settings","unlessVisible":"notification-order"},{"actor":"owner","do":"click","testid":"notification-order"},{"actor":"owner","do":"click","testid":"notification-save"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","requiresFeatures":["ecommerce.progression.notification-preferences"],"role":"guarantee","source":"scenarios/progression-notification-preferences.json"},{"checkGroupId":"notification-preferences-privacy","feature":{"actors":["owner","other"],"criteria":[{"id":"630b","steps":[{"actor":"other","do":"click","testid":"notification-settings","unlessVisible":"notification-order"},{"actor":"other","attribute":"data-state","do":"expect","testid":"notification-order","value":"off"}]}],"id":630,"setup":[{"actor":"owner","do":"signUp","name":"notification-owner"},{"actor":"other","do":"signUp","name":"notification-other"},{"actor":"owner","do":"click","testid":"notification-settings","unlessVisible":"notification-order"},{"actor":"owner","do":"click","testid":"notification-order"},{"actor":"owner","do":"click","testid":"notification-save"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.notification-preferences"],"role":"guarantee","source":"scenarios/progression-notification-preferences.json"}],"id":"selected-source-079","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-notification-preferences.json"},{"checkGroups":[{"checkGroupId":"open-list","feature":{"actors":["reader","reviewer"],"criteria":[{"id":"902a","steps":[{"actor":"reader","do":"openItem","item":"Keyboard"},{"actor":"reader","do":"expect","testid":"item-detail","within":10000},{"actor":"reviewer","do":"click","ifAvailable":true,"testid":"review-toggle","unlessVisible":"review-rating"},{"actor":"reviewer","do":"fill","testid":"review-input","text":"live-review-kbd"},{"actor":"reviewer","do":"click","testid":"review-submit"},{"actor":"reviewer","contains":"live-review-kbd","do":"expectElementCount","equals":1,"testid":"review-item","within":10000},{"actor":"reader","contains":"live-review-kbd","do":"expectElementCount","equals":1,"testid":"review-item","within":10000}]}],"id":902,"setup":[{"actor":"reviewer","do":"signUp","name":"raceR"},{"actor":"reader","do":"signUp","name":"raceL"},{"action":"buy","actor":"reviewer","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"reviewer","do":"expectActionOutcome","outcome":"accepted"},{"actor":"reviewer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"reviewer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"reviewer","contains":"Keyboard","do":"expect","testid":"order-item","within":10000},{"actor":"reviewer","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"reviewer","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"reviewer","do":"openItem","item":"Keyboard"},{"actor":"reviewer","do":"expect","testid":"item-detail","within":10000},{"actor":"reviewer","do":"click","ifAvailable":true,"testid":"review-toggle","unlessVisible":"review-rating"},{"actor":"reviewer","do":"fill","testid":"review-rating","text":"5"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.live-state","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","source":"scenarios/progression-open-list-live.json"}],"id":"selected-source-080","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/progression-open-list-live.json"},{"checkGroups":[{"checkGroupId":"cancellation-and-return","feature":{"actors":["customer"],"criteria":[{"id":"3e","steps":[{"absent":true,"actor":"customer","do":"expect","in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"return-item"}]}],"id":331,"setup":[{"actor":"customer","do":"signUp","name":"return-boundary"},{"action":"buy","actor":"customer","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"customer","do":"expectActionOutcome","outcome":"accepted"},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"return-boundary","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-status","value":"pending"}]},"packId":"ecommerce.l3.order-returns-features","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"feature","source":"scenarios/progression-order-return-boundary.json","stablePackId":"ecommerce.returns-pricing"},{"checkGroupId":"cancellation-and-return","feature":{"actors":["customer","staff","admin"],"criteria":[{"id":"3f","steps":[{"action":"returnItem","actor":"customer","do":"callAction","input":{"attribute":"data-return-input","contains":"Desk Lamp","testid":"order-line"},"namedAction":{"args":[0,0],"id":"returnItem","method":"POST","params":[{"in":"path","name":"orderId","placeholder":"{orderId}","wireType":"u64"},{"in":"path","name":"itemId","placeholder":"{itemId}","wireType":"u64"}],"path":"/api/orders/{orderId}/items/{itemId}/return","reducer":"return_order_item"}},{"actor":"customer","do":"expectActionOutcome","outcome":"validation-refused"},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"return-boundary","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-status","value":"pending"},{"do":"dbExpectStock","item":"Desk Lamp","plus":0,"relativeTo":"pending-East","warehouse":"East"},{"do":"dbExpectStock","item":"Desk Lamp","plus":0,"relativeTo":"pending-West","warehouse":"West"},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"expectNumber","plus":0,"relativeTo":"pending-revenue","testid":"admin-revenue"}]}],"id":332,"setup":[{"actor":"customer","do":"signUp","name":"return-boundary"},{"action":"buy","actor":"customer","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"customer","do":"expectActionOutcome","outcome":"accepted"},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"return-boundary","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-status","value":"pending"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"return-boundary","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"catalog-link","unlessVisible":"item-card"},{"action":"buy","actor":"customer","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"customer","do":"expectActionOutcome","outcome":"accepted"},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"return-boundary","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"action":"ship","actor":"staff","do":"callAction","from":"customer","input":{"attribute":"data-ship-input","contains":"Keyboard","testid":"order-item"},"namedAction":{"args":[0],"id":"ship","params":[{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/fulfilment/ship","reducer":"ship_order"}},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"return-boundary","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"shipped"},{"action":"returnItem","actor":"customer","do":"callAction","input":{"attribute":"data-return-input","contains":"Keyboard","testid":"order-line"},"namedAction":{"args":[0,0],"id":"returnItem","method":"POST","params":[{"in":"path","name":"orderId","placeholder":"{orderId}","wireType":"u64"},{"in":"path","name":"itemId","placeholder":"{itemId}","wireType":"u64"}],"path":"/api/orders/{orderId}/items/{itemId}/return","reducer":"return_order_item"}},{"actor":"customer","do":"expectActionOutcome","outcome":"accepted"},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"return-boundary","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"returned"},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","as":"pending-revenue","do":"recordNumber","testid":"admin-revenue"},{"as":"pending-East","do":"dbRecordStock","item":"Desk Lamp","warehouse":"East"},{"as":"pending-West","do":"dbRecordStock","item":"Desk Lamp","warehouse":"West"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-status","value":"pending"}]},"packId":"ecommerce.l3.order-returns-features","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"feature","source":"scenarios/progression-order-return-boundary.json","stablePackId":"ecommerce.returns-pricing"}],"id":"selected-source-081","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/progression-order-return-boundary.json"},{"checkGroups":[{"checkGroupId":"cancellation-and-return","feature":{"actors":["customer","staff","admin"],"criteria":[{"id":"3c","steps":[{"actor":"customer","do":"click","in":{"contains":"Keyboard","testid":"order-item"},"testid":"return-item"},{"actor":"customer","contains":"Keyboard","containsText":"returned","do":"expect","ignoreCase":true,"testid":"order-item","within":10000},{"actor":"customer","do":"freshClient"},{"actor":"customer-fresh","do":"signIn","name":"return-complete"},{"actor":"customer-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer-fresh","do":"click","testid":"orders-toggle"},{"actor":"customer-fresh","contains":"Keyboard","containsText":"returned","do":"expect","ignoreCase":true,"testid":"order-item","within":10000},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"expectNumber","plus":0,"relativeTo":"return-revenue-before","testid":"admin-revenue","within":10000},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","ifAvailable":true,"testid":"catalog-link","within":1000},{"do":"dbExpectStock","item":"Keyboard","plus":0,"relativeTo":"return-stock-before"}]}],"id":330,"setup":[{"actor":"customer","do":"signUp","name":"return-complete"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"staff","do":"click","testid":"staff-link"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"customer","do":"expectNumber","equals":89,"in":{"contains":"Keyboard","testid":"item-card"},"testid":"item-price"},{"as":"return-stock-before","do":"dbRecordStock","item":"Keyboard"},{"actor":"admin","as":"return-revenue-before","do":"recordNumber","testid":"admin-revenue"},{"actor":"customer","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"buy-now"},{"actor":"staff","do":"reload","settleMs":1000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"queue-item"},{"actor":"staff","contains":"Keyboard","do":"expect","testid":"queue-item","within":10000},{"do":"dbExpectStock","item":"Keyboard","plus":-1,"relativeTo":"return-stock-before"},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"expectNumber","plus":89,"relativeTo":"return-revenue-before","testid":"admin-revenue","within":10000},{"actor":"staff","do":"click","in":{"contains":"Keyboard","testid":"queue-item"},"testid":"ship-submit"},{"actor":"customer","do":"reload","settleMs":1000},{"actor":"customer","do":"ensureSignedIn","name":"return-complete","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"shipped","within":10000}]},"packId":"ecommerce.l3.order-returns-features","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.fulfilment-queue"],"role":"feature","source":"scenarios/progression-order-return-complete.json","stablePackId":"ecommerce.returns-pricing"}],"id":"selected-source-082","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/progression-order-return-complete.json"},{"checkGroups":[{"checkGroupId":"order-support-ownership","feature":{"actors":["owner","other"],"criteria":[{"id":"614b","steps":[{"absent":true,"actor":"other","contains":"Desk Lamp","do":"expect","testid":"support-order-option"},{"action":"linkSupportOrder","actor":"owner","authentication":"actor","do":"callAction","input":{"attribute":"data-action-input","testid":"support-link-order"},"namedAction":{"args":[0,0],"id":"linkSupportOrder","method":"POST","params":[{"in":"path","name":"caseId","placeholder":"{caseId}","wireType":"u64"},{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/support/cases/{caseId}/order","reducer":"link_support_order"},"settleMs":1500},{"actor":"owner","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","contains":"Desk Lamp","do":"expectElementCount","equals":1,"testid":"support-order","within":10000},{"action":"linkSupportOrder","actor":"other","authentication":"actor","do":"callAction","from":"owner","input":{"attribute":"data-action-input","overrides":{"caseId":{"actor":"other","attribute":"data-entity-id","contains":"Other order case","testid":"support-ticket"}},"testid":"support-link-order"},"namedAction":{"args":[0,0],"id":"linkSupportOrder","method":"POST","params":[{"in":"path","name":"caseId","placeholder":"{caseId}","wireType":"u64"},{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/support/cases/{caseId}/order","reducer":"link_support_order"},"settleMs":1500},{"actor":"other","do":"expectActionOutcome","outcome":"refused"},{"actor":"other","do":"freshClient"},{"actor":"other-fresh","do":"signIn","name":"order-boundary-other"},{"actor":"other-fresh","do":"click","testid":"support-link"},{"absent":true,"actor":"other-fresh","contains":"Desk Lamp","do":"expect","in":{"contains":"Other order case","testid":"support-ticket"},"testid":"support-order"},{"actor":"owner","contains":"Desk Lamp","do":"expectElementCount","equals":1,"testid":"support-order","within":10000}]}],"id":614,"setup":[{"actor":"owner","do":"signUp","name":"order-boundary-owner"},{"actor":"owner","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"buy-now"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Owner order case"},{"actor":"owner","do":"fill","testid":"support-message","text":"This case belongs to the order owner."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","contains":"Desk Lamp","do":"click","in":{"contains":"Owner order case","testid":"support-ticket"},"testid":"support-order-option"},{"actor":"other","do":"signUp","name":"order-boundary-other"},{"actor":"other","do":"click","testid":"support-link"},{"actor":"other","do":"fill","testid":"support-subject","text":"Other order case"},{"actor":"other","do":"fill","testid":"support-message","text":"This is a separate case."},{"actor":"other","do":"click","testid":"support-submit"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.order-support"],"role":"guarantee","source":"scenarios/progression-order-support-boundary.json"}],"id":"selected-source-083","scenario":{"level":4,"writeUrlPattern":null},"source":"scenarios/progression-order-support-boundary.json"},{"checkGroups":[{"checkGroupId":"order-support-owned","feature":{"actors":["owner","staff"],"criteria":[{"id":"614a","steps":[{"actor":"owner","contains":"Desk Lamp","do":"click","in":{"contains":"Owned order case","testid":"support-ticket"},"testid":"support-order-option"},{"actor":"owner","do":"click","in":{"contains":"Owned order case","testid":"support-ticket"},"testid":"support-link-order"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee"},{"actor":"staff","contains":"Desk Lamp","do":"expect","in":{"contains":"Owned order case","testid":"support-ticket"},"testid":"support-order","within":10000}]}],"id":614,"setup":[{"actor":"owner","do":"signUp","name":"order-support-owner"},{"actor":"owner","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"buy-now"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Owned order case"},{"actor":"owner","do":"fill","testid":"support-message","text":"Question about my Desk Lamp order."},{"actor":"owner","do":"click","testid":"support-submit"}]},"packId":"ecommerce.progression.order-support","role":"feature","source":"scenarios/progression-order-support-owned.json"}],"id":"selected-source-084","scenario":{"level":4,"writeUrlPattern":null},"source":"scenarios/progression-order-support-owned.json"},{"checkGroups":[{"checkGroupId":"personalized-recommendations","feature":{"actors":["sales-helper","audio-customer","home-customer","computing-customer"],"criteria":[{"id":"403a","steps":[{"actor":"audio-customer","contains":"Headphones","do":"expect","in":{"testid":"recommendations"},"testid":"recommended-item"},{"actor":"home-customer","do":"expectNumber","equals":1,"in":{"contains":"Desk Lamp","testid":"recommended-item"},"testid":"recommendation-rank","within":10000},{"actor":"computing-customer","do":"expectSequence","equals":["Gaming Mouse","Laptop Stand","Webcam"],"in":{"testid":"recommendations"},"testid":"recommended-item","within":10000}]}],"id":403,"setup":[{"actor":"sales-helper","do":"signUp","name":"recommend-sales"},{"actor":"sales-helper","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"settleMs":500,"testid":"buy-now"},{"actor":"sales-helper","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"settleMs":500,"testid":"buy-now"},{"actor":"sales-helper","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"settleMs":500,"testid":"buy-now"},{"actor":"audio-customer","do":"signUp","name":"recommend-audio"},{"actor":"audio-customer","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"buy-now"},{"actor":"home-customer","do":"signUp","name":"recommend-home"},{"actor":"home-customer","do":"click","in":{"contains":"Coffee Grinder","testid":"item-card"},"testid":"buy-now"},{"actor":"computing-customer","do":"signUp","name":"recommend-computing"},{"actor":"computing-customer","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"buy-now"}]},"packId":"ecommerce.progression.personalized-recommendations","role":"feature","source":"scenarios/progression-personalized-recommendations.json"},{"checkGroupId":"recommendation-profile-isolation","feature":{"actors":["sales-helper","audio-customer","home-customer","computing-customer"],"criteria":[{"id":"403b","steps":[{"actor":"audio-customer","do":"click","in":{"contains":"Headphones","testid":"item-card"},"testid":"buy-now"},{"actor":"audio-customer","contains":"Headphones","do":"waitUntilAbsent","in":{"testid":"recommendations"},"testid":"recommended-item","within":10000},{"actor":"home-customer","do":"expectNumber","equals":1,"in":{"contains":"Desk Lamp","testid":"recommended-item"},"testid":"recommendation-rank","within":10000},{"actor":"computing-customer","do":"expectSequence","equals":["Gaming Mouse","Laptop Stand","Webcam"],"in":{"testid":"recommendations"},"testid":"recommended-item","within":10000}]}],"id":403,"setup":[{"actor":"sales-helper","do":"signUp","name":"recommend-sales"},{"actor":"sales-helper","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"settleMs":500,"testid":"buy-now"},{"actor":"sales-helper","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"settleMs":500,"testid":"buy-now"},{"actor":"sales-helper","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"settleMs":500,"testid":"buy-now"},{"actor":"audio-customer","do":"signUp","name":"recommend-audio"},{"actor":"audio-customer","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"buy-now"},{"actor":"home-customer","do":"signUp","name":"recommend-home"},{"actor":"home-customer","do":"click","in":{"contains":"Coffee Grinder","testid":"item-card"},"testid":"buy-now"},{"actor":"computing-customer","do":"signUp","name":"recommend-computing"},{"actor":"computing-customer","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"buy-now"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.personalized-recommendations"],"role":"guarantee","source":"scenarios/progression-personalized-recommendations.json"}],"id":"selected-source-085","scenario":{"level":4,"writeUrlPattern":null},"source":"scenarios/progression-personalized-recommendations.json"},{"checkGroups":[{"checkGroupId":"price-history","feature":{"actors":["admin","customer"],"criteria":[{"id":"4c","steps":[{"actor":"customer","do":"fill","enter":true,"settleMs":1500,"testid":"search-input","text":"Desk Lamp"},{"actor":"customer","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"add-to-cart"},{"actor":"customer","do":"click","testid":"cart-toggle"},{"actor":"customer","do":"expectNumber","equals":42,"testid":"cart-total"},{"actor":"admin","do":"fill","in":{"contains":"Desk Lamp","testid":"admin-item-row"},"testid":"price-input","text":"52.00"},{"actor":"admin","do":"click","in":{"contains":"Desk Lamp","testid":"admin-item-row"},"testid":"price-submit"},{"actor":"customer","do":"expectNumber","equals":52,"testid":"cart-total","within":10000},{"action":"checkout","actor":"customer","do":"callAction","namedAction":{"args":[],"id":"checkout","method":"POST","path":"/api/checkout","reducer":"checkout"},"settleMs":1500},{"actor":"customer","do":"expectActionOutcome","outcome":"accepted"},{"actor":"customer","do":"freshClient"},{"actor":"customer-fresh","do":"signIn","name":"price-cart"},{"actor":"customer-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer-fresh","do":"click","testid":"orders-toggle"},{"actor":"customer-fresh","contains":"Desk Lamp","count":1,"do":"expect","testid":"order-item"},{"actor":"customer-fresh","do":"expectNumber","equals":52,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-total"}]}],"id":420,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"customer","do":"signUp","name":"price-cart"},{"actor":"admin","do":"click","testid":"admin-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.price-history-specifications","requiresFeatures":["ecommerce.feature.checkout","ecommerce.l2.price-history-features"],"role":"guarantee","source":"scenarios/progression-price-cart-checkout.json","stablePackId":"ecommerce.returns-pricing"}],"id":"selected-source-086","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/progression-price-cart-checkout.json"},{"checkGroups":[{"checkGroupId":"product-bundles","feature":{"actors":["admin","visitor"],"criteria":[{"id":"740a","steps":[{"actor":"admin","do":"fill","testid":"bundle-name-input","text":"Office bundle"},{"actor":"admin","do":"fill","testid":"bundle-price-input","text":"75.00"},{"actor":"admin","do":"fill","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"},{"actor":"admin","do":"click","testid":"bundle-save"},{"actor":"admin","contains":"Office bundle","do":"expect","testid":"bundle-card","within":10000},{"actor":"visitor","do":"reload","settleMs":1000},{"actor":"visitor","do":"click","testid":"bundles-link","unlessVisible":"bundle-card"},{"actor":"visitor","do":"expectNumber","equals":75,"in":{"contains":"Office bundle","testid":"bundle-card"},"testid":"bundle-price","within":10000},{"actor":"visitor","do":"expectElementCount","equals":2,"in":{"contains":"Office bundle","testid":"bundle-card"},"testid":"bundle-component"},{"actor":"visitor","attribute":"data-quantity","contains":"Keyboard","do":"expect","in":{"contains":"Office bundle","testid":"bundle-card"},"testid":"bundle-component","value":"2","within":10000},{"actor":"visitor","attribute":"data-quantity","contains":"Desk Lamp","do":"expect","in":{"contains":"Office bundle","testid":"bundle-card"},"testid":"bundle-component","value":"1","within":10000}]}],"id":740,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"bundles-link","unlessVisible":"bundle-name-input"}]},"packId":"ecommerce.feature.product-bundles","role":"feature","source":"scenarios/progression-product-bundles.json"},{"checkGroupId":"bundle-743","feature":{"actors":["admin","customer"],"criteria":[{"id":"743a","steps":[{"actor":"admin","do":"fill","testid":"bundle-price-input","text":"1.00"},{"action":"saveBundle","actor":"customer","do":"callAction","from":"admin","input":{"attribute":"data-bundle-save-input","testid":"bundle-save"},"namedAction":{"args":["Protected bundle",1,"[{\"item\":\"Keyboard\",\"quantity\":1}]"],"id":"saveBundle","params":[{"in":"body","name":"name"},{"in":"body","name":"price"},{"in":"body","name":"componentsJson"}],"path":"/api/bundles","reducer":"save_bundle"},"settleMs":1000},{"actor":"customer","do":"expectActionOutcome","outcome":"application-refused"},{"actor":"admin","do":"reload","settleMs":1000},{"actor":"admin","do":"click","testid":"bundles-link","unlessVisible":"bundle-card"},{"actor":"admin","do":"expectNumber","equals":75,"in":{"contains":"Protected bundle","testid":"bundle-card"},"testid":"bundle-price","within":10000},{"actor":"admin","attribute":"data-quantity","contains":"Keyboard","do":"expect","in":{"contains":"Protected bundle","testid":"bundle-card"},"testid":"bundle-component","value":"2","within":10000}]}],"id":743,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"bundles-link","unlessVisible":"bundle-name-input"},{"actor":"customer","do":"signUp","name":"bundle-management"},{"actor":"admin","do":"fill","testid":"bundle-name-input","text":"Protected bundle"},{"actor":"admin","do":"fill","testid":"bundle-price-input","text":"75.00"},{"actor":"admin","do":"fill","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"},{"actor":"admin","do":"click","testid":"bundle-save"},{"actor":"admin","contains":"Protected bundle","do":"expect","testid":"bundle-card","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.bundle-integrity","requiresFeatures":["ecommerce.feature.product-bundles"],"role":"guarantee","source":"scenarios/progression-product-bundles.json"}],"id":"selected-source-087","scenario":{"level":4,"writeUrlPattern":null},"source":"scenarios/progression-product-bundles.json"},{"checkGroups":[{"checkGroupId":"promotion-checkout-active","feature":{"actors":["staff","activeBuyer","expiredBuyer","firstBuyer","secondBuyer"],"criteria":[{"id":"621a","steps":[{"actor":"activeBuyer","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"actor":"activeBuyer","do":"click","testid":"cart-toggle"},{"actor":"activeBuyer","do":"fill","testid":"cart-promotion","text":"LIVE10"},{"actor":"activeBuyer","do":"click","testid":"apply-promotion"},{"actor":"activeBuyer","do":"click","testid":"checkout-submit"},{"actor":"activeBuyer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"activeBuyer","do":"click","testid":"orders-toggle"},{"actor":"activeBuyer","do":"expectNumber","equals":8.9,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-discount"}]}],"id":621,"setup":[{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"promotions-link"},{"actor":"staff","do":"click","testid":"promotions-link","unlessVisible":"promotion-code"},{"actor":"staff","do":"fill","testid":"promotion-code","text":"LIVE10"},{"actor":"staff","do":"fill","testid":"promotion-discount","text":"10"},{"actor":"staff","do":"fill","testid":"promotion-start","text":"2020-01-01"},{"actor":"staff","do":"fill","testid":"promotion-end","text":"2099-12-31"},{"actor":"staff","do":"fill","testid":"promotion-limit","text":"10"},{"actor":"staff","do":"click","testid":"promotion-submit"},{"actor":"staff","do":"fill","testid":"promotion-code","text":"OLD10"},{"actor":"staff","do":"fill","testid":"promotion-discount","text":"10"},{"actor":"staff","do":"fill","testid":"promotion-start","text":"2000-01-01"},{"actor":"staff","do":"fill","testid":"promotion-end","text":"2000-12-31"},{"actor":"staff","do":"fill","testid":"promotion-limit","text":"10"},{"actor":"staff","do":"click","testid":"promotion-submit"},{"actor":"staff","do":"fill","testid":"promotion-code","text":"ONCE10"},{"actor":"staff","do":"fill","testid":"promotion-discount","text":"10"},{"actor":"staff","do":"fill","testid":"promotion-start","text":"2020-01-01"},{"actor":"staff","do":"fill","testid":"promotion-end","text":"2099-12-31"},{"actor":"staff","do":"fill","testid":"promotion-limit","text":"1"},{"actor":"staff","do":"click","testid":"promotion-submit"},{"actor":"activeBuyer","do":"signUp","name":"promotion-active"},{"actor":"expiredBuyer","do":"signUp","name":"promotion-expired"},{"actor":"firstBuyer","do":"signUp","name":"promotion-first"},{"actor":"secondBuyer","do":"signUp","name":"promotion-second"}]},"packId":"ecommerce.progression.promotion-checkout","role":"feature","source":"scenarios/progression-promotion-checkout.json"},{"checkGroupId":"promotion-checkout-expired","feature":{"actors":["staff","activeBuyer","expiredBuyer","firstBuyer","secondBuyer"],"criteria":[{"id":"621b","steps":[{"actor":"expiredBuyer","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"actor":"expiredBuyer","do":"click","testid":"cart-toggle"},{"actor":"expiredBuyer","do":"fill","testid":"cart-promotion","text":"OLD10"},{"actor":"expiredBuyer","do":"click","testid":"apply-promotion"},{"actor":"expiredBuyer","do":"expect","testid":"promotion-error"}]}],"id":621,"setup":[{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"promotions-link"},{"actor":"staff","do":"click","testid":"promotions-link","unlessVisible":"promotion-code"},{"actor":"staff","do":"fill","testid":"promotion-code","text":"LIVE10"},{"actor":"staff","do":"fill","testid":"promotion-discount","text":"10"},{"actor":"staff","do":"fill","testid":"promotion-start","text":"2020-01-01"},{"actor":"staff","do":"fill","testid":"promotion-end","text":"2099-12-31"},{"actor":"staff","do":"fill","testid":"promotion-limit","text":"10"},{"actor":"staff","do":"click","testid":"promotion-submit"},{"actor":"staff","do":"fill","testid":"promotion-code","text":"OLD10"},{"actor":"staff","do":"fill","testid":"promotion-discount","text":"10"},{"actor":"staff","do":"fill","testid":"promotion-start","text":"2000-01-01"},{"actor":"staff","do":"fill","testid":"promotion-end","text":"2000-12-31"},{"actor":"staff","do":"fill","testid":"promotion-limit","text":"10"},{"actor":"staff","do":"click","testid":"promotion-submit"},{"actor":"staff","do":"fill","testid":"promotion-code","text":"ONCE10"},{"actor":"staff","do":"fill","testid":"promotion-discount","text":"10"},{"actor":"staff","do":"fill","testid":"promotion-start","text":"2020-01-01"},{"actor":"staff","do":"fill","testid":"promotion-end","text":"2099-12-31"},{"actor":"staff","do":"fill","testid":"promotion-limit","text":"1"},{"actor":"staff","do":"click","testid":"promotion-submit"},{"actor":"activeBuyer","do":"signUp","name":"promotion-active"},{"actor":"expiredBuyer","do":"signUp","name":"promotion-expired"},{"actor":"firstBuyer","do":"signUp","name":"promotion-first"},{"actor":"secondBuyer","do":"signUp","name":"promotion-second"}]},"packId":"ecommerce.progression.promotion-checkout","role":"feature","source":"scenarios/progression-promotion-checkout.json"},{"checkGroupId":"promotion-checkout-exhausted","feature":{"actors":["staff","activeBuyer","expiredBuyer","firstBuyer","secondBuyer"],"criteria":[{"id":"621c","steps":[{"actor":"firstBuyer","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"actor":"firstBuyer","do":"click","testid":"cart-toggle"},{"actor":"firstBuyer","do":"fill","testid":"cart-promotion","text":"ONCE10"},{"actor":"firstBuyer","do":"click","testid":"apply-promotion"},{"actor":"firstBuyer","do":"click","testid":"checkout-submit"},{"actor":"secondBuyer","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"actor":"secondBuyer","do":"click","testid":"cart-toggle"},{"actor":"secondBuyer","do":"fill","testid":"cart-promotion","text":"ONCE10"},{"actor":"secondBuyer","do":"click","testid":"apply-promotion"},{"actor":"secondBuyer","do":"expect","testid":"promotion-error"}]}],"id":621,"setup":[{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"promotions-link"},{"actor":"staff","do":"click","testid":"promotions-link","unlessVisible":"promotion-code"},{"actor":"staff","do":"fill","testid":"promotion-code","text":"LIVE10"},{"actor":"staff","do":"fill","testid":"promotion-discount","text":"10"},{"actor":"staff","do":"fill","testid":"promotion-start","text":"2020-01-01"},{"actor":"staff","do":"fill","testid":"promotion-end","text":"2099-12-31"},{"actor":"staff","do":"fill","testid":"promotion-limit","text":"10"},{"actor":"staff","do":"click","testid":"promotion-submit"},{"actor":"staff","do":"fill","testid":"promotion-code","text":"OLD10"},{"actor":"staff","do":"fill","testid":"promotion-discount","text":"10"},{"actor":"staff","do":"fill","testid":"promotion-start","text":"2000-01-01"},{"actor":"staff","do":"fill","testid":"promotion-end","text":"2000-12-31"},{"actor":"staff","do":"fill","testid":"promotion-limit","text":"10"},{"actor":"staff","do":"click","testid":"promotion-submit"},{"actor":"staff","do":"fill","testid":"promotion-code","text":"ONCE10"},{"actor":"staff","do":"fill","testid":"promotion-discount","text":"10"},{"actor":"staff","do":"fill","testid":"promotion-start","text":"2020-01-01"},{"actor":"staff","do":"fill","testid":"promotion-end","text":"2099-12-31"},{"actor":"staff","do":"fill","testid":"promotion-limit","text":"1"},{"actor":"staff","do":"click","testid":"promotion-submit"},{"actor":"activeBuyer","do":"signUp","name":"promotion-active"},{"actor":"expiredBuyer","do":"signUp","name":"promotion-expired"},{"actor":"firstBuyer","do":"signUp","name":"promotion-first"},{"actor":"secondBuyer","do":"signUp","name":"promotion-second"}]},"packId":"ecommerce.progression.promotion-checkout","role":"feature","source":"scenarios/progression-promotion-checkout.json"}],"id":"selected-source-088","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-promotion-checkout.json"},{"checkGroups":[{"checkGroupId":"promotion-report-redemptions","feature":{"actors":["staff","buyer"],"criteria":[{"id":"622a","steps":[{"actor":"staff","do":"expectNumber","equals":1,"in":{"contains":"REPORT10","testid":"promotion-report"},"testid":"promotion-redemptions"}]}],"id":622,"setup":[{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"promotions-link"},{"actor":"staff","do":"click","testid":"promotions-link","unlessVisible":"promotion-code"},{"actor":"staff","do":"fill","testid":"promotion-code","text":"REPORT10"},{"actor":"staff","do":"fill","testid":"promotion-discount","text":"10"},{"actor":"staff","do":"fill","testid":"promotion-start","text":"2020-01-01"},{"actor":"staff","do":"fill","testid":"promotion-end","text":"2099-12-31"},{"actor":"staff","do":"fill","testid":"promotion-limit","text":"2"},{"actor":"staff","do":"click","testid":"promotion-submit"},{"actor":"buyer","do":"signUp","name":"promotion-report-buyer"},{"actor":"buyer","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"actor":"buyer","do":"click","testid":"cart-toggle"},{"actor":"buyer","do":"fill","testid":"cart-promotion","text":"REPORT10"},{"actor":"buyer","do":"click","testid":"apply-promotion"},{"actor":"buyer","do":"click","testid":"checkout-submit"}]},"packId":"ecommerce.progression.promotion-reporting","role":"feature","source":"scenarios/progression-promotion-reporting.json"},{"checkGroupId":"promotion-report-revenue","feature":{"actors":["staff","buyer"],"criteria":[{"id":"622b","steps":[{"actor":"staff","do":"expectNumber","equals":80.1,"in":{"contains":"REPORT10","testid":"promotion-report"},"testid":"promotion-revenue"}]}],"id":622,"setup":[{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"promotions-link"},{"actor":"staff","do":"click","testid":"promotions-link","unlessVisible":"promotion-code"},{"actor":"staff","do":"fill","testid":"promotion-code","text":"REPORT10"},{"actor":"staff","do":"fill","testid":"promotion-discount","text":"10"},{"actor":"staff","do":"fill","testid":"promotion-start","text":"2020-01-01"},{"actor":"staff","do":"fill","testid":"promotion-end","text":"2099-12-31"},{"actor":"staff","do":"fill","testid":"promotion-limit","text":"2"},{"actor":"staff","do":"click","testid":"promotion-submit"},{"actor":"buyer","do":"signUp","name":"promotion-report-buyer"},{"actor":"buyer","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"add-to-cart"},{"actor":"buyer","do":"click","testid":"cart-toggle"},{"actor":"buyer","do":"fill","testid":"cart-promotion","text":"REPORT10"},{"actor":"buyer","do":"click","testid":"apply-promotion"},{"actor":"buyer","do":"click","testid":"checkout-submit"}]},"packId":"ecommerce.progression.promotion-reporting","role":"feature","source":"scenarios/progression-promotion-reporting.json"}],"id":"selected-source-089","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-promotion-reporting.json"},{"checkGroups":[{"checkGroupId":"promotion-rule-values","feature":{"actors":["staff","customer"],"criteria":[{"id":"620a","steps":[{"actor":"staff","do":"click","testid":"promotions-link","unlessVisible":"promotion-code"},{"actor":"staff","do":"fill","testid":"promotion-code","text":"SAVE10"},{"actor":"staff","do":"fill","testid":"promotion-discount","text":"10"},{"actor":"staff","do":"fill","testid":"promotion-start","text":"2099-01-01"},{"actor":"staff","do":"fill","testid":"promotion-end","text":"2099-12-31"},{"actor":"staff","do":"fill","testid":"promotion-limit","text":"2"},{"actor":"staff","do":"click","testid":"promotion-submit"},{"actor":"staff","do":"expectNumber","equals":10,"in":{"contains":"SAVE10","testid":"promotion-item"},"testid":"promotion-discount"},{"actor":"staff","contains":"2099-01-01","do":"expect","in":{"contains":"SAVE10","testid":"promotion-item"},"testid":"promotion-start"},{"actor":"staff","contains":"2099-12-31","do":"expect","in":{"contains":"SAVE10","testid":"promotion-item"},"testid":"promotion-end"},{"actor":"staff","do":"expectNumber","equals":2,"in":{"contains":"SAVE10","testid":"promotion-item"},"testid":"promotion-limit"}]}],"id":620,"setup":[{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"customer","do":"signUp","name":"promotion-customer"},{"actor":"staff","do":"click","testid":"staff-link"}]},"packId":"ecommerce.progression.promotion-rules","role":"feature","source":"scenarios/progression-promotion-rules.json"},{"checkGroupId":"promotion-management-boundary","feature":{"actors":["staff","customer"],"criteria":[{"id":"620b","steps":[{"actor":"staff","do":"click","testid":"promotions-link","unlessVisible":"promotion-code"},{"actor":"staff","do":"fill","testid":"promotion-code","text":"ACCESS10"},{"actor":"staff","do":"fill","testid":"promotion-discount","text":"10"},{"actor":"staff","do":"fill","testid":"promotion-start","text":"2099-01-01"},{"actor":"staff","do":"fill","testid":"promotion-end","text":"2099-12-31"},{"actor":"staff","do":"fill","testid":"promotion-limit","text":"2"},{"actor":"staff","do":"click","testid":"promotion-submit"},{"actor":"staff","do":"reload","settleMs":1000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"staff-link","unlessVisible":"promotion-code"},{"actor":"staff","do":"click","testid":"promotions-link","unlessVisible":"promotion-code"},{"actor":"staff","contains":"ACCESS10","do":"expect","testid":"promotion-item","within":10000},{"absent":true,"actor":"customer","do":"expect","testid":"promotions-link"},{"actor":"customer","do":"replayAs","from":"staff","match":"ACCESS10","namedAction":{"args":["ACCESS10",10,4070908800000000,4102444740000000,2],"id":"createPromotion","params":[{"in":"body","name":"code"},{"in":"body","name":"discountPercent"},{"in":"body","name":"startMicros"},{"in":"body","name":"endMicros"},{"in":"body","name":"usageLimit"}],"path":"/api/promotions","reducer":"create_promotion"},"settleMs":1500,"swap":{"find":"ACCESS10","with":"HACK10"}},{"actor":"customer","do":"expectReplayRejected"},{"absent":true,"actor":"staff","contains":"HACK10","do":"expect","testid":"promotion-item"}]}],"id":620,"setup":[{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"customer","do":"signUp","name":"promotion-customer"},{"actor":"staff","do":"click","testid":"staff-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.promotion-rules"],"role":"guarantee","source":"scenarios/progression-promotion-rules.json"}],"id":"selected-source-090","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-promotion-rules.json"},{"checkGroups":[{"checkGroupId":"purchase-order","feature":{"actors":["buyer"],"criteria":[{"id":"3c","steps":[{"actor":"buyer","do":"click","in":{"contains":"Coffee Grinder","testid":"item-card"},"testid":"buy-now"},{"actor":"buyer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"buyer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"buyer","contains":"Coffee Grinder","do":"expect","testid":"order-item"},{"actor":"buyer","do":"expectNumber","equals":64,"in":{"contains":"Coffee Grinder","testid":"order-item"},"testid":"order-total"}]}],"id":3,"setup":[{"actor":"buyer","do":"signUp","name":"purchase-buyer"}]},"packId":"ecommerce.feature.purchasing","role":"feature","source":"scenarios/progression-purchasing.json"}],"id":"selected-source-091","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/progression-purchasing.json"},{"checkGroups":[{"checkGroupId":"recommendation-feedback","feature":{"actors":["customer","other"],"criteria":[{"id":"504a","steps":[{"absent":true,"actor":"customer","contains":"Headphones","do":"expect","testid":"recommended-item","within":10000}]}],"id":504,"setup":[{"actor":"customer","do":"signUp","name":"feedback-owner"},{"actor":"customer","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"buy-now"},{"actor":"other","do":"signUp","name":"feedback-other"},{"actor":"other","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"buy-now"},{"actor":"customer","contains":"Headphones","do":"expect","testid":"recommended-item"},{"actor":"other","contains":"Headphones","do":"expect","testid":"recommended-item"},{"actor":"customer","do":"click","in":{"contains":"Headphones","testid":"recommended-item"},"testid":"dismiss-recommendation"}]},"packId":"ecommerce.progression.recommendation-feedback","role":"feature","source":"scenarios/progression-recommendation-feedback.json"},{"checkGroupId":"recommendation-feedback-privacy","feature":{"actors":["customer","other"],"criteria":[{"id":"504b","steps":[{"actor":"customer","contains":"Headphones","do":"waitUntilAbsent","testid":"recommended-item","within":10000},{"actor":"other","contains":"Headphones","do":"expect","testid":"recommended-item"}]}],"id":504,"setup":[{"actor":"customer","do":"signUp","name":"feedback-owner"},{"actor":"customer","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"buy-now"},{"actor":"other","do":"signUp","name":"feedback-other"},{"actor":"other","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"buy-now"},{"actor":"customer","contains":"Headphones","do":"expect","testid":"recommended-item"},{"actor":"other","contains":"Headphones","do":"expect","testid":"recommended-item"},{"actor":"customer","do":"click","in":{"contains":"Headphones","testid":"recommended-item"},"testid":"dismiss-recommendation"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.recommendation-feedback"],"role":"guarantee","source":"scenarios/progression-recommendation-feedback.json"},{"checkGroupId":"recommendation-feedback-restart","feature":{"actors":["customer","other"],"criteria":[{"id":"504c","steps":[{"actor":"customer","do":"reload","settleMs":3000},{"actor":"customer","do":"ensureSignedIn","name":"feedback-owner","readyTestid":"current-user"},{"absent":true,"actor":"customer","contains":"Headphones","do":"expect","testid":"recommended-item"},{"do":"restartBackend","settleMs":1000},{"actor":"customer","do":"freshClient"},{"actor":"customer-fresh","do":"signIn","name":"feedback-owner"},{"actor":"customer-fresh","do":"expect","testid":"recommendations","within":10000},{"absent":true,"actor":"customer-fresh","contains":"Headphones","do":"expect","testid":"recommended-item"}]}],"id":504,"setup":[{"actor":"customer","do":"signUp","name":"feedback-owner"},{"actor":"customer","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"buy-now"},{"actor":"other","do":"signUp","name":"feedback-other"},{"actor":"other","do":"click","in":{"contains":"Bluetooth Speaker","testid":"item-card"},"testid":"buy-now"},{"actor":"customer","contains":"Headphones","do":"expect","testid":"recommended-item"},{"actor":"other","contains":"Headphones","do":"expect","testid":"recommended-item"},{"actor":"customer","do":"click","in":{"contains":"Headphones","testid":"recommended-item"},"testid":"dismiss-recommendation"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","requiresFeatures":["ecommerce.progression.recommendation-feedback"],"role":"guarantee","source":"scenarios/progression-recommendation-feedback.json"}],"id":"selected-source-092","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-recommendation-feedback.json"},{"checkGroups":[{"checkGroupId":"review-eligibility-direct","feature":{"actors":["owner","stranger"],"criteria":[{"id":"618a","steps":[{"action":"submitReview","actor":"owner","do":"callAction","from":"owner","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0,5,"eligible progression review"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"}],"path":"/api/items/:id/reviews","reducer":"submit_review"},"settleMs":1500},{"actor":"owner","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"openItem","item":"Keyboard"},{"actor":"owner","contains":"eligible progression review","do":"expect","testid":"review-item","within":10000},{"action":"submitReview","actor":"stranger","do":"callAction","from":"stranger","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0,5,"never bought this"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"}],"path":"/api/items/:id/reviews","reducer":"submit_review"},"settleMs":1500},{"actor":"stranger","do":"expectActionOutcome","outcome":"application-refused","routeProvenBy":"owner"},{"actor":"stranger","do":"freshClient"},{"actor":"stranger-fresh","do":"signIn","name":"review-stranger"},{"actor":"stranger-fresh","do":"openItem","item":"Keyboard"},{"actor":"stranger-fresh","contains":"eligible progression review","do":"expect","testid":"review-item","within":10000},{"absent":true,"actor":"stranger-fresh","contains":"never bought this","do":"expect","testid":"review-item","within":3000}]}],"id":618,"setup":[{"actor":"owner","do":"signUp","name":"review-owner"},{"actor":"stranger","do":"signUp","name":"review-stranger"},{"action":"buy","actor":"owner","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"owner","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner","contains":"Keyboard","do":"expect","testid":"order-item","within":10000},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"catalog-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.review-access-specifications","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","source":"scenarios/progression-review-access.json"}],"id":"selected-source-093","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/progression-review-access.json"},{"checkGroups":[{"checkGroupId":"stored-review-script","feature":{"actors":["owner","writer","reader"],"criteria":[{"id":"9180a","steps":[{"action":"submitReview","actor":"owner","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0,5,"Benign stored review"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"}],"path":"/api/items/:id/reviews","reducer":"submit_review"},"settleMs":0},{"actor":"owner","do":"expectActionOutcome","outcome":"accepted"},{"action":"submitReview","actor":"writer","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0,5,"Stored review marker <img src=\"data:,bad-image\" onerror=\"window.__stackBenchScriptCanary()\">"],"id":"submitReview","method":"POST","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"},{"in":"body","name":"rating"},{"in":"body","name":"comment"}],"path":"/api/items/:id/reviews","reducer":"submit_review"},"settleMs":0},{"actor":"writer","do":"expectActionOutcome","outcome":"completed","routeProvenBy":"owner"},{"actor":"owner","do":"openItem","item":"Keyboard"},{"actor":"owner","contains":"Benign stored review","do":"expect","testid":"review-item","within":10000},{"actor":"owner","do":"wait","ms":1000},{"actor":"owner","do":"expectNoScriptExecution"},{"actor":"reader","do":"freshClient"},{"actor":"reader-fresh","do":"signIn","name":"script-review-reader"},{"actor":"reader-fresh","do":"armScriptCanary"},{"actor":"reader-fresh","do":"openItem","item":"Keyboard"},{"actor":"reader-fresh","contains":"Benign stored review","do":"expect","testid":"review-item","within":10000},{"actor":"reader-fresh","do":"wait","ms":1000},{"actor":"reader-fresh","do":"expectNoScriptExecution"}]}],"id":9180,"setup":[{"actor":"owner","do":"signUp","name":"script-review-owner"},{"actor":"reader","do":"signUp","name":"script-review-reader"},{"actor":"writer","do":"signUp","name":"script-review-writer"},{"action":"buy","actor":"owner","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"owner","do":"expectActionOutcome","outcome":"accepted"},{"action":"buy","actor":"writer","do":"callAction","input":{"attribute":"data-buy-input","contains":"Keyboard","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"}},{"actor":"writer","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"armScriptCanary"}]},"observations":["unmentioned"],"packId":"ecommerce.progression.review-access-specifications","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.reviews"],"role":"guarantee","source":"scenarios/progression-review-script.json"}],"id":"selected-source-094","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/progression-review-script.json"},{"checkGroups":[{"checkGroupId":"search-ordering","feature":{"actors":["visitor","buyer"],"criteria":[{"id":"402b","steps":[{"actor":"visitor","do":"expectSequence","equals":["Air Purifier","Bluetooth Speaker","Coffee Grinder","Desk Lamp","Espresso Machine","Gaming Mouse","Headphones","Induction Cooktop","Keyboard","Laptop Stand"],"in":{"testid":"search-results"},"testid":"item-name"},{"actor":"visitor","do":"fill","enter":true,"testid":"search-input","text":"e"},{"actor":"visitor","do":"fill","testid":"minimum-price","text":""},{"actor":"visitor","do":"click","ifAvailable":true,"testid":"filter-apply"},{"actor":"visitor","do":"expectSequence","equals":["Air Purifier","Bluetooth Speaker","Coffee Grinder","Desk Lamp","Espresso Machine","Gaming Mouse","Headphones","Keyboard","Mirrorless Camera","USB Cable"],"in":{"testid":"search-results"},"testid":"item-name"},{"actor":"visitor","do":"fill","enter":true,"testid":"search-input","text":""},{"actor":"visitor","do":"expectSequence","equals":["Headphones","Air Purifier","Bluetooth Speaker","Coffee Grinder","Desk Lamp","Espresso Machine","Gaming Mouse","Induction Cooktop","Keyboard","Laptop Stand"],"in":{"testid":"item-list"},"testid":"item-name"}]}],"id":402,"setup":[{"actor":"buyer","do":"signUp","name":"search-order-buyer"},{"as":"before-search-purchase","do":"dbRecordStock","item":"Headphones"},{"actor":"buyer","do":"click","in":{"contains":"Headphones","testid":"item-card"},"testid":"buy-now"},{"do":"dbExpectStock","item":"Headphones","plus":-1,"relativeTo":"before-search-purchase"},{"actor":"visitor","do":"reload","settleMs":0},{"actor":"visitor","do":"fill","testid":"minimum-price","text":"1"},{"actor":"visitor","do":"click","ifAvailable":true,"testid":"filter-apply"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.search-ordering","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.progression.faceted-search"],"role":"guarantee","source":"scenarios/progression-search-ordering.json"}],"id":"selected-source-095","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/progression-search-ordering.json"},{"checkGroups":[{"checkGroupId":"shipping-accounting","feature":{"actors":["customer","admin","staff"],"criteria":[{"id":"202e","steps":[{"as":"stock-before-purchase","do":"dbRecordStock","item":"Keyboard"},{"actor":"admin","as":"revenue-before-purchase","do":"recordNumber","testid":"admin-revenue"},{"actor":"customer","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"buy-now"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"pending","within":10000},{"do":"dbExpectStock","item":"Keyboard","plus":-1,"relativeTo":"stock-before-purchase"},{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link"},{"actor":"admin","do":"expectNumber","plus":89,"relativeTo":"revenue-before-purchase","testid":"admin-revenue","within":10000},{"actor":"admin","as":"revenue-before-ship","do":"recordNumber","testid":"admin-revenue"},{"as":"East-before-ship","do":"dbRecordStock","item":"Keyboard","warehouse":"East"},{"as":"West-before-ship","do":"dbRecordStock","item":"Keyboard","warehouse":"West"},{"actor":"staff","do":"reload","settleMs":2000},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"staff-link"},{"action":"ship","actor":"staff","do":"callAction","from":"customer","input":{"attribute":"data-ship-input","contains":"Keyboard","testid":"order-item"},"namedAction":{"args":[0],"id":"ship","params":[{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/fulfilment/ship","reducer":"ship_order"},"settleMs":0},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"customer","do":"reload","settleMs":2000},{"actor":"customer","do":"ensureSignedIn","name":"shipping-accounting","readyTestid":"current-user"},{"actor":"customer","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"customer","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"customer","do":"expect","ignoreCase":true,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"shipped","within":10000},{"do":"dbExpectStock","item":"Keyboard","plus":0,"relativeTo":"East-before-ship","warehouse":"East"},{"do":"dbExpectStock","item":"Keyboard","plus":0,"relativeTo":"West-before-ship","warehouse":"West"},{"actor":"admin","do":"reload","settleMs":2000},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link"},{"actor":"admin","do":"expectNumber","plus":0,"relativeTo":"revenue-before-ship","testid":"admin-revenue","within":10000}]}],"id":202,"setup":[{"actor":"customer","do":"signUp","name":"shipping-accounting"},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.progression.inventory-conservation-specifications","requiresFeatures":["ecommerce.feature.purchasing","ecommerce.feature.warehouse-admin","ecommerce.progression.fulfilment-queue"],"role":"guarantee","source":"scenarios/progression-shipping-accounting.json","stablePackId":"ecommerce.inventory-operations"}],"id":"selected-source-096","scenario":{"level":3,"writeUrlPattern":null},"source":"scenarios/progression-shipping-accounting.json"},{"checkGroups":[{"checkGroupId":"signed-out-purchase","feature":{"actors":["visitor"],"criteria":[{"id":"3a","steps":[{"actor":"visitor","as":"keyboard-before-guest","do":"recordNumber","in":{"contains":"Keyboard","testid":"item-card"},"testid":"item-stock"},{"actor":"visitor","do":"click","ifAvailable":true,"in":{"contains":"Keyboard","testid":"item-card"},"settleMs":1500,"testid":"buy-now"},{"actor":"visitor","do":"reload","settleMs":2000},{"actor":"visitor","do":"click","ifAvailable":true,"testid":"overlay-close"},{"actor":"visitor","do":"click","ifAvailable":true,"testid":"catalog-link"},{"actor":"visitor","do":"expectNumber","in":{"contains":"Keyboard","testid":"item-card"},"plus":0,"relativeTo":"keyboard-before-guest","testid":"item-stock"}]}],"id":3,"setup":[{"actor":"visitor","contains":"Keyboard","do":"expect","testid":"item-card"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/progression-signed-out-purchase.json"}],"id":"selected-source-097","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/progression-signed-out-purchase.json"},{"checkGroups":[{"checkGroupId":"split-tender-refunds-751","feature":{"actors":["owner","staff"],"criteria":[{"id":"751a","steps":[{"actor":"staff","do":"click","in":{"contains":"Split refund 751","testid":"support-ticket"},"testid":"support-refund"},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"credit-owner-751"},{"actor":"owner-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner-fresh","do":"click","testid":"orders-toggle"},{"actor":"owner-fresh","contains":"Desk Lamp","do":"expectElementCount","equals":1,"testid":"refund-entry","within":10000},{"actor":"owner-fresh","do":"expectNumber","equals":42,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-refund-total"},{"actor":"owner-fresh","do":"expectNumber","equals":10,"in":{"contains":"Desk Lamp","testid":"refund-entry"},"testid":"refund-credit-amount"},{"actor":"owner-fresh","do":"expectNumber","equals":32,"in":{"contains":"Desk Lamp","testid":"refund-entry"},"testid":"refund-external-amount"},{"actor":"owner-fresh","do":"click","testid":"credit-link"},{"actor":"owner-fresh","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"owner-fresh","do":"expectNumber","equals":10,"testid":"credit-balance"}]}],"id":751,"setup":[{"actor":"owner","do":"signUp","name":"credit-owner-751"},{"actor":"owner","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"staff","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"staff","do":"fill","testid":"credit-amount-input","text":"10.00"},{"actor":"staff","do":"fill","testid":"credit-reference-input","text":"credit-751"},{"action":"grantCredit","actor":"staff","do":"callAction","from":"staff","input":{"attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","attribute":"data-account-id","testid":"credit-panel"}},"testid":"credit-grant"},"namedAction":{"args":[0,1000,""],"id":"grantCredit","method":"POST","params":[{"in":"body","name":"accountId","wireType":"u64"},{"in":"body","name":"amountMinor"},{"in":"body","name":"reference"}],"path":"/api/staff/credit","reducer":"grant_credit"},"settleMs":0},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"credit-owner-751","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"credit-link","unlessVisible":"credit-balance"},{"actor":"owner","do":"expectNumber","equals":10,"testid":"credit-balance","within":10000},{"actor":"owner","do":"click","testid":"catalog-link"},{"actor":"owner","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"add-to-cart"},{"actor":"owner","do":"click","testid":"cart-toggle"},{"actor":"owner","do":"expectNumber","equals":42,"testid":"cart-total"},{"actor":"owner","do":"click","testid":"credit-checkout"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner","do":"expectNumber","equals":10,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"payment-credit-amount","within":10000},{"actor":"owner","do":"expectNumber","equals":32,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"payment-external-amount"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Split refund 751"},{"actor":"owner","do":"fill","testid":"support-message","text":"Please refund this order."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","contains":"Desk Lamp","do":"click","in":{"contains":"Split refund 751","testid":"support-ticket"},"testid":"support-order-option"},{"actor":"owner","do":"click","in":{"contains":"Split refund 751","testid":"support-ticket"},"testid":"support-link-order"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee","within":1000}]},"packId":"ecommerce.feature.split-tender-refunds","requiresFeatures":["ecommerce.feature.split-tender-refunds","ecommerce.feature.store-credit","ecommerce.progression.support-refunds"],"role":"feature","source":"scenarios/progression-split-tender-refunds.json"},{"checkGroupId":"production-756","feature":{"actors":["owner","staff","staff2"],"criteria":[{"id":"756a","steps":[{"action":"supportRefund","actors":["staff","staff2"],"do":"callConcurrently","from":"staff","input":{"attribute":"data-refund-input","contains":"Split refund 756","testid":"support-ticket"},"namedAction":{"args":[0],"id":"supportRefund","method":"POST","params":[{"in":"path","name":"caseId","placeholder":"{caseId}","wireType":"u64"}],"path":"/api/support/cases/{caseId}/refund","reducer":"support_refund"},"settleMs":0},{"do":"expectCallOutcomes"},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"credit-owner-756"},{"actor":"owner-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner-fresh","do":"click","testid":"orders-toggle"},{"actor":"owner-fresh","contains":"Desk Lamp","do":"expectElementCount","equals":1,"testid":"refund-entry","within":10000},{"actor":"owner-fresh","do":"expectNumber","equals":42,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-refund-total"},{"actor":"owner-fresh","do":"expectNumber","equals":10,"in":{"contains":"Desk Lamp","testid":"refund-entry"},"testid":"refund-credit-amount"},{"actor":"owner-fresh","do":"expectNumber","equals":32,"in":{"contains":"Desk Lamp","testid":"refund-entry"},"testid":"refund-external-amount"},{"actor":"owner-fresh","do":"click","testid":"credit-link"},{"actor":"owner-fresh","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"owner-fresh","do":"expectNumber","equals":10,"testid":"credit-balance"},{"do":"restartBackend","settleMs":1000},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"credit-owner-756"},{"actor":"owner-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner-fresh","do":"click","testid":"orders-toggle"},{"actor":"owner-fresh","contains":"Desk Lamp","do":"expectElementCount","equals":1,"testid":"refund-entry","within":10000},{"actor":"owner-fresh","do":"expectNumber","equals":42,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-refund-total"},{"actor":"owner-fresh","do":"expectNumber","equals":10,"in":{"contains":"Desk Lamp","testid":"refund-entry"},"testid":"refund-credit-amount"},{"actor":"owner-fresh","do":"expectNumber","equals":32,"in":{"contains":"Desk Lamp","testid":"refund-entry"},"testid":"refund-external-amount"},{"actor":"owner-fresh","do":"click","testid":"credit-link"},{"actor":"owner-fresh","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"owner-fresh","do":"expectNumber","equals":10,"testid":"credit-balance"}]}],"id":756,"setup":[{"actor":"owner","do":"signUp","name":"credit-owner-756"},{"actor":"owner","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"staff","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"staff","do":"fill","testid":"credit-amount-input","text":"10.00"},{"actor":"staff","do":"fill","testid":"credit-reference-input","text":"credit-756"},{"action":"grantCredit","actor":"staff","do":"callAction","from":"staff","input":{"attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","attribute":"data-account-id","testid":"credit-panel"}},"testid":"credit-grant"},"namedAction":{"args":[0,1000,""],"id":"grantCredit","method":"POST","params":[{"in":"body","name":"accountId","wireType":"u64"},{"in":"body","name":"amountMinor"},{"in":"body","name":"reference"}],"path":"/api/staff/credit","reducer":"grant_credit"},"settleMs":0},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"credit-owner-756","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"credit-link","unlessVisible":"credit-balance"},{"actor":"owner","do":"expectNumber","equals":10,"testid":"credit-balance","within":10000},{"actor":"owner","do":"click","testid":"catalog-link"},{"actor":"owner","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"add-to-cart"},{"actor":"owner","do":"click","testid":"cart-toggle"},{"actor":"owner","do":"expectNumber","equals":42,"testid":"cart-total"},{"actor":"owner","do":"click","testid":"credit-checkout"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner","do":"expectNumber","equals":10,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"payment-credit-amount","within":10000},{"actor":"owner","do":"expectNumber","equals":32,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"payment-external-amount"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Split refund 756"},{"actor":"owner","do":"fill","testid":"support-message","text":"Please refund this order."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","contains":"Desk Lamp","do":"click","in":{"contains":"Split refund 756","testid":"support-ticket"},"testid":"support-order-option"},{"actor":"owner","do":"click","in":{"contains":"Split refund 756","testid":"support-ticket"},"testid":"support-link-order"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee","within":1000},{"actor":"staff2","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.split-tender-refunds","requiresFeatures":["ecommerce.feature.split-tender-refunds","ecommerce.feature.store-credit","ecommerce.progression.support-refunds"],"role":"guarantee","source":"scenarios/progression-split-tender-refunds.json"}],"id":"selected-source-098","scenario":{"level":6,"writeUrlPattern":null},"source":"scenarios/progression-split-tender-refunds.json"},{"checkGroups":[{"checkGroupId":"staff-access","feature":{"actors":["customer","staff","admin","authorized"],"criteria":[{"id":"601a","steps":[{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link"},{"actor":"staff","do":"expect","testid":"staff-area","within":6000},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"click","testid":"staff-link"},{"actor":"admin","do":"expect","testid":"staff-area","within":6000}]}],"id":601,"setup":[]},"packId":"ecommerce.progression.staff-access","role":"feature","source":"scenarios/progression-staff-access.json"},{"checkGroupId":"staff-area-boundary","feature":{"actors":["customer","staff","admin","authorized"],"criteria":[{"id":"601b","steps":[{"actor":"authorized","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"authorized","do":"click","testid":"staff-link"},{"actor":"authorized","do":"expect","testid":"staff-area"},{"actor":"customer","do":"signIn","exact":true,"name":"customer","password":"stackbench-customer-2026"},{"actor":"customer","do":"click","ifAvailable":true,"settleMs":1500,"testid":"staff-link"},{"absent":true,"actor":"customer","do":"expect","testid":"staff-area"}]}],"id":601,"setup":[]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.staff-access"],"role":"guarantee","source":"scenarios/progression-staff-access.json"}],"id":"selected-source-099","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/progression-staff-access.json"},{"checkGroups":[{"checkGroupId":"staff-activity","feature":{"actors":["admin","staff","customer"],"criteria":[{"id":"624a","steps":[{"actor":"staff","do":"click","testid":"staff-link"},{"actor":"staff","do":"click","testid":"activity-link"},{"actor":"staff","contains":"Activity Mug","do":"expect","testid":"activity-entry","within":10000},{"actor":"staff","contains":"admin","do":"expect","in":{"contains":"Activity Mug","testid":"activity-entry"},"testid":"activity-actor"},{"actor":"staff","contains":"creat","do":"expect","in":{"contains":"Activity Mug","testid":"activity-entry"},"testid":"activity-action"},{"actor":"staff","contains":"Activity Mug","do":"expect","in":{"contains":"Activity Mug","testid":"activity-entry"},"testid":"activity-subject"},{"actor":"staff","do":"expect","in":{"contains":"Activity Mug","testid":"activity-entry"},"testid":"activity-time"}]}],"id":624,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"customer","do":"signIn","exact":true,"name":"customer","password":"stackbench-customer-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"fill","testid":"catalog-name","text":"Activity Mug"},{"actor":"admin","do":"fill","testid":"catalog-category","text":"Kitchen"},{"actor":"admin","do":"fill","testid":"catalog-price","text":"43.00"},{"actor":"admin","do":"fill","testid":"catalog-variants","text":"Green"},{"actor":"admin","do":"click","testid":"catalog-save"}]},"packId":"ecommerce.progression.staff-activity","role":"feature","source":"scenarios/progression-staff-activity.json"},{"checkGroupId":"staff-activity-privacy","feature":{"actors":["admin","staff","customer"],"criteria":[{"id":"624b","steps":[{"absent":true,"actor":"customer","do":"expect","testid":"activity-link"}]}],"id":624,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"customer","do":"signIn","exact":true,"name":"customer","password":"stackbench-customer-2026"},{"actor":"admin","do":"click","testid":"admin-link"},{"actor":"admin","do":"fill","testid":"catalog-name","text":"Activity Mug"},{"actor":"admin","do":"fill","testid":"catalog-category","text":"Kitchen"},{"actor":"admin","do":"fill","testid":"catalog-price","text":"43.00"},{"actor":"admin","do":"fill","testid":"catalog-variants","text":"Green"},{"actor":"admin","do":"click","testid":"catalog-save"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.staff-activity"],"role":"guarantee","source":"scenarios/progression-staff-activity.json"}],"id":"selected-source-100","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-staff-activity.json"},{"checkGroups":[{"checkGroupId":"staff-roles","feature":{"actors":["admin","replayAdmin","staff","roleAdmin","promotedStaff"],"criteria":[{"id":"621c","steps":[{"actor":"admin","do":"expect","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-select","value":"inventory","within":10000}]}],"id":621,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"expect","testid":"admin-link","within":6000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"staff-role-account-staff"},{"actor":"admin","do":"fill","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-select","text":"inventory"},{"actor":"admin","do":"click","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-save"}]},"packId":"ecommerce.progression.staff-roles","role":"feature","source":"scenarios/progression-staff-roles.json"},{"checkGroupId":"staff-role-reload","feature":{"actors":["admin","replayAdmin","staff","roleAdmin","promotedStaff"],"criteria":[{"id":"621a","steps":[{"actor":"admin","do":"reload","settleMs":2500},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","ifAvailable":true,"testid":"admin-link","unlessVisible":"staff-role-account-staff"},{"actor":"admin","do":"expect","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-select","value":"inventory","within":10000},{"do":"restartBackend","settleMs":1000},{"actor":"admin","do":"freshClient"},{"actor":"admin-fresh","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin-fresh","do":"click","testid":"admin-link","unlessVisible":"staff-role-account-staff"},{"actor":"admin-fresh","do":"expect","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-select","value":"inventory","within":10000}]}],"id":621,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"expect","testid":"admin-link","within":6000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"staff-role-account-staff"},{"actor":"admin","do":"fill","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-select","text":"inventory"},{"actor":"admin","do":"click","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-save"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","requiresFeatures":["ecommerce.progression.staff-roles"],"role":"guarantee","source":"scenarios/progression-staff-roles.json"},{"checkGroupId":"staff-role-boundary","feature":{"actors":["admin","replayAdmin","staff","roleAdmin","promotedStaff"],"criteria":[{"id":"621b","steps":[{"actor":"replayAdmin","do":"reload","settleMs":0},{"actor":"replayAdmin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"replayAdmin","do":"expect","testid":"admin-link","within":6000},{"actor":"replayAdmin","do":"click","testid":"admin-link","unlessVisible":"staff-role-account-staff"},{"actor":"replayAdmin","do":"fill","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-select","text":"staff"},{"actor":"replayAdmin","do":"click","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-save"},{"actor":"replayAdmin","do":"reload","settleMs":1000},{"actor":"replayAdmin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"replayAdmin","do":"click","ifAvailable":true,"testid":"admin-link","unlessVisible":"staff-role-account-staff"},{"actor":"replayAdmin","do":"expect","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-select","value":"staff","within":10000},{"actor":"staff","do":"reload","settleMs":0},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"staff-link"},{"absent":true,"actor":"staff","do":"expect","testid":"staff-role-save"},{"actor":"staff","do":"replayAs","from":"replayAdmin","match":"role","namedAction":{"args":[0,"inventory"],"id":"assignStaffRole","method":"PUT","params":[{"in":"path","name":"accountId","placeholder":":id","wireType":"u64"},{"in":"body","name":"role"}],"path":"/api/staff/:id/role","reducer":"assign_staff_role"},"namedTarget":{"attribute":"data-account-id","testid":"staff-role-account-staff","valueType":"string"},"settleMs":1500,"swap":{"find":"\"role\":\"staff\"","with":"\"role\":\"inventory\""}},{"actor":"staff","do":"expectReplayRejected"},{"actor":"replayAdmin","do":"reload","settleMs":1000},{"actor":"replayAdmin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"replayAdmin","do":"click","ifAvailable":true,"testid":"admin-link","unlessVisible":"staff-role-account-staff"},{"actor":"replayAdmin","do":"expect","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-select","value":"staff","within":10000}]}],"id":621,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"expect","testid":"admin-link","within":6000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"staff-role-account-staff"},{"actor":"admin","do":"fill","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-select","text":"inventory"},{"actor":"admin","do":"click","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-save"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.staff-roles"],"role":"guarantee","source":"scenarios/progression-staff-roles.json"},{"checkGroupId":"staff-role-revocation","feature":{"actors":["admin","replayAdmin","staff","roleAdmin","promotedStaff"],"criteria":[{"id":"621d","steps":[{"actor":"roleAdmin","do":"reload","settleMs":0},{"actor":"roleAdmin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"roleAdmin","do":"click","testid":"admin-link","unlessVisible":"staff-role-account-staff"},{"actor":"roleAdmin","do":"replayAs","from":"roleAdmin","match":"__role-revocation-named-action__","namedAction":{"args":[0,"admin"],"id":"assignStaffRole","method":"PUT","params":[{"in":"path","name":"accountId","placeholder":":id","wireType":"u64"},{"in":"body","name":"role"}],"path":"/api/staff/:id/role","reducer":"assign_staff_role"},"namedTarget":{"attribute":"data-account-id","testid":"staff-role-account-staff","valueType":"string"},"settleMs":0},{"actor":"roleAdmin","do":"expectReplayCompleted","requireAccepted":true},{"actor":"roleAdmin","do":"reload","settleMs":0},{"actor":"roleAdmin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"roleAdmin","do":"click","ifAvailable":true,"testid":"admin-link","unlessVisible":"staff-role-account-staff"},{"actor":"roleAdmin","do":"expect","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-select","value":"admin","within":10000},{"actor":"promotedStaff","do":"reload","settleMs":0},{"actor":"promotedStaff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"promotedStaff","do":"replayAs","from":"roleAdmin","match":"__role-revocation-named-action__","namedAction":{"args":[0,"admin"],"id":"assignStaffRole","method":"PUT","params":[{"in":"path","name":"accountId","placeholder":":id","wireType":"u64"},{"in":"body","name":"role"}],"path":"/api/staff/:id/role","reducer":"assign_staff_role"},"namedTarget":{"attribute":"data-account-id","testid":"staff-role-account-staff","valueType":"string"},"settleMs":0},{"actor":"promotedStaff","do":"expectReplayCompleted","requireAccepted":true},{"actor":"roleAdmin","do":"replayAs","from":"roleAdmin","match":"__role-revocation-named-action__","namedAction":{"args":[0,"staff"],"id":"assignStaffRole","method":"PUT","params":[{"in":"path","name":"accountId","placeholder":":id","wireType":"u64"},{"in":"body","name":"role"}],"path":"/api/staff/:id/role","reducer":"assign_staff_role"},"namedTarget":{"attribute":"data-account-id","testid":"staff-role-account-staff","valueType":"string"},"settleMs":0},{"actor":"roleAdmin","do":"expectReplayCompleted","requireAccepted":true},{"actor":"roleAdmin","do":"reload","settleMs":0},{"actor":"roleAdmin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"roleAdmin","do":"click","ifAvailable":true,"testid":"admin-link","unlessVisible":"staff-role-account-staff"},{"actor":"roleAdmin","do":"expect","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-select","value":"staff","within":10000},{"actor":"promotedStaff","do":"replayAs","from":"roleAdmin","match":"__role-revocation-named-action__","namedAction":{"args":[0,"admin"],"id":"assignStaffRole","method":"PUT","params":[{"in":"path","name":"accountId","placeholder":":id","wireType":"u64"},{"in":"body","name":"role"}],"path":"/api/staff/:id/role","reducer":"assign_staff_role"},"namedTarget":{"attribute":"data-account-id","testid":"staff-role-account-staff","valueType":"string"},"settleMs":0},{"actor":"promotedStaff","do":"expectReplayRejected"},{"actor":"roleAdmin","do":"reload","settleMs":0},{"actor":"roleAdmin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"roleAdmin","do":"click","ifAvailable":true,"testid":"admin-link","unlessVisible":"staff-role-account-staff"},{"actor":"roleAdmin","do":"expect","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-select","value":"staff","within":10000}]}],"id":621,"setup":[{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"admin","do":"expect","testid":"admin-link","within":6000},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"staff-role-account-staff"},{"actor":"admin","do":"fill","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-select","text":"inventory"},{"actor":"admin","do":"click","in":{"testid":"staff-role-account-staff"},"testid":"staff-role-save"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.staff-roles"],"role":"guarantee","source":"scenarios/progression-staff-roles.json"}],"id":"selected-source-101","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/progression-staff-roles.json"},{"checkGroups":[{"checkGroupId":"stock-alert-delivery","feature":{"actors":["subscriber","admin"],"criteria":[{"id":"631c","steps":[{"actor":"subscriber","do":"freshClient"},{"actor":"subscriber-fresh","do":"signIn","name":"stock-subscriber"},{"actor":"subscriber-fresh","do":"click","testid":"notifications-toggle","unlessVisible":"notifications-panel"},{"actor":"subscriber-fresh","attribute":"aria-busy","do":"expect","testid":"notifications-panel","value":"false","within":10000},{"actor":"subscriber-fresh","contains":"Air Purifier","do":"expectElementCount","equals":0,"testid":"stock-alert-delivery"},{"actor":"admin","do":"click","testid":"admin-link"},{"action":"restock","actor":"admin","do":"callAction","input":{"attribute":"data-restock-input","contains":"Air Purifier","testid":"admin-location-row"},"namedAction":{"args":[0,0,1],"id":"restock","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"warehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/restock","reducer":"admin_restock"},"settleMs":0},{"actor":"admin","do":"expectActionOutcome","outcome":"accepted"},{"actor":"admin","do":"wait","ms":10000},{"actor":"subscriber","do":"freshClient"},{"actor":"subscriber-fresh","do":"signIn","name":"stock-subscriber"},{"actor":"subscriber-fresh","do":"click","testid":"notifications-toggle","unlessVisible":"notifications-panel"},{"actor":"subscriber-fresh","attribute":"aria-busy","do":"expect","testid":"notifications-panel","value":"false","within":10000},{"actor":"subscriber-fresh","contains":"Air Purifier","do":"expect","testid":"stock-alert-delivery","within":10000}]}],"id":631,"setup":[{"actor":"subscriber","do":"signUp","name":"stock-subscriber"},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"dbSetStock","item":"Air Purifier","quantity":0,"settleMs":1000,"warehouse":"East"},{"do":"dbSetStock","item":"Air Purifier","quantity":0,"settleMs":1000,"warehouse":"West"},{"actor":"subscriber","do":"reload","settleMs":1000},{"actor":"subscriber","do":"ensureSignedIn","name":"stock-subscriber","readyTestid":"current-user"},{"actor":"subscriber","do":"click","in":{"contains":"Air Purifier","testid":"item-card"},"testid":"stock-alert"},{"actor":"subscriber","attribute":"data-submit-state","contains":"Air Purifier","do":"expect","testid":"item-card","value":"succeeded","within":10000}]},"packId":"ecommerce.progression.stock-alerts","requiresFeatures":["ecommerce.feature.warehouse-admin"],"role":"feature","source":"scenarios/progression-stock-alert-delivery.json"}],"id":"selected-source-102","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-stock-alert-delivery.json"},{"checkGroups":[{"checkGroupId":"stock-alert-deduplication","feature":{"actors":["subscriber","other","admin"],"criteria":[{"id":"631a","steps":[{"actor":"subscriber","do":"freshClient"},{"actor":"subscriber-fresh","do":"signIn","name":"stock-subscriber"},{"actor":"subscriber-fresh","do":"click","testid":"notifications-toggle","unlessVisible":"notifications-panel"},{"actor":"subscriber-fresh","attribute":"aria-busy","do":"expect","testid":"notifications-panel","value":"false","within":10000},{"actor":"subscriber-fresh","contains":"Air Purifier","do":"expectElementCount","equals":1,"testid":"stock-alert-delivery","within":10000},{"action":"restock","actor":"admin","do":"callAction","input":{"attribute":"data-restock-input","contains":"Air Purifier","testid":"admin-location-row"},"namedAction":{"args":[0,0,1],"id":"restock","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"warehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/restock","reducer":"admin_restock"},"settleMs":0},{"actor":"admin","do":"expectActionOutcome","outcome":"accepted"},{"actor":"admin","do":"wait","ms":10000},{"actor":"subscriber","do":"freshClient"},{"actor":"subscriber-fresh","do":"signIn","name":"stock-subscriber"},{"actor":"subscriber-fresh","do":"click","testid":"notifications-toggle","unlessVisible":"notifications-panel"},{"actor":"subscriber-fresh","attribute":"aria-busy","do":"expect","testid":"notifications-panel","value":"false","within":10000},{"actor":"subscriber-fresh","contains":"Air Purifier","do":"expectElementCount","equals":1,"testid":"stock-alert-delivery","within":10000}]}],"id":631,"setup":[{"actor":"subscriber","do":"signUp","name":"stock-subscriber"},{"actor":"other","do":"signUp","name":"stock-other"},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"dbSetStock","item":"Air Purifier","quantity":0,"settleMs":1000,"warehouse":"East"},{"do":"dbSetStock","item":"Air Purifier","quantity":0,"settleMs":1000,"warehouse":"West"},{"actor":"subscriber","do":"reload","settleMs":1000},{"actor":"subscriber","do":"ensureSignedIn","name":"stock-subscriber","readyTestid":"current-user"},{"actor":"subscriber","do":"click","in":{"contains":"Air Purifier","testid":"item-card"},"testid":"stock-alert"},{"actor":"subscriber","attribute":"data-submit-state","contains":"Air Purifier","do":"expect","testid":"item-card","value":"succeeded","within":10000},{"actor":"admin","do":"click","testid":"admin-link"},{"action":"restock","actor":"admin","do":"callAction","input":{"attribute":"data-restock-input","contains":"Air Purifier","testid":"admin-location-row"},"namedAction":{"args":[0,0,1],"id":"restock","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"warehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/restock","reducer":"admin_restock"},"settleMs":0},{"actor":"admin","do":"expectActionOutcome","outcome":"accepted"},{"actor":"admin","do":"wait","ms":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","requiresFeatures":["ecommerce.feature.warehouse-admin","ecommerce.progression.stock-alerts"],"role":"guarantee","source":"scenarios/progression-stock-alerts.json"},{"checkGroupId":"stock-alert-privacy","feature":{"actors":["subscriber","other","admin"],"criteria":[{"id":"631b","steps":[{"actor":"subscriber","do":"freshClient"},{"actor":"subscriber-fresh","do":"signIn","name":"stock-subscriber"},{"actor":"subscriber-fresh","do":"click","testid":"notifications-toggle","unlessVisible":"notifications-panel"},{"actor":"subscriber-fresh","attribute":"aria-busy","do":"expect","testid":"notifications-panel","value":"false","within":10000},{"actor":"subscriber-fresh","contains":"Air Purifier","do":"expect","testid":"stock-alert-delivery","within":10000},{"actor":"other","do":"freshClient"},{"actor":"other-fresh","do":"signIn","name":"stock-other"},{"actor":"other-fresh","do":"click","testid":"notifications-toggle","unlessVisible":"notifications-panel"},{"actor":"other-fresh","attribute":"aria-busy","do":"expect","testid":"notifications-panel","value":"false","within":10000},{"absent":true,"actor":"other-fresh","contains":"Air Purifier","do":"expect","testid":"notification-item"}]}],"id":631,"setup":[{"actor":"subscriber","do":"signUp","name":"stock-subscriber"},{"actor":"other","do":"signUp","name":"stock-other"},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"do":"dbSetStock","item":"Air Purifier","quantity":0,"settleMs":1000,"warehouse":"East"},{"do":"dbSetStock","item":"Air Purifier","quantity":0,"settleMs":1000,"warehouse":"West"},{"actor":"subscriber","do":"reload","settleMs":1000},{"actor":"subscriber","do":"ensureSignedIn","name":"stock-subscriber","readyTestid":"current-user"},{"actor":"subscriber","do":"click","in":{"contains":"Air Purifier","testid":"item-card"},"testid":"stock-alert"},{"actor":"subscriber","attribute":"data-submit-state","contains":"Air Purifier","do":"expect","testid":"item-card","value":"succeeded","within":10000},{"actor":"admin","do":"click","testid":"admin-link"},{"action":"restock","actor":"admin","do":"callAction","input":{"attribute":"data-restock-input","contains":"Air Purifier","testid":"admin-location-row"},"namedAction":{"args":[0,0,1],"id":"restock","params":[{"in":"body","name":"itemId","wireType":"u64"},{"in":"body","name":"warehouseId","wireType":"u64"},{"in":"body","name":"quantity"}],"path":"/api/admin/restock","reducer":"admin_restock"},"settleMs":0},{"actor":"admin","do":"expectActionOutcome","outcome":"accepted"},{"actor":"admin","do":"wait","ms":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.feature.warehouse-admin","ecommerce.progression.stock-alerts"],"role":"guarantee","source":"scenarios/progression-stock-alerts.json"}],"id":"selected-source-103","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-stock-alerts.json"},{"checkGroups":[{"checkGroupId":"stock-limit","feature":{"actors":["buyer","watcher","visitor"],"criteria":[{"id":"3d","steps":[{"actor":"buyer","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"buy-now"},{"actor":"buyer","do":"wait","ms":800},{"actor":"buyer","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"buy-now"},{"actor":"buyer","do":"wait","ms":800},{"actor":"buyer","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"buy-now"},{"actor":"buyer","do":"expectNumber","equals":0,"in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock"},{"actor":"buyer","do":"expect","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"out-of-stock"},{"action":"buy","actor":"buyer","do":"callAction","input":{"attribute":"data-buy-input","contains":"Desk Lamp","testid":"item-card"},"namedAction":{"args":[0],"id":"buy","params":[{"in":"path","name":"itemId","placeholder":":id","wireType":"u64"}],"path":"/api/items/:id/buy","reducer":"buy_now"},"settleMs":2000},{"actor":"buyer","do":"expectActionOutcome","outcome":"validation-refused"},{"actor":"buyer","do":"reload","settleMs":3000},{"actor":"buyer","do":"ensureSignedIn","name":"eli","readyTestid":"current-user"},{"actor":"buyer","do":"expectNumber","equals":0,"in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"item-stock"}]}],"id":3,"setup":[{"do":"dbSetStock","item":"Desk Lamp","quantity":2,"settleMs":2000,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":1,"settleMs":2000,"warehouse":"West"},{"actor":"buyer","do":"signUp","name":"eli"},{"actor":"watcher","do":"signUp","name":"fay"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.concurrency-safety","requiresFeatures":["ecommerce.feature.purchasing"],"role":"guarantee","source":"scenarios/progression-stock-limit.json"}],"id":"selected-source-104","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/progression-stock-limit.json"},{"checkGroups":[{"checkGroupId":"store-credit-750","feature":{"actors":["owner","staff","tab"],"criteria":[{"id":"750a","steps":[{"actor":"owner","do":"click","testid":"credit-checkout"},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"credit-owner-750"},{"actor":"owner-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner-fresh","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner-fresh","contains":"Desk Lamp","do":"expectElementCount","equals":1,"testid":"order-item","within":10000},{"actor":"owner-fresh","do":"expectNumber","equals":10,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"payment-credit-amount"},{"actor":"owner-fresh","do":"expectNumber","equals":32,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"payment-external-amount"},{"actor":"owner-fresh","do":"expectNumber","equals":42,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"payment-amount"},{"actor":"owner-fresh","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"owner-fresh","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"owner-fresh","do":"expectNumber","equals":0,"testid":"credit-balance"}]}],"id":750,"setup":[{"actor":"owner","do":"signUp","name":"credit-owner-750"},{"actor":"owner","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"staff","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"staff","do":"fill","testid":"credit-amount-input","text":"10.00"},{"actor":"staff","do":"fill","testid":"credit-reference-input","text":"credit-750"},{"action":"grantCredit","actor":"staff","do":"callAction","from":"staff","input":{"attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","attribute":"data-account-id","testid":"credit-panel"}},"testid":"credit-grant"},"namedAction":{"args":[0,1000,""],"id":"grantCredit","method":"POST","params":[{"in":"body","name":"accountId","wireType":"u64"},{"in":"body","name":"amountMinor"},{"in":"body","name":"reference"}],"path":"/api/staff/credit","reducer":"grant_credit"},"settleMs":0},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"credit-owner-750","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"credit-link","unlessVisible":"credit-balance"},{"actor":"owner","do":"expectNumber","equals":10,"testid":"credit-balance","within":10000},{"actor":"owner","do":"click","testid":"catalog-link"},{"actor":"owner","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"add-to-cart"},{"actor":"owner","do":"click","testid":"cart-toggle"},{"actor":"owner","do":"expectNumber","equals":42,"testid":"cart-total"}]},"packId":"ecommerce.feature.store-credit","role":"feature","source":"scenarios/progression-store-credit.json"},{"checkGroupId":"production-752","feature":{"actors":["owner","staff","tab"],"criteria":[{"id":"752a","steps":[{"action":"grantCredit","actor":"staff","do":"callAction","from":"staff","input":{"attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","attribute":"data-account-id","testid":"credit-panel"}},"testid":"credit-grant"},"namedAction":{"args":[0,1000,""],"id":"grantCredit","method":"POST","params":[{"in":"body","name":"accountId","wireType":"u64"},{"in":"body","name":"amountMinor"},{"in":"body","name":"reference"}],"path":"/api/staff/credit","reducer":"grant_credit"},"settleMs":0},{"actor":"staff","do":"expectActionOutcome","outcome":"completed"},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"credit-owner-752"},{"actor":"owner-fresh","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"owner-fresh","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"owner-fresh","do":"expectNumber","equals":10,"testid":"credit-balance"}]}],"id":752,"setup":[{"actor":"owner","do":"signUp","name":"credit-owner-752"},{"actor":"owner","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"staff","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"staff","do":"fill","testid":"credit-amount-input","text":"10.00"},{"actor":"staff","do":"fill","testid":"credit-reference-input","text":"credit-752"},{"action":"grantCredit","actor":"staff","do":"callAction","from":"staff","input":{"attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","attribute":"data-account-id","testid":"credit-panel"}},"testid":"credit-grant"},"namedAction":{"args":[0,1000,""],"id":"grantCredit","method":"POST","params":[{"in":"body","name":"accountId","wireType":"u64"},{"in":"body","name":"amountMinor"},{"in":"body","name":"reference"}],"path":"/api/staff/credit","reducer":"grant_credit"},"settleMs":0},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"credit-owner-752","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"credit-link","unlessVisible":"credit-balance"},{"actor":"owner","do":"expectNumber","equals":10,"testid":"credit-balance","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.store-credit","requiresFeatures":["ecommerce.feature.store-credit"],"role":"guarantee","source":"scenarios/progression-store-credit.json"},{"checkGroupId":"production-753","feature":{"actors":["owner","staff","tab"],"criteria":[{"id":"753a","steps":[{"actor":"staff","do":"fill","testid":"credit-reference-input","text":"unauthorized-credit-753"},{"action":"grantCredit","actor":"owner","do":"callAction","from":"staff","input":{"attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","attribute":"data-account-id","testid":"credit-panel"}},"testid":"credit-grant"},"namedAction":{"args":[0,1000,""],"id":"grantCredit","method":"POST","params":[{"in":"body","name":"accountId","wireType":"u64"},{"in":"body","name":"amountMinor"},{"in":"body","name":"reference"}],"path":"/api/staff/credit","reducer":"grant_credit"},"settleMs":0},{"actor":"owner","do":"expectActionOutcome","outcome":"refused"},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"credit-owner-753"},{"actor":"owner-fresh","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"owner-fresh","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"owner-fresh","do":"expectNumber","equals":10,"testid":"credit-balance"}]}],"id":753,"setup":[{"actor":"owner","do":"signUp","name":"credit-owner-753"},{"actor":"owner","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"staff","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"staff","do":"fill","testid":"credit-amount-input","text":"10.00"},{"actor":"staff","do":"fill","testid":"credit-reference-input","text":"credit-753"},{"action":"grantCredit","actor":"staff","do":"callAction","from":"staff","input":{"attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","attribute":"data-account-id","testid":"credit-panel"}},"testid":"credit-grant"},"namedAction":{"args":[0,1000,""],"id":"grantCredit","method":"POST","params":[{"in":"body","name":"accountId","wireType":"u64"},{"in":"body","name":"amountMinor"},{"in":"body","name":"reference"}],"path":"/api/staff/credit","reducer":"grant_credit"},"settleMs":0},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"credit-owner-753","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"credit-link","unlessVisible":"credit-balance"},{"actor":"owner","do":"expectNumber","equals":10,"testid":"credit-balance","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.store-credit","requiresFeatures":["ecommerce.feature.store-credit"],"role":"guarantee","source":"scenarios/progression-store-credit.json"},{"checkGroupId":"production-754","feature":{"actors":["owner","staff","tab"],"criteria":[{"id":"754a","steps":[{"action":"checkoutCredit","actors":["owner","tab"],"do":"callConcurrently","namedAction":{"args":[],"id":"checkoutCredit","method":"POST","path":"/api/checkout/credit","reducer":"checkout_credit"},"settleMs":0},{"do":"expectCallOutcomes"},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"credit-owner-754"},{"actor":"owner-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner-fresh","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner-fresh","contains":"Desk Lamp","do":"expectElementCount","equals":1,"testid":"order-item","within":10000},{"actor":"owner-fresh","do":"expectNumber","equals":10,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"payment-credit-amount"},{"actor":"owner-fresh","do":"expectNumber","equals":32,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"payment-external-amount"},{"actor":"owner-fresh","do":"expectNumber","equals":42,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"payment-amount"},{"actor":"owner-fresh","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"owner-fresh","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"owner-fresh","do":"expectNumber","equals":0,"testid":"credit-balance"}]}],"id":754,"setup":[{"actor":"owner","do":"signUp","name":"credit-owner-754"},{"actor":"owner","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"staff","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"staff","do":"fill","testid":"credit-amount-input","text":"10.00"},{"actor":"staff","do":"fill","testid":"credit-reference-input","text":"credit-754"},{"action":"grantCredit","actor":"staff","do":"callAction","from":"staff","input":{"attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","attribute":"data-account-id","testid":"credit-panel"}},"testid":"credit-grant"},"namedAction":{"args":[0,1000,""],"id":"grantCredit","method":"POST","params":[{"in":"body","name":"accountId","wireType":"u64"},{"in":"body","name":"amountMinor"},{"in":"body","name":"reference"}],"path":"/api/staff/credit","reducer":"grant_credit"},"settleMs":0},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"credit-owner-754","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"credit-link","unlessVisible":"credit-balance"},{"actor":"owner","do":"expectNumber","equals":10,"testid":"credit-balance","within":10000},{"actor":"owner","do":"click","testid":"catalog-link"},{"actor":"owner","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"add-to-cart"},{"actor":"owner","do":"click","testid":"cart-toggle"},{"actor":"owner","do":"expectNumber","equals":42,"testid":"cart-total"},{"actor":"tab","do":"signIn","name":"credit-owner-754"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.store-credit","requiresFeatures":["ecommerce.feature.store-credit"],"role":"guarantee","source":"scenarios/progression-store-credit.json"},{"checkGroupId":"production-755","feature":{"actors":["owner","staff","tab"],"criteria":[{"id":"755a","steps":[{"do":"restartBackend","settleMs":1000},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"credit-owner-755"},{"actor":"owner-fresh","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"owner-fresh","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"owner-fresh","do":"expectNumber","equals":10,"testid":"credit-balance"}]}],"id":755,"setup":[{"actor":"owner","do":"signUp","name":"credit-owner-755"},{"actor":"owner","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"credit-link","unlessVisible":"credit-panel"},{"actor":"staff","attribute":"aria-busy","do":"expect","testid":"credit-panel","value":"false","within":10000},{"actor":"staff","do":"fill","testid":"credit-amount-input","text":"10.00"},{"actor":"staff","do":"fill","testid":"credit-reference-input","text":"credit-755"},{"action":"grantCredit","actor":"staff","do":"callAction","from":"staff","input":{"attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","attribute":"data-account-id","testid":"credit-panel"}},"testid":"credit-grant"},"namedAction":{"args":[0,1000,""],"id":"grantCredit","method":"POST","params":[{"in":"body","name":"accountId","wireType":"u64"},{"in":"body","name":"amountMinor"},{"in":"body","name":"reference"}],"path":"/api/staff/credit","reducer":"grant_credit"},"settleMs":0},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"credit-owner-755","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"credit-link","unlessVisible":"credit-balance"},{"actor":"owner","do":"expectNumber","equals":10,"testid":"credit-balance","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.store-credit","requiresFeatures":["ecommerce.feature.store-credit"],"role":"guarantee","source":"scenarios/progression-store-credit.json"}],"id":"selected-source-105","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-store-credit.json"},{"checkGroups":[{"checkGroupId":"subscriptions-760","feature":{"actors":["owner","other"],"criteria":[{"id":"760a","steps":[{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"subscription-760"},{"actor":"owner-fresh","do":"click","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"},{"actor":"owner-fresh","attribute":"aria-busy","do":"expect","testid":"subscriptions-panel","value":"false","within":10000},{"actor":"owner-fresh","do":"expect","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-status","value":"complete","within":90000},{"actor":"owner-fresh","do":"expectElementCount","equals":2,"in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-delivery"},{"actor":"owner-fresh","do":"expectSequence","equals":["paid","paid"],"in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-delivery-status"},{"actor":"owner-fresh","do":"expectNumber","equals":84,"in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-total"},{"do":"dbExpectStock","equals":8,"item":"Desk Lamp"},{"actor":"owner-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner-fresh","do":"click","testid":"orders-toggle"},{"actor":"owner-fresh","contains":"Desk Lamp","do":"expectElementCount","equals":2,"testid":"order-item","within":10000},{"actor":"owner-fresh","do":"expectElementCount","equals":2,"testid":"payment-record"}]}],"id":760,"setup":[{"actor":"owner","do":"signUp","name":"subscription-760"},{"do":"dbSetStock","item":"Desk Lamp","quantity":10,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":0,"warehouse":"West"},{"actor":"owner","do":"click","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"subscriptions-panel","value":"false","within":10000},{"actor":"owner","do":"fill","testid":"subscription-item-input","text":"Desk Lamp"},{"actor":"owner","do":"fill","testid":"subscription-quantity-input","text":"1"},{"actor":"owner","do":"fill","testid":"subscription-interval-input","text":"30"},{"actor":"owner","do":"fill","testid":"subscription-deliveries-input","text":"2"},{"actor":"owner","do":"click","testid":"subscription-create"},{"actor":"owner","do":"expect","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-status","value":"active","within":10000}]},"packId":"ecommerce.feature.subscriptions","role":"feature","source":"scenarios/progression-subscriptions.json"},{"checkGroupId":"production-761","feature":{"actors":["owner","other"],"criteria":[{"id":"761a","steps":[{"do":"restartBackend","settleMs":1000},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"subscription-761"},{"actor":"owner-fresh","do":"click","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"},{"actor":"owner-fresh","attribute":"aria-busy","do":"expect","testid":"subscriptions-panel","value":"false","within":10000},{"actor":"owner-fresh","do":"expect","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-status","value":"complete","within":90000},{"actor":"owner-fresh","do":"expectElementCount","equals":2,"in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-delivery"},{"actor":"owner-fresh","do":"expectSequence","equals":["paid","paid"],"in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-delivery-status"},{"actor":"owner-fresh","do":"expectNumber","equals":84,"in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-total"},{"do":"dbExpectStock","equals":8,"item":"Desk Lamp"},{"actor":"owner-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner-fresh","do":"click","testid":"orders-toggle"},{"actor":"owner-fresh","contains":"Desk Lamp","do":"expectElementCount","equals":2,"testid":"order-item","within":10000},{"actor":"owner-fresh","do":"expectElementCount","equals":2,"testid":"payment-record"},{"do":"restartBackend","settleMs":1000},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"subscription-761"},{"actor":"owner-fresh","do":"click","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"},{"actor":"owner-fresh","attribute":"aria-busy","do":"expect","testid":"subscriptions-panel","value":"false","within":10000},{"actor":"owner-fresh","do":"expect","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-status","value":"complete","within":90000},{"actor":"owner-fresh","do":"expectElementCount","equals":2,"in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-delivery"},{"actor":"owner-fresh","do":"expectSequence","equals":["paid","paid"],"in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-delivery-status"},{"actor":"owner-fresh","do":"expectNumber","equals":84,"in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-total"},{"do":"dbExpectStock","equals":8,"item":"Desk Lamp"},{"actor":"owner-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner-fresh","do":"click","testid":"orders-toggle"},{"actor":"owner-fresh","contains":"Desk Lamp","do":"expectElementCount","equals":2,"testid":"order-item","within":10000},{"actor":"owner-fresh","do":"expectElementCount","equals":2,"testid":"payment-record"}]}],"id":761,"setup":[{"actor":"owner","do":"signUp","name":"subscription-761"},{"do":"dbSetStock","item":"Desk Lamp","quantity":10,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":0,"warehouse":"West"},{"actor":"owner","do":"click","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"subscriptions-panel","value":"false","within":10000},{"actor":"owner","do":"fill","testid":"subscription-item-input","text":"Desk Lamp"},{"actor":"owner","do":"fill","testid":"subscription-quantity-input","text":"1"},{"actor":"owner","do":"fill","testid":"subscription-interval-input","text":"30"},{"actor":"owner","do":"fill","testid":"subscription-deliveries-input","text":"2"},{"actor":"owner","do":"click","testid":"subscription-create"},{"actor":"owner","do":"expect","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-status","value":"active","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.subscriptions","requiresFeatures":["ecommerce.feature.subscriptions"],"role":"guarantee","source":"scenarios/progression-subscriptions.json"},{"checkGroupId":"production-762","feature":{"actors":["owner","other"],"criteria":[{"id":"762a","steps":[{"action":"cancelSubscription","actor":"other","do":"callAction","from":"owner","input":{"attribute":"data-action-input","testid":"subscription-cancel"},"namedAction":{"args":[0],"id":"cancelSubscription","method":"POST","params":[{"in":"path","name":"subscriptionId","placeholder":"{subscriptionId}","wireType":"u64"}],"path":"/api/subscriptions/{subscriptionId}/cancel","reducer":"cancel_subscription"},"settleMs":0},{"actor":"other","do":"expectActionOutcome","outcome":"refused"},{"actor":"owner","do":"expect","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-status","value":"active","within":10000},{"actor":"owner","do":"click","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-cancel"},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"subscription-762"},{"actor":"owner-fresh","do":"click","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"},{"actor":"owner-fresh","attribute":"aria-busy","do":"expect","testid":"subscriptions-panel","value":"false","within":10000},{"actor":"owner-fresh","do":"expect","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-status","value":"cancelled","within":10000},{"actor":"owner-fresh","as":"deliveries-at-cancel","count":true,"do":"recordNumber","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-delivery"},{"as":"stock-at-cancel","do":"dbRecordStock","item":"Desk Lamp"},{"actor":"owner-fresh","do":"wait","ms":65000},{"actor":"owner-fresh","do":"expectElementCount","in":{"contains":"Desk Lamp","testid":"subscription-row"},"plus":0,"relativeTo":"deliveries-at-cancel","testid":"subscription-delivery"},{"do":"dbExpectStock","item":"Desk Lamp","plus":0,"relativeTo":"stock-at-cancel"}]}],"id":762,"setup":[{"actor":"other","do":"signUp","name":"subscription-other-762"},{"actor":"owner","do":"signUp","name":"subscription-762"},{"do":"dbSetStock","item":"Desk Lamp","quantity":10,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":0,"warehouse":"West"},{"actor":"owner","do":"click","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"subscriptions-panel","value":"false","within":10000},{"actor":"owner","do":"fill","testid":"subscription-item-input","text":"Desk Lamp"},{"actor":"owner","do":"fill","testid":"subscription-quantity-input","text":"1"},{"actor":"owner","do":"fill","testid":"subscription-interval-input","text":"30"},{"actor":"owner","do":"fill","testid":"subscription-deliveries-input","text":"2"},{"actor":"owner","do":"click","testid":"subscription-create"},{"actor":"owner","do":"expect","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-status","value":"active","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.subscriptions","requiresFeatures":["ecommerce.feature.subscriptions"],"role":"guarantee","source":"scenarios/progression-subscriptions.json"},{"checkGroupId":"production-763","feature":{"actors":["owner","other"],"criteria":[{"id":"763a","steps":[{"actor":"owner","do":"click","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-pause"},{"actor":"owner","do":"expect","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-status","value":"paused","within":10000},{"do":"restartBackend","settleMs":1000},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"subscription-763"},{"actor":"owner-fresh","do":"click","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"},{"actor":"owner-fresh","attribute":"aria-busy","do":"expect","testid":"subscriptions-panel","value":"false","within":10000},{"actor":"owner-fresh","do":"expect","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-status","value":"paused","within":10000},{"actor":"owner-fresh","as":"deliveries-at-pause","count":true,"do":"recordNumber","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-delivery"},{"as":"stock-at-pause","do":"dbRecordStock","item":"Desk Lamp"},{"actor":"owner-fresh","do":"wait","ms":35000},{"actor":"owner-fresh","do":"expectElementCount","in":{"contains":"Desk Lamp","testid":"subscription-row"},"plus":0,"relativeTo":"deliveries-at-pause","testid":"subscription-delivery"},{"do":"dbExpectStock","item":"Desk Lamp","plus":0,"relativeTo":"stock-at-pause"},{"actor":"owner-fresh","do":"click","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-resume"},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"subscription-763"},{"actor":"owner-fresh","do":"click","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"},{"actor":"owner-fresh","attribute":"aria-busy","do":"expect","testid":"subscriptions-panel","value":"false","within":10000},{"actor":"owner-fresh","do":"expect","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-status","value":"complete","within":90000},{"actor":"owner-fresh","do":"expectElementCount","equals":2,"in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-delivery"},{"actor":"owner-fresh","do":"expectSequence","equals":["paid","paid"],"in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-delivery-status"},{"actor":"owner-fresh","do":"expectNumber","equals":84,"in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-total"},{"do":"dbExpectStock","equals":8,"item":"Desk Lamp"},{"actor":"owner-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner-fresh","do":"click","testid":"orders-toggle"},{"actor":"owner-fresh","contains":"Desk Lamp","do":"expectElementCount","equals":2,"testid":"order-item","within":10000},{"actor":"owner-fresh","do":"expectElementCount","equals":2,"testid":"payment-record"}]}],"id":763,"setup":[{"actor":"owner","do":"signUp","name":"subscription-763"},{"do":"dbSetStock","item":"Desk Lamp","quantity":10,"settleMs":0,"warehouse":"East"},{"do":"dbSetStock","item":"Desk Lamp","quantity":0,"settleMs":0,"warehouse":"West"},{"actor":"owner","do":"click","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"},{"actor":"owner","attribute":"aria-busy","do":"expect","testid":"subscriptions-panel","value":"false","within":10000},{"actor":"owner","do":"fill","testid":"subscription-item-input","text":"Desk Lamp"},{"actor":"owner","do":"fill","testid":"subscription-quantity-input","text":"1"},{"actor":"owner","do":"fill","testid":"subscription-interval-input","text":"30"},{"actor":"owner","do":"fill","testid":"subscription-deliveries-input","text":"2"},{"actor":"owner","do":"click","testid":"subscription-create"},{"actor":"owner","do":"expect","in":{"contains":"Desk Lamp","testid":"subscription-row"},"testid":"subscription-status","value":"active","within":10000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.subscriptions","requiresFeatures":["ecommerce.feature.subscriptions"],"role":"guarantee","source":"scenarios/progression-subscriptions.json"}],"id":"selected-source-106","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-subscriptions.json"},{"checkGroups":[{"checkGroupId":"support-history","feature":{"actors":["owner","other"],"criteria":[{"id":"612c","steps":[{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"support-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"support-link","unlessVisible":"support-ticket"},{"actor":"owner","contains":"Owner ticket {user:ticketmarker}","do":"expect","testid":"support-ticket"}]}],"id":612,"setup":[{"actor":"owner","do":"signUp","name":"support-owner"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-email","text":"owner@example.com"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Owner ticket {user:ticketmarker}"},{"actor":"owner","do":"fill","testid":"support-message","text":"Private account issue."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","do":"expect","nonEmpty":true,"testid":"support-reference"},{"actor":"other","do":"signUp","name":"support-other"}]},"packId":"ecommerce.progression.support-history","role":"feature","source":"scenarios/progression-support-history.json"},{"checkGroupId":"support-history-reload","feature":{"actors":["owner","other"],"criteria":[{"id":"612a","steps":[{"actor":"owner","do":"reload","settleMs":3000},{"actor":"owner","do":"ensureSignedIn","name":"support-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","contains":"Owner ticket {user:ticketmarker}","do":"expect","testid":"support-ticket"},{"do":"restartBackend","settleMs":1000},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"support-owner"},{"actor":"owner-fresh","do":"click","testid":"support-link"},{"actor":"owner-fresh","contains":"Owner ticket {user:ticketmarker}","do":"expect","testid":"support-ticket"}]}],"id":612,"setup":[{"actor":"owner","do":"signUp","name":"support-owner"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-email","text":"owner@example.com"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Owner ticket {user:ticketmarker}"},{"actor":"owner","do":"fill","testid":"support-message","text":"Private account issue."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","do":"expect","nonEmpty":true,"testid":"support-reference"},{"actor":"other","do":"signUp","name":"support-other"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.state-durability","requiresFeatures":["ecommerce.progression.support-history"],"role":"guarantee","source":"scenarios/progression-support-history.json"},{"checkGroupId":"support-history-privacy","feature":{"actors":["owner","other"],"criteria":[{"id":"612b","steps":[{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"support-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-email","text":"owner@example.com"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Private ticket {user:privateticketmarker}"},{"actor":"owner","do":"fill","testid":"support-message","text":"Private account issue."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","do":"expect","nonEmpty":true,"testid":"support-reference"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"support-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","contains":"Private ticket {user:privateticketmarker}","do":"expect","testid":"support-ticket"},{"actor":"owner","contains":"Private ticket {user:privateticketmarker}","do":"expectReceived","within":10000},{"actor":"other","do":"reload","settleMs":0},{"actor":"other","do":"ensureSignedIn","name":"support-other","readyTestid":"current-user"},{"actor":"other","do":"click","testid":"support-link"},{"absent":true,"actor":"other","contains":"Private ticket {user:privateticketmarker}","do":"expect","testid":"support-ticket"},{"actor":"other","contains":"Private ticket {user:privateticketmarker}","do":"expectNotReceived"}]}],"id":612,"setup":[{"actor":"owner","do":"signUp","name":"support-owner"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-email","text":"owner@example.com"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Owner ticket {user:ticketmarker}"},{"actor":"owner","do":"fill","testid":"support-message","text":"Private account issue."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","do":"expect","nonEmpty":true,"testid":"support-reference"},{"actor":"other","do":"signUp","name":"support-other"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.support-history"],"role":"guarantee","source":"scenarios/progression-support-history.json"},{"checkGroupId":"support-history-logout","feature":{"actors":["owner","other"],"criteria":[{"id":"612d","steps":[{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"support-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-email","text":"owner@example.com"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Logout ticket {user:logoutticketmarker}"},{"actor":"owner","do":"fill","testid":"support-message","text":"Private account issue."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","do":"expect","nonEmpty":true,"testid":"support-reference"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"support-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"support-link","unlessVisible":"support-ticket"},{"actor":"owner","contains":"Logout ticket {user:logoutticketmarker}","do":"expect","testid":"support-ticket"},{"actor":"owner","contains":"Logout ticket {user:logoutticketmarker}","do":"expectReceived","within":10000},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"support-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","testid":"current-user","unlessVisible":"signout"},{"actor":"owner","do":"click","testid":"signout"},{"actor":"owner","do":"waitUntilAbsent","testid":"current-user","within":6000},{"actor":"owner","do":"freshClient","preserveStorage":true},{"absent":true,"actor":"owner-fresh","do":"expect","testid":"current-user"},{"actor":"owner-fresh","do":"click","testid":"support-link"},{"actor":"owner-fresh","do":"expect","testid":"support-email"},{"absent":true,"actor":"owner-fresh","contains":"Logout ticket {user:logoutticketmarker}","do":"expect","testid":"support-ticket"},{"actor":"owner-fresh","contains":"Logout ticket {user:logoutticketmarker}","do":"expectNotReceived"}]}],"id":612,"setup":[{"actor":"owner","do":"signUp","name":"support-owner"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-email","text":"owner@example.com"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Owner ticket {user:ticketmarker}"},{"actor":"owner","do":"fill","testid":"support-message","text":"Private account issue."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","do":"expect","nonEmpty":true,"testid":"support-reference"},{"actor":"other","do":"signUp","name":"support-other"}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.support-history"],"role":"guarantee","source":"scenarios/progression-support-history.json"}],"id":"selected-source-107","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/progression-support-history.json"},{"checkGroups":[{"checkGroupId":"support-intake","feature":{"actors":["visitor"],"criteria":[{"id":"610a","steps":[{"actor":"visitor","do":"click","testid":"support-link"},{"actor":"visitor","do":"fill","testid":"support-email","text":"visitor@example.com"},{"actor":"visitor","do":"fill","testid":"support-subject","text":"Damaged package"},{"actor":"visitor","do":"fill","testid":"support-message","text":"The package arrived damaged."},{"actor":"visitor","do":"click","testid":"support-submit"},{"actor":"visitor","do":"expect","nonEmpty":true,"testid":"support-reference"}]}],"id":610,"setup":[]},"packId":"ecommerce.progression.support-intake","role":"feature","source":"scenarios/progression-support-intake.json"}],"id":"selected-source-108","scenario":{"level":1,"writeUrlPattern":null},"source":"scenarios/progression-support-intake.json"},{"checkGroups":[{"checkGroupId":"support-refund-access","feature":{"actors":["owner","staff"],"criteria":[{"id":"615c","steps":[{"absent":true,"actor":"owner","do":"expect","testid":"support-refund"},{"action":"supportRefund","actor":"owner","authentication":"actor","do":"callAction","from":"staff","input":{"attribute":"data-action-input","testid":"support-refund"},"namedAction":{"args":[0],"id":"supportRefund","method":"POST","params":[{"in":"path","name":"caseId","placeholder":"{caseId}","wireType":"u64"}],"path":"/api/support/cases/{caseId}/refund","reducer":"support_refund"},"settleMs":1500},{"actor":"owner","do":"expectActionOutcome","outcome":"refused"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"refund-access-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"support-link","unlessVisible":"support-subject"},{"actor":"owner","do":"expect","in":{"contains":"Access refund case","testid":"support-ticket"},"notContains":"resolved","testid":"support-status"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle"},{"actor":"owner","do":"expect","in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"pending"},{"absent":true,"actor":"owner","contains":"Keyboard","do":"expect","testid":"refund-entry"}]}],"id":615,"setup":[{"actor":"owner","do":"signUp","name":"refund-access-owner"},{"actor":"owner","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"buy-now"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Access refund case"},{"actor":"owner","do":"fill","testid":"support-message","text":"This refund requires staff approval."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","contains":"Keyboard","do":"click","in":{"contains":"Access refund case","testid":"support-ticket"},"testid":"support-order-option"},{"actor":"owner","do":"click","in":{"contains":"Access refund case","testid":"support-ticket"},"testid":"support-link-order"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee","within":1000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.access-control","requiresFeatures":["ecommerce.progression.support-refunds"],"role":"guarantee","source":"scenarios/progression-support-refunds-access.json"}],"id":"selected-source-109","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-support-refunds-access.json"},{"checkGroups":[{"checkGroupId":"support-refund-accounting","feature":{"actors":["owner","staff"],"criteria":[{"id":"615b","steps":[{"actor":"staff","do":"click","in":{"contains":"Accounting refund case","testid":"support-ticket"},"testid":"support-refund"},{"actor":"owner","do":"reload","settleMs":1000},{"actor":"owner","do":"ensureSignedIn","name":"refund-accounting-owner","readyTestid":"current-user"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","within":1000},{"actor":"owner","do":"click","testid":"support-link","unlessVisible":"support-subject"},{"actor":"owner","do":"expectNumber","in":{"contains":"Accounting refund case","testid":"support-ticket"},"plus":0,"relativeTo":"paid-total","testid":"support-refund-total","within":10000},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner","do":"expectNumber","in":{"contains":"Keyboard","testid":"order-item"},"plus":0,"relativeTo":"paid-total","testid":"order-refund-total","within":10000},{"actor":"owner","contains":"Keyboard","do":"expectElementCount","equals":1,"testid":"refund-entry","within":10000},{"actor":"staff","do":"replayAs","from":"staff","match":"refund","namedAction":{"args":[0],"id":"supportRefund","method":"POST","params":[{"in":"path","name":"caseId","placeholder":"{caseId}","wireType":"u64"}],"path":"/api/support/cases/{caseId}/refund","reducer":"support_refund"},"namedTarget":{"attribute":"data-entity-id","contains":"Accounting refund case","testid":"support-ticket","valueType":"string"},"settleMs":1500},{"actor":"staff","do":"expectReplayCompleted"},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"refund-accounting-owner"},{"actor":"owner-fresh","do":"click","testid":"support-link"},{"actor":"owner-fresh","do":"expectNumber","in":{"contains":"Accounting refund case","testid":"support-ticket"},"plus":0,"relativeTo":"paid-total","testid":"support-refund-total","within":10000},{"actor":"owner-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner-fresh","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner-fresh","do":"expectNumber","in":{"contains":"Keyboard","testid":"order-item"},"plus":0,"relativeTo":"paid-total","testid":"order-refund-total","within":10000},{"actor":"owner-fresh","contains":"Keyboard","do":"expectElementCount","equals":1,"testid":"refund-entry","within":10000},{"actor":"owner-fresh","do":"expectNumber","equals":0,"in":{"contains":"Mouse","testid":"order-item"},"testid":"order-refund-total","within":10000},{"actor":"owner-fresh","contains":"Mouse","do":"expectElementCount","equals":0,"testid":"refund-entry","within":10000}]}],"id":615,"setup":[{"actor":"owner","do":"signUp","name":"refund-accounting-owner"},{"actor":"owner","do":"click","in":{"contains":"Mouse","testid":"item-card"},"testid":"buy-now"},{"actor":"owner","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"buy-now"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner","as":"paid-total","do":"recordNumber","in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-total"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Accounting refund case"},{"actor":"owner","do":"fill","testid":"support-message","text":"Refund the exact amount paid."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","contains":"Keyboard","do":"click","in":{"contains":"Accounting refund case","testid":"support-ticket"},"testid":"support-order-option"},{"actor":"owner","do":"click","in":{"contains":"Accounting refund case","testid":"support-ticket"},"testid":"support-link-order"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee","within":1000}]},"observations":["requested","unmentioned"],"packId":"ecommerce.spec.transactional-integrity","requiresFeatures":["ecommerce.progression.support-refunds"],"role":"guarantee","source":"scenarios/progression-support-refunds-accounting.json"}],"id":"selected-source-110","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-support-refunds-accounting.json"},{"checkGroups":[{"checkGroupId":"support-refunds-resolution","feature":{"actors":["owner","staff"],"criteria":[{"id":"615a","steps":[{"actor":"staff","do":"click","in":{"contains":"Resolution refund case","testid":"support-ticket"},"testid":"support-refund"},{"actor":"owner","contains":"resolved","do":"expect","in":{"contains":"Resolution refund case","testid":"support-ticket"},"testid":"support-status","within":10000},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle"},{"actor":"owner","contains":"refunded","do":"expect","in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","within":10000}]}],"id":615,"setup":[{"actor":"owner","do":"signUp","name":"refund-resolution-owner"},{"actor":"owner","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"buy-now"},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Resolution refund case"},{"actor":"owner","do":"fill","testid":"support-message","text":"Please refund my Keyboard order."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","contains":"Keyboard","do":"click","in":{"contains":"Resolution refund case","testid":"support-ticket"},"testid":"support-order-option"},{"actor":"owner","do":"click","in":{"contains":"Resolution refund case","testid":"support-ticket"},"testid":"support-link-order"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee"}]},"packId":"ecommerce.progression.support-refunds","role":"feature","source":"scenarios/progression-support-refunds-resolution.json"}],"id":"selected-source-111","scenario":{"level":5,"writeUrlPattern":null},"source":"scenarios/progression-support-refunds-resolution.json"},{"checkGroups":[{"checkGroupId":"return-refund-interaction","feature":{"actors":["owner","staff","admin"],"criteria":[{"id":"757a","steps":[{"actor":"owner","do":"reload","settleMs":0},{"actor":"owner","do":"ensureSignedIn","name":"return-refund-owner","readyTestid":"current-user","settleMs":0},{"actor":"owner","do":"click","ifAvailable":true,"testid":"catalog-link"},{"as":"757atotal","do":"dbRecordStock","item":"Keyboard"},{"as":"757aEast","do":"dbRecordStock","item":"Keyboard","warehouse":"East"},{"as":"757aWest","do":"dbRecordStock","item":"Keyboard","warehouse":"West"},{"actor":"admin","as":"757arevenue","do":"recordNumber","testid":"admin-revenue"},{"actor":"owner","do":"click","in":{"contains":"Keyboard","testid":"item-card"},"testid":"buy-now"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner","do":"expect","ignoreCase":true,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"pending","within":10000},{"actor":"owner","as":"757apaid","do":"recordNumber","in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-total"},{"action":"ship","actor":"staff","do":"callAction","from":"owner","input":{"attribute":"data-ship-input","contains":"Keyboard","testid":"order-item"},"namedAction":{"args":[0],"id":"ship","params":[{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/fulfilment/ship","reducer":"ship_order"},"settleMs":0},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"expect","ignoreCase":true,"in":{"contains":"Keyboard","testid":"order-item"},"testid":"order-status","value":"shipped","within":10000},{"do":"dbExpectStock","item":"Keyboard","plus":-1,"relativeTo":"757atotal"},{"actor":"owner","do":"reload","settleMs":0},{"actor":"owner","do":"ensureSignedIn","name":"return-refund-owner","readyTestid":"current-user","settleMs":0},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Return refund 757a"},{"actor":"owner","do":"fill","testid":"support-message","text":"Please refund this order."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","contains":"Return refund 757a","do":"expect","testid":"support-ticket","within":10000},{"actor":"owner","contains":"Keyboard","do":"click","in":{"contains":"Return refund 757a","testid":"support-ticket"},"testid":"support-order-option"},{"actor":"owner","do":"click","in":{"contains":"Return refund 757a","testid":"support-ticket"},"testid":"support-link-order"},{"actor":"owner","do":"expect","in":{"contains":"Return refund 757a","testid":"support-ticket"},"testid":"support-order","within":10000},{"action":"supportRefund","actor":"staff","do":"callAction","from":"owner","input":{"attribute":"data-refund-input","contains":"Return refund 757a","testid":"support-ticket"},"namedAction":{"args":[0],"id":"supportRefund","method":"POST","params":[{"in":"path","name":"caseId","placeholder":"{caseId}","wireType":"u64"}],"path":"/api/support/cases/{caseId}/refund","reducer":"support_refund"},"settleMs":0},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"do":"dbExpectStock","item":"Keyboard","plus":-1,"relativeTo":"757atotal"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner","do":"click","in":{"contains":"Keyboard","testid":"order-item"},"testid":"return-item"},{"actor":"owner","contains":"Keyboard","containsText":"returned","do":"expect","ignoreCase":true,"testid":"order-item","within":10000},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"return-refund-owner"},{"actor":"owner-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner-fresh","do":"click","testid":"orders-toggle"},{"actor":"owner-fresh","contains":"Keyboard","containsText":"returned","do":"expect","ignoreCase":true,"testid":"order-item","within":10000},{"actor":"owner-fresh","do":"expectNumber","in":{"contains":"Keyboard","testid":"order-item"},"plus":0,"relativeTo":"757apaid","testid":"order-refund-total","within":10000},{"do":"dbExpectStock","item":"Keyboard","plus":0,"relativeTo":"757aEast","warehouse":"East"},{"do":"dbExpectStock","item":"Keyboard","plus":0,"relativeTo":"757aWest","warehouse":"West"},{"actor":"admin","do":"reload","settleMs":0},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"expectNumber","plus":0,"relativeTo":"757arevenue","testid":"admin-revenue","within":10000}]},{"id":"757b","steps":[{"actor":"owner","do":"reload","settleMs":0},{"actor":"owner","do":"ensureSignedIn","name":"return-refund-owner","readyTestid":"current-user","settleMs":0},{"actor":"owner","do":"click","ifAvailable":true,"testid":"catalog-link"},{"as":"757btotal","do":"dbRecordStock","item":"Desk Lamp"},{"as":"757bEast","do":"dbRecordStock","item":"Desk Lamp","warehouse":"East"},{"as":"757bWest","do":"dbRecordStock","item":"Desk Lamp","warehouse":"West"},{"actor":"admin","as":"757brevenue","do":"recordNumber","testid":"admin-revenue"},{"actor":"owner","do":"click","in":{"contains":"Desk Lamp","testid":"item-card"},"testid":"buy-now"},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner","do":"expect","ignoreCase":true,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-status","value":"pending","within":10000},{"actor":"owner","as":"757bpaid","do":"recordNumber","in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-total"},{"action":"ship","actor":"staff","do":"callAction","from":"owner","input":{"attribute":"data-ship-input","contains":"Desk Lamp","testid":"order-item"},"namedAction":{"args":[0],"id":"ship","params":[{"in":"body","name":"orderId","wireType":"u64"}],"path":"/api/fulfilment/ship","reducer":"ship_order"},"settleMs":0},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"expect","ignoreCase":true,"in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"order-status","value":"shipped","within":10000},{"do":"dbExpectStock","item":"Desk Lamp","plus":-1,"relativeTo":"757btotal"},{"actor":"owner","do":"reload","settleMs":0},{"actor":"owner","do":"ensureSignedIn","name":"return-refund-owner","readyTestid":"current-user","settleMs":0},{"actor":"owner","do":"click","testid":"support-link"},{"actor":"owner","do":"fill","testid":"support-subject","text":"Return refund 757b"},{"actor":"owner","do":"fill","testid":"support-message","text":"Please refund this order."},{"actor":"owner","do":"click","testid":"support-submit"},{"actor":"owner","contains":"Return refund 757b","do":"expect","testid":"support-ticket","within":10000},{"actor":"owner","contains":"Desk Lamp","do":"click","in":{"contains":"Return refund 757b","testid":"support-ticket"},"testid":"support-order-option"},{"actor":"owner","do":"click","in":{"contains":"Return refund 757b","testid":"support-ticket"},"testid":"support-link-order"},{"actor":"owner","do":"expect","in":{"contains":"Return refund 757b","testid":"support-ticket"},"testid":"support-order","within":10000},{"actor":"owner","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner","do":"click","testid":"orders-toggle","unlessVisible":"order-item"},{"actor":"owner","do":"click","in":{"contains":"Desk Lamp","testid":"order-item"},"testid":"return-item"},{"actor":"owner","contains":"Desk Lamp","containsText":"returned","do":"expect","ignoreCase":true,"testid":"order-item","within":10000},{"action":"supportRefund","actor":"staff","do":"callAction","from":"owner","input":{"attribute":"data-refund-input","contains":"Return refund 757b","testid":"support-ticket"},"namedAction":{"args":[0],"id":"supportRefund","method":"POST","params":[{"in":"path","name":"caseId","placeholder":"{caseId}","wireType":"u64"}],"path":"/api/support/cases/{caseId}/refund","reducer":"support_refund"},"settleMs":0},{"actor":"staff","do":"expectActionOutcome","outcome":"accepted"},{"actor":"owner","do":"freshClient"},{"actor":"owner-fresh","do":"signIn","name":"return-refund-owner"},{"actor":"owner-fresh","do":"click","ifAvailable":true,"testid":"overlay-close","unlessVisible":"order-item","within":1000},{"actor":"owner-fresh","do":"click","testid":"orders-toggle"},{"actor":"owner-fresh","contains":"Desk Lamp","containsText":"returned","do":"expect","ignoreCase":true,"testid":"order-item","within":10000},{"actor":"owner-fresh","do":"expectNumber","in":{"contains":"Desk Lamp","testid":"order-item"},"plus":0,"relativeTo":"757bpaid","testid":"order-refund-total","within":10000},{"do":"dbExpectStock","item":"Desk Lamp","plus":0,"relativeTo":"757bEast","warehouse":"East"},{"do":"dbExpectStock","item":"Desk Lamp","plus":0,"relativeTo":"757bWest","warehouse":"West"},{"actor":"admin","do":"reload","settleMs":0},{"actor":"admin","do":"ensureSignedIn","exact":true,"name":"admin","password":"stackbench-admin-2026","readyTestid":"current-user"},{"actor":"admin","do":"click","testid":"admin-link","unlessVisible":"admin-revenue"},{"actor":"admin","do":"expectNumber","plus":0,"relativeTo":"757brevenue","testid":"admin-revenue","within":10000}]}],"id":757,"setup":[{"actor":"owner","do":"signUp","name":"return-refund-owner"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"admin","do":"signIn","exact":true,"name":"admin","password":"stackbench-admin-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee"},{"actor":"admin","do":"click","testid":"admin-link"}]},"packId":"ecommerce.feature.split-tender-refunds","requiresFeatures":["ecommerce.l3.order-returns-features","ecommerce.progression.fulfilment-queue","ecommerce.progression.support-refunds"],"role":"feature","source":"scenarios/progression-support-return-interaction.json"}],"id":"selected-source-112","scenario":{"level":6,"writeUrlPattern":null},"source":"scenarios/progression-support-return-interaction.json"},{"checkGroups":[{"checkGroupId":"support-assignment","feature":{"actors":["visitor","staff"],"criteria":[{"id":"611a","steps":[{"actor":"staff","do":"fill","in":{"contains":"Missing item","testid":"support-ticket"},"testid":"support-assignee","text":"staff"},{"actor":"staff","do":"click","in":{"contains":"Missing item","testid":"support-ticket"},"settleMs":1500,"testid":"support-update"},{"actor":"staff","do":"reload","settleMs":2500},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"expect","in":{"contains":"Missing item","testid":"support-ticket"},"testid":"support-assignee","value":"staff"}]}],"id":611,"setup":[{"actor":"visitor","do":"click","testid":"support-link"},{"actor":"visitor","do":"fill","testid":"support-email","text":"triage@example.com"},{"actor":"visitor","do":"fill","testid":"support-subject","text":"Missing item"},{"actor":"visitor","do":"fill","testid":"support-message","text":"One item is missing."},{"actor":"visitor","do":"click","testid":"support-submit"},{"actor":"visitor","do":"expect","nonEmpty":true,"testid":"support-reference"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee"}]},"packId":"ecommerce.progression.support-triage","role":"feature","source":"scenarios/progression-support-triage.json"},{"checkGroupId":"support-priority","feature":{"actors":["visitor","staff"],"criteria":[{"id":"611b","steps":[{"actor":"staff","do":"fill","in":{"contains":"Missing item","testid":"support-ticket"},"testid":"support-priority","text":"high"},{"actor":"staff","do":"click","in":{"contains":"Missing item","testid":"support-ticket"},"settleMs":1500,"testid":"support-update"},{"actor":"staff","do":"reload","settleMs":2500},{"actor":"staff","do":"ensureSignedIn","exact":true,"name":"staff","password":"stackbench-staff-2026","readyTestid":"current-user"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"expect","in":{"contains":"Missing item","testid":"support-ticket"},"testid":"support-priority","value":"high"}]}],"id":611,"setup":[{"actor":"visitor","do":"click","testid":"support-link"},{"actor":"visitor","do":"fill","testid":"support-email","text":"triage@example.com"},{"actor":"visitor","do":"fill","testid":"support-subject","text":"Missing item"},{"actor":"visitor","do":"fill","testid":"support-message","text":"One item is missing."},{"actor":"visitor","do":"click","testid":"support-submit"},{"actor":"visitor","do":"expect","nonEmpty":true,"testid":"support-reference"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee"}]},"packId":"ecommerce.progression.support-triage","role":"feature","source":"scenarios/progression-support-triage.json"},{"checkGroupId":"support-status","feature":{"actors":["visitor","staff"],"criteria":[{"id":"611c","steps":[{"actor":"staff","do":"fill","in":{"contains":"Missing item","testid":"support-ticket"},"testid":"support-status-input","text":"in progress"},{"actor":"staff","do":"click","in":{"contains":"Missing item","testid":"support-ticket"},"settleMs":1500,"testid":"support-update"},{"actor":"staff","contains":"in progress","do":"expect","in":{"contains":"Missing item","testid":"support-ticket"},"testid":"support-status"}]}],"id":611,"setup":[{"actor":"visitor","do":"click","testid":"support-link"},{"actor":"visitor","do":"fill","testid":"support-email","text":"triage@example.com"},{"actor":"visitor","do":"fill","testid":"support-subject","text":"Missing item"},{"actor":"visitor","do":"fill","testid":"support-message","text":"One item is missing."},{"actor":"visitor","do":"click","testid":"support-submit"},{"actor":"visitor","do":"expect","nonEmpty":true,"testid":"support-reference"},{"actor":"staff","do":"signIn","exact":true,"name":"staff","password":"stackbench-staff-2026"},{"actor":"staff","do":"click","testid":"staff-link","unlessVisible":"support-assignee"},{"actor":"staff","do":"click","ifAvailable":true,"testid":"support-queue-link","unlessVisible":"support-assignee"}]},"packId":"ecommerce.progression.support-triage","role":"feature","source":"scenarios/progression-support-triage.json"}],"id":"selected-source-113","scenario":{"level":2,"writeUrlPattern":null},"source":"scenarios/progression-support-triage.json"}],"fixture":{"accounts":[{"password":"stackbench-admin-2026","roles":["admin"],"username":"admin"},{"password":"stackbench-staff-2026","roles":["staff"],"username":"staff"}],"empty":["carts","orders","reviews","returns"],"items":[{"category":"Home","name":"Air Purifier","price":"189.00","stock":{"East":60,"West":40}},{"category":"Audio","name":"Bluetooth Speaker","price":"79.50","stock":{"East":50,"West":50}},{"category":"Home","name":"Coffee Grinder","price":"64.00","stock":{"East":70,"West":30}},{"category":"Home","name":"Desk Lamp","price":"42.00","stock":{"East":55,"West":45}},{"category":"Home","name":"Espresso Machine","price":"449.00","stock":{"East":80,"West":20}},{"category":"Computing","name":"Gaming Mouse","price":"59.00","stock":{"East":50,"West":50}},{"category":"Audio","name":"Headphones","price":"199.00","stock":{"East":60,"West":40}},{"category":"Home","name":"Induction Cooktop","price":"329.00","stock":{"East":50,"West":50}},{"category":"Computing","name":"Keyboard","price":"89.00","stock":{"East":70,"West":30}},{"category":"Computing","name":"Laptop Stand","price":"29.00","stock":{"East":90,"West":10}},{"category":"Photo","name":"Mirrorless Camera","price":"1299.00","stock":{"East":2,"West":1}},{"category":"Home","name":"USB Cable","price":"65.00","stock":{"East":0,"West":0}},{"category":"Computing","name":"Webcam","price":"69.00","stock":{"East":60,"West":40}}],"warehouses":["East","West"]},"packs":[{"actions":["click","expect","signIn","signUp","waitUntilAbsent"],"budget":{"maxRuntimeMs":18000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.feature.accounts","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","dbExpectStock","dbSetStock","expect","expectNumber","fill","signIn","signUp"],"budget":{"maxRuntimeMs":120000,"status":"bounded"},"capabilities":["browser","database-observation","direct-database-write"],"evidence":["browser-observation","database-observation"],"id":"ecommerce.feature.bundle-checkout","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","dbExpectStock","dbSetStock","expect","expectNumber","fill","signIn","signUp"],"budget":{"maxRuntimeMs":120000,"status":"bounded"},"capabilities":["browser","database-observation","direct-database-write"],"evidence":["browser-observation","database-observation"],"id":"ecommerce.feature.bundle-returns","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","expect","expectNumber","signUp","wait"],"budget":{"maxRuntimeMs":42000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.feature.cart","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.feature.cart-checkout"},{"actions":["expect","expectSequence","fill"],"budget":{"maxRuntimeMs":30000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.feature.catalog-discovery","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.feature.catalog"},{"actions":["expect","expectNumber"],"budget":{"maxRuntimeMs":30000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.feature.catalog-items","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.feature.catalog"},{"actions":["click","ensureSignedIn","expect","expectNumber","reload","signUp","wait"],"budget":{"maxRuntimeMs":42000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.feature.checkout","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.feature.cart-checkout"},{"actions":["click","expect","expectElementCount","expectNumber","fill","reload","signIn"],"budget":{"maxRuntimeMs":120000,"status":"bounded"},"capabilities":["browser","database-observation","direct-database-write"],"evidence":["browser-observation","database-observation"],"id":"ecommerce.feature.product-bundles","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","expect","expectNumber","signUp"],"budget":{"maxRuntimeMs":24000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.feature.purchasing","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","expect","fill","openItem","signUp"],"budget":{"maxRuntimeMs":22000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.feature.reviews","includeRoles":["feature"],"moduleType":"feature"},{"actions":["callAction","click","dbExpectStock","dbRecordStock","ensureSignedIn","expect","expectActionOutcome","expectElementCount","expectNumber","fill","freshClient","recordNumber","reload","signIn","signUp"],"budget":{"maxRuntimeMs":120000,"status":"bounded"},"capabilities":["browser","database-observation","direct-server-call"],"evidence":["browser-observation","database-observation","fresh-client-observation"],"id":"ecommerce.feature.split-tender-refunds","includeRoles":["feature"],"moduleType":"feature"},{"actions":["callAction","click","ensureSignedIn","expect","expectActionOutcome","expectElementCount","expectNumber","fill","freshClient","reload","signIn","signUp"],"budget":{"maxRuntimeMs":180000,"status":"bounded"},"capabilities":["browser","direct-server-call"],"evidence":["browser-observation","fresh-client-observation"],"id":"ecommerce.feature.store-credit","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","dbExpectStock","dbSetStock","expect","expectElementCount","expectNumber","expectSequence","fill","freshClient","signIn","signUp"],"budget":{"maxRuntimeMs":180000,"status":"bounded"},"capabilities":["browser","database-observation","direct-server-call"],"evidence":["browser-observation","database-observation","fresh-client-observation"],"id":"ecommerce.feature.subscriptions","includeRoles":["feature"],"moduleType":"feature"},{"actions":["callAction","click","ensureSignedIn","expect","expectActionOutcome","expectNumber","recordNumber","reload","signIn"],"budget":{"maxRuntimeMs":30000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.feature.warehouse-admin","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","dbSetStock","ensureSignedIn","expect","reload","signIn","signUp"],"budget":{"maxRuntimeMs":86000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.l2.inventory-dashboard","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.inventory-operations"},{"actions":["click","ensureSignedIn","expect","expectNumber","recordNumber","reload","signIn","signUp"],"budget":{"maxRuntimeMs":60000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.l2.order-cancellation-features","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.returns-pricing"},{"actions":["click","expectNumber","fill","signIn"],"budget":{"maxRuntimeMs":94000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.l2.price-history-features","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.returns-pricing"},{"actions":["click","expect","signUp","waitUntilAbsent"],"budget":{"maxRuntimeMs":86000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.l2.recommendations","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.inventory-operations"},{"actions":["click","ensureSignedIn","expect","expectNumber","recordNumber","reload","signIn","signUp"],"budget":{"maxRuntimeMs":86000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.l2.sales-dashboard","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.inventory-operations"},{"actions":["click","dbExpectStock","dbRecordStock","expect","expectNumber","fill","recordNumber","signIn"],"budget":{"maxRuntimeMs":86000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.l2.stock-transfers-features","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.inventory-operations"},{"actions":["click","closeClient","expect","expectNumber","openClient","recordNumber","reload","signUp","wait"],"budget":{"maxRuntimeMs":360000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.l3.cart-expiration-features","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.l3.cart-expiration"},{"actions":["callAction","click","expect","expectActionOutcome","expectReplayRejected","fill","replayAs","signIn","signUp"],"budget":{"maxRuntimeMs":60000,"status":"bounded"},"capabilities":["browser","direct-server-call","request-replay"],"evidence":["browser-observation","server-refusal","server-response"],"id":"ecommerce.l3.deferred-access-specifications","includeRoles":["guarantee"],"moduleType":"specification","stableId":"ecommerce.l3.deferred-access"},{"actions":["click","dbExpectStock","dbRecordStock","ensureSignedIn","expect","expectElapsed","expectNumber","fill","recordNumber","recordTime","reload","restartBackend","signIn","signUp","wait"],"budget":{"maxRuntimeMs":720000,"status":"bounded"},"capabilities":["backend-lifecycle","browser"],"evidence":["browser-observation","database-observation"],"id":"ecommerce.l3.deferred-durability-specifications","includeRoles":["guarantee"],"moduleType":"specification","stableId":"ecommerce.l3.deferred-durability"},{"actions":["click","dbExpectStock","dbRecordStock","ensureSignedIn","expect","expectElementCount","expectNumber","fill","recordNumber","reload","restartBackend","signIn","signUp","wait"],"budget":{"maxRuntimeMs":400000,"status":"bounded"},"capabilities":["backend-lifecycle","browser"],"evidence":["browser-observation","database-observation"],"id":"ecommerce.l3.deferred-integrity-specifications","includeRoles":["guarantee"],"moduleType":"specification","stableId":"ecommerce.l3.deferred-integrity"},{"actions":["click","ensureSignedIn","expect","reload","signIn","signUp","wait"],"budget":{"maxRuntimeMs":190000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.l3.order-delivery-features","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.l3.order-delivery"},{"actions":["callAction","click","dbExpectStock","dbRecordStock","ensureSignedIn","expect","expectActionOutcome","expectNumber","freshClient","recordNumber","reload","signIn","signUp"],"budget":{"maxRuntimeMs":120000,"status":"bounded"},"capabilities":["browser","database-observation"],"evidence":["browser-observation","database-observation","fresh-client-observation"],"id":"ecommerce.l3.order-returns-features","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.returns-pricing"},{"actions":["click","expect","expectNumber","recordNumber","signUp","wait"],"budget":{"maxRuntimeMs":300000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.l3.reservations-features","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.l3.reservations"},{"actions":["click","ensureSignedIn","expect","expectElementCount","expectNumber","fill","recordNumber","reload","signIn","wait"],"budget":{"maxRuntimeMs":150000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.l3.scheduled-restocks-features","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.l3.scheduled-restocks"},{"actions":["click","closeClient","dbExpectStock","dbRecordStock","ensureSignedIn","expect","expectElapsed","expectNumber","fill","recordNumber","recordTime","reload","restartBackend","signIn","signUp","wait"],"budget":{"maxRuntimeMs":300000,"status":"bounded"},"capabilities":["backend-lifecycle","browser"],"evidence":["browser-observation","database-observation"],"id":"ecommerce.l3.server-time-specifications","includeRoles":["guarantee"],"moduleType":"specification","stableId":"ecommerce.l3.server-time"},{"actions":["callAction","click","dbExpectStock","dbSetStock","ensureSignedIn","expect","expectActionOutcome","expectElementCount","fill","reload","signIn","signUp"],"budget":{"maxRuntimeMs":90000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.automatic-reorder","includeRoles":["feature"],"moduleType":"feature"},{"actions":["callConcurrently","click","dbExpectCancellation","dbRecordCheckout","ensureSignedIn","expect","expectCallOutcomes","expectNumber","recordNumber","reload","signIn","signUp","wait"],"budget":{"maxRuntimeMs":24000,"status":"bounded"},"capabilities":["browser","concurrent-actors","direct-server-call"],"evidence":["browser-observation","concurrent-outcome","database-observation"],"id":"ecommerce.progression.cancellation-accounting-specifications","includeRoles":["guarantee"],"moduleType":"specification","stableId":"ecommerce.returns-pricing"},{"actions":["click","expect","signIn","signUp","waitUntilAbsent"],"budget":{"maxRuntimeMs":24000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.cancellation-queue-specifications","includeRoles":["guarantee"],"moduleType":"specification","stableId":"ecommerce.returns-pricing"},{"actions":["click","dbSetStock","ensureSignedIn","expect","expectNumber","recordNumber","reload","signUp","wait"],"budget":{"maxRuntimeMs":350000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.cart-recovery","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","expect","expectElementCount","fill","openItem","reload","signIn"],"budget":{"maxRuntimeMs":30000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.catalog-management","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","expect","fill","signUp"],"budget":{"maxRuntimeMs":30000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.customer-profile","includeRoles":["feature"],"moduleType":"feature"},{"actions":["callAction","click","ensureSignedIn","expect","expectActionOutcome","expectElapsed","expectElementCount","recordTime","reload","signIn","signUp","wait"],"budget":{"maxRuntimeMs":110000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.delivery-notifications","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","dbSetStock","expect","expectElementCount","expectSequence","fill","reload","waitUntilAbsent"],"budget":{"maxRuntimeMs":141000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.faceted-search","includeRoles":["feature"],"moduleType":"feature"},{"actions":["callAction","click","dbSetStock","ensureSignedIn","expect","expectActionOutcome","reload","signIn","signUp"],"budget":{"maxRuntimeMs":76000,"status":"bounded"},"capabilities":["browser","direct-server-call"],"evidence":["browser-observation","server-refusal"],"id":"ecommerce.progression.fulfilment-queue","includeRoles":["feature"],"moduleType":"feature","stableId":"ecommerce.operations-access"},{"actions":["callAction","click","dbExpectStock","dbRecordStock","ensureSignedIn","expect","expectActionOutcome","expectNumber","fill","freshClient","race","recordNumber","reload","signIn","signUp"],"budget":{"maxRuntimeMs":138000,"status":"bounded"},"capabilities":["browser","concurrent-actors","direct-server-call"],"evidence":["browser-observation","server-response"],"id":"ecommerce.progression.inventory-conservation-specifications","includeRoles":["guarantee"],"moduleType":"specification","stableId":"ecommerce.inventory-operations"},{"actions":["click","ensureSignedIn","expect","fill","reload","signIn","signUp"],"budget":{"maxRuntimeMs":55000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.managed-support","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","expect","signUp"],"budget":{"maxRuntimeMs":40000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.notification-preferences","includeRoles":["feature"],"moduleType":"feature"},{"actions":["callAction","click","dbExpectStock","dbRecordStock","ensureSignedIn","expect","expectActionOutcome","expectNumber","fill","recordNumber","reload","signIn","signUp"],"budget":{"maxRuntimeMs":98000,"status":"bounded"},"capabilities":["browser","direct-server-call","request-replay"],"evidence":["server-refusal","server-response"],"id":"ecommerce.progression.operations-access-specifications","includeRoles":["guarantee"],"moduleType":"specification","stableId":"ecommerce.operations-access"},{"actions":["click","expect","fill","signIn","signUp"],"budget":{"maxRuntimeMs":65000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.order-support","includeRoles":["feature"],"moduleType":"feature"},{"actions":["callConcurrently","click","expect","expectCallOutcomes","expectElementCount","expectNumber","freshClient","recordNumber","signIn","signUp"],"budget":{"maxRuntimeMs":30000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.payment-records","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","expect","expectNumber","expectSequence","signUp"],"budget":{"maxRuntimeMs":70000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.personalized-recommendations","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","ensureSignedIn","expect","expectNumber","fill","pressKey","recordNumber","reload","signIn","signUp","wait"],"budget":{"maxRuntimeMs":24000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.price-accounting-specifications","includeRoles":["guarantee"],"moduleType":"specification","stableId":"ecommerce.returns-pricing"},{"actions":["callAction","click","ensureSignedIn","expect","expectActionOutcome","expectNumber","fill","freshClient","recordNumber","reload","signIn","signUp"],"budget":{"maxRuntimeMs":94000,"status":"bounded"},"capabilities":["browser","direct-server-call"],"evidence":["browser-observation","fresh-client-observation"],"id":"ecommerce.progression.price-history-specifications","includeRoles":["guarantee"],"moduleType":"specification","stableId":"ecommerce.returns-pricing"},{"actions":["click","expect","expectNumber","fill","signIn","signUp"],"budget":{"maxRuntimeMs":55000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.promotion-checkout","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","expectNumber","fill","signIn","signUp"],"budget":{"maxRuntimeMs":40000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.promotion-reporting","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","expect","expectNumber","fill","signIn","signUp"],"budget":{"maxRuntimeMs":45000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.promotion-rules","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","expect","signUp"],"budget":{"maxRuntimeMs":130000,"status":"bounded"},"capabilities":["backend-lifecycle","browser"],"evidence":["browser-observation","fresh-client-observation"],"id":"ecommerce.progression.recommendation-feedback","includeRoles":["feature"],"moduleType":"feature"},{"actions":["armScriptCanary","callAction","click","expect","expectActionOutcome","expectNoScriptExecution","freshClient","openItem","signIn","signUp","wait"],"budget":{"maxRuntimeMs":82000,"status":"bounded"},"capabilities":["browser","direct-server-call","request-replay"],"evidence":["browser-observation","fresh-client-observation","server-refusal","server-response"],"id":"ecommerce.progression.review-access-specifications","includeRoles":["guarantee"],"moduleType":"specification"},{"actions":["click","expect","signIn"],"budget":{"maxRuntimeMs":30000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.staff-access","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","expect","fill","signIn"],"budget":{"maxRuntimeMs":30000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.staff-activity","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","expect","fill","signIn"],"budget":{"maxRuntimeMs":30000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.staff-roles","includeRoles":["feature"],"moduleType":"feature"},{"actions":["callAction","click","dbSetStock","ensureSignedIn","expect","expectActionOutcome","expectElementCount","freshClient","reload","signIn","signUp","wait"],"budget":{"maxRuntimeMs":60000,"status":"bounded"},"capabilities":["browser","direct-database-write"],"evidence":["browser-observation"],"id":"ecommerce.progression.stock-alerts","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","ensureSignedIn","expect","fill","reload","signUp"],"budget":{"maxRuntimeMs":50000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.support-history","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","expect","fill"],"budget":{"maxRuntimeMs":30000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.support-intake","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","expect","fill","signIn","signUp"],"budget":{"maxRuntimeMs":70000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.support-refunds","includeRoles":["feature"],"moduleType":"feature"},{"actions":["click","ensureSignedIn","expect","fill","reload","signIn"],"budget":{"maxRuntimeMs":45000,"status":"bounded"},"capabilities":["browser"],"evidence":["browser-observation"],"id":"ecommerce.progression.support-triage","includeRoles":["feature"],"moduleType":"feature"},{"actions":["callAction","click","dbExpectStock","dbRecordStock","dbSetStock","ensureSignedIn","expect","expectActionOutcome","expectElapsed","expectElementCount","expectNotReceived","expectNumber","expectReceived","expectReplayCompleted","expectReplayRejected","expectSequence","fill","freshClient","openItem","recordNumber","recordTime","reload","replayAs","signIn","signUp","wait","waitUntilAbsent"],"budget":{"maxRuntimeMs":464000,"status":"bounded"},"capabilities":["backend-lifecycle","browser","direct-server-call","request-replay"],"evidence":["browser-observation","database-observation","fresh-client-observation","server-response"],"id":"ecommerce.spec.access-control","includeRoles":["guarantee"],"moduleType":"specification"},{"actions":["callAction","callConcurrently","click","dbExpectStock","dbSetStock","ensureSignedIn","expect","expectActionOutcome","expectActorsWith","expectCallOutcomes","expectNumber","fill","reload","restartBackend","signIn","signUp","wait"],"budget":{"maxRuntimeMs":480000,"status":"bounded"},"capabilities":["backend-lifecycle","browser","database-observation","direct-database-write","direct-server-call"],"evidence":["browser-observation","database-observation","fresh-client-observation"],"id":"ecommerce.spec.bundle-integrity","includeRoles":["guarantee"],"moduleType":"specification"},{"actions":["callAction","callConcurrently","click","clickConcurrently","dbExpectCheckout","dbExpectPurchases","dbExpectStock","dbRecordCheckout","dbRecordStock","dbSetStock","ensureSignedIn","expect","expectActionOutcome","expectActorsWith","expectAgreement","expectCallOutcomes","expectNumber","fill","race","recordNumber","reload","signIn","signUp","wait"],"budget":{"maxRuntimeMs":125000,"status":"bounded"},"capabilities":["browser","concurrent-actors","direct-server-call"],"evidence":["concurrent-outcome","database-observation"],"id":"ecommerce.spec.concurrency-safety","includeRoles":["guarantee"],"moduleType":"specification"},{"actions":["dbSetStock","expectNumber","reload","setOffline","startAppServer","stopAppServer"],"budget":{"maxRuntimeMs":105000,"status":"bounded"},"capabilities":["backend-lifecycle","browser","direct-database-write"],"evidence":["database-observation","fresh-client-observation"],"id":"ecommerce.spec.external-data-sync","includeRoles":["guarantee"],"moduleType":"specification"},{"actions":["callAction","click","dbExpectStock","dbRecordStock","dbSetStock","ensureSignedIn","expect","expectActionOutcome","expectAgreement","expectElementCount","expectNumber","expectSequence","fill","openItem","recordNumber","reload","signIn","signUp","waitUntilAbsent"],"budget":{"maxRuntimeMs":184000,"status":"bounded"},"capabilities":["browser","concurrent-actors"],"evidence":["browser-observation","fresh-client-observation"],"id":"ecommerce.spec.live-state","includeRoles":["guarantee"],"moduleType":"specification"},{"actions":["click","dbExpectStock","dbRecordStock","expectSequence","fill","reload","signUp"],"budget":{"maxRuntimeMs":60000,"status":"bounded"},"capabilities":["browser","database-observation"],"evidence":["browser-observation","database-observation"],"id":"ecommerce.spec.search-ordering","includeRoles":["guarantee"],"moduleType":"specification"},{"actions":["callAction","callConcurrently","click","ensureSignedIn","expect","expectActionOutcome","expectCallOutcomes","expectElementCount","expectNumber","fill","freshClient","reload","restartBackend","signIn","signUp"],"budget":{"maxRuntimeMs":300000,"status":"bounded"},"capabilities":["backend-lifecycle","browser","database-observation","direct-server-call"],"evidence":["browser-observation","database-observation","fresh-client-observation"],"id":"ecommerce.spec.split-tender-refunds","includeRoles":["guarantee"],"moduleType":"specification"},{"actions":["click","confirmCheckout","crashCheckout","dbExpectCheckout","dbRecordCheckout","ensureSignedIn","expect","expectCrashCheckout","expectNumber","fill","freshClient","reload","restartBackend","setOffline","signIn","signUp"],"budget":{"maxRuntimeMs":768000,"status":"bounded"},"capabilities":["backend-lifecycle","browser","database-read","process-crash"],"evidence":["browser-observation","fresh-client-observation"],"id":"ecommerce.spec.state-durability","includeRoles":["guarantee"],"moduleType":"specification"},{"actions":["callAction","callConcurrently","click","ensureSignedIn","expect","expectActionOutcome","expectCallOutcomes","expectElementCount","expectNumber","fill","freshClient","reload","restartBackend","signIn","signUp"],"budget":{"maxRuntimeMs":300000,"status":"bounded"},"capabilities":["backend-lifecycle","browser","database-observation","direct-server-call"],"evidence":["browser-observation","database-observation","fresh-client-observation"],"id":"ecommerce.spec.store-credit","includeRoles":["guarantee"],"moduleType":"specification"},{"actions":["callAction","click","dbExpectStock","dbRecordStock","dbSetStock","expect","expectActionOutcome","expectElementCount","expectNumber","expectSequence","fill","freshClient","recordNumber","restartBackend","signIn","signUp","wait"],"budget":{"maxRuntimeMs":480000,"status":"bounded"},"capabilities":["backend-lifecycle","browser","database-observation","direct-server-call"],"evidence":["browser-observation","database-observation","fresh-client-observation"],"id":"ecommerce.spec.subscriptions","includeRoles":["guarantee"],"moduleType":"specification"},{"actions":["callAction","callConcurrently","click","dbExpectStock","dbRecordStock","dbSetStock","ensureSignedIn","expect","expectActionOutcome","expectCallOutcomes","expectElementCount","expectNumber","expectReplayCompleted","fill","freshClient","openItem","recordNumber","reload","replayAs","signIn","signUp","wait"],"budget":{"maxRuntimeMs":100000,"status":"bounded"},"capabilities":["browser","database-observation","direct-server-call"],"evidence":["browser-observation","database-observation","server-response"],"id":"ecommerce.spec.transactional-integrity","includeRoles":["guarantee"],"moduleType":"specification"}],"runtime":{"actions":[{"args":[0,1],"id":"addToCart","path":"/api/cart","reducer":"add_to_cart"},{"args":[0],"id":"buy","path":"/api/items/00000000-0000-0000-0000-000000000000/buy","reducer":"buy_now"},{"args":[],"id":"checkout","path":"/api/checkout","reducer":"checkout"},{"args":[0,0,1],"id":"restock","path":"/api/admin/restock","reducer":"admin_restock"},{"args":["",""],"id":"signIn","path":"/api/auth/signin","reducer":"sign_in"},{"args":["",""],"id":"signUp","params":[{"in":"body","name":"username"},{"in":"body","name":"password"}],"path":"/api/auth/signup","reducer":"sign_up"}],"portOffset":300,"reseedOnReset":true,"restartProbe":"/api/items"},"schemaVersion":3,"task":{"baseExecutionSha256":null,"mode":"action"},"track":"ecommerce"}},"calibration":{"calibrationSchemaVersion":2,"controls":[],"equivalenceDecisions":[],"fixture":{"id":"ecommerce.operations","sourceSha256":"d06444b72dc94fe1ef5e08867d875e1f3bbaa5cd82c35a3558f399c1fcb5ceae"},"id":"ecommerce.dependency-l3-calibration","mutations":[{"backend":"mongodb","path":"grader/mutations/mongodb-ecommerce.json","sha256":"90ae4b351ef5e7b06435c1c99f8df96fb850ccad338eeba4e68910560845d875","referenceId":"ecommerce-reference-mongodb","executionSha256":"927e83bd722707a09c763de85f46c8c4bc58e5ce370490a8ddbaaabde6bbcbf7","targets":[{"id":"staff-admin-access-survives-role-removal","stableKeys":["ecommerce.spec.access-control.staff-role-revocation.621d"]},{"id":"shipping-counts-sale-twice","stableKeys":["ecommerce.inventory-operations.shipping-accounting.202e"]},{"id":"signup-does-not-expose-created-account","stableKeys":["ecommerce.feature.accounts.accounts.1a"]},{"id":"duplicate-signup-reports-success","stableKeys":["ecommerce.feature.accounts.accounts.1b"]},{"id":"signin-skips-password-verification","stableKeys":["ecommerce.feature.accounts.accounts.1c"]},{"id":"signout-keeps-current-account","stableKeys":["ecommerce.feature.accounts.accounts.1d"]},{"id":"session-token-not-persisted","stableKeys":["ecommerce.spec.state-durability.session-reload.1e"]},{"id":"purchase-counts-never-affect-ranking","stableKeys":["ecommerce.spec.live-state.ranking.2c"]},{"id":"signed-out-visitor-purchase-is-accepted","stableKeys":["ecommerce.spec.access-control.signed-out-purchase.3a"]},{"id":"espresso-stock-row-ignores-live-updates","stableKeys":["ecommerce.spec.live-state.purchase-stock.3b"]},{"id":"restock-race-records-wrong-order-total","stableKeys":["ecommerce.spec.concurrency-safety.restock-race.202a"]},{"id":"purchase-order-uses-zero-price","stableKeys":["ecommerce.feature.purchasing.purchase-order.3c"]},{"id":"reload-hydrates-an-empty-cart","stableKeys":["ecommerce.spec.state-durability.cart-reload.4b"]},{"id":"shared-cart-live-events-ignored","stableKeys":["ecommerce.spec.live-state.shared-cart.4c"]},{"id":"review-comment-is-not-persisted","stableKeys":["ecommerce.feature.reviews.reviews.6a"]},{"id":"repeat-review-uses-a-new-owner-key","stableKeys":["ecommerce.spec.transactional-integrity.unique-review.6b"]},{"id":"live-review-average-uses-an-extra-divisor","stableKeys":["ecommerce.spec.live-state.rating.6c"]},{"id":"warehouse-view-omits-one-location","stableKeys":["ecommerce.feature.warehouse-admin.warehouse-view.7b"]},{"id":"unauthenticated-purchase-defaults-to-admin","stableKeys":["ecommerce.spec.access-control.purchase-session.101a"]},{"id":"direct-purchase-total-ignores-store-price","stableKeys":["ecommerce.spec.transactional-integrity.server-price.104a"]},{"id":"cart-hydration-loses-account-state","stableKeys":["ecommerce.spec.state-durability.account-state-recovery.105a"]},{"id":"reconnect-hydration-loses-account-state","stableKeys":["ecommerce.spec.state-durability.account-state-recovery.105b"]},{"id":"order-history-is-not-owner-scoped","stableKeys":["ecommerce.spec.access-control.order-ownership.106a"]},{"id":"revenue-aggregation-ignores-order-totals","stableKeys":["ecommerce.spec.transactional-integrity.books-balance.107a"]},{"id":"unpurchased-review-is-accepted","stableKeys":["ecommerce.spec.access-control.review-eligibility.108a"]},{"id":"purchased-review-ui-does-not-submit","stableKeys":["ecommerce.spec.access-control.review-eligibility.108b"]},{"id":"external-stock-polling-disabled","stableKeys":["ecommerce.spec.external-data-sync.external-stock.901a"]},{"id":"server-restart-disables-catalog-recovery","stableKeys":["ecommerce.spec.external-data-sync.external-stock.901c"]},{"id":"reconnect-generation-ignores-current-catalog","stableKeys":["ecommerce.spec.external-data-sync.external-stock.901d"]},{"id":"open-review-list-ignores-live-update","stableKeys":["ecommerce.spec.live-state.open-list.902a"]},{"id":"cancel-does-not-restore-stock-feature","stableKeys":["ecommerce.returns-pricing.cancellation-and-return.3a"]},{"id":"cancellation-accounting-loses-stock-restoration","stableKeys":["ecommerce.returns-pricing.refund-accounting.203a"]},{"id":"cancel-does-not-restore-stock-fresh-client","stableKeys":["ecommerce.inventory-operations.stock-conservation.202b","ecommerce.inventory-operations.stock-conservation.202c"]},{"id":"cancel-restores-stock-but-keeps-pending-status","stableKeys":["ecommerce.returns-pricing.cancellation-and-return.3b"]},{"id":"cancelled-order-remains-in-revenue-feature","stableKeys":["ecommerce.returns-pricing.cancellation-and-return.3a"]},{"id":"cancelled-order-remains-in-revenue-invariant","stableKeys":["ecommerce.returns-pricing.refund-accounting.203a"]},{"id":"operator-authorization-allows-customer-transfer","stableKeys":["ecommerce.operations-access.operator-authorization.201a"]},{"id":"customer-can-ship-order-direct-1-1","stableKeys":["ecommerce.operations-access.operator-authorization.201c"]},{"id":"customer-can-cancel-foreign-order-1-1","stableKeys":["ecommerce.operations-access.order-owner.204a"]},{"id":"ship-acknowledges-without-changing-status","stableKeys":["ecommerce.operations-access.fulfilment-queue.1c"]},{"id":"progression-customer-sees-fulfilment-content","stableKeys":["ecommerce.spec.access-control.fulfilment-area-boundary.1d"]},{"id":"transfer-debits-source-without-crediting-existing-destination","stableKeys":["ecommerce.inventory-operations.stock-conservation.202a","ecommerce.inventory-operations.warehouse-transfer.2a","ecommerce.operations-access.operator-authorization.201a"]},{"id":"recommendations-ignore-pending-purchases","stableKeys":["ecommerce.inventory-operations.operational-views.5c"]},{"id":"purchases-do-not-affect-best-sellers","stableKeys":["ecommerce.inventory-operations.operational-views.5d"]},{"id":"queue-warehouse-reports-west","stableKeys":["ecommerce.operations-access.fulfilment-queue.1b"]},{"id":"cart-repeat-does-not-increment","stableKeys":["ecommerce.feature.cart-checkout.cart.4a"]},{"id":"checkout-leaves-cart-claimed","stableKeys":["ecommerce.feature.cart-checkout.cart.4d"]},{"id":"catalog-initial-ranking-is-reversed","stableKeys":["ecommerce.feature.catalog.catalog-ranking.2b"]},{"id":"catalog-search-requires-exact-name","stableKeys":["ecommerce.feature.catalog.catalog-search.2d"]},{"id":"catalog-price-is-offset","stableKeys":["ecommerce.feature.catalog.catalog-values.2a"]},{"id":"staff-can-see-admin-navigation","stableKeys":["ecommerce.spec.access-control.warehouse-area-boundary.7a"]},{"id":"staff-can-use-direct-restock","stableKeys":["ecommerce.spec.access-control.warehouse-write-boundary.103b"]},{"id":"restock-adds-the-wrong-quantity","stableKeys":["ecommerce.spec.live-state.warehouse-stock.7c"]},{"id":"transfer-creates-stock-during-race","stableKeys":["ecommerce.inventory-operations.stock-conservation.202d"]},{"id":"customer-can-cancel-scheduled-restock","stableKeys":["ecommerce.l3.deferred-access.scheduled-work-access.317a"]},{"id":"scheduled-restock-never-becomes-due-after-restart","stableKeys":["ecommerce.l3.deferred-durability.restart-survival.311a"]},{"id":"completed-restock-is-replayed","stableKeys":["ecommerce.l3.deferred-integrity.exactly-once.311a"]},{"id":"scheduled-restock-countdown-is-fixed","stableKeys":["ecommerce.l3.scheduled-restocks.scheduled-restocks.302a"]},{"id":"due-restock-does-not-change-stock","stableKeys":["ecommerce.l3.scheduled-restocks.scheduled-restocks.305a"]},{"id":"cancelled-restock-remains-pending","stableKeys":["ecommerce.l3.scheduled-restocks.scheduled-restocks.306a"]},{"id":"server-time-restock-never-completes","stableKeys":["ecommerce.l3.server-time.server-time.312a"]},{"id":"catalog-product-name-is-not-published","stableKeys":["ecommerce.progression.catalog-management.catalog-management.622a","ecommerce.progression.catalog-management.catalog-management.622b"]},{"id":"catalog-variants-are-discarded","stableKeys":["ecommerce.progression.catalog-management.catalog-management.622b"]},{"id":"profile-data-is-lost-on-server-restart","stableKeys":["ecommerce.spec.state-durability.customer-profile-reload.620a"]},{"id":"profile-read-is-not-owner-scoped","stableKeys":["ecommerce.spec.access-control.customer-profile-privacy.620b"]},{"id":"faceted-search-ignores-category","stableKeys":["ecommerce.progression.faceted-search.faceted-search.401a"]},{"id":"active-search-uses-purchase-ranking","stableKeys":["ecommerce.spec.search-ordering.search-ordering.402b"]},{"id":"pagination-repeats-first-page","stableKeys":["ecommerce.progression.faceted-search.faceted-search.402a"]},{"id":"managed-support-live-refresh-keeps-stale-tickets","stableKeys":["ecommerce.spec.live-state.managed-support.613a"]},{"id":"managed-support-allows-another-customer","stableKeys":["ecommerce.spec.access-control.managed-support-privacy.613b"]},{"id":"notification-preference-is-not-saved","stableKeys":["ecommerce.spec.state-durability.notification-preferences-reload.630a","ecommerce.progression.notification-preferences.notification-preferences.630c"]},{"id":"notification-preference-is-not-owner-scoped","stableKeys":["ecommerce.spec.access-control.notification-preferences-privacy.630b"]},{"id":"promotion-save-drops-bounded-values","stableKeys":["ecommerce.progression.promotion-rules.promotion-rule-values.620a"]},{"id":"customer-can-create-promotion","stableKeys":["ecommerce.spec.access-control.promotion-management-boundary.620b"]},{"id":"staff-signin-loses-staff-role","stableKeys":["ecommerce.progression.staff-access.staff-access.601a"]},{"id":"customer-signin-gains-staff-role","stableKeys":["ecommerce.spec.access-control.staff-area-boundary.601b"]},{"id":"role-assignment-drops-role","stableKeys":["ecommerce.spec.state-durability.staff-role-reload.621a"]},{"id":"staff-role-write-precedes-denial","stableKeys":["ecommerce.spec.access-control.staff-role-boundary.621b","ecommerce.spec.access-control.staff-role-revocation.621d"]},{"id":"staff-can-assign-roles","stableKeys":["ecommerce.spec.access-control.staff-role-boundary.621b","ecommerce.spec.access-control.staff-role-revocation.621d"]},{"id":"stock-alert-delivery-is-suppressed","stableKeys":["ecommerce.progression.stock-alerts.stock-alert-delivery.631c"]},{"id":"stock-alert-repeats-while-in-stock","stableKeys":["ecommerce.spec.transactional-integrity.stock-alert-deduplication.631a"]},{"id":"stock-alerts-are-not-owner-scoped","stableKeys":["ecommerce.spec.access-control.stock-alert-privacy.631b"]},{"id":"support-history-is-lost-on-server-restart","stableKeys":["ecommerce.spec.state-durability.support-history-reload.612a"]},{"id":"support-history-is-not-owner-scoped","stableKeys":["ecommerce.spec.access-control.support-history-privacy.612b"]},{"id":"support-intake-returns-no-reference","stableKeys":["ecommerce.progression.support-intake.support-intake.610a"]},{"id":"support-triage-discards-updates","stableKeys":["ecommerce.progression.support-triage.support-assignment.611a","ecommerce.progression.support-triage.support-priority.611b","ecommerce.progression.support-triage.support-status.611c"]},{"id":"cart-add-uses-another-account-cart","stableKeys":["ecommerce.spec.access-control.cart-boundary.109a"]},{"id":"negative-cart-quantity-is-accepted","stableKeys":["ecommerce.spec.access-control.cart-boundary.109b"]},{"id":"direct-purchase-is-attributed-to-another-account","stableKeys":["ecommerce.spec.access-control.purchase-attribution.102a"]},{"id":"concurrent-cart-add-does-not-increment","stableKeys":["ecommerce.spec.concurrency-safety.duplicate-checkout.203a"]},{"id":"checkout-claim-is-not-atomic","stableKeys":["ecommerce.spec.concurrency-safety.duplicate-checkout.203b"]},{"id":"last-unit-allows-negative-stock","stableKeys":["ecommerce.spec.concurrency-safety.last-unit.201a","ecommerce.spec.concurrency-safety.last-unit.201b","ecommerce.spec.concurrency-safety.last-unit.201c"]},{"id":"purchase-read-write-loses-concurrent-stock","stableKeys":["ecommerce.spec.concurrency-safety.restock-race.202a"]},{"id":"purchase-does-not-reduce-warehouse-stock","stableKeys":["ecommerce.spec.transactional-integrity.books-balance.107b"]},{"id":"restock-does-not-increase-stock","stableKeys":["ecommerce.spec.live-state.warehouse-stock.7c"]},{"id":"direct-review-access-is-not-checked","stableKeys":["ecommerce.progression.review-access-specifications.review-eligibility-direct.618a"]},{"id":"support-history-rows-are-hidden","stableKeys":["ecommerce.progression.support-history.support-history.612c","ecommerce.spec.state-durability.support-history-reload.612a","ecommerce.spec.access-control.support-history-privacy.612b","ecommerce.spec.access-control.support-history-logout.612d"]},{"id":"authorized-restock-does-not-change-stock","stableKeys":["ecommerce.feature.warehouse-admin.admin-write.103a"]},{"id":"initial-dashboard-load-omits-low-stock","stableKeys":["ecommerce.inventory-operations.operational-views.5e","ecommerce.spec.live-state.inventory-dashboard.5a"]},{"id":"category-totals-skip-the-newest-order","stableKeys":["ecommerce.inventory-operations.operational-views.5f"]},{"id":"profile-summary-frozen-at-open","stableKeys":["ecommerce.progression.customer-profile.customer-profile.620c"]},{"id":"support-replies-present-at-open-are-hidden","stableKeys":["ecommerce.progression.managed-support.managed-support.613c"]},{"id":"notification-toggle-frozen-at-open","stableKeys":["ecommerce.progression.notification-preferences.notification-preferences.630c"]},{"id":"role-editor-snaps-back-to-stored-role","stableKeys":["ecommerce.progression.staff-roles.staff-roles.621c"]},{"id":"queue-ignores-live-fulfilment-updates","stableKeys":["ecommerce.spec.live-state.fulfilment-queue.1a"]},{"id":"low-stock-boundary-excludes-ten-live","stableKeys":["ecommerce.spec.live-state.inventory-dashboard.5a"]},{"id":"live-admin-updates-keep-stale-category-totals","stableKeys":["ecommerce.spec.live-state.sales-dashboard.5b"]},{"id":"overdraw-transfer-is-accepted","stableKeys":["ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c"]},{"id":"transfer-totals-omit-destination-credit-live","stableKeys":["ecommerce.spec.live-state.stock-transfers.2b"]},{"id":"support-history-leaks-to-signed-out-visitors","stableKeys":["ecommerce.spec.access-control.support-history-logout.612d"]},{"id":"checkout-crash-integrity","stableKeys":["ecommerce.spec.state-durability.checkout-crash-integrity.910a"]},{"id":"checkout-crash-durability","stableKeys":["ecommerce.spec.state-durability.checkout-crash-durability.910b"]},{"id":"review-script-unsafe-render","stableKeys":["ecommerce.progression.review-access-specifications.stored-review-script.9180a"]},{"id":"review-script-reject-all","stableKeys":["ecommerce.progression.review-access-specifications.stored-review-script.9180a"]}]},{"backend":"postgres","path":"grader/mutations/postgres-ecommerce.json","sha256":"ead84228c726f2b69d80de5c7bad2e96757a16b9540874fd291de8ddb7542c75","referenceId":"ecommerce-reference-postgres","executionSha256":"194ecf8d31ee18c1e78b15e84a6ae92d4fd32f0b759f12755ba0ffab42978562","targets":[{"id":"staff-admin-access-survives-role-removal","stableKeys":["ecommerce.spec.access-control.staff-role-revocation.621d"]},{"id":"shipping-counts-sale-twice","stableKeys":["ecommerce.inventory-operations.shipping-accounting.202e"]},{"id":"signup-ui-does-not-enter-created-account","stableKeys":["ecommerce.feature.accounts.accounts.1a"]},{"id":"duplicate-signup-authenticates-existing-account","stableKeys":["ecommerce.feature.accounts.accounts.1b"]},{"id":"password-verification-is-inverted","stableKeys":["ecommerce.feature.accounts.accounts.1c"]},{"id":"correct-signin-is-refused","stableKeys":["ecommerce.feature.accounts.accounts.1d"]},{"id":"reload-discards-session-identity","stableKeys":["ecommerce.spec.state-durability.session-reload.1e"]},{"id":"purchase-does-not-broadcast-ranking","stableKeys":["ecommerce.spec.live-state.ranking.2c"]},{"id":"signed-out-purchase-uses-default-account","stableKeys":["ecommerce.spec.access-control.signed-out-purchase.3a"]},{"id":"purchase-stock-change-is-not-broadcast--01-buying","stableKeys":["ecommerce.spec.live-state.purchase-stock.3b"]},{"id":"restock-race-records-wrong-order-total","stableKeys":["ecommerce.spec.concurrency-safety.restock-race.202a"]},{"id":"direct-purchase-order-total-is-offset","stableKeys":["ecommerce.feature.purchasing.purchase-order.3c"]},{"id":"reload-hydrates-an-empty-cart","stableKeys":["ecommerce.spec.state-durability.cart-reload.4b"]},{"id":"signed-out-visitors-do-not-see-reviews","stableKeys":["ecommerce.feature.reviews.reviews.6a"]},{"id":"review-average-update-is-not-broadcast","stableKeys":["ecommerce.spec.live-state.rating.6c"]},{"id":"admin-warehouse-view-drops-one-location","stableKeys":["ecommerce.feature.warehouse-admin.warehouse-view.7b"]},{"id":"unauthenticated-direct-purchase-uses-default-account","stableKeys":["ecommerce.spec.access-control.purchase-session.101a"]},{"id":"direct-purchase-is-attributed-to-previous-account","stableKeys":["ecommerce.spec.access-control.purchase-attribution.102a"]},{"id":"direct-purchase-uses-constant-price","stableKeys":["ecommerce.spec.transactional-integrity.server-price.104a"]},{"id":"account-state-reload-discards-session","stableKeys":["ecommerce.spec.state-durability.account-state-recovery.105a"]},{"id":"offline-event-clears-account-state","stableKeys":["ecommerce.spec.state-durability.account-state-recovery.105b"]},{"id":"purchase-does-not-decrement-warehouse-stock","stableKeys":["ecommerce.spec.transactional-integrity.books-balance.107b"]},{"id":"review-route-skips-purchase-eligibility","stableKeys":["ecommerce.spec.access-control.review-eligibility.108a"]},{"id":"only-shipped-orders-earn-review-eligibility","stableKeys":["ecommerce.spec.access-control.review-eligibility.108b","ecommerce.spec.access-control.review-eligibility.108a"]},{"id":"cart-update-accepts-negative-quantity","stableKeys":["ecommerce.spec.access-control.cart-boundary.109b"]},{"id":"oversell-no-row-lock","stableKeys":["ecommerce.spec.concurrency-safety.last-unit.201a","ecommerce.spec.concurrency-safety.last-unit.201b","ecommerce.spec.concurrency-safety.last-unit.201c"]},{"id":"purchase-read-write-loses-concurrent-stock","stableKeys":["ecommerce.spec.concurrency-safety.restock-race.202a"]},{"id":"external-stock-polling-disabled","stableKeys":["ecommerce.spec.external-data-sync.external-stock.901a"]},{"id":"server-restart-does-not-resynchronize-catalog","stableKeys":["ecommerce.spec.external-data-sync.external-stock.901c"]},{"id":"reconnect-does-not-send-current-catalog","stableKeys":["ecommerce.spec.external-data-sync.external-stock.901d"]},{"id":"open-review-list-ignores-live-update","stableKeys":["ecommerce.spec.live-state.open-list.902a"]},{"id":"open-review-list-renders-each-review-twice","stableKeys":["ecommerce.spec.live-state.open-list.902a"]},{"id":"cancel-does-not-restore-stock-feature","stableKeys":["ecommerce.returns-pricing.cancellation-and-return.3a"]},{"id":"cancellation-accounting-loses-stock-restoration","stableKeys":["ecommerce.returns-pricing.refund-accounting.203a"]},{"id":"cancel-does-not-restore-stock-fresh-client","stableKeys":["ecommerce.inventory-operations.stock-conservation.202b","ecommerce.inventory-operations.stock-conservation.202c"]},{"id":"cancel-restores-stock-but-keeps-pending-status","stableKeys":["ecommerce.returns-pricing.cancellation-and-return.3b"]},{"id":"operator-authorization-allows-customer-transfer","stableKeys":["ecommerce.operations-access.operator-authorization.201a"]},{"id":"customer-can-ship-order-direct-1-1","stableKeys":["ecommerce.operations-access.operator-authorization.201c"]},{"id":"customer-can-cancel-foreign-order-1-1","stableKeys":["ecommerce.operations-access.order-owner.204a"]},{"id":"progression-customer-sees-fulfilment-content","stableKeys":["ecommerce.spec.access-control.fulfilment-area-boundary.1d"]},{"id":"transfer-debits-source-without-crediting-existing-destination","stableKeys":["ecommerce.inventory-operations.stock-conservation.202a","ecommerce.inventory-operations.warehouse-transfer.2a","ecommerce.operations-access.operator-authorization.201a"]},{"id":"recommendations-ignore-pending-purchases","stableKeys":["ecommerce.inventory-operations.operational-views.5c"]},{"id":"purchases-do-not-affect-best-sellers","stableKeys":["ecommerce.inventory-operations.operational-views.5d"]},{"id":"queue-warehouse-reports-west","stableKeys":["ecommerce.operations-access.fulfilment-queue.1b"]},{"id":"transfer-overwrites-concurrent-purchase-with-stale-stock","stableKeys":["ecommerce.inventory-operations.stock-conservation.202d"]},{"id":"progression-profile-address-is-discarded","stableKeys":["ecommerce.spec.state-durability.customer-profile-reload.620a"]},{"id":"progression-profile-reads-another-account","stableKeys":["ecommerce.spec.access-control.customer-profile-privacy.620b"]},{"id":"progression-staff-tools-are-hidden","stableKeys":["ecommerce.progression.staff-access.staff-access.601a"]},{"id":"progression-customer-sees-staff-tools","stableKeys":["ecommerce.spec.access-control.staff-area-boundary.601b"]},{"id":"progression-staff-role-is-lost-on-restart","stableKeys":["ecommerce.spec.state-durability.staff-role-reload.621a"]},{"id":"staff-role-write-precedes-denial","stableKeys":["ecommerce.spec.access-control.staff-role-boundary.621b","ecommerce.spec.access-control.staff-role-revocation.621d"]},{"id":"progression-staff-can-assign-roles","stableKeys":["ecommerce.spec.access-control.staff-role-boundary.621b","ecommerce.spec.access-control.staff-role-revocation.621d"]},{"id":"progression-catalog-product-name-is-not-published","stableKeys":["ecommerce.progression.catalog-management.catalog-management.622a","ecommerce.progression.catalog-management.catalog-management.622b"]},{"id":"progression-catalog-variants-are-discarded","stableKeys":["ecommerce.progression.catalog-management.catalog-management.622b"]},{"id":"progression-support-intake-is-disabled","stableKeys":["ecommerce.progression.support-intake.support-intake.610a"]},{"id":"progression-support-triage-update-is-disabled","stableKeys":["ecommerce.progression.support-triage.support-assignment.611a","ecommerce.progression.support-triage.support-priority.611b","ecommerce.progression.support-triage.support-status.611c"]},{"id":"progression-support-history-is-not-persisted","stableKeys":["ecommerce.spec.state-durability.support-history-reload.612a"]},{"id":"progression-support-history-leaks","stableKeys":["ecommerce.spec.access-control.support-history-privacy.612b"]},{"id":"progression-managed-support-is-not-shared","stableKeys":["ecommerce.spec.live-state.managed-support.613a"]},{"id":"progression-managed-support-leaks","stableKeys":["ecommerce.spec.access-control.managed-support-privacy.613b"]},{"id":"progression-promotion-discount-is-offset","stableKeys":["ecommerce.progression.promotion-rules.promotion-rule-values.620a"]},{"id":"progression-customer-can-create-promotions","stableKeys":["ecommerce.spec.access-control.promotion-management-boundary.620b"]},{"id":"progression-notification-preferences-do-not-save","stableKeys":["ecommerce.spec.state-durability.notification-preferences-reload.630a"]},{"id":"progression-notifications-leak-across-accounts","stableKeys":["ecommerce.spec.access-control.notification-preferences-privacy.630b"]},{"id":"stock-alert-delivery-is-suppressed","stableKeys":["ecommerce.progression.stock-alerts.stock-alert-delivery.631c"]},{"id":"stock-alert-is-sent-after-every-restock","stableKeys":["ecommerce.spec.transactional-integrity.stock-alert-deduplication.631a"]},{"id":"progression-stock-alerts-leak-across-accounts","stableKeys":["ecommerce.spec.access-control.stock-alert-privacy.631b"]},{"id":"progression-faceted-filter-ignores-category","stableKeys":["ecommerce.progression.faceted-search.faceted-search.401a"]},{"id":"active-search-uses-purchase-ranking","stableKeys":["ecommerce.spec.search-ordering.search-ordering.402b"]},{"id":"progression-pagination-always-shows-first-page","stableKeys":["ecommerce.progression.faceted-search.faceted-search.402a"]},{"id":"progression-restock-countdown-is-fixed","stableKeys":["ecommerce.l3.scheduled-restocks.scheduled-restocks.302a"]},{"id":"progression-due-restock-does-not-run","stableKeys":["ecommerce.l3.scheduled-restocks.scheduled-restocks.305a"]},{"id":"progression-cancelled-restock-still-runs","stableKeys":["ecommerce.l3.scheduled-restocks.scheduled-restocks.306a"]},{"id":"progression-restart-timer-never-runs","stableKeys":["ecommerce.l3.server-time.server-time.312a"]},{"id":"progression-cart-line-does-not-increment","stableKeys":["ecommerce.feature.cart-checkout.cart.4a"]},{"id":"progression-checkout-leaves-cart-lines","stableKeys":["ecommerce.feature.cart-checkout.cart.4d"]},{"id":"progression-cart-update-uses-wrong-room","stableKeys":["ecommerce.spec.live-state.shared-cart.4c"]},{"id":"progression-order-history-ignores-owner","stableKeys":["ecommerce.spec.access-control.order-ownership.106a"]},{"id":"progression-review-conflict-is-not-updated","stableKeys":["ecommerce.spec.transactional-integrity.unique-review.6b"]},{"id":"progression-revenue-double-counts-orders","stableKeys":["ecommerce.spec.transactional-integrity.books-balance.107a"]},{"id":"progression-cancelled-orders-remain-in-revenue","stableKeys":["ecommerce.returns-pricing.refund-accounting.203a"]},{"id":"progression-shipping-keeps-order-pending","stableKeys":["ecommerce.operations-access.fulfilment-queue.1c"]},{"id":"progression-concurrent-cart-line-does-not-increment","stableKeys":["ecommerce.spec.concurrency-safety.duplicate-checkout.203a"]},{"id":"progression-concurrent-checkout-leaves-cart-lines","stableKeys":["ecommerce.spec.concurrency-safety.duplicate-checkout.203b"]},{"id":"progression-catalog-ranking-is-reversed","stableKeys":["ecommerce.feature.catalog.catalog-ranking.2b"]},{"id":"progression-catalog-search-requires-exact-name","stableKeys":["ecommerce.feature.catalog.catalog-search.2d"]},{"id":"progression-catalog-price-is-offset","stableKeys":["ecommerce.feature.catalog.catalog-values.2a"]},{"id":"progression-staff-sees-admin-navigation","stableKeys":["ecommerce.spec.access-control.warehouse-area-boundary.7a"]},{"id":"progression-staff-can-restock-directly","stableKeys":["ecommerce.spec.access-control.warehouse-write-boundary.103b"]},{"id":"progression-restock-adds-wrong-quantity","stableKeys":["ecommerce.spec.live-state.warehouse-stock.7c"]},{"id":"progression-cart-add-uses-another-account","stableKeys":["ecommerce.spec.access-control.cart-boundary.109a"]},{"id":"progression-customers-can-manage-scheduled-work","stableKeys":["ecommerce.l3.deferred-access.scheduled-work-access.317a"]},{"id":"progression-restock-does-not-survive-restart","stableKeys":["ecommerce.l3.deferred-durability.restart-survival.311a"]},{"id":"progression-restock-can-apply-more-than-once","stableKeys":["ecommerce.l3.deferred-integrity.exactly-once.311a"]},{"id":"restock-overwrites-instead-of-increments","stableKeys":["ecommerce.spec.live-state.warehouse-stock.7c"]},{"id":"direct-review-access-is-not-checked","stableKeys":["ecommerce.progression.review-access-specifications.review-eligibility-direct.618a"]},{"id":"support-history-rows-are-hidden","stableKeys":["ecommerce.progression.support-history.support-history.612c","ecommerce.spec.state-durability.support-history-reload.612a","ecommerce.spec.access-control.support-history-privacy.612b","ecommerce.spec.access-control.support-history-logout.612d"]},{"id":"authorized-restock-does-not-change-stock","stableKeys":["ecommerce.feature.warehouse-admin.admin-write.103a"]},{"id":"low-stock-threshold-is-two-units","stableKeys":["ecommerce.inventory-operations.operational-views.5e","ecommerce.spec.live-state.inventory-dashboard.5a"]},{"id":"category-totals-render-as-session-deltas","stableKeys":["ecommerce.inventory-operations.operational-views.5f"]},{"id":"profile-summary-ignores-saved-address","stableKeys":["ecommerce.progression.customer-profile.customer-profile.620c"]},{"id":"support-first-reply-is-hidden","stableKeys":["ecommerce.progression.managed-support.managed-support.613c"]},{"id":"notification-sync-flips-saved-toggle","stableKeys":["ecommerce.progression.notification-preferences.notification-preferences.630c"]},{"id":"staff-role-form-reverts-after-save","stableKeys":["ecommerce.progression.staff-roles.staff-roles.621c"]},{"id":"purchase-does-not-broadcast-fulfilment-queue","stableKeys":["ecommerce.spec.live-state.fulfilment-queue.1a"]},{"id":"admin-state-change-is-not-broadcast","stableKeys":["ecommerce.spec.live-state.inventory-dashboard.5a"]},{"id":"admin-sockets-do-not-join-admin-room","stableKeys":["ecommerce.spec.live-state.sales-dashboard.5b"]},{"id":"transfer-overdraft-guard-skips-bulk-transfers","stableKeys":["ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c"]},{"id":"transfer-does-not-publish-warehouse-totals","stableKeys":["ecommerce.spec.live-state.stock-transfers.2b"]},{"id":"progression-support-history-anonymous-leak","stableKeys":["ecommerce.spec.access-control.support-history-logout.612d"]},{"id":"checkout-crash-integrity","stableKeys":["ecommerce.spec.state-durability.checkout-crash-integrity.910a"]},{"id":"checkout-crash-durability","stableKeys":["ecommerce.spec.state-durability.checkout-crash-durability.910b"]},{"id":"review-script-unsafe-render","stableKeys":["ecommerce.progression.review-access-specifications.stored-review-script.9180a"]},{"id":"review-script-reject-all","stableKeys":["ecommerce.progression.review-access-specifications.stored-review-script.9180a"]}]},{"backend":"spacetime","path":"grader/mutations/spacetime-ecommerce.json","sha256":"158212f01a8b0bfd29976eed0361855bcb1ffbd71f00e4bb22707695098efcba","referenceId":"ecommerce-reference-spacetime","executionSha256":"c845a507863287e3a900498d1cca31e058f1b745bc1b5d8ba048ba79e0734e23","targets":[{"id":"staff-admin-access-survives-role-removal","stableKeys":["ecommerce.spec.access-control.staff-role-revocation.621d"]},{"id":"shipping-counts-sale-twice","stableKeys":["ecommerce.inventory-operations.shipping-accounting.202e"]},{"id":"restock-client-snapshot-overwrites-concurrent-purchases","stableKeys":["ecommerce.spec.concurrency-safety.restock-race.202a"]},{"id":"signup-binds-the-new-account-to-the-admin-session","stableKeys":["ecommerce.feature.accounts.accounts.1a"]},{"id":"duplicate-signup-is-silently-ignored","stableKeys":["ecommerce.feature.accounts.accounts.1b"]},{"id":"signin-does-not-verify-the-password","stableKeys":["ecommerce.feature.accounts.accounts.1c"]},{"id":"signout-keeps-the-account-session","stableKeys":["ecommerce.feature.accounts.accounts.1d"]},{"id":"session-token-is-not-persisted-for-reload","stableKeys":["ecommerce.spec.state-durability.session-reload.1e"]},{"id":"catalog-seeds-the-wrong-air-purifier-price","stableKeys":["ecommerce.feature.catalog.catalog-values.2a"]},{"id":"catalog-tie-breaks-in-reverse-alphabetical-order--01-catalog-ranking","stableKeys":["ecommerce.feature.catalog.catalog-ranking.2b"]},{"id":"catalog-tie-breaks-in-reverse-alphabetical-order--01-core","stableKeys":["ecommerce.spec.live-state.ranking.2c"]},{"id":"purchase-does-not-update-ranking-count","stableKeys":["ecommerce.spec.live-state.ranking.2c"]},{"id":"signed-out-purchase-bypasses-account-check","stableKeys":["ecommerce.spec.access-control.signed-out-purchase.3a"]},{"id":"buy-now-creates-orders-without-reserving-stock--01-buying","stableKeys":["ecommerce.spec.live-state.purchase-stock.3b"]},{"id":"restock-race-records-wrong-order-total","stableKeys":["ecommerce.spec.concurrency-safety.restock-race.202a"]},{"id":"buy-now-records-the-wrong-order-total","stableKeys":["ecommerce.feature.purchasing.purchase-order.3c"]},{"id":"existing-cart-line-does-not-increment-basic-cart","stableKeys":["ecommerce.feature.cart-checkout.cart.4a"]},{"id":"cart-is-deleted-when-owner-disconnects","stableKeys":["ecommerce.spec.state-durability.cart-reload.4b"]},{"id":"signin-binds-the-second-client-to-a-different-account","stableKeys":["ecommerce.spec.live-state.shared-cart.4c"]},{"id":"checkout-does-not-empty-the-basic-cart","stableKeys":["ecommerce.feature.cart-checkout.cart.4d"]},{"id":"new-review-is-accepted-without-being-stored","stableKeys":["ecommerce.feature.reviews.reviews.6a"]},{"id":"repeat-review-inserts-a-second-row","stableKeys":["ecommerce.spec.transactional-integrity.unique-review.6b"]},{"id":"review-average-counts-rows-instead-of-ratings","stableKeys":["ecommerce.spec.live-state.rating.6c"]},{"id":"every-signed-in-customer-is-treated-as-an-admin","stableKeys":["ecommerce.spec.access-control.warehouse-area-boundary.7a"]},{"id":"warehouse-view-omits-west","stableKeys":["ecommerce.feature.warehouse-admin.warehouse-view.7b"]},{"id":"guest-purchase-falls-back-to-the-admin-account","stableKeys":["ecommerce.spec.access-control.purchase-session.101a"]},{"id":"direct-purchases-are-attributed-to-the-system-account","stableKeys":["ecommerce.spec.access-control.purchase-attribution.102a"]},{"id":"direct-restock-does-not-require-an-admin","stableKeys":["ecommerce.spec.access-control.warehouse-write-boundary.103b"]},{"id":"direct-purchase-ignores-the-stored-price","stableKeys":["ecommerce.spec.transactional-integrity.server-price.104a"]},{"id":"account-state-token-is-not-restored-after-reload","stableKeys":["ecommerce.spec.state-durability.account-state-recovery.105a"]},{"id":"reconnect-discards-the-visible-account-state","stableKeys":["ecommerce.spec.state-durability.account-state-recovery.105b"]},{"id":"order-views-return-every-customers-orders","stableKeys":["ecommerce.spec.access-control.order-ownership.106a"]},{"id":"admin-revenue-double-counts-every-order","stableKeys":["ecommerce.spec.transactional-integrity.books-balance.107a"]},{"id":"purchases-do-not-leave-the-warehouses","stableKeys":["ecommerce.spec.transactional-integrity.books-balance.107b"]},{"id":"review-purchase-eligibility-is-not-checked","stableKeys":["ecommerce.spec.access-control.review-eligibility.108a"]},{"id":"eligible-review-is-accepted-without-being-stored","stableKeys":["ecommerce.spec.access-control.review-eligibility.108b","ecommerce.spec.access-control.review-eligibility.108a"]},{"id":"cart-line-lookup-ignores-cart-ownership","stableKeys":["ecommerce.spec.access-control.cart-boundary.109a"]},{"id":"purchase-does-not-reserve-stock-last-unit","stableKeys":["ecommerce.spec.concurrency-safety.last-unit.201a","ecommerce.spec.concurrency-safety.last-unit.201b","ecommerce.spec.concurrency-safety.last-unit.201c"]},{"id":"existing-cart-line-does-not-increment","stableKeys":["ecommerce.spec.concurrency-safety.duplicate-checkout.203a"]},{"id":"checkout-does-not-empty-cart","stableKeys":["ecommerce.spec.concurrency-safety.duplicate-checkout.203b"]},{"id":"stock-subscription-snapshotted-once","stableKeys":["ecommerce.spec.external-data-sync.external-stock.901a"]},{"id":"stock-view-ignores-update-across-app-server-stop","stableKeys":["ecommerce.spec.external-data-sync.external-stock.901c"]},{"id":"stock-view-keeps-pre-reconnect-snapshot","stableKeys":["ecommerce.spec.external-data-sync.external-stock.901d"]},{"id":"open-review-list-snapshots-on-selection","stableKeys":["ecommerce.spec.live-state.open-list.902a"]},{"id":"open-review-list-renders-each-review-twice","stableKeys":["ecommerce.spec.live-state.open-list.902a"]},{"id":"cancel-does-not-restore-stock-feature","stableKeys":["ecommerce.returns-pricing.cancellation-and-return.3a"]},{"id":"cancellation-accounting-loses-stock-restoration","stableKeys":["ecommerce.returns-pricing.refund-accounting.203a"]},{"id":"cancel-does-not-restore-stock-fresh-client","stableKeys":["ecommerce.inventory-operations.stock-conservation.202b","ecommerce.inventory-operations.stock-conservation.202c"]},{"id":"cancel-restores-stock-but-keeps-pending-status","stableKeys":["ecommerce.returns-pricing.cancellation-and-return.3b"]},{"id":"cancelled-order-remains-in-revenue-feature","stableKeys":["ecommerce.returns-pricing.cancellation-and-return.3a"]},{"id":"cancelled-order-remains-in-revenue-invariant","stableKeys":["ecommerce.returns-pricing.refund-accounting.203a"]},{"id":"operator-authorization-allows-customer-transfer","stableKeys":["ecommerce.operations-access.operator-authorization.201a"]},{"id":"customer-can-ship-order-direct-1-1","stableKeys":["ecommerce.operations-access.operator-authorization.201c"]},{"id":"customer-can-cancel-foreign-order-1-1","stableKeys":["ecommerce.operations-access.order-owner.204a"]},{"id":"ship-acknowledges-without-changing-status","stableKeys":["ecommerce.operations-access.fulfilment-queue.1c"]},{"id":"progression-customer-sees-fulfilment-content","stableKeys":["ecommerce.spec.access-control.fulfilment-area-boundary.1d"]},{"id":"operator-authorization-allows-customer-shipping","stableKeys":["ecommerce.operations-access.operator-authorization.201c"]},{"id":"transfer-debits-source-without-crediting-existing-destination","stableKeys":["ecommerce.inventory-operations.stock-conservation.202a","ecommerce.inventory-operations.warehouse-transfer.2a","ecommerce.operations-access.operator-authorization.201a"]},{"id":"recommendations-ignore-pending-purchases","stableKeys":["ecommerce.inventory-operations.operational-views.5c"]},{"id":"purchases-do-not-affect-best-sellers","stableKeys":["ecommerce.inventory-operations.operational-views.5d"]},{"id":"queue-warehouse-reports-west","stableKeys":["ecommerce.operations-access.fulfilment-queue.1b"]},{"id":"transfer-creates-stock-during-race","stableKeys":["ecommerce.inventory-operations.stock-conservation.202d"]},{"id":"catalog-search-ignores-the-query","stableKeys":["ecommerce.feature.catalog.catalog-search.2d"]},{"id":"admin-total-stock-is-not-rendered","stableKeys":["ecommerce.spec.live-state.warehouse-stock.7c"]},{"id":"customers-can-schedule-restocks","stableKeys":["ecommerce.l3.deferred-access.scheduled-work-access.317a"]},{"id":"scheduled-restock-execution-queue-is-process-local","stableKeys":["ecommerce.l3.deferred-durability.restart-survival.311a"]},{"id":"completed-restock-remains-pending","stableKeys":["ecommerce.l3.deferred-integrity.exactly-once.311a"]},{"id":"pending-restock-timer-is-static","stableKeys":["ecommerce.l3.scheduled-restocks.scheduled-restocks.302a"]},{"id":"due-restock-omits-ledger-entry","stableKeys":["ecommerce.l3.scheduled-restocks.scheduled-restocks.305a"]},{"id":"cancelled-restock-remains-pending","stableKeys":["ecommerce.l3.scheduled-restocks.scheduled-restocks.306a"]},{"id":"restart-restock-runs-early","stableKeys":["ecommerce.l3.server-time.server-time.312a"]},{"id":"catalog-product-is-not-published","stableKeys":["ecommerce.progression.catalog-management.catalog-management.622a","ecommerce.progression.catalog-management.catalog-management.622b"]},{"id":"catalog-variants-are-discarded","stableKeys":["ecommerce.progression.catalog-management.catalog-management.622b"]},{"id":"profile-is-lost-on-fresh-account-login","stableKeys":["ecommerce.spec.state-durability.customer-profile-reload.620a"]},{"id":"customer-profile-view-leaks-another-account","stableKeys":["ecommerce.spec.access-control.customer-profile-privacy.620b"]},{"id":"faceted-search-ignores-category","stableKeys":["ecommerce.progression.faceted-search.faceted-search.401a"]},{"id":"active-search-uses-purchase-ranking","stableKeys":["ecommerce.spec.search-ordering.search-ordering.402b"]},{"id":"faceted-search-next-page-does-not-advance","stableKeys":["ecommerce.progression.faceted-search.faceted-search.402a"]},{"id":"managed-support-leaks-and-accepts-cross-account-replies","stableKeys":["ecommerce.spec.access-control.managed-support-privacy.613b"]},{"id":"managed-support-replies-are-empty","stableKeys":["ecommerce.spec.live-state.managed-support.613a","ecommerce.progression.managed-support.managed-support.613c"]},{"id":"managed-support-live-replies-stay-at-initial-snapshot","stableKeys":["ecommerce.spec.live-state.managed-support.613a"]},{"id":"notification-preferences-are-not-saved","stableKeys":["ecommerce.spec.state-durability.notification-preferences-reload.630a"]},{"id":"notification-preferences-leak-across-accounts","stableKeys":["ecommerce.spec.access-control.notification-preferences-privacy.630b"]},{"id":"customers-can-create-promotions","stableKeys":["ecommerce.spec.access-control.promotion-management-boundary.620b"]},{"id":"promotion-rule-stores-the-wrong-discount","stableKeys":["ecommerce.progression.promotion-rules.promotion-rule-values.620a"]},{"id":"staff-cannot-open-staff-tools","stableKeys":["ecommerce.progression.staff-access.staff-access.601a"]},{"id":"customers-can-open-staff-tools","stableKeys":["ecommerce.spec.access-control.staff-area-boundary.601b"]},{"id":"administrator-role-assignment-is-discarded","stableKeys":["ecommerce.spec.state-durability.staff-role-reload.621a"]},{"id":"staff-can-assign-roles","stableKeys":["ecommerce.spec.access-control.staff-role-boundary.621b","ecommerce.spec.access-control.staff-role-revocation.621d"]},{"id":"stock-alert-delivery-is-suppressed","stableKeys":["ecommerce.progression.stock-alerts.stock-alert-delivery.631c"]},{"id":"stock-alert-is-sent-after-every-restock","stableKeys":["ecommerce.spec.transactional-integrity.stock-alert-deduplication.631a"]},{"id":"stock-alerts-are-visible-to-other-customers","stableKeys":["ecommerce.spec.access-control.stock-alert-privacy.631b"]},{"id":"support-history-is-lost-on-fresh-account-login","stableKeys":["ecommerce.spec.state-durability.support-history-reload.612a"]},{"id":"support-history-leaks-across-customers","stableKeys":["ecommerce.spec.access-control.support-history-privacy.612b","ecommerce.spec.access-control.support-history-logout.612d"]},{"id":"visitor-support-reference-is-hidden","stableKeys":["ecommerce.progression.support-intake.support-intake.610a"]},{"id":"support-assignment-is-discarded","stableKeys":["ecommerce.progression.support-triage.support-assignment.611a"]},{"id":"support-priority-is-discarded","stableKeys":["ecommerce.progression.support-triage.support-priority.611b"]},{"id":"support-status-is-discarded","stableKeys":["ecommerce.progression.support-triage.support-status.611c"]},{"id":"nonpositive-cart-quantity-is-treated-as-removal","stableKeys":["ecommerce.spec.access-control.cart-boundary.109b"]},{"id":"admin-restock-preserves-existing-stock","stableKeys":["ecommerce.spec.live-state.warehouse-stock.7c"]},{"id":"direct-review-access-is-not-checked","stableKeys":["ecommerce.progression.review-access-specifications.review-eligibility-direct.618a"]},{"id":"support-history-rows-are-hidden","stableKeys":["ecommerce.progression.support-history.support-history.612c","ecommerce.spec.state-durability.support-history-reload.612a","ecommerce.spec.access-control.support-history-privacy.612b","ecommerce.spec.access-control.support-history-logout.612d"]},{"id":"authorized-restock-does-not-change-stock","stableKeys":["ecommerce.feature.warehouse-admin.admin-write.103a"]},{"id":"low-stock-threshold-is-two-units","stableKeys":["ecommerce.inventory-operations.operational-views.5e","ecommerce.spec.live-state.inventory-dashboard.5a"]},{"id":"category-totals-count-only-since-the-dashboard-opened","stableKeys":["ecommerce.inventory-operations.operational-views.5f"]},{"id":"profile-summary-ignores-a-profile-saved-this-session","stableKeys":["ecommerce.progression.customer-profile.customer-profile.620c"]},{"id":"stored-support-replies-are-hidden-after-reload","stableKeys":["ecommerce.progression.managed-support.managed-support.613c"]},{"id":"saving-notification-preferences-resets-the-toggles","stableKeys":["ecommerce.progression.notification-preferences.notification-preferences.630c"]},{"id":"saving-a-staff-role-snaps-the-input-back-to-the-stored-role","stableKeys":["ecommerce.progression.staff-roles.staff-roles.621c"]},{"id":"fulfilment-queue-is-frozen-at-page-load","stableKeys":["ecommerce.spec.live-state.fulfilment-queue.1a"]},{"id":"low-stock-list-is-frozen-at-page-load","stableKeys":["ecommerce.spec.live-state.inventory-dashboard.5a"]},{"id":"category-totals-are-frozen-at-page-load","stableKeys":["ecommerce.spec.live-state.sales-dashboard.5b"]},{"id":"warehouse-totals-are-frozen-at-page-load","stableKeys":["ecommerce.spec.live-state.stock-transfers.2b"]},{"id":"transfer-skips-the-source-holding-check","stableKeys":["ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c"]},{"id":"support-history-leaks-to-signed-out-visitors","stableKeys":["ecommerce.spec.access-control.support-history-logout.612d"]},{"id":"checkout-crash-integrity","stableKeys":["ecommerce.spec.state-durability.checkout-crash-integrity.910a"]},{"id":"checkout-crash-durability","stableKeys":["ecommerce.spec.state-durability.checkout-crash-durability.910b"]},{"id":"review-script-unsafe-render","stableKeys":["ecommerce.progression.review-access-specifications.stored-review-script.9180a"]},{"id":"review-script-reject-all","stableKeys":["ecommerce.progression.review-access-specifications.stored-review-script.9180a"]}]}],"nullControl":{"pointBearing":"must-fail-conclusively","repetitions":1,"zeroPoint":"typed-policy"},"qualification":{"exactCombinationRequired":true,"referenceRepetitions":1,"mutationRepetitions":1,"checks":["ecommerce.feature.accounts.accounts.1a","ecommerce.feature.accounts.accounts.1b","ecommerce.feature.accounts.accounts.1c","ecommerce.feature.accounts.accounts.1d","ecommerce.feature.cart-checkout.cart.4a","ecommerce.feature.cart-checkout.cart.4d","ecommerce.feature.catalog.catalog-ranking.2b","ecommerce.feature.catalog.catalog-search.2d","ecommerce.feature.catalog.catalog-values.2a","ecommerce.feature.purchasing.purchase-order.3c","ecommerce.feature.reviews.reviews.6a","ecommerce.feature.warehouse-admin.admin-write.103a","ecommerce.feature.warehouse-admin.warehouse-view.7b","ecommerce.inventory-operations.operational-views.5c","ecommerce.inventory-operations.operational-views.5d","ecommerce.inventory-operations.operational-views.5e","ecommerce.inventory-operations.operational-views.5f","ecommerce.inventory-operations.shipping-accounting.202e","ecommerce.inventory-operations.stock-conservation.202a","ecommerce.inventory-operations.stock-conservation.202b","ecommerce.inventory-operations.stock-conservation.202c","ecommerce.inventory-operations.stock-conservation.202d","ecommerce.inventory-operations.warehouse-transfer.2a","ecommerce.l3.deferred-access.scheduled-work-access.317a","ecommerce.l3.deferred-durability.restart-survival.311a","ecommerce.l3.deferred-integrity.exactly-once.311a","ecommerce.l3.scheduled-restocks.scheduled-restocks.302a","ecommerce.l3.scheduled-restocks.scheduled-restocks.305a","ecommerce.l3.scheduled-restocks.scheduled-restocks.306a","ecommerce.l3.server-time.server-time.312a","ecommerce.operations-access.fulfilment-queue.1b","ecommerce.operations-access.fulfilment-queue.1c","ecommerce.operations-access.operator-authorization.201a","ecommerce.operations-access.operator-authorization.201c","ecommerce.operations-access.order-owner.204a","ecommerce.progression.catalog-management.catalog-management.622a","ecommerce.progression.catalog-management.catalog-management.622b","ecommerce.progression.customer-profile.customer-profile.620c","ecommerce.progression.faceted-search.faceted-search.401a","ecommerce.progression.faceted-search.faceted-search.402a","ecommerce.progression.managed-support.managed-support.613c","ecommerce.progression.notification-preferences.notification-preferences.630c","ecommerce.progression.promotion-rules.promotion-rule-values.620a","ecommerce.progression.review-access-specifications.review-eligibility-direct.618a","ecommerce.progression.review-access-specifications.stored-review-script.9180a","ecommerce.progression.staff-access.staff-access.601a","ecommerce.progression.staff-roles.staff-roles.621c","ecommerce.progression.stock-alerts.stock-alert-delivery.631c","ecommerce.progression.support-history.support-history.612c","ecommerce.progression.support-intake.support-intake.610a","ecommerce.progression.support-triage.support-assignment.611a","ecommerce.progression.support-triage.support-priority.611b","ecommerce.progression.support-triage.support-status.611c","ecommerce.returns-pricing.cancellation-and-return.3a","ecommerce.returns-pricing.cancellation-and-return.3b","ecommerce.returns-pricing.refund-accounting.203a","ecommerce.spec.access-control.cart-boundary.109a","ecommerce.spec.access-control.cart-boundary.109b","ecommerce.spec.access-control.customer-profile-privacy.620b","ecommerce.spec.access-control.fulfilment-area-boundary.1d","ecommerce.spec.access-control.managed-support-privacy.613b","ecommerce.spec.access-control.notification-preferences-privacy.630b","ecommerce.spec.access-control.order-ownership.106a","ecommerce.spec.access-control.promotion-management-boundary.620b","ecommerce.spec.access-control.purchase-attribution.102a","ecommerce.spec.access-control.purchase-session.101a","ecommerce.spec.access-control.review-eligibility.108a","ecommerce.spec.access-control.review-eligibility.108b","ecommerce.spec.access-control.signed-out-purchase.3a","ecommerce.spec.access-control.staff-area-boundary.601b","ecommerce.spec.access-control.staff-role-boundary.621b","ecommerce.spec.access-control.staff-role-revocation.621d","ecommerce.spec.access-control.stock-alert-privacy.631b","ecommerce.spec.access-control.support-history-logout.612d","ecommerce.spec.access-control.support-history-privacy.612b","ecommerce.spec.access-control.warehouse-area-boundary.7a","ecommerce.spec.access-control.warehouse-write-boundary.103b","ecommerce.spec.concurrency-safety.duplicate-checkout.203a","ecommerce.spec.concurrency-safety.duplicate-checkout.203b","ecommerce.spec.concurrency-safety.last-unit.201a","ecommerce.spec.concurrency-safety.last-unit.201b","ecommerce.spec.concurrency-safety.last-unit.201c","ecommerce.spec.concurrency-safety.restock-race.202a","ecommerce.spec.external-data-sync.external-stock.901a","ecommerce.spec.external-data-sync.external-stock.901c","ecommerce.spec.external-data-sync.external-stock.901d","ecommerce.spec.live-state.fulfilment-queue.1a","ecommerce.spec.live-state.inventory-dashboard.5a","ecommerce.spec.live-state.managed-support.613a","ecommerce.spec.live-state.open-list.902a","ecommerce.spec.live-state.purchase-stock.3b","ecommerce.spec.live-state.ranking.2c","ecommerce.spec.live-state.rating.6c","ecommerce.spec.live-state.sales-dashboard.5b","ecommerce.spec.live-state.shared-cart.4c","ecommerce.spec.live-state.stock-transfers.2b","ecommerce.spec.live-state.warehouse-stock.7c","ecommerce.spec.search-ordering.search-ordering.402b","ecommerce.spec.state-durability.account-state-recovery.105a","ecommerce.spec.state-durability.account-state-recovery.105b","ecommerce.spec.state-durability.cart-reload.4b","ecommerce.spec.state-durability.checkout-crash-durability.910b","ecommerce.spec.state-durability.checkout-crash-integrity.910a","ecommerce.spec.state-durability.customer-profile-reload.620a","ecommerce.spec.state-durability.notification-preferences-reload.630a","ecommerce.spec.state-durability.session-reload.1e","ecommerce.spec.state-durability.staff-role-reload.621a","ecommerce.spec.state-durability.support-history-reload.612a","ecommerce.spec.transactional-integrity.books-balance.107a","ecommerce.spec.transactional-integrity.books-balance.107b","ecommerce.spec.transactional-integrity.server-price.104a","ecommerce.spec.transactional-integrity.stock-alert-deduplication.631a","ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c","ecommerce.spec.transactional-integrity.unique-review.6b"],"runner":{"schemaVersion":1,"mode":"appliance","platform":"linux","architecture":"x64"},"stacks":["mongodb","postgres","spacetime"],"evidence":[{"kind":"null","repetition":1,"path":"qualification-evidence/ecommerce-l3-7cd96d01b/null.json","sha256":"208efdd6b111d7769c6a03d96786057e4042ae1e6531e3d7bfea2b609cb8c46a"},{"kind":"reference","stack":"postgres","repetition":1,"path":"qualification-evidence/ecommerce-l3-7cd96d01b/postgres-reference.json","sha256":"baaba1487f36fa51907c8ff4fc866997fc2625097e7fe333b3d2250801ed9baf"},{"kind":"mutation","stack":"postgres","repetition":1,"path":"qualification-evidence/ecommerce-l3-7cd96d01b/postgres-mutation.json","sha256":"37ee3c783660a96f44bc9a9f8c5f58968972d0e1a3f8137779720bbaa4fb0f3b"},{"kind":"reference","stack":"mongodb","repetition":1,"path":"qualification-evidence/ecommerce-l3-7cd96d01b/mongodb-reference.json","sha256":"9cef5bf86b29d0ce519fe5b1e05f8596af65518619974b0d64af174ac567dda4"},{"kind":"mutation","stack":"mongodb","repetition":1,"path":"qualification-evidence/ecommerce-l3-7cd96d01b/mongodb-mutation.json","sha256":"2d81f1ab1d611701e6a0c280e4f3f9368bc81932afe6e95e7a84bbda1a00a613"},{"kind":"reference","stack":"spacetime","repetition":1,"path":"qualification-evidence/ecommerce-l3-7cd96d01b/spacetime-reference.json","sha256":"83989faa0da22b7a2bb13e491476813a40be352d08bd79ac617f10d980b26095"},{"kind":"mutation","stack":"spacetime","repetition":1,"path":"qualification-evidence/ecommerce-l3-7cd96d01b/spacetime-mutation.json","sha256":"d84ef17127281df87dad20fceb4efab78a33a59b374dceb206efb98e571fb1eb"}],"featureCatalog":{"path":"progression/ecommerce.json","contentSha256":"8671f7883c2e5a24474a546ef1517407cd9334258e1a7cfe91e292a9af1952f2","id":"ecommerce.questlines"}},"recipe":{"path":"composition/recipes/progression-catalog.json","id":"ecommerce.progression-catalog","meaningSha256":"9724c6dc245ca2cc7d19c9f17938c6edcf3a0234ffbf84b4a48bc8f627aa9337","executionSha256":"2a95cbdeb0deaca8fc6f07167df3c3dbd36541b845133ba34617fc88d54cbd85","contentSha256":"104589b973c1e7dd1a2410659759989eb0de08b16efeda3131777f82858a0a8b"},"references":{"entries":[{"backend":"mongodb","id":"ecommerce-reference-mongodb","sourceSha256":"0d21bbc4b2768f4077a81d676f0d4c87a65067dda8f2c5ffeb2fa9208381dc7e","targetPath":"reference-apps/ecommerce/mongodb"},{"backend":"postgres","id":"ecommerce-reference-postgres","sourceSha256":"f3906fa01050f00a0ed76c69c256a6b0013bbf72eb72c03aa92d79e47b71c2b8","targetPath":"reference-apps/ecommerce/postgres"},{"backend":"spacetime","id":"ecommerce-reference-spacetime","sourceSha256":"7ba1f548e89f461b95b5caf843b0fbcf2b81278428d02869d7f47e1a199eaf6e","targetPath":"reference-apps/ecommerce/spacetime"}],"registryPath":"reference-apps/registry.json"},"selection":{"alias":"L3","coveredAliases":["L1","L2","L3"],"path":"composition/dependency.json","sha256":"e73002df8a7a42e612f2bd1ee5de1f9db2d6c13cb9c819e405a5f2512eca3048"},"title":"Ecommerce dependency L3 calibration","track":"ecommerce"},"mutations":{"mongodb":{"schemaVersion":3,"fixtureSha256":"0d21bbc4b2768f4077a81d676f0d4c87a65067dda8f2c5ffeb2fa9208381dc7e","backend":"mongodb","track":"ecommerce","note":"Mutation definitions for the MongoDB ecommerce reference.","mutations":[{"id":"recommendation-dismissal-lost-on-restart","scenario":"tracks/ecommerce/scenarios/progression-recommendation-feedback.json","targets":["ecommerce.spec.state-durability.recommendation-feedback-restart.504c"],"desc":"Erase saved recommendation dismissals when the application starts again.","file":"server/src/index.ts","edits":[{"find":" await mongoose.connect(DATABASE_URL);","replace":" await mongoose.connect(DATABASE_URL);\n await Dismissal.deleteMany({});"}]},{"id":"pending-order-item-return-accepted","scenario":"tracks/ecommerce/scenarios/progression-order-return-boundary.json","targets":["ecommerce.returns-pricing.cancellation-and-return.3f"],"desc":"Accept a pending order return and restore its stock before shipment.","file":"server/src/index.ts","edits":[{"find":"if (!['shipped', 'delivered'].includes(value.status)) throw new Error('No shipped order found');","replace":"if (!['pending', 'shipped', 'delivered'].includes(value.status)) throw new Error('No shipped order found');"}]},{"id":"staff-admin-access-survives-role-removal","scenario":"tracks/ecommerce/scenarios/progression-staff-roles.json","targets":["ecommerce.spec.access-control.staff-role-revocation.621d"],"desc":"Keep administrator access after changing the assigned role back to staff.","file":"server/src/progression.ts","edits":[{"find":" target.isAdmin = role === \"admin\";","replace":" target.isAdmin = target.isAdmin || role === \"admin\";"}]},{"id":"shipping-counts-sale-twice","scenario":"tracks/ecommerce/scenarios/progression-shipping-accounting.json","targets":["ecommerce.inventory-operations.shipping-accounting.202e"],"desc":"Shipping succeeds but doubles the completed sale value in authoritative revenue.","file":"server/src/index.ts","edits":[{"find":" order.status = \"shipped\";\n await order.save();","replace":" order.status = \"shipped\";\n order.total *= 2;\n await order.save();"}]},{"id":"signup-does-not-expose-created-account","scenario":"tracks/ecommerce/scenarios/01-account-create.json","targets":["ecommerce.feature.accounts.accounts.1a"],"desc":"Signup succeeds but the client discards the created account identity from its current session view.","file":"client/src/App.tsx","edits":[{"find":" saveSession(data.token, data.user);\n };\n\n const handleSignIn","replace":" saveSession(data.token, { ...data.user, username: \"\" });\n };\n\n const handleSignIn"}]},{"id":"duplicate-signup-reports-success","scenario":"tracks/ecommerce/scenarios/01-account-duplicate.json","targets":["ecommerce.feature.accounts.accounts.1b"],"desc":"A duplicate username is reported as a successful empty signup response instead of a refusal.","file":"server/src/index.ts","edits":[{"find":" if (existing) return res.status(409).json({ error: \"Username is already taken\" });","replace":" if (existing) return res.json({}); // mutant: duplicate signup is falsely accepted"}]},{"id":"signin-skips-password-verification","scenario":"tracks/ecommerce/scenarios/01-account-password.json","targets":["ecommerce.feature.accounts.accounts.1c"],"desc":"Signin accepts an existing account without requiring its password to match.","file":"server/src/index.ts","edits":[{"find":" if (!valid) return res.status(401).json({ error: \"Invalid username or password\" });","replace":" if (false && !valid) return res.status(401).json({ error: \"Invalid username or password\" });"}]},{"id":"signout-keeps-current-account","scenario":"tracks/ecommerce/scenarios/01-account-signout.json","targets":["ecommerce.feature.accounts.accounts.1d"],"desc":"Signout disconnects the token state but leaves the current account and persisted credential in place.","file":"client/src/App.tsx","edits":[{"find":" const handleSignOut = () => {\n clearSession();\n };","replace":" const handleSignOut = () => {\n setToken(null); // mutant: visible and persisted account state is not cleared\n };"}]},{"id":"session-token-not-persisted","scenario":"tracks/ecommerce/scenarios/01-account-reload.json","targets":["ecommerce.spec.state-durability.session-reload.1e"],"desc":"The active session is kept only in React state and is unavailable after a page reload.","file":"client/src/App.tsx","edits":[{"find":" localStorage.setItem(TOKEN_KEY, tok);\n setToken(tok);","replace":" void tok; // mutant: the session token is never persisted\n setToken(tok);"}]},{"id":"purchase-counts-never-affect-ranking","scenario":"tracks/ecommerce/scenarios/01-core.json","targets":["ecommerce.spec.live-state.ranking.2c"],"desc":"The catalogue ranking ignores recorded purchases and therefore never promotes the bought item.","file":"server/src/index.ts","edits":[{"find":" purchaseCount: purchaseMap.get(id) || 0,","replace":" purchaseCount: 0, // mutant: ranking ignores durable purchase counts"}]},{"id":"signed-out-visitor-purchase-is-accepted","scenario":"tracks/ecommerce/scenarios/progression-signed-out-purchase.json","targets":["ecommerce.spec.access-control.signed-out-purchase.3a"],"desc":"The UI exposes purchase controls to visitors and the buy route assigns unauthenticated requests an unverified identity, allowing an actual stock-debiting order.","file":"client/src/App.tsx","edits":[{"find":" const isCustomer = !!currentUser && !currentUser.isAdmin && !currentUser.isStaff;","replace":" const isCustomer = !currentUser?.isAdmin && !currentUser?.isStaff;"},{"file":"server/src/index.ts","find":"app.post(\"/api/items/:id/buy\", requireAuth, async (req, res) => {","replace":"app.post(\"/api/items/:id/buy\", async (req, _res, next) => {\n (req as any).user = await userFromToken(extractToken(req)) || { _id: new Types.ObjectId() };\n next();\n}, async (req, res) => {"}]},{"id":"espresso-stock-row-ignores-live-updates","scenario":"tracks/ecommerce/scenarios/01-buying.json","targets":["ecommerce.spec.live-state.purchase-stock.3b"],"desc":"The live catalogue handler preserves a stale Espresso Machine stock projection while applying all other item updates.","file":"client/src/App.tsx","edits":[{"find":" socket.on(\"items:update\", (data: ItemT[]) => setItems(data));","replace":" socket.on(\"items:update\", (data: ItemT[]) => setItems((previous) => data.map((item) => item.name === \"Espresso Machine\" ? { ...item, stock: previous.find((old) => old.id === item.id)?.stock ?? item.stock } : item)));"}]},{"id":"restock-race-records-wrong-order-total","scenario":"tracks/ecommerce/scenarios/01-restock-race.json","targets":["ecommerce.spec.concurrency-safety.restock-race.202a"],"desc":"Purchases preserve stock and visible order counts but record the wrong booked total. Native mixed-race reconciliation must reject them.","file":"server/src/index.ts","edits":[{"find":" total: item.price,\n });","replace":" total: 0, // mutant: purchase receipt loses the authoritative price\n });"}]},{"id":"purchase-order-uses-zero-price","scenario":"tracks/ecommerce/scenarios/progression-purchasing.json","targets":["ecommerce.feature.purchasing.purchase-order.3c"],"desc":"A direct purchase records the item but stores a zero order total instead of the price paid.","file":"server/src/index.ts","edits":[{"find":" total: item.price,\n });","replace":" total: 0, // mutant: purchase receipt loses the authoritative price\n });"}]},{"id":"reload-hydrates-an-empty-cart","scenario":"tracks/ecommerce/scenarios/01-cart.json","targets":["ecommerce.spec.state-durability.cart-reload.4b"],"desc":"Cart hydration discards the persisted server response after reload.","file":"client/src/App.tsx","edits":[{"find":" const refreshCart = useCallback(async (tok: string) => {\n const data = await apiFetch(\"/api/cart\", tok);\n setCart(data);\n }, []);","replace":" const refreshCart = useCallback(async (tok: string) => {\n await apiFetch(\"/api/cart\", tok);\n setCart({ items: [], total: 0 }); // mutant: persisted cart response is discarded\n }, []);"}]},{"id":"shared-cart-live-events-ignored","scenario":"tracks/ecommerce/scenarios/01-cart.json","targets":["ecommerce.spec.live-state.shared-cart.4c"],"desc":"An already-open second session ignores committed cart update events.","file":"client/src/App.tsx","edits":[{"find":" socket.on(\"cart:update\", (data: CartT) => setCart(data));","replace":" socket.on(\"cart:update\", (data: CartT) => setCart(current => current.items.length === 0 ? current : data)); // mutant: an empty second-session cart ignores its first remote update"}]},{"id":"review-comment-is-not-persisted","scenario":"tracks/ecommerce/scenarios/01-review-visibility.json","targets":["ecommerce.feature.reviews.reviews.6a"],"desc":"Review submission persists an empty comment rather than the customer's submitted text.","file":"server/src/index.ts","edits":[{"find":" { itemId, userId: user._id, username: user.username, rating: ratingNum,\n comment: typeof comment === \"string\" ? comment : \"\" },","replace":" { itemId, userId: user._id, username: user.username, rating: ratingNum,\n comment: \"\" },"}]},{"id":"repeat-review-uses-a-new-owner-key","scenario":"tracks/ecommerce/scenarios/01-review-uniqueness.json","targets":["ecommerce.spec.transactional-integrity.unique-review.6b"],"desc":"Each review submission is stored under a fresh owner key, bypassing the one-review-per-customer constraint.","file":"server/src/index.ts","edits":[{"find":" { itemId, userId: user._id },\n { itemId, userId: user._id, username: user.username, rating: ratingNum,\n comment: typeof comment === \"string\" ? comment : \"\" },","replace":" { itemId, userId: new Types.ObjectId() },\n { itemId, userId: new Types.ObjectId(), username: user.username, rating: ratingNum,\n comment: typeof comment === \"string\" ? comment : \"\" },"}]},{"id":"live-review-average-uses-an-extra-divisor","scenario":"tracks/ecommerce/scenarios/01-review-rating-live.json","targets":["ecommerce.spec.live-state.rating.6c"],"desc":"The live review event divides the rating sum by one more review than actually exists.","file":"server/src/index.ts","edits":[{"find":"async function broadcastReviews(itemId: string) {\n const reviews = await Review.find({ itemId }).sort({ createdAt: -1 });\n const average = reviews.length ? reviews.reduce((s, r) => s + r.rating, 0) / reviews.length : 0;","replace":"async function broadcastReviews(itemId: string) {\n const reviews = await Review.find({ itemId }).sort({ createdAt: -1 });\n const average = reviews.length ? reviews.reduce((s, r) => s + r.rating, 0) / (reviews.length + 1) : 0;"}]},{"id":"warehouse-view-omits-one-location","scenario":"tracks/ecommerce/scenarios/01-warehouse-admin-staff.json","targets":["ecommerce.feature.warehouse-admin.warehouse-view.7b"],"desc":"The admin warehouse projection truncates the final item-location row.","file":"client/src/App.tsx","edits":[{"find":" {overview.locations.map((loc) => (","replace":" {overview.locations.slice(0, -1).map((loc) => ("}]},{"id":"unauthenticated-purchase-defaults-to-admin","scenario":"tracks/ecommerce/scenarios/01-purchase-session.json","targets":["ecommerce.spec.access-control.purchase-session.101a"],"desc":"The purchase endpoint drops authentication and assigns sessionless purchases to the seeded administrator.","file":"server/src/index.ts","edits":[{"find":"app.post(\"/api/items/:id/buy\", requireAuth, async (req, res) => {","replace":"app.post(\"/api/items/:id/buy\", async (req, res) => {"},{"find":" const user = (req as any).user;\n const order = await Order.create({","replace":" const user = (req as any).user || await User.findOne({ username: \"admin\" });\n const order = await Order.create({"}]},{"id":"direct-purchase-total-ignores-store-price","scenario":"tracks/ecommerce/scenarios/01-server-price.json","targets":["ecommerce.spec.transactional-integrity.server-price.104a"],"desc":"The direct purchase creates one order but records a zero total rather than the store's current price.","file":"server/src/index.ts","edits":[{"find":" total: item.price,\n });","replace":" total: 0, // mutant: direct purchase ignores the authoritative price\n });"}]},{"id":"cart-hydration-loses-account-state","scenario":"tracks/ecommerce/scenarios/progression-account-state-reload.json","targets":["ecommerce.spec.state-durability.account-state-recovery.105a"],"desc":"Reload hydration discards the account's persisted cart response.","file":"client/src/App.tsx","edits":[{"find":" const refreshCart = useCallback(async (tok: string) => {\n const data = await apiFetch(\"/api/cart\", tok);\n setCart(data);\n }, []);","replace":" const refreshCart = useCallback(async (tok: string) => {\n await apiFetch(\"/api/cart\", tok);\n setCart({ items: [], total: 0 }); // mutant: account state is discarded on hydration\n }, []);"}]},{"id":"reconnect-hydration-loses-account-state","scenario":"tracks/ecommerce/scenarios/progression-account-state-reconnect.json","targets":["ecommerce.spec.state-durability.account-state-recovery.105b"],"desc":"The initial account cart loads correctly, but after network restoration the client ignores both refreshed and pushed cart state.","file":"client/src/App.tsx","edits":[{"find":" useEffect(() => {\n const socket = io({ auth: token ? { token } : {} });","replace":" useEffect(() => {\n const clearAccountOffline = () => {\n setCurrentUser(null);\n setCart({ items: [], total: 0 });\n };\n window.addEventListener(\"offline\", clearAccountOffline, { once: true });\n const socket = io({ auth: token ? { token } : {} });"}]},{"id":"order-history-is-not-owner-scoped","scenario":"tracks/ecommerce/scenarios/01-order-ownership.json","targets":["ecommerce.spec.access-control.order-ownership.106a"],"desc":"Order history returns every customer's orders instead of filtering by the authenticated owner.","file":"server/src/index.ts","edits":[{"find":" const orders = await Order.find({ userId }).sort({ createdAt: -1 });","replace":" const orders = await Order.find({}).sort({ createdAt: -1 });"}]},{"id":"revenue-aggregation-ignores-order-totals","scenario":"tracks/ecommerce/scenarios/progression-books-balance.json","targets":["ecommerce.spec.transactional-integrity.books-balance.107a"],"desc":"The admin revenue aggregation counts every order as zero regardless of its stored total.","file":"server/src/index.ts","edits":[{"find":" { $group: { _id: null, total: { $sum: { $subtract: [\"$total\", { $ifNull: [\"$refundTotal\", 0] }] } } } },","replace":" { $group: { _id: null, total: { $sum: 0 } } },"}]},{"id":"unpurchased-review-is-accepted","scenario":"tracks/ecommerce/scenarios/01-review-eligibility.json","targets":["ecommerce.spec.access-control.review-eligibility.108a"],"desc":"The review endpoint bypasses its completed-purchase eligibility check.","file":"server/src/index.ts","edits":[{"find":" if (!hasPurchased) {\n return res.status(403).json({ error: \"You can only review items you have purchased\" });\n }","replace":" if (false && !hasPurchased) {\n return res.status(403).json({ error: \"You can only review items you have purchased\" });\n }"}]},{"id":"purchased-review-ui-does-not-submit","scenario":"tracks/ecommerce/scenarios/01-review-eligibility.json","targets":["ecommerce.spec.access-control.review-eligibility.108b"],"desc":"The review form discards valid customer submissions. Direct authorized and unauthorized review calls retain their normal behavior.","file":"client/src/App.tsx","edits":[{"find":" const handleReviewSubmit = async (itemId: string, rating: number, comment: string) => {\n setReviewError(\"\");","replace":" const handleReviewSubmit = async (itemId: string, rating: number, comment: string) => {\n setReviewError(\"Review submission unavailable\");\n return;"}]},{"id":"external-stock-polling-disabled","scenario":"tracks/ecommerce/scenarios/01-external-live-sync.json","targets":["ecommerce.spec.external-data-sync.external-stock.901a"],"desc":"The server stops reconciling direct database stock writes into live catalogue events.","file":"server/src/index.ts","edits":[{"find":" broadcastItems().catch((err) => console.error(\"broadcastItems poll failed\", err));","replace":" // mutant: direct database stock changes are never reconciled"}]},{"id":"server-restart-disables-catalog-recovery","scenario":"tracks/ecommerce/scenarios/01-external-server-restart-sync.json","targets":["ecommerce.spec.external-data-sync.external-stock.901c"],"desc":"After a socket disconnect, the existing page ignores both reconnect refreshes and later catalogue snapshots.","file":"client/src/App.tsx","edits":[{"find":" const socketRef = useRef<Socket | null>(null);\n\n const saveSession","replace":" const socketRef = useRef<Socket | null>(null);\n const acceptCatalogRecovery = useRef(true);\n\n const saveSession"},{"find":" socket.on(\"connect\", () => {\n refreshItems().catch((err) => console.error(err));","replace":" socket.on(\"disconnect\", () => { acceptCatalogRecovery.current = false; });\n socket.on(\"connect\", () => {\n if (acceptCatalogRecovery.current) refreshItems().catch((err) => console.error(err));"},{"find":" socket.on(\"items:update\", (data: ItemT[]) => setItems(data));","replace":" socket.on(\"items:update\", (data: ItemT[]) => { if (acceptCatalogRecovery.current) setItems(data); });"}]},{"id":"reconnect-generation-ignores-current-catalog","scenario":"tracks/ecommerce/scenarios/01-external-reconnect-sync.json","targets":["ecommerce.spec.external-data-sync.external-stock.901d"],"desc":"After the browser goes offline, the existing page ignores reconnect refreshes and subsequent catalogue events.","file":"client/src/App.tsx","edits":[{"find":" const socketRef = useRef<Socket | null>(null);\n\n const saveSession","replace":" const socketRef = useRef<Socket | null>(null);\n const acceptCatalogUpdates = useRef(true);\n useEffect(() => {\n const stopCatalogRecovery = () => { acceptCatalogUpdates.current = false; };\n window.addEventListener(\"offline\", stopCatalogRecovery);\n return () => window.removeEventListener(\"offline\", stopCatalogRecovery);\n }, []);\n\n const saveSession"},{"find":" setItems(data.items);","replace":" if (acceptCatalogUpdates.current) setItems(data.items);"},{"find":" socket.on(\"items:update\", (data: ItemT[]) => setItems(data));","replace":" socket.on(\"items:update\", (data: ItemT[]) => {\n if (acceptCatalogUpdates.current) setItems(data);\n });"}]},{"id":"open-review-list-ignores-live-update","scenario":"tracks/ecommerce/scenarios/progression-open-list-live.json","targets":["ecommerce.spec.live-state.open-list.902a"],"desc":"The already-open review list ignores a committed review update from another client.","file":"client/src/App.tsx","edits":[{"find":" setItemDetail((prev) => (prev && prev.id === payload.itemId ? { ...prev, reviews: payload.reviews, average: payload.average } : prev));","replace":" void payload; // mutant: the already-open review list ignores committed updates"}]},{"id":"cancel-does-not-restore-stock-feature","scenario":"tracks/ecommerce/scenarios/02-order-cancellation-core.json","targets":["ecommerce.returns-pricing.cancellation-and-return.3a"],"desc":"Cancellation changes order state but skips restoration of its recorded warehouse allocations.","file":"server/src/index.ts","edits":[{"find":" {item_id:line.itemId,warehouse_id:allocation.warehouseId}, {$inc:{quantity:allocation.quantity}}, {session});","replace":" {item_id:line.itemId,warehouse_id:allocation.warehouseId}, {$inc:{quantity:0}}, {session});"}]},{"id":"cancellation-accounting-loses-stock-restoration","scenario":"tracks/ecommerce/scenarios/02-invariants.json","targets":["ecommerce.returns-pricing.refund-accounting.203a"],"desc":"Cancellation removes revenue and changes order status, but loses the original warehouse stock restoration. The native refund-accounting assertion must detect this.","file":"server/src/index.ts","edits":[{"find":" {item_id:line.itemId,warehouse_id:allocation.warehouseId}, {$inc:{quantity:allocation.quantity}}, {session});","replace":" {item_id:line.itemId,warehouse_id:allocation.warehouseId}, {$inc:{quantity:0}}, {session});"}]},{"id":"cancel-does-not-restore-stock-fresh-client","scenario":"tracks/ecommerce/scenarios/02-self-contained.json","targets":["ecommerce.inventory-operations.stock-conservation.202b","ecommerce.inventory-operations.stock-conservation.202c"],"desc":"Cancellation changes order state but skips restoration, so a fresh client reads the persisted shortfall.","file":"server/src/index.ts","edits":[{"find":" {item_id:line.itemId,warehouse_id:allocation.warehouseId}, {$inc:{quantity:allocation.quantity}}, {session});","replace":" {item_id:line.itemId,warehouse_id:allocation.warehouseId}, {$inc:{quantity:0}}, {session});"}]},{"id":"cancel-restores-stock-but-keeps-pending-status","scenario":"tracks/ecommerce/scenarios/02-order-cancellation-history.json","targets":["ecommerce.returns-pricing.cancellation-and-return.3b"],"desc":"Cancellation restores allocations but writes pending back to order history.","file":"server/src/index.ts","edits":[{"find":" value.status = \"cancelled\";\n await value.save({session});","replace":" value.status = \"pending\";\n await value.save({session});"}]},{"id":"cancelled-order-remains-in-revenue-feature","scenario":"tracks/ecommerce/scenarios/02-order-cancellation-core.json","targets":["ecommerce.returns-pricing.cancellation-and-return.3a"],"desc":"Admin revenue includes cancelled orders even though cancellation otherwise succeeds.","file":"server/src/index.ts","edits":[{"find":"async function getRevenue(): Promise<number> {\n const rows = await Order.aggregate([\n { $match: { status: { $ne: \"cancelled\" } } },","replace":"async function getRevenue(): Promise<number> {\n const rows = await Order.aggregate([\n { $match: {} },"}]},{"id":"cancelled-order-remains-in-revenue-invariant","scenario":"tracks/ecommerce/scenarios/02-invariants.json","targets":["ecommerce.returns-pricing.refund-accounting.203a"],"desc":"Admin revenue includes cancelled orders even though cancellation otherwise succeeds.","file":"server/src/index.ts","edits":[{"find":"async function getRevenue(): Promise<number> {\n const rows = await Order.aggregate([\n { $match: { status: { $ne: \"cancelled\" } } },","replace":"async function getRevenue(): Promise<number> {\n const rows = await Order.aggregate([\n { $match: {} },"}]},{"id":"operator-authorization-allows-customer-transfer","scenario":"tracks/ecommerce/scenarios/02-strengthened.json","targets":["ecommerce.operations-access.operator-authorization.201a"],"desc":"The transfer route keeps authentication but drops its administrator role gate.","file":"server/src/index.ts","edits":[{"find":"app.post(\"/api/admin/transfer\", requireAuth, requireAdmin, async (req, res) => {","replace":"app.post(\"/api/admin/transfer\", requireAuth, async (req, res) => {"}]},{"id":"customer-can-ship-order-direct-1-1","scenario":"tracks/ecommerce/scenarios/02-server-actions.json","targets":["ecommerce.operations-access.operator-authorization.201c"],"desc":"The shipping route keeps authentication but drops its staff role gate.","file":"server/src/index.ts","edits":[{"find":"app.post(\"/api/fulfilment/ship\", requireAuth, requireStaff, async (req, res) => {","replace":"app.post(\"/api/fulfilment/ship\", requireAuth, async (req, res) => {"}]},{"id":"customer-can-cancel-foreign-order-1-1","scenario":"tracks/ecommerce/scenarios/02-server-actions.json","targets":["ecommerce.operations-access.order-owner.204a"],"desc":"Cancellation retains authentication and pending-state validation but drops order ownership.","file":"server/src/index.ts","edits":[{"find":" const value = orderId ? await Order.findOne({ _id: orderId, userId: user._id }).session(session) : null;\n if (!value) return null;\n if (value.status !== \"pending\") throw new Error(\"Order cannot be cancelled\");","replace":" const value = orderId ? await Order.findOne({ _id: orderId }).session(session) : null;\n if (!value) return null;\n if (value.status !== \"pending\") throw new Error(\"Order cannot be cancelled\");"}]},{"id":"queue-depth-lags-one-order","scenario":"tracks/ecommerce/scenarios/02-fulfilment-live.json","targets":["ecommerce.operations-access.fulfilment-queue.1a"],"desc":"The queue renders every order but its visible depth remains one behind.","file":"client/src/App.tsx","edits":[{"find":"Orders waiting: <span className=\"value\" data-role=\"queue-depth\">{queue.depth}</span>","replace":"Orders waiting: <span className=\"value\" data-role=\"queue-depth\">{Math.max(0, queue.depth - 1)}</span>"}]},{"id":"ship-acknowledges-without-changing-status","scenario":"tracks/ecommerce/scenarios/02-fulfilment-ship.json","targets":["ecommerce.operations-access.fulfilment-queue.1c"],"desc":"Shipping returns success but writes pending back to the order, leaving both live views unchanged.","file":"server/src/index.ts","edits":[{"find":" order.status = \"shipped\";\n await order.save();","replace":" order.status = \"pending\";\n await order.save();"}]},{"id":"customer-sees-fulfilment-navigation","scenario":"tracks/ecommerce/scenarios/02-features.json","targets":["ecommerce.operations-access.fulfilment-queue.1d"],"desc":"Expose the protected staff area to signed-in customers, including its navigation and content.","file":"client/src/App.tsx","edits":[{"find":" {(currentUser?.isStaff || currentUser?.isAdmin) && (\n <button className=\"btn btn-ghost\" data-role=\"staff-link\"","replace":" {currentUser && (\n <button className=\"btn btn-ghost\" data-role=\"staff-link\""},{"find":"{fulfilmentOpen && (currentUser?.isStaff || currentUser?.isAdmin) && (","replace":"{fulfilmentOpen && currentUser && ("}]},{"id":"progression-customer-sees-fulfilment-content","scenario":"tracks/ecommerce/scenarios/02-fulfilment-access.json","targets":["ecommerce.spec.access-control.fulfilment-area-boundary.1d"],"desc":"Expose the protected staff area to signed-in customers, including its navigation and content.","file":"client/src/App.tsx","edits":[{"find":" {(currentUser?.isStaff || currentUser?.isAdmin) && (\n <button className=\"btn btn-ghost\" data-role=\"staff-link\"","replace":" {currentUser && (\n <button className=\"btn btn-ghost\" data-role=\"staff-link\""},{"find":"{fulfilmentOpen && (currentUser?.isStaff || currentUser?.isAdmin) && (","replace":"{fulfilmentOpen && currentUser && ("}]},{"id":"transfer-debits-source-without-crediting-existing-destination","scenario":"tracks/ecommerce/scenarios/02-strengthened.json","targets":["ecommerce.inventory-operations.stock-conservation.202a","ecommerce.inventory-operations.warehouse-transfer.2a","ecommerce.operations-access.operator-authorization.201a"],"desc":"A transfer debits the source but credits zero to the destination. This breaks both movement/conservation assertions and the authorized-transfer positive observation inside the authorization check; that last failure is not evidence of an unauthorized transfer.","file":"server/src/index.ts","edits":[{"find":" await Stock.findOneAndUpdate(\n { item_id: itemId, warehouse_id: toWarehouseId },\n { $inc: { quantity: qty } },\n { upsert: true }\n );","replace":" await Stock.findOneAndUpdate(\n { item_id: itemId, warehouse_id: toWarehouseId },\n { $inc: { quantity: 0 } },\n { upsert: true }\n );"}]},{"id":"transfer-warehouse-totals-omit-destination-credit","scenario":"tracks/ecommerce/scenarios/02-transfer-totals.json","targets":["ecommerce.inventory-operations.warehouse-transfer.2b"],"desc":"A transfer debits the source but adds zero to the existing destination, so the two live warehouse totals do not move in opposite directions.","file":"server/src/index.ts","edits":[{"find":" await Stock.findOneAndUpdate(\n { item_id: itemId, warehouse_id: toWarehouseId },\n { $inc: { quantity: qty } },\n { upsert: true }\n );","replace":" await Stock.findOneAndUpdate(\n { item_id: itemId, warehouse_id: toWarehouseId },\n { $inc: { quantity: 0 } },\n { upsert: true }\n );"}]},{"id":"transfer-overdraft-guard-removed","scenario":"tracks/ecommerce/scenarios/02-transfer-overdraw.json","targets":["ecommerce.inventory-operations.warehouse-transfer.2c"],"desc":"The atomic source debit no longer requires sufficient quantity, allowing an overdrawn transfer to succeed and change both rows.","file":"server/src/index.ts","edits":[{"find":"{ item_id: itemId, warehouse_id: fromWarehouseId, quantity: { $gte: qty } }","replace":"{ item_id: itemId, warehouse_id: fromWarehouseId }"}]},{"id":"low-stock-excludes-boundary-ten","scenario":"tracks/ecommerce/scenarios/02-low-stock.json","targets":["ecommerce.inventory-operations.operational-views.5a"],"desc":"The low-stock view uses a strict boundary and omits items with exactly ten units.","file":"server/src/index.ts","edits":[{"find":" .filter((it) => it.stock <= 10)","replace":" .filter((it) => it.stock < 10)"}]},{"id":"category-totals-ignore-pending-purchases","scenario":"tracks/ecommerce/scenarios/02-operational-category-totals.json","targets":["ecommerce.inventory-operations.operational-views.5b"],"desc":"Category totals include only shipped orders, so a newly accepted pending purchase is absent from the live totals.","file":"server/src/index.ts","edits":[{"find":"async function getCategoryTotals() {\n const rows = await Order.aggregate([\n { $match: { status: { $ne: \"cancelled\" } } },","replace":"async function getCategoryTotals() {\n const rows = await Order.aggregate([\n { $match: { status: \"shipped\" } },"}]},{"id":"recommendations-ignore-pending-purchases","scenario":"tracks/ecommerce/scenarios/02-operational-recommendations.json","targets":["ecommerce.inventory-operations.operational-views.5c"],"desc":"Personal recommendations derive categories only from shipped orders, so a newly accepted pending purchase has no influence.","file":"server/src/index.ts","edits":[{"find":" { $match: { userId: new Types.ObjectId(userId), status: { $ne: \"cancelled\" } } },","replace":" { $match: { userId: new Types.ObjectId(userId), status: \"shipped\" } },"}]},{"id":"purchases-do-not-affect-best-sellers","scenario":"tracks/ecommerce/scenarios/02-operational-best-sellers.json","targets":["ecommerce.inventory-operations.operational-views.5d"],"desc":"Rank signed-out recommendations without purchase counts.","file":"server/src/index.ts","edits":[{"find":"async function getPurchaseCounts(): Promise<Map<string, number>> {\n const rows = await Order.aggregate([\n { $match: { status: { $ne: \"cancelled\" } } },\n { $unwind: \"$items\" },\n { $match: { \"items.returned\": { $ne: true } } },\n { $group: { _id: \"$items.itemId\", total: { $sum: \"$items.quantity\" } } },\n ]);\n const map = new Map<string, number>();\n for (const row of rows) map.set(row._id.toString(), row.total);\n return map;\n}","replace":"async function getPurchaseCounts(): Promise<Map<string, number>> {\n return new Map<string, number>();\n}"}]},{"id":"queue-warehouse-reports-west","scenario":"tracks/ecommerce/scenarios/02-queue-warehouse.json","targets":["ecommerce.operations-access.fulfilment-queue.1b"],"desc":"The queue renders West for the deterministic Desk Lamp allocation even though the order reserved stock in East.","file":"client/src/App.tsx","edits":[{"find":"{(l.warehouseNames || []).join(\", \") || \"Unknown\"}","replace":"{\"West\"}"}]},{"id":"cart-repeat-does-not-increment","scenario":"tracks/ecommerce/scenarios/progression-cart-checkout.json","targets":["ecommerce.feature.cart-checkout.cart.4a"],"desc":"An existing cart line remains at its old quantity.","file":"server/src/index.ts","edits":[{"find":"{ $inc: { \"items.$.quantity\": qty },","replace":"{ $inc: { \"items.$.quantity\": 0 },"}]},{"id":"checkout-leaves-cart-claimed","scenario":"tracks/ecommerce/scenarios/progression-cart-checkout.json","targets":["ecommerce.feature.cart-checkout.cart.4d"],"desc":"Checkout creates an order without emptying the cart.","file":"server/src/credit.ts","edits":[{"find":" cart.items = cart.items.filter(line => line.reservationExpiresAt && line.reservationExpiresAt <= now) as any;","replace":" // mutant: purchased cart lines remain"}]},{"id":"catalog-initial-ranking-is-reversed","scenario":"tracks/ecommerce/scenarios/01-catalog-ranking.json","targets":["ecommerce.feature.catalog.catalog-ranking.2b"],"desc":"The initial catalog uses reverse alphabetical order.","file":"client/src/App.tsx","edits":[{"find":".sort((a, b) => (filtering ? 0 : b.purchaseCount - a.purchaseCount) || a.name.localeCompare(b.name))","replace":".sort((a, b) => (filtering ? 0 : b.purchaseCount - a.purchaseCount) || b.name.localeCompare(a.name))"}]},{"id":"catalog-search-requires-exact-name","scenario":"tracks/ecommerce/scenarios/01-catalog-search.json","targets":["ecommerce.feature.catalog.catalog-search.2d"],"desc":"Partial catalog search is replaced with exact matching.","file":"client/src/App.tsx","edits":[{"find":"(!q || it.name.toLowerCase().includes(q))","replace":"(!q || it.name.toLowerCase() === q)"}]},{"id":"catalog-price-is-offset","scenario":"tracks/ecommerce/scenarios/01-catalog-values.json","targets":["ecommerce.feature.catalog.catalog-values.2a"],"desc":"Public catalog prices are reported one unit too high.","file":"server/src/index.ts","edits":[{"find":" ...json,\n id,\n stock: stockMap.get(id) || 0,","replace":" ...json,\n id,\n price: json.price + 1,\n stock: stockMap.get(id) || 0,"}]},{"id":"staff-can-see-admin-navigation","scenario":"tracks/ecommerce/scenarios/01-warehouse-admin-staff.json","targets":["ecommerce.spec.access-control.warehouse-area-boundary.7a"],"desc":"Expose administrator identity and protected inventory data to non-admin accounts.","file":"server/src/index.ts","edits":[{"find":" if (!user || !user.isAdmin) return res.status(403).json({ error: \"Admin access required\" });","replace":" if (!user) return res.status(403).json({ error: \"Admin access required\" });"},{"find":"username: user.username, isAdmin: user.isAdmin,","replace":"username: user.username, isAdmin: true,"}]},{"id":"staff-can-use-direct-restock","scenario":"tracks/ecommerce/scenarios/01-admin-write-staff.json","targets":["ecommerce.spec.access-control.warehouse-write-boundary.103b"],"desc":"The direct restock route accepts staff instead of administrators only.","file":"server/src/index.ts","edits":[{"find":"app.post(\"/api/admin/restock\", requireAuth, requireAdmin, async (req, res) => {","replace":"app.post(\"/api/admin/restock\", requireAuth, requireStaff, async (req, res) => {"}]},{"id":"restock-adds-the-wrong-quantity","scenario":"tracks/ecommerce/scenarios/01-warehouse-stock-live-staff.json","targets":["ecommerce.spec.live-state.warehouse-stock.7c"],"desc":"A direct restock stores one fewer unit than requested.","file":"server/src/index.ts","edits":[{"find":" await Stock.findOneAndUpdate(\n { item_id: item._id, warehouse_id: warehouse._id },\n { $inc: { quantity: qty } },\n { upsert: true }\n );","replace":" await Stock.findOneAndUpdate(\n { item_id: item._id, warehouse_id: warehouse._id },\n { $inc: { quantity: qty - 1 } },\n { upsert: true }\n );"}]},{"id":"transfer-creates-stock-during-race","scenario":"tracks/ecommerce/scenarios/02-server-actions.json","targets":["ecommerce.inventory-operations.stock-conservation.202d"],"desc":"A transfer creates one extra unit while a purchase runs.","file":"server/src/index.ts","edits":[{"find":" { $inc: { quantity: qty } },\n { upsert: true }\n );\n\n await Promise.all([broadcastItems(), broadcastAdmin(), broadcastRecommendedForAll()]);","replace":" { $inc: { quantity: qty + 1 } },\n { upsert: true }\n );\n\n await Promise.all([broadcastItems(), broadcastAdmin(), broadcastRecommendedForAll()]);"}]},{"id":"customer-can-cancel-scheduled-restock","scenario":"tracks/ecommerce/scenarios/03-deferred-access.json","targets":["ecommerce.l3.deferred-access.scheduled-work-access.317a"],"desc":"Scheduled-restock cancellation routes omit their staff checks.","file":"server/src/progression.ts","edits":[{"find":" router.delete(\"/scheduled-restocks/:id\", auth, staff, cancelScheduledRestock);","replace":" router.delete(\"/scheduled-restocks/:id\", auth, cancelScheduledRestock);"},{"find":" adminRouter.delete(\"/scheduled-restocks/:id\", auth, staff, cancelScheduledRestock);","replace":" adminRouter.delete(\"/scheduled-restocks/:id\", auth, cancelScheduledRestock);"}]},{"id":"scheduled-restock-never-becomes-due-after-restart","scenario":"tracks/ecommerce/scenarios/03-deferred-durability.json","targets":["ecommerce.l3.deferred-durability.restart-survival.311a"],"desc":"Application startup discards pending restocks. Ordinary timer execution remains intact; completed and cancelled work is retained.","file":"server/src/index.ts","edits":[{"find":"import { Dismissal, Payment, Promotion } from \"./progression-models.js\";","replace":"import { Dismissal, Payment, Promotion, ScheduledRestock } from \"./progression-models.js\";"},{"find":" await mongoose.connect(DATABASE_URL);","replace":" await mongoose.connect(DATABASE_URL);\n await ScheduledRestock.deleteMany({ status: \"pending\" });"}]},{"id":"reservation-expiry-does-not-release-stock-after-restart","scenario":"tracks/ecommerce/scenarios/03-deferred-durability.json","targets":["ecommerce.l3.deferred-durability.restart-survival.314a"],"desc":"Expired reservations clear their record but do not restore stock.","file":"server/src/progression.ts","edits":[{"find":"if (result.modifiedCount) await releaseStock(line.itemId as Types.ObjectId,\n [...line.reservedWarehouseIds] as Types.ObjectId[]);","replace":"if (result.modifiedCount) await releaseStock(line.itemId as Types.ObjectId, []);"}]},{"id":"completed-restock-is-replayed","scenario":"tracks/ecommerce/scenarios/03-deferred-integrity.json","targets":["ecommerce.l3.deferred-integrity.exactly-once.311a"],"desc":"A completed scheduled restock remains pending and applies repeatedly.","file":"server/src/progression.ts","edits":[{"find":"{ $set: { status: \"applied\" } }, { new: true });","replace":"{ $set: { status: \"pending\" } }, { new: true });"}]},{"id":"expired-reservation-is-released-repeatedly","scenario":"tracks/ecommerce/scenarios/03-deferred-integrity.json","targets":["ecommerce.l3.deferred-integrity.stock-conservation.313a"],"desc":"An expired reservation keeps its allocation and restores it repeatedly.","file":"server/src/progression.ts","edits":[{"find":"} } }, { $set: { \"items.$.reservedWarehouseIds\": [] } }, { timestamps: false });","replace":"} } }, { $set: { \"items.$.reservedWarehouseIds\": line.reservedWarehouseIds } }, { timestamps: false });"}]},{"id":"checkout-decrements-reserved-stock-again","scenario":"tracks/ecommerce/scenarios/03-deferred-integrity.json","targets":["ecommerce.l3.deferred-integrity.stock-conservation.314a"],"desc":"Checkout decrements stock after the reservation already did.","file":"server/src/credit.ts","edits":[{"find":" const [order] = await Order.create([{ userId, items: lines, total: totalMinor / 100, discount,","replace":" await mongoose.connection.collection('stocks').updateOne({ item_id: lines[0].itemId }, { $inc: { quantity: -1 } }, { session });\n const [order] = await Order.create([{ userId, items: lines, total: totalMinor / 100, discount,"}]},{"id":"cart-add-does-not-reserve-stock","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.301a"],"desc":"Adding to the cart records an allocation without reserving stock.","file":"server/src/stock-reservations.ts","edits":[{"find":" { $inc: { quantity: -1 } },","replace":" { $inc: { quantity: 0 } },"}]},{"id":"reservation-timer-does-not-decrease","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.305a"],"desc":"The cart always reports a fresh reservation timer.","file":"server/src/index.ts","edits":[{"find":"? Math.max(0, Math.ceil((line.reservationExpiresAt.getTime() - Date.now()) / 1000)) : 0,","replace":"? 90 : 0,"}]},{"id":"reservation-checkout-does-not-clear-cart","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.306a"],"desc":"Reservation checkout leaves the claimed cart lines in place.","file":"server/src/credit.ts","edits":[{"find":" cart.items = cart.items.filter(line => line.reservationExpiresAt && line.reservationExpiresAt <= now) as any;","replace":" // mutant: purchased cart lines remain"}]},{"id":"expired-reservation-is-not-marked","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.307a"],"desc":"Expired cart lines are never marked as expired.","file":"server/src/index.ts","edits":[{"find":"expired: Boolean(line.reservationExpiresAt && line.reservationExpiresAt.getTime() <= Date.now()),","replace":"expired: false,"}]},{"id":"quantity-change-does-not-renew-reservation","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.308a"],"desc":"Raising cart quantity keeps the old reservation deadline.","file":"server/src/index.ts","edits":[{"find":" $set: { \"items.$.reservationExpiresAt\": new Date(Date.now() + 90_000),\n inactiveExpiresAt: new Date(Date.now() + 300_000) },","replace":" $set: { inactiveExpiresAt: new Date(Date.now() + 300_000) },"}]},{"id":"scheduled-restock-countdown-is-fixed","scenario":"tracks/ecommerce/scenarios/03-scheduled-restocks.json","targets":["ecommerce.l3.scheduled-restocks.scheduled-restocks.302a"],"desc":"The pending restock UI shows a fixed countdown.","file":"client/src/ProgressionPanel.tsx","edits":[{"find":"{Math.max(0, Math.ceil((new Date(entry.dueAt).getTime() - Date.now()) / 1000))}","replace":"{90}"}]},{"id":"due-restock-does-not-change-stock","scenario":"tracks/ecommerce/scenarios/03-scheduled-restock-apply.json","targets":["ecommerce.l3.scheduled-restocks.scheduled-restocks.305a"],"desc":"A due restock is recorded without adding stock.","file":"server/src/progression.ts","edits":[{"find":"{ $inc: { quantity: claimed.quantity } });","replace":"{ $inc: { quantity: 0 } });"}]},{"id":"cancelled-restock-remains-pending","scenario":"tracks/ecommerce/scenarios/03-scheduled-restock-cancel.json","targets":["ecommerce.l3.scheduled-restocks.scheduled-restocks.306a"],"desc":"Cancelling a restock leaves it pending so it later applies.","file":"server/src/progression.ts","edits":[{"find":"{ $set: { status: \"cancelled\" } }, { new: true });","replace":"{ $set: { status: \"pending\" } }, { new: true });"}]},{"id":"server-time-restock-never-completes","scenario":"tracks/ecommerce/scenarios/03-server-time.json","targets":["ecommerce.l3.server-time.server-time.312a"],"desc":"Server time never selects a due restock after restart.","file":"server/src/progression.ts","edits":[{"find":"const due = await ScheduledRestock.find({ status: \"pending\", dueAt: { $lte: now } });","replace":"const due = await ScheduledRestock.find({ status: \"pending\", dueAt: { $lte: new Date(0) } });"}]},{"id":"closed-browser-reservation-never-releases","scenario":"tracks/ecommerce/scenarios/03-server-time.json","targets":["ecommerce.l3.server-time.server-time.313a"],"desc":"A reservation that expires without a browser does not restore stock.","file":"server/src/progression.ts","edits":[{"find":"if (result.modifiedCount) await releaseStock(line.itemId as Types.ObjectId,\n [...line.reservedWarehouseIds] as Types.ObjectId[]);","replace":"if (result.modifiedCount) await releaseStock(line.itemId as Types.ObjectId, []);"}]},{"id":"catalog-product-name-is-not-published","scenario":"tracks/ecommerce/scenarios/progression-catalog-management.json","targets":["ecommerce.progression.catalog-management.catalog-management.622a","ecommerce.progression.catalog-management.catalog-management.622b"],"desc":"The public product card omits the submitted name while retaining its product and variants. Both publication and named variant navigation fail because the latter locates the card by its product name; this does not claim the variants were lost.","file":"client/src/App.tsx","edits":[{"find":" <div className=\"item-name\" data-role=\"item-name\">\n {item.name}\n </div>","replace":" <div className=\"item-name\" data-role=\"item-name\">\n {item.name === \"Travel Mug\" ? \"\" : item.name}\n </div>"}]},{"id":"catalog-variants-are-discarded","scenario":"tracks/ecommerce/scenarios/progression-catalog-management.json","targets":["ecommerce.progression.catalog-management.catalog-management.622b"],"desc":"Catalog management stores the submitted product without its named variants.","file":"server/src/progression.ts","edits":[{"find":"const item = await Item.create({ name, price, category, variants });","replace":"const item = await Item.create({ name, price, category, variants: [] });"}]},{"id":"checkout-records-wrong-payment-amount","scenario":"tracks/ecommerce/scenarios/progression-core-business.json","targets":["ecommerce.progression.payment-records.payment-records.623a"],"desc":"Checkout records a payment above the charged total.","file":"server/src/credit.ts","edits":[{"find":" await Payment.create([{ userId, orderId: order!._id, amount: totalMinor / 100, status: 'paid' }], { session });","replace":" await Payment.create([{ userId, orderId: order!._id, amount: totalMinor / 100 + 1, status: 'paid' }], { session });"}]},{"id":"payment-state-repeats-each-record","scenario":"tracks/ecommerce/scenarios/progression-core-business.json","targets":["ecommerce.spec.transactional-integrity.payment-deduplication.623b"],"desc":"The order response repeats every payment record.","file":"server/src/index.ts","edits":[{"find":"payments: (byOrder.get(String(order._id)) ?? []).map(payment => payment.toJSON()),","replace":"payments: [...(byOrder.get(String(order._id)) ?? []), ...(byOrder.get(String(order._id)) ?? [])].map(payment => payment.toJSON()),"}]},{"id":"profile-data-is-lost-on-server-restart","scenario":"tracks/ecommerce/scenarios/progression-customer-profile.json","targets":["ecommerce.spec.state-durability.customer-profile-reload.620a"],"desc":"Application startup discards saved profiles. Profile creation, reads, and browser reload work before runtime restart.","file":"server/src/index.ts","edits":[{"find":"import { Dismissal, Payment, Promotion } from \"./progression-models.js\";","replace":"import { Dismissal, Payment, Promotion, Profile } from \"./progression-models.js\";"},{"find":" await mongoose.connect(DATABASE_URL);","replace":" await mongoose.connect(DATABASE_URL);\n await Profile.deleteMany({});"}]},{"id":"profile-read-is-not-owner-scoped","scenario":"tracks/ecommerce/scenarios/progression-customer-profile.json","targets":["ecommerce.spec.access-control.customer-profile-privacy.620b"],"desc":"Profile reads return another account profile.","file":"server/src/progression.ts","edits":[{"find":"user ? Profile.findOne({ userId: user._id }) : null,","replace":"user ? Profile.findOne({}) : null,"}]},{"id":"faceted-search-ignores-category","scenario":"tracks/ecommerce/scenarios/progression-faceted-filters.json","targets":["ecommerce.progression.faceted-search.faceted-search.401a"],"desc":"Faceted search ignores the selected category.","file":"client/src/App.tsx","edits":[{"find":"&& (!categoryFilter || it.category === categoryFilter)","replace":"&& (!categoryFilter || true)"}]},{"id":"active-search-uses-purchase-ranking","scenario":"tracks/ecommerce/scenarios/progression-search-ordering.json","targets":["ecommerce.spec.search-ordering.search-ordering.402b"],"desc":"Keep purchase ranking when search text or filters are active.","file":"client/src/App.tsx","edits":[{"find":"const filtering = Boolean(q || categoryFilter || minimumPrice || maximumPrice || inStockOnly);","replace":"const filtering = false;"}]},{"id":"pagination-repeats-first-page","scenario":"tracks/ecommerce/scenarios/progression-faceted-pagination.json","targets":["ecommerce.progression.faceted-search.faceted-search.402a"],"desc":"Every catalog page repeats the first page.","file":"client/src/App.tsx","edits":[{"find":"const searchResults = filteredItems.slice(searchPage * CATALOG_PAGE_SIZE,\n searchPage * CATALOG_PAGE_SIZE + CATALOG_PAGE_SIZE);","replace":"const searchResults = filteredItems.slice(0, CATALOG_PAGE_SIZE);"}]},{"id":"managed-support-live-refresh-keeps-stale-tickets","scenario":"tracks/ecommerce/scenarios/progression-managed-support-shared.json","targets":["ecommerce.spec.live-state.managed-support.613a"],"desc":"Periodic refresh keeps the previous support ticket snapshot. Saved replies remain available on initial load and explicit action refresh, but another open client does not receive updates.","file":"client/src/ProgressionPanel.tsx","edits":[{"find":" const refresh = useCallback(async () => {","replace":" const refresh = useCallback(async (live = false) => {"},{"find":" setState({ ...next, loadedToken: token });","replace":" setState((current: any) => ({ ...next, tickets: live ? current.tickets : next.tickets, loadedToken: token }));"},{"find":" const timer = setInterval(refresh, 1000);","replace":" const timer = setInterval(() => refresh(true), 1000);"}]},{"id":"managed-support-allows-another-customer","scenario":"tracks/ecommerce/scenarios/progression-managed-support-privacy.json","targets":["ecommerce.spec.access-control.managed-support-privacy.613b"],"desc":"Managed support ownership is not checked.","file":"server/src/progression.ts","edits":[{"find":" if (user?.isStaff || user?.isAdmin || (ticket.userId && String(ticket.userId) === String(user?._id))) {\n return ticket;\n }\n return null;","replace":" return ticket;"}]},{"id":"notification-preference-is-not-saved","scenario":"tracks/ecommerce/scenarios/progression-notification-preferences.json","targets":["ecommerce.spec.state-durability.notification-preferences-reload.630a","ecommerce.progression.notification-preferences.notification-preferences.630c"],"desc":"Notification preferences are always saved as disabled, so both the saved-choice view and reload show the wrong value.","file":"server/src/progression.ts","edits":[{"find":"userId: req.progressionUser._id, order: Boolean(req.body?.order), stock: Boolean(req.body?.stock),","replace":"userId: req.progressionUser._id, order: false, stock: false,"}]},{"id":"notification-preference-is-not-owner-scoped","scenario":"tracks/ecommerce/scenarios/progression-notification-preferences.json","targets":["ecommerce.spec.access-control.notification-preferences-privacy.630b"],"desc":"Notification settings read another account preference.","file":"server/src/progression.ts","edits":[{"find":"user ? Preference.findOne({ userId: user._id }) : null,","replace":"user ? Preference.findOne({}) : null,"}]},{"id":"active-promotion-has-zero-discount","scenario":"tracks/ecommerce/scenarios/progression-promotion-checkout.json","targets":["ecommerce.progression.promotion-checkout.promotion-checkout-active.621a"],"desc":"An active promotion is recorded without changing checkout total.","file":"server/src/progression.ts","edits":[{"find":"{ $set: { promotionCode: code, discount: promotion.discount } }, { new: true });","replace":"{ $set: { promotionCode: code, discount: 0 } }, { new: true });"}]},{"id":"unavailable-promotions-are-accepted","scenario":"tracks/ecommerce/scenarios/progression-promotion-checkout.json","targets":["ecommerce.progression.promotion-checkout.promotion-checkout-expired.621b","ecommerce.progression.promotion-checkout.promotion-checkout-exhausted.621c"],"desc":"Expired and exhausted promotions are accepted.","file":"server/src/progression.ts","edits":[{"find":" if (!promotion || promotion.start > now || promotion.end < now\n || promotion.redemptions >= promotion.limit) {","replace":" if (!promotion) {"}]},{"id":"promotion-save-drops-bounded-values","scenario":"tracks/ecommerce/scenarios/progression-promotion-rules.json","targets":["ecommerce.progression.promotion-rules.promotion-rule-values.620a"],"desc":"Promotion creation drops its bounded discount and limit.","file":"server/src/progression.ts","edits":[{"find":"{ code, discount, limit, start, end }, { upsert: true, new: true });","replace":"{ code, discount: 1, limit: 1, start, end }, { upsert: true, new: true });"}]},{"id":"customer-can-create-promotion","scenario":"tracks/ecommerce/scenarios/progression-promotion-rules.json","targets":["ecommerce.spec.access-control.promotion-management-boundary.620b"],"desc":"Promotion creation omits its staff authorization check.","file":"server/src/progression.ts","edits":[{"find":"app.post(\"/api/promotions\", auth, staff, async (req: Request, res) => {","replace":"app.post(\"/api/promotions\", auth, async (req: Request, res) => {"}]},{"id":"staff-signin-loses-staff-role","scenario":"tracks/ecommerce/scenarios/progression-staff-access.json","targets":["ecommerce.progression.staff-access.staff-access.601a"],"desc":"Deny administrators entry to staff tools while keeping ordinary staff access working.","file":"client/src/App.tsx","edits":[{"find":"{(currentUser?.isStaff || currentUser?.isAdmin) && (\n <button className=\"btn btn-ghost\" data-role=\"staff-link\"","replace":"{(currentUser?.isStaff && !currentUser?.isAdmin) && (\n <button className=\"btn btn-ghost\" data-role=\"staff-link\""}]},{"id":"customer-signin-gains-staff-role","scenario":"tracks/ecommerce/scenarios/progression-staff-access.json","targets":["ecommerce.spec.access-control.staff-area-boundary.601b"],"desc":"Signin falsely exposes every customer as staff.","file":"server/src/index.ts","edits":[{"find":" isStaff: user.isStaff, roles: user.roles || [] };","replace":" isStaff: true, roles: user.roles || [] };"}]},{"id":"role-assignment-drops-role","scenario":"tracks/ecommerce/scenarios/progression-staff-roles.json","targets":["ecommerce.spec.state-durability.staff-role-reload.621a"],"desc":"Inventory role assignments are not saved; staff and administrator role transitions still work.","file":"server/src/progression.ts","edits":[{"find":" target.roles = roles;","replace":" if (role !== \"inventory\") target.roles = roles;"}]},{"id":"staff-role-write-precedes-denial","scenario":"tracks/ecommerce/scenarios/progression-staff-roles.json","targets":["ecommerce.spec.access-control.staff-role-boundary.621b","ecommerce.spec.access-control.staff-role-revocation.621d"],"desc":"Save an unauthorized role change before returning a forbidden response, including after administrator access is revoked.","file":"server/src/progression.ts","edits":[{"find":"app.put(\"/api/staff/:id/role\", auth, admin, async (req: Request, res) => {","replace":"app.put(\"/api/staff/:id/role\", auth, staff, async (req: Request, res) => {"},{"find":" await recordActivity(req.progressionUser, \"updated roles\", target.username);","replace":" if (!req.progressionUser.isAdmin) { res.status(403).json({ error: \"Admin access required\" }); return; }\n await recordActivity(req.progressionUser, \"updated roles\", target.username);"}]},{"id":"staff-can-assign-roles","scenario":"tracks/ecommerce/scenarios/progression-staff-roles.json","targets":["ecommerce.spec.access-control.staff-role-boundary.621b","ecommerce.spec.access-control.staff-role-revocation.621d"],"desc":"Staff can call the role assignment endpoint, including after administrator access is revoked.","file":"server/src/progression.ts","edits":[{"find":"app.put(\"/api/staff/:id/role\", auth, admin, async (req: Request, res) => {","replace":"app.put(\"/api/staff/:id/role\", auth, staff, async (req: Request, res) => {"}]},{"id":"stock-alert-delivery-is-suppressed","scenario":"tracks/ecommerce/scenarios/progression-stock-alert-delivery.json","targets":["ecommerce.progression.stock-alerts.stock-alert-delivery.631c"],"desc":"A request is accepted but restored stock does not deliver its alert.","file":"server/src/progression.ts","edits":[{"find":"await Notification.updateOne({ userId: alert.userId, key: `stock:${alert.itemId}` },","replace":"if (false) await Notification.updateOne({ userId: alert.userId, key: `stock:${alert.itemId}` },"}]},{"id":"stock-alert-repeats-while-in-stock","scenario":"tracks/ecommerce/scenarios/progression-stock-alerts.json","targets":["ecommerce.spec.transactional-integrity.stock-alert-deduplication.631a"],"desc":"Keep a delivered stock request active and create a notification with a new key on each timer tick while stock remains available, so its owner receives duplicate alerts.","file":"server/src/progression.ts","edits":[{"find":" alert.sent = true;\n await alert.save();","replace":" alert.sent = false;\n await alert.save();"},{"find":" await Notification.updateOne({ userId: alert.userId, key: `stock:${alert.itemId}` },\n { $setOnInsert: { userId: alert.userId, key: `stock:${alert.itemId}`, type: \"stock\",\n message: `${item?.name || \"Item\"} is back in stock` } }, { upsert: true });","replace":" await Notification.create({ userId: alert.userId, key: `stock:${alert.itemId}:${new mongoose.Types.ObjectId()}`, type: \"stock\",\n message: `${item?.name || \"Item\"} is back in stock` });"}]},{"id":"stock-alerts-are-not-owner-scoped","scenario":"tracks/ecommerce/scenarios/progression-stock-alerts.json","targets":["ecommerce.spec.access-control.stock-alert-privacy.631b"],"desc":"Notification reads expose alerts from other accounts.","file":"server/src/progression.ts","edits":[{"find":"user ? Notification.find({ userId: user._id }).sort({ createdAt: -1 }) : [],","replace":"user ? Notification.find({}).sort({ createdAt: -1 }) : [],"}]},{"id":"support-history-is-lost-on-server-restart","scenario":"tracks/ecommerce/scenarios/progression-support-history.json","targets":["ecommerce.spec.state-durability.support-history-reload.612a"],"desc":"Application startup discards support tickets. Ticket creation and owner reads work before runtime restart; privacy uses independently created history.","file":"server/src/index.ts","edits":[{"find":"import { Dismissal, Payment, Promotion } from \"./progression-models.js\";","replace":"import { Dismissal, Payment, Promotion, SupportTicket } from \"./progression-models.js\";"},{"find":" await mongoose.connect(DATABASE_URL);","replace":" await mongoose.connect(DATABASE_URL);\n await SupportTicket.deleteMany({});"}]},{"id":"support-history-is-not-owner-scoped","scenario":"tracks/ecommerce/scenarios/progression-support-history.json","targets":["ecommerce.spec.access-control.support-history-privacy.612b"],"desc":"Signed-in customers can read other customers support history.","file":"server/src/progression.ts","edits":[{"find":"const ticketFilter = isStaff ? {} : user ? { userId: user._id } : { _id: null };","replace":"const ticketFilter = isStaff ? {} : user ? {} : { _id: null };"}]},{"id":"support-intake-returns-no-reference","scenario":"tracks/ecommerce/scenarios/progression-support-intake.json","targets":["ecommerce.progression.support-intake.support-intake.610a"],"desc":"Support intake creates a ticket but omits its reference from the response.","file":"server/src/progression.ts","edits":[{"find":"res.status(201).json({ ticket });","replace":"res.status(201).json({ ticket: { ...ticket.toJSON(), reference: \"\" } });"}]},{"id":"support-triage-discards-updates","scenario":"tracks/ecommerce/scenarios/progression-support-triage.json","targets":["ecommerce.progression.support-triage.support-assignment.611a","ecommerce.progression.support-triage.support-priority.611b","ecommerce.progression.support-triage.support-status.611c"],"desc":"Support triage accepts but discards assignment, priority, and status updates.","file":"server/src/progression.ts","edits":[{"find":"const ticket = ticketId\n ? await SupportTicket.findByIdAndUpdate(ticketId, { $set: update }, { new: true }) : null;","replace":"const ticket = ticketId\n ? await SupportTicket.findByIdAndUpdate(ticketId, { $set: {} }, { new: true }) : null;"}]},{"id":"cart-add-uses-another-account-cart","scenario":"tracks/ecommerce/scenarios/01-cart-boundary.json","targets":["ecommerce.spec.access-control.cart-boundary.109a"],"desc":"Cart add writes to the first account cart instead of the caller cart.","file":"server/src/index.ts","edits":[{"find":" const incResult = await Cart.updateOne(\n { userId, \"items.itemId\": itemId },","replace":" const incResult = await Cart.updateOne(\n { \"items.itemId\": itemId },"}]},{"id":"negative-cart-quantity-is-accepted","scenario":"tracks/ecommerce/scenarios/01-cart-boundary.json","targets":["ecommerce.spec.access-control.cart-boundary.109b"],"desc":"Cart quantity updates accept values below one.","file":"server/src/index.ts","edits":[{"find":" if (!Number.isInteger(qty) || qty < 1) {\n return res.status(400).json({ error: \"Quantity must be at least 1\" });\n }","replace":" if (!Number.isInteger(qty)) {\n return res.status(400).json({ error: \"Quantity must be numeric\" });\n }"}]},{"id":"direct-purchase-is-attributed-to-another-account","scenario":"tracks/ecommerce/scenarios/01-purchase-attribution.json","targets":["ecommerce.spec.access-control.purchase-attribution.102a"],"desc":"Direct purchase records the order against another account.","file":"server/src/index.ts","edits":[{"find":" userId: user._id,\n items: [","replace":" userId: (await User.findOne({ _id: { $ne: user._id } }))!._id,\n items: ["}]},{"id":"concurrent-cart-add-does-not-increment","scenario":"tracks/ecommerce/scenarios/01-duplicate-checkout.json","targets":["ecommerce.spec.concurrency-safety.duplicate-checkout.203a"],"desc":"Concurrent cart adds leave the line quantity and its reservation list unchanged, so the existing cart remains checkable at checkout.","file":"server/src/index.ts","edits":[{"find":"{ $inc: { \"items.$.quantity\": qty },\n $set: { \"items.$.reservationExpiresAt\": new Date(Date.now() + 90_000),\n inactiveExpiresAt: new Date(Date.now() + 300_000) },\n $push: { \"items.$.reservedWarehouseIds\": { $each: reservedWarehouseIds } } }","replace":"{ $inc: { \"items.$.quantity\": 0 },\n $set: { \"items.$.reservationExpiresAt\": new Date(Date.now() + 90_000),\n inactiveExpiresAt: new Date(Date.now() + 300_000) },\n $push: { \"items.$.reservedWarehouseIds\": { $each: [] } } }"}]},{"id":"checkout-claim-is-not-atomic","scenario":"tracks/ecommerce/scenarios/01-duplicate-checkout.json","targets":["ecommerce.spec.concurrency-safety.duplicate-checkout.203b"],"desc":"Concurrent checkout reads the cart without atomically claiming it.","file":"server/src/credit.ts","edits":[{"find":" return mongoose.connection.transaction(async session => {\n const cart = await Cart.findOne({ userId }).session(session);","replace":" const snapshot = await Cart.findOne({ userId });\n if (snapshot) snapshot.set(\"__v\", undefined);\n await new Promise(resolve => setTimeout(resolve, 500));\n return mongoose.connection.transaction(async session => {\n const cart = snapshot ? Cart.hydrate(snapshot.toObject()) : null;"}]},{"id":"last-unit-allows-negative-stock","scenario":"tracks/ecommerce/scenarios/01-last-unit.json","targets":["ecommerce.spec.concurrency-safety.last-unit.201a","ecommerce.spec.concurrency-safety.last-unit.201b","ecommerce.spec.concurrency-safety.last-unit.201c"],"desc":"Stock reservation can decrement a warehouse below zero.","file":"server/src/stock-reservations.ts","edits":[{"find":"{ item_id: itemId, quantity: { $gte: 1 } },","replace":"{ item_id: itemId },"}]},{"id":"purchase-read-write-loses-concurrent-stock","scenario":"tracks/ecommerce/scenarios/01-restock-race.json","targets":["ecommerce.spec.concurrency-safety.restock-race.202a"],"desc":"Replace atomic stock reservation with an unlocked read and absolute write. A fixed pause widens scheduling overlap; serial purchases and restocks retain their stock effects. Concurrent reservations or restocking can lose updates.","file":"server/src/stock-reservations.ts","edits":[{"find":" const stock = await model.findOneAndUpdate(\n { item_id: itemId, quantity: { $gte: 1 } },\n { $inc: { quantity: -1 } },\n { sort: { quantity: -1 }, ...(session ? { session } : {}) },\n );\n if (!stock) {\n await releaseWithModel(model, itemId, warehouseIds, session);\n return null;\n }\n","replace":" const stock = await model.findOneAndUpdate(\n { item_id: itemId, quantity: { $gte: 1 } },\n { $inc: { quantity: 0 } },\n { sort: { quantity: -1 }, ...(session ? { session } : {}) },\n );\n if (!stock) {\n await releaseWithModel(model, itemId, warehouseIds, session);\n return null;\n }\n // Mutant: widen the unlocked read/write window without changing serial behavior.\n await new Promise(resolve => setTimeout(resolve, 500));\n await model.updateOne({ item_id: itemId, warehouse_id: stock.warehouse_id },\n { $set: { quantity: stock.quantity - 1 } });\n"}]},{"id":"stock-limit-allows-negative-stock","scenario":"tracks/ecommerce/scenarios/progression-stock-limit.json","targets":["ecommerce.spec.concurrency-safety.stock-limit.3d"],"desc":"Stock reservation accepts purchases after sellout.","file":"server/src/stock-reservations.ts","edits":[{"find":"{ item_id: itemId, quantity: { $gte: 1 } },","replace":"{ item_id: itemId },"}]},{"id":"purchase-does-not-reduce-warehouse-stock","scenario":"tracks/ecommerce/scenarios/progression-books-balance.json","targets":["ecommerce.spec.transactional-integrity.books-balance.107b"],"desc":"Purchases create orders without reducing warehouse stock.","file":"server/src/stock-reservations.ts","edits":[{"find":" { $inc: { quantity: -1 } },","replace":" { $inc: { quantity: 0 } },"}]},{"id":"restock-does-not-increase-stock","scenario":"tracks/ecommerce/scenarios/01-warehouse-stock-live-staff.json","targets":["ecommerce.spec.live-state.warehouse-stock.7c"],"desc":"The restock endpoint accepts a positive quantity but increments authoritative stock by zero.","file":"server/src/index.ts","edits":[{"find":" await Stock.findOneAndUpdate(\n { item_id: item._id, warehouse_id: warehouse._id },\n { $inc: { quantity: qty } },\n { upsert: true }\n );","replace":" await Stock.findOneAndUpdate(\n { item_id: item._id, warehouse_id: warehouse._id },\n { $inc: { quantity: 0 } },\n { upsert: true }\n );"}]},{"id":"operator-authorization-allows-customer-price-change","scenario":"tracks/ecommerce/scenarios/02-strengthened.json","targets":["ecommerce.operations-access.operator-authorization.201b"],"desc":"The price route keeps authentication but drops its administrator role gate.","file":"server/src/index.ts","edits":[{"find":"app.post(\"/api/admin/price\", requireAuth, requireAdmin, async (req, res) => {","replace":"app.post(\"/api/admin/price\", requireAuth, async (req, res) => {"}]},{"id":"operator-authorization-allows-customer-shipping","scenario":"tracks/ecommerce/scenarios/02-self-contained.json","targets":["ecommerce.operations-access.fulfilment-queue.1e"],"desc":"The shipping route retains authentication but drops the staff role gate.","file":"server/src/index.ts","edits":[{"find":"app.post(\"/api/fulfilment/ship\", requireAuth, requireStaff, async (req, res) => {","replace":"app.post(\"/api/fulfilment/ship\", requireAuth, async (req, res) => {"}]},{"id":"catalog-price-change-not-broadcast-to-visitors","scenario":"tracks/ecommerce/scenarios/02-live-price.json","targets":["ecommerce.returns-pricing.price-history.4b"],"desc":"The signed-out search result caches detached rows and ignores later catalog updates.","file":"client/src/App.tsx","edits":[{"find":" }, [items, searchQuery, categoryFilter, minimumPrice, maximumPrice, inStockOnly]);","replace":" }, [searchQuery, categoryFilter, minimumPrice, maximumPrice, inStockOnly]);"}]},{"id":"catalog-price-change-not-broadcast-to-open-carts","scenario":"tracks/ecommerce/scenarios/progression-price-cart-checkout.json","targets":["ecommerce.returns-pricing.price-history.4c"],"desc":"The price reaches the catalog but the price route omits its targeted open-cart broadcast.","file":"server/src/index.ts","edits":[{"find":" broadcastRecommendedForAll(),\n broadcastCartsContainingItem(item._id),","replace":" broadcastRecommendedForAll(),\n // mutant: carts containing this item are not refreshed"}]},{"id":"catalog-price-rewrites-receipts","scenario":"tracks/ecommerce/scenarios/02-paid-price-history.json","targets":["ecommerce.returns-pricing.price-history.4a"],"desc":"Changing a catalog price cascades into saved order lines and recomputes historical order totals.","file":"server/src/index.ts","edits":[{"find":" const item = await Item.findByIdAndUpdate(itemId, { price: priceNum }, { new: true });\n if (!item) return res.status(404).json({ error: \"Item not found\" });","replace":" const item = await Item.findByIdAndUpdate(itemId, { price: priceNum }, { new: true });\n if (!item) return res.status(404).json({ error: \"Item not found\" });\n const historicalOrders = await Order.find({ \"items.itemId\": item._id });\n for (const historical of historicalOrders) {\n for (const line of historical.items) {\n if (line.itemId.toString() === item._id.toString()) line.price = priceNum;\n }\n historical.total = historical.items.reduce((sum, line) => sum + line.price * line.quantity, 0);\n await historical.save();\n }"}]},{"id":"catalog-price-rewrites-earned-revenue","scenario":"tracks/ecommerce/scenarios/02-invariants.json","targets":["ecommerce.returns-pricing.refund-accounting.203b"],"desc":"Changing a catalog price cascades into saved order lines and recomputes historical order totals and revenue.","file":"server/src/index.ts","edits":[{"find":" const item = await Item.findByIdAndUpdate(itemId, { price: priceNum }, { new: true });\n if (!item) return res.status(404).json({ error: \"Item not found\" });","replace":" const item = await Item.findByIdAndUpdate(itemId, { price: priceNum }, { new: true });\n if (!item) return res.status(404).json({ error: \"Item not found\" });\n const historicalOrders = await Order.find({ \"items.itemId\": item._id });\n for (const historical of historicalOrders) {\n for (const line of historical.items) {\n if (line.itemId.toString() === item._id.toString()) line.price = priceNum;\n }\n historical.total = historical.items.reduce((sum, line) => sum + line.price * line.quantity, 0);\n await historical.save();\n }"}]},{"id":"returned-line-marker-omitted","scenario":"tracks/ecommerce/scenarios/02-strengthened.json","targets":["ecommerce.returns-pricing.cancellation-and-return.3c"],"desc":"A returned order line keeps its persisted returned state, restored stock, and adjusted revenue but omits the visible returned marker.","file":"client/src/App.tsx","edits":[{"find":"l.returned ? \" (returned)\" : \"\"","replace":"\"\""}]},{"id":"direct-review-access-is-not-checked","scenario":"tracks/ecommerce/scenarios/progression-review-access.json","targets":["ecommerce.progression.review-access-specifications.review-eligibility-direct.618a"],"desc":"The direct review action accepts a review from a customer who did not buy the item.","file":"server/src/index.ts","edits":[{"find":" if (!hasPurchased) {\n return res.status(403).json({ error: \"You can only review items you have purchased\" });\n }","replace":" if (false && !hasPurchased) {\n return res.status(403).json({ error: \"You can only review items you have purchased\" });\n }"}]},{"id":"support-history-rows-are-hidden","scenario":"tracks/ecommerce/scenarios/progression-support-history.json","targets":["ecommerce.progression.support-history.support-history.612c","ecommerce.spec.state-durability.support-history-reload.612a","ecommerce.spec.access-control.support-history-privacy.612b","ecommerce.spec.access-control.support-history-logout.612d"],"desc":"Hide submitted support ticket rows while leaving the submission reference available. History, reload, privacy, and logout require the owner to see the ticket first. This breaks those positive observations; it does not create a privacy leak or remove stored data.","file":"client/src/ProgressionPanel.tsx","edits":[{"find":"data-role=\"support-ticket\"","replace":"data-role=\"support-ticket\" style={{ display: \"none\" }}"}]},{"id":"authorized-restock-does-not-change-stock","scenario":"tracks/ecommerce/scenarios/01-admin-write-staff.json","targets":["ecommerce.feature.warehouse-admin.admin-write.103a"],"desc":"Accept an administrator restock without changing warehouse stock.","file":"server/src/index.ts","edits":[{"find":" await Stock.findOneAndUpdate(\n { item_id: item._id, warehouse_id: warehouse._id },\n { $inc: { quantity: qty } },\n { upsert: true }\n );\n\n","replace":" // mutant: accept restock without changing stock\n\n"}]},{"id":"initial-dashboard-load-omits-low-stock","scenario":"tracks/ecommerce/scenarios/02-low-stock.json","targets":["ecommerce.inventory-operations.operational-views.5e","ecommerce.spec.live-state.inventory-dashboard.5a"],"desc":"The dashboard's initial load reports no low-stock items, so an item already at ten units or fewer is missing until stock moves; 5a's opening precondition re-asserts the same listing, so it necessarily regresses with 5e.","file":"server/src/index.ts","edits":[{"find":"app.get(\"/api/admin/overview\", requireAuth, requireAdmin, async (_req, res) => {\n res.json(await getAdminOverview());","replace":"app.get(\"/api/admin/overview\", requireAuth, requireAdmin, async (_req, res) => {\n res.json({ ...(await getAdminOverview()), lowStock: [] }); // mutant: the initial dashboard load omits current low-stock items"}]},{"id":"category-totals-skip-the-newest-order","scenario":"tracks/ecommerce/scenarios/02-operational-category-totals.json","targets":["ecommerce.inventory-operations.operational-views.5f"],"desc":"Category totals leave out the most recent order, so the reloaded dashboard shows a purchase only once a later order pushes it into the aggregate.","file":"server/src/index.ts","edits":[{"find":"async function getCategoryTotals() {\n const rows = await Order.aggregate([\n { $match: { status: { $ne: \"cancelled\" } } },","replace":"async function getCategoryTotals() {\n const rows = await Order.aggregate([\n { $match: { status: { $ne: \"cancelled\" } } },\n { $sort: { createdAt: -1 } },\n { $skip: 1 }, // mutant: the most recent order is left out of the category totals"}]},{"id":"profile-summary-frozen-at-open","scenario":"tracks/ecommerce/scenarios/progression-customer-profile.json","targets":["ecommerce.progression.customer-profile.customer-profile.620c"],"desc":"The profile summary shows the address loaded when the profile card opened, so a freshly saved address is not visible until the page is reloaded.","file":"client/src/ProgressionPanel.tsx","edits":[{"find":" <p data-role=\"profile-address-summary\">{state.profile?.address || \"\"}</p>","replace":" <ProfileAddressSummary address={state.profile?.address || \"\"} />"},{"find":"function SupportTicket({ ticket, user, orders, act }: any) {","replace":"function ProfileAddressSummary({ address }: { address: string }) {\n const [shown] = useState(address); // mutant: the summary keeps the address loaded when the card opened\n return <p data-role=\"profile-address-summary\">{shown}</p>;\n}\n\nfunction SupportTicket({ ticket, user, orders, act }: any) {"}]},{"id":"support-replies-present-at-open-are-hidden","scenario":"tracks/ecommerce/scenarios/progression-managed-support-shared.json","targets":["ecommerce.progression.managed-support.managed-support.613c"],"desc":"A support case renders only replies that arrive while it is open, so after a reload the replies already exchanged are missing while live replies still appear.","file":"client/src/ProgressionPanel.tsx","edits":[{"find":" const [reply, setReply] = useState(\"\");","replace":" const [reply, setReply] = useState(\"\");\n const [repliesAtOpen] = useState(() => new Set((ticket.replies || []).map((entry: any) => entry.id || entry.createdAt))); // mutant: replies already on the case when it opened are never rendered"},{"find":" {(ticket.replies || []).map((entry: any) => <div data-role=\"support-reply-item\" key={entry.id || entry.createdAt}>{entry.username}: {entry.body}</div>)}","replace":" {(ticket.replies || []).filter((entry: any) => !repliesAtOpen.has(entry.id || entry.createdAt)).map((entry: any) => <div data-role=\"support-reply-item\" key={entry.id || entry.createdAt}>{entry.username}: {entry.body}</div>)}"}]},{"id":"notification-toggle-frozen-at-open","scenario":"tracks/ecommerce/scenarios/progression-notification-preferences.json","targets":["ecommerce.progression.notification-preferences.notification-preferences.630c"],"desc":"The order-notification toggle displays the state loaded when the settings card opened, so the saved choice is reported as off until a reload.","file":"client/src/ProgressionPanel.tsx","edits":[{"find":" <button data-role=\"notification-order\" data-state={preference.order ? \"on\" : \"off\"} className=\"btn btn-ghost\"\n onClick={() => setState((value: any) => ({ ...value, preference: { ...preference, order: !preference.order } }))}>\n Order notifications {preference.order ? \"on\" : \"off\"}\n </button>","replace":" <OrderNotificationToggle on={preference.order}\n onToggle={() => setState((value: any) => ({ ...value, preference: { ...preference, order: !preference.order } }))} />"},{"find":"function SupportTicket({ ticket, user, orders, act }: any) {","replace":"function OrderNotificationToggle({ on, onToggle }: { on: boolean; onToggle: () => void }) {\n const [shown] = useState(on); // mutant: the toggle keeps the state loaded when the card opened\n return <button data-role=\"notification-order\" data-state={shown ? \"on\" : \"off\"} className=\"btn btn-ghost\" onClick={onToggle}>\n Order notifications {shown ? \"on\" : \"off\"}\n </button>;\n}\n\nfunction SupportTicket({ ticket, user, orders, act }: any) {"}]},{"id":"role-editor-snaps-back-to-stored-role","scenario":"tracks/ecommerce/scenarios/progression-staff-roles.json","targets":["ecommerce.progression.staff-roles.staff-roles.621c"],"desc":"Pressing Save persists the role but resets the dropdown to the role that was stored before the save, so the assignment is not visible until a reload.","file":"client/src/ProgressionPanel.tsx","edits":[{"find":" <button data-role=\"staff-role-save\" onClick={() => act(`/api/staff/${entry.id}/role`, {\n method: \"PUT\", body: JSON.stringify({ role: roles[entry.username] ?? entry.roles?.[0] ?? \"staff\" }),\n })}>Save</button>","replace":" <button data-role=\"staff-role-save\" onClick={() => {\n setRoles(value => ({ ...value, [entry.username]: entry.roles?.[0] ?? \"staff\" })); // mutant: the editor snaps back to the previously stored role\n return act(`/api/staff/${entry.id}/role`, {\n method: \"PUT\", body: JSON.stringify({ role: roles[entry.username] ?? entry.roles?.[0] ?? \"staff\" }),\n }); }}>Save</button>"}]},{"id":"queue-ignores-live-fulfilment-updates","scenario":"tracks/ecommerce/scenarios/02-fulfilment-live.json","targets":["ecommerce.spec.live-state.fulfilment-queue.1a"],"desc":"The open staff queue ignores live fulfilment events, so a new order appears only after a reload.","file":"client/src/App.tsx","edits":[{"find":" socket.on(\"fulfilment:update\", (data: FulfilmentQueueT) => setFulfilmentQueue(data));","replace":" socket.on(\"fulfilment:update\", (data: FulfilmentQueueT) => { void data; }); // mutant: the open staff queue ignores live fulfilment updates"}]},{"id":"low-stock-boundary-excludes-ten-live","scenario":"tracks/ecommerce/scenarios/02-low-stock.json","targets":["ecommerce.spec.live-state.inventory-dashboard.5a"],"desc":"The low-stock view uses a strict boundary, so an item that falls to exactly ten units never joins the list.","file":"server/src/index.ts","edits":[{"find":" .filter((it) => it.stock <= 10)","replace":" .filter((it) => it.stock < 10)"}]},{"id":"live-admin-updates-keep-stale-category-totals","scenario":"tracks/ecommerce/scenarios/02-operational-category-totals.json","targets":["ecommerce.spec.live-state.sales-dashboard.5b"],"desc":"Live admin updates keep the category totals loaded at page load, so a purchase does not move units or revenue until a reload.","file":"client/src/App.tsx","edits":[{"find":" socket.on(\"admin:update\", (data: AdminOverviewT) => setAdminOverview(data));","replace":" socket.on(\"admin:update\", (data: AdminOverviewT) => setAdminOverview((previous) => previous ? { ...data, categories: previous.categories } : data)); // mutant: live admin updates keep the category totals loaded at page load"}]},{"id":"overdraw-transfer-is-accepted","scenario":"tracks/ecommerce/scenarios/02-transfer-overdraw.json","targets":["ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c"],"desc":"The atomic source debit no longer requires sufficient quantity, so an overdrawn transfer succeeds and moves both warehouse totals.","file":"server/src/index.ts","edits":[{"find":"{ item_id: itemId, warehouse_id: fromWarehouseId, quantity: { $gte: qty } }","replace":"{ item_id: itemId, warehouse_id: fromWarehouseId }"}]},{"id":"transfer-totals-omit-destination-credit-live","scenario":"tracks/ecommerce/scenarios/02-transfer-totals.json","targets":["ecommerce.spec.live-state.stock-transfers.2b"],"desc":"A transfer debits the source but adds zero to the destination, so the two live warehouse totals do not move in opposite directions.","file":"server/src/index.ts","edits":[{"find":" await Stock.findOneAndUpdate(\n { item_id: itemId, warehouse_id: toWarehouseId },\n { $inc: { quantity: qty } },\n { upsert: true }\n );","replace":" await Stock.findOneAndUpdate(\n { item_id: itemId, warehouse_id: toWarehouseId },\n { $inc: { quantity: 0 } },\n { upsert: true }\n );"}]},{"id":"credit-checkout-ignores-wallet","desc":"A credit checkout pays entirely externally despite available credit.","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.feature.store-credit.store-credit-750.750a"],"file":"server/src/credit.ts","edits":[{"find":" const creditMinor = useCredit ? Math.min(user.creditMinor, totalMinor) : 0;","replace":" const creditMinor = 0;"}]},{"id":"credit-grant-replay-increments-balance","desc":"Replaying a grant applies its credit to the wallet again.","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.spec.store-credit.production-752.752a"],"file":"server/src/credit.ts","edits":[{"find":" if (existing.amountMinor !== amountMinor) throw new Error('Reference already identifies another grant');\n return;","replace":" if (existing.amountMinor !== amountMinor) throw new Error('Reference already identifies another grant');\n await User.updateOne({ _id: accountId }, { $inc: { creditMinor: amountMinor } }, { session });\n return;"}]},{"id":"customer-can-grant-credit","desc":"Customer authentication is accepted without staff authorization.","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.spec.store-credit.production-753.753a"],"file":"server/src/credit.ts","edits":[{"find":" app.post('/api/staff/credit', auth, staff, async (req, res) => {","replace":" app.post('/api/staff/credit', auth, async (req, res) => {"}]},{"id":"credit-checkout-retains-purchased-cart","desc":"A second checkout can reuse the purchased cart and create another order.","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.spec.store-credit.production-754.754a"],"file":"server/src/credit.ts","edits":[{"find":" cart.items = cart.items.filter(line => line.reservationExpiresAt && line.reservationExpiresAt <= now) as any;","replace":" // mutant: purchased cart lines remain"}]},{"id":"split-refund-does-not-restore-credit","desc":"The refund is recorded but its original wallet credit is not restored.","scenario":"tracks/ecommerce/scenarios/progression-split-tender-refunds.json","targets":["ecommerce.feature.split-tender-refunds.split-tender-refunds-751.751a"],"file":"server/src/progression.ts","edits":[{"find":" await refundCredit(order, session);","replace":" // mutant: omit wallet restoration"}]},{"id":"split-refund-duplicates-credit","desc":"A refund credits the wallet twice while recording one refund.","scenario":"tracks/ecommerce/scenarios/progression-split-tender-refunds.json","targets":["ecommerce.spec.split-tender-refunds.production-756.756a"],"file":"server/src/credit.ts","edits":[{"find":"{ $inc: { creditMinor: delta } }, { session });","replace":"{ $inc: { creditMinor: delta * 2 } }, { session });"}]},{"id":"subscription-skips-due-purchase","desc":"Due deliveries are recorded as skipped although stock is available.","scenario":"tracks/ecommerce/scenarios/progression-subscriptions.json","targets":["ecommerce.feature.subscriptions.subscriptions-760.760a"],"file":"server/src/subscriptions.ts","edits":[{"find":" const allocation = await reserveStock(row.itemId, row.quantity, session);","replace":" const allocation = row.quantity < 0 ? await reserveStock(row.itemId, row.quantity, session) : null;"}]},{"id":"subscription-allows-foreign-cancellation","desc":"A customer can cancel another customer subscription.","scenario":"tracks/ecommerce/scenarios/progression-subscriptions.json","targets":["ecommerce.spec.subscriptions.production-762.762a"],"file":"server/src/subscriptions.ts","edits":[{"find":" if (!row || String(row.userId) !== String((req as any).user._id)) return false;","replace":" if (!row) return false;"}]},{"id":"subscription-pause-is-not-recorded","desc":"Pause acknowledges the request but the subscription remains active.","scenario":"tracks/ecommerce/scenarios/progression-subscriptions.json","targets":["ecommerce.spec.subscriptions.production-763.763a"],"file":"server/src/subscriptions.ts","edits":[{"find":" row.status = 'paused'; row.pausedAt = new Date();","replace":" row.status = 'active'; row.pausedAt = new Date();"}]},{"id":"credit-balance-is-cleared-at-startup","desc":"Restart clears an issued wallet balance while leaving accounts present.","file":"server/src/index.ts","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.spec.store-credit.production-755.755a"],"edits":[{"find":" await seed();","replace":" await seed();\n await User.updateMany({}, { $set: { creditMinor: 0 } });"}]},{"id":"pending-subscriptions-are-cleared-at-startup","desc":"Restart erases pending subscription work while preserving ordinary timer execution.","file":"server/src/index.ts","scenario":"tracks/ecommerce/scenarios/progression-subscriptions.json","targets":["ecommerce.spec.subscriptions.production-761.761a"],"edits":[{"find":" await seed();","replace":" await seed();\n await mongoose.connection.collection(\"purchasesubscriptions\").deleteMany({ status: \"active\" });"}]},{"id":"bundle-definition-loses-component-quantity","scenario":"tracks/ecommerce/scenarios/progression-product-bundles.json","targets":["ecommerce.feature.product-bundles.product-bundles.740a"],"desc":"definition loses component quantity","file":"server/src/bundles.ts","edits":[{"find":"quantity: component.quantity });","replace":"quantity: 1 });"}]},{"id":"bundle-catalog-write-allows-customers","scenario":"tracks/ecommerce/scenarios/progression-product-bundles.json","targets":["ecommerce.spec.bundle-integrity.bundle-743.743a"],"desc":"catalog write allows customers","file":"server/src/bundles.ts","edits":[{"find":"if (!actor.isAdmin && !actor.roles?.includes('catalog'))","replace":"if (false)"}]},{"id":"bundle-checkout-price-not-snapshot","scenario":"tracks/ecommerce/scenarios/progression-bundle-checkout.json","targets":["ecommerce.feature.bundle-checkout.bundle-checkout.741a"],"desc":"checkout price not snapshot","file":"server/src/bundles.ts","edits":[{"find":"bundlePrice: bundle.price,","replace":"bundlePrice: bundle.price + 1,"}]},{"id":"bundle-expiry-does-not-release-components","scenario":"tracks/ecommerce/scenarios/progression-bundle-checkout.json","targets":["ecommerce.spec.bundle-integrity.bundle-746.746a"],"desc":"expiry does not release components","file":"server/src/bundles.ts","edits":[{"find":"await releaseBundle(line.componentAllocations as Allocation[], session);\n line.componentAllocations = [] as any;","replace":"line.componentAllocations = [] as any; // mutant: component holds leak"}]},{"id":"bundle-return-loses-original-components","scenario":"tracks/ecommerce/scenarios/progression-bundle-returns.json","targets":["ecommerce.feature.bundle-returns.bundle-returns.742a"],"desc":"return loses original components","file":"server/src/bundles.ts","edits":[{"find":"for (const line of bundles) { await releaseBundle(line.componentAllocations as Allocation[], session); line.returned = true; }","replace":"for (const line of bundles) { line.returned = true; }"}]},{"id":"bundle-return-replay-restocks-again","scenario":"tracks/ecommerce/scenarios/progression-bundle-returns.json","targets":["ecommerce.spec.bundle-integrity.bundle-742.742b"],"desc":"return replay restocks again","file":"server/src/bundles.ts","edits":[{"find":"line.isBundle && !line.returned","replace":"line.isBundle"}]},{"id":"bundle-return-crosses-account-boundary","scenario":"tracks/ecommerce/scenarios/progression-bundle-returns.json","targets":["ecommerce.spec.bundle-integrity.bundle-748.748a"],"desc":"return crosses account boundary","file":"server/src/bundles.ts","edits":[{"find":"{ _id: req.params.orderId, userId, status: { $in: ['shipped', 'delivered'] } }","replace":"{ _id: req.params.orderId, status: { $in: ['shipped', 'delivered'] } }"}]},{"id":"bundle-components-can-overdraw","scenario":"tracks/ecommerce/scenarios/progression-bundle-checkout.json","targets":["ecommerce.spec.bundle-integrity.bundle-744.744a","ecommerce.spec.bundle-integrity.bundle-745.745a"],"desc":"components can overdraw","file":"server/src/stock-reservations.ts","edits":[{"find":"{ item_id: itemId, quantity: { $gte: 1 } }","replace":"{ item_id: itemId }"}]},{"id":"bundle-checkout-reuses-reservation","scenario":"tracks/ecommerce/scenarios/progression-bundle-checkout.json","targets":["ecommerce.spec.bundle-integrity.bundle-747.747a"],"desc":"checkout reuses reservation","file":"server/src/credit.ts","edits":[{"find":"cart.items = cart.items.filter(line => line.reservationExpiresAt && line.reservationExpiresAt <= now) as any;","replace":"// mutant: active cart survives checkout"}]},{"id":"return-after-support-refund-is-blocked","scenario":"tracks/ecommerce/scenarios/progression-support-return-interaction.json","targets":["ecommerce.feature.split-tender-refunds.return-refund-interaction.757a"],"desc":"Reject a valid physical return after a financial refund.","file":"server/src/index.ts","edits":[{"find":" if (!line || line.returned) throw new Error('No returnable item found');","replace":" if (!line || line.returned || value.refundTotal > 0) throw new Error('No returnable item found');"}]},{"id":"support-refund-after-return-pays-twice","scenario":"tracks/ecommerce/scenarios/progression-support-return-interaction.json","targets":["ecommerce.feature.split-tender-refunds.return-refund-interaction.757b"],"desc":"Pay the full order again after a physical return.","file":"server/src/progression.ts","edits":[{"find":" order.refundTotal = order.total;","replace":" order.refundTotal += order.total;"}]},{"id":"support-history-leaks-to-signed-out-visitors","scenario":"tracks/ecommerce/scenarios/progression-support-history.json","targets":["ecommerce.spec.access-control.support-history-logout.612d"],"desc":"Disclose support tickets to signed-out visitors while leaving signed-in ownership filtering unchanged.","file":"server/src/progression.ts","edits":[{"find":"const ticketFilter = isStaff ? {} : user ? { userId: user._id } : { _id: null };","replace":"const ticketFilter = isStaff ? {} : user ? { userId: user._id } : {};"}]},{"id":"checkout-crash-integrity","scenario":"tracks/ecommerce/scenarios/progression-checkout-crash.json","targets":["ecommerce.spec.state-durability.checkout-crash-integrity.910a"],"desc":"A lifecycle callback changes a prepared or cleared cart to quantity two, leaving neither a legal rollback nor a complete checkout.","file":"server/src/index.ts","edits":[{"find":" await initializeOrderData(mongoose.connection);","replace":" await initializeOrderData(mongoose.connection);\n for (const cart of await Cart.find()) { const order = await Order.findOne({ userId: cart.userId }); if (order?.items[0]) await Cart.updateOne({ _id: cart._id }, { $set: { items: [{ itemId: order.items[0].itemId, quantity: 2 }] } }); }"}]},{"id":"checkout-crash-durability","scenario":"tracks/ecommerce/scenarios/progression-checkout-crash.json","targets":["ecommerce.spec.state-durability.checkout-crash-durability.910b"],"desc":"A lifecycle callback corrupts earlier order totals. The unchanged prepared snapshots and acknowledged baseline must expose the damage.","file":"server/src/index.ts","edits":[{"find":" await initializeOrderData(mongoose.connection);","replace":" await initializeOrderData(mongoose.connection);\n await Order.updateMany({}, { $set: { total: 0 } });"}]},{"id":"review-script-unsafe-render","scenario":"tracks/ecommerce/scenarios/progression-review-script.json","targets":["ecommerce.progression.review-access-specifications.stored-review-script.9180a"],"desc":"Render stored review text as HTML so it executes in another customer session.","file":"client/src/App.tsx","edits":[{"find":"<div>{r.comment}</div>","replace":"<div dangerouslySetInnerHTML={{ __html: r.comment }} />"}]},{"id":"review-script-reject-all","scenario":"tracks/ecommerce/scenarios/progression-review-script.json","targets":["ecommerce.progression.review-access-specifications.stored-review-script.9180a"],"desc":"Reject all nonempty reviews, including the required benign review; absence of script is insufficient.","file":"server/src/index.ts","edits":[{"find":"const { rating, comment } = req.body || {};","replace":"const { rating, comment } = req.body || {};\n if (String(comment).length > 0) { res.status(400).json({ error: \"Review rejected\" }); return; }"}]}]},"postgres":{"schemaVersion":3,"fixtureSha256":"f3906fa01050f00a0ed76c69c256a6b0013bbf72eb72c03aa92d79e47b71c2b8","backend":"postgres","track":"ecommerce","note":"Mutation definitions for the PostgreSQL ecommerce reference.","mutations":[{"id":"recommendation-dismissal-lost-on-restart","scenario":"tracks/ecommerce/scenarios/progression-recommendation-feedback.json","targets":["ecommerce.spec.state-durability.recommendation-feedback-restart.504c"],"desc":"Erase saved recommendation dismissals when the application starts again.","file":"server/src/index.ts","edits":[{"find":" await initializeProgressionSchema(pool);","replace":" await initializeProgressionSchema(pool);\n await pool.query('DELETE FROM recommendation_dismissal');"}]},{"id":"pending-order-item-return-accepted","scenario":"tracks/ecommerce/scenarios/progression-order-return-boundary.json","targets":["ecommerce.returns-pricing.cancellation-and-return.3f"],"desc":"Accept a pending order return and restore its stock before shipment.","file":"server/src/index.ts","edits":[{"find":"if (!['shipped', 'delivered'].includes(orderRow.rows[0].status)) {","replace":"if (!['pending', 'shipped', 'delivered'].includes(orderRow.rows[0].status)) {"}]},{"id":"staff-admin-access-survives-role-removal","scenario":"tracks/ecommerce/scenarios/progression-staff-roles.json","targets":["ecommerce.spec.access-control.staff-role-revocation.621d"],"desc":"Keep administrator access after changing the assigned role back to staff.","file":"server/src/progression.ts","edits":[{"find":"is_admin = ($1 = 'admin')","replace":"is_admin = (is_admin OR $1 = 'admin')"}]},{"id":"shipping-counts-sale-twice","scenario":"tracks/ecommerce/scenarios/progression-shipping-accounting.json","targets":["ecommerce.inventory-operations.shipping-accounting.202e"],"desc":"Shipping succeeds but doubles the completed sale value in authoritative revenue.","file":"server/src/index.ts","edits":[{"find":"`UPDATE orders SET status = 'shipped', shipped_at = now()\n WHERE id = $1 AND status = 'pending' RETURNING account_id`","replace":"`UPDATE orders SET status = 'shipped', shipped_at = now(), total = total * 2\n WHERE id = $1 AND status = 'pending' RETURNING account_id`"}]},{"id":"signup-ui-does-not-enter-created-account","scenario":"tracks/ecommerce/scenarios/01-account-create.json","targets":["ecommerce.feature.accounts.accounts.1a"],"desc":"Create the account successfully but discard the returned signed-in identity in the client.","file":"client/src/App.tsx","edits":[{"find":" const r = await api<{ account: Account }>(\"/api/auth/signup\", {\n method: \"POST\",\n body: JSON.stringify({ username, password }),\n });\n setAccount(r.account);","replace":" const r = await api<{ account: Account }>(\"/api/auth/signup\", {\n method: \"POST\",\n body: JSON.stringify({ username, password }),\n });\n void r;\n setAccount(null);"}]},{"id":"duplicate-signup-authenticates-existing-account","scenario":"tracks/ecommerce/scenarios/01-account-duplicate.json","targets":["ecommerce.feature.accounts.accounts.1b"],"desc":"Treat a duplicate signup as a successful session for the pre-existing account.","file":"server/src/index.ts","edits":[{"find":" if (existing.length > 0) {\n res.status(409).json({ error: \"username already taken\" });\n return;\n }","replace":" if (existing.length > 0) {\n const token = newToken();\n await db.insert(session).values({ id: token, accountId: existing[0].id });\n res.cookie(\"sid\", token, { httpOnly: true, sameSite: \"lax\", path: \"/\" });\n res.json({ account: { id: existing[0].id, username, isAdmin: false, isStaff: false } });\n return;\n }"}]},{"id":"password-verification-is-inverted","scenario":"tracks/ecommerce/scenarios/01-account-password.json","targets":["ecommerce.feature.accounts.accounts.1c"],"desc":"Accept a wrong password instead of enforcing password verification.","file":"server/src/index.ts","edits":[{"find":" if (rows.length === 0 || !verifyPassword(password, rows[0].passwordHash)) {","replace":" if (rows.length === 0 || verifyPassword(password, rows[0].passwordHash)) {"}]},{"id":"correct-signin-is-refused","scenario":"tracks/ecommerce/scenarios/01-account-signout.json","targets":["ecommerce.feature.accounts.accounts.1d"],"desc":"Preserve wrong-password refusal but reject an otherwise valid sign-in, preventing a signed-out account from returning.","file":"server/src/index.ts","edits":[{"find":" const acc = rows[0];\n const token = newToken();","replace":" const acc = rows[0];\n if (username === acc.username) {\n res.status(401).json({ error: \"sign in is unavailable\" });\n return;\n }\n const token = newToken();"}]},{"id":"reload-discards-session-identity","scenario":"tracks/ecommerce/scenarios/01-account-reload.json","targets":["ecommerce.spec.state-durability.session-reload.1e"],"desc":"Ignore the authenticated identity returned during initial page hydration.","file":"client/src/App.tsx","edits":[{"find":" setAccount(me.account);","replace":" setAccount(null);"}]},{"id":"purchase-does-not-broadcast-ranking","scenario":"tracks/ecommerce/scenarios/01-core.json","targets":["ecommerce.spec.live-state.ranking.2c"],"desc":"Commit the purchase but omit the catalog broadcast that updates already-open rankings.","file":"server/src/index.ts","edits":[{"find":" lastCatalogJson = json;\n io.emit(\"items:update\", { items: catalog });","replace":" lastCatalogJson = json;\n // mutant: changed catalog state is not broadcast"}]},{"id":"signed-out-purchase-uses-default-account","scenario":"tracks/ecommerce/scenarios/progression-signed-out-purchase.json","targets":["ecommerce.spec.access-control.signed-out-purchase.3a"],"desc":"Expose purchase controls to guests and let the purchase route charge the first stored account when no caller is authenticated.","file":"client/src/App.tsx","edits":[{"find":" const canBuy = !!account && !account.isAdmin && !account.isStaff;","replace":" const canBuy = !account || (!account.isAdmin && !account.isStaff);"},{"file":"server/src/index.ts","find":" \"/api/items/:id/buy\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const itemId = Number(req.params.id);\n const accountId = req.account!.id;","replace":" \"/api/items/:id/buy\",\n asyncHandler(async (req, res) => {\n const itemId = Number(req.params.id);\n const accountId = req.account?.id ?? (await pool.query(`SELECT id FROM account ORDER BY id LIMIT 1`)).rows[0].id;"}]},{"id":"purchase-stock-change-is-not-broadcast--01-buying","scenario":"tracks/ecommerce/scenarios/01-buying.json","targets":["ecommerce.spec.live-state.purchase-stock.3b"],"desc":"Commit purchases without broadcasting their stock changes, breaking live stock and sold-out visibility.","file":"server/src/index.ts","edits":[{"find":" lastCatalogJson = json;\n io.emit(\"items:update\", { items: catalog });","replace":" lastCatalogJson = json;\n // mutant: changed stock is not broadcast"}]},{"id":"purchase-stock-change-is-not-broadcast--stock-limit","scenario":"tracks/ecommerce/scenarios/progression-stock-limit.json","targets":["ecommerce.spec.concurrency-safety.stock-limit.3d"],"desc":"Commit purchases without broadcasting their stock changes, breaking live stock and sold-out visibility.","file":"server/src/index.ts","edits":[{"find":" lastCatalogJson = json;\n io.emit(\"items:update\", { items: catalog });","replace":" lastCatalogJson = json;\n // mutant: changed stock is not broadcast"}]},{"id":"restock-race-records-wrong-order-total","scenario":"tracks/ecommerce/scenarios/01-restock-race.json","targets":["ecommerce.spec.concurrency-safety.restock-race.202a"],"desc":"Purchases preserve stock and visible order counts but record the wrong booked total. Native mixed-race reconciliation must reject them.","file":"server/src/index.ts","edits":[{"find":" [accountId, price]\n );","replace":" [accountId, Number(price) + 1]\n );"}]},{"id":"direct-purchase-order-total-is-offset","scenario":"tracks/ecommerce/scenarios/progression-purchasing.json","targets":["ecommerce.feature.purchasing.purchase-order.3c"],"desc":"Record a direct purchase one dollar above the stored price.","file":"server/src/index.ts","edits":[{"find":" [accountId, price]\n );","replace":" [accountId, Number(price) + 1]\n );"}]},{"id":"reload-hydrates-an-empty-cart","scenario":"tracks/ecommerce/scenarios/01-cart.json","targets":["ecommerce.spec.state-durability.cart-reload.4b"],"desc":"Return an empty cart from both reload hydration paths while preserving later live cart broadcasts.","file":"server/src/index.ts","edits":[{"find":" const state = await buildCartState(req.account!.id);\n res.json(state);","replace":" await buildCartState(req.account!.id);\n res.json({ items: [], total: 0 });"},{"find":" const cartState = await buildCartState(acc.id);\n socket.emit(\"cart:update\", cartState);","replace":" await buildCartState(acc.id);\n socket.emit(\"cart:update\", { items: [], total: 0 });"}]},{"id":"signed-out-visitors-do-not-see-reviews","scenario":"tracks/ecommerce/scenarios/01-review-visibility.json","targets":["ecommerce.feature.reviews.reviews.6a"],"desc":"Hide an item's reviews whenever the viewer is signed out.","file":"client/src/App.tsx","edits":[{"find":" {reviews.length === 0 ? (","replace":" {!account || reviews.length === 0 ? ("}]},{"id":"review-average-update-is-not-broadcast","scenario":"tracks/ecommerce/scenarios/01-review-rating-live.json","targets":["ecommerce.spec.live-state.rating.6c"],"desc":"Return the new average to the submitter but omit the live review update to other viewers.","file":"server/src/index.ts","edits":[{"find":" io.emit(\"review:update\", { itemId, reviews, average });","replace":" // mutant: other open review views do not receive the new average"}]},{"id":"admin-warehouse-view-drops-one-location","scenario":"tracks/ecommerce/scenarios/01-warehouse-admin-staff.json","targets":["ecommerce.feature.warehouse-admin.warehouse-view.7b"],"desc":"Render only 23 of the 24 item-by-warehouse stock locations.","file":"client/src/App.tsx","edits":[{"find":" {admin.locations.map((loc) => {","replace":" {admin.locations.slice(0, 23).map((loc) => {"}]},{"id":"unauthenticated-direct-purchase-uses-default-account","scenario":"tracks/ecommerce/scenarios/01-purchase-session.json","targets":["ecommerce.spec.access-control.purchase-session.101a"],"desc":"Remove purchase authentication and attribute unauthenticated requests to a default account.","file":"server/src/index.ts","edits":[{"find":" \"/api/items/:id/buy\",\n requireAuth,","replace":" \"/api/items/:id/buy\","},{"find":" const itemId = Number(req.params.id);\n const accountId = req.account!.id;\n\n const client = await pool.connect();","replace":" const itemId = Number(req.params.id);\n const accountId = req.account?.id ?? 1;\n\n const client = await pool.connect();"}]},{"id":"direct-purchase-is-attributed-to-previous-account","scenario":"tracks/ecommerce/scenarios/01-purchase-attribution.json","targets":["ecommerce.spec.access-control.purchase-attribution.102a"],"desc":"Create the direct-purchase order for the preceding account id rather than the authenticated caller.","file":"server/src/index.ts","edits":[{"find":" const itemId = Number(req.params.id);\n const accountId = req.account!.id;\n\n const client = await pool.connect();","replace":" const itemId = Number(req.params.id);\n const accountId = req.account!.id - 1;\n\n const client = await pool.connect();"}]},{"id":"direct-purchase-uses-constant-price","scenario":"tracks/ecommerce/scenarios/01-server-price.json","targets":["ecommerce.spec.transactional-integrity.server-price.104a"],"desc":"Create a direct-purchase order at a hard-coded price instead of the current stored price.","file":"server/src/index.ts","edits":[{"find":" [accountId, price]\n );","replace":" [accountId, \"1.00\"]\n );"}]},{"id":"account-state-reload-discards-session","scenario":"tracks/ecommerce/scenarios/progression-account-state-reload.json","targets":["ecommerce.spec.state-durability.account-state-recovery.105a"],"desc":"Discard the authenticated account during reload hydration, making its cart and orders unavailable.","file":"client/src/App.tsx","edits":[{"find":" setAccount(me.account);","replace":" setAccount(null);"}]},{"id":"offline-event-clears-account-state","scenario":"tracks/ecommerce/scenarios/progression-account-state-reconnect.json","targets":["ecommerce.spec.state-durability.account-state-recovery.105b"],"desc":"Treat a temporary offline event as a sign-out and clear the account and cart state.","file":"client/src/App.tsx","edits":[{"find":" useEffect(() => {\n const socket = io({ path: \"/socket.io\" });","replace":" useEffect(() => {\n const clearAccountOffline = () => {\n setAccount(null);\n setCart({ items: [], total: 0 });\n };\n window.addEventListener(\"offline\", clearAccountOffline, { once: true });\n const socket = io({ path: \"/socket.io\" });"}]},{"id":"purchase-does-not-decrement-warehouse-stock","scenario":"tracks/ecommerce/scenarios/progression-books-balance.json","targets":["ecommerce.spec.transactional-integrity.books-balance.107b"],"desc":"Create purchase orders without decrementing their selected warehouse stock row.","file":"server/src/index.ts","edits":[{"find":" UPDATE stock s SET quantity = quantity - 1\n FROM target t","replace":" UPDATE stock s SET quantity = quantity\n FROM target t"}]},{"id":"review-route-skips-purchase-eligibility","scenario":"tracks/ecommerce/scenarios/01-review-eligibility.json","targets":["ecommerce.spec.access-control.review-eligibility.108a"],"desc":"Allow review creation even when the caller has never purchased the item.","file":"server/src/index.ts","edits":[{"find":" if (purchased.rowCount === 0) {","replace":" if (false && purchased.rowCount === 0) {"}]},{"id":"only-shipped-orders-earn-review-eligibility","scenario":"tracks/ecommerce/scenarios/01-review-eligibility.json","targets":["ecommerce.spec.access-control.review-eligibility.108b","ecommerce.spec.access-control.review-eligibility.108a"],"desc":"Incorrectly require an order to be shipped before its buyer may review the item. The same restriction also rejects the required successful buyer control in 108a; it does not independently test non-buyer denial.","file":"server/src/index.ts","edits":[{"find":" WHERE o.account_id = $1 AND oi.item_id = $2 LIMIT 1`,","replace":" WHERE o.account_id = $1 AND oi.item_id = $2 AND o.status = 'shipped' LIMIT 1`,"}]},{"id":"cart-update-accepts-negative-quantity","scenario":"tracks/ecommerce/scenarios/01-cart-boundary.json","targets":["ecommerce.spec.access-control.cart-boundary.109b"],"desc":"Accept a negative cart quantity update and persist it instead of refusing the named action.","file":"server/src/index.ts","edits":[{"find":" if (!Number.isInteger(quantity) || quantity < 1) {","replace":" if (!Number.isInteger(quantity)) {"}]},{"id":"oversell-no-row-lock","scenario":"tracks/ecommerce/scenarios/01-last-unit.json","targets":["ecommerce.spec.concurrency-safety.last-unit.201a","ecommerce.spec.concurrency-safety.last-unit.201b","ecommerce.spec.concurrency-safety.last-unit.201c"],"desc":"Drop the row lock so simultaneous buyers can select the same remaining units.","file":"server/src/index.ts","edits":[{"find":" FOR UPDATE\n LIMIT 1","replace":" LIMIT 1"}]},{"id":"purchase-read-write-loses-concurrent-stock","scenario":"tracks/ecommerce/scenarios/01-restock-race.json","targets":["ecommerce.spec.concurrency-safety.restock-race.202a"],"desc":"Replace atomic stock reservation with an unlocked read and absolute write. A fixed pause widens scheduling overlap; serial purchases and restocks retain their stock effects. Concurrent reservations or restocking can lose updates.","file":"server/src/index.ts","edits":[{"find":" const decrement = await client.query(\n `WITH target AS (\n SELECT item_id, warehouse_id FROM stock\n WHERE item_id = $1 AND quantity > 0\n ORDER BY warehouse_id\n FOR UPDATE\n LIMIT 1\n )\n UPDATE stock s SET quantity = quantity - 1\n FROM target t\n WHERE s.item_id = t.item_id AND s.warehouse_id = t.warehouse_id\n RETURNING s.item_id, s.warehouse_id`,\n [itemId]\n );\n","replace":" const snapshot = await client.query(\n `SELECT item_id, warehouse_id, quantity FROM stock\n WHERE item_id = $1 AND quantity > 0 ORDER BY warehouse_id LIMIT 1`, [itemId]\n );\n // Mutant: widen the unlocked read/write window without changing serial behavior.\n await new Promise(resolve => setTimeout(resolve, 500));\n const decrement = snapshot.rowCount === 0 ? snapshot : await client.query(\n `UPDATE stock SET quantity = $3 WHERE item_id = $1 AND warehouse_id = $2\n RETURNING item_id, warehouse_id`,\n [itemId, snapshot.rows[0].warehouse_id, snapshot.rows[0].quantity - 1]\n );\n"}]},{"id":"external-stock-polling-disabled","scenario":"tracks/ecommerce/scenarios/01-external-live-sync.json","targets":["ecommerce.spec.external-data-sync.external-stock.901a"],"desc":"Stop reconciling direct database changes while the server remains online.","file":"server/src/index.ts","edits":[{"find":" broadcastCatalog().catch((err) => console.error(\"poll broadcast failed\", err));","replace":" // mutant: direct database catalog changes are never reconciled"}]},{"id":"server-restart-does-not-resynchronize-catalog","scenario":"tracks/ecommerce/scenarios/01-external-server-restart-sync.json","targets":["ecommerce.spec.external-data-sync.external-stock.901c"],"desc":"After a server restart, omit both connection-time catalog hydration and periodic authoritative reconciliation.","file":"server/src/index.ts","edits":[{"find":" const catalog = await buildCatalog();\n socket.emit(\"items:update\", { items: catalog });","replace":" // mutant: reconnecting clients retain their pre-restart catalog"},{"find":" broadcastCatalog().catch((err) => console.error(\"poll broadcast failed\", err));","replace":" // mutant: restart recovery does not reconcile authoritative catalog state"}]},{"id":"reconnect-does-not-send-current-catalog","scenario":"tracks/ecommerce/scenarios/01-external-reconnect-sync.json","targets":["ecommerce.spec.external-data-sync.external-stock.901d"],"desc":"Follow catalog changes until the browser goes offline, then ignore updates after restoration.","file":"client/src/App.tsx","edits":[{"find":" const cartObservationRef = useRef(0);\n\n function applyCartResponse","replace":" const cartObservationRef = useRef(0);\n const acceptCatalogUpdates = useRef(true);\n useEffect(() => {\n const stopCatalogRecovery = () => { acceptCatalogUpdates.current = false; };\n window.addEventListener(\"offline\", stopCatalogRecovery);\n return () => window.removeEventListener(\"offline\", stopCatalogRecovery);\n }, []);\n\n function applyCartResponse"},{"find":" socket.on(\"items:update\", (payload: { items: Item[] }) => setItems(payload.items));","replace":" socket.on(\"items:update\", (payload: { items: Item[] }) => {\n if (acceptCatalogUpdates.current) setItems(payload.items);\n });"}]},{"id":"open-review-list-ignores-live-update","scenario":"tracks/ecommerce/scenarios/progression-open-list-live.json","targets":["ecommerce.spec.live-state.open-list.902a"],"desc":"Ignore committed review updates in a detail view that is already open.","file":"client/src/App.tsx","edits":[{"find":" setItemDetail({ reviews: payload.reviews, average: payload.average });","replace":" // mutant: the already-open review list ignores committed updates"}]},{"id":"open-review-list-renders-each-review-twice","scenario":"tracks/ecommerce/scenarios/progression-open-list-live.json","targets":["ecommerce.spec.live-state.open-list.902a"],"desc":"Render every committed review twice in the already-open list.","file":"client/src/App.tsx","edits":[{"find":" reviews.map((r) => (","replace":" [...reviews, ...reviews].map((r) => ("}]},{"id":"cancel-does-not-restore-stock-feature","scenario":"tracks/ecommerce/scenarios/02-order-cancellation-core.json","targets":["ecommerce.returns-pricing.cancellation-and-return.3a"],"desc":"Cancellation commits but restores zero units to each recorded warehouse row.","file":"server/src/index.ts","edits":[{"find":" [l.item_id, l.warehouse_id, l.quantity]\n );\n }\n await refundCredit(client, orderRow.rows[0]);\n await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);","replace":" [l.item_id, l.warehouse_id, 0]\n );\n }\n await refundCredit(client, orderRow.rows[0]);\n await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);"}]},{"id":"cancellation-accounting-loses-stock-restoration","scenario":"tracks/ecommerce/scenarios/02-invariants.json","targets":["ecommerce.returns-pricing.refund-accounting.203a"],"desc":"Cancellation removes revenue and changes order status, but loses the original warehouse stock restoration. The native refund-accounting assertion must detect this.","file":"server/src/index.ts","edits":[{"find":" [l.item_id, l.warehouse_id, l.quantity]\n );\n }\n await refundCredit(client, orderRow.rows[0]);\n await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);","replace":" [l.item_id, l.warehouse_id, 0]\n );\n }\n await refundCredit(client, orderRow.rows[0]);\n await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);"}]},{"id":"cancel-does-not-restore-stock-fresh-client","scenario":"tracks/ecommerce/scenarios/02-self-contained.json","targets":["ecommerce.inventory-operations.stock-conservation.202b","ecommerce.inventory-operations.stock-conservation.202c"],"desc":"Cancellation commits but restores zero units, so a fresh client reads the persisted shortfall.","file":"server/src/index.ts","edits":[{"find":" [l.item_id, l.warehouse_id, l.quantity]\n );\n }\n await refundCredit(client, orderRow.rows[0]);\n await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);","replace":" [l.item_id, l.warehouse_id, 0]\n );\n }\n await refundCredit(client, orderRow.rows[0]);\n await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);"}]},{"id":"cancel-restores-stock-but-keeps-pending-status","scenario":"tracks/ecommerce/scenarios/02-order-cancellation-history.json","targets":["ecommerce.returns-pricing.cancellation-and-return.3b"],"desc":"Cancellation restores allocations but writes pending back to order history.","file":"server/src/index.ts","edits":[{"find":"await client.query(`UPDATE orders SET status = 'cancelled' WHERE id = $1`, [orderId]);","replace":"await client.query(`UPDATE orders SET status = 'pending' WHERE id = $1`, [orderId]);"}]},{"id":"operator-authorization-allows-customer-transfer","scenario":"tracks/ecommerce/scenarios/02-strengthened.json","targets":["ecommerce.operations-access.operator-authorization.201a"],"desc":"The transfer route replaces its administrator gate with ordinary authentication.","file":"server/src/index.ts","edits":[{"find":"app.post(\n \"/api/admin/transfer\",\n requireAdmin,","replace":"app.post(\n \"/api/admin/transfer\",\n requireAuth,"}]},{"id":"customer-can-ship-order-direct-1-1","scenario":"tracks/ecommerce/scenarios/02-server-actions.json","targets":["ecommerce.operations-access.operator-authorization.201c"],"desc":"The shipping route replaces its staff gate with ordinary authentication.","file":"server/src/index.ts","edits":[{"find":"app.post(\n \"/api/fulfilment/ship\",\n requireStaff,","replace":"app.post(\n \"/api/fulfilment/ship\",\n requireAuth,"}]},{"id":"customer-can-cancel-foreign-order-1-1","scenario":"tracks/ecommerce/scenarios/02-server-actions.json","targets":["ecommerce.operations-access.order-owner.204a"],"desc":"Cancellation retains authentication and pending-state validation but drops order ownership.","file":"server/src/index.ts","edits":[{"find":"app.post(\n \"/api/orders/:id/cancel\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const orderId = Number(req.params.id);\n const accountId = req.account!.id;\n const client = await pool.connect();\n try {\n await client.query(\"BEGIN\");\n const orderRow = await client.query(`SELECT * FROM orders WHERE id = $1 FOR UPDATE`, [orderId]);\n if (orderRow.rowCount === 0 || orderRow.rows[0].account_id !== accountId) {","replace":"app.post(\n \"/api/orders/:id/cancel\",\n requireAuth,\n asyncHandler(async (req, res) => {\n const orderId = Number(req.params.id);\n const accountId = req.account!.id;\n const client = await pool.connect();\n try {\n await client.query(\"BEGIN\");\n const orderRow = await client.query(`SELECT * FROM orders WHERE id = $1 FOR UPDATE`, [orderId]);\n if (orderRow.rowCount === 0) {"}]},{"id":"queue-depth-lags-one-order","scenario":"tracks/ecommerce/scenarios/02-fulfilment-live.json","targets":["ecommerce.operations-access.fulfilment-queue.1a"],"desc":"The queue renders every order but its visible depth remains one behind.","file":"client/src/App.tsx","edits":[{"find":"<strong data-role=\"queue-depth\">{queue.depth}</strong>","replace":"<strong data-role=\"queue-depth\">{Math.max(0, queue.depth - 1)}</strong>"}]},{"id":"customer-sees-fulfilment-navigation","scenario":"tracks/ecommerce/scenarios/02-features.json","targets":["ecommerce.operations-access.fulfilment-queue.1d"],"desc":"Expose the protected staff area to signed-in customers, including its navigation and content.","file":"client/src/App.tsx","edits":[{"find":" {account && (account.isStaff || account.isAdmin) && (\n <button className=\"btn btn-ghost btn-sm\" data-role=\"staff-link\"","replace":" {account && (\n <button className=\"btn btn-ghost btn-sm\" data-role=\"staff-link\""},{"find":"{fulfilmentOpen && account && (account.isAdmin || account.isStaff) && (","replace":"{fulfilmentOpen && account && ("}]},{"id":"progression-customer-sees-fulfilment-content","scenario":"tracks/ecommerce/scenarios/02-fulfilment-access.json","targets":["ecommerce.spec.access-control.fulfilment-area-boundary.1d"],"desc":"Expose the protected staff area to signed-in customers, including its navigation and content.","file":"client/src/App.tsx","edits":[{"find":" {account && (account.isStaff || account.isAdmin) && (\n <button className=\"btn btn-ghost btn-sm\" data-role=\"staff-link\"","replace":" {account && (\n <button className=\"btn btn-ghost btn-sm\" data-role=\"staff-link\""},{"find":"{fulfilmentOpen && account && (account.isAdmin || account.isStaff) && (","replace":"{fulfilmentOpen && account && ("}]},{"id":"transfer-debits-source-without-crediting-existing-destination","scenario":"tracks/ecommerce/scenarios/02-strengthened.json","targets":["ecommerce.inventory-operations.stock-conservation.202a","ecommerce.inventory-operations.warehouse-transfer.2a","ecommerce.operations-access.operator-authorization.201a"],"desc":"A transfer debits the source row but inserts or increments the destination by zero, violating both directional movement and total conservation while still committing successfully. The missing credit also breaks the authorized administrator transfer control in 201a; it does not independently test unauthorized access.","file":"server/src/index.ts","edits":[{"find":" await client.query(\n `INSERT INTO stock (item_id, warehouse_id, quantity) VALUES ($1, $2, $3)\n ON CONFLICT (item_id, warehouse_id) DO UPDATE SET quantity = stock.quantity + excluded.quantity`,\n [itemId, toWarehouseId, qty]\n );","replace":" await client.query(\n `INSERT INTO stock (item_id, warehouse_id, quantity) VALUES ($1, $2, $3)\n ON CONFLICT (item_id, warehouse_id) DO UPDATE SET quantity = stock.quantity + excluded.quantity`,\n [itemId, toWarehouseId, 0]\n );"}]},{"id":"transfer-warehouse-totals-omit-destination-credit","scenario":"tracks/ecommerce/scenarios/02-transfer-totals.json","targets":["ecommerce.inventory-operations.warehouse-transfer.2b"],"desc":"A transfer debits the source but inserts or increments the destination by zero, so the live warehouse totals do not move in opposite directions.","file":"server/src/index.ts","edits":[{"find":" await client.query(\n `INSERT INTO stock (item_id, warehouse_id, quantity) VALUES ($1, $2, $3)\n ON CONFLICT (item_id, warehouse_id) DO UPDATE SET quantity = stock.quantity + excluded.quantity`,\n [itemId, toWarehouseId, qty]\n );","replace":" await client.query(\n `INSERT INTO stock (item_id, warehouse_id, quantity) VALUES ($1, $2, $3)\n ON CONFLICT (item_id, warehouse_id) DO UPDATE SET quantity = stock.quantity + excluded.quantity`,\n [itemId, toWarehouseId, 0]\n );"}]},{"id":"transfer-overdraft-guard-removed","scenario":"tracks/ecommerce/scenarios/02-transfer-overdraw.json","targets":["ecommerce.inventory-operations.warehouse-transfer.2c"],"desc":"The transaction no longer rejects a source warehouse with insufficient quantity, allowing an overdrawn transfer to commit.","file":"server/src/index.ts","edits":[{"find":" if (available < qty) {\n await client.query(\"ROLLBACK\");\n res.status(409).json({ error: \"warehouse does not have enough stock to transfer\" });\n return;\n }","replace":" // mutant: insufficient source stock is not rejected"}]},{"id":"low-stock-excludes-boundary-ten","scenario":"tracks/ecommerce/scenarios/02-low-stock.json","targets":["ecommerce.inventory-operations.operational-views.5a"],"desc":"The low-stock view uses a strict boundary and omits items with exactly ten units.","file":"server/src/index.ts","edits":[{"find":" .filter((it) => it.stock <= 10)","replace":" .filter((it) => it.stock < 10)"}]},{"id":"category-totals-ignore-pending-purchases","scenario":"tracks/ecommerce/scenarios/02-operational-category-totals.json","targets":["ecommerce.inventory-operations.operational-views.5b"],"desc":"Category totals include only shipped orders, so a newly accepted pending purchase is absent from the live totals.","file":"server/src/index.ts","edits":[{"find":" COALESCE(SUM(CASE WHEN oi.returned = false AND o.status != 'cancelled' THEN oi.quantity ELSE 0 END), 0)::int AS units,\n COALESCE(SUM(CASE WHEN oi.returned = false AND o.status != 'cancelled' THEN oi.quantity * oi.price ELSE 0 END), 0) AS revenue","replace":" COALESCE(SUM(CASE WHEN oi.returned = false AND o.status = 'shipped' THEN oi.quantity ELSE 0 END), 0)::int AS units,\n COALESCE(SUM(CASE WHEN oi.returned = false AND o.status = 'shipped' THEN oi.quantity * oi.price ELSE 0 END), 0) AS revenue"}]},{"id":"recommendations-ignore-pending-purchases","scenario":"tracks/ecommerce/scenarios/02-operational-recommendations.json","targets":["ecommerce.inventory-operations.operational-views.5c"],"desc":"Personal recommendations derive categories only from shipped orders, so a newly accepted pending purchase has no influence.","file":"server/src/index.ts","edits":[{"find":" WHERE o.account_id = $1 AND oi.returned = false AND o.status != 'cancelled'`,","replace":" WHERE o.account_id = $1 AND oi.returned = false AND o.status = 'shipped'`,"}]},{"id":"purchases-do-not-affect-best-sellers","scenario":"tracks/ecommerce/scenarios/02-operational-best-sellers.json","targets":["ecommerce.inventory-operations.operational-views.5d"],"desc":"Rank signed-out recommendations without purchase counts.","file":"server/src/index.ts","edits":[{"find":" WHERE oi.returned = false AND o.status != 'cancelled'\n GROUP BY oi.item_id","replace":" WHERE FALSE\n GROUP BY oi.item_id"}]},{"id":"queue-warehouse-reports-west","scenario":"tracks/ecommerce/scenarios/02-queue-warehouse.json","targets":["ecommerce.operations-access.fulfilment-queue.1b"],"desc":"The queue renders West for the deterministic Desk Lamp allocation even though the order reserved stock in East.","file":"client/src/App.tsx","edits":[{"find":"{line.warehouse}","replace":"{\"West\"}"}]},{"id":"transfer-overwrites-concurrent-purchase-with-stale-stock","scenario":"tracks/ecommerce/scenarios/02-server-actions.json","targets":["ecommerce.inventory-operations.stock-conservation.202d"],"desc":"The Headphones transfer reads East stock before taking a row lock, signals the concurrent purchase to proceed, waits for that purchase to commit, then writes its stale source result and credits West. Both actions return accepted, but the stale transfer write loses the purchase's decrement, proving the exact-total check detects broken cross-operation synchronization.","file":"server/src/index.ts","edits":[{"find":"const PORT = Number(process.env.PORT) || 6301;\n\nconst app = express();","replace":"const PORT = Number(process.env.PORT) || 6301;\n\nlet mutationTransferCapturedResolve!: () => void;\nconst mutationTransferCaptured = new Promise<void>((resolve) => {\n mutationTransferCapturedResolve = resolve;\n});\nlet mutationBuyCommittedResolve!: () => void;\nconst mutationBuyCommitted = new Promise<void>((resolve) => {\n mutationBuyCommittedResolve = resolve;\n});\n\nconst app = express();"},{"find":" const itemId = Number(req.params.id);\n const accountId = req.account!.id;\n\n const client = await pool.connect();","replace":" const itemId = Number(req.params.id);\n const accountId = req.account!.id;\n const mutationBuyItem = await pool.query(`SELECT name FROM item WHERE id = $1`, [itemId]);\n if (mutationBuyItem.rows[0]?.name === \"Headphones\") await mutationTransferCaptured;\n\n const client = await pool.connect();"},{"find":" await client.query(\n `INSERT INTO order_item (order_id, item_id, item_name, quantity, price, warehouse_id) VALUES ($1, $2, $3, 1, $4, $5)`,\n [orderId, itemId, name, price, warehouseId]\n );\n await client.query(\"COMMIT\");","replace":" await client.query(\n `INSERT INTO order_item (order_id, item_id, item_name, quantity, price, warehouse_id) VALUES ($1, $2, $3, 1, $4, $5)`,\n [orderId, itemId, name, price, warehouseId]\n );\n await client.query(\"COMMIT\");\n if (name === \"Headphones\") mutationBuyCommittedResolve();"},{"find":" if (\n !Number.isInteger(itemId) ||\n !Number.isInteger(fromWarehouseId) ||\n !Number.isInteger(toWarehouseId) ||\n !Number.isInteger(qty) ||\n qty < 1 ||\n fromWarehouseId === toWarehouseId\n ) {\n res.status(400).json({ error: \"invalid transfer request\" });\n return;\n }\n const client = await pool.connect();","replace":" if (\n !Number.isInteger(itemId) ||\n !Number.isInteger(fromWarehouseId) ||\n !Number.isInteger(toWarehouseId) ||\n !Number.isInteger(qty) ||\n qty < 1 ||\n fromWarehouseId === toWarehouseId\n ) {\n res.status(400).json({ error: \"invalid transfer request\" });\n return;\n }\n const mutationTransferItem = await pool.query(`SELECT name FROM item WHERE id = $1`, [itemId]);\n const mutationStaleSource = mutationTransferItem.rows[0]?.name === \"Headphones\"\n ? await pool.query(\n `SELECT quantity FROM stock WHERE item_id = $1 AND warehouse_id = $2`,\n [itemId, fromWarehouseId]\n )\n : null;\n const mutationStaleSourceQuantity = mutationStaleSource?.rows[0]?.quantity ?? null;\n if (mutationStaleSourceQuantity != null) {\n mutationTransferCapturedResolve();\n await mutationBuyCommitted;\n }\n const client = await pool.connect();"},{"find":" await client.query(`UPDATE stock SET quantity = quantity - $1 WHERE item_id = $2 AND warehouse_id = $3`, [\n qty,\n itemId,\n fromWarehouseId,\n ]);","replace":" if (mutationStaleSourceQuantity != null) {\n await client.query(\n `UPDATE stock SET quantity = $1 WHERE item_id = $2 AND warehouse_id = $3`,\n [mutationStaleSourceQuantity - qty, itemId, fromWarehouseId]\n );\n } else {\n await client.query(`UPDATE stock SET quantity = quantity - $1 WHERE item_id = $2 AND warehouse_id = $3`, [\n qty,\n itemId,\n fromWarehouseId,\n ]);\n }"}]},{"id":"progression-profile-address-is-discarded","scenario":"tracks/ecommerce/scenarios/progression-customer-profile.json","targets":["ecommerce.spec.state-durability.customer-profile-reload.620a"],"desc":"Clear stored profile addresses on server initialization, after ordinary saves work. Reload retains the profile; a runtime restart loses it.","file":"server/src/progression.ts","edits":[{"find":" ALTER TABLE account ADD COLUMN IF NOT EXISTS profile_address text NOT NULL DEFAULT '';","replace":" ALTER TABLE account ADD COLUMN IF NOT EXISTS profile_address text NOT NULL DEFAULT '';\n UPDATE account SET profile_address = '';"}]},{"id":"progression-profile-reads-another-account","scenario":"tracks/ecommerce/scenarios/progression-customer-profile.json","targets":["ecommerce.spec.access-control.customer-profile-privacy.620b"],"desc":"Return another account's saved profile.","file":"server/src/progression.ts","edits":[{"find":"account ? one(`SELECT profile_name, profile_address FROM account WHERE id = $1`, [account.id]) : null,","replace":"account ? one(`SELECT profile_name, profile_address FROM account WHERE profile_address != '' ORDER BY id DESC LIMIT 1`) : null,"}]},{"id":"progression-staff-tools-are-hidden","scenario":"tracks/ecommerce/scenarios/progression-staff-access.json","targets":["ecommerce.progression.staff-access.staff-access.601a"],"desc":"Deny administrators entry to staff tools while keeping ordinary staff access working.","file":"client/src/App.tsx","edits":[{"find":"{account && (account.isStaff || account.isAdmin) && (\n <button className=\"btn btn-ghost btn-sm\" data-role=\"staff-link\"","replace":"{account && (account.isStaff && !account.isAdmin) && (\n <button className=\"btn btn-ghost btn-sm\" data-role=\"staff-link\""}]},{"id":"progression-customer-sees-staff-tools","scenario":"tracks/ecommerce/scenarios/progression-staff-access.json","targets":["ecommerce.spec.access-control.staff-area-boundary.601b"],"desc":"Expose the protected staff area to signed-in customers, including its navigation and content.","file":"client/src/App.tsx","edits":[{"find":"{account && (account.isStaff || account.isAdmin) && (\n <button className=\"btn btn-ghost btn-sm\" data-role=\"staff-link\"","replace":"{account && (\n <button className=\"btn btn-ghost btn-sm\" data-role=\"staff-link\""},{"find":"{fulfilmentOpen && account && (account.isAdmin || account.isStaff) && (","replace":"{fulfilmentOpen && account && ("}]},{"id":"progression-staff-role-is-lost-on-restart","scenario":"tracks/ecommerce/scenarios/progression-staff-roles.json","targets":["ecommerce.spec.state-durability.staff-role-reload.621a"],"desc":"Clear stored staff roles on server initialization while preserving authorized assignment and staff access refusal.","file":"server/src/progression.ts","edits":[{"find":" ALTER TABLE account ADD COLUMN IF NOT EXISTS staff_role text NOT NULL DEFAULT '';","replace":" ALTER TABLE account ADD COLUMN IF NOT EXISTS staff_role text NOT NULL DEFAULT '';\n UPDATE account SET staff_role = '';"}]},{"id":"staff-role-write-precedes-denial","scenario":"tracks/ecommerce/scenarios/progression-staff-roles.json","targets":["ecommerce.spec.access-control.staff-role-boundary.621b","ecommerce.spec.access-control.staff-role-revocation.621d"],"desc":"Save an unauthorized role change before returning a forbidden response, including after administrator access is revoked.","file":"server/src/progression.ts","edits":[{"find":"app.put(\"/api/staff/:id/role\", dependencies.requireAdmin,","replace":"app.put(\"/api/staff/:id/role\", dependencies.requireStaff,"},{"find":" await recordActivity(deps.pool, req.account!.id, \"assign role\", updated.rows[0].username);","replace":" if (!req.account!.isAdmin) { res.status(403).json({ error: \"admin only\" }); return; }\n await recordActivity(deps.pool, req.account!.id, \"assign role\", updated.rows[0].username);"}]},{"id":"progression-staff-can-assign-roles","scenario":"tracks/ecommerce/scenarios/progression-staff-roles.json","targets":["ecommerce.spec.access-control.staff-role-boundary.621b","ecommerce.spec.access-control.staff-role-revocation.621d"],"desc":"Allow staff members to assign roles, including after administrator access is revoked.","file":"server/src/progression.ts","edits":[{"find":"app.put(\"/api/staff/:id/role\", dependencies.requireAdmin,","replace":"app.put(\"/api/staff/:id/role\", dependencies.requireStaff,"}]},{"id":"progression-catalog-product-name-is-not-published","scenario":"tracks/ecommerce/scenarios/progression-catalog-management.json","targets":["ecommerce.progression.catalog-management.catalog-management.622a","ecommerce.progression.catalog-management.catalog-management.622b"],"desc":"The public product card omits the submitted name while retaining the product and its variants. The variants assertion also cannot open that same named product card.","file":"client/src/App.tsx","edits":[{"find":" <span className=\"item-name\" data-role=\"item-name\">{item.name}</span>","replace":" <span className=\"item-name\" data-role=\"item-name\">{item.name === \"Travel Mug\" ? \"\" : item.name}</span>"}]},{"id":"progression-catalog-variants-are-discarded","scenario":"tracks/ecommerce/scenarios/progression-catalog-management.json","targets":["ecommerce.progression.catalog-management.catalog-management.622b"],"desc":"Store the submitted product without its named variants.","file":"server/src/progression.ts","edits":[{"find":" [name, category, price.toFixed(2), variants],","replace":" [name, category, price.toFixed(2), []],"}]},{"id":"progression-payment-amount-is-zero","scenario":"tracks/ecommerce/scenarios/progression-core-business.json","targets":["ecommerce.progression.payment-records.payment-records.623a"],"desc":"Record zero as the paid amount.","file":"server/src/progression.ts","edits":[{"find":"[accountId, total, promotionId, discount],","replace":"[accountId, 0, promotionId, discount],"}]},{"id":"progression-payment-record-is-duplicated","scenario":"tracks/ecommerce/scenarios/progression-core-business.json","targets":["ecommerce.spec.transactional-integrity.payment-deduplication.623b"],"desc":"Render a duplicate payment record.","file":"client/src/App.tsx","edits":[{"find":" <div data-role=\"payment-record\">\n <span data-role=\"payment-status\">{o.paymentStatus}</span>","replace":" <div data-role=\"payment-record\">\n <span data-role=\"payment-status\">{o.paymentStatus}</span>\n <span data-role=\"payment-record\"><span data-role=\"payment-status\">{o.paymentStatus}</span></span>"}]},{"id":"progression-support-intake-is-disabled","scenario":"tracks/ecommerce/scenarios/progression-support-intake.json","targets":["ecommerce.progression.support-intake.support-intake.610a"],"desc":"Do not accept new support cases.","file":"server/src/progression.ts","edits":[{"find":"app.post(\"/api/support/cases\",","replace":"app.post(\"/api/mutation-disabled-support/cases\","}]},{"id":"progression-support-triage-update-is-disabled","scenario":"tracks/ecommerce/scenarios/progression-support-triage.json","targets":["ecommerce.progression.support-triage.support-assignment.611a","ecommerce.progression.support-triage.support-priority.611b","ecommerce.progression.support-triage.support-status.611c"],"desc":"Do not persist support triage updates.","file":"server/src/progression.ts","edits":[{"find":"app.put(\"/api/support/cases/:id\",","replace":"app.put(\"/api/mutation-disabled-support/cases/:id\","}]},{"id":"progression-support-history-is-not-persisted","scenario":"tracks/ecommerce/scenarios/progression-support-history.json","targets":["ecommerce.spec.state-durability.support-history-reload.612a"],"desc":"Delete stored support cases on server initialization, preserving case creation, initial history, and reload before a runtime restart.","file":"server/src/progression.ts","edits":[{"find":" UNIQUE (order_id)\n );\n `);","replace":" UNIQUE (order_id)\n );\n DELETE FROM support_case;\n `);"}]},{"id":"progression-support-history-leaks","scenario":"tracks/ecommerce/scenarios/progression-support-history.json","targets":["ecommerce.spec.access-control.support-history-privacy.612b"],"desc":"Return all support cases to each customer.","file":"server/src/progression.ts","edits":[{"find":"const where = account.isAdmin || account.isStaff ? \"TRUE\" : \"sc.account_id = $1\";\n const values = account.isAdmin || account.isStaff ? [] : [account.id];","replace":"const where = \"TRUE\";\n const values: unknown[] = [];"}]},{"id":"progression-managed-support-is-not-shared","scenario":"tracks/ecommerce/scenarios/progression-managed-support-shared.json","targets":["ecommerce.spec.live-state.managed-support.613a"],"desc":"Retain the previous support snapshot on live socket updates. Explicit refresh still loads committed replies and status.","file":"client/src/App.tsx","edits":[{"find":" socket.on(\"progression:update\", setProgression);","replace":" socket.on(\"progression:update\", (state: any) => setProgression((current: any) => ({ ...state, support: current.support })));"}]},{"id":"progression-managed-support-leaks","scenario":"tracks/ecommerce/scenarios/progression-managed-support-privacy.json","targets":["ecommerce.spec.access-control.managed-support-privacy.613b"],"desc":"Return all managed support cases to each customer.","file":"server/src/progression.ts","edits":[{"find":"const where = account.isAdmin || account.isStaff ? \"TRUE\" : \"sc.account_id = $1\";\n const values = account.isAdmin || account.isStaff ? [] : [account.id];","replace":"const where = \"TRUE\";\n const values: unknown[] = [];"}]},{"id":"progression-promotion-discount-is-offset","scenario":"tracks/ecommerce/scenarios/progression-promotion-rules.json","targets":["ecommerce.progression.promotion-rules.promotion-rule-values.620a"],"desc":"Save a promotion with an incorrect discount while preserving the route, other fields, and staff-only authorization.","file":"server/src/progression.ts","edits":[{"find":" [code, discount, start, end, limit, req.account!.id],","replace":" [code, discount + 1, start, end, limit, req.account!.id],"}]},{"id":"progression-customer-can-create-promotions","scenario":"tracks/ecommerce/scenarios/progression-promotion-rules.json","targets":["ecommerce.spec.access-control.promotion-management-boundary.620b"],"desc":"Allow customers to create promotions.","file":"server/src/progression.ts","edits":[{"find":"app.post(\"/api/promotions\", dependencies.requireStaff,","replace":"app.post(\"/api/promotions\", dependencies.requireAuth,"}]},{"id":"progression-active-promotion-cannot-apply","scenario":"tracks/ecommerce/scenarios/progression-promotion-checkout.json","targets":["ecommerce.progression.promotion-checkout.promotion-checkout-active.621a"],"desc":"Do not apply active promotions.","file":"server/src/progression.ts","edits":[{"find":"app.post(\"/api/cart/promotion\",","replace":"app.post(\"/api/mutation-disabled-cart/promotion\","}]},{"id":"progression-notification-preferences-do-not-save","scenario":"tracks/ecommerce/scenarios/progression-notification-preferences.json","targets":["ecommerce.spec.state-durability.notification-preferences-reload.630a"],"desc":"Do not save notification preferences.","file":"server/src/progression.ts","edits":[{"find":"app.put(\"/api/notifications/preferences\",","replace":"app.put(\"/api/mutation-disabled-notifications/preferences\","}]},{"id":"progression-notifications-leak-across-accounts","scenario":"tracks/ecommerce/scenarios/progression-notification-preferences.json","targets":["ecommerce.spec.access-control.notification-preferences-privacy.630b"],"desc":"Save one customer's notification choice to every other account.","file":"server/src/progression.ts","edits":[{"find":"await deps.pool.query(`UPDATE account SET notify_order = $1, notify_stock = $2 WHERE id = $3`,\n [order, stock, req.account!.id]);","replace":"await deps.pool.query(`UPDATE account SET notify_order = $1, notify_stock = $2 WHERE id = $3 OR id != $3`,\n [order, stock, req.account!.id]);"}]},{"id":"stock-alert-delivery-is-suppressed","scenario":"tracks/ecommerce/scenarios/progression-stock-alert-delivery.json","targets":["ecommerce.progression.stock-alerts.stock-alert-delivery.631c"],"desc":"A request is accepted but restored stock does not deliver its alert.","file":"server/src/progression.ts","edits":[{"find":"for (const request of requests.rows) {","replace":"for (const request of requests.rows.slice(0, 0)) {"}]},{"id":"stock-alert-is-sent-after-every-restock","scenario":"tracks/ecommerce/scenarios/progression-stock-alerts.json","targets":["ecommerce.spec.transactional-integrity.stock-alert-deduplication.631a"],"desc":"Keep a delivered stock request active and use a new notification key on every restock, so its owner receives duplicate alerts.","file":"server/src/progression.ts","edits":[{"find":"`UPDATE stock_alert_request SET fulfilled = true\n WHERE item_id = $1 AND fulfilled = false RETURNING account_id`","replace":"`UPDATE stock_alert_request SET fulfilled = false\n WHERE item_id = $1 AND fulfilled = false RETURNING account_id`"},{"find":"VALUES ($1, 'stock', $2, $3) ON CONFLICT DO NOTHING","replace":"VALUES ($1, 'stock', $2::text || ':' || nextval('notification_id_seq')::text, $3) ON CONFLICT DO NOTHING"}]},{"id":"progression-stock-alerts-leak-across-accounts","scenario":"tracks/ecommerce/scenarios/progression-stock-alerts.json","targets":["ecommerce.spec.access-control.stock-alert-privacy.631b"],"desc":"Return stock alerts for all accounts.","file":"server/src/progression.ts","edits":[{"find":"account ? deps.pool.query(`SELECT * FROM notification WHERE account_id = $1 ORDER BY created_at DESC`, [account.id]) : { rows: [] },","replace":"account ? deps.pool.query(`SELECT * FROM notification ORDER BY created_at DESC`) : { rows: [] },"}]},{"id":"progression-faceted-filter-ignores-category","scenario":"tracks/ecommerce/scenarios/progression-faceted-filters.json","targets":["ecommerce.progression.faceted-search.faceted-search.401a"],"desc":"Ignore the selected catalog category.","file":"client/src/App.tsx","edits":[{"find":" return items.filter((item) => (!q || item.name.toLowerCase().includes(q))\n && (!categoryFilter || item.category === categoryFilter)","replace":" return items.filter((item) => (!q || item.name.toLowerCase().includes(q))\n && (!categoryFilter || true)"}]},{"id":"active-search-uses-purchase-ranking","scenario":"tracks/ecommerce/scenarios/progression-search-ordering.json","targets":["ecommerce.spec.search-ordering.search-ordering.402b"],"desc":"Keep purchase ranking when search text or filters are active.","file":"client/src/App.tsx","edits":[{"find":"const filtering = Boolean(q || categoryFilter || minimumPrice || maximumPrice || inStockOnly);","replace":"const filtering = false;"}]},{"id":"progression-pagination-always-shows-first-page","scenario":"tracks/ecommerce/scenarios/progression-faceted-pagination.json","targets":["ecommerce.progression.faceted-search.faceted-search.402a"],"desc":"Always show the first catalog page.","file":"client/src/App.tsx","edits":[{"find":".slice(searchPage * CATALOG_PAGE_SIZE,\n searchPage * CATALOG_PAGE_SIZE + CATALOG_PAGE_SIZE);","replace":".slice(0, CATALOG_PAGE_SIZE);"}]},{"id":"progression-expired-promotion-is-accepted","scenario":"tracks/ecommerce/scenarios/progression-promotion-checkout.json","targets":["ecommerce.progression.promotion-checkout.promotion-checkout-expired.621b"],"desc":"Accept an expired promotion code.","file":"server/src/progression.ts","edits":[{"find":"`SELECT * FROM promotion WHERE code = $1 AND start_at <= now() AND end_at >= now()\n AND redemptions < redemption_limit`, [code]","replace":"`SELECT * FROM promotion WHERE code = $1\n AND redemptions < redemption_limit`, [code]"}]},{"id":"progression-exhausted-promotion-is-accepted","scenario":"tracks/ecommerce/scenarios/progression-promotion-checkout.json","targets":["ecommerce.progression.promotion-checkout.promotion-checkout-exhausted.621c"],"desc":"Accept a promotion after its redemption limit.","file":"server/src/progression.ts","edits":[{"find":"`SELECT * FROM promotion WHERE code = $1 AND start_at <= now() AND end_at >= now()\n AND redemptions < redemption_limit`, [code]","replace":"`SELECT * FROM promotion WHERE code = $1 AND start_at <= now() AND end_at >= now()`, [code]"}]},{"id":"progression-restock-countdown-is-fixed","scenario":"tracks/ecommerce/scenarios/03-scheduled-restocks.json","targets":["ecommerce.l3.scheduled-restocks.scheduled-restocks.302a"],"desc":"Show a fixed restock countdown.","file":"client/src/ProgressionPanel.tsx","edits":[{"find":"{Math.max(0, Math.ceil((new Date(item.dueAt).valueOf() - Date.now()) / 1000))}","replace":"{90}"}]},{"id":"progression-due-restock-does-not-run","scenario":"tracks/ecommerce/scenarios/03-scheduled-restock-apply.json","targets":["ecommerce.l3.scheduled-restocks.scheduled-restocks.305a"],"desc":"Do not select due scheduled restocks.","file":"server/src/progression.ts","edits":[{"find":"`SELECT * FROM scheduled_restock WHERE due_at <= now() AND cancelled = false AND applied = false FOR UPDATE`,","replace":"`SELECT * FROM scheduled_restock WHERE false FOR UPDATE`,"}]},{"id":"progression-cancelled-restock-still-runs","scenario":"tracks/ecommerce/scenarios/03-scheduled-restock-cancel.json","targets":["ecommerce.l3.scheduled-restocks.scheduled-restocks.306a"],"desc":"Leave a scheduled restock active when it is cancelled.","file":"server/src/progression.ts","edits":[{"find":"`UPDATE scheduled_restock SET cancelled = true WHERE id = $1 AND applied = false RETURNING item_id`,","replace":"`UPDATE scheduled_restock SET cancelled = false WHERE id = $1 AND applied = false RETURNING item_id`,"}]},{"id":"progression-restart-timer-never-runs","scenario":"tracks/ecommerce/scenarios/03-server-time.json","targets":["ecommerce.l3.server-time.server-time.312a"],"desc":"Do not process work that becomes due after restart.","file":"server/src/progression.ts","edits":[{"find":"`SELECT * FROM scheduled_restock WHERE due_at <= now() AND cancelled = false AND applied = false FOR UPDATE`,","replace":"`SELECT * FROM scheduled_restock WHERE false FOR UPDATE`,"}]},{"id":"progression-closed-browser-reservation-never-expires","scenario":"tracks/ecommerce/scenarios/03-server-time.json","targets":["ecommerce.l3.server-time.server-time.313a"],"desc":"Do not select expired reservations for server processing.","file":"server/src/progression.ts","edits":[{"find":"`SELECT ci.id, ci.item_id, c.account_id\n FROM cart_item ci JOIN cart c ON c.id = ci.cart_id\n WHERE ci.expired = false AND ci.reserved_until IS NOT NULL AND ci.reserved_until <= now()\n FOR UPDATE OF ci`,","replace":"`SELECT ci.id, ci.item_id, c.account_id\n FROM cart_item ci JOIN cart c ON c.id = ci.cart_id\n WHERE false\n FOR UPDATE OF ci`,"}]},{"id":"progression-cart-line-does-not-increment","scenario":"tracks/ecommerce/scenarios/progression-cart-checkout.json","targets":["ecommerce.feature.cart-checkout.cart.4a"],"desc":"Keep the old cart quantity when the same item is added again.","file":"server/src/progression.ts","edits":[{"find":"ON CONFLICT (cart_id, item_id) DO UPDATE SET quantity = CASE WHEN cart_item.expired THEN excluded.quantity ELSE cart_item.quantity + excluded.quantity END,","replace":"ON CONFLICT (cart_id, item_id) DO UPDATE SET quantity = cart_item.quantity,"}]},{"id":"progression-checkout-leaves-cart-lines","scenario":"tracks/ecommerce/scenarios/progression-cart-checkout.json","targets":["ecommerce.feature.cart-checkout.cart.4d"],"desc":"Leave cart lines after checkout.","file":"server/src/progression.ts","edits":[{"find":" await client.query(`DELETE FROM cart_item WHERE cart_id = $1`, [cart.rows[0].id]);","replace":" // Mutation: keep checked-out cart lines."}]},{"id":"progression-cart-update-uses-wrong-room","scenario":"tracks/ecommerce/scenarios/01-cart.json","targets":["ecommerce.spec.live-state.shared-cart.4c"],"desc":"Send cart updates to the wrong account room.","file":"server/src/index.ts","edits":[{"find":" try {\n await reserveCartItem(accountId, itemId, qty);\n } catch (error) {\n res.status(409).json({ error: error instanceof Error ? error.message : \"could not reserve stock\" });\n return;\n }\n const state = await buildCartState(accountId);\n io.to(`account:${accountId}`).emit(\"cart:update\", state);","replace":" try {\n await reserveCartItem(accountId, itemId, qty);\n } catch (error) {\n res.status(409).json({ error: error instanceof Error ? error.message : \"could not reserve stock\" });\n return;\n }\n const state = await buildCartState(accountId);\n io.to(`account:${accountId}:mutation`).emit(\"cart:update\", state);"}]},{"id":"progression-order-history-ignores-owner","scenario":"tracks/ecommerce/scenarios/01-order-ownership.json","targets":["ecommerce.spec.access-control.order-ownership.106a"],"desc":"Return orders without filtering by account.","file":"server/src/index.ts","edits":[{"find":"FROM orders WHERE account_id = $1 ORDER BY created_at DESC`,","replace":"FROM orders WHERE $1 IS NOT NULL ORDER BY created_at DESC`,"}]},{"id":"progression-review-conflict-is-not-updated","scenario":"tracks/ecommerce/scenarios/01-review-uniqueness.json","targets":["ecommerce.spec.transactional-integrity.unique-review.6b"],"desc":"Render every stored review twice.","file":"client/src/App.tsx","edits":[{"find":"reviews={itemDetail.reviews}","replace":"reviews={[...itemDetail.reviews, ...itemDetail.reviews]}"}]},{"id":"progression-revenue-double-counts-orders","scenario":"tracks/ecommerce/scenarios/progression-books-balance.json","targets":["ecommerce.spec.transactional-integrity.books-balance.107a"],"desc":"Double the order contribution to revenue.","file":"server/src/index.ts","edits":[{"find":"COALESCE((SELECT SUM(total - refund_total) FROM orders WHERE status != 'cancelled'), 0)","replace":"COALESCE((SELECT SUM(total - refund_total) * 2 FROM orders WHERE status != 'cancelled'), 0)"}]},{"id":"progression-cancelled-orders-remain-in-revenue","scenario":"tracks/ecommerce/scenarios/02-invariants.json","targets":["ecommerce.returns-pricing.refund-accounting.203a"],"desc":"Include cancelled orders in revenue.","file":"server/src/index.ts","edits":[{"find":"COALESCE((SELECT SUM(total - refund_total) FROM orders WHERE status != 'cancelled'), 0)","replace":"COALESCE((SELECT SUM(total - refund_total) FROM orders), 0)"}]},{"id":"progression-shipping-keeps-order-pending","scenario":"tracks/ecommerce/scenarios/02-fulfilment-ship.json","targets":["ecommerce.operations-access.fulfilment-queue.1c"],"desc":"Acknowledge shipping without changing order status.","file":"server/src/index.ts","edits":[{"find":"`UPDATE orders SET status = 'shipped', shipped_at = now()\n WHERE id = $1 AND status = 'pending' RETURNING account_id`","replace":"`UPDATE orders SET status = 'pending', shipped_at = now()\n WHERE id = $1 AND status = 'pending' RETURNING account_id`"}]},{"id":"progression-concurrent-cart-line-does-not-increment","scenario":"tracks/ecommerce/scenarios/01-duplicate-checkout.json","targets":["ecommerce.spec.concurrency-safety.duplicate-checkout.203a"],"desc":"Render the old cart quantity after concurrent adds.","file":"client/src/App.tsx","edits":[{"find":" value={line.quantity}","replace":" value={1}"}]},{"id":"progression-concurrent-checkout-leaves-cart-lines","scenario":"tracks/ecommerce/scenarios/01-duplicate-checkout.json","targets":["ecommerce.spec.concurrency-safety.duplicate-checkout.203b"],"desc":"Leave cart lines after concurrent checkout.","file":"server/src/progression.ts","edits":[{"find":" await client.query(`DELETE FROM cart_item WHERE cart_id = $1`, [cart.rows[0].id]);","replace":" // Mutation: keep checked-out cart lines."}]},{"id":"progression-catalog-ranking-is-reversed","scenario":"tracks/ecommerce/scenarios/01-catalog-ranking.json","targets":["ecommerce.feature.catalog.catalog-ranking.2b"],"desc":"Reverse the catalog tie-break order.","file":"client/src/App.tsx","edits":[{"find":".sort((a, b) => (filtering ? 0 : b.purchaseCount - a.purchaseCount) || a.name.localeCompare(b.name))","replace":".sort((a, b) => (filtering ? 0 : b.purchaseCount - a.purchaseCount) || b.name.localeCompare(a.name))"}]},{"id":"progression-catalog-search-requires-exact-name","scenario":"tracks/ecommerce/scenarios/01-catalog-search.json","targets":["ecommerce.feature.catalog.catalog-search.2d"],"desc":"Require an exact catalog name in search.","file":"client/src/App.tsx","edits":[{"find":"return items.filter((item) => (!q || item.name.toLowerCase().includes(q))","replace":"return items.filter((item) => (!q || item.name.toLowerCase() === q)"}]},{"id":"progression-catalog-price-is-offset","scenario":"tracks/ecommerce/scenarios/01-catalog-values.json","targets":["ecommerce.feature.catalog.catalog-values.2a"],"desc":"Report catalog prices one unit too high.","file":"server/src/index.ts","edits":[{"find":" price: Number(r.price),\n category: r.category,\n variants: r.variants,","replace":" price: Number(r.price) + 1,\n category: r.category,\n variants: r.variants,"}]},{"id":"progression-staff-sees-admin-navigation","scenario":"tracks/ecommerce/scenarios/01-warehouse-admin-staff.json","targets":["ecommerce.spec.access-control.warehouse-area-boundary.7a"],"desc":"Give non-admin accounts administrator identity and access to the protected inventory read.","file":"server/src/index.ts","edits":[{"find":" return rows[0];\n}\n\napp.use(async","replace":" return { ...rows[0], isAdmin: true };\n}\n\napp.use(async"},{"find":"isAdmin: acc.isAdmin, isStaff: acc.isStaff","replace":"isAdmin: true, isStaff: acc.isStaff"}]},{"id":"progression-staff-can-restock-directly","scenario":"tracks/ecommerce/scenarios/01-admin-write-staff.json","targets":["ecommerce.spec.access-control.warehouse-write-boundary.103b"],"desc":"Allow staff members to use the direct restock route.","file":"server/src/index.ts","edits":[{"find":"app.post(\n \"/api/admin/restock\",\n requireAdmin,","replace":"app.post(\n \"/api/admin/restock\",\n requireStaff,"}]},{"id":"progression-restock-adds-wrong-quantity","scenario":"tracks/ecommerce/scenarios/01-warehouse-stock-live-staff.json","targets":["ecommerce.spec.live-state.warehouse-stock.7c"],"desc":"Add one fewer unit during direct restock.","file":"server/src/index.ts","edits":[{"find":" [itemId, warehouseId, qty]\n );\n await processImmediateRestock(itemId);","replace":" [itemId, warehouseId, qty - 1]\n );\n await processImmediateRestock(itemId);"}]},{"id":"progression-cart-add-uses-another-account","scenario":"tracks/ecommerce/scenarios/01-cart-boundary.json","targets":["ecommerce.spec.access-control.cart-boundary.109a"],"desc":"Look up a cart containing the same item without checking its owner, so another customer adds to that cart.","file":"server/src/index.ts","edits":[{"find":" const accountId = req.account!.id;\n try {\n await reserveCartItem(accountId, itemId, qty);","replace":" const other = await pool.query(`SELECT c.account_id FROM cart c JOIN cart_item ci ON ci.cart_id = c.id WHERE ci.item_id = $1 ORDER BY c.id LIMIT 1`, [itemId]);\n const accountId = other.rows[0]?.account_id ?? req.account!.id;\n try {\n await reserveCartItem(accountId, itemId, qty);"}]},{"id":"progression-reservation-does-not-reduce-stock","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.301a"],"desc":"Record a reservation without reducing warehouse stock.","file":"server/src/progression.ts","edits":[{"find":" `UPDATE stock SET quantity = quantity - $1 WHERE item_id = $2 AND warehouse_id = $3`,\n [take, itemId, row.warehouse_id],","replace":" `UPDATE stock SET quantity = quantity + ($1 * 0) WHERE item_id = $2 AND warehouse_id = $3`,\n [take, itemId, row.warehouse_id],"}]},{"id":"progression-reservation-countdown-is-fixed","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.305a"],"desc":"Show a fixed reservation countdown.","file":"client/src/App.tsx","edits":[{"find":"{Math.max(0, (line.reservationSeconds ?? 0) - elapsed)}","replace":"{90}"}]},{"id":"progression-reservation-checkout-keeps-cart","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.306a"],"desc":"Leave reserved cart lines after checkout.","file":"server/src/progression.ts","edits":[{"find":" await client.query(`DELETE FROM cart_item WHERE cart_id = $1`, [cart.rows[0].id]);","replace":" // Mutation: keep reserved cart lines after checkout."}]},{"id":"progression-reservation-never-expires","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.307a"],"desc":"Do not select expired reservations.","file":"server/src/progression.ts","edits":[{"find":"`SELECT ci.id, ci.item_id, c.account_id\n FROM cart_item ci JOIN cart c ON c.id = ci.cart_id\n WHERE ci.expired = false AND ci.reserved_until IS NOT NULL AND ci.reserved_until <= now()\n FOR UPDATE OF ci`,","replace":"`SELECT ci.id, ci.item_id, c.account_id\n FROM cart_item ci JOIN cart c ON c.id = ci.cart_id\n WHERE false\n FOR UPDATE OF ci`,"}]},{"id":"progression-reservation-does-not-renew","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.308a"],"desc":"Keep the old reservation deadline when quantity changes.","file":"server/src/progression.ts","edits":[{"find":" reserved_until = excluded.reserved_until, expired = false RETURNING id`, [cartId, itemId, quantity]);","replace":" reserved_until = cart_item.reserved_until, expired = false RETURNING id`, [cartId, itemId, quantity]);"}]},{"id":"progression-customers-can-manage-scheduled-work","scenario":"tracks/ecommerce/scenarios/03-deferred-access.json","targets":["ecommerce.l3.deferred-access.scheduled-work-access.317a"],"desc":"Allow customers to schedule and cancel restocks.","file":"server/src/progression.ts","edits":[{"find":"app.post(\"/api/admin/scheduled-restocks\", dependencies.requireStaff,","replace":"app.post(\"/api/admin/scheduled-restocks\", dependencies.requireAuth,"},{"find":"app.delete(\"/api/admin/scheduled-restocks/:id\", dependencies.requireStaff,","replace":"app.delete(\"/api/admin/scheduled-restocks/:id\", dependencies.requireAuth,"}]},{"id":"progression-restock-does-not-survive-restart","scenario":"tracks/ecommerce/scenarios/03-deferred-durability.json","targets":["ecommerce.l3.deferred-durability.restart-survival.311a"],"desc":"Application startup discards pending restocks. Ordinary timer execution remains intact; completed and cancelled work is retained.","file":"server/src/index.ts","edits":[{"find":" await initializeProgressionSchema(pool);","replace":" await initializeProgressionSchema(pool);\n await pool.query(`DELETE FROM scheduled_restock WHERE cancelled = false AND applied = false`);"}]},{"id":"progression-reservation-does-not-survive-restart","scenario":"tracks/ecommerce/scenarios/03-deferred-durability.json","targets":["ecommerce.l3.deferred-durability.restart-survival.314a"],"desc":"Do not process expired reservations after restart.","file":"server/src/progression.ts","edits":[{"find":"`SELECT ci.id, ci.item_id, c.account_id\n FROM cart_item ci JOIN cart c ON c.id = ci.cart_id\n WHERE ci.expired = false AND ci.reserved_until IS NOT NULL AND ci.reserved_until <= now()\n FOR UPDATE OF ci`,","replace":"`SELECT ci.id, ci.item_id, c.account_id\n FROM cart_item ci JOIN cart c ON c.id = ci.cart_id\n WHERE false\n FOR UPDATE OF ci`,"}]},{"id":"progression-restock-can-apply-more-than-once","scenario":"tracks/ecommerce/scenarios/03-deferred-integrity.json","targets":["ecommerce.l3.deferred-integrity.exactly-once.311a"],"desc":"Leave an applied restock eligible for later timer passes.","file":"server/src/progression.ts","edits":[{"find":"await client.query(`UPDATE scheduled_restock SET applied = true WHERE id = $1`, [restock.id]);","replace":"await client.query(`UPDATE scheduled_restock SET applied = false WHERE id = $1`, [restock.id]);"}]},{"id":"progression-expiry-returns-double-stock","scenario":"tracks/ecommerce/scenarios/03-deferred-integrity.json","targets":["ecommerce.l3.deferred-integrity.stock-conservation.313a"],"desc":"Return twice the reserved quantity on expiry.","file":"server/src/progression.ts","edits":[{"find":" `UPDATE stock SET quantity = quantity + $1 WHERE item_id = $2 AND warehouse_id = $3`,\n [allocation.quantity, item.rows[0].item_id, allocation.warehouse_id],","replace":" `UPDATE stock SET quantity = quantity + $1 WHERE item_id = $2 AND warehouse_id = $3`,\n [allocation.quantity * 2, item.rows[0].item_id, allocation.warehouse_id],"}]},{"id":"progression-checkout-returns-reserved-stock","scenario":"tracks/ecommerce/scenarios/03-deferred-integrity.json","targets":["ecommerce.l3.deferred-integrity.stock-conservation.314a"],"desc":"Return reserved stock during checkout after creating the order.","file":"server/src/progression.ts","edits":[{"find":" if (promotionId) {\n await client.query(","replace":" for (const line of lines.rows) await releaseReservation(client, line.id);\n if (promotionId) {\n await client.query("}]},{"id":"restock-overwrites-instead-of-increments","scenario":"tracks/ecommerce/scenarios/01-warehouse-stock-live-staff.json","targets":["ecommerce.spec.live-state.warehouse-stock.7c"],"desc":"Overwrite the selected warehouse quantity with the restock amount instead of adding it.","file":"server/src/index.ts","edits":[{"find":" ON CONFLICT (item_id, warehouse_id) DO UPDATE SET quantity = stock.quantity + excluded.quantity`,\n [itemId, warehouseId, qty]","replace":" ON CONFLICT (item_id, warehouse_id) DO UPDATE SET quantity = excluded.quantity`,\n [itemId, warehouseId, qty]"}]},{"id":"operator-authorization-allows-customer-price-change","scenario":"tracks/ecommerce/scenarios/02-strengthened.json","targets":["ecommerce.operations-access.operator-authorization.201b"],"desc":"The price route replaces its administrator gate with ordinary authentication.","file":"server/src/index.ts","edits":[{"find":"app.post(\n \"/api/admin/price\",\n requireAdmin,","replace":"app.post(\n \"/api/admin/price\",\n requireAuth,"}]},{"id":"operator-authorization-allows-customer-shipping","scenario":"tracks/ecommerce/scenarios/02-self-contained.json","targets":["ecommerce.operations-access.fulfilment-queue.1e"],"desc":"The shipping route replaces its staff gate with ordinary authentication.","file":"server/src/index.ts","edits":[{"find":"app.post(\n \"/api/fulfilment/ship\",\n requireStaff,","replace":"app.post(\n \"/api/fulfilment/ship\",\n requireAuth,"}]},{"id":"catalog-price-change-not-broadcast-to-visitors","scenario":"tracks/ecommerce/scenarios/02-live-price.json","targets":["ecommerce.returns-pricing.price-history.4b"],"desc":"The signed-out search result caches detached rows and ignores later catalog updates.","file":"client/src/App.tsx","edits":[{"find":" }, [items, search, categoryFilter, minimumPrice, maximumPrice, inStockOnly, searchPage]);","replace":" }, [search, categoryFilter, minimumPrice, maximumPrice, inStockOnly, searchPage]);"}]},{"id":"catalog-price-change-not-broadcast-to-open-carts","scenario":"tracks/ecommerce/scenarios/progression-price-cart-checkout.json","targets":["ecommerce.returns-pricing.price-history.4c"],"desc":"The price reaches the catalog but the price route omits its targeted open-cart broadcasts.","file":"server/src/index.ts","edits":[{"find":" for (const row of cartAccounts.rows) {\n await broadcastCart(row.account_id);\n }","replace":" void cartAccounts; // mutant: carts containing this item are not refreshed"}]},{"id":"catalog-price-rewrites-receipts","scenario":"tracks/ecommerce/scenarios/02-paid-price-history.json","targets":["ecommerce.returns-pricing.price-history.4a"],"desc":"Changing a catalog price cascades into saved order lines and recomputes historical order totals.","file":"server/src/index.ts","edits":[{"find":" await pool.query(`UPDATE item SET price = $1 WHERE id = $2`, [priceNum.toFixed(2), itemId]);\n await broadcastCatalog();","replace":" await pool.query(`UPDATE item SET price = $1 WHERE id = $2`, [priceNum.toFixed(2), itemId]);\n await pool.query(`UPDATE order_item SET price = $1 WHERE item_id = $2`, [priceNum.toFixed(2), itemId]);\n await pool.query(\n `UPDATE orders o SET total = totals.total FROM (\n SELECT order_id, SUM(quantity * price) AS total FROM order_item GROUP BY order_id\n ) totals WHERE o.id = totals.order_id`\n );\n await broadcastCatalog();"}]},{"id":"catalog-price-rewrites-earned-revenue","scenario":"tracks/ecommerce/scenarios/02-invariants.json","targets":["ecommerce.returns-pricing.refund-accounting.203b"],"desc":"Changing a catalog price cascades into saved order lines and recomputes historical order totals and revenue.","file":"server/src/index.ts","edits":[{"find":" await pool.query(`UPDATE item SET price = $1 WHERE id = $2`, [priceNum.toFixed(2), itemId]);\n await broadcastCatalog();","replace":" await pool.query(`UPDATE item SET price = $1 WHERE id = $2`, [priceNum.toFixed(2), itemId]);\n await pool.query(`UPDATE order_item SET price = $1 WHERE item_id = $2`, [priceNum.toFixed(2), itemId]);\n await pool.query(\n `UPDATE orders o SET total = totals.total FROM (\n SELECT order_id, SUM(quantity * price) AS total FROM order_item GROUP BY order_id\n ) totals WHERE o.id = totals.order_id`\n );\n await broadcastCatalog();"}]},{"id":"returned-line-marker-omitted","scenario":"tracks/ecommerce/scenarios/02-strengthened.json","targets":["ecommerce.returns-pricing.cancellation-and-return.3c"],"desc":"A returned order line keeps its persisted returned state, restored stock, and adjusted revenue but omits the visible returned marker.","file":"client/src/App.tsx","edits":[{"find":"l.returned ? \" (returned)\" : \"\"","replace":"\"\""}]},{"id":"direct-review-access-is-not-checked","scenario":"tracks/ecommerce/scenarios/progression-review-access.json","targets":["ecommerce.progression.review-access-specifications.review-eligibility-direct.618a"],"desc":"The direct review action accepts a review from a customer who did not buy the item.","file":"server/src/index.ts","edits":[{"find":" if (purchased.rowCount === 0) {","replace":" if (false && purchased.rowCount === 0) {"}]},{"id":"support-history-rows-are-hidden","scenario":"tracks/ecommerce/scenarios/progression-support-history.json","targets":["ecommerce.progression.support-history.support-history.612c","ecommerce.spec.state-durability.support-history-reload.612a","ecommerce.spec.access-control.support-history-privacy.612b","ecommerce.spec.access-control.support-history-logout.612d"],"desc":"Hide submitted support ticket rows while leaving the submission reference available. History, reload, privacy, and logout require the owner to see the ticket first. This breaks those positive observations; it does not create a privacy leak or remove stored data.","file":"client/src/ProgressionPanel.tsx","edits":[{"find":"data-role=\"support-ticket\"","replace":"data-role=\"support-ticket\" hidden"}]},{"id":"authorized-restock-does-not-change-stock","scenario":"tracks/ecommerce/scenarios/01-admin-write-staff.json","targets":["ecommerce.feature.warehouse-admin.admin-write.103a"],"desc":"Accept an administrator restock without changing warehouse stock.","file":"server/src/index.ts","edits":[{"find":" await pool.query(\n `INSERT INTO stock (item_id, warehouse_id, quantity) VALUES ($1, $2, $3)\n ON CONFLICT (item_id, warehouse_id) DO UPDATE SET quantity = stock.quantity + excluded.quantity`,\n [itemId, warehouseId, qty]\n );\n await processImmediateRestock(itemId);\n","replace":" // mutant: accept restock without changing stock\n"}]},{"id":"low-stock-threshold-is-two-units","scenario":"tracks/ecommerce/scenarios/02-low-stock.json","targets":["ecommerce.inventory-operations.operational-views.5e","ecommerce.spec.live-state.inventory-dashboard.5a"],"desc":"The low-stock list only includes items at two units or fewer, so the seeded three-unit item is missing when the dashboard is read. The live-state sibling 5a opens with the same observation and fails as necessary collateral.","file":"server/src/index.ts","edits":[{"find":" const lowStock = items\n .filter((it) => it.stock <= 10)","replace":" const lowStock = items\n .filter((it) => it.stock <= 2)"}]},{"id":"category-totals-render-as-session-deltas","scenario":"tracks/ecommerce/scenarios/02-operational-category-totals.json","targets":["ecommerce.inventory-operations.operational-views.5f"],"desc":"Category units and revenue render as the change since the admin panel was opened rather than the stored totals, so a reload resets them to zero while live updates still move by the purchase amount.","file":"client/src/App.tsx","edits":[{"find":" const [namedRestock, setNamedRestock] = useState({ item: \"\", warehouse: \"\", quantity: \"\" });\n","replace":" const [namedRestock, setNamedRestock] = useState({ item: \"\", warehouse: \"\", quantity: \"\" });\n const openedTotals = useRef<CategoryTotal[] | null>(null);\n if (admin && !openedTotals.current) openedTotals.current = admin.categoryTotals;\n const openedTotal = (category: string) => openedTotals.current?.find((c) => c.category === category);\n"},{"find":" Units: <span data-role=\"category-units\">{c.units}</span>","replace":" Units: <span data-role=\"category-units\">{c.units - (openedTotal(c.category)?.units ?? 0)}</span>"},{"find":" Revenue: <span data-role=\"category-revenue\">{c.revenue.toFixed(2)}</span>","replace":" Revenue: <span data-role=\"category-revenue\">{(c.revenue - (openedTotal(c.category)?.revenue ?? 0)).toFixed(2)}</span>"}]},{"id":"profile-summary-ignores-saved-address","scenario":"tracks/ecommerce/scenarios/progression-customer-profile.json","targets":["ecommerce.progression.customer-profile.customer-profile.620c"],"desc":"The profile summary shows only the address loaded with the account and ignores addresses saved afterwards, so the saved address appears only after a reload.","file":"client/src/ProgressionPanel.tsx","edits":[{"find":" useEffect(() => {\n if (state?.profile) setProfile({ name: state.profile.name, address: state.profile.address });\n }, [state?.profile?.name, state?.profile?.address]);\n","replace":" useEffect(() => {\n if (state?.profile) setProfile({ name: state.profile.name, address: state.profile.address });\n }, [state?.profile?.name, state?.profile?.address]);\n const [loadedAddress, setLoadedAddress] = useState<string | null>(null);\n useEffect(() => {\n if (loadedAddress === null && state?.profile) setLoadedAddress(state.profile.address);\n }, [state?.profile?.address]);\n"},{"find":"<span data-role=\"profile-address-summary\">{state?.profile?.address}</span>","replace":"<span data-role=\"profile-address-summary\">{loadedAddress ?? \"\"}</span>"}]},{"id":"support-first-reply-is-hidden","scenario":"tracks/ecommerce/scenarios/progression-managed-support-shared.json","targets":["ecommerce.progression.managed-support.managed-support.613c"],"desc":"The reply list drops the first reply on every support case, so the staff member's first reply is never shown to the customer.","file":"client/src/ProgressionPanel.tsx","edits":[{"find":" {entry.replies.map((item: any) => <div data-role=\"support-reply-item\" key={item.id}>{item.username}: {item.message}</div>)}","replace":" {entry.replies.slice(1).map((item: any) => <div data-role=\"support-reply-item\" key={item.id}>{item.username}: {item.message}</div>)}"}]},{"id":"notification-sync-flips-saved-toggle","scenario":"tracks/ecommerce/scenarios/progression-notification-preferences.json","targets":["ecommerce.progression.notification-preferences.notification-preferences.630c"],"desc":"Merging saved preferences into the form flips each toggle against its current value instead of replacing it, so a choice shows off right after saving but on after a reload.","file":"client/src/ProgressionPanel.tsx","edits":[{"find":" if (state?.preferences) setPreferences(state.preferences);","replace":" if (state?.preferences) setPreferences({ order: state.preferences.order !== preferences.order, stock: state.preferences.stock !== preferences.stock });"}]},{"id":"staff-role-form-reverts-after-save","scenario":"tracks/ecommerce/scenarios/progression-staff-roles.json","targets":["ecommerce.progression.staff-roles.staff-roles.621c"],"desc":"After a successful save the role select snaps back to the previously loaded role, so the assignment is only visible after a reload.","file":"client/src/ProgressionPanel.tsx","edits":[{"find":" <button data-role=\"staff-role-save\" onClick={() => run(() => request(`/api/staff/${role.id}/role`, \"PUT\", {\n role: (document.getElementById(`role-${role.id}`) as HTMLSelectElement).value,\n }))}>Save</button></div>)}","replace":" <button data-role=\"staff-role-save\" onClick={() => run(async () => {\n const select = document.getElementById(`role-${role.id}`) as HTMLSelectElement;\n await request(`/api/staff/${role.id}/role`, \"PUT\", { role: select.value });\n select.value = role.role;\n })}>Save</button></div>)}"}]},{"id":"purchase-does-not-broadcast-fulfilment-queue","scenario":"tracks/ecommerce/scenarios/02-fulfilment-live.json","targets":["ecommerce.spec.live-state.fulfilment-queue.1a"],"desc":"A direct purchase commits its pending order but omits the fulfilment queue broadcast, so an open staff queue never learns about the new order.","file":"server/src/index.ts","edits":[{"find":" await broadcastCatalog();\n await broadcastOrders(accountId);\n await broadcastFulfilment();\n await broadcastRecommended(accountId);\n res.json({ ok: true });\n })\n);\n\n// ---------- cart routes ----------","replace":" await broadcastCatalog();\n await broadcastOrders(accountId);\n // mutant: the new pending order is not pushed to open fulfilment queues\n await broadcastRecommended(accountId);\n res.json({ ok: true });\n })\n);\n\n// ---------- cart routes ----------"}]},{"id":"admin-state-change-is-not-broadcast","scenario":"tracks/ecommerce/scenarios/02-low-stock.json","targets":["ecommerce.spec.live-state.inventory-dashboard.5a"],"desc":"Changed admin dashboard state is never broadcast to open admin views, so a customer purchase that drops an item to ten units does not re-enter the low-stock list live.","file":"server/src/index.ts","edits":[{"find":" lastAdminJson = adminJson;\n io.to(\"admin\").emit(\"admin:update\", adminState);","replace":" lastAdminJson = adminJson;\n // mutant: changed admin state is not broadcast"}]},{"id":"admin-sockets-do-not-join-admin-room","scenario":"tracks/ecommerce/scenarios/02-operational-category-totals.json","targets":["ecommerce.spec.live-state.sales-dashboard.5b"],"desc":"Admin sockets receive their dashboard state on connection but never join the admin room, so a customer purchase does not update the open category totals live.","file":"server/src/index.ts","edits":[{"find":" if (acc.isAdmin) socket.join(\"admin\");","replace":" // mutant: admin sockets never join the admin room"}]},{"id":"transfer-overdraft-guard-skips-bulk-transfers","scenario":"tracks/ecommerce/scenarios/02-transfer-overdraw.json","targets":["ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c"],"desc":"The insufficient-stock guard is only evaluated for transfers under 1000 units, so a bulk transfer that overdraws the source warehouse commits instead of being refused.","file":"server/src/index.ts","edits":[{"find":" if (available < qty) {","replace":" if (available < qty && qty < 1000) {"}]},{"id":"transfer-does-not-publish-warehouse-totals","scenario":"tracks/ecommerce/scenarios/02-transfer-totals.json","targets":["ecommerce.spec.live-state.stock-transfers.2b"],"desc":"A transfer commits but answers with the pre-transfer admin snapshot and admin state is never broadcast, so the open warehouse totals do not move.","file":"server/src/index.ts","edits":[{"find":" const client = await pool.connect();\n try {\n await client.query(\"BEGIN\");\n // Lock both warehouse rows","replace":" const state = await buildAdminState();\n const client = await pool.connect();\n try {\n await client.query(\"BEGIN\");\n // Lock both warehouse rows"},{"find":" await broadcastCatalog();\n const state = await buildAdminState();\n res.json(state);\n })\n);\n\napp.post(\n \"/api/admin/price\",","replace":" await broadcastCatalog();\n res.json(state);\n })\n);\n\napp.post(\n \"/api/admin/price\","},{"find":" lastAdminJson = adminJson;\n io.to(\"admin\").emit(\"admin:update\", adminState);","replace":" lastAdminJson = adminJson;\n // mutant: changed admin state is not broadcast"}]},{"id":"credit-checkout-ignores-wallet","desc":"A credit checkout pays entirely externally despite available credit.","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.feature.store-credit.store-credit-750.750a"],"file":"server/src/credit.ts","edits":[{"find":" const creditMinor = Math.min(Number(account.rows[0].credit_minor), totalMinor);","replace":" const creditMinor = 0;"}]},{"id":"credit-grant-replay-increments-balance","desc":"Replaying a grant applies its credit to the wallet again.","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.spec.store-credit.production-752.752a"],"file":"server/src/credit.ts","edits":[{"find":" if (!existing.rows.length) {","replace":" if (existing.rows.length) await client.query('UPDATE account SET credit_minor=credit_minor+$1 WHERE id=$2', [amountMinor, accountId]);\n if (!existing.rows.length) {"}]},{"id":"customer-can-grant-credit","desc":"Customer authentication is accepted without staff authorization.","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.spec.store-credit.production-753.753a"],"file":"server/src/credit.ts","edits":[{"find":" app.post('/api/staff/credit', auth, staff, async (req, res) => {","replace":" app.post('/api/staff/credit', auth, async (req, res) => {"}]},{"id":"split-refund-does-not-restore-credit","desc":"The refund is recorded but its original wallet credit is not restored.","scenario":"tracks/ecommerce/scenarios/progression-split-tender-refunds.json","targets":["ecommerce.feature.split-tender-refunds.split-tender-refunds-751.751a"],"file":"server/src/progression.ts","edits":[{"find":" await refundCredit(client, order.rows[0]);","replace":" // mutant: omit wallet restoration"}]},{"id":"split-refund-duplicates-credit","desc":"A refund credits the wallet twice while recording one refund.","scenario":"tracks/ecommerce/scenarios/progression-split-tender-refunds.json","targets":["ecommerce.spec.split-tender-refunds.production-756.756a"],"file":"server/src/credit.ts","edits":[{"find":"[delta, order.account_id]);","replace":"[delta * 2, order.account_id]);"}]},{"id":"subscription-skips-due-purchase","desc":"Due deliveries are recorded as skipped although stock is available.","scenario":"tracks/ecommerce/scenarios/progression-subscriptions.json","targets":["ecommerce.feature.subscriptions.subscriptions-760.760a"],"file":"server/src/subscriptions.ts","edits":[{"find":" if (stock.rows.reduce((sum, row) => sum + row.quantity, 0) >= subscription.quantity) {","replace":" if (false && stock.rows.reduce((sum, row) => sum + row.quantity, 0) >= subscription.quantity) {"}]},{"id":"subscription-allows-foreign-cancellation","desc":"A customer can cancel another customer subscription.","scenario":"tracks/ecommerce/scenarios/progression-subscriptions.json","targets":["ecommerce.spec.subscriptions.production-762.762a"],"file":"server/src/subscriptions.ts","edits":[{"find":" if (!row || row.account_id !== req.account!.id) {","replace":" if (!row) {"}]},{"id":"subscription-pause-is-not-recorded","desc":"Pause acknowledges the request but the subscription remains active.","scenario":"tracks/ecommerce/scenarios/progression-subscriptions.json","targets":["ecommerce.spec.subscriptions.production-763.763a"],"file":"server/src/subscriptions.ts","edits":[{"find":"UPDATE purchase_subscription SET status='paused',paused_at=now() WHERE id=$1","replace":"UPDATE purchase_subscription SET status='active',paused_at=now() WHERE id=$1"}]},{"id":"credit-checkout-retains-purchased-cart","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.spec.store-credit.production-754.754a"],"desc":"The purchased cart remains available instead of being consumed by checkout.","file":"server/src/progression.ts","edits":[{"find":" await client.query(`DELETE FROM cart_item WHERE cart_id = $1`, [cart.rows[0].id]);","replace":" // Mutation: keep checked-out cart lines."}]},{"id":"pending-subscriptions-are-cleared-at-startup","desc":"Restart erases pending subscription work while preserving ordinary timer execution.","file":"server/src/subscriptions.ts","scenario":"tracks/ecommerce/scenarios/progression-subscriptions.json","targets":["ecommerce.spec.subscriptions.production-761.761a"],"edits":[{"find":" `);\n}\n\nexport function registerSubscriptions","replace":" `);\n await pool.query(\"UPDATE purchase_subscription SET status='cancelled' WHERE status='active'\");\n}\n\nexport function registerSubscriptions"}]},{"id":"credit-balance-is-cleared-at-startup","desc":"Restart clears an issued wallet balance while leaving accounts present.","file":"server/src/credit.ts","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.spec.store-credit.production-755.755a"],"edits":[{"find":" `);\n}\n\nexport async function spendCredit","replace":" `);\n await pool.query(\"UPDATE account SET credit_minor=0\");\n}\n\nexport async function spendCredit"}]},{"id":"bundle-definition-loses-component-quantity","scenario":"tracks/ecommerce/scenarios/progression-product-bundles.json","targets":["ecommerce.feature.product-bundles.product-bundles.740a"],"desc":"definition loses component quantity","file":"server/src/bundles.ts","edits":[{"find":"values.push({ ...component, itemId: item.rows[0].id });","replace":"values.push({ ...component, quantity: 1, itemId: item.rows[0].id });"}]},{"id":"bundle-catalog-write-allows-customers","scenario":"tracks/ecommerce/scenarios/progression-product-bundles.json","targets":["ecommerce.spec.bundle-integrity.bundle-743.743a"],"desc":"catalog write allows customers","file":"server/src/bundles.ts","edits":[{"find":"if (!actor?.is_admin && actor?.staff_role !== 'catalog')","replace":"if (false)"}]},{"id":"bundle-checkout-price-not-snapshot","scenario":"tracks/ecommerce/scenarios/progression-bundle-checkout.json","targets":["ecommerce.feature.bundle-checkout.bundle-checkout.741a"],"desc":"checkout price not snapshot","file":"server/src/bundles.ts","edits":[{"find":"[cart.rows[0].id, bundleId, bundle.price, JSON.stringify(allocations), JSON.stringify(bundle.bundle_components)]","replace":"[cart.rows[0].id, bundleId, bundle.price + 1, JSON.stringify(allocations), JSON.stringify(bundle.bundle_components)]"}]},{"id":"bundle-expiry-does-not-release-components","scenario":"tracks/ecommerce/scenarios/progression-bundle-checkout.json","targets":["ecommerce.spec.bundle-integrity.bundle-746.746a"],"desc":"expiry does not release components","file":"server/src/progression.ts","edits":[{"find":"await releaseBundle(client, bundle.rows[0].component_allocations);","replace":"/* mutant: component holds leak after expiration */"}]},{"id":"bundle-return-loses-original-components","scenario":"tracks/ecommerce/scenarios/progression-bundle-returns.json","targets":["ecommerce.feature.bundle-returns.bundle-returns.742a"],"desc":"return loses original components","file":"server/src/bundles.ts","edits":[{"find":"await releaseBundle(client, line.component_allocations);","replace":"/* mutant: purchased components are not restored */"}]},{"id":"bundle-return-replay-restocks-again","scenario":"tracks/ecommerce/scenarios/progression-bundle-returns.json","targets":["ecommerce.spec.bundle-integrity.bundle-742.742b"],"desc":"return replay restocks again","file":"server/src/bundles.ts","edits":[{"find":"WHERE order_id=$1 AND is_bundle AND NOT returned FOR UPDATE","replace":"WHERE order_id=$1 AND is_bundle FOR UPDATE"}]},{"id":"bundle-return-crosses-account-boundary","scenario":"tracks/ecommerce/scenarios/progression-bundle-returns.json","targets":["ecommerce.spec.bundle-integrity.bundle-748.748a"],"desc":"return crosses account boundary","file":"server/src/bundles.ts","edits":[{"find":"WHERE id=$1 AND account_id=$2 AND status IN ('shipped','delivered')","replace":"WHERE id=$1 AND $2::integer=$2::integer AND status IN ('shipped','delivered')"}]},{"id":"bundle-components-can-overdraw","scenario":"tracks/ecommerce/scenarios/progression-bundle-checkout.json","targets":["ecommerce.spec.bundle-integrity.bundle-744.744a","ecommerce.spec.bundle-integrity.bundle-745.745a"],"desc":"components can overdraw","file":"server/src/bundles.ts","edits":[{"find":"if (rows.rows.reduce((sum, row) => sum + row.quantity, 0) < component.quantity) throw new Error('A component is unavailable');","replace":"// mutant: incomplete component reservation is accepted"}]},{"id":"bundle-checkout-reuses-reservation","scenario":"tracks/ecommerce/scenarios/progression-bundle-checkout.json","targets":["ecommerce.spec.bundle-integrity.bundle-747.747a"],"desc":"checkout reuses reservation","file":"server/src/progression.ts","edits":[{"find":"await client.query(`DELETE FROM cart_item WHERE cart_id = $1`, [cart.rows[0].id]);","replace":"// mutant: cart survives checkout"}]},{"id":"return-after-support-refund-is-blocked","scenario":"tracks/ecommerce/scenarios/progression-support-return-interaction.json","targets":["ecommerce.feature.split-tender-refunds.return-refund-interaction.757a"],"desc":"Reject a valid physical return after a financial refund.","file":"server/src/index.ts","edits":[{"find":" if (lineRow.rows[0].returned) {","replace":" if (lineRow.rows[0].returned || Number(orderRow.rows[0].refund_total) > 0) {"}]},{"id":"support-refund-after-return-pays-twice","scenario":"tracks/ecommerce/scenarios/progression-support-return-interaction.json","targets":["ecommerce.feature.split-tender-refunds.return-refund-interaction.757b"],"desc":"Pay the full order again after a physical return.","file":"server/src/progression.ts","edits":[{"find":" await client.query(`UPDATE orders SET refund_total = total, status = CASE","replace":" await client.query(`UPDATE orders SET refund_total = refund_total + total, status = CASE"}]},{"id":"progression-support-history-anonymous-leak","scenario":"tracks/ecommerce/scenarios/progression-support-history.json","targets":["ecommerce.spec.access-control.support-history-logout.612d"],"desc":"Return private support subjects to signed-out visitors while preserving authenticated ownership filtering.","file":"server/src/progression.ts","edits":[{"find":" if (!account) return [];\n const where = account.isAdmin || account.isStaff ? \"TRUE\" : \"sc.account_id = $1\";\n const values = account.isAdmin || account.isStaff ? [] : [account.id];","replace":" const where = !account || account.isAdmin || account.isStaff ? \"TRUE\" : \"sc.account_id = $1\";\n const values = !account || account.isAdmin || account.isStaff ? [] : [account.id];"}]},{"id":"checkout-crash-integrity","scenario":"tracks/ecommerce/scenarios/progression-checkout-crash.json","targets":["ecommerce.spec.state-durability.checkout-crash-integrity.910a"],"desc":"A lifecycle callback changes a prepared or cleared cart to quantity two, leaving neither a legal rollback nor a complete checkout.","file":"server/src/index.ts","edits":[{"find":" await initializeProgressionSchema(pool);","replace":" await initializeProgressionSchema(pool);\n await pool.query(`INSERT INTO cart_item (cart_id, item_id, quantity) SELECT c.id, MIN(oi.item_id), 2 FROM cart c JOIN orders o ON o.account_id=c.account_id JOIN order_item oi ON oi.order_id=o.id GROUP BY c.id ON CONFLICT (cart_id,item_id) DO UPDATE SET quantity=2`);"}]},{"id":"checkout-crash-durability","scenario":"tracks/ecommerce/scenarios/progression-checkout-crash.json","targets":["ecommerce.spec.state-durability.checkout-crash-durability.910b"],"desc":"A lifecycle callback corrupts earlier order totals. The unchanged prepared snapshots and acknowledged baseline must expose the damage.","file":"server/src/index.ts","edits":[{"find":" await initializeProgressionSchema(pool);","replace":" await initializeProgressionSchema(pool);\n await pool.query('UPDATE orders SET total=0');"}]},{"id":"review-script-unsafe-render","scenario":"tracks/ecommerce/scenarios/progression-review-script.json","targets":["ecommerce.progression.review-access-specifications.stored-review-script.9180a"],"desc":"Render stored review text as HTML so it executes in another customer session.","file":"client/src/App.tsx","edits":[{"find":"<div>{r.comment}</div>","replace":"<div dangerouslySetInnerHTML={{ __html: r.comment }} />"}]},{"id":"review-script-reject-all","scenario":"tracks/ecommerce/scenarios/progression-review-script.json","targets":["ecommerce.progression.review-access-specifications.stored-review-script.9180a"],"desc":"Reject all nonempty reviews, including the required benign review; absence of script is insufficient.","file":"server/src/index.ts","edits":[{"find":"const { rating, comment } = req.body ?? {};","replace":"const { rating, comment } = req.body ?? {};\n if (String(comment).length > 0) { res.status(400).json({ error: \"Review rejected\" }); return; }"}]}]},"spacetime":{"schemaVersion":3,"fixtureSha256":"7ba1f548e89f461b95b5caf843b0fbcf2b81278428d02869d7f47e1a199eaf6e","backend":"spacetime","track":"ecommerce","note":"Mutation definitions for the SpacetimeDB ecommerce reference.","mutations":[{"id":"recommendation-dismissal-lost-on-reconnect","scenario":"tracks/ecommerce/scenarios/progression-recommendation-feedback.json","targets":["ecommerce.spec.state-durability.recommendation-feedback-restart.504c"],"desc":"Erase saved recommendation dismissals when a browser reconnects. This controls reconnect persistence, not backend restart alone.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"export const onConnect = spacetimedb.clientConnected((_ctx) => {});","replace":"export const onConnect = spacetimedb.clientConnected((ctx) => { for (const row of ctx.db.recommendationDismissal.iter()) ctx.db.recommendationDismissal.id.delete(row.id); });"}]},{"id":"pending-order-item-return-accepted","scenario":"tracks/ecommerce/scenarios/progression-order-return-boundary.json","targets":["ecommerce.returns-pricing.cancellation-and-return.3f"],"desc":"Accept a pending order return and restore its stock before shipment.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"if (!['shipped', 'delivered'].includes(order.status)) throw new SenderError('Order has not shipped yet.');","replace":"if (!['pending', 'shipped', 'delivered'].includes(order.status)) throw new SenderError('Order has not shipped yet.');"}]},{"id":"staff-admin-access-survives-role-removal","scenario":"tracks/ecommerce/scenarios/progression-staff-roles.json","targets":["ecommerce.spec.access-control.staff-role-revocation.621d"],"desc":"Keep administrator access after changing the assigned role back to staff.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.account.id.update({ ...target, isAdmin: role === 'admin' });","replace":"ctx.db.account.id.update({ ...target, isAdmin: target.isAdmin || role === 'admin' });"}]},{"id":"shipping-counts-sale-twice","scenario":"tracks/ecommerce/scenarios/progression-shipping-accounting.json","targets":["ecommerce.inventory-operations.shipping-accounting.202e"],"desc":"Shipping succeeds but doubles the completed sale value in authoritative revenue.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" ctx.db.customerOrder.id.update({ ...order, status: 'shipped' });","replace":" ctx.db.customerOrder.id.update({ ...order, status: 'shipped', total: order.total * 2 });"}]},{"id":"restock-client-snapshot-overwrites-concurrent-purchases","scenario":"tracks/ecommerce/scenarios/01-restock-race.json","targets":["ecommerce.spec.concurrency-safety.restock-race.202a"],"desc":"Compute absolute restock quantity when the administrator edits the form and store that captured value in the reducer. A fixed 500 ms submission delay widens the stale-write window. Serial purchases and restocks still work; intervening purchases can be overwritten. This is a stale-form lost update, not a race inside an atomic reducer.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.stock.insert({ ...existing, quantity: existing.quantity + quantity });\n } else {\n ctx.db.stock.insert({ item_id: itemId, warehouse_id: warehouseId, quantity });\n }\n for (const alert of","replace":"ctx.db.stock.insert({ ...existing, quantity });\n } else {\n ctx.db.stock.insert({ item_id: itemId, warehouse_id: warehouseId, quantity });\n }\n for (const alert of"},{"file":"client/src/components/AdminPanel.tsx","find":"onChange={(e) => setRestockInputs((v) => ({ ...v, [k]: e.target.value }))}","replace":"onChange={(e) => setRestockInputs((v) => ({ ...v, [k]: String(stockOf(item.id, wh.id) + Number(e.target.value)) }))}"},{"file":"client/src/App.tsx","find":" await conn?.reducers.adminRestock({ itemId, warehouseId, quantity });","replace":" // Mutant: widen the stale form submission window without changing serial behavior.\n await new Promise(resolve => setTimeout(resolve, 500));\n await conn?.reducers.adminRestock({ itemId, warehouseId, quantity });"}]},{"id":"signup-binds-the-new-account-to-the-admin-session","scenario":"tracks/ecommerce/scenarios/01-account-create.json","targets":["ecommerce.feature.accounts.accounts.1a"],"desc":"Create the requested account but bind the new browser session to the administrator, so account creation no longer signs the visitor in as the account it created.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" const acc = ctx.db.account.insert({\n id: 0n,\n username: uname,\n passwordHash: hashPassword(password),\n isAdmin: false,\n isStaff: false,\n });\n\n const existingSession = ctx.db.session.identity.find(ctx.sender);\n if (existingSession) {\n ctx.db.session.identity.update({ ...existingSession, accountId: acc.id });\n } else {\n ctx.db.session.insert({ identity: ctx.sender, accountId: acc.id });\n }","replace":" const acc = ctx.db.account.insert({\n id: 0n,\n username: uname,\n passwordHash: hashPassword(password),\n isAdmin: false,\n isStaff: false,\n });\n\n const signedInAccount = ctx.db.account.username.find('admin') ?? acc;\n const existingSession = ctx.db.session.identity.find(ctx.sender);\n if (existingSession) {\n ctx.db.session.identity.update({ ...existingSession, accountId: signedInAccount.id });\n } else {\n ctx.db.session.insert({ identity: ctx.sender, accountId: signedInAccount.id });\n }"}]},{"id":"duplicate-signup-is-silently-ignored","scenario":"tracks/ecommerce/scenarios/01-account-duplicate.json","targets":["ecommerce.feature.accounts.accounts.1b"],"desc":"Return success for a taken username without creating a session or surfacing the required refusal.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" if (existing) throw new SenderError('That username is already taken.');","replace":" if (existing) return; // mutant: duplicate signup is silently accepted"}]},{"id":"signin-does-not-verify-the-password","scenario":"tracks/ecommerce/scenarios/01-account-password.json","targets":["ecommerce.feature.accounts.accounts.1c"],"desc":"Accept a known username without comparing the supplied password hash.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" if (!acc || acc.passwordHash !== hashPassword(password)) {","replace":" if (!acc) {"}]},{"id":"signout-keeps-the-account-session","scenario":"tracks/ecommerce/scenarios/01-account-signout.json","targets":["ecommerce.feature.accounts.accounts.1d"],"desc":"Leave the current account session in place when the visitor signs out.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" if (existingSession) ctx.db.session.identity.delete(ctx.sender);","replace":" // mutant: sign out keeps the current account session"}]},{"id":"session-token-is-not-persisted-for-reload","scenario":"tracks/ecommerce/scenarios/01-account-reload.json","targets":["ecommerce.spec.state-durability.session-reload.1e"],"desc":"Discard the connection token instead of persisting it, so a reload receives a new identity with no account session.","file":"client/src/App.tsx","edits":[{"find":" if (token) localStorage.setItem('auth_token', token);","replace":" if (token) localStorage.removeItem('auth_token');"}]},{"id":"catalog-seeds-the-wrong-air-purifier-price","scenario":"tracks/ecommerce/scenarios/01-catalog-values.json","targets":["ecommerce.feature.catalog.catalog-values.2a"],"desc":"Seed Air Purifier with an incorrect stored price while leaving the rest of the catalog intact.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" ['Air Purifier', 189.0, 60, 40, 'Home'],","replace":" ['Air Purifier', 999.0, 60, 40, 'Home'],"}]},{"id":"catalog-tie-breaks-in-reverse-alphabetical-order--01-catalog-ranking","scenario":"tracks/ecommerce/scenarios/01-catalog-ranking.json","targets":["ecommerce.feature.catalog.catalog-ranking.2b"],"desc":"Reverse the specified alphabetical tie-breaker. This necessarily breaks both the initial sequence and the post-purchase sequence in the same scenario.","file":"client/src/App.tsx","edits":[{"find":" return a.name.localeCompare(b.name);","replace":" return b.name.localeCompare(a.name);"}]},{"id":"catalog-tie-breaks-in-reverse-alphabetical-order--01-core","scenario":"tracks/ecommerce/scenarios/01-core.json","targets":["ecommerce.spec.live-state.ranking.2c"],"desc":"Reverse the specified alphabetical tie-breaker. This necessarily breaks both the initial sequence and the post-purchase sequence in the same scenario.","file":"client/src/App.tsx","edits":[{"find":" return a.name.localeCompare(b.name);","replace":" return b.name.localeCompare(a.name);"}]},{"id":"purchase-does-not-update-ranking-count","scenario":"tracks/ecommerce/scenarios/01-core.json","targets":["ecommerce.spec.live-state.ranking.2c"],"desc":"Complete the purchase but leave its popularity count unchanged, so open storefronts cannot rank the purchased item first.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" bumpPurchaseCount(ctx, itemId, quantity);","replace":" // mutant: buy-now never advances the ranking count"}]},{"id":"signed-out-purchase-bypasses-account-check","scenario":"tracks/ecommerce/scenarios/progression-signed-out-purchase.json","targets":["ecommerce.spec.access-control.signed-out-purchase.3a"],"desc":"Expose the guest purchase button and accept its purchase as the existing administrator. The stock observation then exercises the broken account boundary; normal signed-in purchases remain unchanged.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"export const buyNow = spacetimedb.reducer({ itemId: t.u64() }, (ctx, { itemId }) => {\n const acc = requireAccount(ctx);","replace":"export const buyNow = spacetimedb.reducer({ itemId: t.u64() }, (ctx, { itemId }) => {\n const acc = getAccountId(ctx) === null ? ctx.db.account.username.find('admin')! : requireAccount(ctx);"},{"file":"client/src/components/ItemCard.tsx","find":" {isSignedIn && (","replace":" {true && ("}]},{"id":"buy-now-creates-orders-without-reserving-stock--01-buying","scenario":"tracks/ecommerce/scenarios/01-buying.json","targets":["ecommerce.spec.live-state.purchase-stock.3b"],"desc":"Create purchase orders without reserving inventory. The same defect necessarily breaks live purchase stock and the sell-out portion of the stock-limit check.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" const allocations = decrementStockTracked(ctx, itemId, quantity);","replace":" const allocations: Array<{ warehouseId: bigint; quantity: number }> = [];"}]},{"id":"buy-now-creates-orders-without-reserving-stock--stock-limit","scenario":"tracks/ecommerce/scenarios/progression-stock-limit.json","targets":["ecommerce.spec.concurrency-safety.stock-limit.3d"],"desc":"Create purchase orders without reserving inventory. The same defect necessarily breaks live purchase stock and the sell-out portion of the stock-limit check.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" const allocations = decrementStockTracked(ctx, itemId, quantity);","replace":" const allocations: Array<{ warehouseId: bigint; quantity: number }> = [];"}]},{"id":"restock-race-records-wrong-order-total","scenario":"tracks/ecommerce/scenarios/01-restock-race.json","targets":["ecommerce.spec.concurrency-safety.restock-race.202a"],"desc":"Purchases preserve stock and visible order counts but record the wrong booked total. Native mixed-race reconciliation must reject them.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" total: Math.round(price * 100) * quantity / 100,\n status: 'pending',","replace":" total: Math.round(price * 100) * quantity / 100 + 1,\n status: 'pending',"}]},{"id":"buy-now-records-the-wrong-order-total","scenario":"tracks/ecommerce/scenarios/progression-purchasing.json","targets":["ecommerce.feature.purchasing.purchase-order.3c"],"desc":"Record a completed buy-now order one dollar above the stored item price.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" total: Math.round(price * 100) * quantity / 100,\n status: 'pending',","replace":" total: Math.round(price * 100) * quantity / 100 + 1,\n status: 'pending',"}]},{"id":"existing-cart-line-does-not-increment-basic-cart","scenario":"tracks/ecommerce/scenarios/progression-cart-checkout.json","targets":["ecommerce.feature.cart-checkout.cart.4a"],"desc":"Write an existing cart line back without incrementing its quantity.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.cartItem.id.update({ ...existing, quantity: existing.quantity + 1 });","replace":"ctx.db.cartItem.id.update({ ...existing, quantity: existing.quantity });"}]},{"id":"cart-is-deleted-when-owner-disconnects","scenario":"tracks/ecommerce/scenarios/01-cart.json","targets":["ecommerce.spec.state-durability.cart-reload.4b"],"desc":"Delete the account cart on transport disconnect. Reload loses stored cart contents even after the same account signs in again; account and session records remain intact.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"export const onDisconnect = spacetimedb.clientDisconnected((_ctx) => {});","replace":"export const onDisconnect = spacetimedb.clientDisconnected((ctx) => {\n const accountId = getAccountId(ctx);\n if (accountId !== null) for (const row of [...ctx.db.cartItem.byAccountItem.filter(accountId)]) ctx.db.cartItem.id.delete(row.id);\n});"}]},{"id":"signin-binds-the-second-client-to-a-different-account","scenario":"tracks/ecommerce/scenarios/01-cart.json","targets":["ecommerce.spec.live-state.shared-cart.4c"],"desc":"Authenticate valid credentials but bind the second connection to the administrator account, so two sessions for one customer do not share the customer's cart.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" const existingSession = ctx.db.session.identity.find(ctx.sender);\n if (existingSession) {\n ctx.db.session.identity.update({ ...existingSession, accountId: acc.id });\n } else {\n ctx.db.session.insert({ identity: ctx.sender, accountId: acc.id });\n }\n }\n);\n\nexport const signOut","replace":" const wrongAccount = ctx.db.account.username.find('admin') ?? acc;\n const existingSession = ctx.db.session.identity.find(ctx.sender);\n if (existingSession) {\n ctx.db.session.identity.update({ ...existingSession, accountId: wrongAccount.id });\n } else {\n ctx.db.session.insert({ identity: ctx.sender, accountId: wrongAccount.id });\n }\n }\n);\n\nexport const signOut"}]},{"id":"checkout-does-not-empty-the-basic-cart","scenario":"tracks/ecommerce/scenarios/progression-cart-checkout.json","targets":["ecommerce.feature.cart-checkout.cart.4d"],"desc":"Leave completed checkout lines in the durable cart.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" for (const line of lines) ctx.db.cartItem.id.delete(line.id);","replace":" // mutant: checked-out cart lines remain"}]},{"id":"new-review-is-accepted-without-being-stored","scenario":"tracks/ecommerce/scenarios/01-review-visibility.json","targets":["ecommerce.feature.reviews.reviews.6a"],"desc":"Accept an eligible new review but omit its durable insert, so neither author nor visitor can see it.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" ctx.db.review.insert({\n id: 0n,\n itemId,\n accountId: acc.id,\n rating,\n comment,\n createdAt: ctx.timestamp,\n });","replace":" // mutant: accepted review is not persisted"}]},{"id":"repeat-review-inserts-a-second-row","scenario":"tracks/ecommerce/scenarios/01-review-uniqueness.json","targets":["ecommerce.spec.transactional-integrity.unique-review.6b"],"desc":"Insert a second review row instead of updating the customer's existing item review.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" ctx.db.review.id.update({ ...existing, rating, comment, createdAt: ctx.timestamp });","replace":" ctx.db.review.insert({ id: 0n, itemId, accountId: acc.id, rating, comment, createdAt: ctx.timestamp });"}]},{"id":"review-average-counts-rows-instead-of-ratings","scenario":"tracks/ecommerce/scenarios/01-review-rating-live.json","targets":["ecommerce.spec.live-state.rating.6c"],"desc":"Compute the live average from a constant per row rather than each stored rating.","file":"client/src/components/ItemDetail.tsx","edits":[{"find":" : reviews.reduce((sum, r) => sum + r.rating, 0) / reviews.length;","replace":" : reviews.reduce((sum) => sum + 1, 0) / reviews.length;"}]},{"id":"every-signed-in-customer-is-treated-as-an-admin","scenario":"tracks/ecommerce/scenarios/01-warehouse-admin-staff.json","targets":["ecommerce.spec.access-control.warehouse-area-boundary.7a"],"desc":"Use account presence instead of the server-provided administrator flag to expose the admin area.","file":"client/src/App.tsx","edits":[{"find":" const isAdmin = currentUser?.isAdmin ?? false;","replace":" const isAdmin = isSignedIn;"}]},{"id":"warehouse-view-omits-west","scenario":"tracks/ecommerce/scenarios/01-warehouse-admin-staff.json","targets":["ecommerce.feature.warehouse-admin.warehouse-view.7b"],"desc":"Filter one real warehouse out of the administrator's inventory view.","file":"client/src/App.tsx","edits":[{"find":" warehouses={warehouses}","replace":" warehouses={warehouses.filter((warehouse) => warehouse.name !== 'West')}"}]},{"id":"guest-purchase-falls-back-to-the-admin-account","scenario":"tracks/ecommerce/scenarios/01-purchase-session.json","targets":["ecommerce.spec.access-control.purchase-session.101a"],"desc":"Allow an unauthenticated direct purchase by attributing missing sessions to the administrator account, while preserving the authorized control path.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"export const buyNow = spacetimedb.reducer({ itemId: t.u64() }, (ctx, { itemId }) => {\n const acc = requireAccount(ctx);","replace":"export const buyNow = spacetimedb.reducer({ itemId: t.u64() }, (ctx, { itemId }) => {\n const accountId = getAccountId(ctx);\n const acc = (accountId === null ? null : ctx.db.account.id.find(accountId))\n ?? ctx.db.account.username.find('admin')!;"}]},{"id":"direct-purchases-are-attributed-to-the-system-account","scenario":"tracks/ecommerce/scenarios/01-purchase-attribution.json","targets":["ecommerce.spec.access-control.purchase-attribution.102a"],"desc":"Store every buy-now order under the administrator instead of the authenticated caller.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" accountId,\n createdAt: ctx.timestamp,\n total: Math.round(price * 100) * quantity / 100,","replace":" accountId: ctx.db.account.username.find('admin')!.id,\n createdAt: ctx.timestamp,\n total: Math.round(price * 100) * quantity / 100,"}]},{"id":"direct-restock-does-not-require-an-admin","scenario":"tracks/ecommerce/scenarios/01-admin-write-staff.json","targets":["ecommerce.spec.access-control.warehouse-write-boundary.103b"],"desc":"Remove the server-side administrator check from the restock reducer.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"export const adminRestock = spacetimedb.reducer(\n { itemId: t.u64(), warehouseId: t.u64(), quantity: t.u32() },\n (ctx, { itemId, warehouseId, quantity }) => {\n requireAdmin(ctx);","replace":"export const adminRestock = spacetimedb.reducer(\n { itemId: t.u64(), warehouseId: t.u64(), quantity: t.u32() },\n (ctx, { itemId, warehouseId, quantity }) => {\n // mutant: no administrator check"}]},{"id":"direct-purchase-ignores-the-stored-price","scenario":"tracks/ecommerce/scenarios/01-server-price.json","targets":["ecommerce.spec.transactional-integrity.server-price.104a"],"desc":"Create the direct purchase order one dollar above the authoritative stored price.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" total: Math.round(price * 100) * quantity / 100,\n status: 'pending',","replace":" total: 1,\n status: 'pending',"}]},{"id":"account-state-token-is-not-restored-after-reload","scenario":"tracks/ecommerce/scenarios/progression-account-state-reload.json","targets":["ecommerce.spec.state-durability.account-state-recovery.105a"],"desc":"Build a reload connection without the durable identity token, losing the account and its data.","file":"client/src/main.tsx","edits":[{"find":".withToken(localStorage.getItem('auth_token') || undefined)","replace":".withToken(undefined)"}]},{"id":"reconnect-discards-the-visible-account-state","scenario":"tracks/ecommerce/scenarios/progression-account-state-reconnect.json","targets":["ecommerce.spec.state-durability.account-state-recovery.105b"],"desc":"Keep normal reload recovery but discard the client account projection when the browser comes back online.","file":"client/src/App.tsx","edits":[{"find":" const currentUser = currentUserRows[0] ?? null;","replace":" const [discardAccountAfterReconnect, setDiscardAccountAfterReconnect] = useState(false);\n useEffect(() => {\n const discardAccount = () => setDiscardAccountAfterReconnect(true);\n window.addEventListener('online', discardAccount);\n return () => window.removeEventListener('online', discardAccount);\n }, []);\n const currentUser = discardAccountAfterReconnect ? null : currentUserRows[0] ?? null;"}]},{"id":"order-views-return-every-customers-orders","scenario":"tracks/ecommerce/scenarios/01-order-ownership.json","targets":["ecommerce.spec.access-control.order-ownership.106a"],"desc":"Remove account filters from both order views, exposing another customer's order and its line items.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" const rows = [...ctx.db.customerOrder.accountId.filter(accountId)];","replace":" const rows = [...ctx.db.customerOrder.iter()];"},{"find":" for (const o of ctx.db.customerOrder.accountId.filter(accountId)) {","replace":" for (const o of ctx.db.customerOrder.iter()) {"}]},{"id":"admin-revenue-double-counts-every-order","scenario":"tracks/ecommerce/scenarios/progression-books-balance.json","targets":["ecommerce.spec.transactional-integrity.books-balance.107a"],"desc":"Count every completed order twice in the administrator revenue projection.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" total += o.total - o.refundedTotal;","replace":" total += (o.total - o.refundedTotal) * 2;"}]},{"id":"purchases-do-not-leave-the-warehouses","scenario":"tracks/ecommerce/scenarios/progression-books-balance.json","targets":["ecommerce.spec.transactional-integrity.books-balance.107b"],"desc":"Create normal orders and revenue while leaving warehouse stock unchanged.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" const allocations = decrementStockTracked(ctx, itemId, quantity);","replace":" const allocations: Array<{ warehouseId: bigint; quantity: number }> = [];"}]},{"id":"review-purchase-eligibility-is-not-checked","scenario":"tracks/ecommerce/scenarios/01-review-eligibility.json","targets":["ecommerce.spec.access-control.review-eligibility.108a"],"desc":"Allow a signed-in customer to review an item with no matching purchase.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" if (!bought) throw new SenderError('You can only review items you have purchased.');","replace":" // mutant: purchase eligibility is not checked"}]},{"id":"eligible-review-is-accepted-without-being-stored","scenario":"tracks/ecommerce/scenarios/01-review-eligibility.json","targets":["ecommerce.spec.access-control.review-eligibility.108b","ecommerce.spec.access-control.review-eligibility.108a"],"desc":"Keep the non-buyer refusal but omit the insert for a buyer's eligible new review. Both eligibility criteria require a successfully stored eligible review as a positive control; this does not establish a non-buyer authorization defect.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" ctx.db.review.insert({\n id: 0n,\n itemId,\n accountId: acc.id,\n rating,\n comment,\n createdAt: ctx.timestamp,\n });","replace":" // mutant: eligible review is acknowledged but not persisted"}]},{"id":"cart-line-lookup-ignores-cart-ownership","scenario":"tracks/ecommerce/scenarios/01-cart-boundary.json","targets":["ecommerce.spec.access-control.cart-boundary.109a"],"desc":"Find an existing cart line by item alone, so the same named add action from another customer increments the owner's line instead of that customer's cart.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"function findCartLine(ctx: Ctx, accountId: bigint, itemId: bigint) {\n for (const row of ctx.db.cartItem.byAccountItem.filter([accountId, itemId])) {\n return row;\n }\n return null;\n}","replace":"function findCartLine(ctx: Ctx, _accountId: bigint, itemId: bigint) {\n for (const row of ctx.db.cartItem.iter()) {\n if (row.itemId === itemId) return row;\n }\n return null;\n}"}]},{"id":"purchase-does-not-reserve-stock-last-unit","scenario":"tracks/ecommerce/scenarios/01-last-unit.json","targets":["ecommerce.spec.concurrency-safety.last-unit.201a","ecommerce.spec.concurrency-safety.last-unit.201b","ecommerce.spec.concurrency-safety.last-unit.201c"],"desc":"Create purchase orders without reserving stock, proving the focused last-unit stock, order-count, and revenue consequences.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" const allocations = decrementStockTracked(ctx, itemId, quantity);","replace":" const allocations: Array<{ warehouseId: bigint; quantity: number }> = [];"}]},{"id":"existing-cart-line-does-not-increment","scenario":"tracks/ecommerce/scenarios/01-duplicate-checkout.json","targets":["ecommerce.spec.concurrency-safety.duplicate-checkout.203a"],"desc":"Render the old cart quantity after concurrent adds.","file":"client/src/components/CartPanel.tsx","edits":[{"find":" value={line.quantity}","replace":" value={1}"}]},{"id":"checkout-does-not-empty-cart","scenario":"tracks/ecommerce/scenarios/01-duplicate-checkout.json","targets":["ecommerce.spec.concurrency-safety.duplicate-checkout.203b"],"desc":"Keep checked-out cart lines so the next serialized checkout creates a duplicate order.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" for (const line of lines) ctx.db.cartItem.id.delete(line.id);","replace":" // mutant: checked-out lines remain in the cart"}]},{"id":"stock-subscription-snapshotted-once","scenario":"tracks/ecommerce/scenarios/01-external-live-sync.json","targets":["ecommerce.spec.external-data-sync.external-stock.901a"],"desc":"Render the first non-empty stock snapshot forever instead of following committed subscription updates.","file":"client/src/App.tsx","edits":[{"find":"import { useEffect, useMemo, useState } from 'react';","replace":"import { useEffect, useMemo, useRef, useState } from 'react';"},{"find":" const [stocks] = useTable(tables.stock);","replace":" const [liveStocks] = useTable(tables.stock);\n const initialStocks = useRef<typeof liveStocks | null>(null);\n if (initialStocks.current === null && liveStocks.length > 0) {\n initialStocks.current = liveStocks;\n }\n const stocks = initialStocks.current ?? liveStocks;"}]},{"id":"stock-view-ignores-update-across-app-server-stop","scenario":"tracks/ecommerce/scenarios/01-external-server-restart-sync.json","targets":["ecommerce.spec.external-data-sync.external-stock.901c"],"desc":"Persist the first stock quantities in browser session storage and keep rendering them after app-server restart, including any frontend reload. Initial stock remains correct. This validates the stale-view oracle, not SpacetimeDB storage durability.","file":"client/src/App.tsx","edits":[{"find":" const [stocks] = useTable(tables.stock);","replace":" const [liveStocks] = useTable(tables.stock);\n const cacheKey = 'stale-stock-quantities';\n let savedQuantities = sessionStorage.getItem(cacheKey);\n if (!savedQuantities && liveStocks.length > 0) {\n savedQuantities = JSON.stringify(Object.fromEntries(liveStocks.map(row => [`${row.itemId}-${row.warehouseId}`, row.quantity])));\n sessionStorage.setItem(cacheKey, savedQuantities);\n }\n const quantities: Record<string, number> = JSON.parse(savedQuantities ?? '{}');\n const stocks = liveStocks.map(row => ({ ...row, quantity: quantities[`${row.itemId}-${row.warehouseId}`] ?? row.quantity }));"}]},{"id":"stock-view-keeps-pre-reconnect-snapshot","scenario":"tracks/ecommerce/scenarios/01-external-reconnect-sync.json","targets":["ecommerce.spec.external-data-sync.external-stock.901d"],"desc":"Continue following stock until the browser goes offline, then retain the last online snapshot after network restoration.","file":"client/src/App.tsx","edits":[{"find":"import { useEffect, useMemo, useState } from 'react';","replace":"import { useEffect, useMemo, useRef, useState } from 'react';"},{"find":" const [stocks] = useTable(tables.stock);","replace":" const [liveStocks] = useTable(tables.stock);\n const [freezeStockAfterOffline, setFreezeStockAfterOffline] = useState(false);\n const lastOnlineStocks = useRef(liveStocks);\n useEffect(() => {\n const freezeStock = () => setFreezeStockAfterOffline(true);\n window.addEventListener('offline', freezeStock);\n return () => window.removeEventListener('offline', freezeStock);\n }, []);\n if (!freezeStockAfterOffline) {\n lastOnlineStocks.current = liveStocks;\n }\n const stocks = freezeStockAfterOffline ? lastOnlineStocks.current : liveStocks;"}]},{"id":"open-review-list-snapshots-on-selection","scenario":"tracks/ecommerce/scenarios/progression-open-list-live.json","targets":["ecommerce.spec.live-state.open-list.902a"],"desc":"Snapshot the selected item's reviews when the detail opens instead of following later subscription updates.","file":"client/src/App.tsx","edits":[{"find":"import { useEffect, useMemo, useState } from 'react';","replace":"import { useEffect, useMemo, useRef, useState } from 'react';"},{"find":" const selectedItemReviews = selectedItemId !== null ? reviewsByItem.get(selectedItemId) ?? [] : [];","replace":" const openedReviewItem = useRef<bigint | null>(null);\n const openedReviews = useRef<(typeof reviews)[number][]>([]);\n if (selectedItemId !== openedReviewItem.current) {\n openedReviewItem.current = selectedItemId;\n openedReviews.current = selectedItemId !== null ? reviewsByItem.get(selectedItemId) ?? [] : [];\n }\n const selectedItemReviews = openedReviews.current;"}]},{"id":"open-review-list-renders-each-review-twice","scenario":"tracks/ecommerce/scenarios/progression-open-list-live.json","targets":["ecommerce.spec.live-state.open-list.902a"],"desc":"Render every committed review twice in the already-open list.","file":"client/src/components/ItemDetail.tsx","edits":[{"find":" {reviews.map((r) => (","replace":" {[...reviews, ...reviews].map((r) => ("}]},{"id":"cancel-does-not-restore-stock-feature","scenario":"tracks/ecommerce/scenarios/02-order-cancellation-core.json","targets":["ecommerce.returns-pricing.cancellation-and-return.3a"],"desc":"The serialized cancellation reducer changes order state and purchase counts but skips allocation restoration.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" restoreOrderItemStock(ctx, li);\n decrementPurchaseCount(ctx, li.itemId, li.quantity);","replace":" // mutant: cancellation does not restore its reserved stock\n decrementPurchaseCount(ctx, li.itemId, li.quantity);"}]},{"id":"cancellation-accounting-loses-stock-restoration","scenario":"tracks/ecommerce/scenarios/02-invariants.json","targets":["ecommerce.returns-pricing.refund-accounting.203a"],"desc":"Cancellation removes revenue and changes order status, but loses the original warehouse stock restoration. The native refund-accounting assertion must detect this.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" restoreOrderItemStock(ctx, li);\n decrementPurchaseCount(ctx, li.itemId, li.quantity);","replace":" // mutant: cancellation does not restore its reserved stock\n decrementPurchaseCount(ctx, li.itemId, li.quantity);"}]},{"id":"cancel-does-not-restore-stock-fresh-client","scenario":"tracks/ecommerce/scenarios/02-self-contained.json","targets":["ecommerce.inventory-operations.stock-conservation.202b","ecommerce.inventory-operations.stock-conservation.202c"],"desc":"The serialized cancellation reducer skips allocation restoration, so a fresh client reads the persisted shortfall.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" restoreOrderItemStock(ctx, li);\n decrementPurchaseCount(ctx, li.itemId, li.quantity);","replace":" // mutant: cancellation does not restore its reserved stock\n decrementPurchaseCount(ctx, li.itemId, li.quantity);"}]},{"id":"cancel-restores-stock-but-keeps-pending-status","scenario":"tracks/ecommerce/scenarios/02-order-cancellation-history.json","targets":["ecommerce.returns-pricing.cancellation-and-return.3b"],"desc":"The serialized cancellation reducer restores allocations but writes pending back to order history.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" ctx.db.customerOrder.id.update({ ...order, status: 'cancelled' });","replace":" ctx.db.customerOrder.id.update({ ...order, status: 'pending' });"}]},{"id":"cancelled-order-remains-in-revenue-feature","scenario":"tracks/ecommerce/scenarios/02-order-cancellation-core.json","targets":["ecommerce.returns-pricing.cancellation-and-return.3a"],"desc":"The admin revenue view includes cancelled orders even though cancellation otherwise succeeds.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" if (!isOrderCounted(o)) continue;\n total += o.total - o.refundedTotal;","replace":" total += o.total - o.refundedTotal;"}]},{"id":"cancelled-order-remains-in-revenue-invariant","scenario":"tracks/ecommerce/scenarios/02-invariants.json","targets":["ecommerce.returns-pricing.refund-accounting.203a"],"desc":"The admin revenue view includes cancelled orders even though cancellation otherwise succeeds.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" if (!isOrderCounted(o)) continue;\n total += o.total - o.refundedTotal;","replace":" total += o.total - o.refundedTotal;"}]},{"id":"operator-authorization-allows-customer-transfer","scenario":"tracks/ecommerce/scenarios/02-strengthened.json","targets":["ecommerce.operations-access.operator-authorization.201a"],"desc":"The transfer reducer drops its administrator role gate.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" (ctx, { itemId, fromWarehouseId, toWarehouseId, quantity }) => {\n requireAdmin(ctx);","replace":" (ctx, { itemId, fromWarehouseId, toWarehouseId, quantity }) => {\n // mutant: no administrator role check"}]},{"id":"customer-can-ship-order-direct-1-1","scenario":"tracks/ecommerce/scenarios/02-server-actions.json","targets":["ecommerce.operations-access.operator-authorization.201c"],"desc":"The shipping reducer drops its staff role check while retaining pending-state validation.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n requireStaffOrAdmin(ctx);","replace":"export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n // mutant: no staff role check"}]},{"id":"customer-can-cancel-foreign-order-1-1","scenario":"tracks/ecommerce/scenarios/02-server-actions.json","targets":["ecommerce.operations-access.order-owner.204a"],"desc":"Cancellation bypasses the owner helper while retaining missing-order and pending-state validation.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"export const cancelOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n const order = requireOrderOwner(ctx, orderId);","replace":"export const cancelOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n const order = ctx.db.customerOrder.id.find(orderId);\n if (!order) throw new SenderError('Order not found.');"}]},{"id":"queue-depth-lags-one-order","scenario":"tracks/ecommerce/scenarios/02-fulfilment-live.json","targets":["ecommerce.operations-access.fulfilment-queue.1a"],"desc":"The reactive queue renders every order but its visible depth remains one behind.","file":"client/src/components/FulfilmentPanel.tsx","edits":[{"find":"Waiting: <span data-role=\"queue-depth\">{queue.length}</span>","replace":"Waiting: <span data-role=\"queue-depth\">{Math.max(0, queue.length - 1)}</span>"}]},{"id":"ship-acknowledges-without-changing-status","scenario":"tracks/ecommerce/scenarios/02-fulfilment-ship.json","targets":["ecommerce.operations-access.fulfilment-queue.1c"],"desc":"The serialized shipping reducer accepts the call but writes pending back to the order.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" ctx.db.customerOrder.id.update({ ...order, status: 'shipped' });","replace":" ctx.db.customerOrder.id.update({ ...order, status: 'pending' });"}]},{"id":"customer-sees-fulfilment-navigation","scenario":"tracks/ecommerce/scenarios/02-features.json","targets":["ecommerce.operations-access.fulfilment-queue.1d"],"desc":"Expose the protected staff area to signed-in customers, including its navigation and content.","file":"client/src/App.tsx","edits":[{"find":" const isStaff = currentUser?.isStaff ?? false;","replace":" const isStaff = isSignedIn;"}]},{"id":"progression-customer-sees-fulfilment-content","scenario":"tracks/ecommerce/scenarios/02-fulfilment-access.json","targets":["ecommerce.spec.access-control.fulfilment-area-boundary.1d"],"desc":"Expose the protected staff area to signed-in customers, including its navigation and content.","file":"client/src/App.tsx","edits":[{"find":" const isStaff = currentUser?.isStaff ?? false;","replace":" const isStaff = isSignedIn;"}]},{"id":"operator-authorization-allows-customer-shipping","scenario":"tracks/ecommerce/scenarios/02-server-actions.json","targets":["ecommerce.operations-access.operator-authorization.201c"],"desc":"The shipping reducer drops the staff role check while retaining the pending-order guard.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n requireStaffOrAdmin(ctx);","replace":"export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n // mutant: no staff role check"}]},{"id":"transfer-debits-source-without-crediting-existing-destination","scenario":"tracks/ecommerce/scenarios/02-strengthened.json","targets":["ecommerce.inventory-operations.stock-conservation.202a","ecommerce.inventory-operations.warehouse-transfer.2a","ecommerce.operations-access.operator-authorization.201a"],"desc":"A transfer debits the source row but writes the existing destination quantity back unchanged, violating both directional movement and total conservation inside the serialized reducer. It also breaks 201a's authorized-transfer positive control; that coupled failure is not independent evidence of an authorization defect.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" ctx.db.stock.insert({ ...toRow, quantity: toRow.quantity + quantity });","replace":" ctx.db.stock.insert({ ...toRow, quantity: toRow.quantity });"}]},{"id":"transfer-warehouse-totals-omit-destination-credit","scenario":"tracks/ecommerce/scenarios/02-transfer-totals.json","targets":["ecommerce.inventory-operations.warehouse-transfer.2b"],"desc":"The serialized transfer debits the source but writes the existing destination quantity back unchanged.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" ctx.db.stock.insert({ ...toRow, quantity: toRow.quantity + quantity });","replace":" ctx.db.stock.insert({ ...toRow, quantity: toRow.quantity });"}]},{"id":"transfer-overdraft-guard-removed","scenario":"tracks/ecommerce/scenarios/02-transfer-overdraw.json","targets":["ecommerce.inventory-operations.warehouse-transfer.2c"],"desc":"The serialized reducer no longer rejects insufficient source stock and commits negative source quantity.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" if (available < quantity) {\n throw new SenderError(`Not enough stock in source warehouse: only ${available} available.`);\n }","replace":" // mutant: insufficient source stock is not rejected"}]},{"id":"low-stock-excludes-boundary-ten","scenario":"tracks/ecommerce/scenarios/02-low-stock.json","targets":["ecommerce.inventory-operations.operational-views.5a"],"desc":"The reactive low-stock view uses a strict boundary and omits items with exactly ten units.","file":"client/src/App.tsx","edits":[{"find":" .filter((i) => (stockByItem.get(i.id) ?? 0) <= LOW_STOCK_THRESHOLD)","replace":" .filter((i) => (stockByItem.get(i.id) ?? 0) < LOW_STOCK_THRESHOLD)"}]},{"id":"category-totals-ignore-pending-purchases","scenario":"tracks/ecommerce/scenarios/02-operational-category-totals.json","targets":["ecommerce.inventory-operations.operational-views.5b"],"desc":"The category totals view includes only shipped orders, so a newly accepted pending purchase is absent.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" for (const order of ctx.db.customerOrder.iter()) {\n if (!isOrderCounted(order)) continue;","replace":" for (const order of ctx.db.customerOrder.iter()) {\n if (order.status !== 'shipped') continue;"}]},{"id":"recommendations-ignore-pending-purchases","scenario":"tracks/ecommerce/scenarios/02-operational-recommendations.json","targets":["ecommerce.inventory-operations.operational-views.5c"],"desc":"The personal recommendation view derives categories only from shipped orders, so a new pending purchase has no influence.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" for (const order of ctx.db.customerOrder.accountId.filter(accountId)) {\n if (!isOrderCounted(order)) continue;","replace":" for (const order of ctx.db.customerOrder.accountId.filter(accountId)) {\n if (order.status !== 'shipped') continue;"}]},{"id":"purchases-do-not-affect-best-sellers","scenario":"tracks/ecommerce/scenarios/02-operational-best-sellers.json","targets":["ecommerce.inventory-operations.operational-views.5d"],"desc":"Rank signed-out recommendations without purchase counts.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"const purchaseCountOf = (id: bigint) => ctx.db.itemStats.itemId.find(id)?.purchaseCount ?? 0;","replace":"const purchaseCountOf = (_id: bigint) => 0;"}]},{"id":"queue-warehouse-reports-west","scenario":"tracks/ecommerce/scenarios/02-queue-warehouse.json","targets":["ecommerce.operations-access.fulfilment-queue.1b"],"desc":"The queue renders West for the deterministic Desk Lamp allocation even though the order reserved stock in East.","file":"client/src/components/FulfilmentPanel.tsx","edits":[{"find":"{name}: {order.warehouseNames[i]}","replace":"{name}: West"}]},{"id":"transfer-creates-stock-during-race","scenario":"tracks/ecommerce/scenarios/02-server-actions.json","targets":["ecommerce.inventory-operations.stock-conservation.202d"],"desc":"The transfer reducer credits the destination one unit more than it debits from the source, so the item's total after a transfer racing a purchase is the starting total rather than one less. A stored conservation defect; it does not model a lost-update interleaving because SpacetimeDB reducer execution is atomically serialized.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" ctx.db.stock.insert({ ...toRow, quantity: toRow.quantity + quantity });","replace":" ctx.db.stock.insert({ ...toRow, quantity: toRow.quantity + quantity + 1 });"}]},{"id":"catalog-search-ignores-the-query","scenario":"tracks/ecommerce/scenarios/01-catalog-search.json","targets":["ecommerce.feature.catalog.catalog-search.2d"],"desc":"A non-empty catalog query filters out every product instead of matching names.","file":"client/src/App.tsx","edits":[{"find":".filter(item => !q || item.name.toLowerCase().includes(q))","replace":".filter(() => !q) // mutant: non-empty searches return no products"}]},{"id":"admin-total-stock-is-not-rendered","scenario":"tracks/ecommerce/scenarios/01-warehouse-stock-live-staff.json","targets":["ecommerce.spec.live-state.warehouse-stock.7c"],"desc":"The staff stock total always renders zero after a warehouse restock.","file":"client/src/components/AdminPanel.tsx","edits":[{"find":"<td data-role=\"admin-stock\">{totalStockOf(item.id)}</td>","replace":"<td data-role=\"admin-stock\">{0}</td>"}]},{"id":"customers-can-schedule-restocks","scenario":"tracks/ecommerce/scenarios/03-deferred-access.json","targets":["ecommerce.l3.deferred-access.scheduled-work-access.317a"],"desc":"Scheduling a restock no longer checks that the caller is an administrator.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" (ctx, input) => {\n requireAdmin(ctx);\n const itemName = input.item.trim();\n const warehouseName = input.warehouse.trim();\n const item = [...ctx.db.item].find(row => row.name === itemName);\n const warehouse = [...ctx.db.warehouse].find(row => row.name === warehouseName);","replace":" (ctx, input) => {\n // mutant: any signed-in or anonymous caller can schedule work\n const itemName = input.item.trim();\n const warehouseName = input.warehouse.trim();\n const item = [...ctx.db.item].find(row => row.name === itemName);\n const warehouse = [...ctx.db.warehouse].find(row => row.name === warehouseName);"}]},{"id":"scheduled-restock-execution-queue-is-process-local","scenario":"tracks/ecommerce/scenarios/03-deferred-durability.json","targets":["ecommerce.l3.deferred-durability.restart-survival.311a"],"desc":"Keep manual restock execution IDs only in the V8 process. Ordinary timers work, but restart loses the execution queue while pending rows remain. Isolate replacement can also lose this queue.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"export const scheduleRestock = spacetimedb.reducer(","replace":"const pendingRestockExecution = new Set<bigint>();\n\nexport const scheduleRestock = spacetimedb.reducer("},{"find":" ctx.db.scheduledRestock.insert({\n id: 0n,\n itemId: item.id,\n warehouseId: warehouse.id,\n quantity: input.quantity,\n dueMicros: nowMicros(ctx) + BigInt(input.delaySeconds) * SECOND,\n status: 'pending',\n reorderRuleId: undefined,\n });","replace":" const pending = ctx.db.scheduledRestock.insert({\n id: 0n,\n itemId: item.id,\n warehouseId: warehouse.id,\n quantity: input.quantity,\n dueMicros: nowMicros(ctx) + BigInt(input.delaySeconds) * SECOND,\n status: 'pending',\n reorderRuleId: undefined,\n });\n pendingRestockExecution.add(pending.id);"},{"find":" if (pending.status !== 'pending' || pending.dueMicros > now) continue;\n restoreStock(ctx, pending.itemId, pending.warehouseId, pending.quantity);","replace":" if (pending.status !== 'pending' || pending.dueMicros > now) continue;\n if (pending.reorderRuleId === undefined && !pendingRestockExecution.delete(pending.id)) continue;\n restoreStock(ctx, pending.itemId, pending.warehouseId, pending.quantity);"}]},{"id":"reservation-is-delayed-past-the-durability-window","scenario":"tracks/ecommerce/scenarios/03-deferred-durability.json","targets":["ecommerce.l3.deferred-durability.restart-survival.314a"],"desc":"A reservation is persisted with a ten-minute lifetime instead of ninety seconds.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"const expiresMicros = nowMicros(ctx) + 90n * SECOND;","replace":"const expiresMicros = nowMicros(ctx) + 600n * SECOND;"}]},{"id":"completed-restock-remains-pending","scenario":"tracks/ecommerce/scenarios/03-deferred-integrity.json","targets":["ecommerce.l3.deferred-integrity.exactly-once.311a"],"desc":"A completed restock remains pending and is applied again by later maintenance ticks.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.scheduledRestock.id.update({ ...pending, status: 'complete' });","replace":"ctx.db.scheduledRestock.id.update({ ...pending, status: 'pending' });"}]},{"id":"reservation-expiry-restores-stock-twice","scenario":"tracks/ecommerce/scenarios/03-deferred-integrity.json","targets":["ecommerce.l3.deferred-integrity.stock-conservation.313a"],"desc":"Reservation expiry returns twice the quantity that was reserved.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"restoreStock(ctx, row.stockItemId || row.itemId, row.warehouseId, row.quantity);\n ctx.db.reservation.id.update({ ...row, expired: true });","replace":"restoreStock(ctx, row.stockItemId || row.itemId, row.warehouseId, row.quantity * 2);\n ctx.db.reservation.id.update({ ...row, expired: true });"}]},{"id":"checkout-takes-reserved-stock-again","scenario":"tracks/ecommerce/scenarios/03-deferred-integrity.json","targets":["ecommerce.l3.deferred-integrity.stock-conservation.314a"],"desc":"Checkout decrements stock after the cart reservation already took it.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"const allocations = held.map(row => ({ warehouseId: row.warehouseId, quantity: row.quantity, stockItemId: row.stockItemId }));\n const orderItemRow","replace":"const allocations = decrementStockTracked(ctx, p.itemId, p.quantity);\n const orderItemRow"}]},{"id":"reservation-does-not-decrement-stock","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.301a"],"desc":"Creating a reservation leaves the public stock total unchanged.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.stock.insert({ ...row, quantity: row.quantity - allocation.quantity });","replace":"ctx.db.stock.insert({ ...row, quantity: row.quantity });"}]},{"id":"reservation-timer-is-static","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.305a"],"desc":"The cart always renders ninety seconds instead of a decreasing reservation timer.","file":"client/src/components/CartPanel.tsx","edits":[{"find":"const seconds = Math.max(0, Number((reservation.expiresMicros - BigInt(Date.now()) * 1000n) / 1_000_000n));","replace":"const seconds = 90;"}]},{"id":"checkout-leaves-cart-lines","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.306a"],"desc":"Checkout creates an order but leaves the purchased lines in the cart.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"for (const line of lines) ctx.db.cartItem.id.delete(line.id);","replace":"for (const line of lines) void line;"}]},{"id":"expired-reservation-is-still-marked-live","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.307a"],"desc":"Expired reservations keep their live flag, so the cart does not mark them expired.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.reservation.id.update({ ...row, expired: true });","replace":"ctx.db.reservation.id.update({ ...row, expired: false });"}]},{"id":"renewed-reservation-expires-too-soon","scenario":"tracks/ecommerce/scenarios/03-reservations.json","targets":["ecommerce.l3.reservations.reservations.308a"],"desc":"Renewed quantities receive only a twenty-second reservation window.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"reserveUnits(ctx, accountId, itemId, quantity);","replace":"reserveUnits(ctx, accountId, itemId, quantity);\n for (const renewed of findReservations(ctx, accountId, itemId)) {\n ctx.db.reservation.id.update({ ...renewed, expiresMicros: nowMicros(ctx) + 20n * SECOND });\n }"}]},{"id":"pending-restock-timer-is-static","scenario":"tracks/ecommerce/scenarios/03-scheduled-restocks.json","targets":["ecommerce.l3.scheduled-restocks.scheduled-restocks.302a"],"desc":"The pending restock UI always renders ninety seconds instead of the server due time.","file":"client/src/components/ProgressionWorkbench.tsx","edits":[{"find":"{Math.max(0, Number((row.dueMicros - BigInt(Date.now()) * 1000n) / 1_000_000n))}","replace":"{90}"}]},{"id":"due-restock-omits-ledger-entry","scenario":"tracks/ecommerce/scenarios/03-scheduled-restock-apply.json","targets":["ecommerce.l3.scheduled-restocks.scheduled-restocks.305a"],"desc":"A due restock updates stock but does not create its stock ledger record.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.stockLedger.insert({\n id: 0n,\n itemId: pending.itemId,\n warehouseId: pending.warehouseId,\n quantity: pending.quantity,\n createdMicros: now,\n source: 'scheduled restock',\n });","replace":"// mutant: due restocks are not recorded in the ledger"}]},{"id":"cancelled-restock-remains-pending","scenario":"tracks/ecommerce/scenarios/03-scheduled-restock-cancel.json","targets":["ecommerce.l3.scheduled-restocks.scheduled-restocks.306a"],"desc":"Cancelling a restock leaves it pending, so maintenance later applies it.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.scheduledRestock.id.update({ ...row, status: 'cancelled' });","replace":"ctx.db.scheduledRestock.id.update({ ...row, status: 'pending' });"}]},{"id":"restart-restock-runs-early","scenario":"tracks/ecommerce/scenarios/03-server-time.json","targets":["ecommerce.l3.server-time.server-time.312a"],"desc":"A scheduled restock ignores its requested delay and becomes due after one second.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"dueMicros: nowMicros(ctx) + BigInt(input.delaySeconds) * SECOND,","replace":"dueMicros: nowMicros(ctx) + SECOND,"}]},{"id":"closed-browser-reservation-never-expires","scenario":"tracks/ecommerce/scenarios/03-server-time.json","targets":["ecommerce.l3.server-time.server-time.313a"],"desc":"Server maintenance skips reservation expiry when no customer browser is present.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"for (const row of [...ctx.db.reservation.iter()]) {\n if (row.expired || row.expiresMicros > now) continue;","replace":"for (const row of [...ctx.db.reservation.iter()].filter(() => false)) {\n if (row.expired || row.expiresMicros > now) continue;"}]},{"id":"catalog-product-is-not-published","scenario":"tracks/ecommerce/scenarios/progression-catalog-management.json","targets":["ecommerce.progression.catalog-management.catalog-management.622a","ecommerce.progression.catalog-management.catalog-management.622b"],"desc":"The public product card omits the submitted name. Both create visibility and variant navigation locate the submitted product card by its name, so the hidden name prevents both required observations.","file":"client/src/components/ItemCard.tsx","edits":[{"find":"{item.name}","replace":"{item.name === 'Travel Mug' ? '' : item.name}"}]},{"id":"catalog-variants-are-discarded","scenario":"tracks/ecommerce/scenarios/progression-catalog-management.json","targets":["ecommerce.progression.catalog-management.catalog-management.622b"],"desc":"Catalog management discards every submitted product variant.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"for (const variantName of variants.split(',').map(value => value.trim()).filter(Boolean)) {\n ctx.db.itemVariant.insert({ id: 0n, itemId: product.id, name: variantName });\n }","replace":"void variants; // mutant: submitted variants are discarded"}]},{"id":"profile-is-lost-on-fresh-account-login","scenario":"tracks/ecommerce/scenarios/progression-customer-profile.json","targets":["ecommerce.spec.state-durability.customer-profile-reload.620a"],"desc":"Delete a saved profile when its owner signs in from a new transport identity. Initial save, same-identity reload, and the independent privacy owner remain intact.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" throw new SenderError('Incorrect username or password.');\n }\n\n const existingSession = ctx.db.session.identity.find(ctx.sender);\n if (existingSession) {\n ctx.db.session.identity.update({ ...existingSession, accountId: acc.id });","replace":" throw new SenderError('Incorrect username or password.');\n }\n\n const existingSession = ctx.db.session.identity.find(ctx.sender);\n if (!existingSession) { ctx.db.customerProfile.accountId.delete(acc.id); }\n if (existingSession) {\n ctx.db.session.identity.update({ ...existingSession, accountId: acc.id });"}]},{"id":"customer-profile-view-leaks-another-account","scenario":"tracks/ecommerce/scenarios/progression-customer-profile.json","targets":["ecommerce.spec.access-control.customer-profile-privacy.620b"],"desc":"The customer profile view returns the first stored profile without checking its owner.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"return accountId === null ? undefined : ctx.db.customerProfile.accountId.find(accountId) ?? undefined;","replace":"return accountId === null ? undefined : [...ctx.db.customerProfile.iter()][0];"}]},{"id":"faceted-search-ignores-category","scenario":"tracks/ecommerce/scenarios/progression-faceted-filters.json","targets":["ecommerce.progression.faceted-search.faceted-search.401a"],"desc":"Faceted search applies price and stock filters but ignores the selected category.","file":"client/src/App.tsx","edits":[{"find":".filter(item => !categoryFilter || categoryByItem.get(item.id) === categoryFilter)","replace":".filter(() => true) // mutant: category filter is ignored"}]},{"id":"active-search-uses-purchase-ranking","scenario":"tracks/ecommerce/scenarios/progression-search-ordering.json","targets":["ecommerce.spec.search-ordering.search-ordering.402b"],"desc":"Keep purchase ranking when search text or filters are active.","file":"client/src/App.tsx","edits":[{"find":"const catalogItems = showingSearch ? filteredSearchResults : rankedItems;","replace":"const catalogItems = rankedItems;"}]},{"id":"faceted-search-next-page-does-not-advance","scenario":"tracks/ecommerce/scenarios/progression-faceted-pagination.json","targets":["ecommerce.progression.faceted-search.faceted-search.402a"],"desc":"The next-page control keeps the search on its current page.","file":"client/src/App.tsx","edits":[{"find":"onClick={() => setSearchPage(page => page + 1)}>Next</button>","replace":"onClick={() => setSearchPage(page => page)}>Next</button>"}]},{"id":"managed-support-leaks-and-accepts-cross-account-replies","scenario":"tracks/ecommerce/scenarios/progression-managed-support-privacy.json","targets":["ecommerce.spec.access-control.managed-support-privacy.613b"],"desc":"Managed support tickets are visible across accounts and replayed replies are accepted.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":".filter(row => isGuestTicketCreator(sender, row.creatorIdentity.toHexString(), row.accountId) ||\n !!actor && (actor.isAdmin || actor.isStaff || row.accountId === accountId))","replace":".filter(() => true)"},{"file":"backend/spacetimedb/src/index.ts","find":"if (!actor.isAdmin && !actor.isStaff && ticket.accountId !== actor.id) {\n throw new SenderError('That support ticket is private.');\n }","replace":"// mutant: any signed-in account can access any support ticket"}]},{"id":"managed-support-replies-are-empty","scenario":"tracks/ecommerce/scenarios/progression-managed-support-shared.json","targets":["ecommerce.spec.live-state.managed-support.613a","ecommerce.progression.managed-support.managed-support.613c"],"desc":"Managed support stores replies without their message body. Both the ordinary reply and shared-live reply assertions require the stored message body; neither can pass an empty reply.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"body: body.trim(),\n createdMicros: nowMicros(ctx),","replace":"body: '',\n createdMicros: nowMicros(ctx),"}]},{"id":"managed-support-live-replies-stay-at-initial-snapshot","scenario":"tracks/ecommerce/scenarios/progression-managed-support-shared.json","targets":["ecommerce.spec.live-state.managed-support.613a"],"desc":"Keep the initial subscribed reply snapshot on each page. Reducers still save replies, and a reload shows them, but later replies do not reach the rendered conversation live.","file":"client/src/App.tsx","edits":[{"find":" const [supportReplyRows] = useTable(tables.visibleSupportReplies);","replace":" const [liveSupportReplyRows, supportRepliesReady] = useTable(tables.visibleSupportReplies);\n const supportReplyRows = useMemo(() => [...liveSupportReplyRows], [supportRepliesReady]);"}]},{"id":"notification-preferences-are-not-saved","scenario":"tracks/ecommerce/scenarios/progression-notification-preferences.json","targets":["ecommerce.spec.state-durability.notification-preferences-reload.630a"],"desc":"Saving notification preferences discards the selected values.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"if (existing) ctx.db.notificationPreference.accountId.update(row);\n else ctx.db.notificationPreference.insert(row);","replace":"void existing;\n void row; // mutant: notification preferences are discarded"}]},{"id":"notification-preferences-leak-across-accounts","scenario":"tracks/ecommerce/scenarios/progression-notification-preferences.json","targets":["ecommerce.spec.access-control.notification-preferences-privacy.630b"],"desc":"The preference view returns another account's first stored choice.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"const row = ctx.db.notificationPreference.accountId.find(accountId);\n return row ? { orderEnabled: row.orderEnabled, stockEnabled: row.stockEnabled } : undefined;","replace":"const row = [...ctx.db.notificationPreference.iter()][0];\n return row ? { orderEnabled: row.orderEnabled, stockEnabled: row.stockEnabled } : undefined;"}]},{"id":"checkout-records-zero-payment","scenario":"tracks/ecommerce/scenarios/progression-core-business.json","targets":["ecommerce.progression.payment-records.payment-records.623a"],"desc":"Checkout records a paid payment with a zero amount.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.paymentRecord.insert({ id: 0n, orderId: order.id, amount: order.total, status: 'paid' });\n if (promo) {","replace":"ctx.db.paymentRecord.insert({ id: 0n, orderId: order.id, amount: 0, status: 'paid' });\n if (promo) {"}]},{"id":"checkout-records-duplicate-payments","scenario":"tracks/ecommerce/scenarios/progression-core-business.json","targets":["ecommerce.spec.transactional-integrity.payment-deduplication.623b"],"desc":"Checkout inserts two payment records for one order.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.paymentRecord.insert({ id: 0n, orderId: order.id, amount: order.total, status: 'paid' });\n if (promo) {","replace":"ctx.db.paymentRecord.insert({ id: 0n, orderId: order.id, amount: order.total, status: 'paid' });\n ctx.db.paymentRecord.insert({ id: 0n, orderId: order.id, amount: order.total + 0.01, status: 'paid' });\n if (promo) {"}]},{"id":"active-promotion-does-not-discount-checkout","scenario":"tracks/ecommerce/scenarios/progression-promotion-checkout.json","targets":["ecommerce.progression.promotion-checkout.promotion-checkout-active.621a"],"desc":"Checkout ignores an active promotion when it calculates the discount.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"const discount = promo ? total * (promo.discountPercent / 100) : 0;","replace":"const discount = 0;"}]},{"id":"expired-promotion-is-accepted","scenario":"tracks/ecommerce/scenarios/progression-promotion-checkout.json","targets":["ecommerce.progression.promotion-checkout.promotion-checkout-expired.621b"],"desc":"Promotion application does not reject a promotion after its end time.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"if (!promo || promo.startMicros > now || promo.endMicros < now || promo.redemptions >= promo.usageLimit) {","replace":"if (!promo || promo.startMicros > now || promo.redemptions >= promo.usageLimit) {"}]},{"id":"exhausted-promotion-is-accepted","scenario":"tracks/ecommerce/scenarios/progression-promotion-checkout.json","targets":["ecommerce.progression.promotion-checkout.promotion-checkout-exhausted.621c"],"desc":"Promotion application does not reject a promotion at its usage limit.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"if (!promo || promo.startMicros > now || promo.endMicros < now || promo.redemptions >= promo.usageLimit) {","replace":"if (!promo || promo.startMicros > now || promo.endMicros < now) {"}]},{"id":"customers-can-create-promotions","scenario":"tracks/ecommerce/scenarios/progression-promotion-rules.json","targets":["ecommerce.spec.access-control.promotion-management-boundary.620b"],"desc":"Promotion creation does not require staff access.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" (ctx, input) => {\n requireStaffOrAdmin(ctx);\n if (input.discountPercent <= 0 || input.discountPercent > 100) {","replace":" (ctx, input) => {\n if (input.discountPercent <= 0 || input.discountPercent > 100) {"}]},{"id":"promotion-rule-stores-the-wrong-discount","scenario":"tracks/ecommerce/scenarios/progression-promotion-rules.json","targets":["ecommerce.progression.promotion-rules.promotion-rule-values.620a"],"desc":"Promotion creation stores a one-percent discount instead of the submitted value.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.promotion.insert({ id: 0n, ...input, code: input.code.trim(), redemptions: 0 });","replace":"ctx.db.promotion.insert({ id: 0n, ...input, discountPercent: 1, code: input.code.trim(), redemptions: 0 });"}]},{"id":"staff-cannot-open-staff-tools","scenario":"tracks/ecommerce/scenarios/progression-staff-access.json","targets":["ecommerce.progression.staff-access.staff-access.601a"],"desc":"Deny administrators entry to staff tools while keeping ordinary staff access working.","file":"client/src/App.tsx","edits":[{"find":"{(isStaff || isAdmin) && (\n <button\n type=\"button\"\n className=\"btn btn-ghost btn-sm\"\n data-role=\"staff-link\"","replace":"{(isStaff && !isAdmin) && (\n <button\n type=\"button\"\n className=\"btn btn-ghost btn-sm\"\n data-role=\"staff-link\""}]},{"id":"customers-can-open-staff-tools","scenario":"tracks/ecommerce/scenarios/progression-staff-access.json","targets":["ecommerce.spec.access-control.staff-area-boundary.601b"],"desc":"Expose the protected staff area to signed-in customers, including its navigation and content.","file":"client/src/App.tsx","edits":[{"find":" const isStaff = currentUser?.isStaff ?? false;","replace":" const isStaff = isSignedIn;"}]},{"id":"administrator-role-assignment-is-discarded","scenario":"tracks/ecommerce/scenarios/progression-staff-roles.json","targets":["ecommerce.spec.state-durability.staff-role-reload.621a"],"desc":"Inventory role assignments are not saved; staff and administrator role transitions still work.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"if (existing) ctx.db.staffRole.accountId.update(row);\n else ctx.db.staffRole.insert(row);","replace":"if (role !== 'inventory') {\n if (existing) ctx.db.staffRole.accountId.update(row);\n else ctx.db.staffRole.insert(row);\n }"}]},{"id":"staff-can-assign-roles","scenario":"tracks/ecommerce/scenarios/progression-staff-roles.json","targets":["ecommerce.spec.access-control.staff-role-boundary.621b","ecommerce.spec.access-control.staff-role-revocation.621d"],"desc":"Role assignment accepts staff callers instead of requiring an administrator, including after administrator access is revoked.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"const actor = requireAdmin(ctx);\n const target = ctx.db.account.id.find(accountId);","replace":"const actor = requireStaffOrAdmin(ctx);\n const target = ctx.db.account.id.find(accountId);"}]},{"id":"stock-alert-delivery-is-suppressed","scenario":"tracks/ecommerce/scenarios/progression-stock-alert-delivery.json","targets":["ecommerce.progression.stock-alerts.stock-alert-delivery.631c"],"desc":"A request is accepted but restored stock does not deliver its alert.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"notify(ctx, alert.accountId, 'stock', `${it.name} is available again.`);","replace":"// Suppress stock-alert delivery."}]},{"id":"stock-alert-is-sent-after-every-restock","scenario":"tracks/ecommerce/scenarios/progression-stock-alerts.json","targets":["ecommerce.spec.transactional-integrity.stock-alert-deduplication.631a"],"desc":"A fulfilled stock alert remains active and is sent again after another restock.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.stockAlert.id.update({ ...alert, fulfilled: true });\n }\n }\n);\n\nexport const adminChangePrice","replace":"ctx.db.stockAlert.id.update({ ...alert, fulfilled: false });\n }\n }\n);\n\nexport const adminChangePrice"}]},{"id":"stock-alerts-are-visible-to-other-customers","scenario":"tracks/ecommerce/scenarios/progression-stock-alerts.json","targets":["ecommerce.spec.access-control.stock-alert-privacy.631b"],"desc":"The notifications view returns alerts for every account.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"return [...ctx.db.notification.accountId.filter(accountId)].map(row => ({","replace":"return [...ctx.db.notification.iter()].map(row => ({"}]},{"id":"support-history-is-lost-on-fresh-account-login","scenario":"tracks/ecommerce/scenarios/progression-support-history.json","targets":["ecommerce.spec.state-durability.support-history-reload.612a"],"desc":"Resolve customer support ownership by transport identity only, omitting the account ownership path. Initial submission, same-identity reload, and stored tickets remain intact; fresh account login cannot recover the history.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"!!actor && (actor.isAdmin || actor.isStaff || row.accountId === accountId))","replace":"!!actor && (actor.isAdmin || actor.isStaff || row.creatorIdentity.toHexString() === sender))"}]},{"id":"support-history-leaks-across-customers","scenario":"tracks/ecommerce/scenarios/progression-support-history.json","targets":["ecommerce.spec.access-control.support-history-privacy.612b","ecommerce.spec.access-control.support-history-logout.612d"],"desc":"Return all support tickets, exposing them to other customers and signed-out visitors.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":".filter(row => isGuestTicketCreator(sender, row.creatorIdentity.toHexString(), row.accountId) ||\n !!actor && (actor.isAdmin || actor.isStaff || row.accountId === accountId))","replace":".filter(() => true)"}]},{"id":"visitor-support-reference-is-hidden","scenario":"tracks/ecommerce/scenarios/progression-support-intake.json","targets":["ecommerce.progression.support-intake.support-intake.610a"],"desc":"A visitor can create a support ticket but the returned reference is not rendered.","file":"client/src/components/ProgressionWorkbench.tsx","edits":[{"find":"<div data-role=\"support-reference\">{supportReference}</div>","replace":"<div data-role=\"support-reference\"></div>"}]},{"id":"support-assignment-is-discarded","scenario":"tracks/ecommerce/scenarios/progression-support-triage.json","targets":["ecommerce.progression.support-triage.support-assignment.611a"],"desc":"Support triage saves status and priority but discards the assignee.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.supportTicket.id.update({ ...ticket, assigneeId: nextAssigneeId, priority, status });","replace":"ctx.db.supportTicket.id.update({ ...ticket, assigneeId: undefined, priority, status });"}]},{"id":"support-priority-is-discarded","scenario":"tracks/ecommerce/scenarios/progression-support-triage.json","targets":["ecommerce.progression.support-triage.support-priority.611b"],"desc":"Support triage always saves normal priority instead of the submitted value.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.supportTicket.id.update({ ...ticket, assigneeId: nextAssigneeId, priority, status });","replace":"ctx.db.supportTicket.id.update({ ...ticket, assigneeId: nextAssigneeId, priority: 'normal', status });"}]},{"id":"support-status-is-discarded","scenario":"tracks/ecommerce/scenarios/progression-support-triage.json","targets":["ecommerce.progression.support-triage.support-status.611c"],"desc":"Support triage preserves the old status instead of the submitted value.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"ctx.db.supportTicket.id.update({ ...ticket, assigneeId: nextAssigneeId, priority, status });","replace":"ctx.db.supportTicket.id.update({ ...ticket, assigneeId: nextAssigneeId, priority, status: ticket.status });"}]},{"id":"nonpositive-cart-quantity-is-treated-as-removal","scenario":"tracks/ecommerce/scenarios/01-cart-boundary.json","targets":["ecommerce.spec.access-control.cart-boundary.109b"],"desc":"Accept a negative quantity and remove the cart line instead of refusing the request.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" if (quantity < 1) throw new SenderError('Quantity must be at least 1.');\n const existing = findCartLine(ctx, acc.id, itemId);\n if (!existing) throw new SenderError('That item is not in your cart.');\n if (existing.bundlePrice > 0) throw new SenderError('Remove and re-add a whole bundle.');\n replaceReservation(ctx, acc.id, itemId, quantity);\n ctx.db.cartItem.id.update({ ...existing, quantity });","replace":" const existing = findCartLine(ctx, acc.id, itemId);\n if (!existing) throw new SenderError('That item is not in your cart.');\n if (quantity < 1) {\n ctx.db.cartItem.id.delete(existing.id);\n return;\n }\n if (existing.bundlePrice > 0) throw new SenderError('Remove and re-add a whole bundle.');\n replaceReservation(ctx, acc.id, itemId, quantity);\n ctx.db.cartItem.id.update({ ...existing, quantity });"}]},{"id":"admin-restock-preserves-existing-stock","scenario":"tracks/ecommerce/scenarios/01-warehouse-stock-live-staff.json","targets":["ecommerce.spec.live-state.warehouse-stock.7c"],"desc":"Accept an administrator restock but write the existing quantity back unchanged.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" ctx.db.stock.insert({ ...existing, quantity: existing.quantity + quantity });","replace":" ctx.db.stock.insert({ ...existing, quantity: existing.quantity });"}]},{"id":"operator-authorization-allows-customer-price-change","scenario":"tracks/ecommerce/scenarios/02-strengthened.json","targets":["ecommerce.operations-access.operator-authorization.201b"],"desc":"The price reducer drops its administrator role gate.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" (ctx, { itemId, price }) => {\n requireAdmin(ctx);","replace":" (ctx, { itemId, price }) => {\n // mutant: no administrator role check"}]},{"id":"fulfilment-queue-allows-customer-shipping","scenario":"tracks/ecommerce/scenarios/02-self-contained.json","targets":["ecommerce.operations-access.fulfilment-queue.1e"],"desc":"The shipping reducer drops the staff role check while retaining the pending-order guard.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n requireStaffOrAdmin(ctx);","replace":"export const shipOrder = spacetimedb.reducer({ orderId: t.u64() }, (ctx, { orderId }) => {\n // mutant: no staff role check"}]},{"id":"catalog-search-keeps-pre-change-price","scenario":"tracks/ecommerce/scenarios/02-live-price.json","targets":["ecommerce.returns-pricing.price-history.4b"],"desc":"The search result renderer caches each item's first visible price and ignores later live price updates.","file":"client/src/components/ItemCard.tsx","edits":[{"find":"import { ItemRow } from '../types';","replace":"import { useRef } from 'react';\nimport { ItemRow } from '../types';"},{"find":" const outOfStock = stock <= 0;\n const lowStock = !outOfStock && stock <= 5;","replace":" const outOfStock = stock <= 0;\n const lowStock = !outOfStock && stock <= 5;\n // mutant: the card retains the first price it renders\n const firstPrice = useRef(item.price);"},{"find":" {formatMoney(item.price)}","replace":" {formatMoney(firstPrice.current)}"}]},{"id":"open-cart-keeps-pre-change-price","scenario":"tracks/ecommerce/scenarios/progression-price-cart-checkout.json","targets":["ecommerce.returns-pricing.price-history.4c"],"desc":"The open-cart memo ignores reactive item-table price updates while checkout still reads the current server price.","file":"client/src/App.tsx","edits":[{"find":" [cartRows, items, stockByItem]","replace":" [cartRows, stockByItem]"}]},{"id":"catalog-price-rewrites-receipts","scenario":"tracks/ecommerce/scenarios/02-paid-price-history.json","targets":["ecommerce.returns-pricing.price-history.4a"],"desc":"Changing a catalog price cascades into saved order lines and recomputes historical order totals inside the reducer transaction.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" ctx.db.item.id.update({ ...it, price });","replace":" ctx.db.item.id.update({ ...it, price });\n const repricedOrderIds = new Set<bigint>();\n for (const line of ctx.db.orderItem.iter()) {\n if (line.itemId !== itemId) continue;\n ctx.db.orderItem.id.update({ ...line, unitPrice: price });\n repricedOrderIds.add(line.orderId);\n }\n for (const orderId of repricedOrderIds) {\n const historical = ctx.db.customerOrder.id.find(orderId);\n if (!historical) continue;\n let total = 0;\n for (const line of ctx.db.orderItem.orderId.filter(orderId)) total += line.unitPrice * line.quantity;\n ctx.db.customerOrder.id.update({ ...historical, total });\n }"}]},{"id":"catalog-price-rewrites-earned-revenue","scenario":"tracks/ecommerce/scenarios/02-invariants.json","targets":["ecommerce.returns-pricing.refund-accounting.203b"],"desc":"Changing a catalog price cascades into saved order lines and recomputes historical order totals and revenue inside the reducer transaction.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" ctx.db.item.id.update({ ...it, price });","replace":" ctx.db.item.id.update({ ...it, price });\n const repricedOrderIds = new Set<bigint>();\n for (const line of ctx.db.orderItem.iter()) {\n if (line.itemId !== itemId) continue;\n ctx.db.orderItem.id.update({ ...line, unitPrice: price });\n repricedOrderIds.add(line.orderId);\n }\n for (const orderId of repricedOrderIds) {\n const historical = ctx.db.customerOrder.id.find(orderId);\n if (!historical) continue;\n let total = 0;\n for (const line of ctx.db.orderItem.orderId.filter(orderId)) total += line.unitPrice * line.quantity;\n ctx.db.customerOrder.id.update({ ...historical, total });\n }"}]},{"id":"returned-line-marker-omitted","scenario":"tracks/ecommerce/scenarios/02-strengthened.json","targets":["ecommerce.returns-pricing.cancellation-and-return.3c"],"desc":"A returned order line keeps its persisted returned state, restored stock, and adjusted revenue but omits the visible returned marker.","file":"client/src/components/OrdersPanel.tsx","edits":[{"find":"{item.returned && <span className=\"badge badge-muted\" style={{ marginLeft: 6 }}>Returned</span>}","replace":"{false && <span className=\"badge badge-muted\" style={{ marginLeft: 6 }}>Returned</span>}"}]},{"id":"direct-review-access-is-not-checked","scenario":"tracks/ecommerce/scenarios/progression-review-access.json","targets":["ecommerce.progression.review-access-specifications.review-eligibility-direct.618a"],"desc":"The direct review action accepts a review from a customer who did not buy the item.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" if (!bought) throw new SenderError('You can only review items you have purchased.');","replace":" // mutant: purchase eligibility is not checked"}]},{"id":"support-history-rows-are-hidden","scenario":"tracks/ecommerce/scenarios/progression-support-history.json","targets":["ecommerce.progression.support-history.support-history.612c","ecommerce.spec.state-durability.support-history-reload.612a","ecommerce.spec.access-control.support-history-privacy.612b","ecommerce.spec.access-control.support-history-logout.612d"],"desc":"Hide submitted support ticket rows while leaving the submission reference available. History, reload, privacy, and logout require the owner to see the ticket first. This breaks those positive observations; it does not create a privacy leak or remove stored data.","file":"client/src/components/ProgressionWorkbench.tsx","edits":[{"find":"data-role=\"support-ticket\"","replace":"data-role=\"support-ticket\" style={{ display: \"none\" }}"}]},{"id":"authorized-restock-does-not-change-stock","scenario":"tracks/ecommerce/scenarios/01-admin-write-staff.json","targets":["ecommerce.feature.warehouse-admin.admin-write.103a"],"desc":"Accept an administrator restock without changing warehouse stock.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" if (existing) {\n ctx.db.stock.by_item_warehouse.delete([itemId, warehouseId]);\n ctx.db.stock.insert({ ...existing, quantity: existing.quantity + quantity });\n } else {\n ctx.db.stock.insert({ item_id: itemId, warehouse_id: warehouseId, quantity });\n }\n","replace":" // mutant: accept restock without changing stock\n"}]},{"id":"low-stock-threshold-is-two-units","scenario":"tracks/ecommerce/scenarios/02-low-stock.json","targets":["ecommerce.inventory-operations.operational-views.5e","ecommerce.spec.live-state.inventory-dashboard.5a"],"desc":"The dashboard lists only items with two units or fewer, so the seeded three-unit item is missing from the low-stock view. The live check in the same scenario opens with the identical observation and necessarily fails with it.","file":"client/src/App.tsx","edits":[{"find":"const LOW_STOCK_THRESHOLD = 10;","replace":"const LOW_STOCK_THRESHOLD = 2;"}]},{"id":"category-totals-count-only-since-the-dashboard-opened","scenario":"tracks/ecommerce/scenarios/02-operational-category-totals.json","targets":["ecommerce.inventory-operations.operational-views.5f"],"desc":"The category table shows units and revenue accumulated since the dashboard was opened instead of the stored totals, so a reload resets both to zero and the totals recorded before the reload are not reproduced. Live movement within one open dashboard is still correct, so the live check is unaffected.","file":"client/src/components/AdminPanel.tsx","edits":[{"find":"import { useState } from 'react';","replace":"import { useRef, useState } from 'react';"},{"find":" return (\n <div className=\"admin-panel\" data-role=\"admin-panel\">","replace":" const openingTotals = useRef<Map<bigint, { units: number; revenue: number }> | null>(null);\n if (openingTotals.current === null && categoryTotals.length > 0) {\n openingTotals.current = new Map(\n categoryTotals.map((cat): [bigint, { units: number; revenue: number }] => [\n cat.categoryId,\n { units: cat.unitsSold, revenue: cat.revenue },\n ])\n );\n }\n const sessionTotals = categoryTotals.map((cat) => {\n const opening = openingTotals.current?.get(cat.categoryId);\n return {\n ...cat,\n unitsSold: cat.unitsSold - (opening?.units ?? 0),\n revenue: cat.revenue - (opening?.revenue ?? 0),\n };\n });\n\n return (\n <div className=\"admin-panel\" data-role=\"admin-panel\">"},{"find":" {categoryTotals.map((cat) => (","replace":" {sessionTotals.map((cat) => ("}]},{"id":"profile-summary-ignores-a-profile-saved-this-session","scenario":"tracks/ecommerce/scenarios/progression-customer-profile.json","targets":["ecommerce.progression.customer-profile.customer-profile.620c"],"desc":"Hide the profile summary immediately after saving in the current view. Stored profile data and a reopened or reloaded view remain correct, so fresh-login durability and privacy positive controls remain observable.","file":"client/src/components/ProgressionWorkbench.tsx","edits":[{"find":" const [profileName, setProfileName] = useState(profile?.name ?? '');","replace":" const [profileName, setProfileName] = useState(profile?.name ?? '');\n const [profileSavedHere, setProfileSavedHere] = useState(false);"},{"find":"onClick={() => reducers?.saveProfile({ name: profileName, address: profileAddress })}","replace":"onClick={() => { setProfileSavedHere(true); return reducers?.saveProfile({ name: profileName, address: profileAddress }); }}"},{"find":"<div data-role=\"profile-address-summary\">{profile?.name} {profile?.address}</div>","replace":"<div data-role=\"profile-address-summary\">{!profileSavedHere && <>{profile?.name} {profile?.address}</>}</div>"}]},{"id":"stored-support-replies-are-hidden-after-reload","scenario":"tracks/ecommerce/scenarios/progression-managed-support-shared.json","targets":["ecommerce.progression.managed-support.managed-support.613c"],"desc":"Replies already stored when the page loads are hidden and only replies that arrive while the page is open are shown, so a reloaded customer or staff member cannot see the earlier exchange. The live shared-case check exchanges only new replies and is unaffected.","file":"client/src/App.tsx","edits":[{"find":"import { useEffect, useMemo, useState } from 'react';","replace":"import { useEffect, useMemo, useRef, useState } from 'react';"},{"find":" const [supportReplyRows] = useTable(tables.visibleSupportReplies);","replace":" const [liveSupportReplyRows, supportRepliesReady] = useTable(tables.visibleSupportReplies);\n const openedSupportReplies = useRef<Set<string> | null>(null);\n if (openedSupportReplies.current === null && supportRepliesReady) {\n openedSupportReplies.current = new Set(\n liveSupportReplyRows.map((row) => `${row.ticketId}:${row.author}:${row.body}`)\n );\n }\n const supportReplyRows = liveSupportReplyRows.filter(\n (row) => !openedSupportReplies.current?.has(`${row.ticketId}:${row.author}:${row.body}`)\n );"}]},{"id":"saving-notification-preferences-resets-the-toggles","scenario":"tracks/ecommerce/scenarios/progression-notification-preferences.json","targets":["ecommerce.progression.notification-preferences.notification-preferences.630c"],"desc":"Saving sends the chosen preferences but resets both toggles to off and stops the form from following the stored row for the rest of the session, so the saved choice cannot be seen until a reload. The reload and cross-account checks read the stored row on a fresh page and are unaffected.","file":"client/src/components/ProgressionWorkbench.tsx","edits":[{"find":" useEffect(() => {\n setOrderEnabled(preferences?.orderEnabled ?? false);\n setStockEnabled(preferences?.stockEnabled ?? false);\n }, [preferences?.orderEnabled, preferences?.stockEnabled]);","replace":" const [preferencesSubmitted, setPreferencesSubmitted] = useState(false);\n useEffect(() => {\n if (preferencesSubmitted) return;\n setOrderEnabled(preferences?.orderEnabled ?? false);\n setStockEnabled(preferences?.stockEnabled ?? false);\n }, [preferences?.orderEnabled, preferences?.stockEnabled, preferencesSubmitted]);"},{"find":"onClick={() => reducers?.saveNotificationPreferences({ orderEnabled, stockEnabled })}","replace":"onClick={() => { reducers?.saveNotificationPreferences({ orderEnabled, stockEnabled }); setPreferencesSubmitted(true); setOrderEnabled(false); setStockEnabled(false); }}"}]},{"id":"saving-a-staff-role-snaps-the-input-back-to-the-stored-role","scenario":"tracks/ecommerce/scenarios/progression-staff-roles.json","targets":["ecommerce.progression.staff-roles.staff-roles.621c"],"desc":"Saving a role sends the new role to the server but snaps the visible input back to the role stored before the save for the rest of the session, so the administrator cannot see the assignment take. A reload renders the stored role, so the durability and boundary checks are unaffected.","file":"client/src/components/ProgressionWorkbench.tsx","edits":[{"find":" <button className=\"btn btn-ghost btn-sm\" data-role=\"staff-role-save\" onClick={() => reducers?.assignStaffRole({\n accountId: row.accountId,\n role: (document.getElementById(`role-${String(row.accountId)}`) as HTMLSelectElement)?.value ?? 'warehouse',\n })}>Save role</button>","replace":" <button className=\"btn btn-ghost btn-sm\" data-role=\"staff-role-save\" onClick={() => {\n const roleInput = document.getElementById(`role-${String(row.accountId)}`) as HTMLInputElement | null;\n reducers?.assignStaffRole({ accountId: row.accountId, role: roleInput?.value ?? 'warehouse' });\n if (roleInput) roleInput.value = row.role;\n }}>Save role</button>"}]},{"id":"fulfilment-queue-is-frozen-at-page-load","scenario":"tracks/ecommerce/scenarios/02-fulfilment-live.json","targets":["ecommerce.spec.live-state.fulfilment-queue.1a"],"desc":"The fulfilment queue renders the rows delivered with the page's initial subscription and ignores later updates, so an order placed while the queue is open never appears without a reload.","file":"client/src/App.tsx","edits":[{"find":"import { useEffect, useMemo, useState } from 'react';","replace":"import { useEffect, useMemo, useRef, useState } from 'react';"},{"find":" const [queueRows] = useTable(tables.fulfilmentQueue);","replace":" const [liveQueueRows, queueReady] = useTable(tables.fulfilmentQueue);\n const openedQueueRows = useRef<typeof liveQueueRows | null>(null);\n if (openedQueueRows.current === null && queueReady) openedQueueRows.current = liveQueueRows;\n const queueRows = openedQueueRows.current ?? liveQueueRows;"}]},{"id":"low-stock-list-is-frozen-at-page-load","scenario":"tracks/ecommerce/scenarios/02-low-stock.json","targets":["ecommerce.spec.live-state.inventory-dashboard.5a"],"desc":"The low-stock list is computed once from the first complete stock snapshot and never recomputed, so items no longer enter or leave it as stock is restocked or sold. The seeded low item is in that first snapshot, so the static listing check is unaffected.","file":"client/src/App.tsx","edits":[{"find":"import { useEffect, useMemo, useState } from 'react';","replace":"import { useEffect, useMemo, useRef, useState } from 'react';"},{"find":" const lowStockItems = useMemo(\n () =>\n [...items]\n .filter((i) => (stockByItem.get(i.id) ?? 0) <= LOW_STOCK_THRESHOLD)\n .sort((a, b) => (stockByItem.get(a.id) ?? 0) - (stockByItem.get(b.id) ?? 0)),\n [items, stockByItem]\n );","replace":" const liveLowStockItems = useMemo(\n () =>\n [...items]\n .filter((i) => (stockByItem.get(i.id) ?? 0) <= LOW_STOCK_THRESHOLD)\n .sort((a, b) => (stockByItem.get(a.id) ?? 0) - (stockByItem.get(b.id) ?? 0)),\n [items, stockByItem]\n );\n const openedLowStockItems = useRef<typeof liveLowStockItems | null>(null);\n if (openedLowStockItems.current === null && items.length > 0 && stocks.length > 0) {\n openedLowStockItems.current = liveLowStockItems;\n }\n const lowStockItems = openedLowStockItems.current ?? liveLowStockItems;"}]},{"id":"category-totals-are-frozen-at-page-load","scenario":"tracks/ecommerce/scenarios/02-operational-category-totals.json","targets":["ecommerce.spec.live-state.sales-dashboard.5b"],"desc":"The category totals render the first non-empty row set received and ignore later updates, so a purchase does not move units or revenue while the dashboard is open. A reload receives the stored totals, so the reload check is unaffected.","file":"client/src/App.tsx","edits":[{"find":"import { useEffect, useMemo, useState } from 'react';","replace":"import { useEffect, useMemo, useRef, useState } from 'react';"},{"find":" const [categoryTotalRows] = useTable(tables.categoryTotals);","replace":" const [liveCategoryTotalRows] = useTable(tables.categoryTotals);\n const openedCategoryTotalRows = useRef<typeof liveCategoryTotalRows | null>(null);\n if (openedCategoryTotalRows.current === null && liveCategoryTotalRows.length > 0) {\n openedCategoryTotalRows.current = liveCategoryTotalRows;\n }\n const categoryTotalRows = openedCategoryTotalRows.current ?? liveCategoryTotalRows;"}]},{"id":"warehouse-totals-are-frozen-at-page-load","scenario":"tracks/ecommerce/scenarios/02-transfer-totals.json","targets":["ecommerce.spec.live-state.stock-transfers.2b"],"desc":"The per-warehouse totals are computed once from the first stock snapshot and never recomputed, so a transfer moves neither warehouse figure while the dashboard is open.","file":"client/src/App.tsx","edits":[{"find":"import { useEffect, useMemo, useState } from 'react';","replace":"import { useEffect, useMemo, useRef, useState } from 'react';"},{"find":" const stockByWarehouse = useMemo(() => {\n const map = new Map<bigint, number>();\n for (const row of stocks) {\n map.set(row.warehouseId, (map.get(row.warehouseId) ?? 0) + row.quantity);\n }\n return map;\n }, [stocks]);","replace":" const liveStockByWarehouse = useMemo(() => {\n const map = new Map<bigint, number>();\n for (const row of stocks) {\n map.set(row.warehouseId, (map.get(row.warehouseId) ?? 0) + row.quantity);\n }\n return map;\n }, [stocks]);\n const openedStockByWarehouse = useRef<typeof liveStockByWarehouse | null>(null);\n if (openedStockByWarehouse.current === null && liveStockByWarehouse.size > 0) {\n openedStockByWarehouse.current = liveStockByWarehouse;\n }\n const stockByWarehouse = openedStockByWarehouse.current ?? liveStockByWarehouse;"}]},{"id":"transfer-skips-the-source-holding-check","scenario":"tracks/ecommerce/scenarios/02-transfer-overdraw.json","targets":["ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c"],"desc":"The serialized transfer reducer no longer checks that the source warehouse holds the requested quantity, so an overdraw is accepted instead of refused and the source quantity wraps below zero.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" if (available < quantity) {\n throw new SenderError(`Not enough stock in source warehouse: only ${available} available.`);\n }","replace":" // mutant: the source warehouse holding is not checked"}]},{"id":"credit-checkout-ignores-wallet","desc":"A credit checkout pays entirely externally despite available credit.","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.feature.store-credit.store-credit-750.750a"],"file":"backend/spacetimedb/src/index.ts","edits":[{"find":" const creditMinor = useCredit ? Math.min(wallet?.amountMinor ?? 0,totalMinor) : 0;","replace":" const creditMinor = 0;"}]},{"id":"credit-grant-replay-increments-balance","desc":"Replaying a grant applies its credit to the wallet again.","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.spec.store-credit.production-752.752a"],"file":"backend/spacetimedb/src/index.ts","edits":[{"find":" if (existing.amountMinor !== amountMinor) throw new SenderError('Reference identifies another grant.');\n return;","replace":" if (existing.amountMinor !== amountMinor) throw new SenderError('Reference identifies another grant.');\n const wallet = ctx.db.creditWallet.accountId.find(accountId)!;\n ctx.db.creditWallet.accountId.update({ ...wallet, amountMinor: wallet.amountMinor + amountMinor });\n return;"}]},{"id":"customer-can-grant-credit","desc":"Customer authentication is accepted without staff authorization.","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.spec.store-credit.production-753.753a"],"file":"backend/spacetimedb/src/index.ts","edits":[{"find":" (ctx, { accountId, amountMinor, reference }) => {\n requireStaffOrAdmin(ctx);","replace":" (ctx, { accountId, amountMinor, reference }) => {\n requireAccount(ctx);"}]},{"id":"split-refund-does-not-restore-credit","desc":"The refund is recorded but its original wallet credit is not restored.","scenario":"tracks/ecommerce/scenarios/progression-split-tender-refunds.json","targets":["ecommerce.feature.split-tender-refunds.split-tender-refunds-751.751a"],"file":"backend/spacetimedb/src/index.ts","edits":[{"find":" refundOrderCredit(ctx, order, order.total);","replace":" // mutant: omit wallet restoration"}]},{"id":"split-refund-duplicates-credit","desc":"A refund credits the wallet twice while recording one refund.","scenario":"tracks/ecommerce/scenarios/progression-split-tender-refunds.json","targets":["ecommerce.spec.split-tender-refunds.production-756.756a"],"file":"backend/spacetimedb/src/index.ts","edits":[{"find":"amountMinor: wallet.amountMinor + delta","replace":"amountMinor: wallet.amountMinor + delta * 2"}]},{"id":"subscription-skips-due-purchase","desc":"Due deliveries are recorded as skipped although stock is available.","scenario":"tracks/ecommerce/scenarios/progression-subscriptions.json","targets":["ecommerce.feature.subscriptions.subscriptions-760.760a"],"file":"backend/spacetimedb/src/subscriptions.ts","edits":[{"find":" const orderId = purchase(row.accountId, row.itemId, row.quantity, row.price);","replace":" const orderId: bigint | null = null;"}]},{"id":"subscription-allows-foreign-cancellation","desc":"A customer can cancel another customer subscription.","scenario":"tracks/ecommerce/scenarios/progression-subscriptions.json","targets":["ecommerce.spec.subscriptions.production-762.762a"],"file":"backend/spacetimedb/src/subscriptions.ts","edits":[{"find":" if (!row || row.accountId !== accountId) throw new SenderError('Subscription access denied.');","replace":" if (!row) throw new SenderError('Subscription access denied.');"}]},{"id":"subscription-pause-is-not-recorded","desc":"Pause acknowledges the request but the subscription remains active.","scenario":"tracks/ecommerce/scenarios/progression-subscriptions.json","targets":["ecommerce.spec.subscriptions.production-763.763a"],"file":"backend/spacetimedb/src/subscriptions.ts","edits":[{"find":" ctx.db.purchaseSubscription.id.update({ ...row, status: 'paused', pausedMicros: now });","replace":" ctx.db.purchaseSubscription.id.update({ ...row, status: 'active', pausedMicros: now });"}]},{"id":"credit-checkout-retains-purchased-cart","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.spec.store-credit.production-754.754a"],"desc":"The purchased cart remains available instead of being consumed by checkout.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" for (const line of lines) ctx.db.cartItem.id.delete(line.id);","replace":" // mutant: checked-out lines remain in the cart"}]},{"id":"reconnection-erases-stored-credit","desc":"A new connection erases stored wallet credit; the fresh view after restart must catch the data loss.","file":"backend/spacetimedb/src/index.ts","scenario":"tracks/ecommerce/scenarios/progression-store-credit.json","targets":["ecommerce.spec.store-credit.production-755.755a"],"edits":[{"find":"export const onConnect = spacetimedb.clientConnected((_ctx) => {});","replace":"export const onConnect = spacetimedb.clientConnected(ctx => { for (const row of ctx.db.creditWallet.iter()) ctx.db.creditWallet.accountId.update({ ...row, amountMinor: 0 }); });"}]},{"id":"reconnection-cancels-pending-subscriptions","desc":"A new connection cancels pending subscriptions; restarting and reconnecting must preserve this work.","file":"backend/spacetimedb/src/index.ts","scenario":"tracks/ecommerce/scenarios/progression-subscriptions.json","targets":["ecommerce.spec.subscriptions.production-761.761a"],"edits":[{"find":"export const onConnect = spacetimedb.clientConnected((_ctx) => {});","replace":"export const onConnect = spacetimedb.clientConnected(ctx => { for (const row of ctx.db.purchaseSubscription.iter()) if (row.status === 'active') ctx.db.purchaseSubscription.id.update({ ...row, status: 'cancelled' }); });"}]},{"id":"bundle-definition-loses-component-quantity","scenario":"tracks/ecommerce/scenarios/progression-product-bundles.json","targets":["ecommerce.feature.product-bundles.product-bundles.740a"],"desc":"definition loses component quantity","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"return { ...component, itemId: String(product.id) };","replace":"return { ...component, quantity: 1, itemId: String(product.id) };"}]},{"id":"bundle-catalog-write-allows-customers","scenario":"tracks/ecommerce/scenarios/progression-product-bundles.json","targets":["ecommerce.spec.bundle-integrity.bundle-743.743a"],"desc":"catalog write allows customers","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"const actor = requireStaffOrAdmin(ctx);\n if (!actor.isAdmin && ctx.db.staffRole.accountId.find(actor.id)?.role !== 'catalog') {","replace":"const actor = requireAccount(ctx);\n if (false) {"}]},{"id":"bundle-checkout-price-not-snapshot","scenario":"tracks/ecommerce/scenarios/progression-bundle-checkout.json","targets":["ecommerce.feature.bundle-checkout.bundle-checkout.741a"],"desc":"checkout price not snapshot","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"bundlePrice: product.price, bundleComponentsJson: definition.componentsJson","replace":"bundlePrice: product.price + 1, bundleComponentsJson: definition.componentsJson"}]},{"id":"bundle-return-loses-original-components","scenario":"tracks/ecommerce/scenarios/progression-bundle-returns.json","targets":["ecommerce.feature.bundle-returns.bundle-returns.742a"],"desc":"return loses original components","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"for (const line of bundles) {\n restoreOrderItemStock(ctx, line);","replace":"for (const line of bundles) {\n // mutant: original stock is not restored"}]},{"id":"bundle-return-replay-restocks-again","scenario":"tracks/ecommerce/scenarios/progression-bundle-returns.json","targets":["ecommerce.spec.bundle-integrity.bundle-742.742b"],"desc":"return replay restocks again","file":"backend/spacetimedb/src/index.ts","edits":[{"find":".filter(row => row.isBundle && !row.returned);","replace":".filter(row => row.isBundle);"}]},{"id":"bundle-return-crosses-account-boundary","scenario":"tracks/ecommerce/scenarios/progression-bundle-returns.json","targets":["ecommerce.spec.bundle-integrity.bundle-748.748a"],"desc":"return crosses account boundary","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"if (!order || order.accountId !== account.id || !['shipped', 'delivered'].includes(order.status))","replace":"if (!order || !['shipped', 'delivered'].includes(order.status))"}]},{"id":"bundle-components-can-overdraw","scenario":"tracks/ecommerce/scenarios/progression-bundle-checkout.json","targets":["ecommerce.spec.bundle-integrity.bundle-744.744a","ecommerce.spec.bundle-integrity.bundle-745.745a"],"desc":"components can overdraw","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"if (!allocations) throw new SenderError('Not enough stock to reserve.');","replace":"if (!allocations) { if (bundleId) return; throw new SenderError('Not enough stock to reserve.'); }"}]},{"id":"bundle-checkout-reuses-reservation","scenario":"tracks/ecommerce/scenarios/progression-bundle-checkout.json","targets":["ecommerce.spec.bundle-integrity.bundle-747.747a"],"desc":"checkout reuses reservation","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"for (const row of held) ctx.db.reservation.id.delete(row.id);\n processReorderRules(ctx, p.itemId);","replace":"// mutant: reservations survive checkout\n processReorderRules(ctx, p.itemId);"},{"find":"for (const line of lines) ctx.db.cartItem.id.delete(line.id);","replace":"// mutant: cart survives checkout"}]},{"id":"bundle-expiry-does-not-release-components","scenario":"tracks/ecommerce/scenarios/progression-bundle-checkout.json","targets":["ecommerce.spec.bundle-integrity.bundle-746.746a"],"desc":"Expired bundle reservations retain their component stock after restart.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"if (row.expired || row.expiresMicros > now) continue;\n restoreStock(ctx, row.stockItemId || row.itemId, row.warehouseId, row.quantity);","replace":"if (row.expired || row.expiresMicros > now) continue;\n if (!row.stockItemId) restoreStock(ctx, row.itemId, row.warehouseId, row.quantity);"}]},{"id":"return-after-support-refund-is-blocked","scenario":"tracks/ecommerce/scenarios/progression-support-return-interaction.json","targets":["ecommerce.feature.split-tender-refunds.return-refund-interaction.757a"],"desc":"Reject a valid physical return after a financial refund.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" if (target.returned) throw new SenderError('Item already returned.');","replace":" if (target.returned || order.refundedTotal > 0) throw new SenderError('Item already returned.');"}]},{"id":"support-refund-after-return-pays-twice","scenario":"tracks/ecommerce/scenarios/progression-support-return-interaction.json","targets":["ecommerce.feature.split-tender-refunds.return-refund-interaction.757b"],"desc":"Pay the full order again after the physical return already refunded it.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":" const amount = order.total - order.refundedTotal;","replace":" const amount = order.total;"},{"find":"refundedTotal: order.total, status: order.status","replace":"refundedTotal: order.refundedTotal + amount, status: order.status"}]},{"id":"support-history-leaks-to-signed-out-visitors","scenario":"tracks/ecommerce/scenarios/progression-support-history.json","targets":["ecommerce.spec.access-control.support-history-logout.612d"],"desc":"Disclose support tickets to signed-out visitors while leaving signed-in ownership filtering unchanged.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":".filter(row => isGuestTicketCreator(sender, row.creatorIdentity.toHexString(), row.accountId) ||","replace":".filter(row => !actor || isGuestTicketCreator(sender, row.creatorIdentity.toHexString(), row.accountId) ||"}]},{"id":"checkout-crash-integrity","scenario":"tracks/ecommerce/scenarios/progression-checkout-crash.json","targets":["ecommerce.spec.state-durability.checkout-crash-integrity.910a"],"desc":"A lifecycle callback changes a prepared or cleared cart to quantity two, leaving neither a legal rollback nor a complete checkout.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"export const onDisconnect = spacetimedb.clientDisconnected((_ctx) => {});","replace":"export const onDisconnect = spacetimedb.clientDisconnected((ctx) => { for (const row of ctx.db.cartItem.iter()) if ([...ctx.db.customerOrder.iter()].some(order => order.accountId === row.accountId)) ctx.db.cartItem.id.update({ ...row, quantity: 2 }); });"}]},{"id":"checkout-crash-durability","scenario":"tracks/ecommerce/scenarios/progression-checkout-crash.json","targets":["ecommerce.spec.state-durability.checkout-crash-durability.910b"],"desc":"A lifecycle callback corrupts earlier order totals. The unchanged prepared snapshots and acknowledged baseline must expose the damage.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"export const onDisconnect = spacetimedb.clientDisconnected((_ctx) => {});","replace":"export const onDisconnect = spacetimedb.clientDisconnected((ctx) => { for (const order of ctx.db.customerOrder.iter()) if ([...ctx.db.cartItem.iter()].some(row => row.accountId === order.accountId)) ctx.db.customerOrder.id.update({ ...order, total: 0 }); });"}]},{"id":"review-script-unsafe-render","scenario":"tracks/ecommerce/scenarios/progression-review-script.json","targets":["ecommerce.progression.review-access-specifications.stored-review-script.9180a"],"desc":"Render stored review text as HTML so it executes in another customer session.","file":"client/src/components/ItemDetail.tsx","edits":[{"find":"<div>{r.comment}</div>","replace":"<div dangerouslySetInnerHTML={{ __html: r.comment }} />"}]},{"id":"review-script-reject-all","scenario":"tracks/ecommerce/scenarios/progression-review-script.json","targets":["ecommerce.progression.review-access-specifications.stored-review-script.9180a"],"desc":"Reject all nonempty reviews, including the required benign review; absence of script is insufficient.","file":"backend/spacetimedb/src/index.ts","edits":[{"find":"(ctx, { itemId, rating, comment }) => {","replace":"(ctx, { itemId, rating, comment }) => {\n if (comment.length > 0) throw new SenderError(\"Review rejected\");"}]}]}}} diff --git a/tools/stack-bench/tests/fixtures/resource-lock-process.ts b/tools/stack-bench/tests/fixtures/resource-lock-process.ts new file mode 100644 index 00000000000..5ca166dace7 --- /dev/null +++ b/tools/stack-bench/tests/fixtures/resource-lock-process.ts @@ -0,0 +1,23 @@ +import { createBackendLease, acquireResourceLocks, + resourceLockScope } from '../../src/runtime/backend-lease.js'; + +const [root, runId, mode = 'acquire'] = process.argv.slice(2); +if (!root || !runId) process.exit(2); + +try { + const lease = createBackendLease({ + runId, backend: 'stub', track: 'process-test', runIndex: 0, + }); + const scope = resourceLockScope({ + STACK_BENCH_APPLIANCE: '1', + STACK_BENCH_RESOURCE_LOCK_DIR: root, + }); + lease.resources.locks.push(...acquireResourceLocks({ + ...scope, keys: ['slot:process-test:stub:run0'], lease, + })); + process.stdout.write('acquired\n'); + if (mode === 'hold') setInterval(() => {}, 1_000); +} catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 3; +} diff --git a/tools/stack-bench/tests/fixtures/spacetime-module/package.json b/tools/stack-bench/tests/fixtures/spacetime-module/package.json new file mode 100644 index 00000000000..432e3e95b66 --- /dev/null +++ b/tools/stack-bench/tests/fixtures/spacetime-module/package.json @@ -0,0 +1,11 @@ +{ + "name": "stack-bench-container-smoke-module", + "private": true, + "type": "module", + "dependencies": { + "spacetimedb": "file:/deps/bindings-typescript" + }, + "devDependencies": { + "typescript": "5.6.3" + } +} diff --git a/tools/stack-bench/tests/fixtures/spacetime-module/src/index.ts b/tools/stack-bench/tests/fixtures/spacetime-module/src/index.ts new file mode 100644 index 00000000000..4fae5ce63cb --- /dev/null +++ b/tools/stack-bench/tests/fixtures/spacetime-module/src/index.ts @@ -0,0 +1,14 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const app = schema({ + smokeItem: table({ public: true }, { + id: t.u64().primaryKey(), + value: t.string(), + }), +}); + +export default app; + +export const seed = app.reducer({ value: t.string() }, (ctx, { value }) => { + ctx.db.smokeItem.insert({ id: 1n, value }); +}); diff --git a/tools/stack-bench/tests/fixtures/spacetime-module/tsconfig.json b/tools/stack-bench/tests/fixtures/spacetime-module/tsconfig.json new file mode 100644 index 00000000000..41a333124ba --- /dev/null +++ b/tools/stack-bench/tests/fixtures/spacetime-module/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "strict": true, + "skipLibCheck": true, + "moduleResolution": "bundler", + "target": "ESNext", + "lib": ["ES2021"], + "module": "ESNext", + "isolatedModules": true, + "noEmit": true + }, + "include": ["src/**/*.ts"] +} diff --git a/tools/stack-bench/tests/form-input.integration.ts b/tools/stack-bench/tests/form-input.integration.ts new file mode 100644 index 00000000000..0b39d5c5441 --- /dev/null +++ b/tools/stack-bench/tests/form-input.integration.ts @@ -0,0 +1,66 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import test from 'node:test'; +import { chromium } from 'playwright'; +import { ACTION_REGISTRY } from '../src/actions/action-catalog.js'; +import { executeAction } from '../src/actions/action-contract.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; + +// Failure cases, before the change: label/value disagreement, absent choices, +// disabled controls, delayed options, and wrong date normalization. The real +// browser must dispatch the normal input event; direct DOM assignment is not a fill. +test('form entry selects real choices and preserves refusal and delayed-readiness behavior', async () => { + const browser = await chromium.launch({ headless: true }); + const cases = [ + { id: 'label', html: '<select><option value="daily">Daily</option><option value="weekly">Weekly</option></select>', text: 'Weekly', value: 'weekly', status: 'passed' }, + { id: 'value', html: '<select><option value="daily">Daily</option><option value="weekly">Weekly</option></select>', text: 'weekly', value: 'weekly', status: 'passed' }, + { id: 'ambiguous-native-choice', html: '<select><option value="a">Weekly</option><option value="Weekly">Other</option></select>', text: 'Weekly', value: 'a', status: 'passed' }, + { id: 'delayed', html: '<select></select><script>setTimeout(()=>document.querySelector("select").add(new Option("Weekly","weekly")),100)</script>', text: 'weekly', value: 'weekly', status: 'passed' }, + { id: 'missing', html: '<select><option value="daily">Daily</option></select>', text: 'Weekly', status: 'failed', finding: 'choice-missing' }, + { id: 'disabled', html: '<select disabled><option value="daily">Daily</option><option value="weekly">Weekly</option></select>', text: 'Weekly', status: 'failed', finding: 'page-timeout' }, + { id: 'date', html: '<input type="date">', text: '2026-09-22T14:00', value: '2026-09-22', status: 'passed' }, + { id: 'datetime', html: '<input type="datetime-local">', text: '2026-09-22', value: '2026-09-22T00:00', status: 'passed' }, + { id: 'text', html: '<input>', text: 'Volume Item 0999', value: 'Volume Item 0999', status: 'passed' }, + ]; + const observations: unknown[] = []; + let result = 'failed'; + try { + const page = await browser.newPage(); + page.setDefaultTimeout(400); + for (const sample of cases) { + await page.setContent(sample.html); + await page.locator('select,input').evaluate(element => { + element.addEventListener('input', () => element.setAttribute('data-input-seen', 'yes')); + }); + const actor = { page, loc: () => page.locator('select,input') }; + const started = performance.now(); + const action = await executeAction(ACTION_REGISTRY, 'fill', { do: 'fill', actor: 'editor', testid: 'field', text: sample.text }, { + capabilities: { actors: { get: () => actor }, 'browser-interaction': { + defaultWithin: 400, expand: (value: string) => value, sleep: async () => {}, + } }, + }); + const ms = performance.now() - started; + const value = await page.locator('select,input').inputValue(); + observations.push({ id: sample.id, ms, status: action.status, finding: action.finding?.kind, value }); + assert.equal(action.status, sample.status, `${sample.id}: ${action.summary}`); + if (sample.finding) assert.equal(action.finding?.kind, sample.finding); + if (sample.value !== undefined) { + assert.equal(value, sample.value, sample.id); + assert.equal(await page.locator('select,input').getAttribute('data-input-seen'), 'yes'); + } + } + result = 'passed'; + } finally { + await browser.close(); + const evidence = process.env.STACK_BENCH_FORM_EVIDENCE + ?? join(STACK_BENCH_ROOT, 'local-notes', 'form-input-evidence'); + mkdirSync(evidence, { recursive: true }); + writeFileSync(join(evidence, 'receipt.json'), JSON.stringify({ result, + command: 'node --test dist/tests/form-input.integration.js', cases, observations, browserClosed: true, + executorSha256: createHash('sha256').update(readFileSync(join(STACK_BENCH_ROOT, + 'dist/src/actions/browser-action-executors.js'))).digest('hex'), + }, null, 2)); + } +}); diff --git a/tools/stack-bench/tests/grader-cleanup.test.ts b/tools/stack-bench/tests/grader-cleanup.test.ts new file mode 100644 index 00000000000..b2e804241ea --- /dev/null +++ b/tools/stack-bench/tests/grader-cleanup.test.ts @@ -0,0 +1,132 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { EventEmitter } from 'node:events'; +import type { Browser } from 'playwright'; +import { errors } from 'playwright'; +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; + +import { closeActorContexts, gradeFeature } from '../grader/grade.js'; + +test('grader context cleanup records browser failures instead of throwing away the report', async () => { + const context = { + tracing: { stop: async () => { throw new Error('trace target closed'); } }, + close: async () => { throw new Error('browser context closed unexpectedly'); }, + }; + const video = { + saveAs: async () => { throw new Error('video unavailable'); }, + delete: async () => { throw new Error('video already removed'); }, + }; + const failures = await closeActorContexts([ + { context, name: 'buyer', page: { video: () => video }, traceStarted: true }, + ], { trace: true, media: '/tmp/media', slug: 'account-create' }); + + assert.deepEqual(failures.map(failure => failure.stage), + ['trace', 'context-close', 'video-save', 'video-delete']); + assert(failures.every(failure => failure.actor === 'buyer')); +}); + +test('grader context cleanup closes a context when page creation failed', async () => { + let closed = false; + const context = { + tracing: { stop: async () => {} }, + close: async () => { closed = true; }, + }; + const failures = await closeActorContexts([ + { context, name: 'buyer', page: null }, + ], { media: '/tmp/media', slug: 'account-create' }); + assert.equal(closed, true); + assert.deepEqual(failures, []); + + // A trace that was never started is not a cleanup failure. + const untraced = { tracing: { stop: async () => { throw new Error('trace was never started'); } }, close: async () => {} }; + assert.deepEqual(await closeActorContexts([ + { context: untraced, name: 'buyer', page: { video: () => null } }, + ], { trace: true, media: '/tmp/media', slug: 'account-create' }), []); +}); + +test('navigation timeout is inconclusive; connection refusal blocks setup and proven crashes are harness failures', async () => { + const scenario = compileScenarioDefinition({ schemaVersion: 1, track: 'ecommerce', level: 1, + name: 'navigation', features: [{ id: 1, name: 'account', actors: ['buyer'], setup: [], + criteria: [{ id: '1a', desc: 'account exists', points: 1, + steps: [{ do: 'expect', actor: 'buyer', testid: 'current-user' }] }] }] }, { source: 'navigation.json' }); + // Network codes must survive evidence serialization. Raw URLs, credentials, + // headers and browser messages must not enter the diagnostic observation. + for (const [error, expected, networkError] of [ + [new errors.TimeoutError('page.goto: Timeout 20000ms exceeded'), 'inconclusive'], + [new Error('page.goto: net::ERR_CONNECTION_REFUSED'), 'failed', 'net::ERR_CONNECTION_REFUSED'], + [new Error('page.goto: net::ERR_CONNECTION_RESET at https://user:private-secret@failed.example/private-path?token=private-secret'), + 'inconclusive', 'net::ERR_CONNECTION_RESET'], + [new Error('page.reload: net::ERR_ABORTED; Authorization: Bearer private-secret'), 'inconclusive', 'net::ERR_ABORTED'], + [new Error('page.goto: net::ERR_NAME_NOT_RESOLVED'), 'inconclusive', 'net::ERR_NAME_NOT_RESOLVED'], + [new Error('page.goto: Protocol error: invalid parameters'), 'harness_failure'], + [new Error('page.goto: Target crashed'), 'harness_failure'], + ] as const) { + let closed = false; + const context = Object.assign(new EventEmitter(), { newPage: async () => page, serviceWorkers: () => [], + routeWebSocket: async () => {}, + newCDPSession: async () => ({ on() {}, async send() {} }), + close: async () => { closed = true; } }); + const page = Object.assign(new EventEmitter(), { setDefaultTimeout() {}, context: () => context, + goto: async () => { + page.emit('console', { type: () => 'error', text: () => 'password=private-secret failed' }); + const finished = { method: () => 'GET', resourceType: () => 'document', + url: () => 'https://finished.example/private?token=private-secret' }; + page.emit('request', finished); + page.emit('requestfinished', finished); + const failed = { ...finished, url: () => 'https://failed.example/private' }; + page.emit('request', failed); + page.emit('requestfailed', failed); + for (let i = 0; i < 25; i++) page.emit('request', { + method: () => 'GET', resourceType: () => 'stylesheet', + url: () => `https://user:private-secret@fonts.example/private-${i}?token=private-secret#private`, + }); + throw error; + }, video: () => null }); + const browser = { newContext: async () => context } as unknown as Browser; + const result = await gradeFeature(browser, scenario.features[0]!, { + url: 'http://app', level: 1, headed: false, selectedCheckKeys: [], nullControl: false, + }, { runId: 'navigation-test', roomName: name => name, url: 'http://app', + actions: [], spacetime: null, nullControl: false }); + assert.equal(result.setupEvidence.status, expected); + assert.equal(result.criteria[0]!.evidence.status, expected === 'failed' ? 'blocked' : expected); + assert.equal(result.criteria[0]!.evidence.phase, 'setup'); + assert.deepEqual(result.criteria[0]!.evidence.actions, []); + assert.equal(closed, true); + if (error instanceof errors.TimeoutError) { + assert.deepEqual(result.setupEvidence.observation, + { pendingResources: Array(20).fill('stylesheet https://fonts.example') }); + } + if (networkError) assert.deepEqual(JSON.parse(JSON.stringify(result.setupEvidence)).observation, + { networkError }, 'The saved observation must retain only the safe Chromium network code'); + assert.match(result.consoleErrors.join('\n'), /redacted credential/); + assert.doesNotMatch(JSON.stringify([result.consoleErrors, result.setupEvidence.observation]), + /private-secret|private-|finished.example|failed.example/); + assert.equal(page.listenerCount('requestfinished'), 0); + assert.equal(page.listenerCount('requestfailed'), 0); + assert.equal(page.listenerCount('request'), 1); // Only the existing write recorder remains. + } +}); + + +test('a cancelled browser session cannot start later feature work or media collection', async () => { + const calls: string[] = []; + const browser = new Proxy({} as Browser, { get(_target, property) { + calls.push(String(property)); + throw new Error('cancelled browser must not be accessed'); + } }); + const scenario = compileScenarioDefinition({ schemaVersion: 1, track: 'ecommerce', level: 1, + name: 'cancelled feature', features: [{ id: 1, name: 'after cancellation', actors: ['buyer'], + setup: [{ do: 'signUp', actor: 'buyer', name: 'buyer' }], + criteria: [{ id: '1a', desc: 'account exists', points: 1, + steps: [{ do: 'expect', actor: 'buyer', testid: 'current-user' }] }] }] }, { source: 'cancelled.json' }); + const result = await gradeFeature(browser, scenario.features[0]!, { + level: 1, headed: false, selectedCheckKeys: [], nullControl: false, + media: '/unused-media', failureMedia: '/unused-failure-media', trace: true, + }, { actionCancellation: { reason: 'previous action cleanup failed' }, runId: 'cancel-test', + roomName: name => name, url: 'http://unused', actions: [], spacetime: null, nullControl: false }); + assert.deepEqual(calls, []); + assert.equal(result.score, 0); + assert.equal(result.setupEvidence.status, 'harness_failure'); + assert(result.criteria.every(criterion => criterion.evidence.status === 'harness_failure')); + assert.equal(result.cleanupEvidence?.status, 'harness_failure'); +}); diff --git a/tools/stack-bench/tests/grader-entrypoints.test.ts b/tools/stack-bench/tests/grader-entrypoints.test.ts new file mode 100644 index 00000000000..dd54342bc55 --- /dev/null +++ b/tools/stack-bench/tests/grader-entrypoints.test.ts @@ -0,0 +1,124 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { gradeDatabaseLease, parseGradeArgs } from '../grader/grade.js'; +import { parseMutationArgs, remainingMutationBatchMs } from '../grader/mutation-test.js'; +import { createBackendLease, writeBackendLease } from '../src/runtime/backend-lease.js'; +import { createDatabaseWriteCapability } from '../src/actions/runtime-action-executors.js'; +import { attemptDatabaseIdentity } from '../src/stacks/hosted-database-identity.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { loadTrack } from '../src/composition/tracks.js'; +import { requireRecipeRelease } from '../src/composition/recipe-release.js'; +import { createBoundRecipeTaskRequest } from '../src/composition/recipe-selection.js'; +import { readArtifactPayload, writeArtifact } from '../src/evidence/artifacts.js'; + +test('the grader preserves private MongoDB authority through its stock-write caller', t => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-grade-lease-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const lease = createBackendLease({ runId: 'grade-mongo', backend: 'mongodb', + track: 'ecommerce', runIndex: 0, database: 'bench', + container: { name: 'leased-mongodb', id: 'a'.repeat(64) } }); + lease.state = 'active'; + lease.resources.network = { name: 'leased-network', id: 'b'.repeat(64), + namespaceContainerId: lease.resources.container!.id, hostAddresses: [], services: [], + firewallSha256: null, firewallInstalledAt: null }; + const path = join(root, 'lease.json'); + writeBackendLease(path, lease); + const env = { STACK_BENCH_LEASE: path, STACK_BENCH_LEASE_TOKEN: lease.ownershipToken }; + const databaseLease = gradeDatabaseLease('mongodb', env); + assert.deepEqual(databaseLease?.resources.network, lease.resources.network); + const calls: Array<readonly string[]> = []; + const writer = createDatabaseWriteCapability({ backend: 'mongodb', databaseLease, + expand: value => value, exec: (_command, args) => { + calls.push(args); + return args[0] === 'inspect' ? lease.resources.container!.id : 'OK\n'; + } }); + writer.setStock({ item: 'Desk Lamp', warehouse: 'East', quantity: 5, settleMs: 0 }); + const argv = calls.find(args => args.includes('mongosh'))!; + const identity = attemptDatabaseIdentity(lease.ownershipToken); + assert.deepEqual(argv.slice(0, 10), ['exec', lease.resources.container!.id, + 'mongosh', 'bench', '--username', identity.user, '--password', identity.password, + '--authenticationDatabase', 'bench']); + assert.throws(() => gradeDatabaseLease('mongodb', { ...env, STACK_BENCH_LEASE_TOKEN: 'wrong' }), + /token/); + assert.throws(() => gradeDatabaseLease('postgres', env), /backend/); + assert.equal(gradeDatabaseLease('mongodb', {}), null); +}); + +test('grader arguments require an explicit scenario and valid numeric selectors', () => { + assert.throws(() => parseGradeArgs(['node', 'grade', '--url', 'http://localhost:1']), + /--spec/); + assert.throws(() => parseGradeArgs(['node', 'grade', '--url', 'file:///tmp/app', + '--spec', 'scenario.json']), /HTTP or HTTPS/); + assert.throws(() => parseGradeArgs(['node', 'grade', '--url', 'http://localhost:1', + '--spec', 'scenario.json', '--level', '0']), /positive integer/); + assert.equal(parseGradeArgs(['node', 'grade', '--url', 'http://localhost:1', + '--spec', 'scenario.json', '--level', '2']).level, 2); + assert.equal(parseGradeArgs(['node', 'grade', '--url', 'http://localhost:1', + '--spec', 'scenario.json', '--browser-ws-endpoint', 'ws://127.0.0.1:9000/session']) + .browserWsEndpoint, 'ws://127.0.0.1:9000/session'); + assert.throws(() => parseGradeArgs(['node', 'grade', '--url', 'http://localhost:1', + '--spec', 'scenario.json', '--browser-ws-endpoint', 'http://127.0.0.1:9000']), + /must use ws or wss/); +}); + +test('mutation arguments fail before execution when the batch bounds are invalid', () => { + const base = ['node', 'mutation-test', '--app', 'app', '--url', 'http://localhost:1', + '--mutations', 'mutations.json', '--level', '3', '--recipe', 'recipe@1.0.0']; + assert.throws(() => parseMutationArgs([...base, '--mutation-shard-index', '0']), + /must be supplied together/); + assert.throws(() => parseMutationArgs([...base, '--max-runtime-minutes', '0']), + /from 1 through 120/); + assert.equal(parseMutationArgs([...base, '--max-runtime-minutes', '30']).maxRuntimeMinutes, 30); +}); + +test('diagnostic grading cannot bypass the scored recipe boundary', t => { + const args = ['node', 'grade', '--url', 'http://localhost:1', '--spec', 'scenario.json', '--diagnostic']; + assert.equal(parseGradeArgs(args).diagnostic, true); + assert.throws(() => parseGradeArgs([...args, '--recipe', 'ecommerce.progression-catalog']), /scored recipe/); + assert.throws(() => parseGradeArgs([...args, '--recipe-task-json', '{}']), /scored recipe/); + const ordinary = ['node', 'grade', '--url', 'http://localhost:1', '--spec', 'scenario.json']; + assert.throws(() => parseGradeArgs([...ordinary, '--backend', 'postgres', '--saved-diagnostic', '{}']), /zero-point/); + assert.throws(() => parseGradeArgs([...args, '--backend', 'mongodb', '--saved-diagnostic', '{}']), /invalid_type/); + const root = mkdtempSync(join(tmpdir(), 'diagnostic-points-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const spec = JSON.parse(readFileSync(join(STACK_BENCH_ROOT, + 'tracks/ecommerce/scenarios/diagnostic-checkout-database-crash.json'), 'utf8')); + spec.features[0].criteria[0].points = 1; + const path = join(root, 'scenario.json'); writeFileSync(path, JSON.stringify(spec)); + const result = spawnSync(process.execPath, [fileURLToPath(new URL('../grader/grade.js', import.meta.url)), + '--url', 'http://localhost:1', '--spec', path, '--diagnostic'], { encoding: 'utf8', timeout: 10_000 }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /diagnostic grades require zero-point checks/); +}); + +test('the grader rejects changed task contracts and its report preserves the valid request', t => { + const track = loadTrack('ecommerce'); + const binding = requireRecipeRelease(track, 3, 'ecommerce.progression-catalog'); + const request = createBoundRecipeTaskRequest(binding, { taskMode: 'fresh' }).request; + const altered = structuredClone(request) as { task: { contractSha256: string } }; + altered.task.contractSha256 = '0'.repeat(64); + const result = spawnSync(process.execPath, [fileURLToPath(new URL('../grader/grade.js', import.meta.url)), + '--url', 'http://localhost:1', '--spec', join(track.dir, 'scenarios/progression-checkout-crash.json'), + '--track', 'ecommerce', '--level', '3', '--recipe', binding.release.id, + '--recipe-task-json', JSON.stringify(altered)], { encoding: 'utf8', timeout: 15_000 }); + assert.equal(result.status, 2); + assert.match(result.stderr, /task changed after request resolution/); + const root = mkdtempSync(join(tmpdir(), 'grade-task-report-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const path = join(root, 'grade.json'); + writeArtifact(path, { kind: 'grade', id: 'task-report', + payload: { recipeTask: request, total: 0, max: 0, features: [] } }); + assert.deepEqual(readArtifactPayload(path, { expectedKind: 'grade' }).recipeTask, request); +}); + +test('mutation operations use only the remaining batch time', () => { + assert.equal(remainingMutationBatchMs(10_000, 2_500), 7_500); + assert.throws(() => remainingMutationBatchMs(10_000, 10_000), + /deadline reached/); +}); diff --git a/tools/stack-bench/tests/grader-fill-target.test.ts b/tools/stack-bench/tests/grader-fill-target.test.ts new file mode 100644 index 00000000000..9a83c3a267f --- /dev/null +++ b/tools/stack-bench/tests/grader-fill-target.test.ts @@ -0,0 +1,36 @@ +import assert from 'node:assert/strict'; +import { createServer } from 'node:http'; +import test from 'node:test'; +import { chromium } from 'playwright'; +import { gradeFeature } from '../grader/grade.js'; +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; + +test('fill types into the editable control when a listed value reuses its ID', async () => { + // The promotion contract lists saved rules with the same field IDs as the form. + const server = createServer((_request, response) => response.setHeader('content-type', 'text/html') + .end(`<ul><li data-role="promotion-item"><strong data-role="promotion-code">SAVE10</strong></li></ul> + <form data-role="promotion-form"><input data-role="promotion-code"></form>`)); + await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + assert(address && typeof address !== 'string'); + const url = `http://127.0.0.1:${address.port}`; + const browser = await chromium.launch({ headless: true }); + try { + const definition = compileScenarioDefinition({ schemaVersion: 1, track: 'ecommerce', level: 1, + name: 'fill target', features: [{ id: 1, name: 'promotions', actors: ['staff'], setup: [], + criteria: [{ id: '1a', desc: 'a new code is typed into the form', points: 1, steps: [ + { do: 'fill', actor: 'staff', testid: 'promotion-code', text: 'NEW10' }, + { do: 'expect', actor: 'staff', testid: 'promotion-code', in: { testid: 'promotion-form' }, + value: 'NEW10', within: 2000 }, + { do: 'expect', actor: 'staff', testid: 'promotion-item', contains: 'SAVE10', within: 2000 }, + ] }] }] }); + const result = await gradeFeature(browser, definition.features[0]!, { + url, level: 1, headed: false, selectedCheckKeys: [], nullControl: false, + }, { runId: 'fill-target', roomName: name => name, url, actions: [], spacetime: null, nullControl: false }); + assert.equal(result.criteria[0]!.evidence.status, 'passed', result.criteria[0]!.evidence.summary ?? ''); + } finally { + await browser.close(); + server.closeAllConnections(); + await new Promise<void>(resolve => server.close(() => resolve())); + } +}); diff --git a/tools/stack-bench/tests/grader-fresh-client.test.ts b/tools/stack-bench/tests/grader-fresh-client.test.ts new file mode 100644 index 00000000000..3927ad69f45 --- /dev/null +++ b/tools/stack-bench/tests/grader-fresh-client.test.ts @@ -0,0 +1,265 @@ +import assert from 'node:assert/strict'; +import { createServer } from 'node:http'; +import test from 'node:test'; +import { chromium } from 'playwright'; +import { gradeFeature } from '../grader/grade.js'; +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; +import { runApplicationNavigation } from '../src/actions/browser-navigation.js'; +import { ActionInconclusive } from '../src/actions/action-contract.js'; + +test('browser diagnostics survive setup failure without changing check verdicts', async () => { + const server = createServer((_request, response) => response.end(`<button id="trigger">Trigger</button> + <span id="ready">ready</span><script> + document.querySelector('#trigger').onclick = () => { + console.error('query failed password=private-secret'); + throw new Error('account view crashed'); + };</script>`)); + await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + assert(address && typeof address !== 'string'); + const url = `http://127.0.0.1:${address.port}`; + const browser = await chromium.launch({ headless: true }); + try { + for (const mode of ['setup', 'assertion', 'passed'] as const) { + const observation = { do: 'expect', actor: 'buyer', testid: 'ready', + contains: mode === 'passed' ? 'ready' : 'unavailable', within: 100 }; + const definition = compileScenarioDefinition({ schemaVersion: 1, track: 'ecommerce', level: 1, + name: 'browser diagnostics', features: [{ id: 1, name: 'account view', actors: ['buyer'], + setup: [{ do: 'click', actor: 'buyer', testid: 'trigger' }, ...(mode === 'setup' ? [observation] : [])], + criteria: [{ id: '1a', desc: 'account view is ready', points: 1, + steps: [observation] }] }] }); + const result = await gradeFeature(browser, definition.features[0]!, { + url, level: 1, headed: false, selectedCheckKeys: [], nullControl: false, + }, { runId: 'browser-diagnostics', roomName: name => name, url, actions: [], spacetime: null, nullControl: false }); + assert.equal(result.criteria[0]!.evidence.status, + mode === 'setup' ? 'blocked' : mode === 'assertion' ? 'failed' : 'passed'); + assert.equal(result.consoleErrors.filter(line => line.includes('[buyer] query failed')).length, 1); + assert.equal(result.consoleErrors.filter(line => line.includes('[buyer] pageerror: account view crashed')).length, 1); + assert.doesNotMatch(result.consoleErrors.join('\n'), /private-secret/); + } + } finally { + await browser.close(); + server.closeAllConnections(); + await new Promise<void>(resolve => server.close(() => resolve())); + } +}); + +test('a step accepts a confirmation and an explicit reload accepts a leave-page warning', async () => { + const server = createServer((_request, response) => response.end(`<button id="arm">Arm</button> + <button id="confirm">Confirm</button><span id="answer"></span><span id="loads"></span><script> + const loads = Number(sessionStorage.getItem('loads') || 0) + 1; + sessionStorage.setItem('loads', String(loads)); document.querySelector('#loads').textContent = loads; + document.querySelector('#confirm').onclick = () => document.querySelector('#answer').textContent = + confirm('Continue?') ? 'accepted' : 'dismissed'; + document.querySelector('#arm').onclick = () => window.addEventListener('beforeunload', event => { + event.preventDefault(); event.returnValue = 'Pending write'; + });</script>`)); + await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + assert(address && typeof address !== 'string'); + const url = `http://127.0.0.1:${address.port}`; + const browser = await chromium.launch({ headless: true }); + try { + const definition = compileScenarioDefinition({ schemaVersion: 1, track: 'ecommerce', level: 1, + name: 'leave page', features: [{ id: 1, name: 'leave page', actors: ['owner'], setup: [], + criteria: [{ id: 'reload', desc: 'explicit navigation proceeds', points: 1, steps: [ + { do: 'click', actor: 'owner', testid: 'confirm' }, + { do: 'expect', actor: 'owner', testid: 'answer', contains: 'accepted' }, + { do: 'click', actor: 'owner', testid: 'arm' }, + { do: 'reload', actor: 'owner', settleMs: 0 }, + { do: 'expect', actor: 'owner', testid: 'loads', contains: '2' }, + ] }] }] }); + const result = await gradeFeature(browser, definition.features[0]!, { + url, level: 1, headed: false, selectedCheckKeys: [], nullControl: false, + }, { runId: 'leave-page', roomName: name => name, url, actions: [], spacetime: null, nullControl: false }); + const evidence = result.criteria[0]!.evidence; + assert.equal(evidence.status, 'passed', evidence.summary ?? undefined); + } finally { + await browser.close(); + server.closeAllConnections(); + await new Promise<void>(resolve => server.close(() => resolve())); + } +}); + +test('expect reads controls inside collapsed sections, including ones that arrive later', async () => { + const server = createServer((_request, response) => response.end(`<!doctype html><article data-role="support-ticket">Shared case<details><summary>Conversation</summary> + <ol id="thread"></ol></details></article><details><summary>Account</summary><span id="private-note">secret</span></details> + <script>setTimeout(() => { const item = document.createElement('li'); item.dataset.role = 'support-reply-item'; + item.textContent = 'Case received.'; document.querySelector('#thread').append(item); }, 600);</script>`)); + await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + assert(address && typeof address !== 'string'); + const url = `http://127.0.0.1:${address.port}`; + const browser = await chromium.launch({ headless: true }); + try { + const definition = compileScenarioDefinition({ schemaVersion: 1, track: 'ecommerce', level: 1, + name: 'collapsed', features: [{ id: 1, name: 'collapsed', actors: ['owner'], setup: [], criteria: [ + { id: 'reply', desc: 'a later reply is read', points: 1, steps: [ + { do: 'expect', actor: 'owner', testid: 'support-reply-item', contains: 'Case received.', + in: { testid: 'support-ticket', contains: 'Shared case' }, within: 3000 }] }, + { id: 'private', desc: 'a collapsed private value counts as shown', points: 1, steps: [ + { do: 'expect', actor: 'owner', testid: 'private-note', absent: true, within: 500 }] }, + ] }] }); + const result = await gradeFeature(browser, definition.features[0]!, { + url, level: 1, headed: false, selectedCheckKeys: [], nullControl: false, + }, { runId: 'collapsed', roomName: name => name, url, actions: [], spacetime: null, nullControl: false }); + const [reply, hidden] = result.criteria.map(criterion => criterion.evidence); + assert.equal(reply!.status, 'passed', reply!.summary ?? undefined); + assert.equal(hidden!.status, 'failed'); + } finally { + await browser.close(); + server.closeAllConnections(); + await new Promise<void>(resolve => server.close(() => resolve())); + } +}); + +test('a row action that asks for inline confirmation under the same name is confirmed once', async () => { + const server = createServer((_request, response) => response.end(`<!doctype html> + <section data-role="order-item">Coffee Grinder <span data-role="order-status">pending</span><span id="controls"></span></section> + <section data-role="order-item">Desk Lamp <span data-role="order-status">pending</span> + <button data-role="cancel-order" onclick="this.previousElementSibling.textContent='cancelled'">Cancel</button></section> + <script> + const row = document.querySelector('[data-role=order-item]'), controls = row.querySelector('#controls'); + const ask = () => { controls.innerHTML = '<button data-role="cancel-order">Cancel order</button>'; + controls.firstChild.onclick = () => { controls.innerHTML = '<button data-role="cancel-order">Yes, cancel order</button>'; + controls.firstChild.onclick = () => { row.querySelector('[data-role=order-status]').textContent = 'cancelled'; }; }; }; + ask(); + </script>`)); + await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + assert(address && typeof address !== 'string'); + const url = `http://127.0.0.1:${address.port}`; + const browser = await chromium.launch({ headless: true }); + try { + const definition = compileScenarioDefinition({ schemaVersion: 1, track: 'ecommerce', level: 1, + name: 'confirm', features: [{ id: 1, name: 'confirm', actors: ['owner'], setup: [], criteria: [ + { id: 'two-step', desc: 'an inline confirmation is followed', points: 1, steps: [ + { do: 'click', actor: 'owner', testid: 'cancel-order', in: { testid: 'order-item', contains: 'Coffee Grinder' } }, + { do: 'expect', actor: 'owner', testid: 'order-status', in: { testid: 'order-item', contains: 'Coffee Grinder' }, contains: 'cancelled', within: 2000 }] }, + { id: 'one-step', desc: 'a control that stays in place is clicked once', points: 1, steps: [ + { do: 'click', actor: 'owner', testid: 'cancel-order', in: { testid: 'order-item', contains: 'Desk Lamp' } }, + { do: 'expect', actor: 'owner', testid: 'order-status', in: { testid: 'order-item', contains: 'Desk Lamp' }, contains: 'cancelled' }] }, + ] }] }); + const result = await gradeFeature(browser, definition.features[0]!, { + url, level: 1, headed: false, selectedCheckKeys: [], nullControl: false, + }, { runId: 'confirm', roomName: name => name, url, actions: [], spacetime: null, nullControl: false }); + for (const criterion of result.criteria) assert.equal(criterion.evidence.status, 'passed', criterion.evidence.summary ?? undefined); + } finally { + await browser.close(); + server.closeAllConnections(); + await new Promise<void>(resolve => server.close(() => resolve())); + } +}); + +test('a stalled stylesheet leaves navigation unmeasured and the unchanged app can load later', async () => { + let stall = true; + const server = createServer((request, response) => { + if (request.url === '/style.css') { + if (!stall) response.writeHead(200, { 'Content-Type': 'text/css' }).end(''); + return; + } + response.writeHead(200, { 'Content-Type': 'text/html' }).end( + '<link rel="stylesheet" href="/style.css"><script type="module">document.body.textContent="ready"</script>'); + }); + await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + assert(address && typeof address !== 'string'); + const url = `http://127.0.0.1:${address.port}`; + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage(); + for (const operation of [() => page.goto(url, { waitUntil: 'domcontentloaded', timeout: 250 }), + () => page.reload({ waitUntil: 'domcontentloaded', timeout: 250 })]) { + await assert.rejects(runApplicationNavigation(operation, page), error => { + assert(error instanceof ActionInconclusive); + assert.equal(error.details.retryable, true); + assert.deepEqual(error.details.observation, { pendingResources: [`stylesheet ${url}`] }); + return true; + }); + } + stall = false; + await runApplicationNavigation(() => page.goto(url, { waitUntil: 'domcontentloaded' })); + assert.equal(await page.locator('body').textContent(), 'ready'); + } finally { + await browser.close(); + server.closeAllConnections(); + await new Promise<void>(resolve => server.close(() => resolve())); + } +}); + +test('fresh clients can retain identity without retaining authorized transport history', async () => { + const browser = await chromium.launch({ headless: true }); + const secret = 'private-response-before-logout'; + const html = `<span id="identity"></span><span id="loaded"></span> + <button id="login">Login</button><button id="logout">Logout</button><script> + function identity() { document.querySelector('#identity').textContent = + (document.cookie.includes('session=owner') ? 'cookie' : 'no-cookie') + '/' + + (localStorage.getItem('session') === 'owner' ? 'local' : 'no-local') + '/' + + (sessionStorage.getItem('session') === 'owner' ? 'session' : 'no-session'); } + async function read() { await (await fetch('/private')).text(); + document.querySelector('#loaded').textContent = 'done'; } + document.querySelector('#login').onclick = async () => { + document.cookie = 'session=owner; path=/'; localStorage.setItem('session', 'owner'); + sessionStorage.setItem('session', 'owner'); + identity(); await read(); }; + document.querySelector('#logout').onclick = async () => { + document.querySelector('#loaded').textContent = ''; + document.cookie = 'session=; Max-Age=0; path=/'; localStorage.removeItem('session'); + sessionStorage.removeItem('session'); + identity(); await read(); }; + identity();</script>`; + let leaky = false; + const server = createServer((request, response) => { + const privateRead = request.url === '/private'; + const authorized = (request.headers.cookie ?? '').includes('session=owner'); + response.setHeader('Content-Type', privateRead ? 'text/plain' : 'text/html'); + response.end(privateRead ? (authorized || leaky ? secret : 'signed out') : html); + }); + try { + await new Promise<void>((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolve); + }); + const address = server.address(); + assert(address && typeof address !== 'string'); + const url = `http://127.0.0.1:${address.port}`; + for (const mode of ['default', 'preserved', 'leaky'] as const) { + leaky = mode === 'leaky'; + const definition = compileScenarioDefinition({ schemaVersion: 1, track: 'ecommerce', level: 1, + name: 'fresh storage', features: [{ id: 1, name: 'fresh storage', actors: ['owner'], + setup: [ + { do: 'click', actor: 'owner', testid: 'login' }, + { do: 'expectReceived', actor: 'owner', contains: secret, within: 1000 }, + ], criteria: [{ id: 'fresh', desc: 'new transport history with the selected identity', points: 1, + steps: [ + { do: 'freshClient', actor: 'owner', ...(mode === 'default' ? {} : { preserveStorage: true }) }, + { do: 'expect', actor: 'owner-fresh', testid: 'identity', + contains: mode === 'default' ? 'no-cookie/no-local/no-session' : 'cookie/local/session' }, + { do: 'expectNotReceived', actor: 'owner-fresh', contains: secret, within: 25 }, + { do: 'click', actor: 'owner-fresh', testid: 'logout' }, + { do: 'expect', actor: 'owner-fresh', testid: 'loaded', contains: 'done' }, + { do: 'reload', actor: 'owner-fresh', settleMs: 0 }, + { do: 'expect', actor: 'owner-fresh', testid: 'identity', contains: 'no-cookie/no-local/no-session' }, + { do: 'expectNotReceived', actor: 'owner-fresh', contains: secret, within: 100 }, + ] }] }] }); + const result = await gradeFeature(browser, definition.features[0]!, { + url, level: 1, headed: false, selectedCheckKeys: [], nullControl: false, + }, { runId: 'fresh-client', roomName: name => name, url, actions: [], spacetime: null, nullControl: false }); + assert.equal(result.setupEvidence.status, 'passed', result.setupEvidence.summary ?? undefined); + const evidence = result.criteria[0]!.evidence; + assert.equal(evidence.status, mode === 'leaky' ? 'failed' : 'passed', evidence.summary ?? undefined); + if (mode === 'leaky') { + assert.equal(evidence.actions.length, 8); + assert.equal(evidence.finding?.kind, 'message-delivered'); + } + } + assert.throws(() => compileScenarioDefinition({ schemaVersion: 1, track: 'ecommerce', level: 1, + name: 'invalid storage', features: [{ id: 1, name: 'invalid storage', actors: ['owner'], + criteria: [{ id: 'invalid', desc: 'invalid storage type', points: 1, + steps: [{ do: 'freshClient', actor: 'owner', preserveStorage: 'yes' }] }] }] })); + } finally { + await browser.close(); + server.closeAllConnections(); + await new Promise<void>(resolve => server.close(() => resolve())); + } +}); diff --git a/tools/stack-bench/tests/grader-time-limit.test.ts b/tools/stack-bench/tests/grader-time-limit.test.ts new file mode 100644 index 00000000000..c3414a69de9 --- /dev/null +++ b/tools/stack-bench/tests/grader-time-limit.test.ts @@ -0,0 +1,51 @@ +import assert from 'node:assert/strict'; +import { createServer } from 'node:http'; +import test from 'node:test'; +import { chromium } from 'playwright'; +import { gradeFeature, parseGradeArgs } from '../grader/grade.js'; +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; +import { evidenceIsMeasured } from '../src/evidence/check-evidence.js'; + +test('a check past the grading time limit fails and the rest of the grade is kept', async () => { + const server = createServer((_request, response) => response.setHeader('content-type', 'text/html') + .end(`<span id="ready">ready</span> + <script>setTimeout(() => { const late = document.createElement('span'); late.id = 'late'; + late.textContent = 'late'; document.body.append(late); }, 2000);</script>`)); + await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + assert(address && typeof address !== 'string'); + const url = `http://127.0.0.1:${address.port}`; + const browser = await chromium.launch({ headless: true }); + try { + const definition = compileScenarioDefinition({ schemaVersion: 1, track: 'ecommerce', level: 1, + name: 'time limit', features: [{ id: 1, name: 'slow page', actors: ['buyer'], setup: [], + criteria: [ + { id: '1a', desc: 'a step in progress at the limit completes', points: 1, + steps: [{ do: 'expect', actor: 'buyer', testid: 'late', within: 5000 }] }, + { id: '1b', desc: 'the next check is past the limit', points: 1, + steps: [{ do: 'expect', actor: 'buyer', testid: 'ready', within: 5000 }] }, + ] }] }); + const result = await gradeFeature(browser, definition.features[0]!, { + url, level: 1, headed: false, selectedCheckKeys: [], nullControl: false, + }, { runId: 'time-limit', roomName: name => name, url, actions: [], spacetime: null, nullControl: false, + timeLimit: { limitMs: 60_000, deadlineAtMs: Date.now() + 1000 } }); + const [started, late] = result.criteria; + assert.equal(started!.evidence.status, 'passed'); + assert.equal(late!.evidence.status, 'failed'); + assert.equal(late!.evidence.finding?.kind, 'grading-time-limit'); + assert(evidenceIsMeasured(late!.evidence), 'a time-limit failure is measured, not inconclusive'); + assert.equal(result.score, 1); + } finally { + await browser.close(); + server.closeAllConnections(); + await new Promise<void>(resolve => server.close(() => resolve())); + } +}); + +test('the grading time limit must be a positive integer', () => { + const base = ['node', 'grade.js', '--url', 'http://127.0.0.1:1', '--spec', 'spec.json']; + assert.equal(parseGradeArgs([...base, '--time-limit-ms', '120000']).timeLimitMs, 120_000); + for (const bad of ['0', '-1', '1.5', 'soon']) { + assert.throws(() => parseGradeArgs([...base, `--time-limit-ms=${bad}`]), /positive integer/); + } +}); diff --git a/tools/stack-bench/tests/grading-timeout.test.ts b/tools/stack-bench/tests/grading-timeout.test.ts new file mode 100644 index 00000000000..e322b4e8f57 --- /dev/null +++ b/tools/stack-bench/tests/grading-timeout.test.ts @@ -0,0 +1,40 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { GRADER_SOURCE_TIMEOUT_MS, gradingRunTimeoutMs, gradingSourceTimeoutMs, selectedGradingSourceCount } + from '../src/runtime/grading-timeout.js'; +import { mutationGradeTimeoutMs } from '../src/evidence/mutation-control.js'; + +test('grading source count deduplicates checks that share one scenario source', () => { + assert.equal(selectedGradingSourceCount( + [{ source: 'scenarios/accounts.json' }, { source: 'scenarios/accounts.json' }], + [{ source: 'scenarios/orders.json' }], + ), 2); +}); + +test('grading timeout scales per selected source, then caps for a full catalog', () => { + assert.equal(gradingRunTimeoutMs(0), 20 * 60_000); + assert.equal(gradingRunTimeoutMs(1), 20 * 60_000 + GRADER_SOURCE_TIMEOUT_MS); + assert.equal(gradingRunTimeoutMs(6), 110 * 60_000); + assert.equal(gradingRunTimeoutMs(92), 120 * 60_000); + assert.equal(gradingRunTimeoutMs(Number.MAX_SAFE_INTEGER), 120 * 60_000); + assert.throws(() => gradingRunTimeoutMs(-1), /non-negative safe integer/); + assert.throws(() => gradingRunTimeoutMs(1.5), /non-negative safe integer/); +}); + +test('large selected pack fits child and parent deadlines without bypassing the mutation batch limit', () => { + const packs = [ + { id: 'volume', budget: { status: 'bounded', maxRuntimeMs: 2_500_000 } }, + { id: 'accounts', budget: { status: 'bounded', maxRuntimeMs: 40_000 } }, + ] as const; + const checks = [{ source: 'catalog.json', packId: 'volume' }]; + const child = gradingSourceTimeoutMs(packs, checks); + assert.equal(child, 2_560_000); + assert.equal(gradingSourceTimeoutMs(packs, [...checks, ...checks]), child); + assert.equal(gradingSourceTimeoutMs(packs, [{ packId: 'accounts' }]), GRADER_SOURCE_TIMEOUT_MS); + assert.equal(gradingRunTimeoutMs(1, packs, checks), child + 20 * 60_000); + assert.equal(mutationGradeTimeoutMs(4_000_000, 1_000, child), child); + assert.equal(mutationGradeTimeoutMs(31_000, 1_000, child), 30_000); + assert.equal(mutationGradeTimeoutMs(1_000, 1_000, child), 0); + assert.throws(() => mutationGradeTimeoutMs(31_000, 1_000, NaN), /positive safe integer/); +}); diff --git a/tools/stack-bench/tests/grok-login.test.ts b/tools/stack-bench/tests/grok-login.test.ts new file mode 100644 index 00000000000..32355a38602 --- /dev/null +++ b/tools/stack-bench/tests/grok-login.test.ts @@ -0,0 +1,96 @@ +import assert from 'node:assert/strict'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, utimesSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { ensureFreshGrokLogin, readGrokLogin } from '../src/agents/grok-login.js'; + +const HOUR = 3_600_000; +const NOW = Date.parse('2026-10-06T12:00:00Z'); +const SCOPE = 'https://auth.x.ai::client-1'; + +function login(expiresAt: number, extra: Record<string, unknown> = {}) { + return { [SCOPE]: { key: 'access-1', auth_mode: 'oidc', refresh_token: 'refresh-1', principal_id: 'user-1', + expires_at: new Date(expiresAt).toISOString(), oidc_issuer: 'https://auth.x.ai', oidc_client_id: 'client-1', + email: 'kept@example.com', ...extra } }; +} + +function fixture(content: unknown) { + const root = mkdtempSync(join(tmpdir(), 'grok-login-')); + const path = join(root, 'auth.json'); + writeFileSync(path, JSON.stringify(content), { mode: 0o600 }); + return { root, path }; +} + +test('a fresh Grok login is used as it is', async () => { + const { root, path } = fixture(login(NOW + 6 * HOUR)); + try { + const result = await ensureFreshGrokLogin(path, 4.5 * HOUR, + { now: () => NOW, fetch: () => assert.fail('no refresh while enough time remains') }); + assert.equal(result.token, 'access-1'); + assert.equal(result.expiresAtMs, NOW + 6 * HOUR); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('an expiring Grok login is refreshed once, rotated in place, and keeps its account identity', async () => { + const { root, path } = fixture(login(NOW + HOUR)); + try { + const before = readGrokLogin(readFileSync(path, 'utf8')); + const requests: { url: string; body: string }[] = []; + const exchange = async (url: string, init: { body: string }) => { + requests.push({ url, body: init.body }); + await new Promise(resolve => setTimeout(resolve, 20)); + return { ok: true, status: 200, json: async () => ({ access_token: 'access-2', refresh_token: 'refresh-2', expires_in: 21_600 }) }; + }; + // Two attempts starting together share one exchange: the refresh token rotates. + const [first, second] = await Promise.all([1, 2].map(() => + ensureFreshGrokLogin(path, 4.5 * HOUR, { now: () => NOW, fetch: exchange, waitMs: 5 }))); + assert.equal(requests.length, 1); + assert.equal(requests[0]!.url, 'https://auth.x.ai/oauth2/token'); + assert.deepEqual(Object.fromEntries(new URLSearchParams(requests[0]!.body)), + { grant_type: 'refresh_token', refresh_token: 'refresh-1', client_id: 'client-1' }); + assert.equal(first!.token, 'access-2'); + assert.equal(second!.token, 'access-2'); + const stored = JSON.parse(readFileSync(path, 'utf8'))[SCOPE]; + assert.deepEqual([stored.key, stored.refresh_token, stored.email, stored.expires_at], + ['access-2', 'refresh-2', 'kept@example.com', new Date(NOW + 6 * HOUR).toISOString()]); + if (process.platform !== 'win32') assert.equal(statSync(path).mode & 0o777, 0o600); + assert.equal(first!.identity, before.identity, 'a campaign pins the account, not the rotating token'); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('a refused or short refresh leaves the stored login unchanged and asks for a new sign-in', async () => { + const { root, path } = fixture(login(NOW + HOUR)); + try { + const original = readFileSync(path, 'utf8'); + await assert.rejects(ensureFreshGrokLogin(path, 4.5 * HOUR, { now: () => NOW, + fetch: async () => ({ ok: false, status: 400, json: async () => ({ error: 'invalid_grant' }) }) }), /grok login --device-auth/); + assert.equal(readFileSync(path, 'utf8'), original); + await assert.rejects(ensureFreshGrokLogin(path, 4.5 * HOUR, { now: () => NOW, + fetch: async () => ({ ok: true, status: 200, json: async () => ({ access_token: 'short', expires_in: 600 }) }) }), + /shorter than an attempt/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('a lock left by a crashed refresher is reclaimed', async () => { + const { root, path } = fixture(login(NOW + HOUR)); + try { + mkdirSync(`${path}.stack-bench-lock`); + const old = new Date(Date.now() - 10 * 60_000); + utimesSync(`${path}.stack-bench-lock`, old, old); + const result = await ensureFreshGrokLogin(path, 4.5 * HOUR, { now: () => NOW, waitMs: 5, + fetch: async () => ({ ok: true, status: 200, json: async () => ({ access_token: 'access-3', expires_in: 21_600 }) }) }); + assert.equal(result.token, 'access-3'); + assert.equal(JSON.parse(readFileSync(path, 'utf8'))[SCOPE].refresh_token, 'refresh-1', 'an unrotated refresh token is kept'); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('only a single auth.x.ai OIDC sign-in is accepted', () => { + assert.throws(() => readGrokLogin('not json'), /JSON written by `grok login`/); + assert.throws(() => readGrokLogin(JSON.stringify({})), /exactly one auth\.x\.ai sign-in/); + assert.throws(() => readGrokLogin(JSON.stringify({ ...login(NOW), 'https://auth.x.ai::other': login(NOW)[SCOPE] })), + /exactly one/); + const legacy = login(NOW); + delete (legacy[SCOPE] as Record<string, unknown>).refresh_token; + assert.throws(() => readGrokLogin(JSON.stringify(legacy)), /not an auth\.x\.ai OIDC sign-in/); +}); diff --git a/tools/stack-bench/tests/grok-protocol.test.ts b/tools/stack-bench/tests/grok-protocol.test.ts new file mode 100644 index 00000000000..4addb98d07a --- /dev/null +++ b/tools/stack-bench/tests/grok-protocol.test.ts @@ -0,0 +1,76 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { GROK_SUBAGENT_TOOLS, GROK_WEB_TOOLS, grokArguments, parseGrokResult } from '../src/agents/grok-protocol.js'; +import { DEFAULT_TOOL_POLICY } from '../src/agents/tool-policy.js'; + +const OFF = { webSearch: false, subagents: false }; + +const SESSION = '01a112c6-39bd-7d52-a3a9-3158e1b76555'; +const stream = (result: Record<string, unknown>) => [ + { type: 'system', subtype: 'init', session_id: SESSION, cwd: '/app', model: 'grok-4.6' }, + { type: 'assistant', message: { content: [{ type: 'text', text: 'done' }] } }, + { type: 'result', session_id: SESSION, ...result }, +].map(event => JSON.stringify(event)).join('\n'); + +test('Grok runs headless from stdin with files and a shell only, and resumes by session', () => { + const args = grokArguments({ model: 'grok-4.6', effort: 'medium', resumeSession: null, tools: OFF }); + assert.deepEqual(args.slice(0, 11), ['--prompt-file', '/dev/stdin', '--output-format', 'streaming-messages-json', + '-m', 'grok-4.6', '--effort', 'medium', '--always-approve', '--disable-web-search', '--no-subagents']); + const tools = args[args.indexOf('--tools') + 1]!.split(','); + assert(tools.includes('run_terminal_command') && tools.includes('read_file')); + // --no-subagents leaves the subagent tool callable; only the allowlist removes it. + assert(!tools.includes('spawn_subagent')); + // The allowlist does not govern Grok's other built-in tools; they are removed by name. + const removed = args[args.indexOf('--disallowed-tools') + 1]!.split(','); + assert(removed.includes('workflow') && removed.includes('image_gen') && removed.includes('ask_user_question')); + assert.deepEqual(grokArguments({ model: 'grok-4.6', effort: 'high', resumeSession: SESSION, tools: OFF }).slice(-2), + ['--resume', SESSION]); +}); + +test('Grok web search and subagents follow the tool policy', () => { + const view = (tools: { webSearch: boolean; subagents: boolean }) => { + const args = grokArguments({ model: 'grok-4.6', effort: 'medium', resumeSession: null, tools }); + return { flags: args.filter(arg => arg === '--disable-web-search' || arg === '--no-subagents'), + allowed: args[args.indexOf('--tools') + 1]!.split(','), removed: args[args.indexOf('--disallowed-tools') + 1]!.split(',') }; + }; + // The default is how Grok ran in 1.1. + assert.deepEqual(grokArguments({ model: 'grok-4.6', effort: 'medium', resumeSession: null, tools: DEFAULT_TOOL_POLICY }), + grokArguments({ model: 'grok-4.6', effort: 'medium', resumeSession: null, tools: OFF })); + const off = view(OFF); + assert.deepEqual(off.flags, ['--disable-web-search', '--no-subagents']); + assert(GROK_WEB_TOOLS.every(tool => !off.allowed.includes(tool))); + assert(GROK_SUBAGENT_TOOLS.every(tool => !off.allowed.includes(tool))); + const search = view({ webSearch: true, subagents: false }); + assert.deepEqual(search.flags, ['--no-subagents']); + assert(GROK_WEB_TOOLS.every(tool => search.allowed.includes(tool))); + const on = view({ webSearch: true, subagents: true }); + assert.deepEqual(on.flags, []); + assert(GROK_SUBAGENT_TOOLS.every(tool => on.allowed.includes(tool))); + // Scripted workflows, media and MCP tools stay removed whatever the policy. + for (const result of [off, search, on]) assert(result.removed.includes('workflow') && result.removed.includes('search_tool')); +}); + +test('a completed Grok stream maps to the shared result, with cache reads kept separate', () => { + const result = parseGrokResult(stream({ subtype: 'success', is_error: false, num_turns: 2, result: 'done', + stop_reason: 'end_turn', usage: { input_tokens: 400, output_tokens: 50, cache_read_input_tokens: 600, + cache_creation_input_tokens: 0, server_tool_use: { web_search_requests: 0 } } })); + assert.deepEqual(result, { type: 'result', session_id: SESSION, is_error: false, result: 'done', num_turns: 2, + usage: { input_tokens: 400, output_tokens: 50, cache_read_input_tokens: 600, cache_creation_input_tokens: 0 } }); +}); + +test('Grok errors, early stops and missing results are provider errors', () => { + const error = parseGrokResult(stream({ subtype: 'error_during_execution', is_error: true, num_turns: 0, stop_reason: null, + usage: { input_tokens: 0, output_tokens: 0, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 }, + errors: ['Internal error: upstream refused'] })); + assert.equal(error.is_error, true); + assert.match(String(error.result), /upstream refused/); + assert.equal(error.terminal_reason, 'provider_error'); + assert.equal(error.session_id, SESSION, 'a failed session can still be resumed'); + const stopped = parseGrokResult(stream({ is_error: false, stop_reason: 'max_turns', num_turns: 3, + usage: { input_tokens: 1, output_tokens: 1, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 } })); + assert.match(String(stopped.result), /stopped with "max_turns"/); + assert.match(String(parseGrokResult('').result), /no result[\s\S]*no valid session ID/); + assert.match(String(parseGrokResult(stream({ is_error: false, stop_reason: 'end_turn', + usage: { input_tokens: -1, output_tokens: 0, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 } })).result), + /Invalid Grok token usage/); +}); diff --git a/tools/stack-bench/tests/harness-errors.test.ts b/tools/stack-bench/tests/harness-errors.test.ts new file mode 100644 index 00000000000..b37695759c6 --- /dev/null +++ b/tools/stack-bench/tests/harness-errors.test.ts @@ -0,0 +1,39 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { harnessBrowserFailure, harnessProcessFailure, + runBrowserInfrastructureOperation } from '../src/evidence/harness-errors.js'; + +test('child-process timeouts are harness failures, not application findings', () => { + const error = Object.assign(new Error('spawnSync docker ETIMEDOUT'), { + code: 'ETIMEDOUT', + path: 'docker', + status: null, + }); + assert.equal(harnessProcessFailure(error), 'docker failed in the harness (ETIMEDOUT)'); +}); + +test('a child process non-zero exit remains eligible as an application finding', () => { + const error = Object.assign(new Error('command failed'), { status: 1, path: 'docker' }); + assert.equal(harnessProcessFailure(error), null); +}); + +test('a missing harness database container is not blamed on the application', () => { + const error = Object.assign(new Error('docker exec failed'), { + status: 1, + stderr: 'Error response from daemon: No such container: stack-bench-mongodb', + }); + assert.equal(harnessProcessFailure(error), + 'database container selected by the harness is unavailable'); +}); + +test('a crashed browser target is inconclusive harness evidence', async () => { + assert.match(harnessBrowserFailure(new Error('browserContext.setOffline: Target crashed ')) ?? '', + /^browser target failed in the harness/); + assert.equal(harnessBrowserFailure(new Error('expected stock 15, saw 20')), null); + // Browser setup operations are harness failures, but app navigation is not. + const infrastructure = await runBrowserInfrastructureOperation('page creation', async () => { + throw new Error('page allocation failed'); + }).catch((error: unknown) => error); + assert.match(harnessBrowserFailure(infrastructure) ?? '', /browser page creation failed/); + assert.equal(harnessBrowserFailure(new Error('net::ERR_CONNECTION_REFUSED')), null); +}); diff --git a/tools/stack-bench/tests/harness-regressions.test.ts b/tools/stack-bench/tests/harness-regressions.test.ts new file mode 100644 index 00000000000..38cce5d4d6f --- /dev/null +++ b/tools/stack-bench/tests/harness-regressions.test.ts @@ -0,0 +1,79 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { addCostUsd, finalizeRunTotals } from '../src/evidence/benchmark-run.js'; +import { checkDatabaseProvenance } from '../commands/run-suite.js'; +import type { BackendLease } from '../src/runtime/backend-lease.js'; +import { summarizeSessions } from '../src/evidence/session-metrics.js'; + +test('database provenance accepts the leased environment and rejects an unrelated literal', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-database-provenance-')); + try { + const server = join(root, 'server'); + mkdirSync(server); + writeFileSync(join(server, 'db.ts'), + 'export const connectionString = process.env.DATABASE_URL;\n'); + assert.deepEqual(checkDatabaseProvenance({ app: root, backend: 'postgres' }), { + ok: true, url: 'process.env.DATABASE_URL', + reason: 'app reads the database URL supplied by its authenticated backend lease', + }); + writeFileSync(join(server, 'db.ts'), + 'export const connectionString = "postgresql://user:pass@localhost:5433/wrong";\n'); + const wrong = checkDatabaseProvenance({ app: root, backend: 'postgres' }); + assert.equal(wrong.ok, false); + assert.match(wrong.reason, /database supplied for this run is on port 6532/); + writeFileSync(join(server, 'db.ts'), + 'export const connectionString = "postgresql://user:pass@localhost:5433/wrong?note=:6532/";\n'); + assert.equal(checkDatabaseProvenance({ app: root, backend: 'postgres' }).ok, false); + // A port that only contains the benchmark port is not the benchmark database. + for (const [port, ok] of [[6537, true], [16537, false]] as const) { + writeFileSync(join(server, 'db.ts'), `export const url = 'mongodb://localhost:${port}/app_ecom_run0';\n`); + assert.equal(checkDatabaseProvenance({ app: root, backend: 'mongodb' }).ok, ok, String(port)); + } + // Source syntax and host-port allocations cannot establish runtime identity. + for (const backend of ['postgres', 'mongodb']) { + assert.deepEqual(checkDatabaseProvenance({ app: root, backend, + databaseLease: {} as BackendLease }), { + ok: true, reason: 'leased database requires runtime marker verification', + }); + } + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('an interrupted later level keeps completed totals and marks cost incomplete', () => { + const run = { levels: [{ level: 1, graded: true, score: 58, max: 58, + buildCostUsd: 10, repairCostUsd: 2.5, repairs: 1, + sessionTotals: { sessions: 2, tokens: 100, outputTokens: 20, turns: 8, durationMs: 900 } }] }; + assert.deepEqual(finalizeRunTotals(run, 1_000, { now: 11_000, costComplete: false }), { + score: 58, max: 58, costUsd: 12.5, costComplete: false, repairs: 1, + sessions: 2, tokens: 100, outputTokens: 20, turns: 8, modelDurationMs: 900, + durationSec: 10, ungraded: [], + }); +}); + +test('dependency totals use the cumulative progression score', () => { + const run = { + levels: [ + { level: 1, graded: true, score: 8, max: 12 }, + { level: 2, graded: true, score: 13, max: 15 }, + ], + progressionStatus: { score: { uniqueChecks: { passedPoints: 17, availablePoints: 162 } } }, + }; + const totals = finalizeRunTotals(run, 1_000, { now: 2_000 }); + assert.equal(totals.score, 17); + assert.equal(totals.max, 162); +}); + +test('session and run totals keep receipt precision', () => { + const sessions = [{ costUsd: 0.123456 }, { costUsd: 0.234567 }, { costUsd: 0.345678 }]; + assert.equal(summarizeSessions(sessions).costUsd, 0.703701); + const run = { levels: [{ level: 1, buildCostUsd: 0.123456, repairCostUsd: 0.234567 }, + { level: 2, buildCostUsd: 0.345678, repairCostUsd: 0 }] }; + assert.equal(finalizeRunTotals(run, 0, { now: 1 }).costUsd, 0.703701); + assert.equal(addCostUsd(0.123456, 0.234567, 0.345678), 0.703701); +}); diff --git a/tools/stack-bench/tests/helpers/progression-source.ts b/tools/stack-bench/tests/helpers/progression-source.ts new file mode 100644 index 00000000000..75e3fa73be8 --- /dev/null +++ b/tools/stack-bench/tests/helpers/progression-source.ts @@ -0,0 +1,37 @@ +import { readFileSync } from 'node:fs'; + +import { compileProgressionDefinition } from '../../src/progression/progression-definition.js'; +import type { CompiledProgressionDefinition } + from '../../src/progression/progression-definition.js'; + +export interface ValidatedProgressionSource { + definition: CompiledProgressionDefinition; + gradingGroups(nodeId: string): string[]; +} + +export function loadValidatedProgressionSource( + path: string, + trackRoot: string, +): ValidatedProgressionSource { + const source: unknown = JSON.parse(readFileSync(path, 'utf8')); + const definition = compileProgressionDefinition(source, { trackRoot, source: path }); + return { definition, gradingGroups: nodeId => authoredGradingGroups(source, nodeId) }; +} + +function authoredGradingGroups(source: unknown, nodeId: string): string[] { + if (source === null || typeof source !== 'object' || Array.isArray(source) + || !('nodes' in source) || !Array.isArray(source.nodes)) { + throw new Error('validated progression source must contain nodes'); + } + const node = source.nodes.find(candidate => candidate !== null && typeof candidate === 'object' + && !Array.isArray(candidate) && 'id' in candidate && candidate.id === nodeId); + if (!node || !('gradingGroups' in node) || !Array.isArray(node.gradingGroups)) { + throw new Error(`validated progression node ${nodeId} must contain grading groups`); + } + for (const group of node.gradingGroups) { + if (typeof group !== 'string') { + throw new Error(`validated progression node ${nodeId} has an invalid grading group`); + } + } + return node.gradingGroups; +} diff --git a/tools/stack-bench/tests/helpers/supabase-lease.ts b/tools/stack-bench/tests/helpers/supabase-lease.ts new file mode 100644 index 00000000000..6675b7c42f7 --- /dev/null +++ b/tools/stack-bench/tests/helpers/supabase-lease.ts @@ -0,0 +1,55 @@ +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createBackendLease, writeBackendLease, type BackendLease } from '../../src/runtime/backend-lease.js'; +import type { TextCommandExecutor } from '../../src/runtime/command-executor.js'; +import { formatSupabaseSecrets, generateSupabaseSecrets } from '../../src/stacks/backends/supabase-platform.js'; + +export const SUPABASE_GATEWAY = 'http://127.0.0.1:13410'; +export const SUPABASE_SECRETS = generateSupabaseSecrets(); +const STARTED_AT = '2026-09-25T00:00:00.000Z'; + +// An active, isolated Supabase lease with an owned anchor. +export function supabaseLease(): BackendLease { + const lease = createBackendLease({ runId: 'supabase-grading', backend: 'supabase', track: 'ecommerce', + runIndex: 0, serverUri: SUPABASE_GATEWAY, database: 'postgres' }); + const anchor = 'a'.repeat(64); + lease.state = 'active'; + lease.resources.container = { name: 'sb-anchor', id: anchor, owned: true }; + lease.resources.network = { name: 'sb-network', id: 'b'.repeat(64), namespaceContainerId: anchor, + namespaceStartedAt: STARTED_AT, hostAddresses: [], services: [], + firewallSha256: 'c'.repeat(64), firewallInstalledAt: STARTED_AT }; + return lease; +} + +// Docker as the anchor presents it: identity, the root-only secret file, and +// privileged psql, whose SQL arrives on stdin. +export function supabaseExec(lease: BackendLease, psql: (sql: string, args: readonly string[]) => string, + calls: { args: readonly string[] }[] = []): TextCommandExecutor { + return (command, args, options) => { + if (command !== 'docker') throw new Error(`unexpected command ${command}`); + calls.push({ args }); + if (args[0] === 'inspect') return JSON.stringify({ Id: lease.resources.container!.id, + State: { Running: true, StartedAt: lease.resources.network!.namespaceStartedAt } }); + if (args.includes('cat')) return formatSupabaseSecrets(SUPABASE_SECRETS); + if (args.includes('psql')) return psql(String(options.input ?? ''), args); + throw new Error(`unexpected docker ${args.join(' ')}`); + }; +} + +// The grader's authenticated lease environment for the duration of one test. +export function withSupabaseLeaseEnvironment(t: { after(fn: () => void): void }, lease: BackendLease): void { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-supabase-lease-')); + const path = join(root, 'lease.json'); + writeBackendLease(path, lease); + const previous = { path: process.env.STACK_BENCH_LEASE, token: process.env.STACK_BENCH_LEASE_TOKEN }; + process.env.STACK_BENCH_LEASE = path; + process.env.STACK_BENCH_LEASE_TOKEN = lease.ownershipToken; + t.after(() => { + for (const [key, value] of [['STACK_BENCH_LEASE', previous.path], ['STACK_BENCH_LEASE_TOKEN', previous.token]] as const) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + rmSync(root, { recursive: true, force: true }); + }); +} diff --git a/tools/stack-bench/tests/hosted-crash-window.integration.ts b/tools/stack-bench/tests/hosted-crash-window.integration.ts new file mode 100644 index 00000000000..c3ed6918635 --- /dev/null +++ b/tools/stack-bench/tests/hosted-crash-window.integration.ts @@ -0,0 +1,402 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { promisify } from 'node:util'; +import test from 'node:test'; +import { ACTION_REGISTRY } from '../src/actions/action-catalog.js'; +import { executeAction } from '../src/actions/action-contract.js'; +import { checkoutStateSchema, type CheckoutState } from '../src/stacks/checkout-state.js'; +import { prepareRuntimeCrash, recoverRuntimeCrash } from '../src/runtime/backend-control.js'; +import { prepareProcessCrash } from '../src/stacks/process-crash.js'; +import { activateAttemptBackend } from '../src/stacks/hosted-lifecycle.js'; +import { waitFor, answers } from '../src/stacks/lifecycle-readiness.js'; +import { backendResourceLockKeys, claimBackendResources, createBackendLease, publicBackendLease, + readBackendLease, resourceLockScope, updateBackendLease } from '../src/runtime/backend-lease.js'; +import { releaseBackendLease } from '../src/runtime/backend-teardown.js'; +import { attemptDocker, attemptControllerImage, ATTEMPT_CREATION_LABEL, recordAttemptCreation, + requireAttemptNetwork } from '../src/runtime/docker-network.js'; +import { CODING_CONTAINER_AGENT } from '../src/runtime/coding-container-policy.js'; + +// No package installation or model. Real sockets and processes run in the normal +// leased namespace. The state oracle below is a prepared, unchanged cart: this +// fixture qualifies fault admission, not a new checkout/database implementation. +const fixture = `import http from 'node:http'; +import fs from 'node:fs'; +import {spawn} from 'node:child_process'; +const mode=fs.readFileSync('/app/mode','utf8').trim(); +const port=Number(process.env.PORT), api=port+1; +const role=process.argv[2]||'entry'; +const aux=[]; +const ready=()=>{ + if(role!=='api') for(let i=0;i<36;i++) aux.push(spawn('sleep',['infinity']).pid); + if(mode==='late-auxiliary'&&role==='entry') aux.push(spawn(process.execPath,[import.meta.filename,'api'],{stdio:'inherit'}).pid); + fs.writeFileSync('/tmp/'+role+'-ready.json',JSON.stringify({pid:process.pid,port:role==='api'?api:port,aux})); +}; +const handle=(req,res)=>{ + if(req.url==='/health') return res.end('ready'); + fs.appendFileSync('/tmp/entered','request\\n'); + // Held until the real service SIGKILL. There is no fabricated HTTP outcome. +}; +if(mode==='late-auxiliary'&&role==='api'){ + fs.writeFileSync('/tmp/aux-ready','ready'); + const poll=setInterval(()=>{if(fs.existsSync('/tmp/listen-now')){ + clearInterval(poll);http.createServer(handle).listen(api,'0.0.0.0',ready); + }},1); +}else if(mode==='missing'){ready();setInterval(()=>{},1000);} +else if(mode==='proxy'&&role==='entry'){ + const upstream=spawn(process.execPath,[import.meta.filename,'api'],{stdio:'inherit'}); + const server=http.createServer((req,res)=>{ + if(req.url==='/health') return res.end('ready'); + const call=http.request({host:'127.0.0.1',port:api,path:req.url,method:req.method},reply=>reply.pipe(res)); + // Like a proxy with an outstanding upstream operation, do not synthesize a + // completed checkout response when its API dies; entry death closes clients. + call.on('error',()=>{});req.pipe(call); + }); + server.listen(port,'0.0.0.0',ready);upstream.on('error',error=>{throw error;}); +}else http.createServer(handle).listen(role==='api'?api:port,'0.0.0.0',ready); +`; + +const sha = (value: string | Buffer) => createHash('sha256').update(value).digest('hex'); +const asyncExec = promisify(execFile); + +// Run from a Linux controller with the normal Docker socket/cache and pinned +// STACK_BENCH_CONTROLLER_IMAGE_ID. Keep this off the ordinary unit suite. +test('hosted crash uses owned writer freeze while preserving complete process quiescence', { + skip: process.env.STACK_BENCH_HOSTED_CRASH_TEST !== '1', timeout: 300_000, +}, async t => { + assert.equal(process.platform, 'linux'); + const output = process.env.STACK_BENCH_HOSTED_CRASH_EVIDENCE; + assert(output, 'an evidence output path is required'); + const image = attemptControllerImage(); + const buildImage = process.env.STACK_BENCH_HOSTED_CRASH_BUILD_IMAGE; + assert.match(image, /^sha256:[a-f0-9]{64}$/); + assert.match(buildImage ?? '', /^sha256:[a-f0-9]{64}$/, 'use the actual pinned coding/build image'); + const port = Number(process.env.STACK_BENCH_HOSTED_CRASH_PORT ?? 14570); + assert(Number.isInteger(port) && port > 1024 && port < 65534); + mkdirSync(dirname(output), { recursive: true }); + // The private lease must survive a stopped controller's ephemeral /tmp. + const root = mkdtempSync(join(dirname(output), 'stack-bench-hosted-crash-')); + const leasePath = join(root, 'lease.json'), app = join(root, 'app'); + mkdirSync(app); + const start = '#!/bin/sh\ncd /app\nexec node fixture.mjs\n'; + writeFileSync(join(app, 'start.sh'), start); + writeFileSync(join(app, 'fixture.mjs'), fixture); + const ports = { vite: port, express: port + 1, dbPort: null }; + const limits = { cpuCount: 1, memoryBytes: 512 * 1024 * 1024, memorySwapBytes: 512 * 1024 * 1024, pids: 256 }; + const lease = createBackendLease({ runId: root.split('/').at(-1)!, backend: 'postgres', + track: 'ecommerce', runIndex: 0, database: 'crash_window' }); + const previous = { lease: process.env.STACK_BENCH_LEASE, token: process.env.STACK_BENCH_LEASE_TOKEN }; + const evidence: Record<string, unknown> = { result: 'running', image, buildImage, ports, fixtureSha256: sha(fixture), + rerun: 'STACK_BENCH_HOSTED_CRASH_TEST=1 STACK_BENCH_HOSTED_CRASH_EVIDENCE=<file> node --test --test-name-pattern="hosted crash uses owned" dist/tests/hosted-crash-window.integration.js', + runtimeSha256: Object.fromEntries(['../src/actions/crash-action-executors.js', '../src/stacks/process-crash.js', + '../src/runtime/backend-control.js', '../src/stacks/hosted-lifecycle.js'].map(path => + [path, sha(readFileSync(new URL(path, import.meta.url)))])), cases: [] }; + const save = () => writeFileSync(output, JSON.stringify(evidence, null, 2)); + let build = '', network = ''; + const spec = { backend: 'postgres', app, port, probe: '/health' }; + const uid = `${CODING_CONTAINER_AGENT.uid}:${CODING_CONTAINER_AGENT.gid}`; + const exec = (args: string[]) => attemptDocker(['exec', build, ...args]); + const json = (file: string) => JSON.parse(exec(['cat', file])) as { pid: number; port: number; aux: number[] }; + const killUser = () => exec(['sh', '-c', `pkill -KILL -u ${CODING_CONTAINER_AGENT.uid} || [ "$?" = 1 ]`]); + const prepareMode = async (mode: string, foreign = false) => { + killUser(); + exec(['sh', '-c', 'rm -f /tmp/entry-ready.json /tmp/api-ready.json /tmp/entered /tmp/respawn.json /tmp/aux-ready /tmp/listen-now']); + attemptDocker(['exec', '-i', build, 'sh', '-c', 'cat > /app/mode'], mode); + attemptDocker(['exec', '-d', '--user', foreign ? '0:0' : uid, '-e', `PORT=${port}`, build, + 'node', '/app/fixture.mjs']); + await waitFor(async () => { + try { json('/tmp/entry-ready.json'); return mode !== 'proxy' || !!json('/tmp/api-ready.json'); } + catch { return false; } + }, 10_000, `${mode} fixture processes`); + if (mode !== 'missing') await waitFor(() => answers(`http://127.0.0.1:${port}/health`), 10_000, 'entry listener'); + }; + let failure: { error: unknown } | undefined; + try { + claimBackendResources(leasePath, lease, { ...resourceLockScope(), keys: backendResourceLockKeys(lease, ports) }); + activateAttemptBackend({ leasePath, lease, ports }); + let active = readBackendLease(leasePath, { token: lease.ownershipToken, active: true }); + network = active.resources.network!.id; + const intent = recordAttemptCreation(leasePath, active, 'build'); + const networkMode = requireAttemptNetwork(active); + build = attemptDocker(['create', '--name', intent.name, '--label', `${ATTEMPT_CREATION_LABEL}=${intent.creationToken}`, + '--network', networkMode, '--init', '--cap-drop', 'ALL', '--cap-add', 'CHOWN', '--cap-add', 'DAC_OVERRIDE', + '--cap-add', 'FOWNER', '--cap-add', 'KILL', + '--cap-add', 'SETUID', '--cap-add', 'SETGID', '--security-opt', 'no-new-privileges:true', + '--cpus', String(limits.cpuCount), '--memory', String(limits.memoryBytes), + '--memory-swap', String(limits.memorySwapBytes), '--pids-limit', String(limits.pids), + '--entrypoint', 'sleep', buildImage!, 'infinity']); + try { + updateBackendLease(leasePath, { token: lease.ownershipToken }, next => { + next.resources.buildContainer = { name: intent.name, id: build, image: buildImage!, owned: true, networkMode, + resourceLimits: limits }; return next; + }); + } catch (error) { + // This exact, newly created container has not started and has no app files. + attemptDocker(['rm', build]); throw error; + } + const hostConfig = JSON.parse(attemptDocker(['inspect', '--format', '{{json .HostConfig}}', build])); + const actualLimits = { cpuCount: hostConfig.NanoCpus / 1e9, memoryBytes: hostConfig.Memory, + memorySwapBytes: hostConfig.MemorySwap, pids: hostConfig.PidsLimit }; + evidence.resourceLimits = { declared: limits, observed: actualLimits }; + assert.deepEqual(hostConfig.CapAdd.map((capability: string) => capability.replace(/^CAP_/, '')).sort(), + ['CHOWN', 'DAC_OVERRIDE', 'FOWNER', 'KILL', 'SETGID', 'SETUID']); + assert.deepEqual(actualLimits, limits, 'lease limits must match the created container'); + attemptDocker(['start', build]); + assert.equal(exec(['id', '-u', CODING_CONTAINER_AGENT.name]), String(CODING_CONTAINER_AGENT.uid)); + assert.equal(exec(['id', '-g', CODING_CONTAINER_AGENT.name]), String(CODING_CONTAINER_AGENT.gid)); + exec(['sh', '-c', 'mkdir -p /app /run/application; chmod 777 /app /run/application']); + for (const [name, source] of [['fixture.mjs', fixture], ['start.sh', start]]) { + attemptDocker(['exec', '-i', build, 'sh', '-c', `cat > /app/${name}`], source); + } + exec(['chmod', '755', '/app/start.sh']); + process.env.STACK_BENCH_LEASE = leasePath; + process.env.STACK_BENCH_LEASE_TOKEN = lease.ownershipToken; + active = readBackendLease(leasePath, { token: lease.ownershipToken, active: true }); + evidence.lease = publicBackendLease(active); save(); + for (const mode of ['direct', 'proxy', 'late-auxiliary'] as const) await t.test(mode, async () => { + await prepareMode(mode); + const entry = json('/tmp/entry-ready.json'), api = mode === 'proxy' ? json('/tmp/api-ready.json') : null; + const before: CheckoutState = { accountId: 'a', itemId: 'i', priceMinor: 100, cart: [], + stock: [{ warehouseId: 'w', quantity: 10 }], reservations: [], orders: [], payments: [], orphanOrderLines: 0 }; + const prepared = structuredClone(before); prepared.cart.push({ itemId: 'i', quantity: 1 }); + const wrap = (state: CheckoutState) => ({ state: checkoutStateSchema.parse(state), schemaSha256: { fixture: sha(fixture) }, + account: 'a', item: 'i', recordedAtMs: Date.now() }); + let quietBeforeRecovery = false; + const result = await executeAction(ACTION_REGISTRY, 'crashCheckout', { + do: 'crashCheckout', actor: 'buyer', before: 'before', prepared: 'prepared', quantity: 1, + requests: 16, offsetMs: 0, target: 'application', + }, { capabilities: { + actors: { get: () => ({ name: 'buyer', page: { evaluate: async () => [] }, context: { cookies: async () => [] }, + writes: [{ url: `http://127.0.0.1:${port}/session`, headers: { authorization: 'Bearer fixture' } }] }) }, + 'named-actions': { now: Date.now, sleep: async () => {}, resolve: () => ({ id: 'checkout' }), + request: () => ({ url: `http://127.0.0.1:${port}/api/checkout` }), fetch }, + 'database-read': { checkoutSnapshots: new Map([['before', wrap(before)], ['prepared', wrap(prepared)]]), + markCheckoutUnsettled: () => {}, getCheckoutState: () => wrap(prepared) }, + 'browser-observation': { recorded: new Map() }, + 'process-crash': { prepare: async () => { + const runtime = await prepareRuntimeCrash(spec, 'application'); + return { ...runtime, crash: async () => { + await waitFor(async () => { + try { await asyncExec('docker', ['exec', build, 'test', '-s', '/tmp/entered']); return true; } + catch { return false; } + }, 10_000, 'real checkout arrival'); + if (mode === 'late-auxiliary') { + // This existing writer had no listening socket when ARMED. It + // must still belong to the complete writer freeze boundary. + exec(['touch', '/tmp/listen-now']); + await waitFor(async () => { try { json('/tmp/api-ready.json'); return true; } catch { return false; } }, + 5000, 'existing auxiliary becomes a listener after ARMED'); + } + return runtime.crash(); + }, recover: async (signal: AbortSignal) => { + const states = exec(['sh', '-c', `ps -u ${CODING_CONTAINER_AGENT.uid} -o stat= || [ "$?" = 1 ]`]); + quietBeforeRecovery = states.split(/\s+/).filter(Boolean).every(state => state.startsWith('Z')); + assert(quietBeforeRecovery, 'all application processes must be quiet before the real DB drain/restart'); + return runtime.recover(signal); + } }; + } }, + } }); + (evidence.cases as unknown[]).push({ mode, entry, api, quietBeforeRecovery, result }); save(); + assert.equal(result.status, 'passed', JSON.stringify(result)); + const observed = result.observation as { receipt: { + applicationFreeze?: { startedAtMs: number; completedAtMs: number; + processes: Array<{ pid: number; startTicks: number; threads: number[] }> }; + processEvidence: string; clockOffsetBeforeMs: number; + }; outcomes: Array<{ startedAtMs: number; completedAtMs: number }>; + databaseDrain: { settled: boolean }; outstandingAtFault: number }; + assert(quietBeforeRecovery && observed.databaseDrain.settled); + assert(observed.outstandingAtFault > 0); + const freeze = observed.receipt.applicationFreeze; + assert(freeze, 'application admission requires the verified all-writer freeze'); + assert(freeze.startedAtMs <= freeze.completedAtMs); + assert.deepEqual(freeze.processes.map(row => row.pid).sort((a, b) => a - b), + [entry.pid, ...entry.aux, ...(api ? [api.pid] : [])].sort((a, b) => a - b)); + for (const writer of freeze.processes) { + assert(writer.startTicks > 0 && writer.threads.includes(writer.pid)); + assert(writer.threads.every(tid => Number.isSafeInteger(tid) && tid > 1)); + } + assert(freeze.processes.find(row => row.pid === entry.pid)!.threads.length > 1, + 'Node worker threads must be verified as well as the process leader'); + const kills = [...observed.receipt.processEvidence.matchAll(/^KILLED \d+ \d+ (\d+)$/gm)].map(row => Number(row[1])); + assert(freeze.completedAtMs <= Math.min(...kills), 'no KILL may precede the all-stopped observation'); + assert(observed.outcomes.some(row => row.startedAtMs <= freeze.startedAtMs - observed.receipt.clockOffsetBeforeMs + && row.completedAtMs >= freeze.completedAtMs - observed.receipt.clockOffsetBeforeMs), + 'real held HTTP requests must span the full verified freeze'); + for (const pid of entry.aux) assert.match(observed.receipt.processEvidence, new RegExp(`^KILLED ${pid} `, 'm')); + assert.equal(sha(exec(['cat', '/app/fixture.mjs']) + '\n'), sha(fixture), 'frozen fixture source survives recovery'); + }); + await t.test('disarming before CRASH leaves the application running', async () => { + await prepareMode('direct'); + const entry = json('/tmp/entry-ready.json'); + const runtime = await prepareRuntimeCrash(spec, 'application'); + await runtime.close(); + const states = exec(['sh', '-c', `ps -u ${CODING_CONTAINER_AGENT.uid} -o stat=`]).split(/\s+/).filter(Boolean); + const healthy = await answers(`http://127.0.0.1:${port}/health`); + (evidence.cases as unknown[]).push({ mode: 'disarm', entry, states, healthy }); save(); + assert(healthy && states.length > 0 && states.every(state => !/^[TtZX]/.test(state))); + }); + for (const mode of ['missing', 'foreign', 'stale'] as const) await t.test(mode, async () => { + await prepareMode(mode === 'missing' ? 'missing' : 'direct', mode === 'foreign'); + const entry = json('/tmp/entry-ready.json'); + if (mode === 'foreign') attemptDocker(['exec', '-d', '--user', uid, build, 'sleep', 'infinity']); + let runtime: Awaited<ReturnType<typeof prepareRuntimeCrash>> | undefined; + try { + let error: unknown; + try { + runtime = await prepareRuntimeCrash(spec, 'application'); + if (mode === 'stale') { + exec(['kill', '-KILL', String(entry.pid)]); + attemptDocker(['exec', '-d', '--user', uid, '-e', `PORT=${port}`, build, 'node', '/app/fixture.mjs']); + await waitFor(() => answers(`http://127.0.0.1:${port}/health`), 10_000, 'replacement listener'); + await runtime.crash(); + } + } catch (caught) { error = caught; } + (evidence.cases as unknown[]).push({ mode, entry, rejected: !!error, + error: error instanceof Error ? error.message : null }); save(); + assert(error, `${mode} entry must not establish an application fault boundary`); + } finally { + await runtime?.close(); + if (mode === 'foreign') for (const pid of [entry.pid, ...entry.aux]) exec(['sh', '-c', `kill -KILL ${pid} 2>/dev/null || true`]); + killUser(); + } + }); + await t.test('a new listener after arming belongs to the complete writer freeze', async () => { + await prepareMode('direct'); + const entry = json('/tmp/entry-ready.json'); + const runtime = await prepareRuntimeCrash(spec, 'application'); + try { + attemptDocker(['exec', '-d', '--user', uid, '-e', `PORT=${port}`, build, 'node', '/app/fixture.mjs', 'api']); + await waitFor(async () => { try { json('/tmp/api-ready.json'); return true; } catch { return false; } }, + 5000, 'new listener after ARMED'); + exec(['kill', '-0', String(entry.pid)]); + const added = json('/tmp/api-ready.json'); + let receipt: unknown, failure: unknown; + try { receipt = await runtime.crash(); } catch (error) { + failure = error; receipt = error && typeof error === 'object' && 'receipt' in error ? error.receipt : null; + } + const state = exec(['sh', '-c', `ps -u ${CODING_CONTAINER_AGENT.uid} -o stat= || [ "$?" = 1 ]`]); + const quiet = state.split(/\s+/).filter(Boolean).every(value => value.startsWith('Z')); + (evidence.cases as unknown[]).push({ mode: 'added-listener', entry, added, quiet, receipt, + error: failure instanceof Error ? failure.message : null }); save(); + assert(quiet, 'new service must still be stopped by complete UID quiescence'); + assert.equal(failure, undefined, 'a new writer before the freeze is included, not silently omitted'); + const frozen = receipt as { applicationFreeze?: { processes: Array<{ pid: number }> } }; + assert(frozen.applicationFreeze?.processes.some(row => row.pid === added.pid), + 'the late writer must have a verified stopped-state receipt'); + } finally { await runtime.close(); killUser(); } + }); + await t.test('a terminated leader cannot hide a live owned worker', async () => { + await prepareMode('direct'); + // Perl and syscall.ph are already in the pinned coding image. SYS_exit + // ends only this thread; threads->exit would terminate the whole process. + const worker = `use threads; use Time::HiRes qw(sleep); +open my $p, ">", "/tmp/leader.pid" or die $!; print $p "$$\\n"; close $p; +threads->create(sub { my $n=0; while(1){ open my $f, ">", "/tmp/tick" or die $!; print $f ++$n; close $f; sleep 0.05; } }); +require 'syscall.ph'; sleep 0.1; syscall(&SYS_exit, 0); +`; + attemptDocker(['exec', '-i', build, 'sh', '-c', 'cat > /app/dead-leader.pl'], worker); + attemptDocker(['exec', '-d', '--user', uid, build, 'perl', '/app/dead-leader.pl']); + const observe = () => JSON.parse(exec(['node', '-e', `const fs=require('node:fs'); +const pid=Number(fs.readFileSync('/tmp/leader.pid','utf8')); +const state=p=>{try{return fs.readFileSync(p+'/stat','utf8').split(') ')[1].split(' ')[0]}catch{return null}}; +let tids=[];try{tids=fs.readdirSync('/proc/'+pid+'/task').map(Number)}catch{} +console.log(JSON.stringify({pid,state:state('/proc/'+pid),tick:fs.readFileSync('/tmp/tick','utf8'), +tasks:tids.map(tid=>({tid,state:state('/proc/'+pid+'/task/'+tid)}))}));`])) as { + pid: number; state: string | null; tick: string; tasks: Array<{ tid: number; state: string | null }>; + }; + await waitFor(async () => { try { + const row = observe(); return row.state === 'Z' && row.tasks.some(task => task.tid !== row.pid && task.state === 'S'); + } catch { return false; } }, 5000, 'terminated leader with active worker'); + const before = observe(); + const runtime = await prepareRuntimeCrash(spec, 'application'); + try { + let receipt: unknown, failure: unknown; + try { receipt = await runtime.crash(); } catch (error) { + failure = error; receipt = error && typeof error === 'object' && 'receipt' in error ? error.receipt : null; + } + const after = observe(); + (evidence.cases as unknown[]).push({ mode: 'dead-leader', fixtureSha256: sha(worker), before, after, receipt, + error: failure instanceof Error ? failure.message : null }); save(); + assert.equal(failure, undefined, 'a live worker with a terminated leader remains a supported writer'); + assert(after.tasks.every(task => task.state === null || ['Z', 'X'].includes(task.state)), + 'QUIET must not leave the owned worker running'); + const frozen = receipt as { applicationFreeze?: { processes: Array<{ pid: number; threads: number[] }> } }; + const group = frozen.applicationFreeze?.processes.find(row => row.pid === before.pid); + assert(group, 'the thread group must be in the freeze receipt'); + assert.deepEqual(group.threads, before.tasks.filter(task => task.state && !['Z', 'X'].includes(task.state)).map(task => task.tid)); + assert(!group.threads.includes(before.pid), 'the terminated leader is not a live frozen task'); + } finally { await runtime.close(); killUser(); } + }); + await t.test('respawn cannot bypass complete quiescence', async () => { + await prepareMode('direct'); + const entry = json('/tmp/entry-ready.json'); + const runtime = await prepareRuntimeCrash(spec, 'application'); + // One fixture supervisor outside the application UID starts a replacement + // after the initial listener dies. The real rescan must stop that writer. + const watcher = `const fs=require('node:fs'),{spawn}=require('node:child_process'); + fs.writeFileSync('/tmp/watch-ready','ready'); + const until=Date.now()+10000,t=setInterval(()=>{ + let gone=false;try{gone=/\\) [ZX] /.test(fs.readFileSync('/proc/${entry.pid}/stat','utf8'));}catch{gone=true;} + if(gone){clearInterval(t);const p=spawn('setpriv',['--reuid=${CODING_CONTAINER_AGENT.uid}', + '--regid=${CODING_CONTAINER_AGENT.gid}','--clear-groups','node','/app/fixture.mjs'], + {detached:true,stdio:'ignore'});fs.writeFileSync('/tmp/respawn.json',JSON.stringify({pid:p.pid}));p.unref();} + else if(Date.now()>until)process.exit(2); + },10);`; + attemptDocker(['exec', '-d', '-e', `PORT=${port}`, build, 'node', '-e', watcher]); + await waitFor(async () => { try { exec(['test', '-f', '/tmp/watch-ready']); return true; } catch { return false; } }, + 5000, 'one-shot fixture supervisor'); + try { + let receipt: unknown, failure: unknown; + try { receipt = await runtime.crash(); } catch (error) { + failure = error; receipt = error && typeof error === 'object' && 'receipt' in error ? error.receipt : null; + } + await waitFor(async () => { try { json('/tmp/respawn.json'); return true; } catch { return false; } }, + 5000, 'replacement process'); + const replacement = json('/tmp/respawn.json'); + const state = exec(['sh', '-c', `ps -u ${CODING_CONTAINER_AGENT.uid} -o stat= || [ "$?" = 1 ]`]); + const quiet = state.split(/\s+/).filter(Boolean).every(value => value.startsWith('Z')); + (evidence.cases as unknown[]).push({ mode: 'respawn', entry, replacement, quiet, receipt, + error: failure instanceof Error ? failure.message : null }); save(); + assert(quiet, 'a replacement writer must not survive the real all-user rescan'); + assert(failure, 'changed service identities must not establish a measured service boundary'); + } finally { await runtime.close(); killUser(); } + }); + await t.test('database boundary is unchanged', async () => { + await prepareMode('direct'); + const armed = await prepareProcessCrash(active, 'database'); + try { + const receipt = await armed.crash(); + const drain = await recoverRuntimeCrash(spec, 'database'); + (evidence.cases as unknown[]).push({ mode: 'database', receipt, drain }); save(); + assert.equal(receipt.target, 'database'); + assert.match(receipt.processEvidence, /^KILLED \d+ \d+ \d+$/m); + assert.match(receipt.processEvidence, /^QUIET$/m); + assert.equal(drain, null); + assert(await answers(`http://127.0.0.1:${port}/health`)); + } finally { await armed.close(); } + }); + evidence.result = 'cases finished; assertion results are in the accompanying TAP output'; + } catch (error) { + evidence.result = 'failed'; evidence.error = error instanceof Error ? error.message : String(error); + failure = { error }; + } + for (const [key, value] of [['STACK_BENCH_LEASE', previous.lease], ['STACK_BENCH_LEASE_TOKEN', previous.token]]) { + if (value === undefined) delete process.env[key!]; else process.env[key!] = value; + } + // A cleanup failure takes precedence over a case failure; both are in the evidence. + try { + const current = readBackendLease(leasePath, { token: lease.ownershipToken }); + const owned = [current.resources.buildContainer, current.resources.browserContainer, current.resources.container] + .flatMap(container => container ? [container.id] : []); + const released = releaseBackendLease(leasePath, lease.ownershipToken); + const remaining = owned.filter(id => attemptDocker(['ps', '-aq', '--filter', `id=${id}`])); + const networkRemaining = network ? attemptDocker(['network', 'ls', '-q', '--filter', `id=${network}`]) : ''; + evidence.cleanup = { released, network, owned, remaining, networkRemaining }; + assert(released && !remaining.length && !networkRemaining, 'all owned resources must be released'); + } catch (error) { + evidence.cleanupError = error instanceof Error ? error.message : String(error); throw error; + } finally { save(); } + if (failure) throw failure.error; +}); diff --git a/tools/stack-bench/tests/isolation.test.ts b/tools/stack-bench/tests/isolation.test.ts new file mode 100644 index 00000000000..88407bf730b --- /dev/null +++ b/tools/stack-bench/tests/isolation.test.ts @@ -0,0 +1,71 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { parseAgentArgs, refreshCodingInvocationCredentials } from '../commands/agent.js'; +import { compiledEntrypoint } from '../src/package-root.js'; + +const AGENT = compiledEntrypoint('commands', 'agent.js'); + +const args = (app: string) => [AGENT, '--mode', 'build', '--backend', 'spacetime', '--track', 'loop', + '--level', '1', '--app', app]; + +function isCommandFailure(error: unknown): error is Error & { status: number; stderr: unknown } { + return error instanceof Error + && typeof Reflect.get(error, 'status') === 'number' + && Reflect.has(error, 'stderr'); +} + +test('an unavailable isolation image refuses a coding session instead of falling back to the host', () => { + const app = mkdtempSync(join(tmpdir(), 'stack-bench-isolation-')); + try { + assert.throws(() => execFileSync(process.execPath, args(app), { + env: { ...process.env, STACK_BENCH_IMAGE: 'stack-bench-image-that-does-not-exist' }, + stdio: 'pipe', + }), error => isCommandFailure(error) && error.status === 2 + && /isolated build unavailable: cannot verify isolation image stack-bench-image-that-does-not-exist/.test(String(error.stderr)) + && /benchmark coding sessions require the isolation container/.test(String(error.stderr))); + } finally { rmSync(app, { recursive: true, force: true }); } +}); + +test('prompt review does not require Docker or mutate the application directory', () => { + const app = mkdtempSync(join(tmpdir(), 'stack-bench-isolation-')); + try { + const prompt = execFileSync(process.execPath, [...args(app), '--print-prompt'], { + env: { ...process.env, STACK_BENCH_IMAGE: 'stack-bench-image-that-does-not-exist' }, + encoding: 'utf8', stdio: 'pipe', + }); + assert.match(prompt, /Build the application described below/); + assert.equal(readdirSync(app).length, 0); + } finally { rmSync(app, { recursive: true, force: true }); } +}); + +test('agent arguments reject invalid modes and partial numbers', () => { + const base = ['node', 'agent', '--backend', 'postgres', '--app', 'app']; + const codex = [...base, '--mode', 'build', '--provider', 'openai']; + assert.throws(() => parseAgentArgs(codex), /--model is required/); + assert.equal(parseAgentArgs([...codex, '--model', 'gpt-5.3-codex']).provider, 'openai'); + assert.throws(() => parseAgentArgs([...base, '--mode', 'typo']), /--mode must be/); + assert.throws(() => parseAgentArgs([...base, '--mode', 'build', '--level', '2junk']), + /--level must be a positive integer/); + assert.throws(() => parseAgentArgs([...base, '--mode', 'build', '--run-index', '1junk']), + /--run-index must be a non-negative integer/); +}); + +test('coding invocation reloads the selected key file and refuses a billing-mode switch', () => { + const directory = mkdtempSync(join(tmpdir(), 'stack-bench-refresh-')); + const keyFile = join(directory, 'key'); + try { + writeFileSync(keyFile, 'first-fake-key'); + const first = refreshCodingInvocationCredentials({ provider: 'anthropic', keyFile, + expectedMode: null, env: {} }); + writeFileSync(keyFile, 'renewed-fake-key'); + const second = refreshCodingInvocationCredentials({ provider: 'anthropic', keyFile, + expectedMode: first.mode, env: {} }); + assert.equal(second.env.STACK_BENCH_AGENT_API_KEY, 'renewed-fake-key'); + assert.throws(() => refreshCodingInvocationCredentials({ provider: 'anthropic', + expectedMode: first.mode, env: { CLAUDE_CODE_OAUTH_TOKEN: 'fake-subscription-token' } }), /billing mode changed/); + } finally { rmSync(directory, { recursive: true, force: true }); } +}); diff --git a/tools/stack-bench/tests/item-navigation.integration.ts b/tools/stack-bench/tests/item-navigation.integration.ts new file mode 100644 index 00000000000..32b2a61c5e2 --- /dev/null +++ b/tools/stack-bench/tests/item-navigation.integration.ts @@ -0,0 +1,722 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { createServer } from 'node:http'; +import { once } from 'node:events'; +import { dirname, join } from 'node:path'; +import test from 'node:test'; +import { chromium } from 'playwright'; +import { ACTION_REGISTRY } from '../src/actions/action-catalog.js'; +import { executeAction } from '../src/actions/action-contract.js'; +import { stableElementSelector } from '../src/actions/element-selector.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; +import { gradeFeature } from '../grader/grade.js'; + +test('warehouse inventory accepts consistent views and rejects missing or conflicting stock', async t => { + // Failure first: exercise the real 7b criterion against complete and defective + // projections. Authentication and 7a are outside this already-open admin view. + const scenarioPath = 'tracks/ecommerce/scenarios/01-warehouse-admin-staff.json'; + const fixturePath = 'tracks/ecommerce/composition/fixtures/operations.json'; + const scenario = readFileSync(join(STACK_BENCH_ROOT, scenarioPath), 'utf8'); + const fixtureSource = readFileSync(join(STACK_BENCH_ROOT, fixturePath), 'utf8'); + const fixture = JSON.parse(fixtureSource) as { + warehouses: string[]; items: Array<{ name: string; stock: Record<string, number> }>; + }; + const compiled = compileScenarioDefinition(JSON.parse(scenario)).features[0]!; + const feature = { ...compiled, actors: ['admin'], setup: [], + criteria: compiled.criteria.filter(criterion => criterion.id === '7b').map(criterion => ({ + ...criterion, steps: criterion.steps.map(step => ({ ...step, within: 250 })), + })) }; + const cases = [ + { mode: 'single', expected: 'passed' }, + { mode: 'reordered-labelled', expected: 'passed' }, + { mode: 'split-inline-name', expected: 'passed' }, + { mode: 'block-separated-name', expected: 'passed' }, + { mode: 'duplicate-items', expected: 'passed' }, + { mode: 'duplicate-locations', expected: 'passed' }, + { mode: 'nested-warehouse-label', expected: 'passed' }, + { mode: 'nested-holding-label', expected: 'passed' }, + { mode: 'nested-item-label', expected: 'passed' }, + { mode: 'nested-label-quantity', expected: 'passed' }, + { mode: 'nested-label-wrong-quantity', expected: 'failed' }, + { mode: 'nested-label-wrong-warehouse', expected: 'failed' }, + { mode: 'nested-holding-number-only', expected: 'passed' }, + { mode: 'missing-item-equal-count', expected: 'failed' }, + { mode: 'wrong-non-mouse-total', expected: 'failed' }, + { mode: 'wrong-item-copy-first', expected: 'failed' }, + { mode: 'wrong-item-copy-last', expected: 'failed' }, + { mode: 'missing-location-equal-count', expected: 'failed' }, + { mode: 'wrong-location', expected: 'failed' }, + { mode: 'swapped-holdings', expected: 'failed' }, + { mode: 'wrong-location-copy', expected: 'failed' }, + { mode: 'missing-warehouse', expected: 'failed' }, + { mode: 'missing-number', expected: 'failed' }, + { mode: 'missing-copy-number', expected: 'failed' }, + { mode: 'malformed-number', expected: 'failed' }, + { mode: 'hidden-item', expected: 'failed' }, + { mode: 'empty', expected: 'failed' }, + { mode: 'outside-panel', expected: 'failed' }, + { mode: 'hidden-panel', expected: 'failed' }, + { mode: 'unknown-item', expected: 'failed' }, + { mode: 'substring-item', expected: 'failed' }, + { mode: 'unknown-warehouse', expected: 'failed' }, + { mode: 'nested-number-only', expected: 'passed' }, + { mode: 'item-wrapper-empty', expected: 'passed' }, + { mode: 'item-wrapper-copies', expected: 'passed' }, + { mode: 'item-wrapper-labelled', expected: 'passed' }, + { mode: 'item-wrapper-wrong-name', expected: 'failed' }, + { mode: 'item-wrapper-wrong-number', expected: 'failed' }, + { mode: 'hidden-number', expected: 'failed' }, + { mode: 'late-snapshot', expected: 'inconclusive' }, + ]; + let mode = ''; + const evidence: unknown[] = []; + const server = createServer((_request, response) => { + const items: Array<{ name: string; total?: number | string; hidden?: boolean }> = fixture.items.map(item => ({ + name: item.name, total: Object.values(item.stock).reduce((sum, value) => sum + value, 0), + })); + const locations = fixture.items.flatMap(item => fixture.warehouses.map(warehouse => ({ + name: item.name, warehouse, quantity: item.stock[warehouse]!, + }))); + let warehouses = [...fixture.warehouses]; + if (mode === 'reordered-labelled') { items.reverse(); locations.reverse(); warehouses.reverse(); } + if (mode === 'duplicate-items') items.push(...items.map(item => ({ ...item }))); + if (mode === 'duplicate-locations') locations.push(...locations.map(location => ({ ...location }))); + if (mode === 'missing-item-equal-count') { items.pop(); items.push({ ...items[0]! }); } + if (mode === 'wrong-non-mouse-total') items[0]!.total = Number(items[0]!.total) + 1; + if (mode === 'wrong-item-copy-first') items.unshift({ ...items[0]!, total: Number(items[0]!.total) + 1 }); + if (mode === 'wrong-item-copy-last') items.push({ ...items[0]!, total: Number(items[0]!.total) + 1 }); + if (mode === 'missing-location-equal-count') { locations.pop(); locations.push({ ...locations[0]! }); } + if (mode === 'wrong-location') locations[0]!.quantity++; + if (mode === 'swapped-holdings') { + [locations[0]!.quantity, locations[1]!.quantity] = [locations[1]!.quantity, locations[0]!.quantity]; + } + if (mode === 'wrong-location-copy') locations.push({ ...locations[0]!, quantity: locations[0]!.quantity + 1 }); + if (mode === 'missing-warehouse') warehouses = warehouses.filter(warehouse => warehouse !== 'West'); + if (mode === 'missing-number') delete items[0]!.total; + if (mode === 'missing-copy-number') items.push({ name: items[0]!.name }); + if (mode === 'malformed-number') items[0]!.total = 'unknown'; + if (mode === 'hidden-item') items.at(-1)!.hidden = true; + if (mode === 'empty') { items.length = 0; locations.length = 0; warehouses = []; } + response.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' }); + let html = `<!doctype html><main><span id="current-user">admin</span><section id="admin-panel"> + <h1>Warehouse inventory</h1><section aria-label="Items">${items.map(item => + `<article data-role="admin-item-row" ${item.hidden ? 'hidden' : ''}><strong>${item.name}</strong> + ${item.total === undefined ? '' : `<span data-role="admin-stock">${item.total}</span>`} + ${mode === 'reordered-labelled' ? '<small> units available · Inventory summary</small>' : ''}</article>`).join('')}</section> + ${warehouses.map(warehouse => `<section data-role="admin-warehouse-item"><h2>${warehouse}</h2> + ${locations.filter(location => location.warehouse === warehouse).map(location => + `<article data-role="admin-location-row">${mode === 'reordered-labelled' + ? `${location.warehouse} warehouse · ${location.name}` : `${location.name} · ${location.warehouse}`} + <span data-role="admin-location-qty">${location.quantity}</span></article>`).join('')}</section>`).join('')} + </section></main>`; + if (mode === 'outside-panel') html = html.replace('<section id="admin-panel">', '<section id="admin-panel"></section><section>'); + if (mode === 'split-inline-name') html = html.replaceAll('Headphones', '<span>Head</span>phones'); + if (mode === 'block-separated-name') html = html.replace('<strong>Air Purifier</strong>', '<div>Air</div><div>Purifier</div>'); + if (mode === 'hidden-panel') html = html.replace('<section id="admin-panel">', '<section id="admin-panel" hidden>'); + if (mode === 'unknown-item') html = html.replace('<h1>', '<article data-role="admin-item-row">Unknown item<span data-role="admin-stock">100</span></article><h1>'); + if (mode === 'substring-item') html = html.replaceAll('Air Purifier', 'Air PurifierPlus'); + if (mode === 'unknown-warehouse') html = html.replace('<h1>', '<section data-role="admin-warehouse-item">North</section><h1>'); + if (mode === 'nested-number-only') html = html.replace('<span data-role="admin-stock">100</span>', + '<article data-role="admin-item-row">Air Purifier<span data-role="admin-stock">100</span></article>'); + if (mode.startsWith('item-wrapper-')) html = html.replace( + /<article data-role="admin-item-row" >(<strong>Air Purifier<\/strong>\s*<span data-role="admin-stock">100<\/span>\s*)<\/article>/, + `<article data-role="admin-item-row">${mode === 'item-wrapper-wrong-name' ? 'Keyboard' : mode === 'item-wrapper-labelled' ? 'Air Purifier' : ''}${mode === 'item-wrapper-wrong-number' ? 'Air Purifier<span data-role="admin-stock">101</span>' : ''}<article data-role="admin-item-row">$1${mode === 'item-wrapper-labelled' ? ' units available' : ''}</article>${mode === 'item-wrapper-copies' ? '<article data-role="admin-item-row">$1</article>' : ''}</article>`); + if (mode === 'hidden-number') html = html.replace('<span data-role="admin-stock">100</span>', '<span data-role="admin-stock" hidden>100</span>'); + if (mode === 'nested-item-label') html = html.replace('<strong>Air Purifier</strong>', '<span data-role="admin-warehouse-item">Air Purifier · East</span>'); + if (mode.startsWith('nested-') && mode !== 'nested-number-only') { + html = html.replace(/(<article data-role="admin-location-row">)([^<]+) · (East|West)(\s*)(<span data-role="admin-location-qty">\d+<\/span>)/g, + (_match, opening, item, warehouse, spacing, quantity) => { + if (mode === 'nested-holding-number-only') return `${opening}<article data-role="admin-location-row">${item} · ${warehouse}${quantity}</article>`; + const label = mode === 'nested-holding-label' ? `${item} · ${warehouse}` : warehouse; + const prefix = mode === 'nested-holding-label' ? '' : `${item} · `; + return `${opening}${prefix}<span data-role="admin-warehouse-item">${label}${mode === 'nested-label-quantity' ? ` ${quantity}` : ''}</span>${spacing}${mode === 'nested-label-quantity' ? '' : quantity}`; + }); + if (mode === 'nested-label-wrong-quantity') html = html.replace('<span data-role="admin-location-qty">60</span>', '<span data-role="admin-location-qty">61</span>'); + if (mode === 'nested-label-wrong-warehouse') html = html.replace('<span data-role="admin-warehouse-item">East</span>', '<span data-role="admin-warehouse-item">West</span>'); + } + if (mode === 'late-snapshot') html += `<script> + const control = document.querySelector('[data-role="admin-stock"]'); + const measure = control.getBoundingClientRect.bind(control); + control.getBoundingClientRect = () => { + const end = performance.now() + 350; while (performance.now() < end) {} + control.getBoundingClientRect = measure; return measure(); + }; + </script>`; + response.end(html); + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const browser = await chromium.launch({ headless: true }); + try { + for (const entry of cases) await t.test(entry.mode, async () => { + mode = entry.mode; + const grade = await gradeFeature(browser, feature, + { url, level: 2, headed: false, selectedCheckKeys: [], nullControl: false }, + { runId: `warehouse-view-${mode}`, roomName: name => name, url, actions: [], + spacetime: null, backend: 'postgres', nullControl: false, defaultWithin: 250 }); + const row = { ...entry, grade }; evidence.push(row); + assert.equal(grade.setupEvidence?.status, 'passed', JSON.stringify(row)); + assert.equal(grade.criteria[0]!.evidence.status, entry.expected, JSON.stringify(row)); + if (entry.expected === 'failed') { + assert.equal(grade.criteria[0]!.evidence.code, 'application_failure', JSON.stringify(row)); + } + if (entry.expected === 'inconclusive') { + assert.equal(grade.criteria[0]!.evidence.finding?.kind, 'observation-window-missed', JSON.stringify(row)); + } + }); + } finally { + await browser.close(); server.closeAllConnections(); + await new Promise<void>(resolve => server.close(() => resolve())); + if (process.env.STACK_BENCH_WAREHOUSE_VIEW_EVIDENCE) { + const file = process.env.STACK_BENCH_WAREHOUSE_VIEW_EVIDENCE; + mkdirSync(dirname(file), { recursive: true }); + const files = [scenarioPath, fixturePath, 'dist/tests/item-navigation.integration.js', + 'dist/src/actions/browser-action-executors.js', 'dist/src/composition/definition-compiler.js']; + writeFileSync(file, JSON.stringify({ + rerun: 'STACK_BENCH_WAREHOUSE_VIEW_EVIDENCE=<file> node --test --test-name-pattern="warehouse inventory accepts" dist/tests/item-navigation.integration.js', + browserVersion: browser.version(), fixture, scenario: JSON.parse(scenario), + deadlinePolicy: 'A first read that overruns is inconclusive. Expiry preserves the last timely mismatch, even if a later snapshot matches; late evidence never establishes a pass.', + sourceSha256: Object.fromEntries(files.map(path => [path, + createHash('sha256').update(readFileSync(join(STACK_BENCH_ROOT, path))).digest('hex')])), + cases, evidence, + }, null, 2)); + } + } +}); + +test('warehouse totals validate every visible copy against one recorded baseline', async t => { + const scenarioPath = 'tracks/ecommerce/scenarios/02-strengthened.json'; + const scenario = compileScenarioDefinition(JSON.parse(readFileSync(join(STACK_BENCH_ROOT, scenarioPath), 'utf8'))); + const evidence: unknown[] = []; + const browser = await chromium.launch({ headless: true }); + const cases = [ + ['2a', 'single', 'passed'], ['2a', 'copies', 'passed'], ['2a', 'hidden-copy', 'passed'], + ['2a', 'outside-copy', 'passed'], ['2a', 'missing-total', 'failed'], + ['2a', 'bad-baseline', 'failed'], ['2a', 'nonnumeric', 'failed'], ['2a', 'nested-borrow', 'passed'], + ['2a', 'nested-empty', 'passed'], ['2a', 'nested-copies', 'passed'], + ['2a', 'nested-labelled', 'passed'], + ['2a', 'nested-wrong-name', 'failed'], ['2a', 'nested-wrong-number', 'failed'], + ['2a', 'wrong-copy', 'failed'], ['2a', 'reverse', 'failed'], ['2a', 'refused', 'failed'], + ['2a', 'missing-warehouse', 'failed'], ['2a', 'wrong-identity', 'failed'], + ['201a', 'copies', 'passed'], ['201a', 'unauthorized-effect', 'failed'], + ['2a', 'late-record', 'inconclusive'], ['2a', 'late-expect', 'inconclusive'], + ['2a', 'late-correction', 'failed'], + ]; + try { + for (const [criterion, mode, expected] of cases) await t.test(`${criterion}/${mode}`, async () => { + const source = scenario.features.flatMap(feature => feature.criteria).find(item => item.id === criterion)!.steps; + const steps = source.filter(step => step.testid === 'warehouse-total' + || step.do === 'expect' && step.testid === 'admin-warehouse-item'); + const boundary = steps.findIndex(step => step.do === 'expectNumber'); + const page = await browser.newPage(); + const recorded = new Map<string, number>(); + const actions: unknown[] = []; + let result: Awaited<ReturnType<typeof executeAction>> | undefined; + const row = (name: string, value: number | string | null, extra = '') => `<section data-role="admin-warehouse-item" ${extra}>${name} + ${value === null ? '' : `<span data-role="warehouse-total" data-warehouse="${name}">${value}</span>`}</section>`; + let extra = ''; + if (['copies', 'wrong-copy', 'unauthorized-effect'].includes(mode!)) extra = row('East', 600) + row('West', 400); + if (mode === 'missing-total') extra = row('East', null); + if (mode === 'bad-baseline' || mode === 'late-correction') extra = row('East', 601); + if (mode === 'nonnumeric') extra = row('East', 'unknown'); + if (mode === 'hidden-copy') extra = row('East', 999, 'hidden'); + if (mode === 'nested-borrow') extra = `<section data-role="admin-warehouse-item">East ${row('East', 600)}</section>`; + if (mode === 'nested-empty') extra = `<section data-role="admin-warehouse-item">${row('East', 600)}</section>`; + if (mode === 'nested-copies') extra = `<section data-role="admin-warehouse-item">East ${row('East', 600)}${row('East', 600)}</section>`; + if (mode === 'nested-labelled') extra = `<section data-role="admin-warehouse-item">East ${row('East warehouse', 600).replace('data-warehouse="East warehouse"', 'data-warehouse="East"')}</section>`; + if (mode === 'nested-wrong-name') extra = `<section data-role="admin-warehouse-item">East ${row('West', 400)}</section>`; + if (mode === 'nested-wrong-number') extra = `<section data-role="admin-warehouse-item">East <span data-role="warehouse-total">601</span>${row('East', 600)}</section>`; + try { + await page.setContent(`<section id="admin-panel"> + ${row(mode === 'wrong-identity' ? 'Eastside' : 'East', 600)} + ${mode === 'missing-warehouse' ? '' : row('West', 400)}${extra}</section> + ${mode === 'outside-copy' ? row('East', 999) : ''}<button id="apply-transfer">Apply</button> + <script> + let delayRead = ${mode === 'late-record'}; + const control = document.querySelector('[data-role="warehouse-total"]'); + const measure = control.getBoundingClientRect.bind(control); + window.totalSnapshots = 0; + control.getBoundingClientRect = () => { + window.totalSnapshots++; + if (${mode === 'late-correction'} && window.totalSnapshots === 2) { + document.querySelectorAll('[data-warehouse="East"]')[1].textContent = '600'; delayRead = true; + } + if (delayRead) { delayRead = false; const end = performance.now() + 250; while (performance.now() < end) {} } + return measure(); + }; + document.querySelector('button').onclick = () => { + delayRead = ${mode === 'late-expect'}; + for (const node of document.querySelectorAll('#admin-panel [data-role="warehouse-total"]')) { + const delta = ${criterion === '201a'} ? ${mode === 'unauthorized-effect' ? 10 : 0} : ${mode === 'refused' ? 0 : mode === 'reverse' ? -10 : 10}; + node.textContent = Number(node.textContent) + (node.dataset.warehouse === 'East' ? -delta : delta); + } + if (${mode === 'wrong-copy'}) document.querySelectorAll('[data-warehouse="East"]')[1].textContent = '600'; + };</script>`); + const actor = { page, loc: (id: string, options?: { contains?: string; scope?: { testid: string; contains?: string } }) => { + const scope = options?.scope; + const root = scope ? page.locator(stableElementSelector(scope.testid)).filter({ hasText: scope.contains }).first() : page; + const locator = root.locator(stableElementSelector(id)).filter({ visible: true }); + return (options?.contains ? locator.filter({ hasText: options.contains }) : locator).first(); + } }; + const service = { defaultWithin: 150, recorded, expand: (value: string) => value, testId: stableElementSelector, + sleep: (ms: number) => new Promise(resolve => setTimeout(resolve, Math.min(ms, 10))) }; + const capabilities = { actors: { get: () => actor }, 'browser-interaction': service, 'browser-observation': service }; + for (const [index, step] of steps.entries()) { + if (index === boundary) await page.locator('#apply-transfer').click(); + result = await executeAction(ACTION_REGISTRY, step.do, { ...step, within: 150 }, { capabilities }); + actions.push(result); + if (result.status !== 'passed') break; + } + const snapshots = await page.evaluate(() => (window as unknown as { totalSnapshots: number }).totalSnapshots); + evidence.push({ criterion, mode, expected, snapshots, recorded: [...recorded], actions }); + assert.equal(result?.status, expected, JSON.stringify(evidence.at(-1))); + if (expected === 'failed') { + assert.equal(result?.code, 'application_failure'); + assert.equal(result?.action.id, ['wrong-copy', 'reverse', 'refused', 'unauthorized-effect'].includes(mode!) + ? 'expectNumber' : 'recordNumber', 'fail at the missing or wrong owned total, not row cardinality'); + if (mode === 'late-correction') { + assert.equal(snapshots, 2, 'one timely mismatch must precede the delayed correct snapshot'); + assert.equal(result?.finding?.kind, 'number-mismatch'); + } + } + if (expected === 'inconclusive') { + assert.equal(result?.finding?.kind, 'observation-window-missed'); + assert.equal(result?.action.id, mode === 'late-record' ? 'recordNumber' : 'expectNumber'); + } + } finally { await page.close(); } + }); + } finally { + await browser.close(); + if (process.env.STACK_BENCH_WAREHOUSE_VIEW_EVIDENCE) { + const file = process.env.STACK_BENCH_WAREHOUSE_VIEW_EVIDENCE; + mkdirSync(dirname(file), { recursive: true }); + writeFileSync(file, JSON.stringify({ + rerun: 'STACK_BENCH_WAREHOUSE_VIEW_EVIDENCE=<file> node --test --test-name-pattern="warehouse totals validate" dist/tests/item-navigation.integration.js', + fixture: 'Compiled 2a/201a numeric browser slices. A button models the confirmed effect; native database and authorization checks are outside this fixture.', + deadlinePolicy: 'A first read that overruns is inconclusive. Expiry preserves the last timely mismatch, even if a later snapshot matches; late evidence never establishes a pass.', + sourceSha256: Object.fromEntries([scenarioPath, 'dist/tests/item-navigation.integration.js', + 'dist/src/actions/browser-action-executors.js'].map(path => [path, + createHash('sha256').update(readFileSync(join(STACK_BENCH_ROOT, path))).digest('hex')])), evidence, + }, null, 2) + '\n'); + } + } +}); + +test('catalog ranking distinguishes exact name hooks from surrounding decoration', async () => { + // Failure cases: scoped fallback requires the delivered contract and cannot + // conceal hidden results, wrong visible primary contents, or a stale observer. + const names = ['Air Purifier', 'Bluetooth Speaker', 'Coffee Grinder', 'Desk Lamp', + 'Espresso Machine', 'Gaming Mouse', 'Headphones', 'Induction Cooktop', 'Keyboard', 'Laptop Stand']; + const scenario = readFileSync(join(STACK_BENCH_ROOT, + 'tracks/ecommerce/scenarios/01-catalog-ranking.json'), 'utf8'); + const compiled = compileScenarioDefinition(JSON.parse(scenario)).features[0]!; + const feature = { ...compiled, setup: compiled.setup.map(step => ({ ...step, within: 250 })), + criteria: compiled.criteria.map(criterion => ({ ...criterion, + steps: criterion.steps.map(step => ({ ...step, within: 250 })), + })) }; + const liveSource = JSON.parse(readFileSync(join(STACK_BENCH_ROOT, + 'tracks/ecommerce/scenarios/01-core.json'), 'utf8')); + const live = compileScenarioDefinition(liveSource).features.find(candidate => candidate.id === 2)!; + // Exercise the registered two-client ranking steps without unrelated auth/stock setup. + const liveFeature = { ...live, actors: ['buyer', 'visitor'], setup: [], + criteria: live.criteria.filter(criterion => criterion.id === '2c').map(criterion => ({ + ...criterion, steps: criterion.steps.filter(step => step.do !== 'expectAgreement') + .map(step => ({ ...step, within: 500 })), + })) }; + let mode = 'plain'; + let purchased = false; + const evidence: unknown[] = []; + const server = createServer((request, response) => { + if (request.url === '/buy') { + purchased = true; response.writeHead(204); response.end(); return; + } + if (request.url === '/ranking') { + response.writeHead(200, { 'Content-Type': 'application/json' }); + response.end(JSON.stringify(purchased ? [names[2], ...names.filter(name => name !== names[2])] : names)); + return; + } + const rows = [...names]; + if (mode.endsWith('wrong-order')) [rows[0], rows[1]] = [rows[1]!, rows[0]!]; + if (mode.endsWith('missing')) rows.pop(); + if (mode.endsWith('duplicate')) rows[9] = rows[0]!; + if (mode === 'changed-name') rows[0] = 'Air Purifier Pro'; + const cards = (values: string[]) => values.map(name => + `<article data-role="item-card"><button><span data-role="item-name">${name}${mode === 'inside-icon' ? ' ↗' : ''}</span>${mode === 'outside-icon' ? '<span aria-hidden="true"> ↗</span>' : ''}</button><button data-role="buy-now" onclick="fetch('/buy')">Buy</button></article>`).join(''); + const fallback = mode.startsWith('fallback') || mode === 'no-gate' || mode === 'hidden-only' + || mode === 'live' || mode === 'stale'; + let html = mode === 'fallback-absent' ? '' + : `<section id="item-list" ${fallback ? 'hidden' : ''}>${cards(mode === 'primary-wrong' ? [...rows].reverse() : rows)}</section>`; + if (fallback || mode === 'primary-wrong') html += `<section id="search-results" ${mode === 'hidden-only' ? 'hidden' : ''}>${cards(rows)}</section>`; + if (mode === 'nested') html = `<section id="search-results">${html}</section>`; + if (mode === 'live' || mode === 'stale') html += `<script> + const update = ${cards.toString()}; + const mode = ${JSON.stringify(mode)}; + let previous = ${JSON.stringify(JSON.stringify(names))}; + if (mode === 'live') setInterval(async () => { + const rows = await (await fetch('/ranking')).json(), next = JSON.stringify(rows); + if (next !== previous) document.querySelector('#search-results').innerHTML = update(rows); + previous = next; + }, 30); + </script>`; + response.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' }); + response.end(html); + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const browser = await chromium.launch({ headless: true }); + try { + for (mode of ['plain', 'outside-icon', 'inside-icon', 'wrong-order', 'missing', 'duplicate', 'changed-name', + 'fallback', 'fallback-absent', 'no-gate', 'nested', 'hidden-only', 'primary-wrong', + 'fallback-wrong-order', 'fallback-missing', 'fallback-duplicate', 'live', 'stale']) { + purchased = false; + const expected = ['plain', 'outside-icon', 'fallback', 'fallback-absent', 'nested', 'live'].includes(mode) ? 'passed' : 'failed'; + const gated = !['plain', 'outside-icon', 'inside-icon', 'wrong-order', 'missing', 'duplicate', 'changed-name', 'no-gate'].includes(mode); + const grade = await gradeFeature(browser, mode === 'live' || mode === 'stale' ? liveFeature : feature, + { url, level: 1, headed: false, selectedCheckKeys: [], nullControl: false }, + { runId: 'catalog-name-contract', roomName: name => name, url, actions: [], + spacetime: null, backend: 'postgres', nullControl: false, defaultWithin: 250, + contractIds: gated ? ['ecommerce.progression.faceted-search-hooks'] : [] }); + evidence.push({ mode, expected, gated, grade }); + if (mode === 'stale') assert.equal(purchased, true, 'the stale observer case must reach the purchase'); + assert.equal(grade.criteria[0]!.evidence.status, expected, JSON.stringify(evidence.at(-1))); + } + } finally { + await browser.close(); server.closeAllConnections(); + await new Promise<void>(resolve => server.close(() => resolve())); + if (process.env.STACK_BENCH_ITEM_NAVIGATION_EVIDENCE) { + const file = process.env.STACK_BENCH_ITEM_NAVIGATION_EVIDENCE; + mkdirSync(dirname(file), { recursive: true }); + writeFileSync(file, JSON.stringify({ + rerun: 'STACK_BENCH_ITEM_NAVIGATION_EVIDENCE=<file> node --test --test-name-pattern="catalog ranking distinguishes" dist/tests/item-navigation.integration.js', + browserVersion: browser.version(), names, scenario: JSON.parse(scenario), liveSource, evidence, + }, null, 2)); + } + } +}); + +test('restock race accepts an open catalog and still requires catalog contents', async () => { + const definition = compileScenarioDefinition(JSON.parse(readFileSync(join(STACK_BENCH_ROOT, + 'tracks/ecommerce/scenarios/01-restock-race.json'), 'utf8'))); + const feature = definition.features[0]!; + const steps = [...feature.setup, ...feature.criteria.flatMap(criterion => criterion.steps)] + .filter(step => step.do === 'click' && step.testid === 'catalog-link'); + const browser = await chromium.launch({ headless: true }); + const evidence: unknown[] = []; + try { + const page = await browser.newPage(); + const actor = { page, loc: (id: string) => page.locator(stableElementSelector(id)).filter({ visible: true }) }; + const service = { defaultWithin: 150, expand: (text: string) => text, testId: stableElementSelector, + sleep: (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) }; + const capabilities = { actors: { get: () => actor }, 'browser-interaction': service, 'browser-observation': service }; + for (const layout of ['inline', 'closed', 'missing']) for (const step of steps) { + await page.setContent(`${layout === 'closed' ? '<button id="catalog-link" onclick="document.querySelector(\'#item-list\').hidden=false">Catalog</button>' : ''} + ${layout === 'missing' ? '' : `<section id="item-list" ${layout === 'closed' ? 'hidden' : ''}>Products</section>`}`); + const clicked = await executeAction(ACTION_REGISTRY, 'click', { ...step, within: 150 }, { capabilities }); + const observed = clicked.status === 'passed' ? await executeAction(ACTION_REGISTRY, 'expect', + { do: 'expect', actor: step.actor, testid: 'item-list', contains: 'Products', within: 150 }, { capabilities }) : clicked; + evidence.push({ layout, step, clicked, observed }); + assert.equal(observed.status, layout === 'missing' ? 'failed' : 'passed', JSON.stringify(evidence.at(-1))); + } + } finally { + await browser.close(); + if (process.env.STACK_BENCH_ITEM_NAVIGATION_EVIDENCE) writeFileSync(process.env.STACK_BENCH_ITEM_NAVIGATION_EVIDENCE, + JSON.stringify({ rerun: 'node --test --test-name-pattern="restock race accepts" dist/tests/item-navigation.integration.js', evidence }, null, 2)); + } +}); + +test('catalog management reaches a declared inner tab and verifies the created product', async t => { + // Failure first: a valid inner tab must work; broken navigation and missing fields must not pass. + let layout = '', products: Array<{ name: string; variants: string[] }> = []; + const visits: string[] = [], evidence: unknown[] = []; + const server = createServer(async (request, response) => { + const path = new URL(request.url!, 'http://fixture.test').pathname; + visits.push(`${request.method} ${path}`); + if (path === '/save') { + let body = ''; for await (const chunk of request) body += String(chunk); + const form = new URLSearchParams(body); + products.push({ name: form.get('name')!, variants: form.get('variants')!.split(',').map(v => v.trim()) }); + response.writeHead(303, { Location: '/admin/catalog' }); response.end(); return; + } + response.writeHead(200, { 'Content-Type': 'text/html' }); + response.end(`<!doctype html><main></main><script> + const layout=${JSON.stringify(layout)}, path=${JSON.stringify(path)}, products=${JSON.stringify(products)}; + const user=sessionStorage.getItem('user'), main=document.querySelector('main'); + main.innerHTML='<input id="search-input"><section id="item-list">'+products.map(p=>'<article data-role="item-card"><span data-role="item-name">'+p.name+'</span>'+p.variants.map(v=>'<span data-role="item-variant">'+v+'</span>').join('')+'</article>').join('')+'</section>'; + if (!user) { + main.innerHTML+='<form id="login"><input id="signin-username"><input id="signin-password"><button id="signin-submit">Sign in</button></form>'; + document.querySelector('#login').onsubmit=e=>{e.preventDefault();sessionStorage.setItem('user',document.querySelector('#signin-username').value);location.href='/';}; + } else { + main.innerHTML+='<a id="current-user" href="/account">'+user+'</a>'; + const area=path.startsWith('/admin')||layout==='already-open'; + if (!area&&(layout!=='menu-tab'||path==='/account')) main.innerHTML+='<a id="admin-link" href="/admin">Admin</a>'; + const tab=['menu-tab','broken-tab','missing-tab'].includes(layout); + if (area&&tab&&path!=='/admin/catalog'&&layout!=='missing-tab') main.innerHTML+=layout==='broken-tab' + ?'<button id="catalog-management-link">Products</button>':'<a id="catalog-management-link" href="/admin/catalog">Products</a>'; + if (area&&(!tab||path==='/admin/catalog')) main.innerHTML+='<form method="post" action="/save"><input id="catalog-name" name="name"><input id="catalog-category" name="category">'+(layout==='missing-field'?'':'<input id="catalog-price" name="price">')+'<input id="catalog-variants" name="variants"><button id="catalog-save">Save</button></form>'; + } + </script>`); + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const browser = await chromium.launch({ headless: true }); + try { + for (layout of ['direct', 'menu-tab', 'already-open', 'broken-tab', 'missing-tab', 'missing-field']) { + await t.test(layout, async () => { + products = []; visits.length = 0; + const definition = JSON.parse(readFileSync(join(STACK_BENCH_ROOT, + 'tracks/ecommerce/scenarios/progression-catalog-management.json'), 'utf8')); + const selected = definition.features[0]; + for (const step of [...selected.setup, ...selected.criteria.flatMap((c: { steps: Record<string, unknown>[] }) => c.steps)]) { + if ('within' in step || ['click', 'fill', 'expect'].includes(step.do)) step.within = 700; + if ('settleMs' in step) step.settleMs = 0; + } + const grade = await gradeFeature(browser, compileScenarioDefinition(definition).features[0]!, + { url, level: definition.level, headed: false, selectedCheckKeys: [], nullControl: false }, + { runId: `catalog-management-${layout}`, roomName: name => name, url, actions: [], + spacetime: null, backend: 'postgres', nullControl: false, defaultWithin: 700 }); + const passed = ['direct', 'menu-tab', 'already-open'].includes(layout); + const row = { layout, grade, visits: [...visits], products: [...products] }; evidence.push(row); + assert.equal(grade.criteria.every(c => c.evidence.status === 'passed'), passed, JSON.stringify(row)); + assert.equal(products.length, passed ? 1 : 0, 'navigation must not add or repeat product writes'); + if (passed) assert.deepEqual(products[0], { name: 'Travel Mug', variants: ['Black', 'Silver'] }); + if (layout === 'menu-tab') assert.ok(visits.includes('GET /account') && visits.includes('GET /admin/catalog')); + if (['direct', 'already-open'].includes(layout)) assert.equal(visits.includes('GET /account'), false); + }); + } + } finally { + await browser.close(); server.closeAllConnections(); + await new Promise<void>(resolve => server.close(() => resolve())); + if (process.env.STACK_BENCH_CATALOG_MANAGEMENT_EVIDENCE) { + writeFileSync(process.env.STACK_BENCH_CATALOG_MANAGEMENT_EVIDENCE, JSON.stringify({ + rerun: 'STACK_BENCH_CATALOG_MANAGEMENT_EVIDENCE=<file> node --test --test-name-pattern="catalog management reaches" dist/tests/item-navigation.integration.js', evidence, + }, null, 2)); + } + } +}); + +test('catalog search reaches its controls when the home page also has an item list', async () => { + let layout = '', routes: string[] = []; + const evidence: unknown[] = []; + const server = createServer((request, response) => { + routes.push(request.url ?? ''); + const searchable = layout === 'direct' || request.url === '/catalog'; + response.writeHead(200, { 'Content-Type': 'text/html' }); + response.end(`<!doctype html> + ${layout === 'separate' && !searchable ? '<a id="catalog-link" href="/catalog">Browse catalog</a>' : ''} + <section id="item-list"><article data-role="item-card">Desk Lamp</article></section> + ${searchable ? `<input id="search-input"><section id="search-results"></section> + <script>document.querySelector('input').oninput = event => { + document.querySelector('#search-results').innerHTML = 'mirrorless camera'.includes(event.target.value.toLowerCase()) + ? '<article data-role="item-card">Mirrorless Camera</article>' : ''; + };</script>` : ''}`); + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const browser = await chromium.launch({ headless: true }); + let result = 'failed'; + try { + const definition = JSON.parse(readFileSync(join(STACK_BENCH_ROOT, + 'tracks/ecommerce/scenarios/01-catalog-search.json'), 'utf8')); + const feature = compileScenarioDefinition(definition).features[0]!; + for (layout of ['separate', 'direct', 'missing-link']) { + routes = []; + const grade = await gradeFeature(browser, feature, + { url, level: 1, headed: false, selectedCheckKeys: [], nullControl: false }, + { runId: 'catalog-search-navigation', roomName: name => name, url, actions: [], + spacetime: null, backend: 'postgres', nullControl: false, defaultWithin: 1000 }); + evidence.push({ layout, grade, routes: [...routes], setup: feature.setup, criteria: feature.criteria }); + assert.equal(grade.criteria[0]!.evidence.status, layout === 'missing-link' ? 'failed' : 'passed', + JSON.stringify({ layout, grade, routes })); + assert.equal(routes.includes('/catalog'), layout === 'separate', JSON.stringify({ layout, routes })); + } + result = 'passed'; + } finally { + await browser.close(); server.closeAllConnections(); + await new Promise<void>(resolve => server.close(() => resolve())); + if (process.env.STACK_BENCH_ITEM_NAVIGATION_EVIDENCE) { + const file = process.env.STACK_BENCH_ITEM_NAVIGATION_EVIDENCE; + mkdirSync(dirname(file), { recursive: true }); + writeFileSync(file, JSON.stringify({ result, + rerun: 'node --test dist/tests/item-navigation.integration.js', evidence }, null, 2)); + } + } +}); + +test('filter setup supports input, change, and explicit Apply without accepting wrong results', async t => { + const files = ['progression-faceted-filters.json', 'progression-faceted-pagination.json', 'progression-search-ordering.json']; + const catalog = JSON.parse(readFileSync(join(STACK_BENCH_ROOT, + 'tracks/ecommerce/composition/fixtures/operations.json'), 'utf8')).items; + const evidence: unknown[] = []; + const browser = await chromium.launch({ headless: true }); + try { + for (const file of files) for (const mode of ['input', 'change', 'apply', 'wrong-filter', 'wrong-sort', 'empty']) { + // Sorting does not belong to 401a; its negative controls concern filtering. + if (file === files[0] && mode === 'wrong-sort') continue; + if (file === files[1] && mode === 'wrong-filter') continue; // All seeded prices already exceed 1. + await t.test(`${file}/${mode}`, async () => { + const feature = compileScenarioDefinition(JSON.parse(readFileSync(join(STACK_BENCH_ROOT, + 'tracks/ecommerce/scenarios', file), 'utf8'))).features[0]!; + // 402b's purchase and database receipt are separate prerequisites. Seed + // their proven result here, then run its actual post-purchase browser slice. + const ranked = file === files[2]; + const setup = ranked ? feature.setup.slice(feature.setup.findIndex(step => step.do === 'reload')) : feature.setup; + const page = await browser.newPage(); + const writes: unknown[] = [], actions: unknown[] = []; + let setupFilters = ''; + let result: Awaited<ReturnType<typeof executeAction>> | undefined; + try { + await page.route('http://filters.test/', route => route.fulfill({ contentType: 'text/html', body: ` + <input id="search-input"><select id="category-filter"><option value="">All</option><option>Home</option></select> + <input id="minimum-price"><input id="maximum-price"><input type="checkbox" id="in-stock-filter"> + ${mode === 'apply' ? '<button id="filter-apply">Apply</button>' : ''} + <output id="filter-state"></output><section id="item-list"><section id="search-results"></section></section> + <button id="search-next-page">Next</button><button id="search-previous-page">Previous</button> + <script> + const items = ${JSON.stringify(catalog)}, mode = ${JSON.stringify(mode)}, ranked = ${ranked}; + const $ = id => document.getElementById(id); + let filters = { category: '', min: '', max: '', stock: false }, page = 0, search = ''; + function render() { + const active = search || mode !== 'wrong-filter' && (filters.category || filters.min || filters.max || filters.stock); + let rows = items.filter(item => (!search || item.name.toLowerCase().includes(search.toLowerCase())) + && (mode === 'wrong-filter' || ((!filters.category || item.category === filters.category) + && (!filters.min || Number(item.price) >= Number(filters.min)) + && (!filters.max || Number(item.price) <= Number(filters.max)) + && (!filters.stock || item.name !== 'Coffee Grinder' && item.stock.East + item.stock.West > 0)))); + rows.sort((a,b) => !active && ranked ? Number(b.name === 'Headphones') - Number(a.name === 'Headphones') || a.name.localeCompare(b.name) : a.name.localeCompare(b.name)); + if (mode === 'wrong-sort') rows.reverse(); + if (mode === 'empty') rows = []; + $('filter-state').textContent = [filters.category, filters.min, filters.max].join('|'); + $('search-results').innerHTML = rows.slice(page*10, page*10+10).map(item => '<article data-role="item-card"><span data-role="item-name">'+item.name+'</span></article>').join(''); + } + function apply() { + filters = { category: $('category-filter').value, min: $('minimum-price').value, + max: $('maximum-price').value, stock: $('in-stock-filter').checked }; page = 0; render(); + } + for (const id of ['category-filter','minimum-price','maximum-price','in-stock-filter']) + $(id).addEventListener(mode === 'change' ? 'change' : 'input', () => { if (mode !== 'apply') apply(); }); + if ($('filter-apply')) $('filter-apply').onclick = apply; + $('search-input').onkeydown = event => { if (event.key === 'Enter') { search = $('search-input').value; page = 0; render(); } }; + $('search-next-page').onclick = () => { page++; render(); }; + $('search-previous-page').onclick = () => { page--; render(); }; + render(); + </script>` })); + await page.goto('http://filters.test/'); + const actor = { page, loc: (id: string, options?: { contains?: string; scope?: { testid: string; contains?: string } }) => { + const scope = options?.scope; + const root = scope ? page.locator(stableElementSelector(scope.testid)).filter({ hasText: scope.contains }).first() : page; + const locator = root.locator(stableElementSelector(id)).filter({ visible: true }); + return (options?.contains ? locator.filter({ hasText: options.contains }) : locator).first(); + } }; + const sleep = async (ms: number) => { if (ms) await new Promise(resolve => setTimeout(resolve, Math.min(ms, 10))); }; + const service = { defaultWithin: 250, expand: (value: string) => value, testId: stableElementSelector, sleep }; + const capabilities = { actors: { get: () => actor }, 'browser-interaction': service, + 'browser-observation': service, clock: { sleep }, + 'database-write': { setStock: async (input: unknown) => { writes.push(input); return input; } } }; + for (const [index, step] of [...setup, ...feature.criteria[0]!.steps].entries()) { + result = await executeAction(ACTION_REGISTRY, step.do, + { ...step, ...(step.testid ? { within: 250 } : {}), ...(step.do === 'reload' ? { settleMs: 0 } : {}) }, { capabilities }); + actions.push(result); + if (index === setup.length - 1) setupFilters = await page.locator('#filter-state').innerText(); + if (result.status !== 'passed') break; + } + const expected = ['input', 'change', 'apply'].includes(mode) ? 'passed' : 'failed'; + evidence.push({ file, mode, expected, setupFilters, actions, writes }); + assert.equal(result?.status, expected, JSON.stringify(evidence.at(-1))); + if (expected === 'passed') { + assert.equal(setupFilters, file === files[0] ? 'Home|50|200' : '|1|', 'the edit batch must be committed before its assertions'); + assert.equal(await page.locator('#filter-state').innerText(), file === files[0] ? 'Home|50|200' : ranked ? '||' : '|1|'); + } else assert.equal(result?.code, 'application_failure'); + assert.equal(writes.length, file === files[0] ? 2 : 0); + } finally { await page.close(); } + }); + } + } finally { + await browser.close(); + if (process.env.STACK_BENCH_ITEM_NAVIGATION_EVIDENCE) { + const file = process.env.STACK_BENCH_ITEM_NAVIGATION_EVIDENCE; + mkdirSync(dirname(file), { recursive: true }); + writeFileSync(file, JSON.stringify({ + rerun: 'STACK_BENCH_ITEM_NAVIGATION_EVIDENCE=<file> node --test --test-name-pattern="filter setup supports" dist/tests/item-navigation.integration.js', + fixture: 'Operations catalog; Coffee Grinder has zero stock for 401a; one prior Headphones purchase for 402b. Browser slices only.', + sourceSha256: Object.fromEntries([...files.map(name => `tracks/ecommerce/scenarios/${name}`), + 'dist/tests/item-navigation.integration.js', 'dist/src/actions/browser-action-executors.js'].map(path => [path, + createHash('sha256').update(readFileSync(join(STACK_BENCH_ROOT, path))).digest('hex')])), evidence, + }, null, 2) + '\n'); + } + } +}); + +test('openItem uses the declared name hook on links, buttons, and clickable cards', async () => { + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage(); + for (const control of ['button', 'a', 'card']) { + await page.setContent(` + <article data-role="item-card">Other item<button>View</button></article> + <article data-role="item-card" id="selected-card"> + <${control === 'card' ? 'h3' : control} data-role="item-name">Desk Lamp</${control === 'card' ? 'h3' : control}> + <button id="buy">Buy now</button> + </article> + <section id="item-detail" hidden></section> + <script> + document.querySelector('#buy').onclick = () => { throw Error('must not buy'); }; + document.querySelector(${JSON.stringify(control === 'card' + ? '#selected-card' : '[data-role="item-name"]')}).onclick = event => { + event.preventDefault(); + const detail = document.querySelector('#item-detail'); + detail.textContent = 'Desk Lamp details'; detail.hidden = false; + }; + </script>`); + const actor = { page, loc: (id: string, options?: { contains?: string }) => { + const locator = page.locator(stableElementSelector(id)); + return options?.contains ? locator.filter({ hasText: options.contains }) : locator; + } }; + const result = await executeAction(ACTION_REGISTRY, 'openItem', + { do: 'openItem', actor: 'visitor', item: 'Desk Lamp' }, { capabilities: { + actors: { get: () => actor }, + 'browser-interaction': { defaultWithin: 1000, expand: (value: string) => value, + testId: stableElementSelector }, + } }); + assert.equal(result.status, 'passed', result.summary ?? JSON.stringify(result)); + assert.equal(await page.locator('#item-detail').innerText(), 'Desk Lamp details'); + } + } finally { await browser.close(); } +}); + +test('openItem preserves inline variants and otherwise requires successful detail navigation', async () => { + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage(); + for (const placement of ['inline', 'detail', 'broken']) { + await page.setContent(` + <article data-role="item-card">Other item<span data-role="item-variant">Other variant</span></article> + <article data-role="item-card"> + <button data-role="item-name">Travel Mug</button> + <span data-role="item-variant" ${placement === 'inline' ? '' : 'hidden'}>Black</span> + </article> + <section id="item-detail" hidden><span data-role="item-variant">Black</span></section> + <script> + document.querySelector('[data-role="item-name"]').onclick = () => { + document.body.dataset.clicked = 'true'; + if (${JSON.stringify(placement)} === 'detail') document.querySelector('#item-detail').hidden = false; + }; + </script>`); + const actor = { page, loc: (id: string, options?: { contains?: string }) => { + const locator = page.locator(stableElementSelector(id)); + return options?.contains ? locator.filter({ hasText: options.contains }) : locator; + } }; + const result = await executeAction(ACTION_REGISTRY, 'openItem', { + do: 'openItem', actor: 'visitor', item: 'Travel Mug', unlessVisible: 'item-variant', within: 1000, + }, { capabilities: { + actors: { get: () => actor }, + 'browser-interaction': { defaultWithin: 1000, expand: (value: string) => value, + testId: stableElementSelector }, + } }); + assert.equal(result.status, placement === 'broken' ? 'failed' : 'passed', result.summary ?? undefined); + assert.equal(await page.locator('body').getAttribute('data-clicked'), placement === 'inline' ? null : 'true'); + if (placement === 'detail') assert.equal(await page.locator('#item-detail').isVisible(), true); + } + } finally { await browser.close(); } +}); diff --git a/tools/stack-bench/tests/l1-concurrency-completion.mutation.ts b/tools/stack-bench/tests/l1-concurrency-completion.mutation.ts new file mode 100644 index 00000000000..d12000738c7 --- /dev/null +++ b/tools/stack-bench/tests/l1-concurrency-completion.mutation.ts @@ -0,0 +1,170 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, readdirSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import ts from 'typescript'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { mutationFileEdits, mutationScenario, mutationTargetKeys, + readMutationManifest, type LoadedMutationDefinition } + from '../src/evidence/mutation-analysis.js'; +import { loadReferenceRegistry, prepareReferenceFixtureSource, + selectReferenceFixture } from '../src/references/reference-fixtures.js'; +import { buildRecipeRelease } from '../src/composition/recipe-release.js'; +import { selectScenarioChecks } from '../src/composition/recipe-selection.js'; +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; + +const ROOT = STACK_BENCH_ROOT; +const TRACK = join(ROOT, 'tracks', 'ecommerce'); +const RECIPES = join(TRACK, 'composition', 'recipes'); +const release = buildRecipeRelease(join(RECIPES, 'sequential-l1.json')); +// Shared manifests describe every recipe; each qualification keeps only its own targets. +const recipeCheckKeys = new Set(readdirSync(RECIPES).flatMap(name => + buildRecipeRelease(join(RECIPES, name)).checkCatalog.map(check => check.stableKey))); + +interface CandidateCase { + backend: string; + manifest: string; + lastUnitMutation: string; +} +const cases: CandidateCase[] = [ + { + backend: 'mongodb', + manifest: 'mongodb-ecommerce.json', + lastUnitMutation: 'last-unit-allows-negative-stock', + }, + { + backend: 'postgres', + manifest: 'postgres-ecommerce.json', + lastUnitMutation: 'oversell-no-row-lock', + }, + { + backend: 'spacetime', + manifest: 'spacetime-ecommerce.json', + lastUnitMutation: 'purchase-does-not-reserve-stock-last-unit', + }, +]; + +test('each last-unit score retains the shared purchase race when selected alone', () => { + const scenarioPath = join(TRACK, 'scenarios', '01-last-unit.json'); + const scenario = compileScenarioDefinition(readJson(scenarioPath), { source: scenarioPath }); + for (const criterionId of ['201a', '201b', '201c']) { + const key = `ecommerce.spec.concurrency-safety.last-unit.${criterionId}`; + const selected = selectScenarioChecks(scenario, { checks: release.checkCatalog }, [key]); + const [feature] = selected.features; + assert(feature, 'the selected scenario must have a feature'); + assert.deepEqual(feature.criteria.map(criterion => criterion.id), [criterionId]); + const races = feature.setup.filter(step => step.do === 'callConcurrently'); + assert.equal(races.length, 1); + const [race] = races; + assert(race, 'the selected scenario must have a race'); + assert.deepEqual(race.actors, ['a', 'b', 'c', 'd', 'e', 'f']); + assert.equal(race.action, 'buy'); + assert(feature.setup.some(step => step.do === 'expectCallOutcomes')); + assert.equal(recordValue(race.input, 'race input').contains, 'Air Purifier'); + const selectedCriterion = feature.criteria.find(criterion => criterion.id === criterionId); + assert(selectedCriterion, 'the selected feature must retain its criterion'); + assert.equal(selectedCriterion.steps.some(step => step.do === 'clickConcurrently' + && step.testid === 'buy-now'), false); + const baseline = feature.setup.findIndex(step => step.as === 'revenue-before-last-unit'); + assert(baseline >= 0 && baseline < feature.setup.indexOf(race), + 'the revenue baseline must be recorded before the shared purchase race'); + } +}); + +test('the restock race retains its admin page prerequisite when selected alone', () => { + const scenarioPath = join(TRACK, 'scenarios', '01-restock-race.json'); + const scenario = compileScenarioDefinition(readJson(scenarioPath), { source: scenarioPath }); + const key = 'ecommerce.spec.concurrency-safety.restock-race.202a'; + const selected = selectScenarioChecks(scenario, { checks: release.checkCatalog }, [key]); + const [feature] = selected.features; + assert(feature, 'the selected scenario must have a feature'); + assert.deepEqual(feature.criteria.map(criterion => criterion.id), ['202a']); + assert(feature.setup.some(step => step.do === 'click' && step.actor === 'admin' + && step.testid === 'admin-link')); + assert(feature.setup.some(step => step.do === 'click' && step.testid === 'restock-submit'), + 'the uncontended restock must run even when the zero-point control is not selected'); + assert(feature.setup.some(step => step.do === 'dbExpectStock' && step.plus === 5), + 'ordinary restocking must have a verified stored effect before the race'); + assert(feature.setup.some(step => step.do === 'click' && step.actor === 'a' + && step.testid === 'buy-now'), 'ordinary purchasing must precede the race'); + assert(feature.setup.some(step => step.do === 'expect' && step.actor === 'a' + && step.testid === 'order-item' && step.count === 1)); + assert(feature.setup.some(step => step.do === 'dbExpectStock' && step.plus === -1), + 'a missing serial decrement must be invalid setup, not a caught concurrency defect'); + const [criterion] = feature.criteria; + assert(criterion, 'the selected feature must have a criterion'); + assert(criterion.steps.some(step => step.do === 'race')); + assert(criterion.steps.some(step => step.do === 'dbExpectStock' && step.plus === 2)); + for (const actor of ['a', 'b', 'c']) assert(criterion.steps.some(step => step.do === 'expect' + && step.actor === actor && step.testid === 'order-item' && step.count === (actor === 'a' ? 2 : 1))); +}); + +for (const entry of cases) { +test(`${entry.backend} binds the current L1 mutation inventory to its effective source`, () => { + const work = mkdtempSync(join(tmpdir(), `stack-bench-l1-concurrency-${entry.backend}-`)); + try { + const app = join(work, 'app'); + prepareReferenceFixtureSource(selectReferenceFixture(loadReferenceRegistry(), { + backend: entry.backend, track: 'ecommerce', level: 1, recipe: 'ecommerce.sequential-l1', + }), app); + + const manifest = readMutationManifest(join(ROOT, 'grader', 'mutations', entry.manifest)); + + assert.equal(manifest.mutations.some(mutation => mutationTargetKeys(mutation) + .includes('ecommerce.spec.external-data-sync.external-stock.901b')), false, '901b has no deterministic candidate mutation'); + const mutations = new Map(manifest.mutations.map(mutation => [mutation.id, mutation])); + const lastUnit = requiredMutation(mutations, entry.lastUnitMutation); + assert.equal(mutationScenario(manifest, lastUnit), + 'tracks/ecommerce/scenarios/01-last-unit.json'); + assert.deepEqual(mutationTargetKeys(lastUnit), ['ecommerce.spec.concurrency-safety.last-unit.201a', 'ecommerce.spec.concurrency-safety.last-unit.201b', 'ecommerce.spec.concurrency-safety.last-unit.201c']); + + assert.equal(mutations.has('purchase-does-not-reserve-stock-restock-race'), false, + 'a defect that breaks serial purchasing cannot qualify this race'); + const purchase = requiredMutation(mutations, entry.backend === 'spacetime' + ? 'restock-client-snapshot-overwrites-concurrent-purchases' + : 'purchase-read-write-loses-concurrent-stock'); + assert.equal(mutationScenario(manifest, purchase), + 'tracks/ecommerce/scenarios/01-restock-race.json'); + assert.deepEqual(mutationTargetKeys(purchase), ['ecommerce.spec.concurrency-safety.restock-race.202a']); + const releaseKeys = new Set(release.checkCatalog.map(check => check.stableKey)); + for (const mutation of manifest.mutations.filter(candidate => + mutationTargetKeys(candidate).some(key => releaseKeys.has(key)))) { + for (const key of mutationTargetKeys(mutation)) { + assert(recipeCheckKeys.has(key), `${mutation.id} target ${key} is not a check in any recipe`); + } + for (const edit of mutationFileEdits(mutation)) { + const file = edit.file; + assert(file, `${mutation.id} must declare a source file for each edit`); + const source = readFileSync(join(app, file), 'utf8'); + const mutated = source.replace(edit.find, edit.replace); + const transpiled = ts.transpileModule(mutated, { + compilerOptions: { module: ts.ModuleKind.ESNext, target: ts.ScriptTarget.ES2022 }, + fileName: file, + reportDiagnostics: true, + }); + assert.deepEqual((transpiled.diagnostics ?? []) + .filter(diagnostic => diagnostic.category === ts.DiagnosticCategory.Error) + .map(diagnostic => ts.flattenDiagnosticMessageText(diagnostic.messageText, '\n')), []); + } + } + } finally { + rmSync(work, { recursive: true, force: true }); + } + }); +} + +function readJson(path: string): unknown { return JSON.parse(readFileSync(path, 'utf8')); } +function recordValue(value: unknown, label: string): Record<string, unknown> { + if (!isRecord(value)) throw new Error(`${label} must be an object`); + return value; +} +function isRecord(value: unknown): value is Record<string, unknown> { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} +function requiredMutation(mutations: Map<string, LoadedMutationDefinition>, id: string): LoadedMutationDefinition { + const mutation = mutations.get(id); + if (!mutation) throw new Error(`mutation ${id} is required`); + return mutation; +} diff --git a/tools/stack-bench/tests/l3-composition.contract.ts b/tools/stack-bench/tests/l3-composition.contract.ts new file mode 100644 index 00000000000..3f5f4477460 --- /dev/null +++ b/tools/stack-bench/tests/l3-composition.contract.ts @@ -0,0 +1,299 @@ +import assert from 'node:assert/strict'; +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { compilePackDefinition, compileRecipeFile, type CompiledPackDefinition } + from '../src/composition/composition-compiler.js'; +import { compileScenarioDefinition, type CompiledFeature, type CompiledStep } + from '../src/composition/definition-compiler.js'; +import { declaredPackWaitMs } from '../src/composition/pack-budget.js'; + +const trackRoot = join(STACK_BENCH_ROOT, 'tracks', 'ecommerce'); +const packRoot = join(trackRoot, 'composition', 'packs'); +const readJson = (path: string): unknown => JSON.parse(readFileSync(path, 'utf8')); +const l3Recipe = recipePacks(readJson(join(trackRoot, 'composition', 'recipes', 'sequential-l3.json'))); +const packNames = l3Recipe + .map(pack => pack.path.split('/').at(-1)) + .filter((name): name is string => name !== undefined && name.startsWith('l3-')) + .sort(); +const packs = packNames.map(name => compilePackDefinition(readJson(join(packRoot, name)), { source: name })); +const selected = packs.flatMap(pack => pack.checks.map(check => ({ pack, check }))); +const scenarioFor = (check: CompiledPackDefinition['checks'][number]) => { + const source = join(trackRoot, check.source); + return compileScenarioDefinition(readJson(source), { source, expectedLevel: 3 }); +}; +const featureFor = (check: CompiledPackDefinition['checks'][number]): CompiledFeature => { + const feature = scenarioFor(check).features.find(candidate => candidate.id === check.feature); + if (!feature) throw new Error(`${check.id} must select a feature`); + return feature; +}; +const nestedSteps = (feature: CompiledFeature): CompiledStep[] => + [...feature.setup, ...feature.criteria.flatMap(criterion => criterion.steps)].flatMap(flatten); + +function flatten(step: CompiledStep): CompiledStep[] { + if (!step.branches) return [step]; + if (!Array.isArray(step.branches)) throw new Error('nested step must have branches'); + return [step, ...step.branches.flatMap(branch => branch.flatMap(flatten))]; +} + +function requiredPack(id: string): CompiledPackDefinition { + const pack = packs.find(candidate => candidate.id === id); + if (!pack) throw new Error(`missing pack ${id}`); + return pack; +} + +function requiredCheck(pack: CompiledPackDefinition, id?: string): CompiledPackDefinition['checks'][number] { + const check = id === undefined ? pack.checks[0] : pack.checks.find(candidate => candidate.id === id); + if (!check) throw new Error(`missing check ${id ?? 'first'} in ${pack.id}`); + return check; +} + +function firstCriterion(feature: CompiledFeature) { + const criterion = feature.criteria[0]; + if (!criterion) throw new Error(`feature ${feature.id} must have a criterion`); + return criterion; +} + +function lastStep(steps: CompiledStep[]): CompiledStep { + const step = steps.at(-1); + if (!step) throw new Error('scenario must have at least one step'); + return step; +} + +function atLeast(value: unknown, minimum: number): boolean { + return typeof value === 'number' && value >= minimum; +} + +function atMost(value: unknown, maximum: number): boolean { + return typeof value === 'number' && value <= maximum; +} + +function maxRuntimeMs(pack: CompiledPackDefinition): number { + const runtime = pack.budget.maxRuntimeMs; + if (typeof runtime !== 'number') throw new Error(`${pack.id} must declare maxRuntimeMs`); + return runtime; +} + +function resolveFragment(fragment: CompiledPackDefinition['task']['requirements'][number]): string { + const path = join(trackRoot, fragment.path); + assert(existsSync(path), `missing fragment source ${fragment.path}`); + const text = readFileSync(path, 'utf8'); + const start = fragment.from ? text.indexOf(fragment.from) : 0; + assert(start >= 0, `${fragment.id} has no start marker ${fragment.from}`); + const end = fragment.until ? text.indexOf(fragment.until, start + (fragment.from?.length ?? 0)) : text.length; + assert(end > start, `${fragment.id} resolves to empty or reversed text`); + return text.slice(start, end).trim(); +} + +test('L3 product work and production specifications are separate modules', () => { + for (const pack of packs) { + const promptPaths = pack.task.requirements.map(fragment => fragment.path); + if (pack.moduleType === 'feature') { + assert(promptPaths.every(path => path.startsWith('prompts/modular/'))); + assert(pack.checks.every(check => check.role === 'feature')); + } else { + assert(promptPaths.every(path => path === 'prompts/modular/l3-specifications.md')); + assert(pack.task.requirements.every(fragment => fragment.requiresFeatures?.length)); + assert(pack.checks.every(check => check.role === 'guarantee')); + assert(pack.checks.every(check => check.observations?.includes('unmentioned'))); + } + } +}); + +test('every L3 scored check has isolated setup and one criterion', () => { + assert.equal(selected.length, 21); + const stableKeys = new Set(); + + for (const { pack, check } of selected) { + const feature = featureFor(check); + assert(feature, `${pack.id}.${check.id} must select a real feature`); + assert.equal(feature.criteria.length, 1, + `${pack.id}.${check.id} must not depend on another criterion's side effects`); + assert(feature.setup.length > 0, `${pack.id}.${check.id} must own its setup`); + const criterion = firstCriterion(feature); + const key = `${pack.stableId ?? pack.id}.${check.stableId ?? check.id}.${criterion.id}`; + assert(!stableKeys.has(key), `duplicate stable check ${key}`); + stableKeys.add(key); + } +}); + +test('L3 feature requests and production specifications do not claim the same work', () => { + const featureText = packs.filter(pack => pack.moduleType === 'feature') + .flatMap(pack => pack.task.requirements).map(resolveFragment).join('\n'); + const specificationText = readFileSync( + join(trackRoot, 'prompts', 'modular', 'l3-specifications.md'), 'utf8'); + assert.doesNotMatch(featureText, /survive a backend restart|more than once|without an open browser|Only an admin/); + assert.doesNotMatch(specificationText, /shows the remaining|enters the stock ledger|returns empty/); + + const featureCriteria = selected.filter(entry => entry.pack.moduleType === 'feature') + .map(entry => firstCriterion(featureFor(entry.check))); + const specificationCriteria = selected.filter(entry => entry.pack.moduleType === 'specification') + .map(entry => firstCriterion(featureFor(entry.check))); + const featureSteps = new Set(featureCriteria.map(criterion => JSON.stringify(criterion.steps))); + assert(specificationCriteria.every(criterion => !featureSteps.has(JSON.stringify(criterion.steps))), + 'a production check must not duplicate a product check byte for byte'); +}); + +test('the sequential L3 brief agrees with cumulative L2 shipping behavior', () => { + const sequentialBrief = readFileSync(join(trackRoot, 'prompts', '03-scheduled.md'), 'utf8'); + assert.match(sequentialBrief, /Shipping remains the immediate staff action introduced at level 2/); + assert.match(sequentialBrief, /shipped order moves to `delivered` \*\*60 seconds\*\* after shipping/); + assert.doesNotMatch(sequentialBrief, /pending.*moves to.*shipped/is); + assert.doesNotMatch(sequentialBrief, /time it actually ran/); +}); + +test('L3 capability and evidence declarations match the selected actions', () => { + const knownCapabilities = new Set([ + 'backend-lifecycle', 'browser', 'concurrency', 'concurrent-actors', 'database-observation', + 'direct-database-write', 'direct-server-call', 'request-replay', + ]); + const knownEvidence = new Set([ + 'browser-observation', 'concurrent-outcome', 'database-observation', 'fresh-client-observation', + 'server-action-observation', 'server-refusal', 'server-response', + ]); + for (const pack of packs) { + assert(pack.capabilities.every(value => knownCapabilities.has(value))); + assert(pack.evidence.every(value => knownEvidence.has(value))); + const actions = selected.filter(entry => entry.pack.id === pack.id) + .flatMap(entry => nestedSteps(featureFor(entry.check)).map(step => step.do)); + if (actions.includes('restartBackend')) assert(pack.capabilities.includes('backend-lifecycle')); + if (actions.includes('callAction')) { + assert(pack.capabilities.includes('direct-server-call')); + assert(pack.evidence.includes('server-response')); + } + if (actions.includes('replayAs')) { + assert(pack.capabilities.includes('request-replay')); + assert(pack.evidence.includes('server-refusal')); + } + assert(!pack.evidence.includes('fresh-client-observation'), + `${pack.id} does not run freshClient and must not claim fresh-client evidence`); + } +}); + +test('timed and restarted work has conclusive before-and-after observations', () => { + const durationText = packs.filter(pack => pack.moduleType === 'feature') + .flatMap(pack => pack.task.requirements).map(resolveFragment).join('\n'); + assert.match(durationText, /90 seconds/); + + const durability = requiredPack('ecommerce.l3.deferred-durability-specifications'); + for (const check of durability.checks) { + const feature = featureFor(check); + const steps = nestedSteps(feature); + const restart = steps.findIndex(step => step.do === 'restartBackend'); + assert(restart > 0); + assert(steps.slice(0, restart).some(step => ['expect', 'expectNumber'].includes(step.do)), + `${check.id} needs a pre-restart observation`); + assert(steps.slice(restart + 1).some(step => ['expect', 'expectNumber'].includes(step.do)), + `${check.id} needs a post-restart completion observation`); + } + + const timedChecks: ReadonlyArray<readonly [packId: string, checkId: string]> = [ + ['ecommerce.l3.reservations-features', 'countdown'], + ['ecommerce.l3.scheduled-restocks-features', 'pending'], + ]; + for (const [packId, checkId] of timedChecks) { + const pack = requiredPack(packId); + const feature = featureFor(requiredCheck(pack, checkId)); + const steps = nestedSteps(feature); + const baseline = steps.findIndex(step => step.do === 'recordNumber'); + assert(baseline >= 0); + assert(steps.slice(0, baseline).some(step => step.do === 'expectNumber' + && atLeast(step.atLeast, 1) && atMost(step.atMost, 90))); + assert(steps.slice(baseline + 1).some(step => step.do === 'wait' && atLeast(step.ms, 1000))); + assert(steps.slice(baseline + 1).some(step => step.do === 'expectNumber' + && step.relativeTo === steps[baseline]?.as && step.comparison === 'atMost' && step.plus === -1)); + } +}); + +test('pending-work checks cancel or complete the work they create', () => { + const pendingPack = requiredPack('ecommerce.l3.scheduled-restocks-features'); + const pendingSteps = nestedSteps(featureFor( + requiredCheck(pendingPack, 'pending'))); + assert.equal(lastStep(pendingSteps).do, 'click'); + assert.equal(lastStep(pendingSteps).testid, 'pending-restock-cancel'); + + const accessSteps = nestedSteps(featureFor(requiredCheck( + requiredPack('ecommerce.l3.deferred-access-specifications')))); + assert.equal(lastStep(accessSteps).do, 'click'); + assert.equal(lastStep(accessSteps).testid, 'pending-restock-cancel'); + + const timePack = requiredPack('ecommerce.l3.server-time-specifications'); + const timeSteps = nestedSteps(featureFor(requiredCheck(timePack, 'not-early'))); + assert(timeSteps.some(step => step.do === 'dbExpectStock' && step.plus === 4)); + assert.equal(timeSteps.some(step => step.testid === 'pending-restock-remaining'), false); + assert.equal(lastStep(timeSteps).do, 'expect'); + assert.equal(lastStep(timeSteps).absent, true); +}); + +test('L3 budgets cover declared waits without becoming unbounded estimates', () => { + for (const pack of packs) { + const declaredDelay = declaredPackWaitMs(pack, trackRoot); + assert(maxRuntimeMs(pack) >= declaredDelay, + `${pack.id} budget is below its declared waits`); + assert(maxRuntimeMs(pack) <= declaredDelay + 180000, + `${pack.id} budget has more than three minutes of unexplained slack`); + } +}); + +test('L3 covers each declared product area and production behavior', () => { + assert.deepEqual(new Set(packs.filter(pack => pack.moduleType === 'feature').map(pack => pack.id)), new Set([ + 'ecommerce.l3.reservations-features', + 'ecommerce.l3.scheduled-restocks-features', + 'ecommerce.l3.order-delivery-features', + 'ecommerce.l3.cart-expiration-features', + ])); + assert.deepEqual(new Set(packs.filter(pack => pack.moduleType === 'specification').map(pack => pack.id)), new Set([ + 'ecommerce.l3.deferred-durability-specifications', + 'ecommerce.l3.deferred-integrity-specifications', + 'ecommerce.l3.server-time-specifications', + 'ecommerce.l3.deferred-access-specifications', + ])); +}); + +test('the cumulative L3 recipe adds every L3 check', () => { + const plan = compileRecipeFile( + join(trackRoot, 'composition', 'recipes', 'sequential-l3.json'), + { trackRoot }, + ); + assert.equal(plan.checks.length, 93); + assert.equal(plan.scoring.points, 170); + + const plannedKeys = new Set(plan.checks.map(check => check.stableKey)); + const expectedL3Keys = selected.flatMap(({ pack, check }) => { + const feature = featureFor(check); + const criteria = check.criteria === undefined + ? feature.criteria + : check.criteria.map(id => feature.criteria.find(criterion => criterion.id === id)); + return criteria.map(criterion => { + if (!criterion) throw new Error(`${check.id} must select a real criterion`); + return `${pack.stableId ?? pack.id}.${check.stableId ?? check.id}.${criterion.id}`; + }); + }); + assert.equal(expectedL3Keys.length, 21); + assert(expectedL3Keys.every(key => plannedKeys.has(key))); + + const reservationHooks = plan.recipe.task.contracts.find(fragment => + fragment.id === 'ecommerce.l3.reservation-hooks'); + assert(reservationHooks, 'L3 recipe must contain reservation hooks'); + assert.deepEqual(reservationHooks.owners, ['ecommerce.l3.reservations-features']); + const cartExpirationHooks = plan.recipe.task.contracts.find(fragment => + fragment.id === 'ecommerce.l3.cart-expiration-hooks'); + assert(cartExpirationHooks, 'L3 recipe must contain cart expiration hooks'); + assert.deepEqual(cartExpirationHooks.owners, ['ecommerce.l3.cart-expiration-features']); +}); + +function recipePacks(value: unknown): Array<{ path: string }> { + if (!isRecord(value) || !Array.isArray(value.packs)) throw new Error('recipe must have packs'); + return value.packs.map((pack, index) => { + if (!isRecord(pack) || typeof pack.path !== 'string') { + throw new Error(`recipe pack ${index} must have a path`); + } + return { path: pack.path }; + }); +} + +function isRecord(value: unknown): value is Record<string, unknown> { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} diff --git a/tools/stack-bench/tests/leak-audit.test.ts b/tools/stack-bench/tests/leak-audit.test.ts new file mode 100644 index 00000000000..9d3fdc9813d --- /dev/null +++ b/tools/stack-bench/tests/leak-audit.test.ts @@ -0,0 +1,410 @@ +import assert from 'node:assert/strict'; +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { runAuditNetworkContext } from '../commands/bench.js'; +import { createBackendLease } from '../src/runtime/backend-lease.js'; +import { loadTrack } from '../src/composition/tracks.js'; +import { auditTranscript, networkTargetsFromBash, STACKBENCH_SOURCE, stackBenchSourceReferences } + from '../commands/leak-audit.js'; +import { archiveTranscripts, transcriptDirectories } from '../src/agents/transcript-archive.js'; +import { codexTranscriptDirectory } from '../src/agents/codex-protocol.js'; + +test('Codex tool events use the shared restricted access and refusal audit', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-codex-audit-')); + const transcript = join(root, 'session.events.jsonl'); + try { + const events = [ + { type: 'stack_bench_context', cwd: '/app' }, + { type: 'item.completed', item: { id: 'read', type: 'command_execution', + command: "/bin/bash -lc 'cat /tools/stack-bench/grader/grade.ts'", + status: 'completed', exit_code: 0, aggregated_output: 'private contents' } }, + { type: 'item.completed', item: { id: 'net', type: 'command_execution', + command: 'curl http://host.docker.internal:7331/api/overview', + status: 'completed', exit_code: 0, aggregated_output: '{}' } }, + { type: 'item.completed', item: { id: 'edit', type: 'file_change', + status: 'completed', changes: [{ path: '/app/src/../stack-bench/bundle.json', kind: 'update' }] } }, + { type: 'item.completed', item: { id: 'refused', type: 'file_change', + status: 'failed', changes: [{ path: '/tools/stack-bench/grader/grade.ts', kind: 'update' }] } }, + { type: 'item.started', item: { id: 'cut', type: 'command_execution', + command: 'cat /tools/stack-bench/scenarios/test.json', status: 'in_progress' } }, + ]; + writeFileSync(transcript, events.map(event => JSON.stringify(event)).join('\n')); + const audit = auditTranscript(transcript, 'C:/host/app'); + assert.equal(audit.cwd, '/app'); + assert.deepEqual(audit.hits.map(hit => hit.path), [ + '/tools/stack-bench/grader/grade.ts', 'host.docker.internal:7331', + '/app/stack-bench/bundle.json', '/tools/stack-bench/scenarios/test.json', + ]); + assert.equal(audit.hits.at(-1)?.unresolved, true); + assert.equal(audit.refused.length, 1); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('failed Codex commands retain partial reads but refuse empty output', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-codex-refusal-')); + const transcript = join(root, 'session.events.jsonl'); + try { + writeFileSync(transcript, ['partial', 'empty'].map(id => JSON.stringify({ + cwd: '/app', type: 'item.completed', item: { id, type: 'command_execution', + command: 'cat /tools/stack-bench/grader/grade.ts; false', status: 'failed', + exit_code: 1, aggregated_output: id === 'partial' ? 'private contents' : '' }, + })).join('\n')); + const audit = auditTranscript(transcript, '/app'); + assert.equal(audit.hits.length, 1); + assert.equal(audit.refused.length, 1); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('Codex archive copies controller events and excludes native session files', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-codex-archive-')); + const store = codexTranscriptDirectory(root); + assert(!existsSync(store)); + try { + mkdirSync(join(store, 'sessions'), { recursive: true }); + writeFileSync(join(store, 'run.events.jsonl'), '{}\n'); + writeFileSync(join(store, 'sessions', 'agent.events.jsonl'), '{}\n'); + const output = join(root, 'archive'); + assert.deepEqual(transcriptDirectories(root, join(root, 'no-claude')), [store]); + const result = archiveTranscripts(root, 'attempt', output, join(root, 'no-claude')); + assert.equal(result.copied, 1); + assert(existsSync(join(output, 'attempt', 'run.events.jsonl'))); + assert(!existsSync(join(output, 'attempt', 'sessions__agent.events.jsonl'))); + } finally { + rmSync(store, { recursive: true, force: true }); + rmSync(root, { recursive: true, force: true }); + } +}); + +test('a completed external Bash read contaminates the transcript', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-leak-audit-')); + const transcript = join(root, 'session.jsonl'); + try { + const events = [ + { cwd: '/app', message: { content: [{ type: 'tool_use', id: 'read-1', name: 'Bash', + input: { command: 'cat /tools/stack-bench/grader/grade.ts; rg secret /outside/notes.md' } }] } }, + { message: { content: [{ type: 'tool_result', tool_use_id: 'read-1', is_error: false }] } }, + { message: { content: [{ type: 'tool_use', id: 'read-2', name: 'Bash', + input: { command: 'cat stack-bench/bundle.json' } }] } }, + { message: { content: [{ type: 'tool_result', tool_use_id: 'read-2', is_error: false }] } }, + ]; + writeFileSync(transcript, `${events.map(event => JSON.stringify(event)).join('\n')}\n`); + + const result = auditTranscript(transcript, '/app'); + + assert.equal(result.cwd, '/app'); + assert.equal(result.refused.length, 0); + assert.deepEqual(result.hits.map(hit => ({ path: hit.path, kind: hit.kind })), [ + { path: '/tools/stack-bench/grader/grade.ts', kind: 'GRADER / TEST SPECS' }, + { path: '/outside/notes.md', kind: 'other' }, + { path: '/app/stack-bench/bundle.json', kind: 'GRADER / TEST SPECS' }, + ]); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('network targets come from URLs and raw sockets in a shell command', () => { + assert.deepEqual(networkTargetsFromBash( + 'curl -s http://127.0.0.1:7331/api/overview && wget https://registry.npmjs.org/react; ' + + 'nc -z host.docker.internal 6532; node -e "fetch(\'http://localhost:6173/\')"'), [ + { host: '127.0.0.1', port: 7331 }, + { host: 'registry.npmjs.org', port: null }, + { host: 'localhost', port: 6173 }, + { host: 'host.docker.internal', port: 6532 }, + ]); + assert.deepEqual(networkTargetsFromBash('npm install && npm run build'), []); +}); + +test('a local network read outside the run\'s own ports contaminates the transcript', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-network-leak-audit-')); + const transcript = join(root, 'session.jsonl'); + try { + const events = [ + { cwd: '/app', message: { content: [{ type: 'tool_use', id: 'own', name: 'Bash', + input: { command: 'curl -s http://localhost:6173/api/items' } }] } }, + { message: { content: [{ type: 'tool_result', tool_use_id: 'own', is_error: false }] } }, + { message: { content: [{ type: 'tool_use', id: 'own-wildcard', name: 'Bash', + input: { command: 'curl -s http://0.0.0.0:6173/api/items' } }] } }, + { message: { content: [{ type: 'tool_result', tool_use_id: 'own-wildcard', is_error: false }] } }, + { message: { content: [{ type: 'tool_use', id: 'other-wildcard', name: 'Bash', + input: { command: 'curl -s http://0.0.0.0:6174/api/items' } }] } }, + { message: { content: [{ type: 'tool_result', tool_use_id: 'other-wildcard', is_error: false }] } }, + { message: { content: [{ type: 'tool_use', id: 'other', name: 'Bash', + input: { command: 'curl -s http://127.0.0.1:3211/v1/database/app-ecom-run1/schema' } }] } }, + { message: { content: [{ type: 'tool_result', tool_use_id: 'other', is_error: false }] } }, + { message: { content: [{ type: 'tool_use', id: 'blocked', name: 'Bash', + input: { command: 'curl -s http://127.0.0.1:7331/api/overview' } }] } }, + { message: { content: [{ type: 'tool_result', tool_use_id: 'blocked', is_error: true }] } }, + { message: { content: [{ type: 'tool_use', id: 'registry', name: 'Bash', + input: { command: 'curl -sI https://registry.npmjs.org/express' } }] } }, + { message: { content: [{ type: 'tool_result', tool_use_id: 'registry', is_error: false }] } }, + ]; + writeFileSync(transcript, `${events.map(event => JSON.stringify(event)).join('\n')}\n`); + + const result = auditTranscript(transcript, '/app', { ownEndpoints: ['127.0.0.1:6173', '127.0.0.1:6001', '127.0.0.1:3210'] }); + + assert.deepEqual(result.hits.map(hit => ({ path: hit.path, kind: hit.kind })), [ + { path: '0.0.0.0:6174', kind: 'NETWORK / OTHER RUN' }, + { path: '127.0.0.1:3211', kind: 'NETWORK / OTHER RUN' }, + { path: 'registry.npmjs.org', kind: 'network (internet)' }, + ]); + assert.deepEqual(result.refused.map(hit => ({ path: hit.path, kind: hit.kind })), [ + { path: '127.0.0.1:7331', kind: 'NETWORK / OTHER RUN' }, + ]); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + + +test('leased database endpoints permit the supplied namespace, not another host at the same port', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-owned-endpoints-')); + try { + for (const [backend, port] of [['mongodb', 27017], ['postgres', 5432], ['spacetime', 3210]] as const) { + const lease = createBackendLease({ runId: 'endpoint-test', backend, track: 'ecommerce', + runIndex: 1, ...(backend === 'spacetime' + ? { serverUri: 'http://127.0.0.1:3210', module: 'app-ecom-run1', dataDir: join(root, 'spacetime') } + : { database: 'app_ecom_run1' }) }); + lease.resources.network = { name: 'owned', id: 'network', namespaceContainerId: 'namespace', + hostAddresses: [], services: [{ address: '172.20.0.2', port: 4873 }], + ownAddresses: ['172.20.0.3'], + cacheContainerId: 'cache', firewallSha256: null, firewallInstalledAt: null }; + const context = () => runAuditNetworkContext(loadTrack('ecommerce'), { backend, runIndex: 1 }, lease); + const { ownEndpoints } = context(); + assert.equal(context().isolatedLoopback, false); + lease.resources.container = { name: 'backend', id: 'namespace', owned: true }; + lease.resources.buildContainer = { name: 'build', id: 'build', owned: true, networkMode: 'container:namespace' }; + lease.resources.network.firewallSha256 = 'a'.repeat(64); + lease.resources.network.firewallInstalledAt = new Date().toISOString(); + assert.equal(context().isolatedLoopback, true); + for (const mode of ['host', 'bridge', 'container:foreign']) { + lease.resources.buildContainer.networkMode = mode; + assert.equal(context().isolatedLoopback, false, mode); + } + assert(ownEndpoints.includes(`127.0.0.1:${port}`)); + // The attempt's own bridge address owns the same ports; a probe there is not another run. + assert(ownEndpoints.includes(`172.20.0.3:${port}`)); + assert(!ownEndpoints.includes(`172.20.0.99:${port}`)); + assert(!ownEndpoints.some(value => value.includes('@') || value.includes(lease.ownershipToken))); + const transcript = join(root, `${backend}.jsonl`); + const events = [ + { cwd: '/app', message: { content: [{ type: 'tool_use', id: 'own', name: 'Bash', + input: { command: `which mongosh mongo; ls /usr/bin | grep -i mongo; nc -zv 127.0.0.1 ${port} 2>&1; npm config get registry` } }] } }, + { message: { content: [{ type: 'tool_result', tool_use_id: 'own', is_error: false, + content: '/bin/bash: line 1: nc: command not found' }] } }, + { message: { content: [{ type: 'tool_use', id: 'foreign', name: 'Bash', + input: { command: `nc -zv 172.20.0.99 ${port}` } }] } }, + { message: { content: [{ type: 'tool_result', tool_use_id: 'foreign', is_error: false }] } }, + ]; + writeFileSync(transcript, events.map(event => JSON.stringify(event)).join('\n')); + const audit = auditTranscript(transcript, '/app', { ownEndpoints }); + assert.deepEqual(audit.hits.map(hit => ({ path: hit.path, via: hit.via, kind: hit.kind })), [ + { path: `172.20.0.99:${port}`, via: 'Bash network attempt', kind: 'NETWORK / OTHER RUN' }, + ]); + } + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('verified isolated loopback allows temporary test ports but keeps private hosts restricted', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-isolated-network-')); + try { + const transcript = join(root, 'session.jsonl'); + const loopback = ['localhost:6574', '127.0.0.1:6579', '127.0.0.2:7331', '[::1]:6574', '0.0.0.0:6574']; + const privateHosts = ['host.docker.internal:6574', '10.0.0.1:6574', '172.20.0.99:6574', '192.168.0.1:6574']; + writeFileSync(transcript, [...loopback, ...privateHosts].flatMap((endpoint, i) => [ + { cwd: '/app', message: { content: [{ type: 'tool_use', id: String(i), name: 'Bash', + input: { command: `curl http://${endpoint}/` } }] } }, + { message: { content: [{ type: 'tool_result', tool_use_id: String(i), is_error: false }] } }, + ]).map(event => JSON.stringify(event)).join('\n')); + // /app in a transcript is not proof of isolation; the default remains strict. + assert.equal(auditTranscript(transcript, '/app').hits.length, loopback.length + privateHosts.length); + const isolated = auditTranscript(transcript, '/app', { isolatedLoopback: true }); + assert.deepEqual(isolated.hits.map(hit => hit.path), privateHosts); + assert(isolated.hits.every(hit => hit.kind === 'NETWORK / OTHER RUN')); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('a Grok Build stream is audited through its own tool names', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-grok-audit-')); + const transcript = join(root, '01a112c6-39bd-7d52-a3a9-3158e1b76555.events.jsonl'); + try { + // The shapes grok 1.0.46 prints with --output-format streaming-messages-json. + const events = [ + { type: 'system', subtype: 'init', session_id: '01a112c6-39bd-7d52-a3a9-3158e1b76555', cwd: '/app' }, + { type: 'assistant', message: { content: [{ type: 'tool_use', id: 'call_1', name: 'run_terminal_command', + input: { command: 'cat /tools/stack-bench/grader/grade.ts', description: 'read' } }] } }, + { type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'call_1', content: 'x', is_error: false }] } }, + { type: 'assistant', message: { content: [{ type: 'tool_use', id: 'call_2', name: 'read_file', + input: { target_file: '/outside/notes.md' } }] } }, + { type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'call_2', content: 'x', is_error: false }] } }, + { type: 'assistant', message: { content: [{ type: 'tool_use', id: 'call_3', name: 'search_replace', + input: { file_path: 'src/app.ts', old_string: 'a', new_string: 'b' } }] } }, + { type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'call_3', content: 'ok', is_error: false }] } }, + { type: 'assistant', message: { content: [{ type: 'tool_use', id: 'call_4', name: 'list_dir', + input: { target_directory: '/tools/stack-bench/tracks' } }] } }, + { type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'call_4', content: 'denied', is_error: true }] } }, + { type: 'result', subtype: 'success', is_error: false, session_id: '01a112c6-39bd-7d52-a3a9-3158e1b76555' }, + ]; + writeFileSync(transcript, `${events.map(event => JSON.stringify(event)).join('\n')}\n`); + const audit = auditTranscript(transcript, 'C:/host/app'); + assert.equal(audit.cwd, '/app'); + assert.deepEqual(audit.hits.map(hit => [hit.path, hit.via]), [ + ['/tools/stack-bench/grader/grade.ts', 'Bash'], ['/outside/notes.md', 'Read']]); + assert.deepEqual(audit.refused.map(hit => [hit.path, hit.via]), [['/tools/stack-bench/tracks', 'Glob']]); + assert.equal(audit.fileTool, 3, 'the in-app edit is a file tool use, not a hit'); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('an Antigravity stream is audited through its tool steps', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-antigravity-audit-')); + const id = '86eb1dda-1452-4d14-ab41-9afdc260bdd2'; + const transcript = join(root, `${id}.events.jsonl`); + // The shapes agy 1.3.2 prints with --output-format stream-json: each tool starts, then ends. + const step = (index: number, state: string, name: string, parameters: Record<string, string>, output?: string) => + ({ event: 'step_update', step_update: { conversation_id: id, step_index: index, state, step_type: 'tool', + tool_name: name, tool_info: { name, parameters, ...(output === undefined ? {} : { output }) } } }); + try { + const events = [ + { event: 'init', conversation_id: id, init: { model: 'gemini-3.8-flash', cwd: '/app' } }, + step(2, 'ACTIVE', 'run_command', { CommandLine: 'cat /tools/stack-bench/grader/grade.ts' }), + step(2, 'DONE', 'run_command', { CommandLine: 'cat /tools/stack-bench/grader/grade.ts' }, 'x'), + step(3, 'ACTIVE', 'view_file', { AbsolutePath: '/outside/notes.md' }), + step(3, 'DONE', 'view_file', { AbsolutePath: '/outside/notes.md' }, 'x'), + step(4, 'ACTIVE', 'write_to_file', { TargetFile: '/app/src/app.ts' }), + step(4, 'DONE', 'write_to_file', { TargetFile: '/app/src/app.ts' }), + step(5, 'ACTIVE', 'view_file', { AbsolutePath: '/tools/stack-bench/tracks/spec.md' }), + step(5, 'ERROR', 'view_file', { AbsolutePath: '/tools/stack-bench/tracks/spec.md' }), + { event: 'result', result: { conversation_id: id, status: 'SUCCESS' } }, + ]; + writeFileSync(transcript, `${events.map(event => JSON.stringify(event)).join('\n')}\n`); + const audit = auditTranscript(transcript, 'C:/host/app'); + assert.equal(audit.cwd, '/app'); + assert.deepEqual(audit.hits.map(hit => [hit.path, hit.via]), [ + ['/tools/stack-bench/grader/grade.ts', 'Bash'], ['/outside/notes.md', 'Read']]); + assert.deepEqual(audit.refused.map(hit => [hit.path, hit.via]), [['/tools/stack-bench/tracks/spec.md', 'Read']]); + assert.equal(audit.fileTool, 3, 'the in-app write is a file tool use, not a hit'); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('StackBench source online is matched on any branch, by URL, gh and git, and nothing else of SpacetimeDB is', () => { + const flagged = [ + 'curl -sL https://raw.githubusercontent.com/clockworklabs/SpacetimeDB/master/tools/stack-bench/grader/grade.ts', + 'curl https://raw.githubusercontent.com/clockworklabs/SpacetimeDB/refs/heads/bradley/stackbench-test/tools/stack-bench/README.md', + 'wget https://github.com/clockworklabs/SpacetimeDB/blob/bradley/stackbench-test/tools/stack-bench/tracks/x.json', + 'curl "https://api.github.com/repos/clockworklabs/SpacetimeDB/contents/tools%2Fstack-bench?ref=bradley/stackbench-test"', + 'curl https://api.github.com/repos/clockworklabs/spacetimedb/contents/tools/stack-bench/grader?ref=master', + 'curl -L https://github.com/clockworklabs/SpacetimeDB/pull/5766.diff', + 'curl https://github.com/clockworklabs/SpacetimeDB/pull/5766/files', + 'curl https://api.github.com/repos/clockworklabs/SpacetimeDB/pulls/5766/files', + 'curl https://patch-diff.githubusercontent.com/raw/clockworklabs/SpacetimeDB/pull/5766.patch', + 'gh pr diff 5766 --repo clockworklabs/SpacetimeDB', + 'gh api repos/clockworklabs/SpacetimeDB/contents/tools/stack-bench', + 'git fetch https://github.com/clockworklabs/SpacetimeDB pull/5766/head:pr', + '{"query":"clockworklabs/SpacetimeDB tools/stack-bench grader"}', + // Mirrors serve the same files from other hosts, so the harness path counts on any host. + 'curl https://example.com/clockworklabs/SpacetimeDB/tools/stack-bench', + 'curl https://cdn.jsdelivr.net/gh/clockworklabs/SpacetimeDB@bradley/stackbench-test/tools/stack-bench/grader/grade.ts', + 'curl https://raw.githack.com/clockworklabs/SpacetimeDB/master/tools/stack-bench/README.md', + 'curl "https://sourcegraph.com/github.com/clockworklabs/SpacetimeDB@master/-/blob/tools/stack-bench/grader/grade.ts"', + // A search for the pull request. + '{"query":"clockworklabs SpacetimeDB pull 5766"}', + '{"query":"SpacetimeDB PR #5766 stack bench"}', + '{"query":"clockworklabs/SpacetimeDB pull/5766 files"}', + ]; + for (const command of flagged) assert.equal(stackBenchSourceReferences(command).length > 0, true, command); + const allowed = [ + 'curl -sL https://raw.githubusercontent.com/clockworklabs/SpacetimeDB/master/crates/schema/src/auto_migrate.rs', + 'curl https://github.com/clockworklabs/SpacetimeDB/blob/master/crates/bindings-typescript/README.md', + 'git clone --depth 1 https://github.com/clockworklabs/SpacetimeDB', + 'curl https://github.com/clockworklabs/SpacetimeDB/pull/5767', + 'curl https://github.com/clockworklabs/SpacetimeDB/pull/57660', + 'curl https://cdn.jsdelivr.net/gh/clockworklabs/SpacetimeDB@master/crates/schema/src/auto_migrate.rs', + '{"query":"clockworklabs SpacetimeDB pull 57660"}', + 'curl https://github.com/clockworklabs/spacetimedb-typescript-sdk/tree/main/tools/stack-bench', + 'curl https://github.com/someone/benchmarks/tree/main/tools/stack-bench', + 'gh pr view 5766 --repo someone/other', + '{"query":"spacetimedb auto migration add column"}', + ]; + for (const command of allowed) assert.deepEqual(stackBenchSourceReferences(command), [], command); +}); + +test('fetching StackBench source contaminates through each agent\'s web tools and shell', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-source-audit-')); + const raw = 'https://raw.githubusercontent.com/clockworklabs/SpacetimeDB/bradley/stackbench-test/tools/stack-bench/grader/grade.ts'; + const allowed = 'https://raw.githubusercontent.com/clockworklabs/SpacetimeDB/master/crates/schema/src/auto_migrate.rs'; + const use = (id: string, name: string, input: Record<string, unknown>) => + ({ cwd: '/app', message: { content: [{ type: 'tool_use', id, name, input }] } }); + const result = (id: string, is_error = false) => + ({ message: { content: [{ type: 'tool_result', tool_use_id: id, is_error }] } }); + const write = (name: string, events: unknown[]): string => { + const file = join(root, name); + writeFileSync(file, `${events.map(event => JSON.stringify(event)).join('\n')}\n`); + return file; + }; + const sources = (file: string) => { + const audit = auditTranscript(file, '/app'); + const pick = (hits: typeof audit.hits) => hits.filter(hit => hit.kind === STACKBENCH_SOURCE) + .map(hit => [hit.path, hit.via]); + return { hits: pick(audit.hits), refused: pick(audit.refused), all: audit.hits }; + }; + try { + // Claude Code and Grok Build record tool calls as message content. + const claude = sources(write('claude.jsonl', [ + use('f1', 'WebFetch', { url: raw, prompt: 'read it' }), result('f1'), + use('f2', 'WebFetch', { url: allowed, prompt: 'read it' }), result('f2'), + use('s1', 'WebSearch', { query: 'spacetimedb auto migrate' }), result('s1'), + use('b1', 'Bash', { command: 'curl -sL https://github.com/clockworklabs/SpacetimeDB/pull/5766.diff | head' }), result('b1'), + use('b2', 'Bash', { command: `curl -sL ${raw}` }), result('b2', true), + use('b3', 'Bash', { command: 'cat vendor/SpacetimeDB/tools/stack-bench/grader/grade.ts' }), result('b3'), + ])); + assert.deepEqual(claude.hits, [[raw, 'WebFetch network attempt'], + ['https://github.com/clockworklabs/SpacetimeDB/pull/5766.diff', 'Bash network attempt']]); + assert.deepEqual(claude.refused, [[raw, 'Bash network attempt']]); + // A local copy of the harness is never the app's own file. + assert.deepEqual(claude.all.filter(hit => hit.kind === 'GRADER / TEST SPECS').map(hit => hit.path), + ['/app/vendor/spacetimedb/tools/stack-bench/grader/grade.ts']); + + const grok = sources(write('grok.events.jsonl', [ + use('g1', 'web_fetch', { url: raw }), result('g1'), + use('g2', 'web_search', { query: 'spacetimedb typescript procedures' }), result('g2'), + ])); + assert.deepEqual(grok.hits, [[raw, 'web_fetch network attempt']]); + + // Codex records its hosted search, and each page it opens, as an item. + const codex = sources(write('codex.events.jsonl', [ + { type: 'stack_bench_context', cwd: '/app' }, + { type: 'item.started', item: { id: 'w1', type: 'web_search', query: '' } }, + { type: 'item.completed', item: { id: 'w1', type: 'web_search', query: raw, + action: { type: 'open_page', url: raw } } }, + { type: 'item.completed', item: { id: 'w2', type: 'web_search', query: 'spacetimedb views', + action: { type: 'search', query: 'spacetimedb views' } } }, + ])); + assert.deepEqual(codex.hits, [[raw, 'WebSearch network attempt']]); + + // Antigravity reports each tool as a step that starts, then ends. + const id = '86eb1dda-1452-4d14-ab41-9afdc260bdd2'; + const step = (index: number, state: string, name: string, parameters: Record<string, string>) => + ({ event: 'step_update', step_update: { conversation_id: id, step_index: index, state, step_type: 'tool', + tool_name: name, tool_info: { name, parameters, ...(state === 'DONE' ? { output: 'x' } : {}) } } }); + const antigravity = sources(write(`${id}.events.jsonl`, [ + { event: 'init', conversation_id: id, init: { cwd: '/app' } }, + step(1, 'ACTIVE', 'read_url_content', { Url: raw }), step(1, 'DONE', 'read_url_content', { Url: raw }), + step(2, 'ACTIVE', 'search_web', { query: 'spacetimedb auto migrate' }), + step(2, 'DONE', 'search_web', { query: 'spacetimedb auto migrate' }), + ])); + assert.deepEqual(antigravity.hits, [[raw, 'WebFetch network attempt']]); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/tools/stack-bench/tests/lifecycle-readiness.test.ts b/tools/stack-bench/tests/lifecycle-readiness.test.ts new file mode 100644 index 00000000000..b65bb9d5f23 --- /dev/null +++ b/tools/stack-bench/tests/lifecycle-readiness.test.ts @@ -0,0 +1,17 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { waitFor } from '../src/stacks/lifecycle-readiness.js'; + +test('readiness polling preserves cancellation reasons before and during its delay', async () => { + const reason = new Error('operator stopped the run'); + await assert.rejects(waitFor(async () => false, 1000, 'app', AbortSignal.abort(reason)), + error => error === reason); + const controller = new AbortController(); + const pending = waitFor(async () => false, 1000, 'app', controller.signal); + setImmediate(() => controller.abort(reason)); + await assert.rejects(pending, error => error === reason); + await assert.rejects(waitFor(async () => false, 1000, 'app', AbortSignal.abort(null)), + /backend control cancelled/); + await waitFor(async () => true, 1000, 'app', null); + await assert.rejects(waitFor(async () => false, 0, 'app'), /timed out waiting for app/); +}); diff --git a/tools/stack-bench/tests/linter-coverage.contract.ts b/tools/stack-bench/tests/linter-coverage.contract.ts new file mode 100644 index 00000000000..302d24c6170 --- /dev/null +++ b/tools/stack-bench/tests/linter-coverage.contract.ts @@ -0,0 +1,109 @@ +import assert from 'node:assert/strict'; +import { resolve } from 'node:path'; +import test from 'node:test'; + +import type { Page } from 'playwright'; +import { type LintResult, checkHook, completeAbortedHooks, completeUnvisitedHooks, loadHooks, selectHooks } from '../linter/lint.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; + +interface TestHook { + id: string; + element: string; + stage: string; + check: 'visible' | 'attached'; + note: string; + revealedBy?: string; +} + +test('a selected lint surface excludes unrelated hooks and keeps unknown hooks for scenario grading', () => { + const hooks: TestHook[] = [ + { id: 'accounts', element: 'accounts', stage: 'landing', check: 'visible', note: '' }, + { id: 'cart', element: 'cart', stage: 'cart', check: 'visible', note: '' }, + ]; + assert.deepEqual(selectHooks(hooks), hooks); + assert.deepEqual(selectHooks(hooks, []), []); + const selected = selectHooks(hooks, ['support-link', 'accounts']); + assert.deepEqual(selected.map(hook => hook.id), ['accounts', 'support-link']); + assert.equal(selected[1]?.stage, 'scenario'); + // Selected hooks keep contract order when one control reveals another. + assert.deepEqual(selectHooks([ + { id: 'signin-toggle', element: 'sign-in toggle', stage: 'landing', check: 'visible', note: '' }, + { id: 'signin-username', element: 'sign-in user', stage: 'landing', check: 'attached', note: '', revealedBy: 'signin-toggle' }, + { id: 'signin-password', element: 'sign-in password', stage: 'landing', check: 'attached', note: '', revealedBy: 'signin-toggle' }, + ], ['signin-password', 'signin-toggle', 'signin-username']).map(hook => hook.id), + ['signin-toggle', 'signin-username', 'signin-password']); + // Selected scenario controls do not require retired JSON contract files. + assert.deepEqual(loadHooks(3, { contracts: resolve(STACK_BENCH_ROOT, 'tracks/ecommerce/contracts') }, + ['current-user']), [{ + id: 'current-user', + element: 'the selected application control current-user', + stage: 'scenario', + check: 'visible', + note: 'checked by the selected feature suite', + }]); +}); + +test('contract lint fails closed when a core flow forgets a lintable stage', () => { + const hooks: TestHook[] = [ + { id: 'seen', element: 'seen control', stage: 'landing', check: 'visible', note: '' }, + { id: 'forgotten', element: 'forgotten control', stage: 'operations', check: 'visible', note: '' }, + { id: 'scenario-only', element: 'scenario control', stage: 'scenario', check: 'visible', note: 'requires two actors' }, + ]; + const results: LintResult[] = [{ id: 'seen', status: 'PASS' }]; + + completeUnvisitedHooks(hooks, results); + + assert.deepEqual(results, [ + { id: 'seen', status: 'PASS' }, + { id: 'forgotten', status: 'BLOCKED', + detail: 'the core flow did not visit contract stage "operations"' }, + { id: 'scenario-only', status: 'SCENARIO', detail: 'requires two actors' }, + ]); +}); + +test('an unexpected walk error records one failure before blocking later hooks', () => { + const hooks: TestHook[] = [ + { id: 'seen', element: 'seen control', stage: 'landing', check: 'visible', note: '' }, + { id: 'cart-panel', element: 'cart panel', stage: 'cart', check: 'visible', note: '' }, + { id: 'order-list', element: 'order list', stage: 'after-checkout', check: 'visible', note: '' }, + { id: 'scenario-only', element: 'scenario control', stage: 'scenario', check: 'visible', note: 'requires setup' }, + ]; + const results: LintResult[] = [{ id: 'seen', status: 'PASS' }]; + + completeAbortedHooks(hooks, results, new Error('target product was not visible\nlocator details')); + + assert.deepEqual(results, [ + { id: 'seen', status: 'PASS' }, + { id: 'core-flow', status: 'FAIL', + detail: 'core flow aborted: target product was not visible locator details' }, + { id: 'cart-panel', status: 'BLOCKED', detail: 'core flow aborted' }, + { id: 'order-list', status: 'BLOCKED', detail: 'core flow aborted' }, + { id: 'scenario-only', status: 'SCENARIO', detail: 'requires setup' }, + ]); +}); + +test('browser, script, and navigation faults in the core flow are not app failures', () => { + const hooks: TestHook[] = [{ id: 'cart-panel', element: 'cart panel', stage: 'cart', check: 'visible', note: '' }]; + const coreFlow = (error: Error): string => { + const results: LintResult[] = []; + completeAbortedHooks(hooks, results, error); + return results[0]!.status; + }; + assert.equal(coreFlow(new Error('Target page, context or browser has been closed')), 'HARNESS'); + assert.equal(coreFlow(new TypeError('Cannot read properties of undefined')), 'HARNESS'); + assert.equal(coreFlow(new Error('page.goto: Timeout 15000ms exceeded.')), 'UNMEASURED'); +}); + +test('a hook fails only when its control does not appear', async () => { + const hook: TestHook = { id: 'cart-panel', element: 'cart panel', stage: 'cart', check: 'visible', note: '' }; + const page = (error: Error) => ({ locator: () => ({ first: () => ({ + count: async () => 1, waitFor: async () => { throw error; } }) }) }) as unknown as Page; + const timeout = Object.assign(new Error('locator.waitFor: Timeout 5000ms exceeded.'), { name: 'TimeoutError' }); + for (const [error, status] of [[timeout, 'FAIL'], + [new Error('Target page, context or browser has been closed'), 'HARNESS'], + [new Error('Unexpected token "[" while parsing css selector'), 'HARNESS']] as const) { + const results: LintResult[] = []; + await checkHook(page(error), hook, results); + assert.equal(results[0]!.status, status, error.message); + } +}); diff --git a/tools/stack-bench/tests/live-progression.test.ts b/tools/stack-bench/tests/live-progression.test.ts new file mode 100644 index 00000000000..59ded947aeb --- /dev/null +++ b/tools/stack-bench/tests/live-progression.test.ts @@ -0,0 +1,763 @@ +import { privateGradingDirectory } from '../src/evidence/repair-evidence.js'; +import assert from 'node:assert/strict'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } + from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { loadTrack } from '../src/composition/tracks.js'; +import { requireRecipeRelease as resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { artifactPayload, createArtifact, emptyArtifactIdentities, readArtifact, writeArtifact } + from '../src/evidence/artifacts.js'; +import { createCheckEvidence } from '../src/evidence/check-evidence.js'; +import type { CheckEvidenceStatus } from '../src/evidence/check-evidence.js'; +import { hashAppSource } from '../src/runtime/source-snapshot.js'; +import { compileDependencyPolicyInput, compileFeatureCatalogInput, compileProgressionInput } + from '../src/progression/progression-definition.js'; +import type { ProgressionInput } from '../src/progression/progression-definition.js'; +import { createLiveProgressionExecution, type LiveProgressionExecution, + type LiveProgressionStatus, clearTimeContinuationBoundary, + commitTimeContinuationBoundary, timeContinuationEligibility } + from '../src/progression/live-progression.js'; +import { auditProgressionReferenceRun } + from '../src/progression/progression-reference-audit.js'; +import type { ProgressionWorkAction } from '../src/progression/progression-engine.js'; +import type { ProgressionRecipeAction, ProgressionRecipeSelections } + from '../src/progression/progression-recipe-selection.js'; +import type { ProgressionState } from '../src/progression/progression-state.js'; +import type { ProgressionNodeState } from '../src/progression/progression-state.js'; +import { validateCampaignRun } from '../src/campaigns/campaign-run-validation.js'; +import { compileCampaignFile } from '../src/campaigns/campaign-compiler.js'; +import { createCampaignState, claimNextAttempt, finishCampaignExecution, + scheduleTimeContinuation } from '../src/campaigns/campaign-scheduler.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import type { RepairPlan } from '../src/progression/repair-plan.js'; +import { progressionEngine } from '../src/progression/progression-engine.js'; +import { synchronizeProgressionSummary } from '../commands/bench.js'; +import { aggregateRunOutcome, runExitCode } from '../src/evidence/outcomes.js'; +import type { BenchmarkRunRecord, RunLevelRecord } from '../src/evidence/benchmark-run.js'; + +type RepairPlanInput = Pick<RepairPlan, 'selection' | 'budget'> & { order?: RepairPlan['order'] }; + +test('interrupted repair summaries retain failure identity and charge the recorded depth', () => { + const compiled = compileProgressionInput(definition()); + let state = progressionEngine.initialize(compiled.definition); + state = progressionEngine.recordResult(state, { + attemptId: 'initial', outcome: 'conclusive', selectionSha256: 'a'.repeat(64), + nodes: [{ id: 'accounts', checks: [ + { id: 'ecommerce.feature.accounts.accounts.1a', outcome: 'fail' }, + ] }], + }); + state = progressionEngine.recordResult(state, { + attemptId: 'repair', outcome: 'inconclusive', category: 'harness_failure', + reason: 'target grading failed', completedRepair: true, + }); + const run: Pick<BenchmarkRunRecord, 'validation' | 'levels'> = { + validation: { ladder: { policy: 'dependency-graph', requestedLevels: [1, 2, 3], + completedLevels: [], stoppedAfterLevel: null, blockedLevels: [] } }, + levels: [1, 3].map((level): RunLevelRecord => ({ level, graded: true, score: 1, max: 1, + selection: { sha256: 'a'.repeat(64) }, outcome: { kind: 'passed', reason: null }, + repairs: level === 3 ? 1 : 0, + repair: { status: 'not-needed', used: level === 3 ? 1 : 0, limit: 2, stopReason: null }, + })), + }; + synchronizeProgressionSummary(run, state); + assert.deepEqual(run.validation.ladder.completedLevels, [1]); + assert.equal(run.levels[0]?.graded, false); + assert.equal(run.levels[0]?.score, null); + assert.equal(run.levels[0]?.selection, null); + assert.equal(run.levels[0]?.outcome.reason, 'target grading failed'); + assert.equal(run.levels[0]?.repairs, 1); + assert.equal(run.levels[0]?.repair?.used, 1); + assert.equal(run.levels[1]?.repairs, 0); + assert.equal(run.levels[1]?.repair?.used, 0); + assert.equal(aggregateRunOutcome(run.levels).kind, 'harness_failure'); + assert.notEqual(runExitCode(aggregateRunOutcome(run.levels)), 0); + const once = structuredClone(run); + synchronizeProgressionSummary(run, state); + assert.deepEqual(run, once); + state = progressionEngine.recordResult(state, { + attemptId: 'regraded', outcome: 'conclusive', selectionSha256: 'b'.repeat(64), + nodes: [{ id: 'accounts', checks: [ + { id: 'ecommerce.feature.accounts.accounts.1a', outcome: 'pass' }, + ] }], + }); + Object.assign(run.levels[0]!, { graded: true, score: 1, max: 1, + selection: { sha256: 'b'.repeat(64) }, outcome: { kind: 'passed', reason: null } }); + synchronizeProgressionSummary(run, state); + assert.equal(run.levels[0]?.repair?.status, 'corrected'); + assert.equal(run.levels[0]?.repair?.used, 1); + assert.equal(run.levels[0]?.stalled, false); +}); + +interface FixtureDependency { + id: string; + reason: string; +} + +interface FixtureNode { + id: string; + title: string; + questline: string; + dependencies: FixtureDependency[]; + featureRefs: string[]; + promptModules: string[]; + gradingChecks: Array<{ id: string; points: number; role: 'feature' | 'guarantee' }>; +} + +interface FixtureQuestline { + id: string; + title: string; + nodes?: string[]; +} + +interface FixtureDefinition { + schemaVersion: number; + kind: string; + id: string; + title: string; + policy: string; + repair: RepairPlanInput; + workSelection?: 'feature' | 'progressive' | 'all-at-once'; + nodes: FixtureNode[]; + questlines: FixtureQuestline[]; +} + +type WorkSelection = { action: ProgressionWorkAction } & ProgressionRecipeSelections; + +const object = (value: unknown): value is Record<string, unknown> => + value !== null && typeof value === 'object' && !Array.isArray(value); + +function workSelection(selected: ProgressionRecipeAction): WorkSelection { + assert('grader' in selected, 'expected a progression work action'); + return selected; +} + +function executionState(execution: LiveProgressionExecution): ProgressionState { + assert(execution.state, 'expected initialized progression state'); + return execution.state; +} + +function nodeState(state: ProgressionState, id: string): ProgressionNodeState { + const node = state.nodes[id]; + assert(node, `expected progression node ${id}`); + return node; +} + +function latestStatus(states: LiveProgressionStatus[]): LiveProgressionStatus { + const status = states.at(-1); + assert(status, 'expected a reported progression status'); + return status; +} + +function workAction(action: ProgressionRecipeAction['action']): ProgressionWorkAction { + assert.notEqual(action.type, 'terminal'); + return action as ProgressionWorkAction; +} + +const definition = (): FixtureDefinition => ({ + schemaVersion: 7, + kind: 'progression-mode', + id: 'live-fixture', + title: 'Live fixture', + policy: 'dependency-graph', + repair: { selection: 'feature', budget: { perFeature: 2 } }, + workSelection: 'progressive', + nodes: [{ + id: 'accounts', + title: 'Accounts', + questline: 'identity', + dependencies: [], + featureRefs: ['ecommerce.feature.accounts'], + promptModules: [], + gradingChecks: [{ id: 'ecommerce.feature.accounts.accounts.1a', points: 1, + role: 'feature' }], + }], + questlines: [{ id: 'identity', title: 'Identity' }], +}); +const splitIdentities = (progression: ProgressionInput) => { + const progressionDefinition = progression.definition as typeof progression.definition & { + repair?: RepairPlanInput; + workSelection?: 'feature' | 'progressive' | 'all-at-once'; + }; + const { policy: _policy, repair, unchangedFailureLimit: _limit, workSelection, + ...catalogDefinition } = progressionDefinition; + assert(repair); + const featureCatalog = compileFeatureCatalogInput({ ...catalogDefinition, + schemaVersion: 1, kind: 'feature-catalog' }); + const dependencyPolicy = compileDependencyPolicyInput(repair, featureCatalog, + { workSelection }); + return { featureCatalog, dependencyPolicy, + featureCatalogIdentity: featureCatalog.identity, + dependencyPolicyIdentity: dependencyPolicy.identity }; +}; + +const evidence = (status: CheckEvidenceStatus) => createCheckEvidence({ + status, + code: status === 'passed' ? 'completed' : 'test_result', + phase: 'assertion', + startedAtMs: 1, + completedAtMs: 2, +}); +const setupEvidence = () => createCheckEvidence({ + status: 'passed', + code: 'completed', + phase: 'setup', + startedAtMs: 1, + completedAtMs: 2, +}); + +test('live progression binds and persists one exact accepted action', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-live-progression-')); + try { + const appDir = join(root, 'app'); + const outputDir = join(root, 'result'); + mkdirSync(appDir, { recursive: true }); + mkdirSync(privateGradingDirectory(appDir), { recursive: true }); + mkdirSync(outputDir, { recursive: true }); + writeFileSync(join(appDir, 'index.js'), 'export const ready = true;\n'); + + const progression = compileProgressionInput(definition()); + const split = splitIdentities(progression); + const binding = resolveRecipeRelease(loadTrack('ecommerce'), 1, + 'ecommerce.sequential-l1'); + const owner = { + schemaVersion: 1, + campaign: { id: 'campaign', version: '1.0.0', sha256: 'a'.repeat(64) }, + attempt: { id: 'campaign-r1', track: 'ecommerce', stack: 'postgres', + agentAdapter: 'claude-code', model: 'test-model', conditionSha256: 'b'.repeat(64) }, + workspace: { appDirectory: 'source' }, + }; + const identities = emptyArtifactIdentities({ + experiment: { ...owner.campaign, state: 'draft' }, + agentAdapter: { id: owner.attempt.agentAdapter }, + stackAdapter: { id: owner.attempt.stack, version: null }, + }); + const runArtifact = createArtifact({ + kind: 'benchmark_run', + id: 'run-1', + attempt: { id: 'run-1', parentId: owner.attempt.id }, + identities, + payload: { + mode: { id: 'dependency' }, + backend: owner.attempt.stack, + model: owner.attempt.model, + condition: { contentSha256: owner.attempt.conditionSha256 }, + featureCatalog: split.featureCatalogIdentity, + dependencyPolicy: split.dependencyPolicyIdentity, + progressionOwner: { schemaVersion: 1, campaign: owner.campaign, attempt: owner.attempt }, + }, + }); + const states: LiveProgressionStatus[] = []; + const execution = createLiveProgressionExecution({ + progression, + ...split, + owner, + statePath: join(outputDir, 'progression-state.json'), + runId: 'run-1', + outputDir, + appDir, + track: 'ecommerce', + backend: owner.attempt.stack, + identities, + recipeBindings: new Map([[1, binding]]), + getRunArtifact: () => runArtifact, + onState: status => states.push(status), + }); + execution.initialize(); + const selected = workSelection(execution.bind()); + const source = hashAppSource(appDir); + const key = selected.grader.checkKeys[0]; + const recipe = selected.grader.request.recipe; + const grade = { + observation: 'scored', + source: { sha256: source.sha256 }, + selection: { + sha256: selected.grader.selectionSha256, + checks: [{ stableKey: key, points: 1 }], + attemptedChecks: [key], + reportedChecks: [key], + notRun: [], + }, + totals: { score: 1, max: 1, regression: null }, + suites: { application: { features: [{ setupEvidence: setupEvidence(), criteria: [ + { stableKey: key, points: 1, evidence: evidence('passed') }, + ] }] } }, + outcome: { kind: 'passed' }, + }; + const gradeArtifact = { + kind: 'grade_bundle', + id: 'grade-1', + attempt: { id: 'grade-1', parentId: 'run-1' }, + identities: emptyArtifactIdentities({ + recipe: { id: recipe.id, sha256: recipe.contentSha256 }, + stackAdapter: { id: owner.attempt.stack }, + }), + payload: grade, + }; + writeArtifact(join(privateGradingDirectory(appDir), 'bundle.json'), gradeArtifact); + mkdirSync(join(outputDir, 'grading'), { recursive: true }); + writeArtifact(join(outputDir, 'grading', 'bundle.json'), gradeArtifact); + + const next = execution.record({ selected, + bundle: { ...grade, totals: { score: 0, max: 1, regression: null }, + outcome: { kind: 'app_failure' } }, level: 1 }); + assert(next); + assert.equal(next.type, 'terminal'); + assert.equal(nodeState(executionState(execution), 'accounts').status, 'passed'); + assert.equal(latestStatus(states).score.questlineAveragePercentage, 100); + assert(existsSync(join(outputDir, 'progression-state.json'))); + assert(existsSync(join(outputDir, 'progression', 'attempt-001', 'bundle.json'))); + assert(!existsSync(join(outputDir, 'level-l1-checkpoint.json'))); + assert(existsSync(join(outputDir, 'source', 'index.js'))); + assert.doesNotThrow(() => createArtifact({ + ...runArtifact, + attempt: { id: 'run-1', parentId: owner.attempt.id }, + payload: { ...runArtifact.payload, progressionStatus: states.at(-1) }, + })); + assert.throws(() => createArtifact({ + ...runArtifact, + attempt: { id: 'run-1', parentId: owner.attempt.id }, + payload: { ...runArtifact.payload, + progressionStatus: { ...states.at(-1), stateArtifact: '../state.json' } }, + }), /stateArtifact is invalid/); + + const attempt = { + id: owner.attempt.id, + mode: { id: 'dependency' }, + levels: [1], + featureCatalog: split.featureCatalogIdentity, + dependencyPolicy: split.dependencyPolicyIdentity, + stack: owner.attempt.stack, + agentAdapter: owner.attempt.agentAdapter, + model: owner.attempt.model, + guidance: 'neutral', + condition: { contentSha256: owner.attempt.conditionSha256 }, + skills: [], + }; + const agentIdentity = identities.agentAdapter; + const stackIdentity = identities.stackAdapter; + const engineIdentity = identities.engine; + assert(agentIdentity); + assert(stackIdentity); + assert(engineIdentity); + const plan = { + id: owner.campaign.id, + version: owner.campaign.version, + state: 'draft', + contentSha256: owner.campaign.sha256, + featureCatalog: split.featureCatalog, + dependencyPolicy: split.dependencyPolicy, + definition: { track: owner.attempt.track, selection: { levels: [] }, + runtime: { buildImage: null }, + repair: { selection: 'feature' as const, budget: { perFeature: 2 }, + order: 'declared' as const }, + budgets: { maxCostUsdPerAttempt: null } }, + agents: [{ adapter: owner.attempt.agentAdapter, model: owner.attempt.model, + costLimit: 'non-billable', identity: agentIdentity }], + stacks: [{ id: owner.attempt.stack, version: null }], + conditions: [attempt.condition], + identities: { engine: engineIdentity }, + }; + const completedRun = artifactPayload(createArtifact({ + ...runArtifact, + attempt: { id: 'run-1', parentId: owner.attempt.id }, + payload: { + ...runArtifact.payload, + track: owner.attempt.track, + guidance: attempt.guidance, + condition: attempt.condition, + selectionRequest: plan.definition.selection, + skills: attempt.skills, + runtime: { buildImage: null }, + progressionStatus: states.at(-1), + validation: { ladder: { policy: 'dependency-graph', requestedLevels: [1], + completedLevels: [1], stoppedAfterLevel: null, blockedLevels: [] } }, + levels: [{ level: 1, selection: selected.grader.selection, + graded: true, score: 1, max: 1, repairs: 0, + repair: { status: 'not-needed', limit: 0, used: 0, + stopReason: 'not-needed', nodeRepairs: [ + { nodeId: 'accounts', used: 0, exhaustionReason: null }, + ] }, + outcome: { kind: 'passed' } }], + totals: { score: 1, max: 1, costUsd: 0 }, + outcome: { kind: 'passed' }, + }, + })); + assert.equal(validateCampaignRun(plan, attempt, completedRun, { resultDir: outputDir }), + completedRun); + assert.throws(() => validateCampaignRun(plan, attempt, { + ...completedRun, + totals: { ...completedRun.totals, score: 2 }, + }, { resultDir: outputDir }), /totals\.score/); + assert.throws(() => validateCampaignRun(plan, attempt, { + ...completedRun, + progressionStatus: { ...completedRun.progressionStatus, attempts: 0 }, + }, { resultDir: outputDir }), /progressionStatus/); + assert.throws(() => validateCampaignRun(plan, attempt, { + ...completedRun, + levels: completedRun.levels?.map(level => ({ ...level, + repair: { ...level.repair, nodeRepairs: level.repair?.nodeRepairs?.map(node => ({ + ...node, used: node.nodeId === 'accounts' ? 1 : node.used, + })) } })), + }, { resultDir: outputDir }), /levels\.L1\.repair\.nodeRepairs/); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('live progression records provider interruptions without consuming a repair', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-live-provider-failure-')); + try { + const appDir = join(root, 'app'); + const outputDir = join(root, 'result'); + mkdirSync(appDir, { recursive: true }); + mkdirSync(outputDir, { recursive: true }); + const progression = compileProgressionInput(definition()); + const split = splitIdentities(progression); + const binding = resolveRecipeRelease(loadTrack('ecommerce'), 1, + 'ecommerce.sequential-l1'); + const owner = { schemaVersion: 1, + campaign: { id: 'campaign', version: '1.0.0', sha256: 'a'.repeat(64) }, + attempt: { id: 'campaign-r1', track: 'ecommerce', stack: 'postgres', + agentAdapter: 'claude-code', model: 'test-model', conditionSha256: 'b'.repeat(64) }, + workspace: { appDirectory: 'source' } }; + const identities = emptyArtifactIdentities({ + agentAdapter: { id: owner.attempt.agentAdapter }, + stackAdapter: { id: owner.attempt.stack }, + }); + const runArtifact = createArtifact({ kind: 'benchmark_run', id: 'run-1', + attempt: { id: 'run-1', parentId: owner.attempt.id }, identities, + payload: { backend: owner.attempt.stack, model: owner.attempt.model, + condition: { contentSha256: owner.attempt.conditionSha256 }, + featureCatalog: split.featureCatalogIdentity, + dependencyPolicy: split.dependencyPolicyIdentity, + progressionOwner: { schemaVersion: 1, campaign: owner.campaign, attempt: owner.attempt } } }); + const execution = createLiveProgressionExecution({ progression, ...split, owner, + statePath: join(outputDir, 'progression-state.json'), runId: 'run-1', + outputDir, appDir, track: owner.attempt.track, backend: owner.attempt.stack, + identities, recipeBindings: new Map([[1, binding]]), + getRunArtifact: () => runArtifact }); + execution.initialize(); + const selected = workSelection(execution.bind()); + execution.record({ selected, bundle: null, level: 1, + failure: { kind: 'provider_failure', reason: 'provider-connection-error' } }); + const state = executionState(execution); + const attempt = state.attempts[0]; + assert(attempt); + assert.equal(state.attempts.length, 1); + assert.equal(attempt.outcome, 'inconclusive'); + assert.equal(attempt.category, 'provider_failure'); + assert.equal(nodeState(state, 'accounts').repairs.used, 0); + assert.equal(existsSync(join(outputDir, 'progression', 'attempt-001')), false); + assert.equal(existsSync(join(outputDir, 'progression-state.json')), true); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('an interrupted execution restores the saved source and resumes the next graph action', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-live-resume-')); + try { + const firstOutput = join(root, 'execution-1'); + const secondOutput = join(root, 'execution-2'); + const firstApp = join(root, 'first-app'); + const secondApp = join(root, 'second-app'); + for (const path of [firstOutput, secondOutput, firstApp, secondApp]) { + mkdirSync(path, { recursive: true }); + } + writeFileSync(join(firstApp, 'index.js'), 'export const level = 1;\n'); + writeFileSync(join(secondApp, 'index.js'), 'export const interrupted = true;\n'); + + const value = definition(); + const accounts = value.nodes[0]; + assert(accounts); + value.nodes = [ + accounts, + { id: 'catalog', title: 'Catalog', questline: 'catalog', dependencies: [], + featureRefs: ['ecommerce.feature.catalog-items'], promptModules: [], + gradingChecks: [{ id: 'ecommerce.feature.catalog.catalog-values.2a', points: 1, + role: 'feature' }] }, + { id: 'purchasing', title: 'Purchasing', questline: 'identity', dependencies: [ + { id: 'accounts', reason: 'Purchasing requires an account.' }, + { id: 'catalog', reason: 'Purchasing requires a catalog item.' }, + ], featureRefs: ['ecommerce.feature.purchasing'], promptModules: [], + gradingChecks: [{ id: 'ecommerce.feature.purchasing.purchase-order.3c', points: 1, + role: 'feature' }] }, + ]; + value.questlines = [ + { id: 'identity', title: 'Identity', nodes: ['accounts', 'purchasing'] }, + { id: 'catalog', title: 'Catalog', nodes: ['catalog'] }, + ]; + const progression = compileProgressionInput(value); + const split = splitIdentities(progression); + const binding = resolveRecipeRelease(loadTrack('ecommerce'), 1, + 'ecommerce.sequential-l1'); + const owner = { + schemaVersion: 1, + campaign: { id: 'campaign', version: '1.0.0', sha256: 'a'.repeat(64) }, + attempt: { id: 'campaign-r1', track: 'ecommerce', stack: 'postgres', + agentAdapter: 'claude-code', model: 'test-model', conditionSha256: 'b'.repeat(64) }, + workspace: { appDirectory: 'source' }, + }; + const identities = emptyArtifactIdentities({ + agentAdapter: { id: owner.attempt.agentAdapter }, + stackAdapter: { id: owner.attempt.stack }, + }); + const runArtifact = createArtifact({ + kind: 'benchmark_run', id: 'run-1', + attempt: { id: 'run-1', parentId: owner.attempt.id }, identities, + payload: { backend: owner.attempt.stack, model: owner.attempt.model, + condition: { contentSha256: owner.attempt.conditionSha256 }, + featureCatalog: split.featureCatalogIdentity, + dependencyPolicy: split.dependencyPolicyIdentity, + progressionOwner: { schemaVersion: 1, campaign: owner.campaign, attempt: owner.attempt } }, + }); + const first = createLiveProgressionExecution({ progression, ...split, owner, + statePath: join(firstOutput, 'progression-state.json'), runId: 'run-1', + outputDir: firstOutput, appDir: firstApp, track: owner.attempt.track, + backend: owner.attempt.stack, identities, + recipeBindings: new Map([[1, binding], [2, binding]]), + getRunArtifact: () => runArtifact }); + assert.equal(first.initialize().resumed, false); + const selected = workSelection(first.bind()); + const source = hashAppSource(firstApp); + const checks = selected.grader.checkKeys.map(stableKey => ({ stableKey, points: 1 })); + const recipe = selected.grader.request.recipe; + const grade = { + observation: 'scored', source: { sha256: source.sha256 }, + selection: { sha256: selected.grader.selectionSha256, checks, + attemptedChecks: checks.map(check => check.stableKey), + reportedChecks: checks.map(check => check.stableKey), notRun: [] }, + totals: { score: checks.length, max: checks.length, regression: null }, + suites: { application: { features: [{ setupEvidence: setupEvidence(), criteria: + checks.map(check => ({ ...check, evidence: evidence('passed') })) }] } }, + outcome: { kind: 'passed' }, + }; + mkdirSync(firstApp, { recursive: true }); + mkdirSync(privateGradingDirectory(firstApp), { recursive: true }); + writeArtifact(join(privateGradingDirectory(firstApp), 'bundle.json'), { + kind: 'grade_bundle', id: 'grade-1', + attempt: { id: 'grade-1', parentId: 'run-1' }, + identities: emptyArtifactIdentities({ recipe: { id: recipe.id, + sha256: recipe.contentSha256 }, stackAdapter: { id: owner.attempt.stack } }), + payload: grade, + }); + const next = first.record({ selected, bundle: grade, level: 1 }); + assert(next); + assert.equal(next.level, 2); + writeArtifact(join(firstOutput, 'run.json'), { + ...runArtifact, + payload: { ...runArtifact.payload, progressionStatus: first.status(), + totals: { costUsd: 1.25, costComplete: true } }, + }); + assert.equal(timeContinuationEligibility(firstOutput).eligible, false); + commitTimeContinuationBoundary(firstOutput, executionState(first), 1); + assert.equal(timeContinuationEligibility(firstOutput).eligible, true); + const savedRun = readArtifact<Record<string, unknown>>(join(firstOutput, 'run.json')); + savedRun.payload.backendLease = { state: 'released' }; + savedRun.payload.backendDiagnostics = { captured: true }; + writeArtifact(join(firstOutput, 'run.json'), savedRun); + assert.equal(timeContinuationEligibility(firstOutput).eligible, true); + savedRun.payload.totals = { costUsd: 1.5, costComplete: true }; + writeArtifact(join(firstOutput, 'run.json'), savedRun); + assert.equal(timeContinuationEligibility(firstOutput).eligible, false); + savedRun.payload.totals = { costUsd: 1.25, costComplete: true }; + writeArtifact(join(firstOutput, 'run.json'), savedRun); + // Same-depth snapshots cannot preserve the active level's session ledger. + commitTimeContinuationBoundary(firstOutput, executionState(first), 2); + assert.equal(timeContinuationEligibility(firstOutput).eligible, false); + commitTimeContinuationBoundary(firstOutput, executionState(first), 1); + const sourcePath = join(firstOutput, 'source', 'index.js'); + const sourceBytes = readFileSync(sourcePath); + writeFileSync(sourcePath, 'changed'); + assert.equal(timeContinuationEligibility(firstOutput).eligible, false); + writeFileSync(sourcePath, sourceBytes); + assert.equal(timeContinuationEligibility(firstOutput).eligible, true); + // Invalidate before newer work, even if that work never returns a receipt. + clearTimeContinuationBoundary(firstOutput); + assert.equal(timeContinuationEligibility(firstOutput).eligible, false); + commitTimeContinuationBoundary(firstOutput, executionState(first), 1); + const proof = timeContinuationEligibility(firstOutput); + assert(proof.eligible); + const campaign = compileCampaignFile(join(STACK_BENCH_ROOT, 'tests', 'fixtures', + 'campaign.deterministic.json')); + const scheduled = createCampaignState(campaign); + const target = scheduled.attempts[0]!; + target.plan.mode = { id: 'dependency', version: '3.2.0' }; + const started = claimNextAttempt(scheduled, { admissionId: 'boundary-test' }); + assert(started.claim); + const stopped = finishCampaignExecution(started.state, started.claim.executionId, + { exitCode: 1, timedOut: true }); + const oldExecution = structuredClone(stopped.attempts[0]!.executions[0]!); + const added = scheduleTimeContinuation(stopped, target.plan.id, { + request: { campaignSha256: campaign.contentSha256, attemptId: target.plan.id, + executionId: started.claim.executionId, grantId: 'more-time', minutes: 60, + requestedAt: new Date().toISOString() }, + disposition: 'accepted', acceptedAt: new Date().toISOString(), + previousMinutes: campaign.definition.budgets.attemptTimeoutMinutes, + effectiveMinutes: campaign.definition.budgets.attemptTimeoutMinutes + 60, + }, proof.stateSha256); + const continuation = claimNextAttempt(added, { admissionId: 'boundary-resume' }); + assert(continuation.claim); + assert.equal(continuation.claim.resumeFrom, oldExecution.output); + assert.deepEqual(continuation.claim.priorOutputs, [oldExecution.output]); + assert.equal(continuation.state.attempts[0]!.executions[0]!.outcome, 'timed_out'); + assert.equal(continuation.state.attempts[0]!.executions.length, 2); + writeFileSync(join(firstApp, 'index.js'), 'export const interrupted = true;\n'); + + const resumed = createLiveProgressionExecution({ progression, ...split, owner, + statePath: join(secondOutput, 'progression-state.json'), runId: 'run-2', + outputDir: secondOutput, appDir: secondApp, track: owner.attempt.track, + backend: owner.attempt.stack, identities, resumeFrom: proof.resumeFrom, + recipeBindings: new Map([[1, binding], [2, binding]]), + getRunArtifact: () => { throw new Error('grading is not part of this resume check'); } }); + const restored = resumed.initialize(); + assert.equal(restored.resumed, true); + assert.equal(restored.action.type, 'build'); + assert.equal(restored.action.level, 2); + assert.deepEqual(restored.action, next); + assert.equal(restored.priorRun?.payload.totals?.costUsd, 1.25); + const restoredAction = workAction(restored.action); + assert(object(restoredAction.prompt)); + assert.deepEqual(restoredAction.prompt.nodeIds, ['purchasing']); + const resumedState = executionState(resumed); + assert.equal(resumedState.attempts.length, 1); + assert.equal(nodeState(resumedState, 'accounts').status, 'passed'); + assert.equal(nodeState(resumedState, 'catalog').status, 'passed'); + assert.equal(readFileSync(join(secondApp, 'index.js'), 'utf8'), + 'export const level = 1;\n'); + assert(existsSync(join(secondOutput, 'progression-state.json'))); + + writeFileSync(join(secondOutput, 'source', 'index.js'), 'tampered\n'); + const rejected = createLiveProgressionExecution({ progression, ...split, owner, + statePath: join(secondOutput, 'progression-state.json'), runId: 'run-2', + outputDir: secondOutput, appDir: secondApp, track: owner.attempt.track, + backend: owner.attempt.stack, identities, + recipeBindings: new Map([[1, binding], [2, binding]]), + getRunArtifact: () => runArtifact }); + assert.throws(() => rejected.initialize(), /source does not match its state/); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('an interrupted repair resumes with its failed evidence and regression feedback', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-live-repair-resume-')); + try { + const firstOutput = join(root, 'execution-1'); + const secondOutput = join(root, 'execution-2'); + const firstApp = join(root, 'first-app'); + const secondApp = join(root, 'second-app'); + for (const path of [firstOutput, secondOutput, firstApp, secondApp]) { + mkdirSync(path, { recursive: true }); + } + writeFileSync(join(firstApp, 'index.js'), 'export const broken = true;\n'); + const input = definition(); + input.repair.budget.perFeature = 3; + const progression = compileProgressionInput(input); + const split = splitIdentities(progression); + const binding = resolveRecipeRelease(loadTrack('ecommerce'), 1, + 'ecommerce.sequential-l1'); + const owner = { schemaVersion: 1, + campaign: { id: 'campaign', version: '1.0.0', sha256: 'a'.repeat(64) }, + attempt: { id: 'campaign-r1', track: 'ecommerce', stack: 'postgres', + agentAdapter: 'claude-code', model: 'test-model', conditionSha256: 'b'.repeat(64) }, + workspace: { appDirectory: 'source' } }; + const identities = emptyArtifactIdentities({ + agentAdapter: { id: owner.attempt.agentAdapter }, + stackAdapter: { id: owner.attempt.stack }, + }); + const runArtifact = createArtifact({ kind: 'benchmark_run', id: 'run-1', + attempt: { id: 'run-1', parentId: owner.attempt.id }, identities, + payload: { backend: owner.attempt.stack, model: owner.attempt.model, + condition: { contentSha256: owner.attempt.conditionSha256 }, + featureCatalog: split.featureCatalogIdentity, + dependencyPolicy: split.dependencyPolicyIdentity, + progressionOwner: { schemaVersion: 1, campaign: owner.campaign, attempt: owner.attempt } } }); + const first = createLiveProgressionExecution({ progression, ...split, owner, + statePath: join(firstOutput, 'progression-state.json'), runId: 'run-1', + outputDir: firstOutput, appDir: firstApp, track: owner.attempt.track, + backend: owner.attempt.stack, identities, recipeBindings: new Map([[1, binding]]), + getRunArtifact: () => runArtifact }); + first.initialize(); + const selected = workSelection(first.bind()); + const source = hashAppSource(firstApp); + const key = selected.grader.checkKeys[0]; + const recipe = selected.grader.request.recipe; + const grade = { observation: 'scored', source: { sha256: source.sha256 }, + selection: { sha256: selected.grader.selectionSha256, + checks: [{ stableKey: key, points: 1 }], attemptedChecks: [key], + reportedChecks: [key], notRun: [] }, + totals: { score: 0, max: 1, regression: null }, + suites: { application: { features: [{ setupEvidence: setupEvidence(), criteria: [ + { stableKey: key, points: 1, evidence: evidence('failed') }, + ] }] } } }; + mkdirSync(firstApp, { recursive: true }); + mkdirSync(privateGradingDirectory(firstApp), { recursive: true }); + writeArtifact(join(privateGradingDirectory(firstApp), 'bundle.json'), { + kind: 'grade_bundle', id: 'grade-failed', + attempt: { id: 'grade-failed', parentId: 'run-1' }, + identities: emptyArtifactIdentities({ recipe: { id: recipe.id, + sha256: recipe.contentSha256 }, stackAdapter: { id: owner.attempt.stack } }), + payload: grade, + }); + const next = first.record({ selected, bundle: grade, level: 1 }); + assert(next); + assert.equal(next.type, 'repair'); + const repairSelected = workSelection(first.bind()); + const repairRecipe = repairSelected.grader.request.recipe; + const repairGrade = { ...grade, selection: { + ...grade.selection, sha256: repairSelected.grader.selectionSha256, + } }; + writeArtifact(join(privateGradingDirectory(firstApp), 'bundle.json'), { + kind: 'grade_bundle', id: 'grade-regression', + attempt: { id: 'grade-regression', parentId: 'run-1' }, + identities: emptyArtifactIdentities({ recipe: { id: repairRecipe.id, + sha256: repairRecipe.contentSha256 }, + stackAdapter: { id: owner.attempt.stack } }), + payload: repairGrade, + }); + const afterRegression = first.record({ selected: repairSelected, bundle: repairGrade, level: 1, + completedRepair: true, + repairRegression: { ownerNodeIds: ['accounts'], + report: '# Previous repair regression\n\nThe earlier account behavior stopped working.\n' } }); + assert(afterRegression); + assert.equal(afterRegression.type, 'repair'); + writeArtifact(join(firstOutput, 'run.json'), { ...runArtifact, + payload: { ...runArtifact.payload, progressionStatus: first.status(), + totals: { costUsd: 1.25, costComplete: true }, levels: [] } }); + const audit = auditProgressionReferenceRun({ outputDir: firstOutput, progression, + ...split, owner, recipeBindings: new Map([[1, binding]]), release: binding.release }); + assert.equal(audit.actions.length, 2); + + const resumed = createLiveProgressionExecution({ progression, ...split, owner, + statePath: join(secondOutput, 'progression-state.json'), runId: 'run-2', + outputDir: secondOutput, appDir: secondApp, track: owner.attempt.track, + backend: owner.attempt.stack, identities, resumeFrom: firstOutput, + recipeBindings: new Map([[1, binding]]), + getRunArtifact: () => { throw new Error('grading is not part of this resume check'); } }); + mkdirSync(secondApp, { recursive: true }); + mkdirSync(privateGradingDirectory(secondApp), { recursive: true }); + writeFileSync(join(privateGradingDirectory(secondApp), 'stale.json'), '{}\n'); + const restored = resumed.initialize(); + assert.equal(restored.action.type, 'repair'); + assert.equal(restored.action.level, 1); + assert.equal(executionState(resumed).attempts.length, 2); + assert.deepEqual(executionState(resumed).attempts.at(-1)?.repairRegression, + { ownerNodeIds: ['accounts'], + report: '# Previous repair regression\n\nThe earlier account behavior stopped working.\n' }); + assert.equal(readFileSync(join(secondApp, 'index.js'), 'utf8'), + 'export const broken = true;\n'); + assert(existsSync(join(privateGradingDirectory(secondApp), 'bundle.json'))); + assert.equal(existsSync(join(privateGradingDirectory(secondApp), 'stale.json')), false); + assert(existsSync(join(secondOutput, 'progression', 'attempt-002', 'bundle.json'))); + assert(restored.priorRun); + const totals = restored.priorRun.payload.totals; + assert(object(totals)); + assert.equal(totals.costUsd, 1.25); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/tools/stack-bench/tests/managed-dev.test.ts b/tools/stack-bench/tests/managed-dev.test.ts new file mode 100644 index 00000000000..136edacbec6 --- /dev/null +++ b/tools/stack-bench/tests/managed-dev.test.ts @@ -0,0 +1,52 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { execFileSync, spawn } from 'node:child_process'; +import test from 'node:test'; +import { validateProject } from '../container/spacetime-dev.js'; + +test('managed development validates the assigned target and application paths', () => { + const root = mkdtempSync(join(tmpdir(), 'managed-dev-')); + const config = { server: 'http://localhost:3210', database: 'app', 'module-path': 'module', + generate: [{ language: 'typescript', 'out-dir': 'client/bindings' }] }; + mkdirSync(join(root, 'module')); + const save = (value: unknown) => writeFileSync(join(root, 'spacetime.json'), JSON.stringify(value)); + try { + save(config); assert.doesNotThrow(() => validateProject(root, config.server, 'app')); + assert.throws(() => validateProject(root, config.server, 'other'), /supplied/); + save({ ...config, generate: [] }); assert.throws(() => validateProject(root, config.server, 'app'), /generate/); + save({ ...config, 'module-path': '../outside' }); assert.throws(() => validateProject(root, config.server, 'app'), /inside/); + save(config); writeFileSync(join(root, 'spacetime.local.json'), '{"database":"other"}'); + assert.throws(() => validateProject(root, config.server, 'app'), /supplied/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('managed watcher starts once, reports readiness, stops and exposes startup failures', + { skip: process.platform !== 'linux' }, async () => { + const root = mkdtempSync(join(tmpdir(), 'managed-dev-')); + const state = join(root, 'state'); mkdirSync(state); mkdirSync(join(root, 'module')); + const cli = join(root, 'cli'); + writeFileSync(join(root, 'spacetime.json'), JSON.stringify({ server: 'http://localhost:3210', + database: 'app', 'module-path': 'module', generate: [{ language: 'typescript', 'out-dir': 'bindings' }] })); + writeFileSync(cli, '#!/usr/bin/env node\nconst fs=require("node:fs"); fs.writeFileSync(process.argv[process.argv.indexOf("--ready-file")+1],"ready"); setInterval(()=>{},1000);', { mode: 0o755 }); + const args = ['-w', '15', join(state, 'command.lock'), process.execPath, + resolve('dist/container/spacetime-dev.js'), root, state, cli, 'http://localhost:3210', 'app']; + const run = (command: string) => execFileSync('flock', [...args, command], { encoding: 'utf8', timeout: 20000 }); + try { + const start = () => new Promise<string>((done, reject) => { + const child = spawn('flock', [...args, 'start']); let output = ''; + child.stdout.on('data', data => { output += data; }); + child.once('error', reject); child.once('exit', code => code === 0 ? done(output) : reject(new Error(output))); + }); + const results = await Promise.all([start(), start()]); + assert(results.every(value => value.includes('Running'))); + const before = readFileSync(join(state, 'process.json'), 'utf8'); + assert.match(run('start'), /Running/); + assert.equal(readFileSync(join(state, 'process.json'), 'utf8'), before); + assert.match(run('stop'), /Stopped/); assert.match(run('status'), /Not running/); + writeFileSync(cli, '#!/bin/sh\necho compile-failed >&2\nexit 1\n'); + assert.throws(() => run('start'), /Watcher exited/); + assert.match(readFileSync(join(state, 'watcher.log'), 'utf8'), /compile-failed/); + } finally { run('stop'); rmSync(root, { recursive: true, force: true }); } + }); diff --git a/tools/stack-bench/tests/modular-selection.test.ts b/tools/stack-bench/tests/modular-selection.test.ts new file mode 100644 index 00000000000..d80e5772968 --- /dev/null +++ b/tools/stack-bench/tests/modular-selection.test.ts @@ -0,0 +1,201 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { createModularRecipeTaskRequest, resolveModularRecipeSelection, + resolveModularRecipeTaskRequest } from '../src/composition/recipe-selection.js'; +import type { ModularRecipePack, ModularRecipeRelease, ModularRecipeTaskBinding, + RecipeTaskPlan } from '../src/composition/recipe-selection.js'; +import type { RecipeCheck } from '../src/composition/recipe-release.js'; + +const module = (id: string, moduleType: 'feature' | 'specification', requiresPacks: string[] = []): + ModularRecipePack => ({ + id, moduleType, requiresPacks, +}); +const check = (packId: string, suffix: string, observations?: string[], + requiresFeatures?: string[]): RecipeCheck => ({ + stableKey: `${packId}.${suffix}`, packId, executionId: packId, points: 1, + ...(observations ? { observations } : {}), + ...(requiresFeatures ? { requiresFeatures } : {}), +}); +const release: ModularRecipeRelease = { + id: 'example.modular', + contentSha256: 'a'.repeat(64), + components: { packs: [ + module('example.accounts', 'feature'), + module('example.cart', 'feature', ['example.accounts']), + module('example.durability', 'specification'), + module('example.concurrency', 'specification'), + module('example.reconnect', 'specification'), + ] }, + checkCatalog: [ + check('example.accounts', 'works'), + check('example.cart', 'works'), + check('example.durability', 'survives', ['requested', 'unmentioned'], ['example.cart']), + check('example.concurrency', 'safe', ['requested', 'unmentioned'], ['example.cart']), + check('example.reconnect', 'catches-up', ['requested', 'unmentioned'], ['example.cart']), + ], +}; + +const plan: RecipeTaskPlan = { packs: release.components.packs, recipe: { task: { + mode: 'fresh', + requirements: [ + { id: 'frame', owners: ['recipe'], modes: ['fresh'], text: 'Build this product.\n' }, + { id: 'accounts', owners: ['example.accounts'], modes: ['fresh'], text: 'Support accounts.\n' }, + { id: 'cart', owners: ['example.cart'], modes: ['fresh'], text: 'Support a cart.\n' }, + { id: 'durability', owners: ['example.durability'], modes: ['fresh'], text: 'Survive restarts.\n' }, + { id: 'concurrency', owners: ['example.concurrency'], modes: ['fresh'], text: 'Handle races.\n' }, + { id: 'reconnect', owners: ['example.reconnect'], modes: ['fresh'], text: 'Catch up after reconnect.\n' }, + ], + contracts: [ + { id: 'cart-hook', owners: ['example.cart'], modes: ['fresh'], text: 'Expose a cart button.\n' }, + { id: 'probe-hook', owners: ['example.concurrency'], modes: ['fresh'], text: 'DO NOT LEAK.\n' }, + ], +} } }; +const binding: ModularRecipeTaskBinding = { release, plan }; + +test('requested, expected, and observed specification treatments stay independent', () => { + const selection = resolveModularRecipeSelection(release, { + featureIds: ['example.cart'], + requestedSpecifications: ['example.durability'], + expectedSpecifications: ['example.concurrency'], + observedSpecifications: ['example.reconnect'], + }); + assert.deepEqual(selection.features, ['example.accounts', 'example.cart']); + assert.deepEqual(selection.specifications, { + requested: ['example.durability'], + expected: ['example.concurrency'], + observed: ['example.reconnect'], + }); + assert.deepEqual(selection.promptPacks, + ['example.accounts', 'example.cart', 'example.durability']); + assert.deepEqual(selection.scoredChecks.map(item => item.stableKey), [ + 'example.accounts.works', 'example.cart.works', 'example.durability.survives', + 'example.concurrency.safe', + ]); + const expectedCheck = selection.scoredChecks.find(item => + item.stableKey === 'example.concurrency.safe'); + assert(expectedCheck); + assert.equal(expectedCheck.treatment, 'expected'); + assert.deepEqual(selection.observedChecks.map(item => item.stableKey), + ['example.reconnect.catches-up']); + assert.match(selection.sha256, /^[a-f0-9]{64}$/); +}); + +test('recursive feature dependencies contribute their stable family', () => { + const familyRelease: ModularRecipeRelease = { + ...release, + components: { packs: [ + { ...module('example.accounts-v1', 'feature'), stableId: 'example.accounts' }, + { ...module('example.accounts-v2', 'feature'), stableId: 'example.accounts' }, + module('example.cart-v2', 'feature', ['example.accounts-v2']), + ] }, + checkCatalog: [ + check('example.accounts-v2', 'works'), + check('example.cart-v2', 'works', undefined, ['example.accounts-v1']), + ], + }; + const selection = resolveModularRecipeSelection(familyRelease, { + featureIds: ['example.cart-v2'], + }); + assert.deepEqual(selection.features, ['example.accounts-v2', 'example.cart-v2']); + assert.deepEqual(selection.scoredChecks.map(item => item.stableKey), [ + 'example.accounts-v2.works', 'example.cart-v2.works', + ]); +}); + +test('modular selection rejects overlap, wrong module kinds, and unobservable checks', () => { + assert.throws(() => resolveModularRecipeSelection(release, { + requestedSpecifications: ['example.durability'], + expectedSpecifications: ['example.durability'], + }), /both requested and expected/); + assert.throws(() => resolveModularRecipeSelection(release, { + requestedSpecifications: ['example.cart'], + }), /no requested specification/); + const noProbe = structuredClone(release); + const noProbeCheck = noProbe.checkCatalog.find(item => item.packId === 'example.concurrency'); + assert(noProbeCheck); + noProbeCheck.observations = ['requested']; + assert.throws(() => resolveModularRecipeSelection(noProbe, { + expectedSpecifications: ['example.concurrency'], + }), /has no evaluation without prompting/); + const noRequested = structuredClone(release); + const noRequestedCheck = noRequested.checkCatalog.find(item => item.packId === 'example.durability'); + assert(noRequestedCheck); + noRequestedCheck.observations = ['unmentioned']; + assert.throws(() => resolveModularRecipeSelection(noRequested, { + requestedSpecifications: ['example.durability'], + }), /has no prompted evaluation/); + assert.throws(() => resolveModularRecipeSelection(release, { + featureIds: ['example.accounts'], + requestedSpecifications: ['example.durability'], + }), /has no prompted evaluation/); + const featureAddingSpec = structuredClone(release); + const durabilityPack = featureAddingSpec.components.packs.find(item => item.id === 'example.durability'); + assert(durabilityPack); + durabilityPack.requiresPacks = ['example.cart']; + assert.throws(() => resolveModularRecipeSelection(featureAddingSpec, { + featureIds: ['example.accounts'], + requestedSpecifications: ['example.durability'], + }), /cannot add feature example.cart/); +}); + +test('scored check filters can select expected checks but cannot reach observed-only scope', () => { + const expected = resolveModularRecipeSelection(release, { + expectedSpecifications: ['example.concurrency'], + checkKeys: ['example.concurrency.safe'], + }); + assert.deepEqual(expected.scoredChecks.map(check => check.stableKey), + ['example.concurrency.safe']); + assert.throws(() => resolveModularRecipeSelection(release, { + observedSpecifications: ['example.reconnect'], + checkKeys: ['example.reconnect.catches-up'], + }), /outside the requested\/expected/); +}); + +test('modular task composition includes requested specs and withholds expected and observed specs', () => { + const compiled = createModularRecipeTaskRequest(binding, { + featureIds: ['example.cart'], + requestedSpecifications: ['example.durability'], + expectedSpecifications: ['example.concurrency'], + observedSpecifications: ['example.reconnect'], + }); + assert.equal(compiled.task.requirementText, + 'Build this product.\n\nSupport accounts.\n\nSupport a cart.\n\nSurvive restarts.\n'); + assert.equal(compiled.task.contractText, 'Expose a cart button.\n'); + assert.equal(compiled.task.requirementText.includes('Handle races'), false); + assert.equal(compiled.task.requirementText.includes('Catch up after reconnect'), false); + assert.equal(compiled.task.contractText.includes('DO NOT LEAK'), false); + assert.deepEqual(compiled.request.selection.scoredChecks, [ + 'example.accounts.works', 'example.cart.works', 'example.durability.survives', + 'example.concurrency.safe', + ]); + assert.deepEqual(compiled.request.selection.observedChecks, ['example.reconnect.catches-up']); + assert.deepEqual(resolveModularRecipeTaskRequest(binding, compiled.request).request, + compiled.request); + + const requestedOnly = createModularRecipeTaskRequest(binding, { + featureIds: ['example.cart'], + requestedSpecifications: ['example.durability'], + }); + assert.equal(requestedOnly.task.requirementSha256, compiled.task.requirementSha256); + assert.equal(requestedOnly.task.contractSha256, compiled.task.contractSha256); + assert.equal(requestedOnly.task.sha256, compiled.task.sha256); + assert.notEqual(requestedOnly.selection.sha256, compiled.selection.sha256); +}); + +test('a fresh action includes directly selected upgrade-only feature text', () => { + const actionPlan = structuredClone(plan); + actionPlan.recipe.task.mode = 'action'; + actionPlan.recipe.task.requirements.push({ + id: 'upgrade-frame', owners: ['recipe'], modes: ['upgrade'], text: 'Update this product.\n', + }); + const cart = actionPlan.recipe.task.requirements.find(fragment => fragment.id === 'cart')!; + cart.modes = ['upgrade']; + const task = createModularRecipeTaskRequest({ release, plan: actionPlan }, { + featureIds: ['example.cart'], taskMode: 'fresh', + }).task; + assert.match(task.requirementText, /Build this product/); + assert.match(task.requirementText, /Support accounts/); + assert.match(task.requirementText, /Support a cart/); + assert.doesNotMatch(task.requirementText, /Update this product/); +}); diff --git a/tools/stack-bench/tests/module-layout.contract.ts b/tools/stack-bench/tests/module-layout.contract.ts new file mode 100644 index 00000000000..672c14d97f9 --- /dev/null +++ b/tools/stack-bench/tests/module-layout.contract.ts @@ -0,0 +1,113 @@ +import assert from 'node:assert/strict'; +import { existsSync, readdirSync, readFileSync } from 'node:fs'; +import { dirname, extname, join, relative, resolve } from 'node:path'; +import test from 'node:test'; + +import { STACK_BENCH_ROOT as ROOT } from '../src/package-root.js'; +import { DATABASE_IMAGES } from '../src/stacks/database-containers.js'; + +const SKIPPED_DIRECTORIES = new Set([ + 'archive', 'dist', 'local-notes', 'node_modules', 'results', +]); +const RELATIVE_MODULE = /(?:from\s+|import\s*\()(['"])(\.\.?\/[^'"\r\n]+?\.js)\1/g; + +function filesBelow(directory: string, output: string[] = []): string[] { + for (const entry of readdirSync(directory, { withFileTypes: true })) { + if (entry.isDirectory() && SKIPPED_DIRECTORIES.has(entry.name)) continue; + const path = join(directory, entry.name); + if (entry.isDirectory()) filesBelow(path, output); + else if (entry.isFile()) output.push(path); + } + return output; +} + +function modulesBelow(directory: string): string[] { + return filesBelow(directory) + .filter(path => !relative(ROOT, path).split(/[\\/]/).includes('tests')) + .filter(path => extname(path) === '.ts' && !path.endsWith('.d.ts')); +} + +function relativeModuleTargets(file: string): string[] { + const runtimeSource = readFileSync(file, 'utf8') + .replace(/(?:import|export)\s+type\b[\s\S]*?\bfrom\s+['"][^'"]+['"];?/g, ''); + return [...runtimeSource.matchAll(RELATIVE_MODULE)] + .map(match => resolve(dirname(file), `${match[2]?.slice(0, -3)}.ts`)); +} + +test('production libraries do not import command entrypoints', () => { + const violations: string[] = []; + for (const area of ['src', 'grader', 'dashboard']) { + const directory = join(ROOT, area); + if (!existsSync(directory)) continue; + for (const file of modulesBelow(directory)) { + const source = readFileSync(file, 'utf8'); + if (/from\s+['"][^'"]*commands\/|import\s*\(['"][^'"]*commands\//.test(source)) { + violations.push(relative(ROOT, file)); + } + } + } + assert.deepEqual(violations, []); +}); + +test('production modules have no circular imports', () => { + const files = modulesBelow(ROOT); + const known = new Set(files); + const graph = new Map(files.map(file => [file, + relativeModuleTargets(file).filter(target => known.has(target))])); + const visited = new Set(); + const active = new Set(); + const stack: string[] = []; + function visit(file: string): void { + if (active.has(file)) { + const cycle = [...stack.slice(stack.indexOf(file)), file] + .map(path => relative(ROOT, path)).join(' -> '); + assert.fail(`circular import: ${cycle}`); + } + if (visited.has(file)) return; + visited.add(file); + active.add(file); + stack.push(file); + for (const target of graph.get(file) ?? []) visit(target); + stack.pop(); + active.delete(file); + } + for (const file of files) visit(file); +}); + +test('package command entrypoints exist', () => { + const parsed: unknown = JSON.parse(readFileSync(join(ROOT, 'package.json'), 'utf8')); + assert(parsed && typeof parsed === 'object' && 'scripts' in parsed); + const scripts = parsed.scripts; + assert(scripts && typeof scripts === 'object'); + for (const [name, command] of Object.entries(scripts)) { + if (typeof command !== 'string') continue; + const match = command.match(/^node\s+([^\s]+\.js)(?:\s|$)/); + if (!match) continue; + const entrypoint = match[1]; + assert(entrypoint); + assert.equal(existsSync(join(ROOT, entrypoint)), true, + `${name} points to missing entrypoint ${entrypoint}`); + } +}); + +test('owned appliance backends use the pinned source images without shared database services', () => { + const local = readFileSync(join(ROOT, 'docker-compose.yaml'), 'utf8'); + const appliance = readFileSync(join(ROOT, 'appliance', 'docker-compose.yaml'), 'utf8'); + const image = (source: string, service: string): string | null => source.match(new RegExp( + `^ ${service}:\\r?\\n(?: .*\\r?\\n)*? image: (\\S+)$`, 'm'))?.[1] ?? null; + for (const service of ['postgres', 'mongodb'] as const) { + assert.match(image(local, service) ?? '', /@sha256:[a-f0-9]{64}$/); + assert.equal(image(local, service), DATABASE_IMAGES[service]); + assert.equal(image(appliance, service), null); + } + for (const port of ['6532:5432', '6537:27017']) { + assert.match(local, new RegExp(`127\\.0\\.0\\.1:${port}`)); + assert.doesNotMatch(appliance, new RegExp(`127\\.0\\.0\\.1:${port}`)); + } + assert.doesNotMatch(appliance, /STACK_BENCH_DASHBOARD_CONTROL_SECRET/); + for (const resource of ['postgres', 'mongodb', 'pgdata', 'mongodata']) { + assert.match(local, new RegExp(`stack-bench-dev-${resource}`)); + assert.doesNotMatch(appliance, new RegExp(`stack-bench-(?:appliance-)?${resource}\\b`)); + assert.doesNotMatch(appliance, new RegExp(`stack-bench-dev-${resource}`)); + } +}); diff --git a/tools/stack-bench/tests/mongodb-progression-reference.contract.ts b/tools/stack-bench/tests/mongodb-progression-reference.contract.ts new file mode 100644 index 00000000000..23d608a42b0 --- /dev/null +++ b/tools/stack-bench/tests/mongodb-progression-reference.contract.ts @@ -0,0 +1,97 @@ +import assert from 'node:assert/strict'; +import { readFileSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; +import test from 'node:test'; +import { compilePackDefinition } from '../src/composition/composition-compiler.js'; +import { compileProgressionDefinitionFile } from '../src/progression/progression-definition.js'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +const stackBenchRoot = STACK_BENCH_ROOT; +const trackRoot = join(stackBenchRoot, 'tracks', 'ecommerce'); +const packRoot = join(trackRoot, 'composition', 'packs'); +const appRoot = join(stackBenchRoot, 'reference-apps', 'ecommerce', 'mongodb'); +const readJson = (path: string): unknown => JSON.parse(readFileSync(path, 'utf8')); + +function record(value: unknown): value is Record<string, unknown> { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function visit(value: unknown, callback: (item: Record<string, unknown>) => void): void { + if (Array.isArray(value)) value.forEach(entry => visit(entry, callback)); + else if (record(value)) { + callback(value); + Object.values(value).forEach(entry => visit(entry, callback)); + } +} + +test('the MongoDB reference source inventory includes graph hooks and the declared account prefix', () => { + const graph = compileProgressionDefinitionFile(join(trackRoot, 'progression', 'ecommerce.json')); + const packs = new Map(readdirSync(packRoot).filter(name => name.endsWith('.json')).map(name => { + const pack = compilePackDefinition(readJson(join(packRoot, name)), { source: name }); + return [pack.id, pack]; + })); + const packRefs = new Set(graph.nodes.flatMap(node => node.featureRefs)); + const testIds = new Set<string>(); + + for (const reference of packRefs) { + const pack = packs.get(reference); + assert(pack, `missing graph pack ${reference}`); + for (const check of pack.checks ?? []) { + const scenario = readJson(join(trackRoot, check.source)); + visit(scenario, value => { + if (typeof value.testid === 'string' && !(value.do === 'click' && value.ifAvailable === true)) { + testIds.add(value.testid); + } + }); + } + } + + const clientSource = readdirSync(join(appRoot, 'client', 'src')) + .filter(name => name.endsWith('.tsx')) + .map(name => readFileSync(join(appRoot, 'client', 'src', name), 'utf8')) + .join('\n'); + const missing = [...testIds].filter(id => id.startsWith('staff-role-account-') + ? !clientSource.includes('staff-role-account-${encodeURIComponent(') + : /^subscription-(pause|resume|cancel)$/.test(id) + ? !(clientSource.includes('data-role={`subscription-${action}`}') + && clientSource.includes("['pause', 'resume', 'cancel']")) + : !clientSource.includes(id)).sort(); + assert.deepEqual(missing, []); +}); + +test('the MongoDB progression reference binds named actions and its runtime', () => { + const clientSource = readFileSync(join(appRoot, 'client', 'src', 'ProgressionPanel.tsx'), 'utf8'); + const serverSource = [ + readFileSync(join(appRoot, 'server', 'src', 'index.ts'), 'utf8'), + readFileSync(join(appRoot, 'server', 'src', 'progression.ts'), 'utf8'), + ].join('\n'); + const reference = readJson(join(appRoot, 'reference.json')); + const serverPackage = readJson(join(appRoot, 'server', 'package.json')); + const clientPackage = readJson(join(appRoot, 'client', 'package.json')); + assert(record(reference)); + assert(record(serverPackage)); + assert(record(clientPackage)); + assert(Array.isArray(reference.installDirectories)); + assert(record(reference.server)); + assert(record(reference.client)); + assert(record(serverPackage.scripts)); + assert(record(clientPackage.scripts)); + const viteConfig = readFileSync(join(appRoot, 'client', 'vite.config.ts'), 'utf8'); + + assert.match(clientSource, /data-action-input/); + assert.match(clientSource, /\/api\/support\/cases\/\$\{ticket\.id\}\/order/); + assert.match(clientSource, /\/api\/support\/cases\/\$\{ticket\.id\}\/refund/); + assert.match(clientSource, /\/api\/admin\/scheduled-restocks/); + assert.match(serverSource, /app\.use\("\/api\/support", supportRouter\)/); + assert.match(serverSource, /app\.use\("\/api\/admin", adminRouter\)/); + assert.match(serverSource, /JWT_SECRET \|\| "stack-bench-reference-only-secret-2026"/); + assert.deepEqual(reference.installDirectories, ['server', 'client']); + assert.deepEqual(reference.server, { directory: 'server' }); + assert.deepEqual(reference.client, { directory: 'client' }); + assert.equal(serverPackage.scripts.typecheck, 'tsc --noEmit'); + assert.equal(serverPackage.scripts.test, 'tsx --test src/*.test.ts'); + assert.equal(clientPackage.scripts.build, 'vite build'); + assert.match(viteConfig, /Number\(process\.env\.API_PORT\) \|\| 6401/); + assert.match(viteConfig, /Number\(process\.env\.VITE_PORT\) \|\| 6723/); + assert.match(viteConfig, /host: "0\.0\.0\.0"/); +}); diff --git a/tools/stack-bench/tests/mongodb-saved-checkout.test.ts b/tools/stack-bench/tests/mongodb-saved-checkout.test.ts new file mode 100644 index 00000000000..32e6a139f38 --- /dev/null +++ b/tools/stack-bench/tests/mongodb-saved-checkout.test.ts @@ -0,0 +1,87 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { hashAppSource } from '../src/runtime/source-snapshot.js'; +import type { TextCommandExecutor } from '../src/runtime/command-executor.js'; +import { getSavedMongoDbCheckoutState } from '../src/stacks/backends/mongodb-saved-checkout.js'; +import { attemptDatabaseIdentity } from '../src/stacks/hosted-database-identity.js'; +import type { BackendLeaseNetwork } from '../src/runtime/backend-lease.js'; + +function fixture() { + const root = mkdtempSync(join(tmpdir(), 'saved-mongodb-reader-')), app = join(root, 'app'); + mkdirSync(app); writeFileSync(join(app, 'server.js'), 'source'); + const sourceSha256 = hashAppSource(app).sha256; + const mapping = JSON.stringify({ sourceSha256, script: "return {};" }); + const reader = { path: join(root, 'reader.json'), sha256: createHash('sha256').update(mapping).digest('hex') }; + writeFileSync(reader.path, mapping); + const state = { accountId: 'buyer', itemId: 'keyboard', priceMinor: 8900, cart: [], + stock: [{ warehouseId: 'east', quantity: 10 }], reservations: [], payments: [], orphanOrderLines: 0, + orphanAllocations: 0, orders: [{ id: 'old', accountId: 'buyer', totalMinor: 8900, + refundedMinor: 0, status: 'pending', lines: [{ itemId: 'keyboard', quantity: 1, priceMinor: 8900, + allocations: [{ warehouseId: 'east', quantity: 1 }] }] }] }; + return { root, sourceSha256, state, args: { app, reader, account: "buyer'\\name", item: 'Keyboard', + lease: { resources: { container: { id: 'owned-id', name: 'owned-name' }, database: 'owned-db' } } } }; +} + +test('saved reader uses the owned Mongo container and aborts its consistent snapshot', t => { + const f = fixture(); t.after(() => rmSync(f.root, { recursive: true, force: true })); + let queries = 0; + const exec: TextCommandExecutor = (file, args, options) => { + assert.equal(file, 'docker'); + if (args[0] === 'inspect') { assert.equal(args.at(-1), 'owned-name'); return 'owned-id'; } + queries++; + assert.deepEqual(args.slice(0, 4), ['exec', 'owned-id', 'mongosh', 'owned-db']); + const script = String(args.at(-1)); + assert(script.includes(`account=${JSON.stringify(f.args.account)}`)); + assert.match(script, /startTransaction\(\{readConcern:\{level:'snapshot'\}\}\)/); + assert.match(script, /session.getDatabase\(db.getName\(\)\)/); + assert.match(script, /finally \{ try \{ session.abortTransaction\(\); \} finally \{ session.endSession\(\); \} \}/); + assert(!script.includes('commitTransaction')); + assert.equal(options.timeout, 60_000); + return JSON.stringify({ accountMatches: 1, itemMatches: 1, state: f.state }); + }; + const result = getSavedMongoDbCheckoutState({ ...f.args, exec }); + assert.equal(queries, 1); assert.deepEqual(result.state, f.state); + assert.deepEqual(result.schemaSha256, { source: f.sourceSha256, reader: f.args.reader.sha256 }); + assert.equal(result.scope, 'orders'); +}); + +test('isolated Mongo reads authenticate with private lease authority', t => { + const f = fixture(); t.after(() => rmSync(f.root, { recursive: true, force: true })); + const lease = { ...f.args.lease, ownershipToken: 'private-authority', + resources: { ...f.args.lease.resources, network: {} as BackendLeaseNetwork } }; + const exec: TextCommandExecutor = (_file, args) => { + if (args[0] === 'inspect') return 'owned-id'; + const identity = attemptDatabaseIdentity(lease.ownershipToken); + assert(args.includes(identity.user)); assert(args.includes(identity.password)); + assert.equal(args[args.indexOf('--authenticationDatabase') + 1], 'owned-db'); + return JSON.stringify({ accountMatches: 1, itemMatches: 1, state: f.state }); + }; + getSavedMongoDbCheckoutState({ ...f.args, lease, exec }); + assert.throws(() => getSavedMongoDbCheckoutState({ ...f.args, + lease: { ...lease, ownershipToken: '' }, exec }), /private lease authority/); +}); + +test('database errors retain stderr without exposing credentials or the evaluated program', t => { + const f = fixture(); t.after(() => rmSync(f.root, { recursive: true, force: true })); + const lease = { ...f.args.lease, ownershipToken: 'private-authority', + resources: { ...f.args.lease.resources, network: {} as BackendLeaseNetwork } }; + const password = attemptDatabaseIdentity(lease.ownershipToken).password; + const exec: TextCommandExecutor = (_file, args) => { + if (args[0] === 'inspect') return 'owned-id'; + throw Object.assign(new Error(`command --password ${password} --eval ${args.at(-1)}`), { + stderr: `MongoServerError: snapshot unavailable ${password}\n${args.at(-1)}`, + }); + }; + assert.throws(() => getSavedMongoDbCheckoutState({ ...f.args, lease, exec }), error => { + assert(error instanceof Error); + assert.match(error.message, /snapshot unavailable/); + assert(!error.message.includes(password)); + assert(!error.message.includes('startTransaction')); + assert(!error.message.includes('--eval')); + return true; + }); +}); diff --git a/tools/stack-bench/tests/mutation-analysis.contract.ts b/tools/stack-bench/tests/mutation-analysis.contract.ts new file mode 100644 index 00000000000..b45a7fa61a3 --- /dev/null +++ b/tools/stack-bench/tests/mutation-analysis.contract.ts @@ -0,0 +1,336 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { classifyMutationResult, groupMutationsByScenario, mutationScenario, + indexMutationReport, mutationFileEdits, validateMutationBaseline, isRetryableMutationBaseline, + isRetryableMutationResult, readMutationManifest, + releaseScenarioCheckKeys, resolveMutationFile, resolveMutationScenarioPath, + reusableMutationBaseline, + validateMutationDefinitions } from '../src/evidence/mutation-analysis.js'; +import { createCheckEvidence } from '../src/evidence/check-evidence.js'; +import { buildRecipeRelease, requireRecipeRelease as resolveRecipeRelease } + from '../src/composition/recipe-release.js'; +import { loadTrack } from '../src/composition/tracks.js'; + +type CriterionValue = boolean | 'inconclusive'; + +// A failed action record, as the grader stores the step a criterion stopped at. +const failedStep = (id: string) => { + const { actor: _actor, actions: _actions, ...base } = createCheckEvidence({ status: 'failed', + code: 'test_result', phase: 'assertion', startedAtMs: 1, completedAtMs: 2 }); + return { actor: null, evidence: { ...base, action: { id, version: '1' }, type: 'browser-observation-evidence', + phase: 'execute', timing: { startedAtMs: 1, completedAtMs: 2, durationMs: 1, deadlineMs: 100 } } }; +}; + +const report = (criteria: Record<string, CriterionValue>, setupError: string | null = null, + failedAction = 'expect') => ({ + total: Object.values(criteria).filter(value => value === true).length, + max: Object.keys(criteria).length, + features: [{ + id: 7, + criteria: Object.entries(criteria).map(([id, value]) => ({ id, stableKey: `check.${id}`, + evidence: createCheckEvidence({ + status: value === true ? 'passed' : value === 'inconclusive' ? 'inconclusive' : 'failed', + code: value === true ? 'completed' : 'test_result', + phase: setupError ? 'setup' : 'assertion', summary: setupError, + startedAtMs: 1, completedAtMs: 2, + actions: value === false ? [failedStep(failedAction)] : [], + }) })), + }], +}); +const mutation = { id: 'break-b', targets: ['check.b'] }; + +test('a clean scenario report is reused only for the exact run identity and check set', () => { + const scenario = { total: 2, max: 2, selection: { checks: [ + { stableKey: 'check.a' }, { stableKey: 'check.b' }, + ] }, features: [] }; + const bundle = { + backend: 'postgres', track: 'ecommerce', level: 3, + source: { sha256: 'fixture' }, + recipeRelease: { id: 'recipe', contentSha256: 'recipe-sha' }, + artifactEnvelope: { identities: { + engine: { id: 'stack-bench', version: null, sha256: 'engine' }, + calibration: { id: 'calibration', sha256: 'calibration-sha' }, + stackAdapter: { id: 'postgres', version: null, sha256: null }, + } }, + suites: { scenario, another: { selection: { checks: [{ stableKey: 'check.c' }] } } }, + }; + const expected = { backend: 'postgres', track: 'ecommerce', level: 3, + fixtureSha256: 'fixture', recipe: { id: 'recipe', sha256: 'recipe-sha' }, + identities: structuredClone(bundle.artifactEnvelope.identities), + selectedCheckKeys: ['check.b', 'check.a'] }; + + assert.deepEqual(reusableMutationBaseline(bundle, expected), { ok: true, report: scenario }); + const fixtureMismatch = reusableMutationBaseline(bundle, { ...expected, fixtureSha256: 'other' }); + assert.equal(fixtureMismatch.ok, false); + if (fixtureMismatch.ok) assert.fail('fixture mismatch was accepted'); + assert.match(fixtureMismatch.reason, /fixture/); + const selectionMismatch = reusableMutationBaseline(bundle, { ...expected, + selectedCheckKeys: ['check.a'] }); + assert.equal(selectionMismatch.ok, false); + if (selectionMismatch.ok) assert.fail('selection mismatch was accepted'); + assert.match(selectionMismatch.reason, /0 exact scenario matches/); + const engineMismatch = reusableMutationBaseline(bundle, { ...expected, identities: { + ...expected.identities, + engine: { ...expected.identities.engine, sha256: 'new-engine' }, + } }); + assert.equal(engineMismatch.ok, false); + if (engineMismatch.ok) assert.fail('engine mismatch was accepted'); + assert.match(engineMismatch.reason, /engine/); + const calibrationMismatch = reusableMutationBaseline(bundle, { ...expected, identities: { + ...expected.identities, + calibration: { ...expected.identities.calibration, sha256: 'new-calibration' }, + } }); + assert.equal(calibrationMismatch.ok, false); + if (calibrationMismatch.ok) assert.fail('calibration mismatch was accepted'); + assert.match(calibrationMismatch.reason, /calibration/); +}); + +test('mutation definitions name exact criteria and contain real edits', () => { + const valid = { ...mutation, file: 'src/app.ts', find: 'correct', replace: 'broken' }; + assert.equal(validateMutationDefinitions([valid]).ok, true); + const invalid = validateMutationDefinitions([ + { id: 'duplicate', file: '', targets: [], find: 'same', replace: 'same' }, + { id: 'duplicate', file: 'src/app.ts', targets: ['check.b', 'check.b'], edits: [] }, + ]); + assert.equal(invalid.ok, false); + assert.deepEqual([...new Set(invalid.issues.map(issue => issue.kind))], + ['bad_file', 'bad_targets', 'bad_edit', 'duplicate_id', 'missing_edits']); +}); + +test('one mutation may atomically edit multiple application files', () => { + const multiFile = { + ...mutation, + edits: [ + { file: 'server/schema.sql', find: 'UNIQUE (item_id, account_id)', replace: '-- removed' }, + { file: 'server/src/index.ts', find: 'ON CONFLICT', replace: '/* no conflict guard */' }, + ], + }; + assert.equal(validateMutationDefinitions([multiFile]).ok, true); + assert.deepEqual(mutationFileEdits(multiFile).map(edit => edit.file), + ['server/schema.sql', 'server/src/index.ts']); + + const inheritedFile = { + ...mutation, + file: 'client/src/App.tsx', + edits: [{ find: 'first', replace: 'broken first' }, + { file: 'server/src/index.ts', find: 'second', replace: 'broken second' }], + }; + assert.deepEqual(mutationFileEdits(inheritedFile).map(edit => edit.file), + ['client/src/App.tsx', 'server/src/index.ts']); + assert.equal(validateMutationDefinitions([{ ...multiFile, + edits: [{ file: '', find: 'x', replace: 'y' }] }]).ok, false); +}); + +test('mutation scenarios may override a manifest default and are required for execution', () => { + const valid = { ...mutation, file: 'src/app.ts', find: 'correct', replace: 'broken' }; + assert.equal(mutationScenario({ scenario: 'scenarios/base.json' }, valid), 'scenarios/base.json'); + assert.equal(mutationScenario({ scenario: 'scenarios/base.json' }, + { ...valid, scenario: 'scenarios/upgrade.json' }), 'scenarios/upgrade.json'); + assert.equal(validateMutationDefinitions([valid], { requireScenario: true }).ok, false); + assert.equal(validateMutationDefinitions([{ ...valid, scenario: 'scenarios/upgrade.json' }], + { requireScenario: true }).ok, true); + const groups = groupMutationsByScenario({ scenario: 'scenarios/base.json', mutations: [ + valid, { ...valid, id: 'upgrade', scenario: 'scenarios/upgrade.json' }, + ] }); + assert.deepEqual([...groups.keys()], ['scenarios/base.json', 'scenarios/upgrade.json']); + assert.deepEqual([...groups.values()].map(entries => entries.map(entry => entry.id)), + [['break-b'], ['upgrade']]); +}); + +test('mutation scenarios resolve from the package assets after TypeScript compilation', () => { + assert.equal(resolveMutationScenarioPath( + 'tracks/ecommerce/scenarios/01-account-create-2.4.0.json'), + join(loadTrack('ecommerce').dir, 'scenarios', '01-account-create-2.4.0.json')); + assert.throws(() => resolveMutationScenarioPath('dist/tracks/ecommerce/scenarios/example.json'), + /escapes the tracks directory/); + assert.throws(() => resolveMutationScenarioPath('../outside.json'), + /escapes the tracks directory/); +}); + +test('recipe-bound mutation grading selects only checks owned by the scenario', () => { + const track = loadTrack('ecommerce'); + const release = buildRecipeRelease(join(track.dir, 'composition', 'recipes', + 'sequential-l2.json'), { trackRoot: track.dir }); + const keys = releaseScenarioCheckKeys(release, track.dir, + join(track.dir, 'scenarios', '02-features.json')); + assert(keys.includes('ecommerce.operations-access.fulfilment-queue.1a')); + assert(!keys.includes('ecommerce.operations-access.fulfilment-queue.1b'), + '1b moved to a self-contained scenario and must not be graded from the legacy source'); + assert(!keys.includes('ecommerce.inventory-operations.operational-views.5a'), + '5a moved to an isolated scenario and must not share state with the remaining views'); + assert.deepEqual(releaseScenarioCheckKeys(release, track.dir, + join(track.dir, 'scenarios', '02-low-stock.json')), [ + 'ecommerce.inventory-operations.operational-views.5a', + ]); + assert.throws(() => releaseScenarioCheckKeys(release, track.dir, + join(track.dir, 'scenarios', '03-features.json')), + /has no checks/); +}); + +test('recipe-bound mutation grading excludes zero-point controls', () => { + const track = loadTrack('ecommerce'); + const binding = resolveRecipeRelease(track, 3, 'ecommerce.progression-catalog'); + assert.deepEqual(releaseScenarioCheckKeys(binding.release, track.dir, + join(track.dir, 'scenarios', '01-restock-race.json')), [ + 'ecommerce.spec.concurrency-safety.restock-race.202a', + ]); +}); + +test('recipe-bound mutation grading keeps only checks selected for the run', () => { + const track = loadTrack('ecommerce'); + const binding = resolveRecipeRelease(track, 3, 'ecommerce.progression-catalog'); + assert.deepEqual(releaseScenarioCheckKeys(binding.release, track.dir, + join(track.dir, 'scenarios', '02-strengthened.json'), [ + 'ecommerce.inventory-operations.warehouse-transfer.2a', + ]), [ + 'ecommerce.inventory-operations.warehouse-transfer.2a', + ]); +}); + +test('a mutation can declare exact targets across multiple features', () => { + const crossFeature = { + id: 'cross-feature', file: 'src/app.ts', find: 'correct', replace: 'broken', + targets: ['check.b', 'check.c'], + }; + assert.equal(validateMutationDefinitions([crossFeature]).ok, true); + const directory = mkdtempSync(join(tmpdir(), 'stack-bench-mutation-manifest-')); + try { + const manifest = join(directory, 'mutations.json'); + writeFileSync(manifest, JSON.stringify({ schemaVersion: 3, fixtureSha256: 'a'.repeat(64), + backend: 'postgres', track: 'ecommerce', mutations: [{ id: 'cross-feature', + edits: [{ file: 'src/app.ts', find: 'correct', replace: 'broken' }], + targets: ['check.b', 'check.c'], breaks: 7, kills: ['b'] }] })); + assert.throws(() => readMutationManifest(manifest), /unknown fields: breaks, kills/); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + const baseline = { total: 2, max: 2, features: [ + { id: 7, criteria: [report({ b: true }).features[0]!.criteria[0]!] }, + { id: 8, criteria: [report({ c: true }).features[0]!.criteria[0]!] }, + ] }; + const mutant = { total: 0, max: 2, features: [ + { id: 7, criteria: [report({ b: false }).features[0]!.criteria[0]!] }, + { id: 8, criteria: [report({ c: false }).features[0]!.criteria[0]!] }, + ] }; + assert.equal(validateMutationBaseline(baseline, [crossFeature]).ok, true); + assert.equal(classifyMutationResult(baseline, mutant, crossFeature).status, 'CAUGHT'); +}); + +test('mutation files cannot escape the application directory', () => { + assert.match(resolveMutationFile('/fixture/app', 'server/src/index.ts'), /server[\\/]src[\\/]index\.ts$/); + assert.throws(() => resolveMutationFile('/fixture/app', '../outside.ts'), /escapes the app directory/); + assert.throws(() => resolveMutationFile('/fixture/app', '/outside.ts'), /escapes the app directory/); +}); + +test('a mutation baseline must pass every criterion and contain every declared target', () => { + assert.equal(validateMutationBaseline(report({ a: true, b: true }), [mutation]).ok, true); + const invalid = validateMutationBaseline(report({ a: true, b: 'inconclusive' }), [ + mutation, { id: 'missing', targets: ['check.c'] }, + ]); + assert.equal(invalid.ok, false); + assert.deepEqual(invalid.issues.map(issue => issue.kind), + ['score_not_full', 'inconclusive', 'missing_target']); +}); + +test('mutation reports reject duplicate criterion identities', () => { + const duplicate = report({ a: true }); + duplicate.features[0]!.criteria.push({ ...duplicate.features[0]!.criteria[0]! }); + assert.throws(() => indexMutationReport(duplicate), /duplicate mutation criterion identity/); +}); + +test('only transient baseline failures are retried', () => { + assert.equal(isRetryableMutationBaseline([ + { kind: 'score_not_full' }, { kind: 'setup_failure' }, { kind: 'inconclusive' }, + ]), true); + assert.equal(isRetryableMutationBaseline([ + { kind: 'score_not_full' }, { kind: 'criterion_failure' }, + ]), false); + assert.equal(isRetryableMutationBaseline([{ kind: 'missing_target' }]), false); +}); + +test('only unusable mutation results are retried', () => { + for (const status of ['INVALID_SETUP', 'INVALID_HARNESS_FAILURE', + 'INVALID_INCONCLUSIVE'] as const) { + assert.equal(isRetryableMutationResult(status), true); + } + for (const status of ['CAUGHT', 'INVALID_REPORT', 'WRONG_CRITERION', 'SURVIVED', + 'CAUGHT_COLLATERAL', 'CAUGHT_OFF_ASSERTION'] as const) { + assert.equal(isRetryableMutationResult(status), false); + } +}); + +test('nested failure attribution requires the exact recorded child within its wrapper', () => { + for (const mode of ['verified', 'unrelated', 'changed-copy', 'different-status', 'different-finding', + 'outside-wrapper', 'unrecorded']) { + const child = failedStep('expectNumber'), wrapper = failedStep('probeSignupClaims'); + child.evidence.timing = { startedAtMs: 2, completedAtMs: 3, durationMs: 1, deadlineMs: 100 }; + wrapper.evidence.timing = { startedAtMs: 1, completedAtMs: 4, durationMs: 3, deadlineMs: 100 }; + if (mode === 'outside-wrapper') wrapper.evidence.timing = { + startedAtMs: 3, completedAtMs: 4, durationMs: 1, deadlineMs: 100, + }; + if (mode === 'different-status') child.evidence.status = 'passed'; + if (mode === 'different-finding') wrapper.evidence.finding = { + kind: 'action-failed', fields: { action: 'probeSignupClaims' }, + }; + const nestedAction = structuredClone(child.evidence); + if (mode === 'changed-copy') nestedAction.code = 'other_failure'; + wrapper.evidence.observation = mode === 'unrelated' ? null : { nestedAction }; + const cleanup = failedStep('closeClient'); cleanup.evidence.status = 'passed'; + const mutant = report({ a: true, b: false }); + mutant.features[0]!.criteria[1]!.evidence.actions = [ + ...(mode === 'unrecorded' ? [] : [child]), cleanup, wrapper, + ]; + const result = classifyMutationResult(report({ a: true, b: true }), mutant, mutation); + assert.equal(result.status, mode === 'verified' ? 'CAUGHT' : 'CAUGHT_OFF_ASSERTION', mode); + assert.equal(indexMutationReport(mutant).criteria.get('check.b')!.failedAction, + mode === 'verified' ? 'expectNumber' : 'probeSignupClaims', mode); + } +}); + +test('only a conclusive failure of the declared criterion is a clean kill', () => { + const harnessFailure = (summary: string) => createCheckEvidence({ status: 'harness_failure', + code: 'browser_failure', phase: 'assertion', summary, startedAtMs: 1, completedAtMs: 2 }); + const targetHarness = report({ a: true, b: true }); + targetHarness.total = 1; + targetHarness.features[0]!.criteria[1] = { id: 'b', stableKey: 'check.b', + evidence: harnessFailure('not an application observation') }; + const collateralHarness = report({ a: true, b: false }); + collateralHarness.features[0]!.criteria[0] = { id: 'a', stableKey: 'check.a', + evidence: harnessFailure('browser disappeared') }; + const cases: Array<{ name: string; mutant: ReturnType<typeof report>; status: string; + fields?: Record<string, unknown>; regressions?: string[]; collateral?: string[] }> = [ + { name: 'the declared criterion fails conclusively', mutant: report({ a: true, b: false }), + status: 'CAUGHT', regressions: ['check.b'] }, + { name: 'the target fails at a database observation', + mutant: report({ a: true, b: false }, null, 'dbExpectStock'), status: 'CAUGHT' }, + { name: 'the target fails before any observation', + mutant: report({ a: true, b: false }, null, 'click'), status: 'CAUGHT_OFF_ASSERTION', + fields: { targetOffAssertion: [{ key: 'check.b', action: 'click' }] } }, + { name: 'a score drop caused by setup failure', + mutant: report({ a: false, b: false }, 'sign in failed'), status: 'INVALID_SETUP' }, + { name: 'an inconclusive target', mutant: report({ a: true, b: 'inconclusive' }), + status: 'INVALID_INCONCLUSIVE' }, + { name: 'a typed harness failure on the target', mutant: targetHarness, + status: 'INVALID_HARNESS_FAILURE', fields: { targetHarnessFailures: ['check.b'] } }, + { name: 'inconclusive evidence outside the target', mutant: report({ a: 'inconclusive', b: false }), + status: 'INVALID_INCONCLUSIVE', fields: { collateralInconclusive: ['check.a'] } }, + { name: 'a harness failure outside the target', mutant: collateralHarness, + status: 'INVALID_HARNESS_FAILURE', fields: { collateralHarnessFailures: ['check.a'] } }, + { name: 'failure in the wrong criterion', mutant: report({ a: false, b: true }), + status: 'WRONG_CRITERION', collateral: ['check.a'] }, + { name: 'collateral damage beside a caught target', mutant: report({ a: false, b: false }), + status: 'CAUGHT_COLLATERAL' }, + ]; + for (const { name, mutant, status, fields = {}, regressions, collateral } of cases) { + const result = classifyMutationResult(report({ a: true, b: true }), mutant, mutation); + assert.equal(result.status, status, name); + for (const [field, value] of Object.entries(fields)) { + assert.deepEqual((result as unknown as Record<string, unknown>)[field], value, name); + } + if (regressions) assert.deepEqual(result.regressions.map(item => item.key), regressions, name); + if (collateral) assert.deepEqual(result.collateral.map(item => item.key), collateral, name); + } +}); diff --git a/tools/stack-bench/tests/mutation-checkpoint.test.ts b/tools/stack-bench/tests/mutation-checkpoint.test.ts new file mode 100644 index 00000000000..275cc50420c --- /dev/null +++ b/tools/stack-bench/tests/mutation-checkpoint.test.ts @@ -0,0 +1,89 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { mutationHarnessFailureArtifact } from '../grader/mutation-test.js'; + +import { reusableMutationEvidence } from '../src/evidence/mutation-checkpoint.js'; +import type { MutationCheckpointEvidence, MutationCheckpointIdentity } + from '../src/evidence/mutation-checkpoint.js'; + +function identity(): MutationCheckpointIdentity { + return { + schemaVersion: 1, + engineSha256: 'a'.repeat(64), + recipeSha256: 'b'.repeat(64), + fixtureSha256: 'c'.repeat(64), + calibrationSha256: '8'.repeat(64), + imageId: 'sha256:image', + backend: 'mongodb', + track: 'ecommerce', + level: 3, + trackSha256: '9'.repeat(64), + shard: { index: 0, count: 1, mutationIds: ['m1', 'm2'] }, + groups: [ + { scenario: 'first.json', identitySha256: 'd'.repeat(64), mutationIds: ['m1'] }, + { scenario: 'second.json', identitySha256: 'e'.repeat(64), mutationIds: ['m2'] }, + ], + }; +} + +function evidence(checkpoint: MutationCheckpointIdentity = identity()): MutationCheckpointEvidence { + return { checkpoint, results: [ + { id: 'm1', scenario: 'first.json', status: 'CAUGHT' }, + { id: 'm2', scenario: 'second.json', status: 'CAUGHT' }, + ], baseline: { scenarios: [ + { scenario: 'first.json', identitySha256: 'd'.repeat(64), total: 1, max: 1 }, + { scenario: 'second.json', identitySha256: 'e'.repeat(64), total: 1, max: 1 }, + ] } }; +} + +test('mutation checkpoints reuse only unchanged scenario groups', () => { + const current = identity(); + current.groups[1]!.identitySha256 = 'f'.repeat(64); + const reusable = reusableMutationEvidence(evidence(), current); + assert.deepEqual(reusable.results.map(result => result.id), ['m1']); + assert.deepEqual(reusable.baselines.map(result => result.scenario), ['first.json']); +}); + +test('mutation checkpoints fail closed when a global identity changes', () => { + const fields = ['engineSha256', 'recipeSha256', 'fixtureSha256', + 'calibrationSha256', 'imageId', 'backend', 'track', 'level', 'trackSha256'] as const; + for (const field of fields) { + const current = identity(); + current[field] = `changed-${field}`; + assert.throws(() => reusableMutationEvidence(evidence(), current), + new RegExp(`checkpoint ${field} does not match`)); + } + const current = identity(); + current.shard = { index: 0, count: 2, mutationIds: ['m1'] }; + assert.throws(() => reusableMutationEvidence(evidence(), current), + /checkpoint shard does not match/); +}); + +test('checkpoint identity comparison ignores object property order', () => { + const current = identity(); + current.shard = { mutationIds: ['m1', 'm2'], count: 1, index: 0 }; + assert.deepEqual(reusableMutationEvidence(evidence(), current).results.map(result => result.id), + ['m1', 'm2']); +}); + +test('mutation checkpoints reject duplicate scenario groups', () => { + const current = identity(); + current.groups.push({ ...current.groups[0]! }); + assert.throws(() => reusableMutationEvidence(evidence(), current), /duplicates scenario first\.json/); +}); + + +test('a later grader failure preserves completed mutation evidence without claiming success', () => { + const current = { ...evidence(), ok: true, + summary: { caught: 2, completed: 2, total: 3, remaining: 1 }, + gradeReports: [{ mutationId: 'm3', status: 'threw' }] }; + const failed = mutationHarnessFailureArtifact(current, 'grader timed out', '2026-09-10T14:00:00Z'); + assert.deepEqual(failed.results, current.results); + assert.deepEqual(failed.baseline, current.baseline); + assert.deepEqual(failed.checkpoint, current.checkpoint); + assert.deepEqual(failed.summary, current.summary); + assert.deepEqual(failed.gradeReports, current.gradeReports); + assert.equal(failed.ok, false); + assert.deepEqual(failed.outcome, { kind: 'harness_failure', phase: 'mutation-control', reason: 'grader timed out' }); + assert.equal(current.ok, true); +}); diff --git a/tools/stack-bench/tests/mutation-manifests.test.ts b/tools/stack-bench/tests/mutation-manifests.test.ts new file mode 100644 index 00000000000..4e20a29b021 --- /dev/null +++ b/tools/stack-bench/tests/mutation-manifests.test.ts @@ -0,0 +1,122 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync, readdirSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { extname, join } from 'node:path'; +import test from 'node:test'; +import ts from 'typescript'; +import { mutationFileEdits, mutationScenario, mutationTargetKeys, + validateMutationDefinitions, type MutationManifest } + from '../src/evidence/mutation-analysis.js'; +import { loadReferenceRegistry, prepareReferenceFixtureSource } from '../src/references/reference-fixtures.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { STACK_IDS } from '../src/stacks/stack-identities.js'; + +const MUTATIONS = join(STACK_BENCH_ROOT, 'grader', 'mutations'); + +function readJson(path: string): unknown { + return JSON.parse(readFileSync(path, 'utf8')); +} + +function record(value: unknown): value is Record<string, unknown> { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function isMutationManifest(value: unknown): value is MutationManifest & Record<string, unknown> { + return record(value) && (value.mutations === undefined + || Array.isArray(value.mutations) && value.mutations.every(record)); +} + +function manifestAt(path: string): MutationManifest & Record<string, unknown> { + const value = readJson(path); + if (!isMutationManifest(value)) throw new Error(`${path} mutations must be an array`); + return value; +} + +function syntaxErrors(source: string, file: string): string[] { + if (!['.ts', '.tsx'].includes(extname(file))) return []; + return (ts.transpileModule(source, { + compilerOptions: { jsx: ts.JsxEmit.ReactJSX, module: ts.ModuleKind.ESNext, + target: ts.ScriptTarget.ES2022 }, + fileName: file, + reportDiagnostics: true, + }).diagnostics ?? []).filter(diagnostic => diagnostic.category === ts.DiagnosticCategory.Error) + .map(diagnostic => ts.flattenDiagnosticMessageText(diagnostic.messageText, '\n')); +} + +test('every mutation manifest binds valid edits to exact scenario criteria', () => { + const files = readdirSync(MUTATIONS).filter(file => file.endsWith('.json')).sort(); + assert.ok(files.length > 0, 'expected mutation manifests'); + for (const file of files) { + const manifest = manifestAt(join(MUTATIONS, file)); + const definitions = validateMutationDefinitions(manifest.mutations, + { defaultScenario: typeof manifest.scenario === 'string' ? manifest.scenario : null, requireScenario: true }); + assert.deepEqual(definitions.issues, [], `${file} has an invalid mutation definition`); + assert.equal(manifest.schemaVersion, 3); + assert.deepEqual(Object.keys(manifest).filter(field => !new Set([ + 'schemaVersion', 'fixtureSha256', 'backend', 'track', 'scenario', 'note', 'mutations', + ]).has(field)), [], `${file} has unknown root fields`); + assert(typeof manifest.backend === 'string'); + assert.ok(STACK_IDS.includes(manifest.backend), `${file} names an unregistered stack`); + assert.equal(typeof manifest.track, 'string'); + assert.equal(manifest.level, undefined); + assert(typeof manifest.fixtureSha256 === 'string'); + assert.match(manifest.fixtureSha256, /^[a-f0-9]{64}$/); + + for (const mutation of manifest.mutations ?? []) { + const scenarioPath = mutationScenario(manifest, mutation); + assert.equal(typeof scenarioPath, 'string'); + for (const target of mutationTargetKeys(mutation)) { + assert.match(target, /^[a-z0-9][a-z0-9.-]+$/, + `${file}/${mutation.id} has an invalid stable check ID`); + } + } + } +}); + +test('current mutation anchors match their hash-bound canonical fixture exactly once and keep it syntactically valid', () => { + const registry = loadReferenceRegistry(); + for (const fixture of registry.fixtures) { + const manifests = fixture.mutationManifests ?? []; + if (!manifests.length) continue; + if ((!fixture.source && !fixture.targetPath) || !fixture.imported?.sourceSha256) { + throw new Error(`${fixture.id} mutation fixture has incomplete source identity`); + } + const temporary = fixture.source + ? mkdtempSync(join(tmpdir(), `stack-bench-mutation-fixture-${fixture.backend}-`)) : null; + try { + let sourceRoot: string; + if (temporary) { + prepareReferenceFixtureSource(fixture, temporary); + sourceRoot = temporary; + } else { + if (!fixture.targetPath) throw new Error(`${fixture.id} has no target path`); + sourceRoot = join(STACK_BENCH_ROOT, fixture.targetPath); + } + for (const relativeManifest of manifests) { + const manifest = manifestAt(join(STACK_BENCH_ROOT, relativeManifest)); + assert(typeof manifest.fixtureSha256 === 'string'); + assert.equal(manifest.fixtureSha256, fixture.imported.sourceSha256, + `${fixture.id} manifest is bound to different source bytes`); + for (const mutation of manifest.mutations ?? []) { + const editsByFile = new Map<string, ReturnType<typeof mutationFileEdits>>(); + for (const edit of mutationFileEdits(mutation)) { + editsByFile.set(edit.file, [...editsByFile.get(edit.file) ?? [], edit]); + } + for (const [file, edits] of editsByFile) { + let source = readFileSync(join(sourceRoot, file), 'utf8'); + for (const edit of edits) { + const matches = source.split(edit.find).length - 1; + assert.equal(matches, 1, + `${fixture.id}/${mutation.id} anchor matched ${matches} times in ${file}`); + source = source.replace(edit.find, edit.replace); + } + assert.deepEqual(syntaxErrors(source, file), [], + `${fixture.id}/${mutation.id} must remain syntactically valid in ${file}`); + } + } + } + } finally { + if (temporary) rmSync(temporary, { recursive: true, force: true }); + } + } +}); diff --git a/tools/stack-bench/tests/mutation-preflight.test.ts b/tools/stack-bench/tests/mutation-preflight.test.ts new file mode 100644 index 00000000000..bc9947a8078 --- /dev/null +++ b/tools/stack-bench/tests/mutation-preflight.test.ts @@ -0,0 +1,320 @@ +import { GRADER_SOURCE_TIMEOUT_MS } from '../src/runtime/grading-timeout.js'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { mutationControlArgv, mutationControlTimeoutMs, + mutationGradeTimeoutMs } from '../src/evidence/mutation-control.js'; +import { loadTrack } from '../src/composition/tracks.js'; +import { STACK_BENCH_ROOT, compiledEntrypoint } from '../src/package-root.js'; +import { mutationClientCommand, mutationFailureMessage, mutationGradeArguments, restoreMutationSource, + retainMutationGrade, resetMutationDatabase } from '../grader/mutation-test.js'; +import { STACK_ADAPTER_REGISTRY } from '../src/stacks/stack-adapters.js'; +import { createBackendLease, writeBackendLease } from '../src/runtime/backend-lease.js'; +import type { TextCommandExecutor } from '../src/runtime/command-executor.js'; +import { parseGradeArgs } from '../grader/grade.js'; +import { createArtifact } from '../src/evidence/artifacts.js'; + +test('mutation reset stops hosted apps before clearing data and fails closed', async t => { + const root = mkdtempSync(join(tmpdir(), 'mutation-reset-order-')); + const path = join(root, 'lease.json'); + const priorPath = process.env.STACK_BENCH_LEASE; + const priorToken = process.env.STACK_BENCH_LEASE_TOKEN; + t.after(() => { + if (priorPath === undefined) delete process.env.STACK_BENCH_LEASE; + else process.env.STACK_BENCH_LEASE = priorPath; + if (priorToken === undefined) delete process.env.STACK_BENCH_LEASE_TOKEN; + else process.env.STACK_BENCH_LEASE_TOKEN = priorToken; + rmSync(root, { recursive: true, force: true }); + }); + for (const backend of ['postgres', 'mongodb', 'spacetime'] as const) { + const lease = createBackendLease({ runId: 'mutation-reset', backend, + track: 'ecommerce', runIndex: 0, database: 'mutation_reset', + container: { name: 'database', id: 'a'.repeat(64) }, + serverUri: 'http://127.0.0.1:3211', module: 'mutation_reset', dataDir: root }); + lease.state = 'active'; + writeBackendLease(path, lease); + process.env.STACK_BENCH_LEASE = path; + process.env.STACK_BENCH_LEASE_TOKEN = lease.ownershipToken; + const calls: string[] = []; + let failure: string | null = null; + const adapter = STACK_ADAPTER_REGISTRY.get(backend); + const control: NonNullable<Parameters<typeof resetMutationDatabase>[2]> = async (_spec, mode) => { + calls.push(mode!); + if (failure === mode) throw new Error(`${mode} failed`); + }; + t.mock.method(adapter.reset, 'run', () => { + calls.push('reset'); + if (failure === 'reset') throw new Error('reset failed'); + }); + const args = { backend, app: root, track: 'ecommerce', reseedOnReset: true, + restartSpec: { backend, app: root, port: 3000, probe: '/' } }; + await resetMutationDatabase(args, null, control); + assert.deepEqual(calls.splice(0), ['stop', 'reset', 'start']); + for (const phase of ['stop', 'reset']) { + failure = phase; + await assert.rejects(resetMutationDatabase(args, null, control), new RegExp(`${phase} failed`)); + assert.deepEqual(calls.splice(0), phase === 'stop' ? ['stop'] : ['stop', 'reset']); + } + await assert.rejects(resetMutationDatabase({ ...args, restartSpec: undefined }, null, control), + /requires a lease-authenticated --restart-spec/); + assert.deepEqual(calls, [], 'missing control must not clear the database'); + } +}); + +test('mutation grading retains each report, including errors, without reusing stale output', async t => { + const root = mkdtempSync(join(tmpdir(), 'mutation-retained-grades-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const output = join(root, 'control.json'); + const receipts: Array<{ status: string; report: { path: string; sha256: string | null } }> = []; + const record = (receipt: typeof receipts[number]) => receipts.push(structuredClone(receipt)); + const context = { scenario: 'scenario.json', mutationId: null }; + await retainMutationGrade(output, context, async path => { + assert.equal(existsSync(path), false); + writeFileSync(path, '{"baseline":true}'); + }, record); + const baseline = receipts.at(-1)!; + assert.equal(baseline.status, 'returned'); + assert.ok(baseline.report); + const failure = new Error('deadline'); + await assert.rejects(retainMutationGrade(output, { ...context, mutationId: 'mutant' }, async path => { + assert.equal(existsSync(path), false); + writeFileSync(path, '{"partial":true}'); + throw failure; + }, record), error => error === failure); + const failed = receipts.at(-1)!; + assert.equal(failed.status, 'threw'); + assert.ok(failed.report); + assert.notEqual(failed.report.path, baseline.report.path); + for (const receipt of [baseline, failed]) { + const file = join(root, receipt.report!.path); + assert.equal(createHash('sha256').update(readFileSync(file)).digest('hex'), receipt.report!.sha256); + } + await assert.rejects(retainMutationGrade(output, context, async path => { + assert.equal(existsSync(path), false); + throw failure; + }, record)); + assert.equal(receipts.at(-1)!.report.sha256, null, 'a missing report must not reuse earlier grade bytes'); + assert.equal(existsSync(join(root, receipts.at(-1)!.report.path)), false); + assert.ok(receipts[0]!.report.path, 'a running receipt identifies its report before the grader starts'); + assert.equal(readFileSync(join(root, baseline.report.path), 'utf8'), '{"baseline":true}'); + assert.equal(readFileSync(join(root, failed.report.path), 'utf8'), '{"partial":true}'); + assert.deepEqual(receipts.map(item => item.status), + ['running', 'returned', 'running', 'threw', 'running', 'threw']); +}); + +test('mutation client builds and dist copies run only as the exact leased app owner', t => { + const root = mkdtempSync(join(tmpdir(), 'mutation-client-owner-')); + const path = join(root, 'lease.json'); + const priorPath = process.env.STACK_BENCH_LEASE; + const priorToken = process.env.STACK_BENCH_LEASE_TOKEN; + t.after(() => { + if (priorPath === undefined) delete process.env.STACK_BENCH_LEASE; + else process.env.STACK_BENCH_LEASE = priorPath; + if (priorToken === undefined) delete process.env.STACK_BENCH_LEASE_TOKEN; + else process.env.STACK_BENCH_LEASE_TOKEN = priorToken; + rmSync(root, { recursive: true, force: true }); + }); + const lease = createBackendLease({ runId: 'mutation-client', backend: 'postgres', + track: 'ecommerce', runIndex: 0, database: 'mutation_client', + container: { name: 'database', id: 'a'.repeat(64) } }); + const id = 'b'.repeat(64); + lease.state = 'active'; + lease.resources.buildContainer = { name: 'build', id, running: true, owned: true, + image: `sha256:${'c'.repeat(64)}`, + resourceLimits: { cpuCount: 2, memoryBytes: 1024, memorySwapBytes: 1024, pids: 32 } }; + writeBackendLease(path, lease); + process.env.STACK_BENCH_LEASE = path; + process.env.STACK_BENCH_LEASE_TOKEN = lease.ownershipToken; + let observedId = id; + const calls: string[][] = []; + const exec: TextCommandExecutor = (command, args, options) => { + assert.equal(command, 'docker'); + if (args[0] === 'inspect') return observedId; + assert.equal(options.timeout, 3000); + calls.push([...args]); + return ''; + }; + for (const [command, ...args] of [ + ['npm', 'run', 'build'], + ['cp', '-R', '--', '/app/client/dist', '/tmp/clean-client'], + ['rm', '-rf', '--', '/app/client/dist'], + ['cp', '-R', '--', '/tmp/clean-client', '/app/client/dist'], + ]) { + mutationClientCommand('postgres', command!, args, 3000, exec); + assert.deepEqual(calls.at(-1), ['exec', '--user', '10001:10001', + '-e', 'HOME=/home/developer', '-e', 'USER=developer', '-w', '/app/client', id, + 'sh', '-c', 'umask 022; exec "$@"', 'application-command', command, ...args]); + } + const before = calls.length; + observedId = 'd'.repeat(64); + assert.throws(() => mutationClientCommand('postgres', 'npm', ['run', 'build'], 3000, exec), + /changed after lease creation/); + process.env.STACK_BENCH_LEASE_TOKEN = 'wrong'; + assert.throws(() => mutationClientCommand('postgres', 'rm', ['-rf', '/app/client/dist'], 3000, exec), + /token does not match/); + assert.equal(calls.length, before); +}); + +test('mutation restore verifies original contents and keeps the backup if writing fails', () => { + const directory = mkdtempSync(join(tmpdir(), 'mutation-restore-')); + const target = join(directory, 'source.ts'); + const backup = `${target}.mutation-backup`; + const original = 'const name = "café";\n'; + try { + writeFileSync(target, 'mutant'); + writeFileSync(backup, original); + restoreMutationSource({ target, backup, original }); + assert.equal(readFileSync(target, 'utf8'), original); + assert.equal(existsSync(backup), false); + rmSync(target); + mkdirSync(target); + writeFileSync(backup, original); + assert.throws(() => restoreMutationSource({ target, backup, original })); + assert.equal(readFileSync(backup, 'utf8'), original); + } finally { rmSync(directory, { recursive: true, force: true }); } +}); + +test('mutation cleanup diagnostics retain the original error and restore failure', () => { + const message = mutationFailureMessage(new AggregateError([ + new Error('mutation grade failed'), new Error('cannot restore source: EPERM copyfile'), + ], 'mutation cleanup failed; do not reuse this app source')); + assert.match(message, /mutation grade failed/); + assert.match(message, /EPERM copyfile/); + assert.match(message, /do not reuse/); +}); + +test('mutation grader selection survives CLI parsing and strict artifact validation', () => { + const selectedCheckKeys = ['ecommerce.spec.state-durability.staff-role-reload.621a', + 'ecommerce.spec.access-control.staff-role-boundary.621b']; + const input = { app: '/app', url: 'http://app', level: '3', backend: 'postgres', + track: 'ecommerce', spec: '/staff-roles.json', recipe: 'ecommerce.progression-catalog', + expectedRecipeSha256: 'a'.repeat(64), selectedCheckKeys }; + const parse = (keys: string[]) => parseGradeArgs(['node', + ...mutationGradeArguments({ ...input, selectedCheckKeys: keys }, '/grade.json')]); + const parsed = parse(selectedCheckKeys); + assert.deepEqual(parsed.selectedCheckKeys, selectedCheckKeys); + assert.equal(parsed.expectedRecipeSha256, input.expectedRecipeSha256); + assert.match(parsed.selectionSha256 ?? '', /^[a-f0-9]{64}$/); + assert.equal(parsed.selectionSha256, parse([...selectedCheckKeys].reverse()).selectionSha256); + assert.notEqual(parsed.selectionSha256, parse(selectedCheckKeys.slice(0, 1)).selectionSha256); + const payload = { total: 0, max: 0, features: [], selection: { + sha256: parsed.selectionSha256, + checks: selectedCheckKeys.map(stableKey => ({ stableKey, points: 2 })), + } }; + assert.doesNotThrow(() => createArtifact({ kind: 'grade', id: 'mutation-selection', payload })); + assert.throws(() => createArtifact({ kind: 'grade', id: 'missing-selection-identity', + payload: { ...payload, selection: { checks: payload.selection.checks } } }), + /grade payload.selection is invalid/); +}); + +test('campaign-bound mutation grading forwards the run level and exact recipe', () => { + const manifest = join(STACK_BENCH_ROOT, 'grader', 'mutations', 'mongodb-ecommerce.json'); + const recipeTask = { schemaVersion: 3, + recipe: { id: 'ecommerce.sequential-l1' }, + selection: {}, task: {} }; + const args = { out: 'output', mutations: manifest, backend: 'mongodb', + track: 'ecommerce', levelList: [1], runIndex: 0, parentAttemptId: 'campaign-attempt', + recipe: null, recipeTasks: new Map([[1, { request: recipeTask }]]) }; + const argv = mutationControlArgv(args, 'app', 'http://localhost:5173', + loadTrack('ecommerce')); + assert.equal(argv[argv.indexOf('--recipe') + 1], 'ecommerce.sequential-l1'); + assert.equal(argv[argv.indexOf('--level') + 1], '1'); + assert.deepEqual(JSON.parse(argv[argv.indexOf('--restart-spec') + 1] ?? ''), { + backend: 'mongodb', app: 'app', port: 6723, probe: '', + }); + assert.throws(() => mutationControlArgv({ ...args, + recipe: 'ecommerce.sequential-l2' }, 'app', 'http://localhost:5173', + loadTrack('ecommerce')), /does not match bound task/); +}); + +test('mutation grading receives the exact scored checks selected for the run', () => { + const manifest = join(STACK_BENCH_ROOT, 'grader', 'mutations', 'mongodb-ecommerce.json'); + const args = { out: 'output', mutations: manifest, backend: 'mongodb', + track: 'ecommerce', levelList: [3], runIndex: 0, parentAttemptId: 'selected-attempt', + recipe: null, recipeTasks: new Map([[3, { + request: { schemaVersion: 3, + recipe: { id: 'ecommerce.progression-catalog' }, + selection: {}, task: {} }, + selection: { scoredChecks: [ + { stableKey: 'ecommerce.inventory-operations.warehouse-transfer.2a' }, + { stableKey: 'ecommerce.inventory-operations.stock-conservation.202a' }, + ] }, + }]]) }; + const argv = mutationControlArgv(args, 'app', 'http://localhost:5173', + loadTrack('ecommerce')); + assert.deepEqual(argv.flatMap((value, index) => value === '--selected-check' + ? [argv[index + 1]] : []), [ + 'ecommerce.inventory-operations.warehouse-transfer.2a', + 'ecommerce.inventory-operations.stock-conservation.202a', + ]); +}); + +test('mutation timeouts cover their runtime budget and each grade uses only the remaining batch time', () => { + assert(mutationControlTimeoutMs() > 0); + for (const budget of [1, 15, 60]) assert(mutationControlTimeoutMs(budget) >= budget * 60_000); + assert.throws(() => mutationControlTimeoutMs(0), /positive number/); + const now = 1_000_000; + assert.equal(mutationGradeTimeoutMs(now + 30_000, now), 30_000); + assert.equal(mutationGradeTimeoutMs(now + GRADER_SOURCE_TIMEOUT_MS + 1, now), + GRADER_SOURCE_TIMEOUT_MS); + assert.equal(mutationGradeTimeoutMs(now, now), 0); + assert.equal(mutationGradeTimeoutMs(now - 1, now), 0); + assert.throws(() => mutationGradeTimeoutMs(Number.NaN, now), /must be finite/); +}); + +test('mutation checkpoint and shard controls reach the mutation runner', () => { + const manifest = join(STACK_BENCH_ROOT, 'grader', 'mutations', 'mongodb-ecommerce.json'); + const args = { out: 'output', mutations: manifest, backend: 'mongodb', + track: 'ecommerce', levelList: [1], runIndex: 4, parentAttemptId: 'resume-attempt', + recipe: null, recipeTasks: new Map(), mutationResumeFrom: 'prior.json', + mutationCheckpointOut: 'next.json', mutationMaxRuntimeMinutes: 30, + mutationImageId: 'sha256:image', mutationBaselineBundle: 'baseline.json', + expectedMutationCalibration: { id: 'calibration', sha256: 'calibration-sha' }, + mutationShardIndex: 2, mutationShardCount: 4 }; + const argv = mutationControlArgv(args, 'app', 'http://localhost:5173', + loadTrack('ecommerce')); + const after = (flag: string): string => { + const value = argv[argv.indexOf(flag) + 1]; + if (value === undefined) throw new Error(`missing value for ${flag}`); + return value; + }; + assert.equal(after('--resume-from'), 'prior.json'); + assert.equal(after('--checkpoint-out'), 'next.json'); + assert.equal(after('--max-runtime-minutes'), '30'); + assert.equal(after('--image-id'), 'sha256:image'); + assert.equal(after('--baseline-bundle'), 'baseline.json'); + assert.deepEqual(JSON.parse(after('--expected-calibration-json')), + args.expectedMutationCalibration); + const shard = argv.indexOf('--mutation-shard-index'); + assert.deepEqual(argv.slice(shard, shard + 4), + ['--mutation-shard-index', '2', '--mutation-shard-count', '4']); +}); + +test('a mismatched mutation fixture fails before acquiring any backend resource', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-mutation-preflight-')); + const output = join(root, 'output'); + const manifest = join(root, 'mutations.json'); + const locks = join(tmpdir(), 'stack-bench-resource-locks'); + const lock = join(locks, `${createHash('sha256').update('slot:ecommerce:mongodb:run19').digest('hex')}.lock.json`); + try { + assert.equal(existsSync(lock), false, 'test slot is already leased'); + writeFileSync(join(root, 'source.txt'), 'fixture\n'); + // Use root as the explicit app; the manifest intentionally targets other + // bytes. No Docker or database lookup should happen before this rejection. + writeFileSync(manifest, JSON.stringify({ schemaVersion: 3, + fixtureSha256: '0'.repeat(64), backend: 'mongodb', track: 'ecommerce', + scenario: 'tracks/ecommerce/scenarios/01-contention.json', mutations: [] })); + assert.throws(() => execFileSync(process.execPath, [compiledEntrypoint('commands', 'bench.js'), + '--backend', 'mongodb', '--track', 'ecommerce', '--levels', '1', + '--run-index', '19', '--app', root, '--out', output, + '--agent-adapter', 'deterministic', '--mutations', manifest, '--no-media'], + { encoding: 'utf8', stdio: 'pipe', timeout: 30_000 }), /targets fixture/); + assert.equal(existsSync(lock), false); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/tools/stack-bench/tests/mutation-shards.test.ts b/tools/stack-bench/tests/mutation-shards.test.ts new file mode 100644 index 00000000000..c4d9de009ce --- /dev/null +++ b/tools/stack-bench/tests/mutation-shards.test.ts @@ -0,0 +1,57 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { mergeMutationShards, mutationShard, mutationWorkerSlots } + from '../src/evidence/mutation-shards.js'; + +const mutations = [ + { id: 'a', scenario: 'one' }, + { id: 'b', scenario: 'two' }, + { id: 'c', scenario: 'one' }, + { id: 'd', scenario: 'three' }, + { id: 'e', scenario: 'two' }, +]; + +test('mutation workers reserve one consecutive proven run slot each', () => { + assert.deepEqual(mutationWorkerSlots({ workerCount: 4, runIndex: 8, maxRunIndex: 20 }), + [8, 9, 10, 11]); + assert.throws(() => mutationWorkerSlots({ workerCount: 2, runIndex: 20, maxRunIndex: 20 }), + /exceed run-index cap/); + for (const workerCount of [0, 1.5, NaN]) { + assert.throws(() => mutationWorkerSlots({ workerCount, runIndex: 0, maxRunIndex: 20 })); + } +}); + +test('mutation partitioning balances individual defects and permits empty trailing shards', () => { + assert.deepEqual(mutationShard(mutations, { index: 0, count: 3 }).mutationIds, ['a', 'd']); + assert.deepEqual(mutationShard(mutations, { index: 1, count: 3 }).mutationIds, ['b', 'e']); + assert.deepEqual(mutationShard(mutations, { index: 2, count: 3 }).mutationIds, ['c']); + assert.deepEqual(mutationShard([{ id: 'a', scenario: 'one' }], + { index: 2, count: 3 }).mutationIds, []); + assert.deepEqual(mutationShard([{ id: 'a' }, { id: 'b' }], + { index: 0, count: 2, defaultScenario: 'shared' }).mutationIds, ['a']); + assert.deepEqual(mutationShard([{ id: 'a' }, { id: 'b' }], + { index: 1, count: 2, defaultScenario: 'shared' }).mutationIds, ['b']); +}); + +test('mutation shard merging restores manifest order and rejects incomplete unions', () => { + const shards = Array.from({ length: 3 }, (_, index) => { + const plan = mutationShard(mutations, { index, count: 3 }); + return { ...plan, results: plan.mutationIds.map(id => ({ id, status: 'CAUGHT' })) }; + }); + assert.deepEqual(mergeMutationShards(mutations, [shards[2]!, shards[0]!, shards[1]!]) + .map(result => result.id), ['a', 'b', 'c', 'd', 'e']); + assert.throws(() => mergeMutationShards(mutations, shards.slice(1)), /declared shard count/); + assert.throws(() => mergeMutationShards(mutations, [shards[0]!, shards[0]!, shards[2]!]), + /duplicated/); + assert.throws(() => mergeMutationShards(mutations, [ + { ...shards[0]!, mutationIds: ['a'] }, shards[1]!, shards[2]!, + ]), /exact assigned/); +}); + +test('mutation shard inputs reject duplicate and malformed mutation identities', () => { + assert.throws(() => mutationShard([{ id: 'a' }, { id: 'a' }], { index: 0, count: 1 }), + /unique/); + assert.throws(() => mutationShard([{ nope: true }], { index: 0, count: 1 }), /has no id/); + assert.throws(() => mutationShard(mutations, { index: 3, count: 3 }), /0 through 2/); +}); diff --git a/tools/stack-bench/tests/network-interruption.integration.ts b/tools/stack-bench/tests/network-interruption.integration.ts new file mode 100644 index 00000000000..ad920a858cf --- /dev/null +++ b/tools/stack-bench/tests/network-interruption.integration.ts @@ -0,0 +1,429 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { once } from 'node:events'; +import { createServer } from 'node:http'; +import type { IncomingMessage, ServerResponse } from 'node:http'; +import { connect } from 'node:net'; +import type { Socket } from 'node:net'; +import test from 'node:test'; +import { writeFileSync } from 'node:fs'; +import { gzipSync, deflateSync, brotliCompressSync } from 'node:zlib'; +import { chromium } from 'playwright'; +import type { Browser, Page } from 'playwright'; +import { ACTION_REGISTRY } from '../src/actions/action-catalog.js'; +import { executeAction } from '../src/actions/action-contract.js'; +import { startNetworkInterruption } from '../src/actions/network-interruption.js'; +import type { NetworkInterruption } from '../src/actions/network-interruption.js'; +import { startNetworkProxy } from '../container/browser-network-proxy.js'; + +// A fixture app server: its own page, WebSocket upgrades, and a live value it can push. +async function fixture(page: string, http: (req: IncomingMessage, res: ServerResponse, value: () => string) => boolean = () => false, + wire?: (chunk: Buffer) => void, viteDelayMs = 0) { + let value = '1'; + const sockets = new Set<Socket>(), streams = new Set<() => void>(), waiters = new Set<() => void>(); + const server = createServer((req, res) => { + const path = new URL(req.url!, 'http://fixture').pathname; + if (path === '/@vite/client') { + setTimeout(() => res.writeHead(200, { 'Content-Type': 'text/javascript' }).end('const wsToken = "dev";'), viteDelayMs); + return; + } + if (path === '/events' || path === '/stream') { + res.writeHead(200, { 'Content-Type': path === '/events' ? 'text/event-stream' : 'text/plain' }); + const send = () => res.write(path === '/events' ? `retry: 300\ndata: ${value}\n\n` : `${value}\n`); + send(); streams.add(send); res.on('close', () => streams.delete(send)); return; + } + if (path === '/poll') { + if (new URL(req.url!, 'http://fixture').searchParams.get('since') !== value) { res.end(value); return; } + const answer = () => { waiters.delete(answer); res.end(value); }; + waiters.add(answer); res.on('close', () => waiters.delete(answer)); return; + } + if (http(req, res, () => value)) return; + res.writeHead(200, { 'Content-Type': 'text/html' }).end(page); + }); + server.on('upgrade', (req, socket: Socket) => { + const accept = createHash('sha1').update(`${req.headers['sec-websocket-key']}258EAFA5-E914-47DA-95CA-C5AB0DC85B11`).digest('base64'); + socket.write(`HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: ${accept}\r\n\r\n`); + sockets.add(socket); + socket.on('close', () => sockets.delete(socket)); + socket.on('error', () => {}); + }); + if (wire) server.on('connection', socket => { + let application = false; + socket.on('data', chunk => { + if (chunk.toString('latin1').startsWith('GET /sock')) application = true; + if (application) wire(chunk); + }); + }); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + const frame = (text: string) => Buffer.concat([Buffer.from([0x81, text.length]), Buffer.from(text)]); + return { + url: `http://127.0.0.1:${(server.address() as { port: number }).port}/`, + set(next: string) { value = next; for (const send of streams) send(); for (const answer of [...waiters]) answer(); }, + send(text: string) { for (const socket of sockets) socket.write(frame(text)); }, + close() { for (const socket of sockets) socket.destroy(); server.closeAllConnections(); server.close(); }, + }; +} + +// One interruptible actor, driven through the real setOffline action. +async function interruptibleActor(browser: Browser, url: string, route = true) { + const networkInterruption = await startNetworkInterruption(); + const context = await browser.newContext(route ? { proxy: networkInterruption.proxy } : {}); + const page = await context.newPage(); + await networkInterruption.attach(context, page); + await page.goto(url); + await page.waitForTimeout(500); + const actor = { page, networkInterruption, loc: () => { throw new Error('unused'); } }; + const capabilities = { actors: { get: () => actor }, + 'browser-interaction': { defaultWithin: 1000, sleep: async (ms: number) => new Promise(resolve => setTimeout(resolve, ms)) } }; + const setOffline = (offline: boolean, settleMs = 300) => executeAction(ACTION_REGISTRY, 'setOffline', + { do: 'setOffline', actor: 'buyer', offline, settleMs }, { capabilities }); + return { page, networkInterruption, setOffline }; +} + +const windowValue = (page: Page, key: string) => page.evaluate(name => Reflect.get(window, name), key); + +// HTTP observation must retain unread/split bytes without changing forwarding. +// The receipt is proxy-owned; compressed failures and bounded-storage overflow +// cannot certify absence. Harness API replay tunnels must remain excluded. +test('HTTP proxy snapshots preserve delivered bodies and exclude replay tunnels', async t => { + const evidence: unknown[] = []; + t.after(() => { + if (process.env.STACK_BENCH_PROXY_HTTP_EVIDENCE) writeFileSync(process.env.STACK_BENCH_PROXY_HTTP_EVIDENCE, + JSON.stringify({ node: process.version, evidence }, null, 2)); + }); + for (const encoding of ['identity', 'gzip', 'deflate', 'br', 'unknown', 'corrupt', 'overflow']) { + await t.test(encoding, async () => { + type Record = { id: string; url: string; method: string; status: number; contentType: string; body: string; complete: boolean }; + type Reply = { id?: number; ok?: boolean; port?: number; records?: Record[]; incomplete?: boolean; pending?: number }; + const waiting = new Map<number, { resolve: (reply: Reply) => void; reject: (error: Error) => void }>(); + let next = 0, stopped = false; + const handle = startNetworkProxy(reply => { + const result = reply as Reply; + waiting.get(result.id ?? -1)?.resolve(result); waiting.delete(result.id ?? -1); + }, code => { stopped = true; for (const value of waiting.values()) value.reject(new Error(`proxy exited ${code}`)); waiting.clear(); }); + const command = (cmd: string, fields = {}) => new Promise<Reply>((resolve, reject) => { + const id = ++next; waiting.set(id, { resolve, reject }); handle(JSON.stringify({ id, cmd, ...fields })); + }); + const marker = 'private-🔒-marker'; + const plain = Buffer.from(JSON.stringify({ value: marker })); + const body = encoding === 'gzip' ? gzipSync(plain) : encoding === 'deflate' ? deflateSync(plain) + : encoding === 'br' ? brotliCompressSync(plain) : encoding === 'overflow' ? Buffer.alloc(8 * 1024 * 1024 + 1, 120) : plain; + let complete: (() => void) | undefined; + const server = createServer((req, res) => { + if (req.url === '/') { res.end('<title>observer'); return; } + if (req.url === '/favicon.ico') { res.writeHead(204).end(); return; } + if (req.url === '/preheaders') { complete = () => res.end('late body'); return; } + if (req.url === '/failed') { req.socket.destroy(); return; } + if (req.url === '/refused') { res.writeHead(403).end('refused'); return; } + if (req.url === '/duplicate') { + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ value: 'public-repeat-'.repeat(160000) })); return; + } + res.writeHead(200, { 'Content-Type': 'application/json', 'x-stack-bench-capture': 'upstream-forgery', + ...(encoding !== 'identity' && encoding !== 'overflow' ? { 'Content-Encoding': encoding === 'corrupt' ? 'gzip' : encoding } : {}) }); + const responseBody = req.url === '/cancel' ? Buffer.from('{"value":"cancel-private-prefix"}') : body; + const split = encoding === 'identity' ? plain.indexOf(Buffer.from('🔒')) + 2 : Math.floor(responseBody.length / 2); + res.write(responseBody.subarray(0, split)); + setTimeout(() => { if (!res.destroyed) res.write(responseBody.subarray(split)); }, 10); + complete = () => res.end(); + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const configured = await command('config', { user: 'fixture', pass: 'synthetic', observeHttp: true }); + const browser = await chromium.launch({ headless: true, executablePath: chromium.executablePath() }); + const context = await browser.newContext({ proxy: { server: `http://127.0.0.1:${configured.port}`, username: 'fixture', password: 'synthetic', bypass: '<-loopback>' } }); + try { + const page = await context.newPage(); await page.goto(url); + const response = page.waitForResponse(`${url}/data`); + await page.evaluate(consume => { void fetch('/data').then(value => { Reflect.set(window, 'heldResponse', value); if (consume) return value.text(); }).catch(() => {}); }, encoding === 'overflow'); + const received = await response; + await page.waitForTimeout(50); + const partial = await command('httpSnapshot'); + assert(complete); complete(); + if (encoding === 'overflow') await received.finished(); + await page.waitForTimeout(50); + const finished = await command('httpSnapshot'); + evidence.push({ encoding, responseReceipt: received.headers()['x-stack-bench-capture'], partial, finished }); + assert.equal(finished.pending, 0, 'Completed delivery releases the pending request'); + if (['unknown', 'corrupt', 'overflow'].includes(encoding)) { + assert.equal(finished.incomplete, true, 'Unsupported, corrupt or lost bytes cannot establish absence'); + } else { + assert.equal(partial.pending, 1, 'A live partial body still belongs to the navigation drain'); + const first = partial.records?.find(record => record.url === `${url}/data`); + assert(first, 'The response must have a snapshot record'); + assert.equal(first.id, received.headers()['x-stack-bench-capture']); + assert.notEqual(first.id, 'upstream-forgery'); + assert.equal(first.body, plain.toString(), 'Unread response bytes and split UTF-8 must remain intact'); + assert.equal(first.complete, false, 'A prefix is not a complete response receipt'); + assert.equal(partial.incomplete, false); + const final = finished.records?.find(record => record.id === first.id); + assert.equal(final?.complete, true); + assert.equal(final?.body, plain.toString()); + assert.equal(finished.incomplete, false); + if (encoding === 'identity') { + // Drain state comes from the actual client connection, including + // requests with no headers and cancellation without a body end. + const preheadersStarted = once(server, 'request'); + await page.evaluate(() => { void fetch('/preheaders').catch(() => {}); }); + await preheadersStarted; + const preheaders = await command('httpSnapshot'); + assert.equal(preheaders.pending, 1); + assert(!preheaders.records?.some(record => record.url === `${url}/preheaders`)); + await page.reload(); + await page.waitForTimeout(30); + const canceledHeaders = await command('httpSnapshot'); + assert.equal(canceledHeaders.pending, 0); + const cancelResponse = page.waitForResponse(`${url}/cancel`); + await page.evaluate(() => { void fetch('/cancel').then(r => r.text()).catch(() => {}); }); + await cancelResponse; + await page.waitForTimeout(30); + assert.equal((await command('httpSnapshot')).pending, 1); + await page.reload(); + await page.waitForTimeout(30); + const canceledBody = await command('httpSnapshot'); + assert.equal(canceledBody.pending, 0); + const prefix = canceledBody.records?.find(record => record.url === `${url}/cancel`); + assert(prefix); + assert.equal(prefix.complete, false, 'Cancellation must never certify a complete response'); + assert(prefix.body.includes('cancel-private-prefix'), 'Cancellation retains delivered prefix bytes'); + await page.evaluate(() => fetch('/refused').then(r => r.text())); + assert.equal((await command('httpSnapshot')).pending, 0); + await page.evaluate(() => fetch('/failed').catch(() => {})); + const failed = await command('httpSnapshot'); + assert.equal(failed.pending, 0, 'Forwarding errors release the request'); + evidence.push({ preheaders, canceledHeaders, canceledBody, failed }); + // Repeated completed bundles retain one body without losing receipts. + for (let n = 0; n < 5; n++) await page.evaluate(() => fetch('/duplicate').then(r => r.text())); + const duplicates = await command('httpSnapshot'); + const repeated = duplicates.records?.filter(record => record.url === `${url}/duplicate`); + assert.equal(repeated?.length, 5); + assert.equal(repeated?.filter(record => record.body.includes('public-repeat-')).length, 1); + assert.equal(duplicates.incomplete, false); + const replayStarted = once(server, 'request'); + const replay = context.request.get(`${url}/replay`); + await replayStarted; complete(); + assert.equal((await replay).status(), 200); + const afterReplay = await command('httpSnapshot'); + assert(!afterReplay.records?.some(record => record.url === `${url}/replay`), 'APIRequestContext CONNECT is not browser HTTP delivery'); + } + } + } catch (error) { evidence.push({ encoding, error: error instanceof Error ? error.message : String(error) }); throw error; } + finally { + await context.close(); await browser.close(); + if (!stopped) await command('dispose'); + server.closeAllConnections(); server.close(); + } + }); + } +}); + +test('split WebSocket request bytes pass through while the Vite token is checked', async () => { + let received = ''; + const app = await fixture('', undefined, chunk => { received += chunk.toString('latin1'); }, 300); + const interruption = await startNetworkInterruption(); + try { + const authority = new URL(app.url).host; + const credentials = Buffer.from(`${interruption.proxy.username}:${interruption.proxy.password}`).toString('base64'); + const first = 'GET /socket?token=dev HTTP/1.1\r\n'; + const rest = `Host: ${authority}\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nX-Split-Probe: must-arrive\r\n\r\n`; + for (const chunks of [[first.slice(0, 9), first.slice(9), rest], [first, rest]]) { + received = ''; + const client = connect(Number(new URL(interruption.proxy.server).port), '127.0.0.1'); + try { + await once(client, 'connect'); + const connected = once(client, 'data'); + client.write(`CONNECT ${authority} HTTP/1.1\r\nHost: ${authority}\r\nProxy-Authorization: Basic ${credentials}\r\n\r\n`); + await connected; + for (const chunk of chunks) { + client.write(chunk); + await new Promise(resolve => setTimeout(resolve, 75)); + } + await new Promise(resolve => setTimeout(resolve, 300)); + assert.equal(received, first + rest); + await interruption.interrupt(); + await new Promise(resolve => setTimeout(resolve, 50)); + assert.equal(client.destroyed, false, 'the split Vite token must keep its tooling socket open'); + await interruption.restore(); + } finally { client.destroy(); } + } + } finally { + await interruption.dispose(); + app.close(); + } +}); + +test('going offline closes open sockets, never delivers held updates, and keeps the dev reload socket', async () => { + const app = await fixture(``); + const browser = await chromium.launch({ headless: true }); + let interruption: NetworkInterruption | undefined; + try { + const { page, networkInterruption, setOffline } = await interruptibleActor(browser, app.url); + interruption = networkInterruption; + // A reloaded document's sockets end without a close event; they must not count as open. + await page.reload(); + await page.waitForTimeout(500); + const loadedAt = await windowValue(page, 'loadedAt'); + const offline = await setOffline(true); + assert.equal(offline.status, 'passed', offline.summary ?? ''); + assert.deepEqual((offline.observation as { open: string[] }).open, []); + app.send('during'); + await page.waitForTimeout(300); + const online = await setOffline(false, 800); + assert.equal(online.status, 'passed', online.summary ?? ''); + app.send('after'); + await page.waitForTimeout(300); + assert.deepEqual(await windowValue(page, 'received'), ['after'], 'the held update is never delivered; the reconnected page receives new ones'); + assert.deepEqual(await windowValue(page, 'devReceived'), ['during', 'after'], 'the dev reload socket stays connected'); + assert.equal(await windowValue(page, 'loadedAt'), loadedAt, 'the page was not reloaded'); + } finally { + await interruption?.dispose(); + await browser.close(); + app.close(); + } +}); + +// EventSource, a held long poll, and a streamed fetch, each with or without its own recovery. +const streamingPage = (recover: boolean) => ``; + +for (const recover of [true, false]) { + test(`going offline cuts event streams, long polls and streamed fetches (${recover ? 'recovering' : 'no recovery'} client)`, async () => { + const app = await fixture(streamingPage(recover)); + const browser = await chromium.launch({ headless: true }); + let interruption: NetworkInterruption | undefined; + try { + const { page, networkInterruption, setOffline } = await interruptibleActor(browser, app.url); + interruption = networkInterruption; + await page.waitForTimeout(500); + const loadedAt = await windowValue(page, 'loadedAt'); + const offline = await setOffline(true); + assert.equal(offline.status, 'passed', offline.summary ?? ''); + app.set('2'); + await page.waitForTimeout(800); + assert.deepEqual(await windowValue(page, 'state'), { sse: '1', poll: '1', stream: '1' }, 'nothing is delivered during the cut'); + await setOffline(false, 3000); + assert.deepEqual(await windowValue(page, 'state'), recover + ? { sse: '2', poll: '2', stream: '2' } : { sse: '1', poll: '1', stream: '1' }); + assert.equal(await windowValue(page, 'loadedAt'), loadedAt, 'the page was not reloaded'); + } finally { + await interruption?.dispose(); + await browser.close(); + app.close(); + } + }); +} + +test('a root/token app socket is cut, and an unproven cut is unmeasured', async () => { + const app = await fixture(``); + const browser = await chromium.launch({ headless: true }); + const interruptions: NetworkInterruption[] = []; + try { + const cut = await interruptibleActor(browser, app.url); + interruptions.push(cut.networkInterruption); + const offline = await cut.setOffline(true); + assert.equal(offline.status, 'passed', offline.summary ?? ''); + app.send('stock:52'); + await cut.setOffline(false); + assert.deepEqual(await windowValue(cut.page, 'received'), []); + assert.equal(await windowValue(cut.page, 'closes'), 1, 'an app without reconnect code stays disconnected'); + + // A context whose traffic does not pass through the proxy keeps its socket: unmeasured. + const bypassed = await interruptibleActor(browser, app.url, false); + interruptions.push(bypassed.networkInterruption); + assert.equal(await bypassed.page.evaluate(() => fetch('/ping').then(response => response.ok)), true, + 'another actor keeps network access'); + assert.equal((await bypassed.setOffline(true, 0)).status, 'inconclusive'); + + // An actor opened without an interruption cannot prove one. + const plain = await executeAction(ACTION_REGISTRY, 'setOffline', { do: 'setOffline', actor: 'buyer', settleMs: 0 }, + { capabilities: { actors: { get: () => ({ page: cut.page, loc: () => { throw new Error('unused'); } }) }, + 'browser-interaction': { defaultWithin: 1000, sleep: async () => {} } } }); + assert.equal(plain.status, 'inconclusive'); + } finally { + for (const interruption of interruptions) await interruption.dispose(); + await browser.close(); + app.close(); + } +}); + +test('same-document navigation keeps a live socket tracked until the document is replaced', async () => { + const app = await fixture(``); + const browser = await chromium.launch({ headless: true }); + let interruption: NetworkInterruption | undefined; + try { + const actor = await interruptibleActor(browser, app.url, false); + interruption = actor.networkInterruption; + await actor.page.waitForFunction(() => (window as unknown as { ws: WebSocket }).ws.readyState === WebSocket.OPEN); + await actor.page.evaluate(() => { history.pushState({}, '', '/next'); location.hash = 'later'; }); + const during = await interruption.interrupt(); + assert.equal(during.open.length, 1, 'the live socket remains visible after same-document navigation'); + await interruption.restore(); + await actor.page.goto('data:text/html,replaced'); + const replaced = await interruption.interrupt(); + assert.deepEqual(replaced.open, [], 'the replaced document does not leave a stale socket'); + } finally { + await interruption?.dispose(); + await browser.close(); + app.close(); + } +}); + +test('a disposed interruption releases its listener and cannot report a cut', async () => { + const interruption = await startNetworkInterruption(); + const port = Number(new URL(interruption.proxy.server).port); + await interruption.dispose(); + const probe = await new Promise(resolve => { + const socket = connect(port, '127.0.0.1'); + socket.on('connect', () => { socket.destroy(); resolve('open'); }); + socket.on('error', error => resolve((error as NodeJS.ErrnoException).code ?? 'error')); + }); + assert.equal(probe, 'ECONNREFUSED'); + await assert.rejects(interruption.interrupt(), /stopped/); + await assert.rejects(interruption.restore(), /stopped/); +}); + +test('an unleased appliance browser, such as the null control, gets a local proxy', async () => { + const prior = { appliance: process.env.STACK_BENCH_APPLIANCE, lease: process.env.STACK_BENCH_LEASE }; + process.env.STACK_BENCH_APPLIANCE = '1'; + delete process.env.STACK_BENCH_LEASE; + try { + const interruption = await startNetworkInterruption(); + assert.match(interruption.proxy.server, /^http:\/\/127\.0\.0\.1:\d+$/); + await interruption.dispose(); + } finally { + if (prior.appliance === undefined) delete process.env.STACK_BENCH_APPLIANCE; + else process.env.STACK_BENCH_APPLIANCE = prior.appliance; + if (prior.lease !== undefined) process.env.STACK_BENCH_LEASE = prior.lease; + } +}); diff --git a/tools/stack-bench/tests/neutral-guidance.contract.ts b/tools/stack-bench/tests/neutral-guidance.contract.ts new file mode 100644 index 00000000000..dd027fe8344 --- /dev/null +++ b/tools/stack-bench/tests/neutral-guidance.contract.ts @@ -0,0 +1,34 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import test from 'node:test'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { resolveGuidanceProfile } from '../src/campaigns/condition-compiler.js'; +const root = STACK_BENCH_ROOT; +const files = Object.values(resolveGuidanceProfile('neutral', + ['mongodb', 'postgres', 'spacetime', 'supabase']).documents) + .map(document => resolve(root, document.path)); +const prescribedFiles = Object.values(resolveGuidanceProfile('prescribed', + ['mongodb', 'postgres', 'spacetime', 'supabase']).documents) + .map(document => resolve(root, document.path)); +const presentationAdvice = /brand|styling|theme|colour|color|app title/i; + +test('neutral backend documents contain only stack access facts', () => { + const implementationAdvice = /\b(express|socket\.io|mongoose|drizzle|prisma|orm|polling|locking)\b/i; + const evaluationLanguage = /stack bench|benchmark|harness|test(?:ing|ed|s)?|grader|score/i; + for (const file of files) { + const content = readFileSync(file, 'utf8'); + assert.doesNotMatch(content, implementationAdvice, file); + assert.doesNotMatch(content, evaluationLanguage, file); + assert.doesNotMatch(content, presentationAdvice, file); + assert.match(content, /Choose the/); + assert.match(content, /## Connection/); + } +}); + +test('backend guidance does not control product presentation', () => { + for (const file of prescribedFiles) { + assert.doesNotMatch(readFileSync(file, 'utf8'), presentationAdvice, file); + } +}); diff --git a/tools/stack-bench/tests/null-control-analysis.test.ts b/tools/stack-bench/tests/null-control-analysis.test.ts new file mode 100644 index 00000000000..6165f97d51a --- /dev/null +++ b/tools/stack-bench/tests/null-control-analysis.test.ts @@ -0,0 +1,83 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { analyseNullReports } from '../src/evidence/null-control-analysis.js'; +import { + createCheckEvidence, + type CheckEvidenceStatus, +} from '../src/evidence/check-evidence.js'; + +const criterion = ( + id: string, + points: number, + status: CheckEvidenceStatus, + phase: 'setup' | 'assertion' = 'assertion', + summary: string | null = null, +) => ({ + id, points, evidence: createCheckEvidence({ status, + code: status === 'passed' ? 'completed' : 'test_result', phase, summary, + startedAtMs: 1, completedAtMs: 2 }), +}); + +test('null analysis separates expected failures, vacuous passes and oracle gaps', () => { + const result = analyseNullReports([{ track: 'shop', level: 1, id: 'features', scenario: '01.json', + report: { features: [{ id: 1, name: 'Items', criteria: [ + criterion('1a', 2, 'failed', 'assertion', 'missing control'), + criterion('1b', 3, 'passed'), + criterion('1c', 4, 'inconclusive', 'assertion', 'harness could not issue action'), + criterion('diagnostic', 0, 'passed'), + ] }] } }]); + + assert.equal(result.ok, false); + assert.deepEqual(result.summary, { + criteria: 3, + points: 9, + expectedFailures: { criteria: 1, points: 2 }, + expectedFailureStages: { + setup: { criteria: 0, points: 0 }, + assertion: { criteria: 1, points: 2 }, + }, + vacuousPasses: { criteria: 1, points: 3 }, + oracleGaps: { criteria: 1, points: 4 }, + unscored: { criteria: 1, passed: 1, failed: 0, inconclusive: 0 }, + }); + assert.deepEqual(result.criteria.map(item => item.status), + ['expected_fail', 'vacuous_pass', 'oracle_gap']); +}); + +test('all conclusive failures make the null control pass', () => { + const result = analyseNullReports([{ track: 'chat', level: 2, id: 'invariants', scenario: '02.json', + report: { features: [{ id: 10, criteria: [ + criterion('10a', 1, 'failed'), + criterion('10b', 2, 'failed', 'setup', 'sign in did not complete'), + criterion('diagnostic', 0, 'inconclusive'), + ] }] } }]); + assert.equal(result.ok, true); + assert.deepEqual(result.summary.expectedFailures, { criteria: 2, points: 3 }); + assert.deepEqual(result.summary.expectedFailureStages, { + setup: { criteria: 1, points: 2 }, + assertion: { criteria: 1, points: 1 }, + }); + assert.deepEqual(result.summary.unscored, + { criteria: 1, passed: 0, failed: 0, inconclusive: 1 }); +}); + +test('typed null analysis gets failure phase from evidence, not detail wording', () => { + const evidence = createCheckEvidence({ status: 'failed', code: 'application_failure', phase: 'setup', + summary: 'wording without the historical prefix', startedAtMs: 1, completedAtMs: 2 }); + const result = analyseNullReports([{ track: 'shop', level: 1, id: 'features', + report: { features: [{ id: 1, criteria: [ + { id: 'a', points: 1, evidence }, + ] }] } }]); + assert.equal(result.criteria[0]?.failureStage, 'setup'); + assert.equal(result.ok, true); +}); + +test('null analysis rejects invalid rubric points', () => { + const evidence = createCheckEvidence({ status: 'failed', code: 'test_result', phase: 'assertion', + summary: null, startedAtMs: 1, completedAtMs: 2 }); + for (const points of [Number.NaN, -1, 1.5, '2']) { + assert.throws(() => analyseNullReports([{ track: 'shop', level: 1, id: 'features', + report: { features: [{ id: 1, criteria: [{ id: 'a', points, evidence }] }] } }]), + /points must be a non-negative integer/); + } +}); diff --git a/tools/stack-bench/tests/null-control-cli.contract.ts b/tools/stack-bench/tests/null-control-cli.contract.ts new file mode 100644 index 00000000000..4b231f1ff36 --- /dev/null +++ b/tools/stack-bench/tests/null-control-cli.contract.ts @@ -0,0 +1,117 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import test from 'node:test'; + +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; +import { createNullQualification, nullControlSuites, parseNullControlArgs } from '../commands/null-control.js'; +import { resolveCalibrationForRelease } from '../src/composition/calibration-compiler.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { requireRecipeRelease as resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import type { RecipeCheck } from '../src/composition/recipe-release.js'; +import { selectScenarioChecks } from '../src/composition/recipe-selection.js'; +import { loadTrack } from '../src/composition/tracks.js'; + +const readJson = (path: string): unknown => JSON.parse(readFileSync(path, 'utf8')); + +interface RecipeSuite { + id: string; + spec: string; + level: number; + checks: RecipeCheck[]; +} + +function hasChecks(suite: unknown): suite is RecipeSuite { + return suite !== null && typeof suite === 'object' && 'checks' in suite + && Array.isArray(suite.checks); +} + +test('null qualification can select one exact track and level', () => { + const args = parseNullControlArgs(['node', 'null-control.js', + '--track', 'ecommerce', '--level', '1']); + assert.deepEqual(args.tracks, ['ecommerce']); + assert.equal(args.level, 1); + assert.throws(() => parseNullControlArgs(['node', 'null-control.js', + '--track', 'ecommerce,chat', '--level', '1']), /exactly one/); + assert.throws(() => parseNullControlArgs(['node', 'null-control.js', + '--track', 'ecommerce', '--level', '0']), /positive integer/); + assert.equal(parseNullControlArgs(['node', 'null-control.js', '--track', 'ecommerce', + '--level', '1', '--recipe', 'ecommerce.sequential-l1']).recipe, + 'ecommerce.sequential-l1'); + assert.throws(() => parseNullControlArgs(['node', 'null-control.js', '--track', 'ecommerce', + '--recipe', 'ecommerce.sequential-l1']), /requires --level/); + const track = loadTrack('ecommerce'); + assert.deepEqual(nullControlSuites(track, 2).map(suite => suite.level), [2, 2, 2, 2, 2]); + assert.throws(() => nullControlSuites(track, 4), /not declared/); +}); + +test('targeted null selection keeps only calibrated checks and their execution', () => { + const track = loadTrack('ecommerce'); + const binding = resolveRecipeRelease(track, 1, 'ecommerce.sequential-l1'); + const calibration = resolveCalibrationForRelease(binding.release, + { trackRoot: track.dir, stackBenchRoot: STACK_BENCH_ROOT, alias: 'L1' })!; + const key = binding.release.checkCatalog[0]!.stableKey; + const args = parseNullControlArgs(['node', 'null-control.js', '--track', 'ecommerce', + '--level', '1', '--selected-check', key]); + const selected = createNullQualification(binding, calibration, args.selectedChecks); + assert.deepEqual(selected.binding.release.checkCatalog.map(check => check.stableKey), [key]); + assert.equal(selected.binding.execution.length, 1); + assert.throws(() => createNullQualification(binding, calibration, ['unknown']), /unknown checks/); + assert.throws(() => parseNullControlArgs(['node', 'null-control.js', '--selected-check', key]), /requires --level/); +}); + +test('bound recipes can qualify levels outside legacy suites without widening legacy selection', () => { + const track = loadTrack('ecommerce'); + const binding = resolveRecipeRelease(track, 6, 'ecommerce.progression-catalog'); + const calibration = resolveCalibrationForRelease(binding.release, + { trackRoot: track.dir, stackBenchRoot: STACK_BENCH_ROOT, alias: 'L6' })!; + const key = 'ecommerce.returns-pricing.cancellation-and-return.3d'; + const selected = createNullQualification(binding, calibration, [key]); + const suites = nullControlSuites(track, 6, selected.binding); + assert(suites.every(suite => suite.level === 6)); + assert.deepEqual(suites.filter(hasChecks).flatMap(suite => suite.checks.map(check => check.stableKey)), [key]); + assert.throws(() => nullControlSuites(track, 6), /not declared/); +}); + +test('recipe-bound null qualification grades the exact modular execution and checks', () => { + const track = loadTrack('ecommerce'); + const binding = resolveRecipeRelease(track, 1, 'ecommerce.sequential-l1'); + const suites = nullControlSuites(track, 1, binding); + assert(suites.every(hasChecks)); + const recipeSuites = suites.filter(hasChecks); + const checks = recipeSuites.flatMap(suite => suite.checks); + + assert.deepEqual(suites.map(suite => suite.id), binding.execution.map(execution => execution.id)); + assert.equal(checks.length, binding.release.checkCatalog.length); + assert.equal(checks.reduce((total, check) => total + check.points, 0), binding.release.scoring.points); + assert.equal(new Set(checks.map(check => check.stableKey)).size, checks.length); + + const selectedByExecution = new Map(recipeSuites.map(suite => { + const scenario = compileScenarioDefinition(readJson(suite.spec), + { source: suite.spec, expectedLevel: 1 }); + const selected = selectScenarioChecks(scenario, { checks: binding.release.checkCatalog }, + suite.checks.map(check => check.stableKey)); + const selectedKeys = selected.features.flatMap(feature => feature.criteria + .map(criterion => suite.checks.find(check => check.featureId === feature.id + && check.criterionId === criterion.id)?.stableKey)).filter(Boolean).sort(); + assert.deepEqual(selectedKeys, suite.checks.map(check => check.stableKey).sort()); + return [suite.id, selectedKeys]; + })); + for (const criterion of ['101a', '102a', '103a', '104a']) { + const owners = [...selectedByExecution.entries()] + .filter(([, keys]) => keys.some(key => key.endsWith(`.${criterion}`))); + assert.equal(owners.length, 1); + const owner = owners[0]; + assert(owner); + assert.equal(owner[0].includes('invariants'), false); + } +}); + +test('recipe-bound null qualification rejects an execution with no mapped checks', () => { + const track = loadTrack('ecommerce'); + const binding = structuredClone(resolveRecipeRelease(track, 1, 'ecommerce.sequential-l1')); + const firstExecution = binding.execution[0]; + assert(firstExecution); + binding.execution.push({ id: 'empty-execution', source: firstExecution.source, + ownership: { kind: 'current', level: 1 } }); + assert.throws(() => nullControlSuites(track, 1, binding), /maps no checks/); +}); diff --git a/tools/stack-bench/tests/open-list-live.mutation.ts b/tools/stack-bench/tests/open-list-live.mutation.ts new file mode 100644 index 00000000000..7b478fc6be6 --- /dev/null +++ b/tools/stack-bench/tests/open-list-live.mutation.ts @@ -0,0 +1,47 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; + +const SCENARIO = join(STACK_BENCH_ROOT, 'tracks/ecommerce/scenarios/01-open-list-live.json'); + +test('the focused 902a candidate deterministically checks an already-open live list', () => { + const scenario = compileScenarioDefinition(readJson(SCENARIO), + { source: SCENARIO, expectedLevel: 1 }); + const feature = scenario.features.find(candidate => candidate.id === 902); + assert(feature, 'scenario must include feature 902'); + const criterion = feature.criteria.find(candidate => String(candidate.id) === '902a'); + assert(criterion, 'feature 902 must include criterion 902a'); + assert.equal(criterion.points, 1); + assert.deepEqual(criterion.steps.map(step => step.do), + ['openItem', 'expect', 'click', 'fill', 'click', 'expectElementCount', 'expectElementCount']); + assert.deepEqual(criterion.steps.slice(0, 2).map(step => step.actor), ['reader', 'reader'], + 'the reader view must be visibly open before the write begins'); + assert.deepEqual(criterion.steps[2], { + do: 'click', actor: 'reviewer', testid: 'review-toggle', + unlessVisible: 'review-rating', ifAvailable: true, + }, 'only the writer may open its review form; the reader must remain open'); + assert.equal(criterion.steps.some(action => action.do === 'race' || action.do === 'wait'), false); + assert.deepEqual(criterion.steps.at(-2), { + do: 'expectElementCount', actor: 'reviewer', testid: 'review-item', + contains: 'live-review-kbd', equals: 1, within: 10000, + }, 'the writer must prove the review committed before grading the reader'); + assert.deepEqual(criterion.steps.at(-1), { + do: 'expectElementCount', actor: 'reader', testid: 'review-item', + contains: 'live-review-kbd', equals: 1, within: 10000, + }); + assert.match(stringValue(criterion.note, 'criterion 902a note'), + /without assuming HTTP, WebSockets, subscriptions, or any project layout/); +}); + +function readJson(path: string): unknown { + return JSON.parse(readFileSync(path, 'utf8')); +} + +function stringValue(value: unknown, label: string): string { + if (typeof value !== 'string') throw new Error(`${label} must be a string`); + return value; +} diff --git a/tools/stack-bench/tests/operational-paths.test.ts b/tools/stack-bench/tests/operational-paths.test.ts new file mode 100644 index 00000000000..50a816e3ca5 --- /dev/null +++ b/tools/stack-bench/tests/operational-paths.test.ts @@ -0,0 +1,46 @@ +import assert from 'node:assert/strict'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, + writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import test from 'node:test'; + +import { archiveTranscripts } from '../src/agents/transcript-archive.js'; +import { stackBenchResultsRoot } from '../src/runtime/operational-paths.js'; + +test('configured operational output must be an exact absolute path', () => { + const packageRoot = resolve(tmpdir(), 'stack-bench-module'); + const absolute = resolve(tmpdir(), 'stack-bench-results'); + assert.equal(stackBenchResultsRoot(packageRoot, {}), resolve(packageRoot, 'results')); + assert.equal(stackBenchResultsRoot(packageRoot, { STACK_BENCH_RESULTS_DIR: absolute }), absolute); + assert.throws( + () => stackBenchResultsRoot('/opt/stack-bench', { STACK_BENCH_RESULTS_DIR: 'results' }), + /absolute path/, + ); + assert.throws( + () => stackBenchResultsRoot('/opt/stack-bench', { STACK_BENCH_RESULTS_DIR: ` ${absolute}` }), + /surrounding whitespace/, + ); +}); + +test('transcript archiving includes nested agent sessions without replacing a longer archive', t => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-archive-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const app = join(root, 'app'); + const storeRoot = join(root, 'store'); + const storeName = resolve(app).replace(/[\\/:]/g, '-').toLowerCase().replace(/^-+/, ''); + const nested = join(storeRoot, storeName, 'session', 'subagents'); + const output = join(root, 'output'); + mkdirSync(nested, { recursive: true }); + writeFileSync(join(nested, 'agent-1.jsonl'), 'first\n'); + + const first = archiveTranscripts(app, 'attempt-1', output, storeRoot); + const archived = join(output, 'attempt-1', 'session__subagents__agent-1.jsonl'); + assert.deepEqual(first, { copied: 1, missing: 0, outputDirectory: output }); + assert.equal(readFileSync(archived, 'utf8'), 'first\n'); + + writeFileSync(archived, 'longer preserved archive\n'); + const second = archiveTranscripts(app, 'attempt-1', output, storeRoot); + assert.equal(second.copied, 0); + assert.equal(readFileSync(archived, 'utf8'), 'longer preserved archive\n'); +}); diff --git a/tools/stack-bench/tests/order-data.test.ts b/tools/stack-bench/tests/order-data.test.ts new file mode 100644 index 00000000000..f34727f2273 --- /dev/null +++ b/tools/stack-bench/tests/order-data.test.ts @@ -0,0 +1,647 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { createHash, randomUUID } from 'node:crypto'; +import { createServer } from 'node:http'; +import { once } from 'node:events'; +import { chromium } from 'playwright'; +import { setTimeout as delay } from 'node:timers/promises'; +import test from 'node:test'; +import { ORDER_DATA_COLUMNS, readOrderDataSnapshot } from '../src/stacks/order-data.js'; +import { orderCheckoutDifferences, orderPurchaseDifferences, orderCancellationDifferences, checkoutCrashDifferences } from '../src/stacks/checkout-state.js'; +import { getPostgresCheckoutState } from '../src/stacks/backends/postgres-operations.js'; +import { getSupabaseCheckoutState } from '../src/stacks/backends/supabase-operations.js'; +import { supabaseAdapter } from '../src/stacks/backends/supabase-adapter.js'; +import { supabaseExec, supabaseLease } from './helpers/supabase-lease.js'; +import { getMongoDbCheckoutState } from '../src/stacks/backends/mongodb-operations.js'; +import { getSpacetimeCheckoutState } from '../src/stacks/backends/spacetime-operations.js'; +import { createDatabaseReadCapability } from '../src/actions/runtime-action-executors.js'; +import { ACTION_REGISTRY } from '../src/actions/action-catalog.js'; +import { executeAction } from '../src/actions/action-contract.js'; +import { readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { runInNewContext } from 'node:vm'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; +import { loadTrack } from '../src/composition/tracks.js'; +import { requireRecipeRelease } from '../src/composition/recipe-release.js'; +import { createBoundRecipeTaskRequest, resolveBoundRecipeTaskRequest } from '../src/composition/recipe-selection.js'; +import { Actor, parseGradeArgs } from '../grader/grade.js'; +import { createNamedActionsCapability } from '../src/actions/named-action-runtime.js'; + +const fullStorage = { kind: 'order-data' as const, cart: true, warehouses: true }; + +test('crash stock scope follows the validated delivered task, including retained upgrade contracts', async () => { + const binding = requireRecipeRelease(loadTrack('ecommerce'), 3, 'ecommerce.progression-catalog'); + const path = join(STACK_BENCH_ROOT, 'tracks/ecommerce/scenarios/progression-checkout-crash.json'); + const scenario = compileScenarioDefinition(JSON.parse(readFileSync(path, 'utf8')), { source: path }); + const step = scenario.features[0]!.setup.find(step => step.do === 'dbRecordCheckout')!; + assert.deepEqual(step.storage, { kind: 'order-data', cart: true, warehouses: 'if-requested' }); + const warehouseContract = 'ecommerce.feature.warehouse-admin.hooks'; + const checkKeys = ['ecommerce.spec.state-durability.checkout-crash-integrity.910a']; + for (const mode of ['checkout', 'warehouses', 'retained', 'missing-task'] as const) { + const request = createBoundRecipeTaskRequest(binding, { + featureIds: ['ecommerce.feature.checkout', ...(mode === 'warehouses' ? ['ecommerce.feature.warehouse-admin'] : [])], + taskMode: mode === 'retained' ? 'upgrade' : 'fresh', + priorContractIds: mode === 'retained' ? [warehouseContract] : [], + expectedSpecifications: ['ecommerce.spec.state-durability'], checkKeys, + }).request; + const parsed = parseGradeArgs(['node', 'grade', '--url', 'http://localhost:1', '--spec', path, + '--recipe-task-json', JSON.stringify(request)]); + const task = resolveBoundRecipeTaskRequest(binding, parsed.recipeTask!).task; + const warehouses = mode === 'warehouses' || mode === 'retained'; + assert.equal(task.contractIds.includes(warehouseContract), warehouses); + let reads = 0; + const capability = createDatabaseReadCapability({ backend: 'spacetime', app: '/app', + expand: value => value.startsWith('{user:') ? 'buyer' : value, + contractIds: mode === 'missing-task' ? undefined : task.contractIds, + spacetime: { buildContainer: { id: 'owned', name: 'owned' }, mod: 'app', + uri: 'http://localhost:3000', containerUri: 'http://localhost:3000' }, + exec: (_command, args) => { + reads++; + if (args[0] === 'inspect') return 'owned'; + const tables = args.filter(arg => arg.startsWith('SELECT * FROM ')).map(arg => arg.slice('SELECT * FROM '.length)); + assert.equal(tables.length, warehouses ? 9 : 5); + assert.equal(tables.includes('stock'), warehouses); + const raw = data(); + return JSON.stringify(Object.fromEntries(tables.map(table => [table, { inserts: raw[table as keyof typeof raw], deletes: [] }]))); + } }); + const result = await executeAction(ACTION_REGISTRY, step.do, step, { capabilities: { 'database-read': capability } }); + if (mode === 'missing-task') { + assert.equal(result.status, 'harness_failure'); + assert.equal(reads, 0); + assert.equal(capability.checkoutSnapshots.size, 0); + } else { + assert.equal(result.status, 'passed', mode); + const snapshot = [...capability.checkoutSnapshots.values()][0]!; + assert.deepEqual(snapshot.storage, { kind: 'order-data', cart: true, warehouses }); + assert.deepEqual((await capability.getCheckoutState(snapshot)).state, snapshot.state); + } + } +}); + +function data(): Record[]> { + return { + order_account: [{ id: '1', username: 'buyer' }], item: [{ id: '2', name: 'Keyboard', price: '19.99' }], + warehouse: [{ id: '3' }], + stock: [{ item_id: '2', warehouse_id: '3', quantity: 10 }], + order_cart: [], order_reservation: [], order_header: [], order_line: [], order_allocation: [], + }; +} + +test('compiled UI checkout waits for complete stored effects and preserves defects and reader failures', async t => { + const source = join(STACK_BENCH_ROOT, 'tracks/ecommerce/scenarios/progression-cart-checkout.json'); + const scenario = compileScenarioDefinition(JSON.parse(readFileSync(source, 'utf8')), { source }); + const step = scenario.features[0]!.criteria.find(check => check.id === '4d')!.steps + .find(step => step.do === 'dbExpectCheckout' && step.before === 'normal-before')!; + let now = 0; + t.mock.method(Date, 'now', () => now); + for (const mode of ['delayed', 'reject-all', 'missing-line', 'wrong-total', 'duplicate', 'retained-cart', + 'reader-error', 'late-read', 'late-retry', 'late-wake', 'slow-missing-line', 'slow-wrong-total', 'late-reader-error']) await t.test(mode, async () => { + now = 0; + const raw = data(); + raw.item = [{ id: '2', name: 'Desk Lamp', price: 19.99 }, { id: '4', name: 'Coffee Grinder', price: 3.25 }]; + const storage = { kind: 'order-data' as const, cart: true, warehouses: false }; + const read = () => ({ ...readOrderDataSnapshot(raw, 'buyer', 'Desk Lamp', storage), account: 'buyer', item: 'Desk Lamp' }); + const before = read(); + raw.order_cart = [{ account_id: '1', item_id: '2', quantity: 1 }, { account_id: '1', item_id: '4', quantity: 1 }]; + const prepared = read(); + let reads = 0; + const result = await executeAction(ACTION_REGISTRY, step.do, step, { capabilities: { + actors: { get: () => undefined }, + clock: { sleep: async (ms: number) => { now += mode === 'late-wake' ? 10001 : ms; } }, + 'database-read': { checkoutSnapshots: new Map([['normal-before', before], ['normal-prepared', prepared]]), + getCheckoutState: () => { + reads++; + assert(now < 10000, 'no checkout read starts after the retry deadline'); + if (mode.startsWith('slow-') || mode === 'late-reader-error') now += 450; + if (reads > 1 && mode === 'reader-error' || mode === 'late-reader-error' && now > 10000) throw new Error('database unavailable'); + if (mode === 'late-read' || (mode === 'late-retry' && reads > 1)) now += 10001; + if (now >= 2000 && !['reject-all', 'late-reader-error'].includes(mode)) { + raw.order_header = [{ id: 'new', account_id: '1', total: ['wrong-total', 'slow-wrong-total'].includes(mode) ? 1 : 23.24, refunded: 0, status: 'pending' }]; + raw.order_line = [{ id: 'one', order_id: 'new', item_id: '2', quantity: 1, unit_price: 19.99 }, + { id: 'two', order_id: 'new', item_id: '4', quantity: 1, unit_price: 3.25 }]; + if (['missing-line', 'slow-missing-line'].includes(mode)) raw.order_line.pop(); + if (mode === 'duplicate') raw.order_header.push({ ...raw.order_header[0]!, id: 'extra' }); + if (mode !== 'retained-cart') raw.order_cart = []; + } + return read(); + } }, + } }); + assert.equal(result.status, ['delayed', 'late-read', 'late-retry'].includes(mode) ? 'passed' + : ['reader-error', 'late-reader-error'].includes(mode) ? 'harness_failure' : 'failed', `${mode}: ${JSON.stringify(result)}`); + if (mode.startsWith('slow-')) { + assert(now > 10000, 'the final native read completes after the retry deadline'); + assert.equal(result.finding?.kind, 'number-mismatch'); + } + if (mode === 'late-wake') assert.equal(reads, 1, 'do not start another read after the deadline'); + }); +}); + +// Cart preparation must complete before the immutable native snapshot is taken. +// An optimistic UI or a returned Playwright click is not a committed cart write. +test('checkout preparation waits for delayed cart commits without excusing broken writes', async t => { + const source = join(STACK_BENCH_ROOT, 'tracks/ecommerce/scenarios/progression-checkout-crash.json'); + const scenario = compileScenarioDefinition(JSON.parse(readFileSync(source, 'utf8')), { source }); + const setup = scenario.features[0]!.setup; + const steps = setup.slice(setup.findIndex(step => step.do === 'dbRecordCheckout'), + setup.findIndex(step => step.do === 'dbExpectCheckout') + 1); + const evidence: unknown[] = []; + const browser = await chromium.launch({ headless: true }); + try { + for (const mode of ['immediate', 'delayed', 'missing-line', 'no-op-add', 'lost-response']) await t.test(mode, async () => { + const raw = data(); + raw.order_account = [{ id: 'a', username: 'database' }]; + raw.item = [{ id: 'k', name: 'Keyboard', price: 19.99 }, { id: 'c', name: 'Coffee Grinder', price: 3.25 }]; + const storage = { kind: 'order-data' as const, cart: true, warehouses: false }; + const events: unknown[] = [], pendingAdds: Promise[] = []; + const started = performance.now(); + const record = (event: string, details: object = {}) => events.push({ event, elapsedMs: Math.round(performance.now() - started), ...details }); + const server = createServer(async (req, res) => { + if (req.url === '/state') { + record('independent-read', { cart: structuredClone(raw.order_cart), orders: raw.order_header.length }); + res.writeHead(200, { 'Content-Type': 'application/json' }).end(JSON.stringify(raw)); return; + } + if (req.url === '/api/add') { + let body = ''; for await (const part of req) body += String(part); + const { item } = JSON.parse(body) as { item: string }; record('add-dispatched', { item }); + const pending = (async () => { + if (item === 'c' && mode !== 'immediate') await delay(900); + if (!(mode === 'no-op-add' && item === 'c')) raw.order_cart.push({ account_id: 'a', item_id: item, quantity: 1 }); + record('add-finished', { item, committed: !(mode === 'no-op-add' && item === 'c') }); + if (mode === 'lost-response' && item === 'c') res.destroy(); + else res.writeHead(200, { 'Content-Type': 'application/json' }).end('{}'); + })(); + pendingAdds.push(pending); await pending; return; + } + if (req.url === '/api/checkout') { + for await (const _part of req) { /* Receive the checkout request. */ } + await Promise.all(pendingAdds); + const lines = raw.order_cart.map((line, i) => ({ id: 'line-' + i, order_id: 'order', item_id: line.item_id, quantity: Number(line.quantity), + unit_price: Number(raw.item.find(item => item.id === line.item_id)!.price) })); + raw.order_header.push({ id: 'order', account_id: 'a', status: 'pending', + total: lines.reduce((sum, line) => sum + Math.round(line.unit_price * 100) * line.quantity, 0) / 100, refunded: 0 }); + raw.order_line = mode === 'missing-line' ? lines.slice(0, 1) : lines; + raw.order_cart = []; record('checkout-committed', { lines: structuredClone(raw.order_line) }); + res.writeHead(200, { 'Content-Type': 'application/json' }).end('{}'); return; + } + res.writeHead(200, { 'Content-Type': 'text/html' }).end(` + +
Keyboard
+
Coffee Grinder
+ `); + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const context = await browser.newContext(); + const page = await context.newPage(); + const actor = new Actor('a', page, context, true); + await actor.ready; await page.goto(url); + const observation = { defaultWithin: 1000, recorded: new Map(), expand: (value: string) => value, + sleep: (ms: number) => delay(ms), testId: (id: string) => id }; + const database = { checkoutSnapshots: new Map(), markCheckoutUnsettled: () => assert.fail('fixture checkout did not complete'), + async getCheckoutState(input: { item: string }) { + const rows = await (await fetch(url + '/state')).json() as ReturnType; + return readOrderDataSnapshot(rows, 'database', input.item, storage); + } }; + const capabilities = { actors: { get: () => actor }, 'browser-interaction': observation, 'browser-observation': observation, + clock: { sleep: (ms: number) => delay(ms) }, 'database-read': database, + 'named-actions': createNamedActionsCapability({ backend: 'postgres', url, + lastCalls: { get: () => null, set() {} }, sleep: async ms => { await delay(ms); } }) }; + const actions: { input: unknown; evidence: Awaited> }[] = []; + try { + for (const step of steps) { + const result = await executeAction(ACTION_REGISTRY, step.do, step, { capabilities }); + actions.push({ input: step, evidence: result }); + if (result.status !== 'passed') break; + } + const final = readOrderDataSnapshot(raw, 'database', 'Keyboard', storage).state; + const actual = actions.at(-1)!.evidence; + const expected = ['immediate', 'delayed'].includes(mode) ? 'passed' : mode === 'lost-response' ? 'inconclusive' : 'failed'; + evidence.push({ mode, expected, actual: actual.status, events, actions, finalState: final }); + assert.equal(actual.status, expected, `${mode}: ${actual.summary}`); + if (expected === 'passed') { + assert.equal(final.orders[0]!.lines.length, 2); + assert.equal(final.orders[0]!.totalMinor, 2324); + assert.equal(final.cart.length, 0); + } + if (mode === 'lost-response') assert.equal(actual.action.id, 'click', 'do not use a cart snapshot after unmeasured delivery'); + } finally { + await Promise.all(pendingAdds); await actor.prepareNavigation(1000, new AbortController().signal); + await context.close(); server.closeAllConnections(); await new Promise(resolve => server.close(() => resolve())); + } + }); + } finally { + await browser.close(); + if (process.env.STACK_BENCH_PREPARED_CART_EVIDENCE) writeFileSync(process.env.STACK_BENCH_PREPARED_CART_EVIDENCE, + JSON.stringify({ rerun: 'STACK_BENCH_PREPARED_CART_EVIDENCE= node --test --test-name-pattern="checkout preparation" dist/tests/order-data.test.js', + source, scenarioSha256: createHash('sha256').update(readFileSync(source)).digest('hex'), node: process.version, + scope: 'real browser, action dispatcher and independent order-data parser/oracle; local HTTP storage, no external database or crash', + cleanup: 'owned contexts, browser and loopback servers closed', cases: evidence }, null, 2)); + } +}); + +test('multi-item checkout reconciles every cart line, stored price and selected warehouse effect, including refused writes', async () => { + for (const lazy of [false, true]) for (const warehouses of [false, true]) for (const reserved of warehouses ? [false, true] : [false]) for (const mode of ['accepted', 'refused', 'missing-line', 'wrong-price', 'wrong-total', 'swapped-quantity', + 'extra-line', 'retained-cart', 'duplicate-order', 'refused-cart-change', 'committed-refusal', 'reject-all', + ...(warehouses ? ['missing-stock-effect', 'swapped-stock-effects', 'unrelated-stock-change', 'missing-stock-row', 'duplicate-stock-row', 'wrong-allocation'] : [])]) { + const raw = data(); + raw.item.push({ id: '4', name: 'Lamp', price: '3.25' }, { id: '5', name: 'Untouched', price: '2.00' }); + raw.stock.push({ item_id: '4', warehouse_id: '3', quantity: 10 }, { item_id: '5', warehouse_id: '3', quantity: 10 }); + const storage = { kind: 'order-data' as const, cart: true, warehouses }; + const read = () => ({ ...readOrderDataSnapshot(raw, 'buyer', 'Keyboard', storage), account: 'buyer', item: 'Keyboard' }); + const accounts = raw.order_account; + if (lazy) raw.order_account = []; + const before = read(); + raw.order_account = accounts; + raw.order_cart = [{ account_id: '1', item_id: '2', quantity: 1 }, { account_id: '1', item_id: '4', quantity: 2 }]; + if (reserved) { + raw.stock[0]!.quantity = 9; raw.stock[1]!.quantity = 8; + raw.order_reservation = [{ account_id: '1', item_id: '2', warehouse_id: '3', quantity: 1 }, + { account_id: '1', item_id: '4', warehouse_id: '3', quantity: 2 }]; + } + const prepared = read(); + const refused = ['refused', 'refused-cart-change', 'committed-refusal'].includes(mode); + if (!refused && mode !== 'reject-all' || mode === 'committed-refusal') { + raw.order_cart = []; + raw.order_reservation = []; + raw.order_header = [{ id: 'new', account_id: '1', total: 26.49, refunded: 0, status: 'pending' }]; + raw.order_line = [{ id: 'a', order_id: 'new', item_id: '2', quantity: 1, unit_price: 19.99 }, + { id: 'b', order_id: 'new', item_id: '4', quantity: 2, unit_price: 3.25 }]; + raw.stock[0]!.quantity = 9; raw.stock[1]!.quantity = 8; + raw.order_allocation = [{ order_line_id: 'a', warehouse_id: '3', quantity: 1 }, + { order_line_id: 'b', warehouse_id: '3', quantity: 2 }]; + } + if (mode === 'missing-line') raw.order_line.pop(); + if (mode === 'wrong-price') raw.order_line[1]!.unit_price = 0.01; + if (mode === 'wrong-total') raw.order_header[0]!.total = 0.01; + if (mode === 'swapped-quantity') { raw.order_line[0]!.quantity = 2; raw.order_line[1]!.quantity = 1; } + if (mode === 'extra-line') raw.order_line.push({ ...raw.order_line[0]!, id: 'extra' }); + if (mode === 'retained-cart') raw.order_cart = [{ account_id: '1', item_id: '4', quantity: 2 }]; + if (mode === 'duplicate-order') raw.order_header.push({ ...raw.order_header[0]!, id: 'extra' }); + if (mode === 'refused-cart-change') raw.order_cart.pop(); + if (mode === 'missing-stock-effect') raw.stock[1]!.quantity = 10; + if (mode === 'swapped-stock-effects') { raw.stock[0]!.quantity = 8; raw.stock[1]!.quantity = 9; } + if (mode === 'unrelated-stock-change') raw.stock[2]!.quantity = 9; + if (mode === 'missing-stock-row') raw.stock.pop(); + if (mode === 'duplicate-stock-row') raw.stock.push({ ...raw.stock[1]! }); + if (mode === 'wrong-allocation') raw.order_allocation[1]!.quantity = 1; + const result = await executeAction(ACTION_REGISTRY, 'dbExpectCheckout', { + do: 'dbExpectCheckout', before: 'before', prepared: 'prepared', + quantity: [{ item: 'Keyboard', quantity: 1 }, { item: 'Lamp', quantity: 2 }], + ...(mode !== 'reject-all' ? { actor: 'buyer' } : {}), + }, { capabilities: { + actors: { get: () => ({ actionCall: { action: 'checkout', accepted: !refused, status: refused ? 400 : 200 } }) }, + clock: { sleep: async () => {} }, + 'database-read': { ...createDatabaseReadCapability({ expand: value => value, + checkoutSnapshots: new Map([['before', before], ['prepared', prepared]]) }), getCheckoutState: read }, + } }); + assert.equal(result.status, ['accepted', 'refused'].includes(mode) ? 'passed' : 'failed', `${warehouses}:${reserved}:${mode}`); + if (warehouses && ['accepted', 'refused', 'missing-stock-effect', 'swapped-stock-effects', 'missing-line'].includes(mode)) { + const after = read().state; + const quantities = [{ itemId: '2', priceMinor: 1999, quantity: 1 }, { itemId: '4', priceMinor: 325, quantity: 2 }]; + for (const confirmed of [false, true]) { + const verdict = checkoutCrashDifferences(before.state, after, confirmed, + (state, allow) => orderCheckoutDifferences(before.state, prepared.state, state, quantities, allow, true)); + assert.equal(verdict.atomicity.length === 0, ['accepted', 'refused'].includes(mode), `crash atomicity ${reserved}:${mode}`); + assert.equal(verdict.durability.length > 0, confirmed && mode === 'refused', `crash durability ${reserved}:${mode}`); + } + } + } +}); + +test('compiled duplicate checkout reconciles real order effects without requesting warehouse records', async () => { + const binding = requireRecipeRelease(loadTrack('ecommerce'), 3, 'ecommerce.progression-catalog'); + const request = createBoundRecipeTaskRequest(binding, { + featureIds: ['ecommerce.feature.checkout'], taskMode: 'fresh', + expectedSpecifications: ['ecommerce.spec.concurrency-safety'], + checkKeys: ['ecommerce.spec.concurrency-safety.duplicate-checkout.203b'], + }); + assert(!request.selection.features?.includes('ecommerce.feature.warehouse-admin')); + assert.match(request.task.contractText, /item\(id, name, price\)/); + assert.match(request.task.contractText, /order_cart\(account_id, item_id, quantity\)/); + assert(!request.task.requirementText.includes('Warehouse administration')); + const path = join(STACK_BENCH_ROOT, 'tracks/ecommerce/scenarios/01-duplicate-checkout.json'); + const scenario = compileScenarioDefinition(JSON.parse(readFileSync(path, 'utf8')), { source: path }); + const criterion = scenario.features[0]!.criteria.find(row => row.id === '203b')!; + const steps = criterion.steps.filter(step => ['dbRecordCheckout', 'dbExpectCheckout'].includes(step.do) + && [step.as, step.before].some(key => key === 'checkout-before' || key === 'checkout-prepared')); + for (const step of criterion.steps.filter(step => step.do === 'dbRecordCheckout')) { + assert.equal((step.storage as { warehouses: boolean }).warehouses, false); + } + const loss = criterion.steps.findIndex(step => step.do === 'loseCheckoutResponse'); + const retry = criterion.steps.findIndex(step => step.do === 'callAction'); + assert(loss > 0 && retry > loss); + assert(criterion.steps.slice(loss + 1, retry).some(step => step.do === 'reload')); + assert(criterion.steps.slice(retry + 1).some(step => step.do === 'expectNumber' + && step.relativeTo === 'reply-stock-before' && step.plus === -1)); + assert.equal(steps.length, 3); + const raceIndex = criterion.steps.findIndex(step => step.do === 'callConcurrently'); + assert(criterion.steps.indexOf(steps[1]!) < raceIndex && criterion.steps.indexOf(steps[2]!) > raceIndex); + for (const defect of ['none', 'placed', 'completed', 'no-op', 'duplicate', 'wrong-owner', 'wrong-price', 'retained-cart', 'lost-prior']) { + const initial = data(); + for (const key of ['warehouse', 'stock', 'order_reservation', 'order_allocation'] as const) delete (initial as Partial)[key]; + initial.order_header = [{ id: 'old', account_id: '1', total: 0, refunded: 0, status: 'cancelled' }]; + const prepared = structuredClone(initial); prepared.order_cart = [{ account_id: '1', item_id: '2', quantity: 1 }]; + const after = structuredClone(initial); + after.order_header.push({ id: 'new', account_id: '1', total: 19.99, refunded: 0, status: 'pending' }); + if (['placed', 'completed'].includes(defect)) after.order_header[1]!.status = defect; + after.order_line.push({ id: 'line', order_id: 'new', item_id: '2', quantity: 1, unit_price: 19.99 }); + if (defect === 'no-op') Object.assign(after, prepared); + if (defect === 'duplicate') after.order_header.push({ ...after.order_header[1]!, id: 'extra' }); + if (defect === 'wrong-owner') { + after.order_account.push({ id: 'other', username: 'other' }); after.order_header[1]!.account_id = 'other'; + } + if (defect === 'wrong-price') after.order_line[0]!.unit_price = 0; + if (defect === 'retained-cart') after.order_cart = prepared.order_cart; + if (defect === 'lost-prior') after.order_header.shift(); + const queue = [initial, prepared, after]; + const capability = createDatabaseReadCapability({ backend: 'spacetime', app: '/minimal-checkout', expand: value => value === '{user:twin}' ? 'buyer' : value, + spacetime: { buildContainer: { id: 'owned', name: 'owned' }, mod: 'app', uri: 'http://localhost:3000', containerUri: 'http://localhost:3000' }, + exec: (_command, args) => { + if (args[0] === 'inspect') return 'owned'; + const tables = args.filter(arg => arg.startsWith('SELECT * FROM ')).map(arg => arg.slice('SELECT * FROM '.length)); + assert.deepEqual(tables.sort(), ['item', 'order_account', 'order_cart', 'order_header', 'order_line']); + const raw = queue.shift()!; + return JSON.stringify(Object.fromEntries(tables.map(table => [table, { inserts: raw[table as keyof typeof raw], deletes: [] }]))); + } }); + for (const [index, step] of steps.entries()) { + const result = await executeAction(ACTION_REGISTRY, step.do, step, { capabilities: { 'database-read': capability, + actors: { get: () => undefined }, clock: { sleep: async () => {} } } }); + assert.equal(result.status, index < 2 || ['none', 'placed', 'completed'].includes(defect) ? 'passed' : 'failed', defect); + } + } +}); + +test('order read scope requires selected data and never infers features from missing tables', () => { + for (const cart of [false, true]) for (const warehouses of [false, true]) { + const storage = { kind: 'order-data' as const, cart, warehouses }, raw = data(); + if (!cart) delete (raw as Partial).order_cart; + if (!cart || !warehouses) delete (raw as Partial).order_reservation; + if (!warehouses) for (const key of ['warehouse', 'stock', 'order_allocation'] as const) delete (raw as Partial)[key]; + const snapshot = readOrderDataSnapshot(raw, 'buyer', 'Keyboard', storage); + assert.deepEqual(snapshot.storage, storage); + assert.equal(snapshot.state.stock.length, warehouses ? 1 : 0); + if (!cart || !warehouses) assert.throws(() => readOrderDataSnapshot(raw, 'buyer', 'Keyboard', fullStorage)); + delete (raw as Partial).order_line; + assert.throws(() => readOrderDataSnapshot(raw, 'buyer', 'Keyboard', storage), /missing fields/); + } +}); + +test('order data preserves empty state, orphan effects and exact identifiers without a reference source', () => { + const raw = data(); + const read = () => readOrderDataSnapshot(raw, 'buyer', 'Keyboard', fullStorage); + assert.equal(read().state.priceMinor, 1999); + assert.deepEqual(read().state.orders, []); + raw.order_account = []; + assert.deepEqual(read().state.cart, [], 'a customer with no shopping records has an empty cart'); + assert.deepEqual(read().state.orders, []); + raw.order_account = [{ id: '1', username: 'buyer' }, { id: '9', username: 'buyer' }]; + assert.throws(read, /ambiguous/, 'two native identities must not merge into one customer'); + raw.order_account.pop(); + raw.order_header.push({ id: '9007199254740993', account_id: '1', total: '19.99', refunded: 0, status: 'pending' }); + raw.order_line.push({ id: '5', order_id: '9007199254740993', item_id: '2', quantity: 1, unit_price: '19.99' }); + raw.order_allocation.push({ order_line_id: '5', warehouse_id: '3', quantity: 1 }); + assert.equal(read().state.orders[0]!.id, '9007199254740993'); + raw.order_account = []; + assert.throws(read, /link is missing/, 'an orphan order is not an empty account'); + raw.order_account = [{ id: '1', username: 'buyer' }]; + raw.order_line[0]!.order_id = 'absent'; + assert.equal(read().state.orphanOrderLines, 1); + raw.order_line = []; + assert.equal(read().state.orphanAllocations, 1); + raw.stock = []; + assert.deepEqual(read().state.stock, [], 'lost stock must reach reconciliation'); + raw.order_header[0]!.total = '1.001'; + assert.throws(read, /invalid values/); + raw.order_header[0]!.total = 19.99; + raw.order_header.push({ ...raw.order_header[0]! }); + assert.throws(read, /duplicate ids/); +}); + +test('order data uses one SpacetimeDB subscription and preserves the explicit reader in later snapshots', async () => { + const raw = data(); + raw.order_header.push({ id: '9007199254740993', account_id: '1', total: 19.99, refunded: 0, status: 'pending' }); + const exec = (_: string, args: readonly string[]) => { + if (args[0] === 'inspect') return 'owned'; + assert(args.includes('subscribe')); + assert.equal(args.filter(arg => arg.startsWith('SELECT *')).length, 9); + return JSON.stringify(Object.fromEntries(Object.entries(raw).filter(([, rows]) => rows.length) + .map(([table, inserts]) => [table, { inserts, deletes: [] }]))).replace('"9007199254740993"', '9007199254740993'); + }; + const capability = createDatabaseReadCapability({ backend: 'spacetime', app: '/not-a-reference', expand: value => value, exec, + spacetime: { buildContainer: { id: 'owned', name: 'owned' }, mod: 'app', uri: 'http://localhost:3000', containerUri: 'http://localhost:3000' } }); + const result = await executeAction(ACTION_REGISTRY, 'dbRecordCheckout', { + do: 'dbRecordCheckout', storage: { kind: 'order-data', cart: true, warehouses: true }, account: 'buyer', item: 'Keyboard', as: 'before', + }, { capabilities: { 'database-read': capability } }); + assert.equal(result.status, 'passed'); + const snapshot = capability.checkoutSnapshots.get('before')!; + assert.deepEqual(snapshot.storage, fullStorage); + assert.equal(snapshot.state.orders[0]!.id, '9007199254740993'); + assert.deepEqual((await capability.getCheckoutState(snapshot)).state, snapshot.state); + raw.item[0]!.price = 0.001; + const invalid = await executeAction(ACTION_REGISTRY, 'dbRecordCheckout', { + do: 'dbRecordCheckout', storage: { kind: 'order-data', cart: true, warehouses: true }, account: 'buyer', item: 'Keyboard', as: 'bad', + }, { capabilities: { 'database-read': capability } }); + assert.equal(invalid.status, 'failed', 'bad declared data is an app failure'); + assert.throws(() => getSpacetimeCheckoutState({ account: 'buyer', item: 'Keyboard', app: '/not-a-reference' }), + /ENOENT/, 'old diagnostic readers still require verified reference source'); +}); + +test('order data preserves infrastructure failures instead of blaming the app', async () => { + const capability = createDatabaseReadCapability({ backend: 'postgres', app: '/any-app', expand: value => value, + databaseLease: { resources: { container: { id: 'owned', name: 'owned' }, database: 'app' } }, + exec: (_command, args) => { if (args[0] === 'inspect') return 'owned'; throw new Error('connection refused'); } }); + const result = await executeAction(ACTION_REGISTRY, 'dbRecordCheckout', { + do: 'dbRecordCheckout', storage: { kind: 'order-data', cart: true, warehouses: true }, account: 'buyer', item: 'Keyboard', as: 'before', + }, { capabilities: { 'database-read': capability } }); + assert.notEqual(result.status, 'passed'); + assert.notEqual(result.status, 'failed'); +}); + +test('Supabase order data reads the PostgreSQL order interface in public and classifies it identically', () => { + const platform = supabaseLease(); + const lease = supabaseAdapter.grading.databaseLease(platform); + const postgres = { resources: { container: { id: 'owned', name: 'owned' }, database: 'app' } }; + for (const storage of [fullStorage, { kind: 'order-data' as const, cart: false, warehouses: false }]) { + const raw = data(); + raw.order_header.push({ id: '9007199254740993', account_id: '1', total: '19.99', refunded: '0', status: 'pending' }); + raw.order_line.push({ id: '5', order_id: '9007199254740993', item_id: '2', quantity: 1, unit_price: '19.99' }); + const sql: string[] = []; + const supabase = getSupabaseCheckoutState({ account: 'buyer', item: 'Keyboard', app: '/not-a-reference', storage, lease, + exec: supabaseExec(platform, input => { sql.push(input); return JSON.stringify(raw); }) }); + const expected = getPostgresCheckoutState({ account: 'buyer', item: 'Keyboard', app: '/not-a-reference', storage, + lease: postgres, exec: (_command, args, options) => { + if (args[0] === 'inspect') return 'owned'; + sql.push(options.input ?? ''); + return JSON.stringify(raw); + } }); + assert.equal(sql[0], sql[1]); + assert.deepEqual(supabase, expected); + assert.equal(supabase.state.orders[0]!.id, '9007199254740993'); + } + const missing = Object.assign(new Error('psql failed'), { stderr: 'ERROR: relation "public.order_line" does not exist' }); + assert.throws(() => getSupabaseCheckoutState({ account: 'buyer', item: 'Keyboard', storage: fullStorage, lease, + exec: supabaseExec(platform, () => { throw missing; }) }), (error: unknown) => error instanceof Error + && 'orderDataInterface' in error); + const refused = new Error('connection refused'); + assert.throws(() => getSupabaseCheckoutState({ account: 'buyer', item: 'Keyboard', storage: fullStorage, lease, + exec: supabaseExec(platform, () => { throw refused; }) }), (error: unknown) => error === refused); + assert.throws(() => getSupabaseCheckoutState({ account: 'buyer', item: 'Keyboard', lease, + exec: supabaseExec(platform, () => '{}') }), /declared order data interface/); +}); + +test('MongoDB order data reads a not-yet-created collection as empty and keeps real read failures', () => { + const lease = { resources: { container: { id: 'owned', name: 'owned' }, database: 'bench' } }; + const mongo = (collections: Record[]>, failure?: string) => + (_command: string, args: readonly string[]) => { + if (args[0] === 'inspect') return 'owned'; + let output = ''; + const collection = (name: string) => ({ find: () => ({ toArray: () => { + if (name === failure) throw new Error(`MongoServerError: read of ${name} failed`); + return collections[name]!; + } }) }); + const session = { getDatabase: () => ({ getCollection: collection }), startTransaction() {}, commitTransaction() {}, + endSession() {} }; + runInNewContext(args.at(-1)!, { db: { getName: () => 'bench', getCollectionNames: () => Object.keys(collections), + getMongo: () => ({ startSession: () => session }) }, print: (value: string) => { output += value; } }); + return output; + }; + const read = (collections: Record[]>, failure?: string) => getMongoDbCheckoutState({ + account: 'buyer', item: 'Keyboard', app: '/not-a-reference', lease, storage: fullStorage, exec: mongo(collections, failure) }); + const catalog = { item: data().item, warehouse: data().warehouse, stock: data().stock }; + const empty = read(catalog).state; + assert.deepEqual([empty.cart, empty.reservations, empty.orders], [[], [], []]); + assert.equal(empty.stock[0]!.quantity, 10); + assert.deepEqual(read({ item: data().item }).state.stock, [], 'no stock collection yet is no stock rows'); + assert.throws(() => read(catalog, 'stock'), (error: unknown) => error instanceof Error + && /read of stock failed/.test(error.message) && !('orderDataInterface' in error)); +}); + +test('order-only purchase and cancellation reject no-op, wrong allocation and wrong refund effects', () => { + const storage = { kind: 'order-data' as const, cart: false, warehouses: true }; + const raw = data(); + raw.item.push({ id: 'other', name: 'Untouched', price: 1 }); + raw.stock.push({ item_id: 'other', warehouse_id: '3', quantity: 7 }); + const before = readOrderDataSnapshot(raw, 'buyer', 'Keyboard', storage).state; + raw.stock[0]!.quantity = 9; + raw.order_header.push({ id: '4', account_id: '1', total: 19.99, refunded: 0, status: 'pending' }); + raw.order_line.push({ id: '5', order_id: '4', item_id: '2', quantity: 1, unit_price: 19.99 }); + raw.order_allocation.push({ order_line_id: '5', warehouse_id: '3', quantity: 1 }); + const after = readOrderDataSnapshot(raw, 'buyer', 'Keyboard', storage).state; + assert.deepEqual(orderPurchaseDifferences(before, after, new Map([['buyer', 1]]), new Map()), []); + raw.warehouse = []; + assert.throws(() => readOrderDataSnapshot(raw, 'buyer', 'Keyboard', fullStorage), /warehouse link is missing/); + raw.warehouse = [{ id: '3' }]; + assert(orderPurchaseDifferences(before, before, new Map([['buyer', 1]]), new Map()).length); + raw.order_header[0]!.status = 'cancelled'; + raw.order_header[0]!.refunded = 19.99; + raw.stock[0]!.quantity = 10; + const cancelled = readOrderDataSnapshot(raw, 'buyer', 'Keyboard', storage).state; + assert.deepEqual(orderCancellationDifferences(after, cancelled), []); + const damaged = structuredClone(cancelled); damaged.stock[1]!.quantity++; + assert(orderCancellationDifferences(after, damaged).length, 'cancellation must not restore another product'); + const wrongPurchase = structuredClone(after); wrongPurchase.stock[1]!.quantity--; + assert(orderPurchaseDifferences(before, wrongPurchase, new Map([['buyer', 1]]), new Map()).length, 'purchase must preserve other products'); + for (const defect of ['missing-order', 'missing-lines', 'missing-allocation']) { + const broken = structuredClone(after); + if (defect === 'missing-order') broken.orders = []; + if (defect === 'missing-lines') broken.orders[0]!.lines = []; + if (defect === 'missing-allocation') broken.orders[0]!.lines[0]!.allocations = []; + assert(orderCancellationDifferences(broken, cancelled).length, `${defect} is an app failure, not a harness exception`); + } + assert(orderCancellationDifferences(after, after).length); + cancelled.orders[0]!.refundedMinor = 0; + assert(orderCancellationDifferences(after, cancelled).some(row => row.control === 'cancellation orders')); + cancelled.orders[0]!.refundedMinor = 1999; + cancelled.stock[0]!.quantity = 11; + assert(orderCancellationDifferences(after, cancelled).some(row => row.control === 'cancellation stock')); +}); + +const docker = (args: string[], input?: string) => execFileSync('docker', args, + { encoding: 'utf8', input, stdio: 'pipe', timeout: 30000, windowsHide: true }).trim(); +for (const backend of ['postgres', 'mongodb'] as const) { + test(`${backend} native order interface reads live data, detects lost effects and rejects missing interfaces`, { + skip: process.env.STACK_BENCH_CHECKOUT_READ_DOCKER !== '1', timeout: 120000, + }, async () => { + const pg = backend === 'postgres', name = `stack-bench-order-data-${backend}-${randomUUID()}`; + const id = docker(['run', '--pull=never', '--rm', '-d', '--name', name, '--network', 'none', + '--cpus', '1', '--memory', '512m', ...(pg ? ['-e', 'POSTGRES_HOST_AUTH_METHOD=trust', + '-e', 'POSTGRES_USER=appuser', '-e', 'POSTGRES_DB=bench', 'postgres:16-alpine'] + : ['mongo:7', '--replSet', 'rs0', '--bind_ip_all'])]); + const run = (script: string) => pg + ? docker(['exec', '-i', id, 'psql', '-U', 'appuser', '-d', 'bench', '-v', 'ON_ERROR_STOP=1', '-At'], script) + : docker(['exec', id, 'mongosh', 'bench', '--quiet', '--eval', script]); + try { + for (let attempt = 0; ; attempt++) { + try { run(pg ? 'SELECT 1;' : 'db.runCommand({ping:1})'); break; } + catch (error) { if (attempt === 30) throw error; await delay(250); } + } + if (!pg) { + run("rs.initiate({_id:'rs0',members:[{_id:0,host:'127.0.0.1:27017'}]})"); + for (let attempt = 0; ; attempt++) { + if (run('print(db.hello().isWritablePrimary)') === 'true') break; + if (attempt === 30) throw new Error('test replica did not become writable'); + await delay(250); + } + } + run(pg ? ` + CREATE TABLE order_account(id bigint, username text); + CREATE TABLE item(id bigint, name text, price numeric); + CREATE TABLE warehouse(id bigint); + CREATE TABLE stock(item_id bigint, warehouse_id bigint, quantity integer); + CREATE TABLE order_cart(account_id bigint, item_id bigint, quantity integer); + CREATE TABLE order_reservation(account_id bigint, item_id bigint, warehouse_id bigint, quantity integer); + CREATE TABLE order_header(id bigint, account_id bigint, total numeric, refunded numeric, status text); + CREATE TABLE order_line(id bigint, order_id bigint, item_id bigint, quantity integer, unit_price numeric); + CREATE TABLE order_allocation(order_line_id bigint, warehouse_id bigint, quantity integer); + INSERT INTO order_account VALUES(1,'buyer'); INSERT INTO item VALUES(2,'Keyboard',19.99); + INSERT INTO stock VALUES(2,3,10); INSERT INTO warehouse VALUES(3);` + : `for (const [table, rows] of Object.entries(${JSON.stringify(data())})) { + db.createCollection(table); if (rows.length) db.getCollection(table).insertMany(rows); + }`); + const read = (storage = fullStorage) => (pg ? getPostgresCheckoutState : getMongoDbCheckoutState)({ account: 'buyer', item: 'Keyboard', + storage, app: '/not-a-reference', lease: { resources: { container: { id, name }, database: 'bench' } } }); + run(pg ? 'DELETE FROM order_account;' : 'db.order_account.deleteMany({})'); + const before = read().state; + run(pg ? "INSERT INTO order_account VALUES(1,'buyer'); INSERT INTO order_cart VALUES(1,2,1);" + : "db.order_account.insertOne({id:'1',username:'buyer'}); db.order_cart.insertOne({account_id:'1',item_id:'2',quantity:1})"); + const prepared = read().state; + run(pg ? `INSERT INTO order_header VALUES(9007199254740993,1,19.99,0,'pending'); + INSERT INTO order_line VALUES(5,9007199254740993,2,1,19.99); + INSERT INTO order_allocation VALUES(5,3,1); UPDATE stock SET quantity=9; DELETE FROM order_cart;` + : `db.order_header.insertOne({id:Long('9007199254740993'),account_id:'1',total:Decimal128('19.99'),refunded:0,status:'pending'}); + db.order_line.insertOne({id:'5',order_id:Long('9007199254740993'),item_id:'2',quantity:1,unit_price:19.99}); + db.order_allocation.insertOne({order_line_id:'5',warehouse_id:'3',quantity:1}); + db.stock.updateOne({},{$set:{quantity:9}}); db.order_cart.deleteMany({});`); + const after = read().state; + assert.equal(after.orders[0]!.id, '9007199254740993'); + assert.deepEqual(orderCheckoutDifferences(before, prepared, after, 1), []); + run(pg ? 'ALTER TABLE order_header RENAME TO stored_orders; CREATE VIEW order_header AS SELECT * FROM stored_orders;' + : "db.order_header.renameCollection('stored_orders'); db.createView('order_header','stored_orders',[]);"); + assert.deepEqual(read().state, after, 'native views may expose the application records without duplicate storage'); + run(pg ? 'DELETE FROM order_line;' : 'db.order_line.deleteMany({})'); + assert.equal(read().state.orphanAllocations, 1); + assert(orderCheckoutDifferences(before, prepared, read().state, 1).length > 0); + run(pg ? 'DROP TABLE order_line;' : 'db.order_line.drop()'); + // MongoDB creates a collection on first insert, so an absent collection has no rows. + if (pg) assert.throws(read, error => error instanceof Error && 'orderDataInterface' in error); + else assert.equal(read().state.orphanAllocations, 1); + run(pg ? `CREATE TABLE order_line(id bigint, order_id bigint, item_id bigint, quantity integer, unit_price numeric); + DROP TABLE stock, warehouse, order_allocation, order_reservation, order_cart;` + : `db.createCollection('order_line'); for (const table of ['stock','warehouse','order_allocation','order_reservation','order_cart']) db[table].drop();`); + const minimal = { kind: 'order-data' as const, cart: false, warehouses: false }; + assert.deepEqual(read(minimal).state.stock, []); + if (pg) { + assert.throws(() => read({ ...minimal, cart: true }), error => error instanceof Error && 'orderDataInterface' in error); + assert.throws(() => read({ ...minimal, warehouses: true }), error => error instanceof Error && 'orderDataInterface' in error); + } else { + assert.deepEqual(read({ ...minimal, cart: true }).state.cart, []); + assert.deepEqual(read({ ...minimal, warehouses: true }).state.stock, []); + } + run(pg ? 'CREATE TABLE order_cart(account_id bigint, item_id bigint, quantity integer);' : "db.createCollection('order_cart')"); + assert.deepEqual(read({ ...minimal, cart: true }).state.cart, []); + } finally { docker(['rm', '-f', id]); } + }); +} diff --git a/tools/stack-bench/tests/order-operation-history.test.ts b/tools/stack-bench/tests/order-operation-history.test.ts new file mode 100644 index 00000000000..d712facd983 --- /dev/null +++ b/tools/stack-bench/tests/order-operation-history.test.ts @@ -0,0 +1,154 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { orderOperationDifferences, type CheckoutState, type OrderOperation } from '../src/stacks/checkout-state.js'; +import { executeAction } from '../src/actions/action-contract.js'; +import { ACTION_REGISTRY } from '../src/actions/action-catalog.js'; +import { createDatabaseReadCapability } from '../src/actions/runtime-action-executors.js'; + +const catalog = [{ itemId: 'i', priceMinor: 200 }, { itemId: 'j', priceMinor: 300 }]; + +function initial(): CheckoutState { + return { accountId: 'a', itemId: 'i', priceMinor: 200, cart: [], reservations: [], orders: [], payments: [], + orphanOrderLines: 0, orphanAllocations: 0, + stock: ['i', 'j'].flatMap(itemId => ['E', 'W'].map(warehouseId => ({ itemId, warehouseId, quantity: 100 }))) }; +} + +test('checkout snapshot timing uses the request evidence clock across a wall-clock correction', async context => { + context.mock.method(Date, 'now', () => 0); + const snapshots = new Map(); + const result = await executeAction(ACTION_REGISTRY, 'dbRecordCheckout', { + do: 'dbRecordCheckout', account: 'a', item: 'i', as: 'before', + }, { capabilities: { 'database-read': { + ...createDatabaseReadCapability({ expand: value => value, checkoutSnapshots: snapshots }), + getCheckoutState: () => ({ state: initial(), schemaSha256: {} }), + } } }); + assert.equal(result.status, 'passed'); + assert(snapshots.get('before').recordedAtMs >= result.timing.startedAtMs); + assert(snapshots.get('before').recordedAtMs <= result.timing.completedAtMs); +}); + +test('three 30-operation histories preserve cumulative orders, ownership, stock and single cancellation refunds', () => { + for (const seed of [17, 43, 91]) { + let state = initial(), steps = 0; + const history: OrderOperation[] = []; + const advance = (operation: OrderOperation, effect: (state: CheckoutState) => void) => { + const before = structuredClone(state), after = structuredClone(state); effect(after); + history.push(operation); + assert.deepEqual(orderOperationDifferences(before, after, operation, catalog), [], JSON.stringify({ seed, history })); + if (JSON.stringify(before) !== JSON.stringify(after)) { + assert(orderOperationDifferences(before, before, operation, catalog).length, 'ignored completed writes must fail'); + } + const corrupt = structuredClone(after); corrupt.stock[0]!.quantity++; + assert(orderOperationDifferences(before, corrupt, operation, catalog).length, 'unexplained stock must fail at this step'); + if (after.orders.length > before.orders.length) for (const field of ['owner', 'price', 'duplicate'] as const) { + const broken = structuredClone(after), order = broken.orders.at(-1)!; + if (field === 'owner') order.accountId = 'wrong'; + if (field === 'price') order.totalMinor++; + if (field === 'duplicate') broken.orders.push({ ...order, id: 'duplicate' }); + assert(orderOperationDifferences(before, broken, operation, catalog).length, field); + } + state = after; steps++; + }; + for (let round = 0; round < 3; round++) { + state.accountId = round % 2 ? 'b' : 'a'; + const quantity = 2 + (seed + round) % 4, transfer = 1 + (seed * 3 + round) % 7; + const owner = state.accountId, id = `${seed}-${round}`; + const buy = (itemId: 'i' | 'j', suffix: string) => advance({ kind: 'buy', itemId }, next => { + next.stock[itemId === 'i' ? 0 : 2]!.quantity--; + next.orders.push({ id: id + suffix, accountId: owner, status: 'pending', refundedMinor: 0, + totalMinor: itemId === 'i' ? 200 : 300, + lines: [{ itemId, quantity: 1, priceMinor: itemId === 'i' ? 200 : 300, + allocations: [{ warehouseId: 'E', quantity: 1 }] }] }); + }); + buy('i', '-buy'); + advance({ kind: 'cart', itemId: 'j', quantity }, next => { + next.cart = [{ itemId: 'j', quantity }]; next.stock[2]!.quantity -= quantity; + next.reservations = [{ itemId: 'j', warehouseId: 'E', quantity }]; + }); + advance({ kind: 'transfer', itemId: 'j', fromWarehouseId: 'W', toWarehouseId: 'E', quantity: transfer }, next => { + next.stock[2]!.quantity += transfer; next.stock[3]!.quantity -= transfer; + }); + advance({ kind: 'restock', itemId: 'i', warehouseId: 'W', quantity: 7 }, next => { next.stock[1]!.quantity += 7; }); + advance({ kind: 'cart', itemId: 'j', quantity: quantity + 1 }, next => { + next.cart[0]!.quantity++; next.stock[2]!.quantity--; next.reservations[0]!.quantity++; + }); + advance({ kind: 'checkout' }, next => { + next.orders.push({ id, accountId: owner, status: 'pending', refundedMinor: 0, totalMinor: 300 * (quantity + 1), + lines: [{ itemId: 'j', quantity: quantity + 1, priceMinor: 300, allocations: [{ warehouseId: 'E', quantity: quantity + 1 }] }] }); + next.cart = []; next.reservations = []; + }); + advance({ kind: 'cancel', orderId: id }, next => { + const order = next.orders.find(order => order.id === id)!; + order.status = 'cancelled'; order.refundedMinor = 300 * (quantity + 1); next.stock[2]!.quantity += quantity + 1; + }); + advance({ kind: 'cancel', orderId: id }, () => {}); + advance({ kind: 'reconnect' }, () => {}); + buy('j', '-last'); + } + assert.equal(steps, 30); assert.equal(state.orders.length, 9); + assert.equal(state.orders.filter(order => order.status === 'cancelled').length, 3); + assert.equal(state.stock[0]!.quantity, 97); assert.equal(state.stock[1]!.quantity, 121); + assert.equal(state.stock[2]!.quantity + state.stock[3]!.quantity, 197); + } +}); + +test('history accepts unreserved carts and rejects invalid or missing observations', () => { + const before = initial(), after = initial(); after.cart = [{ itemId: 'j', quantity: 3 }]; + assert.deepEqual(orderOperationDifferences(before, after, { kind: 'cart', itemId: 'j', quantity: 3 }, catalog), []); + for (const change of [ + (state: CheckoutState) => { state.reservations = [{ itemId: 'j', warehouseId: 'unknown', quantity: 1 }]; }, + (state: CheckoutState) => { state.reservations = [{ itemId: 'j', warehouseId: 'E', quantity: 4 }]; state.stock[2]!.quantity -= 4; }, + (state: CheckoutState) => { state.cart.push({ itemId: 'j', quantity: 3 }); }, + ]) { + const broken = structuredClone(after); change(broken); + assert(orderOperationDifferences(before, broken, { kind: 'cart', itemId: 'j', quantity: 3 }, catalog).length); + } + assert.throws(() => orderOperationDifferences(before, undefined as unknown as CheckoutState, { kind: 'reconnect' }, catalog)); + assert.throws(() => orderOperationDifferences(before, after, { kind: 'cart', itemId: 'j', quantity: -1 }, catalog)); +}); + +test('history action binds complete fresh requests and keeps both customer snapshots in failed evidence', async () => { + for (const operation of [['buy'], 'unknown']) { + const invalid = await executeAction(ACTION_REGISTRY, 'dbExpectOperation', { + do: 'dbExpectOperation', before: 'before', otherBefore: 'other', actor: 'buyer', operation, + }, { capabilities: {} }); + assert.equal(invalid.status, 'harness_failure'); assert.equal(invalid.code, 'invalid_input'); + } + for (const mode of ['valid', 'reconnect', 'ignored', 'other-cart', 'other-hold', 'divergent-reads', 'catalog-change', 'incomplete', 'stale', 'refused', 'wrong-action', 'missing', 'reader-error']) { + const before = initial(), other = initial(); other.accountId = 'b'; + const after = initial(), otherAfter = structuredClone(other); + if (!['ignored', 'refused', 'reconnect'].includes(mode)) after.cart = [{ itemId: 'j', quantity: 1 }]; + if (mode === 'other-cart') otherAfter.cart = [{ itemId: 'j', quantity: 1 }]; + if (mode === 'other-hold') otherAfter.reservations = [{ itemId: 'j', warehouseId: 'E', quantity: 1 }]; + if (mode === 'divergent-reads') otherAfter.stock[0]!.quantity++; + const snapshot = (state: CheckoutState) => ({ state, scope: 'orders' as const, + storage: { kind: 'order-data' as const, cart: true, warehouses: true }, recordedAtMs: 10, + account: state.accountId, item: 'i', schemaSha256: { contract: 'fixed' }, + catalog: catalog.map(row => ({ ...row, name: row.itemId })) }); + const snapshots = new Map([['before', snapshot(before)], ['other', snapshot(other)]]); + if (mode === 'missing') snapshots.delete('before'); + const result = await executeAction(ACTION_REGISTRY, 'dbExpectOperation', { + do: 'dbExpectOperation', before: 'before', otherBefore: 'other', actor: 'buyer', operation: mode === 'reconnect' ? 'reconnect' : 'cart-add', + }, { capabilities: { + 'database-read': { + ...createDatabaseReadCapability({ expand: value => value, checkoutSnapshots: snapshots }), + getCheckoutState: (input: { account: string }) => { + if (mode === 'reader-error') throw new Error('observer unavailable'); + const value = snapshot(input.account === 'a' ? after : otherAfter); + if (mode === 'catalog-change') value.catalog[0]!.priceMinor++; + return value; + }, + }, + 'named-actions': { lastCalls: { get: () => ({ fired: 1, action: 'cart-add', ms: 1, outcomes: [{ + name: 'buyer', action: mode === 'wrong-action' ? 'buy' : 'cart-add', values: { itemId: 'j' }, + ok: mode !== 'refused', status: mode === 'refused' ? 409 : 200, text: '{}', complete: mode !== 'incomplete', + startedAtMs: mode === 'stale' ? 5 : 11, completedAtMs: 12, + }] }) } }, + } }); + assert.equal(result.status, ['valid', 'reconnect'].includes(mode) ? 'passed' : ['incomplete', 'stale', 'missing', 'divergent-reads'].includes(mode) ? 'inconclusive' + : ['wrong-action', 'reader-error'].includes(mode) ? 'harness_failure' : 'failed', mode); + if (['ignored', 'other-cart', 'other-hold', 'refused'].includes(mode)) { + assert(result.observation && typeof result.observation === 'object'); assert('otherAfter' in result.observation); + } + } +}); diff --git a/tools/stack-bench/tests/outcomes.test.ts b/tools/stack-bench/tests/outcomes.test.ts new file mode 100644 index 00000000000..155c1cfda7b --- /dev/null +++ b/tools/stack-bench/tests/outcomes.test.ts @@ -0,0 +1,281 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { aggregateRunOutcome, classifyBundle, ladderMayAdvance, ladderMayContinue, + mutationControlEligible, runExitCode, runOutcomeKind, + type OutcomeCriterion } from '../src/evidence/outcomes.js'; +import { createCheckEvidence, type CheckEvidence, + type CheckEvidenceStatus } from '../src/evidence/check-evidence.js'; + +interface TestFeature { + cleanupEvidence?: { status: string; failures: readonly unknown[] }; + criteria: OutcomeCriterion[]; + id: string; +} + +interface TestBundle { + readonly totals: { score: number; max: number }; + readonly suites: { + readonly feature: { readonly features: TestFeature[] }; + readonly lint: { readonly pass: boolean }; + }; +} + +const bundle = (criteria: OutcomeCriterion[]): TestBundle => ({ + totals: { score: 1, max: 2 }, + suites: { feature: { features: [{ id: 'f', criteria }] }, lint: { pass: true } }, +}); + +const typed = ( + id: string, + status: CheckEvidenceStatus, + summary: string | null, + points = 1, +): OutcomeCriterion & { readonly evidence: CheckEvidence; readonly id: string; readonly points: number } => { + const evidence = createCheckEvidence({ status, code: status === 'passed' ? 'completed' : 'test_result', + phase: 'assertion', summary, startedAtMs: 1, completedAtMs: 2 }); + return { id, points, evidence }; +}; + +test('missing output and explicit harness failure are not app failures', () => { + assert.equal(classifyBundle(null).kind, 'ungraded'); + assert.equal(classifyBundle({ outcome: { kind: 'harness_failure', phase: 'reset', reason: 'down' } }).kind, + 'harness_failure'); +}); + +test('provider failures preserve their cause and stop the run', () => { + const provider = { kind: 'provider_failure', phase: 'coding-session', + reason: 'provider-connection-error', provider: { providerStatus: null } }; + assert.deepEqual(classifyBundle({ outcome: provider }), { + ...provider, appFailures: [], inconclusive: [], harnessFailures: [], + }); + const outcome = aggregateRunOutcome([{ level: 1, outcome: provider }]); + assert.equal(outcome.kind, 'provider_failure'); + assert.equal(outcome.reason, 'provider-connection-error'); + assert.deepEqual(outcome.provider, { providerStatus: null }); + assert.equal(runExitCode(outcome), 1); + assert.equal(ladderMayContinue(outcome), false); +}); + +test('ungraded and harness-failed runs return a failing process status', () => { + assert.equal(runExitCode({ kind: 'harness_failure' }), 1); + assert.equal(runExitCode({ kind: 'provider_failure' }), 1); + assert.equal(runExitCode({ kind: 'ungraded' }), 1); + assert.equal(runExitCode({ kind: 'app_failure' }), 0); + assert.equal(runExitCode({ kind: 'passed' }), 0); + assert.equal(runExitCode({ kind: 'inconclusive' }), 1); + assert.equal(runExitCode({ kind: 'incomplete' }), 1); +}); + +test('a ladder stops after an ungraded or harness-failed level', () => { + assert.equal(ladderMayContinue({ kind: 'app_failure' }), true); + assert.equal(ladderMayContinue({ kind: 'inconclusive' }), false); + assert.equal(ladderMayContinue({ kind: 'app_failure', inconclusive: ['feature/check'] }), false); + assert.equal(ladderMayContinue({ kind: 'passed' }), true); + assert.equal(ladderMayContinue({ kind: 'harness_failure' }), false); + assert.equal(ladderMayContinue({ kind: 'provider_failure' }), false); + assert.equal(ladderMayContinue({ kind: 'ungraded' }), false); +}); + +test('a ladder advances only from a level that actually passed', () => { + assert.equal(ladderMayAdvance({ kind: 'passed' }), true); + assert.equal(ladderMayAdvance({ kind: 'app_failure' }), false); + assert.equal(ladderMayAdvance({ kind: 'inconclusive' }), false); + assert.equal(ladderMayAdvance({ kind: 'harness_failure' }), false); + assert.equal(ladderMayAdvance({ kind: 'ungraded' }), false); + for (const outcome of [{ kind: 'passed', inconclusive: ['unmeasured'] }, + { kind: 'passed', harnessFailures: ['cleanup'] }]) { + assert.equal(ladderMayAdvance(outcome), false); + assert.equal(mutationControlEligible(outcome), false); + } +}); + +test('a later application abort cannot hide earlier unmeasured observations', () => { + for (const status of ['harness_failure', 'inconclusive'] as const) { + const partial = { ...bundle([typed('first', status, 'observation unavailable')]), + outcome: { kind: 'app_failure', phase: 'application-start', reason: 'restart failed' }, + selection: { checks: [{ stableKey: 'first', points: 1 }, { stableKey: 'later', points: 1 }], + reportedChecks: ['first'], notRun: ['later'] } }; + assert.equal(classifyBundle(partial).kind, status); + assert.equal(ladderMayContinue(classifyBundle(partial)), false); + } +}); + +test('app failures and inconclusive evidence remain separately visible', () => { + const outcome = classifyBundle(bundle([ + typed('a', 'failed', 'not observed'), + typed('b', 'inconclusive', 'not measurable'), + ])); + assert.equal(outcome.kind, 'inconclusive'); + assert.deepEqual(outcome.appFailures, ['feature/f/a']); + assert.deepEqual(outcome.inconclusive, ['feature/f/b']); + assert.equal(ladderMayContinue(outcome), false, 'incomplete grades cannot enter repair'); + assert.equal(aggregateRunOutcome([{ level: 1, outcome: { kind: 'app_failure' } }, + { level: 2, outcome }]).kind, 'inconclusive'); +}); + +test('inconclusive-only grades are not reported as passing', () => { + assert.equal(classifyBundle(bundle([typed('a', 'inconclusive', 'not measurable')])).kind, + 'inconclusive'); +}); + +test('zero-point evidence cannot fail or invalidate a scored benchmark outcome', () => { + const outcome = classifyBundle(bundle([ + typed('scored', 'passed', null), + typed('candidate', 'failed', 'candidate behavior failed', 0), + typed('control', 'inconclusive', 'supporting control unavailable', 0), + ])); + assert.equal(outcome.kind, 'passed'); + assert.deepEqual(outcome.appFailures, []); + assert.deepEqual(outcome.inconclusive, []); +}); + +test('an explicitly selected zero-only scope still has a qualification outcome', () => { + const scoped = { + totals: { score: 0, max: 0 }, + selection: { checks: [{ stableKey: 'candidate', points: 0 }], attemptedChecks: ['candidate'], + reportedChecks: ['candidate'], notRun: [] }, + suites: { development: { features: [{ id: 'f', criteria: [ + { ...typed('candidate', 'failed', 'candidate behavior failed', 0), stableKey: 'candidate' }, + ] }] }, lint: { pass: true } }, + }; + assert.equal(classifyBundle(scoped).kind, 'app_failure'); + assert.deepEqual(classifyBundle(scoped).appFailures, ['development/f/candidate']); +}); + +test('typed harness failures outrank prose and application failures', () => { + const outcome = classifyBundle(bundle([ + typed('app', 'failed', 'INCONCLUSIVE: wording must not control status'), + typed('harness', 'harness_failure', 'this sentence sounds harmless'), + ])); + assert.equal(outcome.kind, 'harness_failure'); + assert.deepEqual(outcome.harnessFailures, ['feature/f/harness']); + assert.deepEqual(outcome.appFailures, ['feature/f/app']); +}); + +test('completed check evidence outranks a declared application failure', () => { + const criterion = { ...typed('check', 'harness_failure', + 'grader could not observe the result'), stableKey: 'feature.check' }; + const scoped = { + ...bundle([criterion]), + totals: { score: 0, max: 1 }, + selection: { checks: [{ stableKey: 'feature.check', points: 1 }], + attemptedChecks: ['feature.check'], reportedChecks: ['feature.check'], notRun: [] }, + outcome: { kind: 'app_failure', phase: 'grading', appFailures: ['declared-failure'] }, + }; + const outcome = classifyBundle(scoped); + assert.equal(outcome.kind, 'harness_failure'); + assert.deepEqual(outcome.harnessFailures, ['feature/f/check']); +}); + +test('grader cleanup failures invalidate the bundle', () => { + const scoped = bundle([typed('works', 'passed', null)]); + const feature = scoped.suites.feature.features[0]; + assert(feature); + feature.cleanupEvidence = { + status: 'harness_failure', failures: [{ stage: 'context-close' }], + }; + const outcome = classifyBundle(scoped); + assert.equal(outcome.kind, 'harness_failure'); + assert.equal(outcome.phase, 'grading-cleanup'); + assert.deepEqual(outcome.harnessFailures, ['feature/f/cleanup']); +}); + +test('a fully reported zero-point selection is graded rather than mistaken for missing output', () => { + const scoped = { + totals: { score: 0, max: 0 }, + selection: { + checks: [{ stableKey: 'pack.control.zero', points: 0 }], + attemptedChecks: ['pack.control.zero'], + reportedChecks: ['pack.control.zero'], + notRun: [], + }, + suites: { control: { features: [{ id: 'f', criteria: [ + { ...typed('zero', 'passed', null), stableKey: 'pack.control.zero', points: 0 }, + ] }] }, lint: { pass: true } }, + }; + assert.equal(classifyBundle(scoped).kind, 'passed'); + scoped.selection.reportedChecks = []; + assert.equal(classifyBundle(scoped).kind, 'ungraded'); +}); + +test('recipe-bound scores must exactly match their check evidence', () => { + const passed = { ...typed('works', 'passed', null, 2), stableKey: 'pack.feature.works' }; + const scoped = { + totals: { score: 1, max: 2, regression: null }, + selection: { + checks: [{ stableKey: 'pack.feature.works', points: 2 }], + attemptedChecks: ['pack.feature.works'], + reportedChecks: ['pack.feature.works'], + notRun: [], + }, + suites: { feature: { features: [{ id: 'f', criteria: [passed] }] }, lint: { pass: true } }, + }; + const outcome = classifyBundle(scoped); + assert.equal(outcome.kind, 'harness_failure'); + assert.equal(outcome.phase, 'grading'); + assert.match(outcome.reason ?? '', /reported score 1\/2 disagrees with check evidence 2\/2/); + + scoped.totals.score = 2; + assert.equal(classifyBundle(scoped).kind, 'passed'); + + scoped.selection.checks.push({ stableKey: 'pack.feature.other', points: 2 }); + scoped.selection.reportedChecks.push('pack.feature.other'); + scoped.selection.attemptedChecks.push('pack.feature.other'); + const feature = scoped.suites.feature.features[0]; + assert(feature); + feature.criteria.push({ ...passed, id: 'duplicate' }); + scoped.totals = { score: 4, max: 4, regression: null }; + assert.equal(classifyBundle(scoped).kind, 'harness_failure', + 'duplicating one check must not stand in for a different selected check'); +}); + +test('mutation control runs only after a conclusive passing pristine grade', () => { + assert.equal(mutationControlEligible({ kind: 'passed' }), true); + for (const kind of ['app_failure', 'inconclusive', 'harness_failure', 'ungraded']) { + assert.equal(mutationControlEligible({ kind }), false, kind); + } +}); + +test('run aggregation preserves every level and prioritizes harness failure', () => { + const outcome = aggregateRunOutcome([ + { level: 1, outcome: { kind: 'app_failure' } }, + { level: 2, outcome: { kind: 'harness_failure', phase: 'grading' } }, + ]); + assert.equal(outcome.kind, 'harness_failure'); + assert.equal(outcome.levels['1']?.kind, 'app_failure'); +}); + +test('an incomplete selected scope is never accepted through a positive denominator', () => { + const scoped = { + totals: { score: 1, max: 2 }, + selection: { + checks: [{ stableKey: 'pack.feature.a', points: 1 }, + { stableKey: 'pack.feature.b', points: 1 }], + reportedChecks: ['pack.feature.a'], + notRun: [{ stableKey: 'pack.feature.b' }], + }, + suites: { feature: { features: [{ id: 'f', criteria: [ + { ...typed('a', 'passed', null), stableKey: 'pack.feature.a' }, + ] }] }, lint: { pass: true } }, + }; + assert.equal(classifyBundle(scoped).kind, 'ungraded'); +}); + +test('unknown outcome names fail closed', () => { + assert.equal(runExitCode({ kind: 'typo' }), 1); + assert.equal(ladderMayContinue({ kind: 'typo' }), false); + assert.throws(() => runOutcomeKind('typo'), /invalid run outcome kind/); +}); + +test('contract lint harness and unmeasured results are not app failures', () => { + const graded = bundle([typed('a', 'passed', null)]); + const withLint = (lint: object) => classifyBundle({ ...graded, suites: { ...graded.suites, lint } }); + assert.equal(withLint({ pass: false }).kind, 'app_failure'); + assert.equal(withLint({ pass: false, harness: true }).kind, 'harness_failure'); + assert.equal(withLint({ pass: false, unmeasured: true }).kind, 'inconclusive'); +}); + +test('a declared inconclusive run outcome is not reported from the checks that did run', () => { + assert.equal(classifyBundle({ ...bundle([typed('a', 'passed', null)]), + outcome: { kind: 'inconclusive', phase: 'application-readiness' } }).kind, 'inconclusive'); +}); diff --git a/tools/stack-bench/tests/owned-backend.integration.ts b/tools/stack-bench/tests/owned-backend.integration.ts new file mode 100644 index 00000000000..cb535b3c283 --- /dev/null +++ b/tools/stack-bench/tests/owned-backend.integration.ts @@ -0,0 +1,140 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { randomBytes } from 'node:crypto'; +import test from 'node:test'; +import { fileURLToPath } from 'node:url'; + +// Explicit, model-free gate. Uses installed immutable images and briefly attaches +// the trusted cache to each owned bridge. It does not reset or stop the cache. +for (const backend of ['postgres', 'mongodb', 'spacetime']) test(`owned ${backend} activation, native authentication, smoke, restart, and exact cleanup`, { + skip: process.env.STACK_BENCH_OWNED_BACKEND_TEST !== '1', timeout: 240_000, +}, () => { + const image = process.env.STACK_BENCH_NETWORK_CONTROLLER_IMAGE; + assert.match(image ?? '', /^sha256:[a-f0-9]{64}$/); + const prefix = `stack-bench-owned-probe-${randomBytes(6).toString('hex')}`; + const docker = (args: string[], input?: string): string => { + const result = spawnSync('docker', args, { encoding: 'utf8', input, timeout: 210_000 }); + assert.equal(result.status, 0, `${args.slice(0, 3).join(' ')}: ${result.stderr || result.error}\n${result.stdout}`); + return result.stdout.trim(); + }; + const cache = docker(['inspect', '--format', '{{.Id}}', 'stack-bench-npm-cache']); + const networksBefore = docker(['inspect', '--format', '{{json .NetworkSettings.Networks}}', cache]); + const volume = docker(['volume', 'create', `${prefix}-state`]); + const state = docker(['volume', 'inspect', '--format', '{{.Mountpoint}}', volume]); + let controller: string | undefined; + try { + controller = docker(['create', '--name', `${prefix}-controller`, '--network', 'host', + '--mount', 'type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock', + '--mount', `type=bind,source=${state},target=${state}`, + '--mount', `type=bind,source=${fileURLToPath(new URL('../', import.meta.url))},target=/opt/stack-bench/dist,readonly`, + '-e', 'STACK_BENCH_APPLIANCE=1', '-e', `STACK_BENCH_CONTROLLER_IMAGE_ID=${image}`, + '--entrypoint', 'node', image!, '--input-type=module', '-e', ` + import assert from 'node:assert/strict'; + import { mkdirSync } from 'node:fs'; + import { createBackendLease, writeBackendLease, readBackendLease } from './dist/src/runtime/backend-lease.js'; + import { releaseBackendLease } from './dist/src/runtime/backend-teardown.js'; + import { loadTrack, portsFor } from './dist/src/composition/tracks.js'; + import { activateAttemptBackend, startAttemptDatabaseProcess } from './dist/src/stacks/hosted-lifecycle.js'; + import { controlSpacetime } from './dist/src/stacks/spacetime-lifecycle.js'; + import { attemptDatabaseIdentity } from './dist/src/stacks/hosted-database-identity.js'; + import { resetMongoDb } from './dist/src/stacks/backends/mongodb-operations.js'; + import { requireLeasedDatabase } from './dist/src/stacks/backend-reset-guard.js'; + import { attemptDocker, requireAttemptNetwork } from './dist/src/runtime/docker-network.js'; + import { runContainerSmoke } from './dist/src/runtime/container-smoke.js'; + for (const backend of [${JSON.stringify(backend)}]) { + const directory = ${JSON.stringify(state)} + '/' + backend; + mkdirSync(directory, {recursive:true}); + const path = directory + '/lease.json'; + const lease = createBackendLease({runId:${JSON.stringify(prefix)}+'-'+backend, + backend,track:'chat',runIndex:20, + database:backend==='spacetime'?null:'owned_probe', + serverUri:backend==='spacetime'?'http://127.0.0.1:3299':null, + module:backend==='spacetime'?'owned-probe':null, + dataDir:backend==='spacetime'?directory+'/data':null}); + writeBackendLease(path,lease,{exclusive:true}); + let active; + try { + activateAttemptBackend({leasePath:path,lease,ports:portsFor(loadTrack(lease.track),backend,lease.runIndex)}); + active=readBackendLease(path,{token:lease.ownershipToken,active:true}); + const id=active.resources.container.id; + const secret=attemptDatabaseIdentity(lease.ownershipToken); + if(backend==='postgres') { + const query='CREATE TABLE proof(id integer); INSERT INTO proof VALUES(1); SELECT count(*) FROM proof;'; + attemptDocker(['exec','-e','PGPASSWORD='+secret.password,id,'psql','-h','127.0.0.1','-U','appuser','-d','owned_probe','-v','ON_ERROR_STOP=1','-c',query]); + } else if(backend==='mongodb') { + const applicationShell=['exec',id,'mongosh','--quiet','--username','appuser','--password',secret.password, + '--authenticationDatabase','owned_probe','owned_probe','--eval']; + attemptDocker([...applicationShell,'db.proof.insertOne({id:1}); if(db.proof.countDocuments()!==1) quit(1)']); + attemptDocker([...applicationShell, + 'if(db.hello().setName!=="rs0" || !db.hello().isWritablePrimary) throw new Error("not a primary replica-set member"); ' + +'const session=db.getMongo().startSession(); const tx=session.getDatabase("owned_probe"); ' + +'session.startTransaction(); tx.proof.insertOne({id:10}); session.commitTransaction(); ' + +'session.startTransaction(); tx.proof.insertOne({id:11}); session.abortTransaction(); session.endSession(); ' + +'if(!db.proof.findOne({id:10}) || db.proof.findOne({id:11})) throw new Error("transaction commit or rollback failed");']); + for(let restart=0;restart<2;restart++) { + attemptDocker(['exec','--user','mongodb',id,'mongod','--shutdown','--dbpath','/data/db']); + startAttemptDatabaseProcess(active); + let restarted=false; + for(let retry=0;retry<60;retry++) { + try { + attemptDocker([...applicationShell,'if(!db.hello().isWritablePrimary) quit(1); if(!db.proof.findOne({id:10})) throw new Error("committed data lost after restart");']); + restarted=true; + break; + } catch { await new Promise(resolve=>setTimeout(resolve,500)); } + } + if(!restarted) console.error(attemptDocker(['exec',id,'tail','-n','20','/run/application/stack-bench-backend.log'])); + assert.equal(restarted,true,'MongoDB did not recover committed data as primary after restart'); + } + resetMongoDb({lease:requireLeasedDatabase(active)}); + attemptDocker([...applicationShell, + 'if(db.proof.countDocuments()!==0) throw new Error("reset left application data"); ' + +'db.proof.insertOne({id:2}); if(!db.proof.findOne({id:2})) throw new Error("read failed"); ' + +'if(db.proof.updateOne({id:2},{$set:{id:3}}).modifiedCount!==1) throw new Error("update failed"); ' + +'if(db.proof.deleteOne({id:3}).deletedCount!==1) throw new Error("delete failed"); ' + +'let denied; try { db.dropDatabase(); } catch(error) { denied=error.code; } ' + +'if(denied!==13) throw new Error("application role did not reject dropDatabase");']); + console.log('mongodb: trusted reset preserved application CRUD and denied dropDatabase'); + } else { + const started=active.resources.network.namespaceStartedAt; + await controlSpacetime({lease:active}); + assert.equal(readBackendLease(path).resources.network.namespaceStartedAt,started); + assert.equal(requireAttemptNetwork(active),'container:'+id); + } + const smoke=runContainerSmoke({command:(_file,args)=>attemptDocker(args), + imageId:process.env.STACK_BENCH_CONTROLLER_IMAGE_ID,resultsDir:directory, + destinations:['http://'+active.resources.network.services[0].address+':4873/-/ping'], + tcpPorts:[backend==='postgres'?5432:backend==='mongodb'?27017:3299], + requiredExecutables:['node'],credentialStatusCommand:null,credentialMount:null, + credentialEnvironment:null,marker:'smoke-proof',networkMode:requireAttemptNetwork(active), + leaseContext:{path,lease:active}}); + assert.equal(smoke.tcpReached.length,1); + console.log(backend+': native backend, isolated smoke'+(backend==='spacetime'?', process restart':'')+' passed'); + } catch(error) { + const current=readBackendLease(path); + if(current.resources.container) { + try { console.error(attemptDocker(['exec',current.resources.container.id,'tail','-n','30','/run/application/stack-bench-backend.log'])); } catch {} + } + throw error; + } finally { + assert.equal(releaseBackendLease(path,lease.ownershipToken),true); + const released=readBackendLease(path); + assert.equal(released.state,'released'); + for(const kind of ['container','browserContainer']) { + const resource=released.resources[kind]; + if(resource) assert.throws(()=>attemptDocker(['inspect',resource.id])); + } + if(released.resources.network) assert.throws(()=>attemptDocker(['network','inspect',released.resources.network.id])); + } + } + `]); + const output = docker(['start', '--attach', controller]); + if (backend === 'mongodb') assert.match(output, /mongodb: trusted reset preserved application CRUD and denied dropDatabase/); + if (backend === 'spacetime') assert.match(output, /spacetime: native backend, isolated smoke, process restart passed/); + console.log(output); + } finally { + if (controller) docker(['rm', '-f', '--volumes', controller]); + assert.equal(docker(['inspect', '--format', '{{.Id}}', 'stack-bench-npm-cache']), cache); + assert.equal(docker(['inspect', '--format', '{{json .NetworkSettings.Networks}}', cache]), networksBefore); + docker(['volume', 'rm', volume]); + } +}); diff --git a/tools/stack-bench/tests/pack-budget.test.ts b/tools/stack-bench/tests/pack-budget.test.ts new file mode 100644 index 00000000000..9b5a231833f --- /dev/null +++ b/tools/stack-bench/tests/pack-budget.test.ts @@ -0,0 +1,266 @@ +import assert from 'node:assert/strict'; +import { mkdirSync, mkdtempSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import test from 'node:test'; + +import { createArtifact, currentEngineIdentity, recipeArtifactIdentities, writeArtifact } from '../src/evidence/artifacts.js'; +import { calibrationQualificationIdentity, calibrationQualificationRelease, + resolveCalibrationForRelease } from '../src/composition/calibration-compiler.js'; +import { PACK_RUNTIME_METRIC } from '../src/composition/pack-runtime.js'; +import { parsePackBudgetArgs } from '../commands/pack-budget.js'; +import { loadPackBudgetEvidence, PACK_BUDGET_POLICY, recommendPackBudgets } + from '../src/composition/pack-budget.js'; +import type { PackBudgetEvidence, PackRuntime, ReferenceQualificationPayload, + RunnerObservation } from '../src/composition/pack-budget.js'; +import type { Artifact, ArtifactIdentity } from '../src/evidence/artifacts.js'; +import { requireRecipeRelease as resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { loadTrack } from '../src/composition/tracks.js'; + +const track = loadTrack('ecommerce'); +const binding = resolveRecipeRelease(track, 3, 'ecommerce.progression-catalog'); +const resolvedCalibration = resolveCalibrationForRelease(binding.release, { trackRoot: track.dir, alias: 'L3' }); +assert(resolvedCalibration, `no L3 calibration matches ${binding.release.id} ` + + `(${binding.release.contentSha256}); recalibrate after changing its scenarios`); +const calibration = structuredClone(resolvedCalibration); +const selectedChecks = calibrationQualificationRelease(calibration, + binding.release, binding.execution).release.checkCatalog; +calibration.qualification.runner = { + schemaVersion: 1, mode: 'appliance', platform: 'linux', architecture: 'x64', +}; +const applianceRunner: RunnerObservation = Object.freeze({ ...calibration.qualification.runner, + dockerEngineVersion: '29.1.2', dockerOs: 'linux', dockerArchitecture: 'x86_64', + kernelVersion: '6.8.0-test', cpuCount: 8, memoryBytes: 16_000_000_000 }); + +function runtime(stackIndex: number, repetition: number): PackRuntime { + const counts = new Map(binding.plan.packs.map(pack => [pack.id, 0])); + for (const check of selectedChecks) { + if (!check.packId) throw new Error(`check ${check.stableKey} has no pack`); + counts.set(check.packId, (counts.get(check.packId) ?? 0) + 1); + } + return { schemaVersion: 1, metric: PACK_RUNTIME_METRIC, + packs: [...counts].filter(([, count]) => count > 0) + .sort(([a], [b]) => a.localeCompare(b)).map(([id, checkCount], packIndex) => { + const measuredRuntimeMs = 1_000 + stackIndex * 100 + repetition * 10 + packIndex; + return { id, checkCount, setupRuntimeMs: 100, criterionRuntimeMs: measuredRuntimeMs - 100, + measuredRuntimeMs, budget: { status: 'unmeasured' }, exceeded: null }; + }) }; +} + +function reference(stack: string, stackIndex: number, + overrides: Partial = {}): Artifact { + const fixture = calibration.references.entries.find(entry => entry.backend === stack); + assert(fixture); + return createArtifact({ kind: 'reference_qualification', id: `reference-${stack}`, + identities: recipeArtifactIdentities(binding.release, { + engine: currentEngineIdentity(), calibration: { id: calibration.id, + sha256: calibrationQualificationIdentity(calibration).contentSha256 }, stackAdapter: { id: stack }, + fixture: { id: fixture.id, sha256: fixture.sourceSha256 }, + }), + payload: { fixture: fixture.id, fixtureSha256: fixture.sourceSha256, + requiredRepetitions: 1, isolation: 'docker', mutationControl: false, + runner: { ...applianceRunner }, + stable: true, sameImage: true, sameHarness: true, harnessSha256: 'b'.repeat(64), ok: true, + runs: [1].map(repetition => ({ repetition, ok: true, packRuntime: runtime(stackIndex, repetition) })), + ...overrides } }); +} + +function exactEvidence(): PackBudgetEvidence[] { + return calibration.qualification.stacks.map((stack, index) => ({ + path: `${stack}.json`, sha256: String(index).repeat(64), artifact: reference(stack, index), + runtimeCalibration: { id: calibration.id, + sha256: calibration.contentSha256 }, + })); +} + +function evidenceAt(evidence: PackBudgetEvidence[], index: number): PackBudgetEvidence { + const item = evidence[index]; + assert(item); + return item; +} + +function identity(value: ArtifactIdentity | null): ArtifactIdentity { + assert(value); + return value; +} + +test('budget recommendation requires every exact reference repetition and applies the published rule', () => { + const evidence = exactEvidence(); + evidence.forEach((item, index) => { + assert(item.artifact.payload.runner); + item.artifact.payload.runner.containersRunning = 18 + index; + }); + const original = structuredClone(evidence); + const result = recommendPackBudgets({ binding, calibration, trackRoot: track.dir, evidence }); + const measuredPackCount = new Set(selectedChecks.map(check => check.packId)).size; + assert.equal(result.samples.length, measuredPackCount * evidence.length); + assert.equal(result.recommendations.length, measuredPackCount); + assert(result.recommendations.every(item => item.sampleCount === evidence.length)); + // Twice the slowest run, but never below the time the pack's scenarios allow. + assert(result.recommendations.every(item => + item.maxRuntimeMs === Math.max(3_000, Math.ceil(item.declaredWaitMs / 1_000) * 1_000))); + assert(result.recommendations.some(item => item.declaredWaitMs > 3_000 && item.maxRuntimeMs > 3_000)); + assert(result.recommendations.some(item => item.maxRuntimeMs === 3_000)); + assert.equal(PACK_BUDGET_POLICY.multiplier, 2); + assert.deepEqual(result.measuredEngine, currentEngineIdentity()); + assert.deepEqual(result.measuredRunner, { ...applianceRunner, containersRunning: 18 }); + assert.deepEqual(evidence, original); + assert.notEqual(evidence[0]!.artifact.identities.calibration!.sha256, evidence[0]!.runtimeCalibration!.sha256, + 'qualification and runtime calibration hashes have different meanings'); +}); + +test('budget recommendation rejects mutation, duplicate, incomplete, and cross-scope evidence', () => { + const diagnostic = exactEvidence(); + evidenceAt(diagnostic, 0).artifact.payload.diagnostic = true; + assert.throws(() => recommendPackBudgets({ binding, calibration, trackRoot: track.dir, evidence: diagnostic }), /targeted diagnostic/); + evidenceAt(diagnostic, 0).artifact.payload.timingOnly = true; + assert.doesNotThrow(() => recommendPackBudgets({ binding, calibration, trackRoot: track.dir, evidence: diagnostic })); + const mutation = exactEvidence(); + evidenceAt(mutation, 0).artifact.payload.mutationControl = true; + assert.throws(() => recommendPackBudgets({ binding, calibration, trackRoot: track.dir, evidence: mutation }), /mutation evidence/); + const duplicate = exactEvidence(); + identity(evidenceAt(duplicate, 2).artifact.identities.stackAdapter).id = identity(evidenceAt(duplicate, 0).artifact.identities.stackAdapter).id; + assert.throws(() => recommendPackBudgets({ binding, calibration, trackRoot: track.dir, evidence: duplicate }), /repeats stack/); + assert.throws(() => recommendPackBudgets({ binding, calibration, trackRoot: track.dir, evidence: exactEvidence().slice(1) }), + /cover each supported stack/); + const stale = exactEvidence(); + identity(evidenceAt(stale, 0).artifact.identities.recipe).sha256 = 'f'.repeat(64); + assert.throws(() => recommendPackBudgets({ binding, calibration, trackRoot: track.dir, evidence: stale }), + /does not match the selected qualification scope/); + const staleRuntime = exactEvidence(); + const staleRuntimeIdentity = evidenceAt(staleRuntime, 0).runtimeCalibration; + assert(staleRuntimeIdentity); + staleRuntimeIdentity.sha256 = 'f'.repeat(64); + assert.throws(() => recommendPackBudgets({ binding, calibration, trackRoot: track.dir, evidence: staleRuntime }), + /retainedRuntimeCalibration.sha256/); + const wrongQualification = exactEvidence(); + identity(evidenceAt(wrongQualification, 0).artifact.identities.calibration).sha256 = calibration.contentSha256; + assert.throws(() => recommendPackBudgets({ binding, calibration, trackRoot: track.dir, evidence: wrongQualification }), + /identities.calibration.sha256/); +}); + +test('progression L3 budgets require exactly the selected pack counts, not the full catalog', () => { + const binding = resolveRecipeRelease(track, 3, 'ecommerce.progression-catalog'); + const calibration = resolveCalibrationForRelease(binding.release, { trackRoot: track.dir, alias: 'L3' }); + assert(calibration); + const selected = new Set(calibration.qualification.checks); + const counts = new Map(); + for (const check of binding.release.checkCatalog.filter(check => selected.has(check.stableKey))) { + assert(check.packId); + counts.set(check.packId, (counts.get(check.packId) ?? 0) + 1); + } + assert(selected.size < binding.release.checkCatalog.length); + const evidence = exactEvidence(); + for (const item of evidence) { + const fixture = calibration.references.entries.find(entry => entry.backend === item.artifact.identities.stackAdapter?.id); + assert(fixture); + item.runtimeCalibration = { id: calibration.id, sha256: calibration.contentSha256 }; + item.artifact.identities = recipeArtifactIdentities(binding.release, { + ...item.artifact.identities, recipe: { id: binding.release.id, sha256: binding.release.contentSha256 }, + calibration: { id: calibration.id, sha256: calibrationQualificationIdentity(calibration).contentSha256 }, + fixture: { id: fixture.id, sha256: fixture.sourceSha256 }, + }); + item.artifact.payload.fixture = fixture.id; + item.artifact.payload.fixtureSha256 = fixture.sourceSha256; + for (const run of item.artifact.payload.runs) run.packRuntime.packs = [...counts].map(([id, checkCount]) => ({ + id, checkCount, setupRuntimeMs: 100, criterionRuntimeMs: 900, measuredRuntimeMs: 1_000, + })); + } + assert.equal(recommendPackBudgets({ binding, calibration, trackRoot: track.dir, evidence }).recommendations.length, counts.size); + const missing = structuredClone(evidence); + evidenceAt(missing, 0).artifact.payload.runs[0]!.packRuntime.packs.pop(); + assert.throws(() => recommendPackBudgets({ binding, calibration, trackRoot: track.dir, evidence: missing }), /missing packs/); + const extra = structuredClone(evidence); + const unselected = binding.plan.packs.find(pack => !counts.has(pack.id)); + assert(unselected); + evidenceAt(extra, 0).artifact.payload.runs[0]!.packRuntime.packs.push({ id: unselected.id, + checkCount: 1, setupRuntimeMs: 100, criterionRuntimeMs: 900, measuredRuntimeMs: 1_000 }); + assert.throws(() => recommendPackBudgets({ binding, calibration, trackRoot: track.dir, evidence: extra }), /unknown pack/); + const wrongCount = structuredClone(evidence); + evidenceAt(wrongCount, 0).artifact.payload.runs[0]!.packRuntime.packs[0]!.checkCount = 0; + assert.throws(() => recommendPackBudgets({ binding, calibration, trackRoot: track.dir, evidence: wrongCount }), /checks for .* expected/); +}); + +test('budget recommendation rejects timing captured outside the Linux appliance', () => { + const local = exactEvidence(); + const localRunner = evidenceAt(local, 0).artifact.payload.runner; + assert(localRunner); + localRunner.mode = 'local-controller'; + localRunner.platform = 'win32'; + assert.throws(() => recommendPackBudgets({ binding, calibration, trackRoot: track.dir, evidence: local }), + /not supported appliance timing evidence/); + + const legacy = exactEvidence(); + delete evidenceAt(legacy, 0).artifact.payload.runner; + assert.throws(() => recommendPackBudgets({ binding, calibration, trackRoot: track.dir, evidence: legacy }), + /runner\.schemaVersion must be 1/); + + const unobserved = exactEvidence(); + evidenceAt(unobserved, 0).artifact.payload.runner = { ...calibration.qualification.runner }; + assert.throws(() => recommendPackBudgets({ binding, calibration, trackRoot: track.dir, evidence: unobserved }), + /runner observation is missing/); + + for (const [field, value] of Object.entries({ cpuCount: 16, memoryBytes: 32_000_000_000, + dockerEngineVersion: '30.0.0', kernelVersion: 'other-kernel', hostname: 'other-host', + packageRegistry: 'https://other-registry.example/', futureIdentityField: 'other-setting' })) { + const mixed = exactEvidence(); + const mixedRunner = evidenceAt(mixed, 1).artifact.payload.runner; + assert(mixedRunner); + mixedRunner[field] = value; + assert.throws(() => recommendPackBudgets({ binding, calibration, trackRoot: track.dir, evidence: mixed }), + /different appliance runner environment/); + } +}); + +test('budget CLI parsing requires explicit unique evidence and output', () => { + const parsed = parsePackBudgetArgs(['node', 'pack-budget.js', 'recommend', '--track', 'ecommerce', + '--level', '1', '--evidence', 'mongo.json', '--out', 'budgets.json']); + assert.equal(parsed.command, 'recommend'); + assert.equal(parsed.evidence.length, 1); + assert.equal(parsePackBudgetArgs(['node', 'pack-budget.js', 'recommend', '--track', 'ecommerce', + '--level', '1', '--recipe', 'ecommerce.sequential-l1', + '--evidence', 'mongo.json', '--out', 'budgets.json']).recipe, + 'ecommerce.sequential-l1'); + assert.throws(() => parsePackBudgetArgs(['node', 'pack-budget.js', 'recommend', + '--track', 'ecommerce', '--level', '1']), /usage/); +}); + +test('budget evidence loader verifies retained raw runs against their summary', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-pack-budget-')); + try { + const path = join(root, 'mongodb.json'); + const artifact = reference('mongodb', 0); + for (const run of artifact.payload.runs) { + run.output = `mongodb.runs/r${run.repetition}`; + const output = join(root, 'mongodb.runs', `r${run.repetition}`); + mkdirSync(join(output, 'grading'), { recursive: true }); + const raw = createArtifact({ kind: 'benchmark_run', id: `run-${run.repetition}`, + identities: recipeArtifactIdentities(null, { engine: artifact.identities.engine, + agentAdapter: { id: 'reference-fixture' }, stackAdapter: { id: 'mongodb' } }) }); + writeArtifact(join(output, 'run.json'), raw); + writeArtifact(join(output, 'grading', 'bundle.json'), { kind: 'grade_bundle', + id: `bundle-${run.repetition}`, identities: recipeArtifactIdentities(binding.release, { + engine: artifact.identities.engine, stackAdapter: artifact.identities.stackAdapter, + calibration: { id: calibration.id, + sha256: calibration.contentSha256 } }), + payload: { packRuntime: run.packRuntime } }); + } + writeArtifact(path, artifact); + const loaded = loadPackBudgetEvidence([path]); + assert.equal(loaded.length, 1); + const loadedEvidence = evidenceAt(loaded, 0); + assert.match(loadedEvidence.sha256, /^[a-f0-9]{64}$/); + assert(loadedEvidence.runtimeCalibration); + assert.equal(loadedEvidence.runtimeCalibration.sha256, calibration.contentSha256); + + const bundlePath = join(root, 'mongodb.runs', 'r1', 'grading', 'bundle.json'); + const changed = createArtifact({ kind: 'grade_bundle', id: 'changed', + identities: recipeArtifactIdentities(binding.release, { + engine: artifact.identities.engine, stackAdapter: artifact.identities.stackAdapter, + calibration: { id: calibration.id, + sha256: calibration.contentSha256 } }), + payload: { packRuntime: runtime(2, 2) } }); + writeArtifact(bundlePath, changed); + assert.throws(() => loadPackBudgetEvidence([path]), /summary differs/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/tools/stack-bench/tests/pack-runtime.test.ts b/tools/stack-bench/tests/pack-runtime.test.ts new file mode 100644 index 00000000000..cb4cac7ee31 --- /dev/null +++ b/tools/stack-bench/tests/pack-runtime.test.ts @@ -0,0 +1,98 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { createCheckEvidence } from '../src/evidence/check-evidence.js'; +import { aggregatePackRuntime, exceededPackBudgets, measureGradePackRuntime, + PACK_RUNTIME_METRIC } from '../src/composition/pack-runtime.js'; + +const evidence = (durationMs: number) => createCheckEvidence({ status: 'passed', code: 'completed', + phase: 'assertion', startedAtMs: 100, completedAtMs: 100 + durationMs }); +const setupEvidence = (durationMs: number) => createCheckEvidence({ status: 'passed', code: 'completed', + phase: 'setup', startedAtMs: 10, completedAtMs: 10 + durationMs }); + +function report() { + return { + total: 3, + max: 3, + selection: { checks: [ + { stableKey: 'pack-a/check-1', packId: 'pack-a' }, + { stableKey: 'pack-b/check-1', packId: 'pack-b' }, + { stableKey: 'pack-a/check-2', packId: 'pack-a' }, + ] }, + features: [{ id: 1, name: 'feature', score: 3, max: 3, consoleErrors: [], + setupEvidence: setupEvidence(5), criteria: [ + { id: 'check-1', desc: 'check 1', points: 1, + stableKey: 'pack-a/check-1', evidence: evidence(30) }, + { id: 'check-2', desc: 'check 2', points: 1, + stableKey: 'pack-b/check-1', evidence: evidence(10) }, + { id: 'check-3', desc: 'check 3', points: 1, + stableKey: 'pack-a/check-2', evidence: evidence(20) }, + ] }], + }; +} + +function requireFirst(items: readonly T[]): T { + const item = items[0]; + assert.ok(item); + return item; +} + +test('grade timing is attributed by permanent check key and pack id', () => { + assert.deepEqual(measureGradePackRuntime(report()), { + schemaVersion: 1, + metric: PACK_RUNTIME_METRIC, + packs: [ + { id: 'pack-a', checkCount: 2, setupRuntimeMs: 5, criterionRuntimeMs: 50, + measuredRuntimeMs: 55 }, + { id: 'pack-b', checkCount: 1, setupRuntimeMs: 5, criterionRuntimeMs: 10, + measuredRuntimeMs: 15 }, + ], + }); +}); + +test('pack timing refuses missing, repeated, and out-of-scope evidence', () => { + const missing = report(); + requireFirst(missing.features).criteria.pop(); + assert.throws(() => measureGradePackRuntime(missing), /missing pack-a\/check-2/); + const repeated = report(); + const repeatedCriteria = requireFirst(repeated.features).criteria; + repeatedCriteria.push(requireFirst(repeatedCriteria)); + assert.throws(() => measureGradePackRuntime(repeated), /repeats stable key/); + const unknown = report(); + requireFirst(requireFirst(unknown.features).criteria).stableKey = 'other/check'; + assert.throws(() => measureGradePackRuntime(unknown), /unknown stable key/); +}); + +test('suite timing accumulates across suites and enforces only declared bounds', () => { + const first = { packRuntime: measureGradePackRuntime(report()) }; + const secondReport = report(); + const secondCriteria = requireFirst(secondReport.features).criteria; + requireFirst(secondCriteria).evidence = evidence(11); + const secondCriterion = secondCriteria[1]; + assert.ok(secondCriterion); + secondCriterion.evidence = evidence(9); + const thirdCriterion = secondCriteria[2]; + assert.ok(thirdCriterion); + thirdCriterion.evidence = evidence(10); + const runtime = aggregatePackRuntime([first, { packRuntime: measureGradePackRuntime(secondReport) }], [ + { id: 'pack-a', budget: { status: 'bounded', maxRuntimeMs: 80 } }, + { id: 'pack-b', budget: { status: 'unmeasured' } }, + ]); + assert.deepEqual(runtime.packs, [ + { id: 'pack-a', checkCount: 4, setupRuntimeMs: 10, criterionRuntimeMs: 71, + measuredRuntimeMs: 81, budget: { status: 'bounded', maxRuntimeMs: 80 }, exceeded: true }, + { id: 'pack-b', checkCount: 2, setupRuntimeMs: 10, criterionRuntimeMs: 19, measuredRuntimeMs: 29, + budget: { status: 'unmeasured' }, exceeded: null }, + ]); + assert.deepEqual(exceededPackBudgets(runtime).map(pack => pack.id), ['pack-a']); +}); + +test('suite timing rejects inconsistent measurements', () => { + const packRuntime = measureGradePackRuntime(report()); + const first = packRuntime.packs[0]; + assert(first); + first.measuredRuntimeMs += 1; + assert.throws(() => aggregatePackRuntime([{ packRuntime }], [ + { id: 'pack-a', budget: { status: 'unmeasured' } }, + { id: 'pack-b', budget: { status: 'unmeasured' } }, + ]), /must equal setupRuntimeMs \+ criterionRuntimeMs/); +}); diff --git a/tools/stack-bench/tests/package-registry.test.ts b/tools/stack-bench/tests/package-registry.test.ts new file mode 100644 index 00000000000..71394192eeb --- /dev/null +++ b/tools/stack-bench/tests/package-registry.test.ts @@ -0,0 +1,40 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { packageRegistry, packageRegistryEnvironment } + from '../src/runtime/package-registry.js'; +import type { BackendLeaseNetwork } from '../src/runtime/backend-lease.js'; + +test('the package registry cache is optional and must be a loopback http origin', () => { + assert.equal(packageRegistry({}), null); + assert.equal(packageRegistry({ STACK_BENCH_NPM_REGISTRY: 'http://127.0.0.1:4873/' })?.href, + 'http://127.0.0.1:4873/'); + for (const value of ['registry', 'https://127.0.0.1:4873/', 'http://registry.npmjs.org/', + 'http://127.0.0.1/', 'http://127.0.0.1:4873/npm/', 'http://user:pw@127.0.0.1:4873/']) { + assert.throws(() => packageRegistry({ STACK_BENCH_NPM_REGISTRY: value }), /STACK_BENCH_NPM_REGISTRY/, value); + } +}); + +test('coding containers reach the cache through the address their network mode allows', () => { + const registry = packageRegistry({ STACK_BENCH_NPM_REGISTRY: 'http://127.0.0.1:4873/' }); + assert.deepEqual(packageRegistryEnvironment(registry, 'host'), + { NPM_CONFIG_REGISTRY: 'http://127.0.0.1:4873/' }); + assert.deepEqual(packageRegistryEnvironment(registry, 'bridge'), + { NPM_CONFIG_REGISTRY: 'http://host.docker.internal:4873/' }); + assert.deepEqual(packageRegistryEnvironment(null, 'host'), {}); +}); + +test('owned attempts use the same registry origin across fresh cache IP addresses', () => { + const network: BackendLeaseNetwork = { name: 'attempt', id: 'a'.repeat(64), + namespaceContainerId: 'b'.repeat(64), cacheContainerId: 'c'.repeat(64), + hostAddresses: ['172.20.0.1'], services: [], firewallSha256: null, firewallInstalledAt: null }; + for (const address of ['172.20.0.3', '192.168.48.3']) { + network.services = [{ address, port: 4873 }]; + assert.deepEqual(packageRegistryEnvironment(null, `container:${network.namespaceContainerId}`, network), + { NPM_CONFIG_REGISTRY: 'http://stack-bench-npm-cache:4873/' }); + } + assert.throws(() => packageRegistryEnvironment(null, null, { ...network, cacheContainerId: undefined }), + /authenticated cache connection/); + assert.throws(() => packageRegistryEnvironment(null, null, { ...network, services: [] }), + /authenticated cache connection/); +}); diff --git a/tools/stack-bench/tests/page-failure.test.ts b/tools/stack-bench/tests/page-failure.test.ts new file mode 100644 index 00000000000..c9ab09df4d8 --- /dev/null +++ b/tools/stack-bench/tests/page-failure.test.ts @@ -0,0 +1,83 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { isFinding } from '../src/actions/action-findings.js'; +import { browserApplicationBoundary, pageFailure } from '../src/actions/browser-action-executors.js'; +import type { Finding } from '../src/actions/action-findings.js'; +import { createActionRegistry, executeAction } from '../src/actions/action-contract.js'; +import { chromium } from 'playwright'; + +function findingOf(message: string): Finding { + const failure = pageFailure(message); + assert(isFinding(failure.details.finding)); + return failure.details.finding; +} + +test('a locator timeout names the awaited control, not its container', () => { + const message = 'locator.waitFor: Timeout 5000ms exceeded.\nCall log:\n' + + ' - waiting for locator(\'[data-role="category-row"],#category-row\').filter({ hasText: \'Audio\' })' + + '.first().locator(\'[data-role="category-units"],#category-units\').first() to be visible\n'; + const finding = findingOf(message); + assert.equal(finding.kind, 'page-timeout'); + assert.equal(finding.fields.control, 'category-units'); + assert.equal(pageFailure(message).message, 'the category-units control inside the category-row control did not become available in time'); + + // A timeout without a named control stays a page timeout. + const unnamed = findingOf('page.goto: Timeout 30000ms exceeded.'); + assert.equal(unnamed.kind, 'page-timeout'); + assert.equal(unnamed.fields.control, undefined); + assert.equal(pageFailure('page.goto: Timeout 30000ms exceeded.').message, 'the page did not respond in time'); + + // Alternative controls are not reported as nested controls. + for (const operation of ['or', 'and']) { + const alternative = `locator.waitFor: Timeout waiting for locator('[data-role="signup-username"]').${operation}(locator('[data-role="signup-toggle"]'))`; + const failure = pageFailure(alternative); + assert.doesNotMatch(failure.message, /inside/); + if (operation === 'or') assert.match(failure.message, /signup-username, signup-toggle/); + else assert.doesNotMatch(failure.message, /signup-toggle|signup-username/); + const finding = findingOf(alternative); + assert.equal(finding.kind, 'page-timeout'); + if (finding.kind === 'page-timeout') assert.equal(finding.fields.scope, undefined); + } +}); + + +test('scoped failures expose the parent control without probe text or raw errors', async () => { + const message = 'locator.fill: Timeout waiting for [data-role="restock-input"] SECRET_PROBE'; + const run = browserApplicationBoundary(async (_args: { scope: string }) => { + throw Object.assign(new Error(message), { name: 'TimeoutError' }); + }, args => args.scope); + await assert.rejects(run({ scope: 'admin-location-row' }), error => { + assert(error instanceof Error); + assert.equal(error.message, + 'the restock-input control inside the admin-location-row control did not become available in time'); + assert(!error.message.includes('SECRET_PROBE')); + return true; + }); + for (const message of ['locator.selectOption: failed', 'locator.click: element intercepts pointer events']) { + assert(pageFailure(message, 'admin-location-row').message.includes('inside the admin-location-row control')); + } +}); + + +test('real browser control failures score, but grader selector and script errors do not', async () => { + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage(); + page.setDefaultTimeout(100); + await page.setContent(''); + for (const [operation, status] of [ + [() => page.locator('#missing').click(), 'failed'], + [() => page.locator('button').fill('value'), 'failed'], + [() => page.locator('[').count(), 'harness_failure'], + [() => page.evaluate(() => { throw new Error('broken grader observation'); }), 'harness_failure'], + [() => page.keyboard.press('NotAKey'), 'harness_failure'], + ] as const) { + const registry = createActionRegistry([{ id: 'probe', version: '1', category: 'browser-observation', + capabilities: [], sensitivity: [], timeoutMs: 5000, compile: input => input, + execute: browserApplicationBoundary(async () => { await operation(); }) }]); + const result = await executeAction(registry, 'probe', {}, {}); + assert.equal(result.status, status, result.summary ?? undefined); + } + } finally { await browser.close(); } +}); diff --git a/tools/stack-bench/tests/password-boundary.integration.ts b/tools/stack-bench/tests/password-boundary.integration.ts new file mode 100644 index 00000000000..cbd68e07d4e --- /dev/null +++ b/tools/stack-bench/tests/password-boundary.integration.ts @@ -0,0 +1,94 @@ +import assert from 'node:assert/strict'; +import { once } from 'node:events'; +import { readFileSync, writeFileSync } from 'node:fs'; +import { createServer } from 'node:http'; +import { join } from 'node:path'; +import test from 'node:test'; +import { chromium } from 'playwright'; +import { gradeFeature } from '../grader/grade.js'; +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; + +// Failure cases, defined before the action change: full-byte verification and +// explicit refusal above 72 bytes pass. Truncation, accepting every password, +// rejecting ordinary registration/login, and creating an account despite a +// registration refusal fail. Refusal must not leave a session after reload. +test('password scenario grades accepted bytes and safe unsupported-input refusal', async t => { + let mode = ''; + const accounts = new Map(); + const calls: { path: string; bytes: number; accepted: boolean }[] = []; + const evidence: unknown[] = []; + const server = createServer(async (req, res) => { + const path = new URL(req.url!, 'http://fixture.test').pathname; + if (path === '/signup' || path === '/signin') { + let raw = ''; for await (const chunk of req) raw += String(chunk); + const { username, password } = JSON.parse(raw) as { username: string; password: string }; + const bytes = Buffer.byteLength(password); + const stored = mode === 'truncation' ? Buffer.from(password).subarray(0, 72).toString('hex') : password; + const refusal = mode === 'reject-all' || bytes > 72 && mode.startsWith('refusal'); + let accepted: boolean; + if (path === '/signup') { + if (!refusal || mode === 'refusal-creates-account') accounts.set(username, stored); + accepted = !refusal; + } else accepted = mode !== 'reject-login' && accounts.has(username) + && (mode === 'any-password' || accounts.get(username) === stored); + calls.push({ path, bytes, accepted }); + if (accepted || mode === 'refusal-keeps-session' && bytes > 72) { + res.setHeader('Set-Cookie', `session=${encodeURIComponent(username)}; HttpOnly; Path=/`); + } + res.writeHead(accepted ? 200 : 400, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(accepted ? { user: username } : { error: 'Credentials refused' })); + return; + } + const user = decodeURIComponent(/(?:^|;\s*)session=([^;]*)/.exec(req.headers.cookie ?? '')?.[1] ?? ''); + res.writeHead(200, { 'Content-Type': 'text/html' }); + res.end(`
+ + +
`); + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const browser = await chromium.launch({ headless: true }); + try { + for (const [variant, expected] of [ + ['full-password', 'passed'], ['refusal-safe', 'passed'], + ['truncation', 'failed'], ['any-password', 'failed'], + ['reject-all', 'failed'], ['reject-login', 'failed'], + ['refusal-creates-account', 'failed'], ['refusal-keeps-session', 'failed'], + ]) await t.test(variant!, async () => { + mode = variant!; accounts.clear(); calls.length = 0; + const scenario = compileScenarioDefinition(JSON.parse(readFileSync(join(STACK_BENCH_ROOT, + 'tracks/ecommerce/scenarios/01-account-password.json'), 'utf8'))); + const grade = await gradeFeature(browser, scenario.features[0]!, + { url, level: 1, headed: false, selectedCheckKeys: [], nullControl: false }, + { url, runId: `password-${variant}`, roomName: name => name, actions: [], spacetime: null, + backend: 'postgres', nullControl: false, defaultWithin: 200 }); + evidence.push({ variant, grade, calls: [...calls] }); + assert.equal(grade.criteria[0]!.evidence.status, expected, JSON.stringify(evidence.at(-1))); + if (expected === 'passed') { + assert(calls.some(call => call.path === '/signin' && call.bytes <= 72 && call.accepted), + 'ordinary account must work before safe refusal can earn credit'); + assert(calls.some(call => call.path === '/signup' && call.bytes === 74), + 'the truncation boundary must reach the application'); + } + }); + } finally { + await browser.close(); server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + if (process.env.STACK_BENCH_PASSWORD_BOUNDARY_EVIDENCE) { + writeFileSync(process.env.STACK_BENCH_PASSWORD_BOUNDARY_EVIDENCE, + `${JSON.stringify({ schemaVersion: 1, modelCalls: 0, evidence }, null, 2)}\n`); + } + } +}); diff --git a/tools/stack-bench/tests/path-safety.test.ts b/tools/stack-bench/tests/path-safety.test.ts new file mode 100644 index 00000000000..4ccaaa7492f --- /dev/null +++ b/tools/stack-bench/tests/path-safety.test.ts @@ -0,0 +1,49 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, + writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; + +import { assertNewOrEmptyDirectory } from '../src/runtime/path-safety.js'; + +test('new and empty output directories are accepted', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-path-')); + try { + const missing = join(root, 'missing'); + assert.equal(assertNewOrEmptyDirectory(missing, 'output'), missing); + const empty = join(root, 'empty'); + mkdirSync(empty); + assert.equal(assertNewOrEmptyDirectory(empty, 'output'), empty); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('existing content is refused and left untouched', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-path-')); + try { + const target = join(root, 'owned-by-caller'); + mkdirSync(target); + const sentinel = join(target, 'keep.txt'); + writeFileSync(sentinel, 'keep me'); + assert.throws(() => assertNewOrEmptyDirectory(target, 'output'), /refusing to overwrite/); + assert.equal(existsSync(sentinel), true); + assert.equal(readFileSync(sentinel, 'utf8'), 'keep me'); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('symlink destinations are refused even when their target is empty', { skip: process.platform === 'win32' }, () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-path-')); + try { + const target = join(root, 'target'); + const link = join(root, 'link'); + mkdirSync(target); + symlinkSync(target, link, 'dir'); + assert.throws(() => assertNewOrEmptyDirectory(link, 'output'), /real directory/); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/tools/stack-bench/tests/phase-timing.test.ts b/tools/stack-bench/tests/phase-timing.test.ts new file mode 100644 index 00000000000..3e2657d10b1 --- /dev/null +++ b/tools/stack-bench/tests/phase-timing.test.ts @@ -0,0 +1,24 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { measurePhase, type PhaseTiming } from '../src/evidence/phase-timing.js'; +import { createArtifact } from '../src/evidence/artifact-schema.js'; + +test('phase measurement preserves results and failures and records both', async () => { + const timings: PhaseTiming[] = []; + const failure = new Error('reset failed'); + assert.equal(await measurePhase(timings, 'reset', null, () => false), false); + await assert.rejects(measurePhase(timings, 'grader', 'check-1', async () => { + throw failure; + }), error => error === failure); + assert.deepEqual(timings.map(({ phase, suite, threw }) => ({ phase, suite, threw })), [ + { phase: 'reset', suite: null, threw: false }, + { phase: 'grader', suite: 'check-1', threw: true }, + ]); + assert.ok(timings.every(timing => Number.isFinite(timing.durationMs) && timing.durationMs >= 0)); + const artifact = (phaseTimings: unknown) => createArtifact({ kind: 'grade_bundle', + id: 'timing-check', payload: { suites: {}, phaseTimings } }); + assert.doesNotThrow(() => artifact(timings)); + for (const durationMs of [-1, NaN, Infinity, '1']) { + assert.throws(() => artifact([{ ...timings[0], durationMs }]), /phaseTimings/); + } +}); diff --git a/tools/stack-bench/tests/platform-ports.test.ts b/tools/stack-bench/tests/platform-ports.test.ts new file mode 100644 index 00000000000..0b94b418850 --- /dev/null +++ b/tools/stack-bench/tests/platform-ports.test.ts @@ -0,0 +1,47 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import childProcess from 'node:child_process'; +import { syncBuiltinESMExports } from 'node:module'; + +test('port probes retain strict failure handling through the shared Linux fallback', async t => { + const platform = Object.getOwnPropertyDescriptor(process, 'platform')!; + let probe: typeof import('../src/runtime/platform.js'); + try { + // Exercise the Linux branch on Windows without running host commands. + Object.defineProperty(process, 'platform', { value: 'linux' }); + probe = await import(new URL('../src/runtime/platform.js?port-test', import.meta.url).href); + } finally { + Object.defineProperty(process, 'platform', platform); + } + const failed = (status: number, stdout = '') => Object.assign(new Error('probe failed'), { status, stdout }); + let lsof: string | Error; + let ss: string | Error; + const calls: string[] = []; + const command = t.mock.method(childProcess, 'execFileSync', (name: string) => { + calls.push(name); + assert.ok(name === 'lsof' || name === 'ss'); + const result = name === 'lsof' ? lsof : ss; + if (result instanceof Error) throw result; + return result; + }); + syncBuiltinESMExports(); + try { + lsof = failed(2, '999\n'); + ss = 'LISTEN 0 10 [::]:7331 [::]:* users:(("app",pid=42,fd=3))\n' + + 'LISTEN 0 10 0.0.0.0:7332 0.0.0.0:* users:(("other",pid=99,fd=3))'; + assert.deepEqual(probe.pidsOnPort(7331, { strict: true }), ['42']); + assert.deepEqual(calls, ['lsof', 'ss']); + ss = failed(127); + assert.throws(() => probe.pidsOnPort(7331, { strict: true }), /could not inspect listeners/); + assert.deepEqual(probe.pidsOnPort(7331), ['999']); + lsof = failed(1); + assert.deepEqual(probe.pidsOnPort(7331, { strict: true }), []); + lsof = '42\n42\n'; + calls.length = 0; + assert.deepEqual(probe.pidsOnPort(7331, { strict: true }), ['42']); + assert.deepEqual(calls, ['lsof']); + } finally { + command.mock.restore(); + syncBuiltinESMExports(); + } +}); diff --git a/tools/stack-bench/tests/platform-process-tree.test.ts b/tools/stack-bench/tests/platform-process-tree.test.ts new file mode 100644 index 00000000000..ad373d245d8 --- /dev/null +++ b/tools/stack-bench/tests/platform-process-tree.test.ts @@ -0,0 +1,27 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { processTreePids } from '../src/runtime/platform.js'; + +test('process trees are ordered deepest-child-first for safe teardown', () => { + const rows = ` + 10 1 + 11 10 + 12 10 + 13 11 + 14 13 + 20 1 + `; + assert.deepEqual(processTreePids(10, rows), [14, 13, 11, 12, 10]); +}); + +test('malformed, unrelated, and cyclic process rows cannot broaden teardown', () => { + const rows = ` + not-a-row + 31 30 + 30 31 + 40 1 + `; + assert.deepEqual(processTreePids(30, rows), [31, 30]); + assert.deepEqual(processTreePids(0, rows), []); +}); diff --git a/tools/stack-bench/tests/port-windows.test.ts b/tools/stack-bench/tests/port-windows.test.ts new file mode 100644 index 00000000000..609db40d40a --- /dev/null +++ b/tools/stack-bench/tests/port-windows.test.ts @@ -0,0 +1,39 @@ +import { RESTRICTED_PORTS } from '../src/composition/product-config.js'; +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { RUN_INDEX_CAP, listTracks, loadTrack, + portsFor } from '../src/composition/tracks.js'; +import { STACK_IDS } from '../src/stacks/stack-identities.js'; + +test('dynamic indices validate actual ports, including browser restrictions and TCP bounds', () => { + for (const name of listTracks({ includeInternal: true })) { + const track = loadTrack(name); + for (const backend of STACK_IDS) { + const ports = portsFor(track, backend, 30); + for (const port of [ports.vite, ports.express]) { + if (port != null) assert(!RESTRICTED_PORTS.has(port)); + } + assert.throws(() => portsFor(track, backend, RUN_INDEX_CAP), /TCP port/); + } + } + assert.throws(() => portsFor(loadTrack('chat'), 'mongodb', 256), /6679/); +}); + +test('Supabase application ports stay clear of every other stack and of its platform services', () => { + const tracks = listTracks({ includeInternal: true }).map(name => loadTrack(name)); + // Indices whose ports a browser refuses are never assigned. + const assigned = (track: ReturnType, backend: string, runIndex: number) => { + try { return portsFor(track, backend, runIndex); } catch { return null; } + }; + const window = (backend: string) => new Set(tracks.flatMap(track => Array.from({ length: 64 }, (_, runIndex) => { + const ports = assigned(track, backend, runIndex); + return ports ? [ports.vite, ports.express, ports.dbPort].filter((port): port is number => port !== null) : []; + }).flat())); + const supabase = window('supabase'); + for (const backend of STACK_IDS.filter(id => id !== 'supabase')) { + assert.deepEqual([...window(backend)].filter(port => supabase.has(port)), [], backend); + } + // Services listening inside the attempt namespace, and the appliance dashboard. + for (const port of [3000, 3001, 4000, 5000, 5432, 7331, 9000, 9901, 9999]) assert(!supabase.has(port), String(port)); +}); diff --git a/tools/stack-bench/tests/postgres-saved-checkout.test.ts b/tools/stack-bench/tests/postgres-saved-checkout.test.ts new file mode 100644 index 00000000000..434640e3d3b --- /dev/null +++ b/tools/stack-bench/tests/postgres-saved-checkout.test.ts @@ -0,0 +1,89 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { hashAppSource } from '../src/runtime/source-snapshot.js'; +import type { TextCommandExecutor } from '../src/runtime/command-executor.js'; +import { getSavedMongoDbCheckoutState } from '../src/stacks/backends/mongodb-saved-checkout.js'; +import { getSavedPostgresCheckoutState } from '../src/stacks/backends/postgres-saved-checkout.js'; + +const READERS = { postgres: getSavedPostgresCheckoutState, mongodb: getSavedMongoDbCheckoutState } as const; + +function fixture(backend: keyof typeof READERS = 'postgres') { + const root = mkdtempSync(join(tmpdir(), `saved-${backend}-reader-`)), app = join(root, 'app'); + mkdirSync(app); writeFileSync(join(app, 'server.js'), 'source'); + const sourceSha256 = hashAppSource(app).sha256; + const mapping = JSON.stringify(backend === 'postgres' + ? { sourceSha256, sql: "SELECT :'account', :'item'" } : { sourceSha256, script: 'return {};' }); + const reader = { path: join(root, 'reader.json'), sha256: createHash('sha256').update(mapping).digest('hex') }; + writeFileSync(reader.path, mapping); + const state = { accountId: 'buyer', itemId: 'keyboard', priceMinor: 8900, cart: [], + stock: [{ warehouseId: 'east', quantity: 10 }], reservations: [], payments: [], orphanOrderLines: 0, + orphanAllocations: 0, orders: [{ id: 'old', accountId: 'buyer', totalMinor: 8900, + refundedMinor: 0, status: 'pending', lines: [{ itemId: 'keyboard', quantity: 1, priceMinor: 8900, + allocations: [{ warehouseId: 'east', quantity: 1 }] }] }] }; + return { root, sourceSha256, state, args: { app, reader, account: "buyer'\\name", item: 'Keyboard', + lease: { resources: { container: { id: 'owned-id', name: 'owned-name' }, database: 'owned-db' } } } }; +} + +test('saved reader uses the exact owned container and one read-only repeatable snapshot with psql variables', t => { + const f = fixture(); t.after(() => rmSync(f.root, { recursive: true, force: true })); + let queries = 0; + const exec: TextCommandExecutor = (file, args, options) => { + assert.equal(file, 'docker'); + if (args[0] === 'inspect') { assert.equal(args.at(-1), 'owned-name'); return 'owned-id'; } + queries++; + assert.deepEqual(args.slice(0, 6), ['exec', '-i', 'owned-id', 'psql', '-X', '-qAt']); + assert(args.includes('owned-db')); assert(args.includes('ON_ERROR_STOP=1')); + assert(args.includes(`account=${f.args.account}`)); assert(args.includes('item=Keyboard')); + const sql = String(options.input); + assert.match(sql, /^BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY;/); + assert.match(sql, /SELECT :'account', :'item'/); assert.match(sql, /COMMIT;\n$/); + assert(!sql.includes(f.args.account)); + return JSON.stringify({ accountMatches: 1, itemMatches: 1, state: f.state }); + }; + const result = getSavedPostgresCheckoutState({ ...f.args, exec }); + assert.equal(queries, 1); assert.deepEqual(result.state, f.state); + assert.deepEqual(result.schemaSha256, { source: f.sourceSha256, reader: f.args.reader.sha256 }); + assert.equal(result.scope, 'orders'); +}); + +test('reader and saved source mismatches fail before querying; changed container ownership prevents reads', t => { + for (const backend of ['postgres', 'mongodb'] as const) { + const f = fixture(backend); t.after(() => rmSync(f.root, { recursive: true, force: true })); + const read = READERS[backend]; + const noExec: TextCommandExecutor = () => { throw new Error('must not execute'); }; + assert.throws(() => read({ ...f.args, + reader: { ...f.args.reader, sha256: '0'.repeat(64) }, exec: noExec }), /reader hash mismatch/); + writeFileSync(join(f.args.app, 'server.js'), 'changed'); + assert.throws(() => read({ ...f.args, exec: noExec }), /source hash mismatch/); + writeFileSync(join(f.args.app, 'server.js'), 'source'); + const wrongOwner: TextCommandExecutor = (_file, args) => { + assert.equal(args[0], 'inspect'); return 'replacement-id'; + }; + assert.throws(() => read({ ...f.args, exec: wrongOwner }), /changed after lease creation/); + } +}); + +test('reader rejects missing or ambiguous entities, omitted accounting and fabricated payment state', t => { + for (const backend of ['postgres', 'mongodb'] as const) { + const f = fixture(backend); t.after(() => rmSync(f.root, { recursive: true, force: true })); + const read = READERS[backend]; + const valid = { accountMatches: 1, itemMatches: 1, state: f.state }; + for (const value of [ + { ...valid, accountMatches: 0 }, { ...valid, itemMatches: 2 }, + { ...valid, state: { ...f.state, orphanAllocations: undefined } }, + { ...valid, state: { ...f.state, orders: [{ ...f.state.orders[0], refundedMinor: undefined }] } }, + { ...valid, state: { ...f.state, payments: [{ id: 'made-up', orderId: 'old', amountMinor: 8900, status: 'paid' }] } }, + ]) { + const exec: TextCommandExecutor = (_file, args) => args[0] === 'inspect' ? 'owned-id' : JSON.stringify(value); + assert.throws(() => read({ ...f.args, exec })); + } + const reservations = [{ itemId: 'keyboard', warehouseId: 'east', quantity: 1 }]; + const exec: TextCommandExecutor = (_file, args) => args[0] === 'inspect' ? 'owned-id' + : JSON.stringify({ ...valid, state: { ...f.state, reservations } }); + assert.deepEqual(read({ ...f.args, exec }).state.reservations, reservations); + } +}); diff --git a/tools/stack-bench/tests/preflight.test.ts b/tools/stack-bench/tests/preflight.test.ts new file mode 100644 index 00000000000..e3bc1f0a3ac --- /dev/null +++ b/tools/stack-bench/tests/preflight.test.ts @@ -0,0 +1,713 @@ +import assert from 'node:assert/strict'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import test from 'node:test'; +import { runInNewContext } from 'node:vm'; +import { runContainerSmoke } from '../src/runtime/container-smoke.js'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { parsePreflightArgs } from '../commands/preflight-cli.js'; +import { credentialReady, probeLoopbackPort, runPreflight, verifyPostgresServiceIdentity } + from '../src/runtime/preflight.js'; +import { AGENT_ADAPTER_REGISTRY } from '../src/agents/agent-adapters.js'; +import type { PreflightCheck, PreflightReport } from '../src/runtime/preflight.js'; +import { isExactImageReference } from '../src/runtime/container-image.js'; +import { createArtifact, validateArtifact } from '../src/evidence/artifacts.js'; +import { compileCampaignFile, validateCampaignDefinition } + from '../src/campaigns/campaign-compiler.js'; +import { PREFLIGHT_RESOURCE_FLOORS } from '../src/composition/product-config.js'; +import { createBackendLease, writeBackendLease } from '../src/runtime/backend-lease.js'; +import { SUPERVISOR_STATE_VERSION } from '../src/runtime/recovery.js'; + +const IMAGE_ID = `sha256:${'a'.repeat(64)}`; +const EXACT_IMAGE = `registry.example/stack-bench/build@${IMAGE_ID}`; + +test('preflight requires only the selected provider route and keeps its failures visible', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-provider-smoke-')); + try { + const authFile = join(root, 'auth.json'); + writeFileSync(authFile, JSON.stringify({ auth_mode: 'chatgpt', tokens: { + account_id: 'synthetic-account', access_token: `test.${Buffer.from(JSON.stringify({ exp: 4102444800 })).toString('base64url')}.test`, + } })); + for (const [adapter, env, origin] of [ + ['codex', { CODEX_AUTH_FILE: authFile }, 'https://chatgpt.com'], + ['codex', { OPENAI_API_KEY: 'synthetic-api-key' }, 'https://api.openai.com'], + ['claude-code', { ANTHROPIC_API_KEY: 'synthetic-api-key' }, 'https://api.anthropic.com'], + ['openrouter', { OPENROUTER_API_KEY: 'synthetic-api-key' }, 'https://openrouter.ai'], + ] as const) { + for (const unavailable of [false, true]) { + const report = runPreflight(request(root, ['--smoke', '--agent-adapter', adapter, + ...(adapter === 'openrouter' ? ['--provider-route', 'openai', '--max-output-tokens', '8192'] : [])]), { + env, home: root, statfs: () => ({ bavail: 20n, bsize: 1024n ** 3n }), + pidsOnPort: () => [], probePort: () => ({ free: true }), + run: (_file, args) => { + if (args[0] === 'info') return dockerInfo(); + if (args[0] === 'compose') return '2.40.0'; + if (args[0] === 'ps') return ''; + if (args[0] === 'image') return args[3] === '{{.Os}}/{{.Architecture}}' ? 'linux/amd64' : IMAGE_ID; + assert.equal(args[0], 'run'); + const destinations = JSON.parse(requiredArgument(args, args.indexOf('-e') + 2)) as string[]; + assert.deepEqual(destinations.sort(), [origin, 'https://registry.npmjs.org'].sort()); + if (unavailable) throw new Error('selected endpoint unavailable'); + writeFileSync(join(root, requiredArgument(args, args.length - 1)), 'container-write-ok'); + return JSON.stringify({ platform: 'linux', arch: 'x64', node: 'v22.0.0', + reached: destinations.map(url => ({ url, status: 403 })), tcpReached: [], + executables: Object.fromEntries((JSON.parse(requiredArgument(args, args.indexOf('-e') + 4)) as string[]) + .map(name => [name, `/usr/bin/${name}`])), credentialStatus: 'not-checked', diskFreeBytes: 20 * 1024 ** 3 }); + }, + }); + assert.equal(requiredCheck(report, 'smoke.container').status, unavailable ? 'fail' : 'pass', + JSON.stringify(requiredCheck(report, 'smoke.container'))); + assert.doesNotMatch(JSON.stringify(report), /synthetic-api-key|synthetic-account/); + } + } + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('container smoke keeps safe transport causes without error bodies', async () => { + let script = ''; + runContainerSmoke({ command: (_file, args) => { script = requiredArgument(args, args.indexOf('-e') + 1); return '{}'; }, + imageId: IMAGE_ID, resultsDir: '/unused', destinations: [], tcpPorts: [], requiredExecutables: [], + credentialStatusCommand: null, credentialMount: null, credentialEnvironment: null, marker: 'unused', networkMode: 'host' }); + const setTimeout = (resume: () => void) => resume(); + for (const code of ['ENOTFOUND', 'CERT_HAS_EXPIRED', 'ECONNRESET', 'UND_ERR_CONNECT_TIMEOUT']) { + const failure = new Error('private response body', { cause: new AggregateError([ + Object.assign(new Error('private credential'), { code, syscall: 'connect' }), + ], 'private aggregate') }); + let calls = 0; + await assert.rejects(runInNewContext(script, { require: () => ({}), AbortSignal, URL, setTimeout, + process: { argv: ['node', '["https://provider.invalid"]', '[]', '[]', 'null', 'null', 'unused'], env: {} }, + fetch: async () => { calls++; throw failure; }, + }), error => { + const text = String(error); + assert.match(text, new RegExp(code)); + assert.match(text, /AggregateError/); + assert.match(text, /"attempts":3/); + assert.doesNotMatch(text, /private/); + return true; + }); + assert.equal(calls, 3, 'a failed destination is retried before preflight fails'); + } + await assert.rejects(runInNewContext(script, { require: () => ({}), AbortSignal, URL, setTimeout, + process: { argv: ['node', '["https://provider.invalid"]', '[]', '[]', 'null', 'null', 'unused'], env: {} }, + fetch: async () => { throw Object.assign(new Error('private'), { name: 'TimeoutError' }); }, + }), /TimeoutError/); +}); +const progressionCampaign = join(STACK_BENCH_ROOT, 'appliance', + 'campaign.ecommerce-progression-reference.json'); +let progressionPlanCache: ReturnType | null = null; +const progressionPlan = (): ReturnType => + structuredClone(progressionPlanCache ??= compileCampaignFile(progressionCampaign)); + +type DockerCommand = (file: string, args: string[]) => string; + +function dockerInfo(overrides: Record = {}): string { + return JSON.stringify({ OSType: 'linux', Architecture: 'x86_64', ServerVersion: '29.0.0', + NCPU: 8, MemTotal: 16 * 1024 ** 3, SystemTime: '2026-08-12T12:00:00.000Z', ...overrides }); +} + +function request(root: string, extra: string[] = []) { + return parsePreflightArgs(['node', 'preflight.js', '--backend', 'stub', '--track', 'loop', + '--levels', '1', '--agent-adapter', 'deterministic', '--results-dir', root, ...extra]); +} + +function requiredArgument(args: string[], index: number): string { + const value = args[index]; + assert.ok(value, `missing command argument at index ${index}`); + return value; +} + +function requiredCheck(report: PreflightReport, id: string): PreflightCheck { + const check = report.checks.find(candidate => candidate.id === id); + assert.ok(check, `missing preflight check ${id}`); + return check; +} + +function record(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function stringArray(value: unknown, field: string): string[] { + assert.ok(record(value), 'expected evidence object'); + const selected = value[field]; + assert.ok(Array.isArray(selected) && selected.every(item => typeof item === 'string'), + `expected ${field} string array`); + return selected; +} + +test('PostgreSQL preflight proves the configured role, password, and database', () => { + let command: string[] | undefined; + const identity = verifyPostgresServiceIdentity('stack-bench-postgres', { + execute: (file, args) => { + command = [file, ...args]; + return 'appuser|app\n'; + }, + }); + assert.equal(identity, 'appuser|app'); + assert.ok(command); + assert.deepEqual(command.slice(0, 4), + ['docker', 'exec', '-e', 'PGPASSWORD=local-app-password']); + assert(command.includes('stack-bench-postgres')); + assert(command.includes('127.0.0.1')); + assert.throws(() => verifyPostgresServiceIdentity('stack-bench-postgres', { + execute: () => 'stackbench|stackbench\n', + }), /expected appuser\|app/); +}); + +test('preflight validates exact scope and a model-free container/result-volume smoke', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-preflight-')); + try { + let smokeArgs: string[] | undefined; + const run: DockerCommand = (_file, args) => { + if (args[0] === 'info') return dockerInfo(); + if (args[0] === 'compose') return '2.40.0'; if (args[0] === 'ps') return ''; + if (args[0] === 'image') return args[3] === '{{.Os}}/{{.Architecture}}' + ? 'linux/amd64' : `${IMAGE_ID}\n`; + if (args[0] === 'run') { + smokeArgs = args; + const mount = requiredArgument(args, args.indexOf('-v') + 1).split(':/results')[0]; + const marker = requiredArgument(args, args.length - 1); + assert.ok(mount); + writeFileSync(join(mount, marker), 'container-write-ok'); + return JSON.stringify({ platform: 'linux', arch: 'x64', node: 'v22.0.0', + reached: [{ url: 'https://registry.npmjs.org', status: 200 }], + tcpReached: [], + executables: {}, credentialStatus: 'not-checked', + diskFreeBytes: 20 * 1024 ** 3 }); + } + throw new Error(`unexpected docker command: ${args.join(' ')}`); + }; + const report = runPreflight(request(root, ['--smoke']), { + run, now: Date.parse('2026-08-12T12:00:00.100Z'), env: {}, home: root, + statfs: () => ({ bavail: 20n, bsize: 1024n ** 3n }), + pidsOnPort: () => [], probePort: () => ({ free: true }), + }); + assert.equal(report.ok, true, JSON.stringify(report.checks, null, 2)); + assert.equal(report.request.smoke, true); + assert.equal(requiredCheck(report, 'smoke.container').status, 'pass'); + assert.match(requiredCheck(report, 'smoke.container').summary, + /0 database route\(s\); 0 agent executable\(s\)/); + assert.equal(requiredCheck(report, 'storage.container').status, 'pass'); + assert.equal(report.checks.some(check => check.id === 'outbound.container'), false); + assert.ok(smokeArgs); + assert.equal(requiredArgument(smokeArgs, smokeArgs.indexOf('--network') + 1), 'bridge'); + assert.equal(requiredArgument(smokeArgs, smokeArgs.indexOf('--add-host') + 1), + 'host.docker.internal:host-gateway'); + const artifact = createArtifact({ kind: 'preflight', id: 'preflight-test', payload: report }); + assert.equal(validateArtifact(artifact).payload.ok, true); + const regrade = runPreflight({ ...request(root, ['--smoke']), + agentAdapter: 'claude-code', modelFree: true }, { + run, now: Date.parse('2026-08-12T12:00:00.100Z'), env: {}, home: root, + statfs: () => ({ bavail: 20n, bsize: 1024n ** 3n }), + pidsOnPort: () => [], probePort: () => ({ free: true }), + }); + assert.equal(regrade.ok, true, JSON.stringify(regrade.checks)); + assert.equal(requiredCheck(regrade, 'agent.credentials').status, 'pass'); + assert.equal(regrade.checks.some(check => check.id === 'agent.authentication'), false); + assert.ok(smokeArgs); + assert.equal(smokeArgs.some(arg => /anthropic|oauth|claude/.test(arg)), false); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('appliance preflight accepts immutable local and registry images but rejects mutable or malformed references', () => { + assert.equal(isExactImageReference(EXACT_IMAGE), true); + assert.equal(isExactImageReference('stack-bench-controller:latest'), false); + const root = mkdtempSync(join(tmpdir(), 'stack-bench-preflight-images-')); + try { + const run: DockerCommand = (_file, args) => { + if (args[0] === 'info') return dockerInfo(); + if (args[0] === 'compose') return '2.40.0'; if (args[0] === 'ps') return ''; + if (args[0] === 'image') return args[3] === '{{.Os}}/{{.Architecture}}' + ? 'linux/amd64' : `${IMAGE_ID}\n`; + throw new Error(`unexpected docker command: ${args.join(' ')}`); + }; + for (const [controller, build, controllerStatus, buildStatus] of [ + [IMAGE_ID, IMAGE_ID, 'pass', 'pass'], + [EXACT_IMAGE, EXACT_IMAGE, 'pass', 'pass'], + ['stack-bench-controller:latest', IMAGE_ID, 'fail', 'pass'], + [IMAGE_ID, 'stack-bench-build:latest', 'pass', 'fail'], + [IMAGE_ID.slice(0, -1), ` ${IMAGE_ID}`, 'fail', 'fail'], + ] as const) { + const report = runPreflight({ ...request(root), image: build }, { + run, now: Date.parse('2026-08-12T12:00:00.100Z'), + env: { STACK_BENCH_APPLIANCE: '1', STACK_BENCH_CONTROLLER_IMAGE: controller }, + home: root, statfs: () => ({ bavail: 20n, bsize: 1024n ** 3n }), + pidsOnPort: () => [], probePort: () => ({ free: true }), + }); + assert.equal(requiredCheck(report, 'image.controller-reference').status, controllerStatus, controller); + assert.equal(requiredCheck(report, 'image.build-reference').status, buildStatus, build); + } + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('preflight does not count Docker info latency as clock skew', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-preflight-clock-')); + try { + const times = [ + Date.parse('2026-08-12T12:00:00.000Z'), + Date.parse('2026-08-12T12:00:00.100Z'), + Date.parse('2026-08-12T12:00:10.100Z'), + ]; + const run: DockerCommand = (_file, args) => { + if (args[0] === 'info') return dockerInfo({ SystemTime: '2026-08-12T12:00:05.000Z' }); + if (args[0] === 'compose') return '2.40.0'; if (args[0] === 'ps') return ''; + if (args[0] === 'image') return args[3] === '{{.Os}}/{{.Architecture}}' + ? 'linux/amd64' : `${IMAGE_ID}\n`; + throw new Error(`unexpected docker command: ${args.join(' ')}`); + }; + const report = runPreflight(request(root), { + run, now: () => { + const time = times.shift(); + if (time === undefined) { + throw new Error('preflight requested more clock samples than expected'); + } + return time; + }, + env: {}, home: root, + statfs: () => ({ bavail: 20n, bsize: 1024n ** 3n }), + pidsOnPort: () => [], probePort: () => ({ free: true }), + }); + const clock = requiredCheck(report, 'docker.clock'); + assert.equal(clock.status, 'pass', clock.summary); + assert.equal(clock.summary, 'host/engine skew 0 ms'); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('appliance pool caps warn without replacing the single-worker startup baseline', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-preflight-parallel-capacity-')); + try { + let allocation = { NCPU: 32, MemTotal: 46 * 1024 ** 3 }; + const run: DockerCommand = (_file, args) => { + if (args[0] === 'info') return dockerInfo(allocation); + if (args[0] === 'compose') return '2.40.0'; if (args[0] === 'ps') return ''; + if (args[0] === 'image') return args[3] === '{{.Os}}/{{.Architecture}}' + ? 'linux/amd64' : `${IMAGE_ID}\n`; + throw new Error(`unexpected docker command: ${args.join(' ')}`); + }; + const dependencies = { + run, now: Date.parse('2026-08-12T12:00:00.100Z'), + env: { STACK_BENCH_APPLIANCE: '1', + STACK_BENCH_CONTROLLER_IMAGE: IMAGE_ID }, home: root, + statfs: () => ({ bavail: 20n, bsize: 1024n ** 3n }), + pidsOnPort: () => [], probePort: () => ({ free: true }), + }; + const selected = { ...request(root, ['--parallelism', '9']), image: IMAGE_ID }; + const report = runPreflight(selected, dependencies); + assert.equal(report.request.parallelism, 9); + assert.equal(requiredCheck(report, 'docker.cpu').status, 'pass'); + assert.equal(requiredCheck(report, 'docker.memory').status, 'pass'); + assert.match(requiredCheck(report, 'docker.memory').summary, /total memory allocation/); + assert.equal(requiredCheck(report, 'docker.capacity').status, 'warn'); + assert.match(requiredCheck(report, 'docker.capacity').summary, /9-worker.*36 CPUs.*92\.3 GiB/); + assert.equal(requiredCheck(report, 'storage.results').status, 'pass'); + + allocation = { NCPU: 3, MemTotal: 7 * 1024 ** 3 }; + const insufficient = runPreflight(selected, dependencies); + assert.equal(requiredCheck(insufficient, 'docker.cpu').status, 'fail'); + assert.match(requiredCheck(insufficient, 'docker.cpu').remediation ?? '', + new RegExp(`at least ${PREFLIGHT_RESOURCE_FLOORS.cpuCount} CPUs`)); + assert.equal(requiredCheck(insufficient, 'docker.memory').status, 'fail'); + assert.match(requiredCheck(insufficient, 'docker.memory').remediation ?? '', /at least 8\.0 GiB/); + for (const parallelism of [0, -1, Number.NaN, 1.5]) { + assert.throws(() => runPreflight({ ...selected, parallelism }, { + ...dependencies, run: () => assert.fail('invalid parallelism must fail before Docker probes'), + }), /parallelism must be a positive integer/); + } + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('pure preflight creates no containers and does not claim smoke evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-preflight-admitted-')); + try { + const run: DockerCommand = (_file, args) => { + if (args[0] === 'info') return dockerInfo(); + if (args[0] === 'compose') return '2.40.0'; if (args[0] === 'ps') return ''; + if (args[0] === 'image') return args[3] === '{{.Os}}/{{.Architecture}}' + ? 'linux/amd64' : `${IMAGE_ID}\n`; + if (args[0] === 'run') throw new Error('duplicate container smoke ran'); + throw new Error(`unexpected docker command: ${args.join(' ')}`); + }; + const report = runPreflight({ ...request(root), smoke: false }, { + run, now: Date.parse('2026-08-12T12:00:00.100Z'), env: {}, home: root, + statfs: () => ({ bavail: 20n, bsize: 1024n ** 3n }), + pidsOnPort: () => [], probePort: () => ({ free: true }), + }); + assert.equal(report.checks.some(check => check.id === 'smoke.container'), false); + assert.equal(requiredCheck(report, 'outbound.container').status, 'warn'); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('preflight validates campaign-owned dependency selections through the progression graph', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-preflight-progression-')); + try { + const manifest = validateCampaignDefinition( + JSON.parse(readFileSync(progressionCampaign, 'utf8')), + { source: progressionCampaign }, + ); + manifest.id = 'sequential-preflight-proof'; + manifest.mode = { id: 'sequential' }; + manifest.repair = { selection: 'batch', budget: { total: 0 }, order: 'declared' }; + manifest.levels = [1, 2]; + assert.ok(manifest.selection.levels, 'campaign must select progression levels'); + manifest.selection.levels = manifest.selection.levels.slice(0, 2); + const path = join(root, 'campaign.json'); + writeFileSync(path, `${JSON.stringify(manifest, null, 2)}\n`); + for (const plan of [progressionPlan(), compileCampaignFile(path)]) { + const report = runPreflight({ + backends: ['mongodb'], + track: plan.definition.track, + levelList: plan.definition.levels, + runIndex: 0, + agentAdapter: 'reference-fixture', + guidance: plan.conditions[0].guidance.mode, + packIds: [], + checkKeys: [], + requestedScopes: plan.conditions.map(condition => condition.requested), + featureCatalog: plan.featureCatalog, + mode: plan.definition.mode, + smoke: false, + image: 'unavailable-in-focused-test', + resultsDir: root, + }, { + run: () => { throw new Error('Docker unavailable in this focused test'); }, + env: {}, home: root, + statfs: () => ({ bavail: 20n, bsize: 1024n ** 3n }), + pidsOnPort: () => [], probePort: () => ({ free: true }), + }); + const scope = requiredCheck(report, 'request.scope'); + assert.equal(scope.status, 'pass', scope.summary); + } + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('preflight fails before a paid run when the selected agent executable is absent', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-preflight-agent-')); + try { + const selected = parsePreflightArgs(['node', 'preflight.js', '--backend', 'stub', + '--track', 'loop', '--levels', '1', '--agent-adapter', 'claude-code', + '--results-dir', root, '--smoke']); + let requestedExecutables: unknown; + const run: DockerCommand = (_file, args) => { + if (args[0] === 'info') return dockerInfo(); + if (args[0] === 'compose') return '2.40.0'; if (args[0] === 'ps') return ''; + if (args[0] === 'image') return args[3] === '{{.Os}}/{{.Architecture}}' + ? 'linux/amd64' : `${IMAGE_ID}\n`; + if (args[0] === 'run') { + const mount = requiredArgument(args, args.indexOf('-v') + 1).split(':/results')[0]; + assert.ok(mount); + writeFileSync(join(mount, requiredArgument(args, args.length - 1)), 'container-write-ok'); + requestedExecutables = JSON.parse(requiredArgument(args, args.length - 4)); + return JSON.stringify({ platform: 'linux', arch: 'x64', node: 'v22.0.0', + reached: [], tcpReached: [], executables: {}, credentialStatus: 'not-checked', + diskFreeBytes: 20 * 1024 ** 3 }); + } + throw new Error(`unexpected docker command: ${args.join(' ')}`); + }; + const report = runPreflight(selected, { run, now: Date.parse('2026-08-12T12:00:00.100Z'), + env: { ANTHROPIC_API_KEY: '' }, home: root, + statfs: () => ({ bavail: 20n, bsize: 1024n ** 3n }), pidsOnPort: () => [], + probePort: () => ({ free: true }) }); + assert.deepEqual(requestedExecutables, ['claude']); + assert.equal(report.ok, false); + assert.equal(requiredCheck(report, 'smoke.container').status, 'fail'); + assert.doesNotMatch(JSON.stringify(report), /test-present/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('expired OpenAI account credentials fail preflight without exposing the token', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-preflight-codex-')); + try { + const codexHome = join(root, 'codex'); + const claudeHome = join(root, 'claude'); + mkdirSync(codexHome); + mkdirSync(join(claudeHome, '.claude'), { recursive: true }); + const codexAuth = join(codexHome, 'auth.json'); + const token = 'header.eyJleHAiOjF9.signature'; + writeFileSync(codexAuth, JSON.stringify({ auth_mode: 'chatgpt', + tokens: { access_token: token, account_id: 'account' } })); + const rotating = 'rotating-interactive-secret'; + writeFileSync(join(claudeHome, '.claude', '.credentials.json'), + JSON.stringify({ claudeAiOauth: { accessToken: rotating } })); + for (const { adapter, home, env, secret, summary } of [ + { adapter: 'codex', home: codexHome, env: { CODEX_AUTH_FILE: codexAuth }, secret: token, summary: /expired/ }, + { adapter: 'claude-code', home: claudeHome, env: {}, secret: rotating, + summary: /No declared credential source/ }, + ]) { + const selected = parsePreflightArgs(['node', 'preflight.js', '--backend', 'stub', + '--track', 'loop', '--levels', '1', '--agent-adapter', adapter, '--results-dir', home]); + const report = runPreflight(selected, { run: (_file, args) => { + if (args[0] === 'info') return dockerInfo(); + if (args[0] === 'compose') return '2.40.0'; + if (args[0] === 'ps') return ''; + if (args[0] === 'image') return args[3] === '{{.Os}}/{{.Architecture}}' ? 'linux/amd64' : `${IMAGE_ID}\n`; + throw new Error('unexpected Docker command'); + }, now: Date.parse('2026-08-12T12:00:00.100Z'), env, home, + statfs: () => ({ bavail: 20n, bsize: 1024n ** 3n }), pidsOnPort: () => [], + probePort: () => ({ free: true }) }); + assert.equal(report.ok, false, adapter); + assert.equal(requiredCheck(report, 'agent.credentials').status, 'fail', adapter); + assert.match(requiredCheck(report, 'agent.credentials').summary, summary); + assert.doesNotMatch(JSON.stringify(report), new RegExp(secret)); + } + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('long-lived subscription token is mounted read-only and checked without exposing its value', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-preflight-subscription-token-')); + try { + const tokenFile = join(root, 'claude-token'); + writeFileSync(tokenFile, 'must-not-appear-in-report\n'); + const selected = parsePreflightArgs(['node', 'preflight.js', '--backend', 'stub', + '--track', 'loop', '--levels', '1', '--agent-adapter', 'claude-code', + '--results-dir', root, '--smoke']); + let dockerArgs: string[] | undefined; + const run: DockerCommand = (_file, args) => { + if (args[0] === 'info') return dockerInfo(); + if (args[0] === 'compose') return '2.40.0'; if (args[0] === 'ps') return ''; + if (args[0] === 'image') return args[3] === '{{.Os}}/{{.Architecture}}' + ? 'linux/amd64' : `${IMAGE_ID}\n`; + if (args[0] === 'run') { + dockerArgs = args; + const mount = requiredArgument(args, args.indexOf('-v') + 1).split(':/results')[0]; + assert.ok(mount); + writeFileSync(join(mount, requiredArgument(args, args.length - 1)), 'container-write-ok'); + return JSON.stringify({ platform: 'linux', arch: 'x64', node: 'v22.0.0', reached: [], + tcpReached: [], executables: { claude: '/usr/local/bin/claude' }, + credentialStatus: 'ready', diskFreeBytes: 20 * 1024 ** 3 }); + } + throw new Error(`unexpected docker command: ${args.join(' ')}`); + }; + const report = runPreflight(selected, { run, now: Date.parse('2026-08-12T12:00:00.100Z'), + env: { CLAUDE_CODE_OAUTH_TOKEN_FILE: tokenFile }, home: root, + statfs: () => ({ bavail: 20n, bsize: 1024n ** 3n }), pidsOnPort: () => [], + probePort: () => ({ free: true }) }); + assert.equal(report.ok, true, JSON.stringify(report.checks, null, 2)); + assert.equal(requiredCheck(report, 'agent.authentication').status, 'pass'); + assert.ok(dockerArgs); + assert.match(requiredArgument(dockerArgs, dockerArgs.indexOf('--mount') + 1), + /src=.*claude-token,dst=\/run\/secrets\/agent-credential,readonly$/); + assert.deepEqual(JSON.parse(requiredArgument(dockerArgs, dockerArgs.length - 2)), { + name: 'CLAUDE_CODE_OAUTH_TOKEN', file: '/run/secrets/agent-credential', + }); + assert.doesNotMatch(JSON.stringify({ report, dockerArgs }), /must-not-appear-in-report/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('preflight fails before launch when a stack default skill document is absent', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-preflight-materials-')); + try { + for (const [guidance, missing, skills] of [ + ['prescribed', 'typescript-server', ['typescript-server', 'typescript-client', 'cli']], + ['neutral', 'cli', ['typescript-server', 'typescript-client', 'cli']], + ] as const) { + const selected = parsePreflightArgs(['node', 'preflight.js', '--backend', 'spacetime', + '--track', 'ecommerce', '--levels', '1', '--agent-adapter', 'claude-code', + '--guidance', guidance, '--results-dir', root]); + const report = runPreflight(selected, { + run: () => { throw new Error('Docker unavailable in this focused test'); }, + env: { ANTHROPIC_API_KEY: '' }, home: root, + exists: path => !String(path).includes(`${join('skills', missing)}`), + statfs: () => ({ bavail: 20n, bsize: 1024n ** 3n }), pidsOnPort: () => [], + probePort: () => ({ free: true }), + }); + const materials = requiredCheck(report, 'materials.spacetime.skills'); + assert.equal(materials.status, 'fail'); + assert.deepEqual(stringArray(materials.evidence, 'skills'), skills); + assert.equal(stringArray(materials.evidence, 'missing').length, 1); + } + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('preflight fails closed on inherited ownership, unavailable Docker, and occupied ports', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-preflight-')); + try { + const report = runPreflight(request(root), { + run: () => { throw new Error('daemon unavailable'); }, + now: Date.parse('2026-08-12T12:00:00Z'), + env: { STACK_BENCH_LEASE_TOKEN: 'must-not-be-reported' }, home: root, + pidsOnPort: () => ['4242'], probePort: () => ({ free: false }), + }); + assert.equal(report.ok, false); + assert.equal(requiredCheck(report, 'ambient.stack_bench_lease_token').status, 'fail'); + assert.equal(requiredCheck(report, 'docker.engine').status, 'fail'); + assert.equal(requiredCheck(report, 'port.stub.vite').status, 'fail'); + assert.doesNotMatch(JSON.stringify(report), /must-not-be-reported/); + assert.equal(requiredCheck(report, 'outbound.container').status, 'warn'); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('a supervised benchmark accepts only its exact fresh handoff path', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-preflight-supervisor-')); + try { + const supervisorState = join(root, 'supervisor.json'); + const dependencies = { run: () => { throw new Error('daemon unavailable'); }, + now: Date.parse('2026-08-12T12:00:00Z'), home: root, pidsOnPort: () => [], + probePort: () => ({ free: true }) }; + const accepted = runPreflight({ ...request(root), supervisorState }, { + ...dependencies, env: { STACK_BENCH_SUPERVISOR_STATE: supervisorState }, + }); + assert.equal(requiredCheck(accepted, 'ambient.stack_bench_supervisor_state').status, 'pass'); + + writeFileSync(supervisorState, '{}'); + const stale = runPreflight({ ...request(root), supervisorState }, { + ...dependencies, env: { STACK_BENCH_SUPERVISOR_STATE: supervisorState }, + }); + assert.equal(requiredCheck(stale, 'ambient.stack_bench_supervisor_state').status, 'fail'); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('admitted child preflight accepts its private lease and supervisor but rejects foreign ownership', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-preflight-child-')); + try { + const runtimeDir = join(root, 'runtime'); + const path = join(runtimeDir, 'backend-lease.json'); + const output = join(root, 'execution-1'); + const supervisorState = join(root, 'execution-1.supervisor.json'); + const lease = createBackendLease({ runId: 'admitted-child', backend: 'postgres', + track: 'ecommerce', runIndex: 1, database: 'child_db' }); + lease.campaignDelegation = { path: join(root, 'execution-1.delegation.json'), token: 'private-delegation-token' }; + writeBackendLease(path, lease); + const supervisor = { version: SUPERVISOR_STATE_VERSION, runId: lease.runId, backend: lease.backend, + runtimeDir, leasePath: path, ownershipToken: lease.ownershipToken, output }; + writeFileSync(supervisorState, JSON.stringify(supervisor)); + const env = { STACK_BENCH_APPLIANCE: '1', STACK_BENCH_CONTROLLER_IMAGE: IMAGE_ID, + STACK_BENCH_LEASE: path, STACK_BENCH_LEASE_TOKEN: lease.ownershipToken, + STACK_BENCH_SUPERVISOR_STATE: supervisorState, + STACK_BENCH_CAMPAIGN_DELEGATION: lease.campaignDelegation.path, + STACK_BENCH_CAMPAIGN_DELEGATION_TOKEN: lease.campaignDelegation.token, + STACK_BENCH_CAMPAIGN_EXECUTION: 'execution-1' }; + const selected = { ...request(output), backends: ['postgres'], track: lease.track, + runIndex: lease.runIndex, agentAdapter: 'reference-fixture', supervisorState }; + const dependencies = { run: () => { throw new Error('Docker unavailable in this focused test'); }, + env, home: root, statfs: () => ({ bavail: 20n, bsize: 1024n ** 3n }), + pidsOnPort: () => [], probePort: () => ({ free: true }), ownedLease: { path, runId: lease.runId } }; + const report = runPreflight(selected, dependencies); + assert.equal(requiredCheck(report, 'ownership.lease').status, 'pass'); + assert.equal(requiredCheck(report, 'ambient.stack_bench_supervisor_state').status, 'pass'); + assert.equal(report.checks.some(check => check.id.startsWith('ambient.stack_bench_lease')), false); + assert.doesNotMatch(JSON.stringify(report), new RegExp(`${lease.ownershipToken}|private-delegation-token`)); + for (const changed of [ + { ...dependencies, env: { ...env, STACK_BENCH_LEASE_TOKEN: 'foreign-token' } }, + { ...dependencies, ownedLease: { path, runId: 'another-run' } }, + { ...dependencies, ownedLease: { path: join(root, 'foreign.json'), runId: lease.runId } }, + ]) assert.equal(requiredCheck(runPreflight(selected, changed), 'ownership.lease').status, 'fail'); + assert.equal(requiredCheck(runPreflight({ ...selected, smoke: true }, { + ...dependencies, env: { ...env, STACK_BENCH_LEASE_TOKEN: 'foreign-token' }, + }), 'smoke.lease').status, 'fail'); + assert.equal(requiredCheck(runPreflight({ ...selected, runIndex: 2 }, dependencies), 'ownership.lease').status, 'fail'); + for (const change of [{ output: join(root, 'foreign-output') }, { ownershipToken: 'foreign-token' }]) { + writeFileSync(supervisorState, JSON.stringify({ ...supervisor, ...change })); + assert.equal(requiredCheck(runPreflight(selected, dependencies), 'ambient.stack_bench_supervisor_state').status, 'fail'); + } + writeFileSync(supervisorState, JSON.stringify(supervisor)); + lease.state = 'active'; + writeBackendLease(path, lease); + assert.equal(requiredCheck(runPreflight({ ...selected, smoke: true }, dependencies), 'ownership.lease').status, 'pass'); + lease.state = 'released'; + writeBackendLease(path, lease); + assert.equal(requiredCheck(runPreflight(selected, dependencies), 'ownership.lease').status, 'fail'); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('preflight argument parsing rejects ambiguous ranges and missing backends', () => { + assert.throws(() => parsePreflightArgs(['node', 'preflight.js']), /--backend is required/); + assert.throws(() => parsePreflightArgs(['node', 'preflight.js', '--backend', 'stub', + '--levels', '3-1']), /--levels/); + assert.throws(() => parsePreflightArgs(['node', 'preflight.js', '--backend', 'stub', + '--mystery']), /Unknown option/); + assert.equal(parsePreflightArgs(['node', 'preflight.js', '--backend', 'spacetime', + '--guidance', 'neutral']).guidance, 'neutral'); + assert.throws(() => parsePreflightArgs(['node', 'preflight.js', '--backend', 'spacetime', + '--guidance', 'unknown']), /--guidance/); + assert.throws(() => parsePreflightArgs(['node', 'preflight.js', '--backend', 'spacetime', + '--guidance', '']), /--guidance/); + assert.throws(() => parsePreflightArgs(['node', 'preflight.js', '--backend', 'spacetime', + '--parallelism', '']), /--parallelism/); + assert.equal(parsePreflightArgs(['node', 'preflight.js', '--backend', 'postgres', + '--track', 'ecommerce', '--levels', '1', '--recipe', 'ecommerce.sequential-l1']) + .recipe, 'ecommerce.sequential-l1'); + assert.throws(() => parsePreflightArgs(['node', 'preflight.js', '--backend', 'postgres', + '--track', 'ecommerce', '--levels', '1-2', '--recipe', 'ecommerce.sequential-l1']), + /exactly one/); + const resultsDir = resolve(tmpdir(), 'stack-bench-appliance-results'); + const parsed = parsePreflightArgs(['node', 'preflight.js', '--backend', 'postgres'], { + env: { STACK_BENCH_RESULTS_DIR: resultsDir, STACK_BENCH_IMAGE: 'exact-build-image' }, + }); + assert.equal(parsed.resultsDir, resultsDir); + assert.equal(parsed.image, 'exact-build-image'); +}); + +test('unknown requested scope becomes a failed report instead of terminating the process', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-preflight-')); + try { + const report = runPreflight({ ...request(root), track: 'not-a-track' }, { + run: (_file, args) => args[0] === 'info' ? dockerInfo() + : args[0] === 'compose' ? '2.40.0' : args[0] === 'ps' ? '' + : args[3] === '{{.Os}}/{{.Architecture}}' ? 'linux/amd64' : IMAGE_ID, + now: Date.parse('2026-08-12T12:00:00Z'), env: {}, home: root, pidsOnPort: () => [], + probePort: () => ({ free: true }), + }); + assert.equal(report.ok, false); + assert.equal(requiredCheck(report, 'request.scope').status, 'fail'); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('loopback port proof detects a listener without relying on process visibility', async t => { + const { createServer } = await import('node:net'); + const server = createServer(); + await new Promise((resolveListen, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolveListen); + }); + t.after(() => server.close()); + const address = server.address(); + assert.ok(address && typeof address !== 'string'); + assert.deepEqual(probeLoopbackPort(address.port), { free: false }); +}); + +test('OpenRouter preflight requires its own key and a pinned route without revealing secrets', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-preflight-openrouter-')); + try { + const keyFile = join(root, 'key'); + writeFileSync(keyFile, 'private-router-key'); + const selected = request(root, ['--agent-adapter', 'openrouter', '--provider-route', 'openai', + '--max-output-tokens', '8192']); + const options = { run: (_file: string, args: string[]) => { + if (args[0] === 'info') return dockerInfo(); + if (args[0] === 'compose') return '2.40.0'; + if (args[0] === 'ps') return ''; + if (args[0] === 'image') return args[3] === '{{.Os}}/{{.Architecture}}' ? 'linux/amd64' : `${IMAGE_ID}\n`; + throw new Error('unexpected Docker command'); + }, home: root, statfs: () => ({ bavail: 20n, bsize: 1024n ** 3n }), + pidsOnPort: () => [], probePort: () => ({ free: true }) }; + for (const [env, expected] of [ + [{ OPENROUTER_API_KEY_FILE: keyFile }, 'pass'], + [{ OPENAI_API_KEY: 'private-other-key' }, 'fail'], + [{ OPENROUTER_API_KEY_FILE: keyFile, OPENROUTER_API_KEY: 'private-duplicate' }, 'fail'], + ] as const) { + const report = runPreflight(selected, { ...options, env }); + assert.equal(requiredCheck(report, 'agent.credentials').status, expected); + assert.doesNotMatch(JSON.stringify(report), /private-router-key|private-other-key|private-duplicate/); + } + const missingRoute = runPreflight({ ...selected, providerRoute: undefined }, + { ...options, env: { OPENROUTER_API_KEY_FILE: keyFile } }); + assert.equal(missingRoute.ok, false); + assert(missingRoute.checks.some(check => check.status === 'fail' + && check.summary.includes('providerRoute'))); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('a key read from its file reaches preflight as the one selected source', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-preflight-key-file-')); + try { + const keyFile = join(root, 'openrouter.key'); + writeFileSync(keyFile, 'sk-or-test'); + const adapter = AGENT_ADAPTER_REGISTRY.get('openrouter'); + // Bench reads the file itself, then passes a placeholder and clears the file variable. + assert.equal(credentialReady(adapter, { OPENROUTER_API_KEY: '', + OPENROUTER_API_KEY_FILE: undefined }, root, () => true).ok, true); + assert.match(String(credentialReady(adapter, { OPENROUTER_API_KEY: '', + OPENROUTER_API_KEY_FILE: keyFile }, root, () => true).reason), /only one API key source/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/tools/stack-bench/tests/pricing-authority.test.ts b/tools/stack-bench/tests/pricing-authority.test.ts new file mode 100644 index 00000000000..4f64943920b --- /dev/null +++ b/tools/stack-bench/tests/pricing-authority.test.ts @@ -0,0 +1,32 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { CLAUDE_SONNET_RATES, claudeRatesForModel, priceClaudeUsage } + from '../src/evidence/claude-usage-cost.js'; +import { PRICING_UNIT, validatePricingAuthority } + from '../src/evidence/pricing-authority.js'; + +test('the direct Sonnet 5 fallback uses the recorded five-rate schedule', () => { + assert.deepEqual(CLAUDE_SONNET_RATES, { + input: 2, output: 10, cacheWrite5m: 2.5, cacheWrite1h: 4, cacheRead: 0.2, + }); + assert.equal(claudeRatesForModel('claude-sonnet-5'), CLAUDE_SONNET_RATES); + assert.equal(priceClaudeUsage({ + input_tokens: 1_000_000, + output_tokens: 1_000_000, + cache_read_input_tokens: 1_000_000, + cache_creation: { + ephemeral_5m_input_tokens: 1_000_000, + ephemeral_1h_input_tokens: 1_000_000, + }, + }), 18.7); +}); + +test('pricing authority requires one unit and the exact five-rate shape', () => { + const value = { unit: PRICING_UNIT, rates: CLAUDE_SONNET_RATES }; + assert.deepEqual(validatePricingAuthority(value), value); + assert.throws(() => validatePricingAuthority({ ...value, unit: 'USD-per-token' }), + /pricing\.unit/); + assert.throws(() => validatePricingAuthority({ ...value, + rates: { ...value.rates, cacheWrite: 3 } }), /contain only/); +}); diff --git a/tools/stack-bench/tests/private-delivery.test.ts b/tools/stack-bench/tests/private-delivery.test.ts new file mode 100644 index 00000000000..903d46ba73a --- /dev/null +++ b/tools/stack-bench/tests/private-delivery.test.ts @@ -0,0 +1,53 @@ +import assert from 'node:assert/strict'; +import { brotliCompressSync } from 'node:zlib'; +import test from 'node:test'; +import { deliveredRecords, sameRecord } from '../src/actions/private-delivery.js'; + +const codec = await import(new URL('../src/stacks/spacetime-wire-codec.js', import.meta.url).href); +const frame = (message: unknown) => { + const writer = new codec.BinaryWriter(256); + codec.ServerMessage.serialize(writer, message); + return Buffer.concat([Buffer.from([1]), brotliCompressSync(writer.getBuffer())]); +}; +const rows = (table: string, data: number[][]) => ({ + table, + rows: { sizeHint: { tag: 'RowOffsets', value: data.reduce((starts, _row, index) => + [...starts, index ? starts.at(-1)! + BigInt(data[index - 1]!.length) : 0n], []) }, + rowsData: Uint8Array.from(data.flat()) }, +}); +// An alert row as one real app stored it: id, account id, item id, delivered flag, created time. +const alert = [...Array(8).fill(0).map((_, i) => i ? 0 : 7), ...Array(8).fill(0).map((_, i) => i ? 0 : 42), + ...Array(8).fill(0).map((_, i) => i ? 0 : 3), 0, ...Array(8).fill(9)]; + +test('a SpacetimeDB row reaches another client whether it arrives as an update or a subscription', async () => { + const owner = frame({ tag: 'TransactionUpdate', value: { querySets: [{ querySetId: { id: 1 }, tables: [{ + tableName: 'stock_alert', rows: [{ tag: 'PersistentTable', value: { + inserts: { sizeHint: { tag: 'FixedSize', value: alert.length }, rowsData: Uint8Array.from(alert) }, + deletes: { sizeHint: { tag: 'FixedSize', value: alert.length }, rowsData: new Uint8Array() } } }] }] }] } }); + const other = frame({ tag: 'SubscribeApplied', value: { requestId: 3, querySetId: { id: 1 }, + rows: { tables: [rows('item', [[3, 0, 0, 0, 0, 0, 0, 0, 65, 105, 114]]), rows('stock_alert', [alert])] } } }); + const mine = await deliveredRecords([{ atMs: 1, payload: owner }]); + const theirs = await deliveredRecords([{ atMs: 2, payload: other }]); + assert.deepEqual([mine.unreadable, theirs.unreadable], [0, 0]); + assert.equal(mine.records.length, 1); + assert.equal(mine.records[0]!.kind, 'table:stock_alert'); + assert(theirs.records.some(record => sameRecord(mine.records[0]!, record)), 'the leaked row is found by its bytes'); + assert(!theirs.records.some(record => record.kind === 'table:item' && sameRecord(mine.records[0]!, record))); +}); + +test('JSON records match whole objects, not shared text such as a product name', async () => { + const owner = await deliveredRecords([{ atMs: 1, + payload: JSON.stringify({ alert: { id: 7, accountId: 42, itemId: 3, delivered: false }, ok: true }) }]); + const leak = await deliveredRecords([{ atMs: 2, + payload: `42["alerts",${JSON.stringify([{ delivered: false, itemId: 3, accountId: 42, id: 7 }])}]` }]); + const publicItem = await deliveredRecords([{ atMs: 2, payload: JSON.stringify({ id: 3, name: 'Air Purifier', price: 189 }) }]); + const own = owner.records.find(record => record.kind === 'json:accountId,delivered,id,itemId')!; + assert(own, 'the alert object is a record; the bare status flag is not'); + assert(leak.records.some(record => sameRecord(own, record)), 'key order and Socket.IO framing do not hide it'); + assert(!publicItem.records.some(record => sameRecord(own, record))); +}); + +test('an unreadable binary delivery is counted, so absence cannot be claimed over it', async () => { + const { unreadable } = await deliveredRecords([{ atMs: 1, payload: Buffer.from([9, 0xff, 0xfe, 0x00, 0x81]) }]); + assert.equal(unreadable, 1); +}); diff --git a/tools/stack-bench/tests/progression-behavior-evidence-packs.contract.ts b/tools/stack-bench/tests/progression-behavior-evidence-packs.contract.ts new file mode 100644 index 00000000000..d10e2bfc4c7 --- /dev/null +++ b/tools/stack-bench/tests/progression-behavior-evidence-packs.contract.ts @@ -0,0 +1,228 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { compilePackDefinition, resolveTaskFragment, type CompiledPackDefinition } + from '../src/composition/composition-compiler.js'; +import { compileScenarioDefinition, type CompiledCriterion, type CompiledFeature } + from '../src/composition/definition-compiler.js'; + +const trackRoot = join(STACK_BENCH_ROOT, 'tracks', 'ecommerce'); +const packRoot = join(trackRoot, 'composition', 'packs'); +const readJson = (path: string): unknown => JSON.parse(readFileSync(path, 'utf8')); +const readPack = (name: string): CompiledPackDefinition => + compilePackDefinition(readJson(join(packRoot, name)), { source: name }); +const packNames = [ + 'progression-automatic-reorder.json', + 'progression-cart-recovery.json', + 'progression-personalized-recommendations.json', +]; + +function fragmentText( + fragment: CompiledPackDefinition['task']['requirements'][number], +): string { + return resolveTaskFragment(fragment, { trackRoot, source: fragment.id }).text; +} + +function featureFor(pack: CompiledPackDefinition, checkId?: string): CompiledFeature { + const check = checkId === undefined ? pack.checks[0] : pack.checks.find(check => check.id === checkId); + if (!check) throw new Error(`${pack.id} must have a check`); + const source = check.source; + const scenario = compileScenarioDefinition(readJson(join(trackRoot, source)), { source }); + const feature = scenario.features.find(candidate => candidate.id === check.feature); + assert(feature, `${pack.id}.${check.id} must select a feature`); + return feature; +} + +function criterion(feature: CompiledFeature, id: string): CompiledCriterion { + const selected = feature.criteria.find(item => item.id === id); + assert(selected, `feature ${feature.id} must contain criterion ${id}`); + return selected; +} + +test('the three behavior packs have isolated, non-prescriptive contracts', () => { + const packs = packNames.map(readPack); + for (const pack of packs) { + assert.equal(pack.moduleType, 'feature'); + assert.equal(pack.task.requirements.length, 1); + assert.equal(pack.task.contracts.length, 1); + for (const fragment of [...pack.task.requirements, ...pack.task.contracts]) { + assert.deepEqual(fragment.modes, ['upgrade']); + assert.equal(fragment.from, undefined); + assert.equal(fragment.until, undefined); + } + const text = `${fragmentText(requiredRequirement(pack))}\n${fragmentText(requiredContract(pack))}`; + assert.doesNotMatch(text, /\b(?:framework|ORM|database|websocket|table|query|endpoint)\b/i); + } + assert.equal(new Set(packs.map(pack => requiredRequirement(pack).path)).size, packs.length); + assert.equal(new Set(packs.map(pack => requiredContract(pack).path)).size, packs.length); +}); + +test('the three behavior packs declare exact dependencies and check ownership', () => { + const packs = new Map(packNames.map(name => { + const pack = readPack(name); + return [pack.id, pack]; + })); + + const reorder = requiredPack(packs, 'ecommerce.progression.automatic-reorder'); + const recovery = requiredPack(packs, 'ecommerce.progression.cart-recovery'); + const recommendations = requiredPack(packs, 'ecommerce.progression.personalized-recommendations'); + assert.deepEqual(reorder.requiresPacks, + ['ecommerce.feature.purchasing', 'ecommerce.l3.scheduled-restocks-features', + 'ecommerce.progression.staff-roles']); + assert.deepEqual(recovery.requiresPacks, ['ecommerce.l3.cart-expiration-features']); + assert.deepEqual(recommendations.requiresPacks, ['ecommerce.l2.recommendations']); + + assert.deepEqual(reorder.checks + .map(check => [check.id, check.criteria]), + [['threshold', ['502a']]]); + assert.deepEqual(recovery.checks + .map(check => [check.id, check.criteria]), + [['available', ['503a']], ['partial', ['503b']]]); + assert.deepEqual(recommendations.checks + .map(check => [check.id, check.criteria]), + [['ordering', ['403a']]]); + + const access = readPack('spec-access-control.json'); + const integrity = readPack('spec-transactional-integrity.json'); + for (const [owner, group, ids, required] of [ + [access, 'automatic-reorder-access', ['502c'], reorder.id], + [integrity, 'automatic-reorder-deduplication', ['502b'], reorder.id], + [access, 'recommendation-profile-isolation', ['403b'], recommendations.id], + ] as const) { + const check = owner.checks.find(check => check.id === group); + assert(check, `${owner.id} owns ${group}`); + assert.equal(owner.moduleType, 'specification'); + assert.equal(check.role, 'guarantee'); + assert.deepEqual(check.criteria, ids); + assert(check.requiresFeatures?.includes(required)); + assert.deepEqual(featureFor(owner, group).criteria.filter(criterion => ids.some(id => id === criterion.id)) + .map(criterion => criterion.id), ids); + } + + for (const pack of packs.values()) { + const feature = featureFor(pack); + const criteria = new Set(feature.criteria.map(item => item.id)); + const source = pack.checks[0]!.source; + const selected = [pack, access, integrity].flatMap(owner => owner.checks) + .filter(check => check.source === source && check.feature === feature.id).flatMap(check => { + if (!check.criteria) throw new Error(`${pack.id}.${check.id} must select criteria`); + return check.criteria; + }); + assert.equal(new Set(selected).size, selected.length); + assert.deepEqual(new Set(selected), criteria); + } +}); + +test('automatic reorder proves staff access and rejects a customer replay', () => { + const pack = readPack(requiredPackName(0)); + const feature = featureFor(pack); + assert.equal(feature.setup.filter(step => step.do === 'dbSetStock' + && step.item === 'Desk Lamp').reduce((total, step) => + total + (typeof step.quantity === 'number' ? step.quantity : 0), 0), 3); + assert(feature.setup.some(step => step.do === 'signIn' && step.actor === 'staff')); + assert(feature.setup.some(step => step.do === 'callAction' && step.actor === 'staff' + && step.action === 'saveReorderRule' && step.input?.testid === 'reorder-submit')); + assert(feature.setup.some(step => step.do === 'expectActionOutcome' && step.actor === 'staff' + && step.outcome === 'accepted')); + const accessFeature = featureFor(readPack('spec-access-control.json'), 'automatic-reorder-access'); + assert(accessFeature.setup.some(step => step.do === 'expectActionOutcome' + && step.actor === 'staff' && step.outcome === 'accepted')); + const access = criterion(accessFeature, '502c').steps; + assert(access.some(step => step.do === 'expect' && step.actor === 'customer' + && step.testid === 'reorder-link' && step.absent === true)); + assert(access.some(step => step.do === 'callAction' && step.actor === 'customer' + && step.from === 'staff' && step.action === 'saveReorderRule')); + assert(access.some(step => step.do === 'expectActionOutcome' && step.actor === 'customer' + && step.outcome === 'refused' && step.routeProvenBy === 'staff')); + assert(access.some(step => step.do === 'expect' + && step.testid === 'reorder-rule-item' && step.count === 1 + && step.attribute === 'data-threshold' && step.value === '2')); + assert(access.some(step => step.do === 'expect' + && step.testid === 'reorder-rule-item' && step.attribute === 'data-quantity' && step.value === '5')); +}); + +test('cart recovery proves complete and partial restoration', () => { + const pack = readPack(requiredPackName(1)); + const feature = featureFor(pack); + const available = criterion(feature, '503a').steps; + assert(available.some(step => step.do === 'expect' && step.testid === 'cart-item' + && step.contains === 'Keyboard')); + assert(available.some(step => step.do === 'expect' && step.testid === 'cart-restore-warning' + && step.absent === true)); + + const partial = criterion(feature, '503b').steps; + assert.equal(partial.filter(step => step.do === 'dbSetStock' + && step.item === 'Desk Lamp' && step.quantity === 0).length, 2); + assert(partial.some(step => step.do === 'expect' && step.testid === 'cart-item' + && step.contains === 'Gaming Mouse')); + assert(partial.some(step => step.do === 'expect' && step.testid === 'cart-item' + && step.contains === 'Desk Lamp' && step.absent === true)); + assert(partial.some(step => step.do === 'expect' && step.testid === 'cart-restore-warning' + && step.contains === 'Desk Lamp')); +}); + +test('personalized recommendations prove sales ordering, name ties, and isolation', () => { + const pack = readPack(requiredPackName(2)); + const feature = featureFor(pack); + assert.equal(feature.setup.filter(step => step.do === 'click' + && step.actor === 'sales-helper' && step.contains === undefined + && step.in?.contains === 'Desk Lamp').length, 3); + // Each item holds its rank, so the list order is the rank of each named item in a list of three. + const expectedNameOrder = ['Gaming Mouse', 'Laptop Stand', 'Webcam']; + const assertRankedList = (steps: typeof feature.setup) => { + assert(steps.some(step => step.do === 'expectElementCount' && step.actor === 'computing-customer' + && step.testid === 'recommended-item' && step.equals === expectedNameOrder.length)); + expectedNameOrder.forEach((name, index) => assert(steps.some(step => step.do === 'expectNumber' + && step.actor === 'computing-customer' && step.testid === 'recommendation-rank' + && step.in?.contains === name && step.equals === index + 1), name)); + }; + const ordering = criterion(feature, '403a').steps; + assert(ordering.some(step => step.do === 'expectNumber' + && step.actor === 'home-customer' && step.testid === 'recommendation-rank' + && step.in?.contains === 'Desk Lamp' && step.equals === 1)); + assertRankedList(ordering); + assert(ordering.some(step => step.do === 'expect' && step.actor === 'audio-customer' + && step.contains === 'Headphones')); + + const isolation = criterion(feature, '403b').steps; + assert(isolation.some(step => step.do === 'waitUntilAbsent' && step.actor === 'audio-customer' + && step.contains === 'Headphones')); + assert(isolation.some(step => step.do === 'expectNumber' + && step.actor === 'home-customer' && step.testid === 'recommendation-rank' + && step.in?.contains === 'Desk Lamp' && step.equals === 1)); + assertRankedList(isolation); +}); + +function requiredPackName(index: number): string { + const name = packNames[index]; + if (!name) throw new Error(`behavior pack name ${index} is required`); + return name; +} + +function requiredPack( + packs: ReadonlyMap, + packId: string, +): CompiledPackDefinition { + const pack = packs.get(packId); + if (!pack) throw new Error(`pack ${packId} is required`); + return pack; +} + +function requiredRequirement( + pack: CompiledPackDefinition, +): CompiledPackDefinition['task']['requirements'][number] { + const requirement = pack.task.requirements[0]; + if (!requirement) throw new Error(`${pack.id} must have a product requirement`); + return requirement; +} + +function requiredContract( + pack: CompiledPackDefinition, +): CompiledPackDefinition['task']['contracts'][number] { + const contract = pack.task.contracts[0]; + if (!contract) throw new Error(`${pack.id} must have a testing contract`); + return contract; +} diff --git a/tools/stack-bench/tests/progression-depth3-composition.contract.ts b/tools/stack-bench/tests/progression-depth3-composition.contract.ts new file mode 100644 index 00000000000..cc6eb00498d --- /dev/null +++ b/tools/stack-bench/tests/progression-depth3-composition.contract.ts @@ -0,0 +1,20 @@ +import assert from 'node:assert/strict'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { compileFeatureCatalogInput, compileProgressionDefinitionFile, + selectFeatureCatalogLevels } from '../src/progression/progression-definition.js'; + +const TRACK = join(STACK_BENCH_ROOT, 'tracks', 'ecommerce'); + +test('the first three progression depths are a valid prefix of the catalog', () => { + const full = compileFeatureCatalogInput(compileProgressionDefinitionFile( + join(TRACK, 'progression', 'ecommerce.json'), { trackRoot: TRACK })); + const selected = selectFeatureCatalogLevels(full, [1, 2, 3]); + + assert(selected.definition.nodes.length > 0); + assert(selected.definition.nodes.every(node => node.level <= 3)); + assert(selected.definition.nodes.length < full.definition.nodes.length); + assert(selected.definition.questlines.every(questline => questline.nodes.length > 0)); +}); diff --git a/tools/stack-bench/tests/progression-feature-packs.contract.ts b/tools/stack-bench/tests/progression-feature-packs.contract.ts new file mode 100644 index 00000000000..eafa793b93a --- /dev/null +++ b/tools/stack-bench/tests/progression-feature-packs.contract.ts @@ -0,0 +1,220 @@ +import assert from 'node:assert/strict'; +import { readFileSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { compilePackDefinition, resolveTaskFragment, + type CompiledPackDefinition } from '../src/composition/composition-compiler.js'; +import { compileScenarioDefinition, type CompiledCriterion, type CompiledFeature } + from '../src/composition/definition-compiler.js'; +import { compileProgressionDefinitionFile, type CompiledProgressionNode } + from '../src/progression/progression-definition.js'; + +// Rules every feature pack in the dependency catalog must hold, checked once +// over the whole catalog rather than restated per pack. Exact ids, points, +// paths and hook names are data the compiler already binds; they are not +// asserted here. + +const trackRoot = join(STACK_BENCH_ROOT, 'tracks', 'ecommerce'); +const packRoot = join(trackRoot, 'composition', 'packs'); +const readJson = (path: string): unknown => JSON.parse(readFileSync(path, 'utf8')); +const packs = new Map(readdirSync(packRoot) + .filter(name => name.endsWith('.json')).map(name => { + const pack = compilePackDefinition(readJson(join(packRoot, name)), { source: name }); + return [pack.id, pack]; + })); +const definition = compileProgressionDefinitionFile( + join(trackRoot, 'progression', 'ecommerce.json'), { trackRoot }); + +test('stored review script safety belongs to reviews and review access is verified at the server boundary', () => { + const pack = packs.get('ecommerce.progression.review-access-specifications')!; + const check = pack.checks.find(check => check.id === 'stored-review-script')!; + assert.equal(check.role, 'guarantee'); + assert.deepEqual(check.requiresFeatures, ['ecommerce.feature.purchasing', 'ecommerce.feature.reviews']); + assert.deepEqual(definition.nodes.filter(node => node.gradingChecks.some(item => + item.id === `${pack.id}.${check.id}.9180a`)).map(node => node.id), ['reviews']); + const feature = scenarioFeature(check, check.id).feature; + const steps = feature.criteria[0]!.steps; + assert(steps.some(step => step.do === 'expectActionOutcome' && step.actor === 'owner' + && step.outcome === 'accepted'), 'reject-all cannot pass'); + assert.deepEqual(steps.filter(step => step.do === 'expectNoScriptExecution').map(step => step.actor), + ['owner', 'reader-fresh']); + + const [reviewCriterion] = selectedCriteria(pack); + assert(reviewCriterion); + assert(reviewCriterion.steps.some(step => step.do === 'callAction' && step.actor === 'owner')); + assert(reviewCriterion.steps.some(step => step.do === 'expectActionOutcome' && step.outcome === 'accepted')); + assert(reviewCriterion.steps.some(step => step.do === 'callAction' && step.actor === 'stranger')); + assert(reviewCriterion.steps.some(step => step.do === 'expectActionOutcome' + && step.outcome === 'application-refused' && step.routeProvenBy === 'owner')); +}); + +test('shipping accounting is an unprompted production check owned only by fulfilment', () => { + const pack = packs.get('ecommerce.progression.inventory-conservation-specifications')!; + const check = pack.checks.find(check => check.id === 'shipping-accounting')!; + assert.equal(check.role, 'guarantee'); + const stableKey = `${pack.stableId}.${check.stableId}.202e`; + assert.deepEqual(definition.nodes.filter(node => node.gradingChecks.some(check => check.id === stableKey)) + .map(node => node.id), ['fulfilment-queue']); + const criterion = scenarioFeature(check, check.id).feature.criteria.find(criterion => criterion.id === '202e')!; + assert.equal(criterion.category, 'production'); + assert(pack.task.requirements.every(fragment => !fragment.requiresFeatures?.includes('ecommerce.progression.fulfilment-queue'))); +}); + +function requiredPack(reference: string): CompiledPackDefinition { + const pack = packs.get(reference); + if (!pack) throw new Error(`the catalog references missing pack ${reference}`); + return pack; +} + +const featurePacks: Array<{ node: CompiledProgressionNode; pack: CompiledPackDefinition }> = + definition.nodes.flatMap(node => node.featureRefs.map(reference => + ({ node, pack: requiredPack(reference) }))); + +type Fragment = CompiledPackDefinition['task']['requirements'][number]; + +function fragmentText(fragment: Fragment): string { + return resolveTaskFragment(fragment, { trackRoot, source: fragment.id }).text; +} + +function scenarioFeature(check: CompiledPackDefinition['checks'][number], + at: string): { feature: CompiledFeature } { + const scenario = compileScenarioDefinition(readJson(join(trackRoot, check.source)), + { source: check.source }); + const feature = scenario.features.find(candidate => candidate.id === check.feature); + assert(feature, `${at} must select a feature that exists in ${check.source}`); + return { feature }; +} + +function selectedCriteria(pack: CompiledPackDefinition): CompiledCriterion[] { + return pack.checks.flatMap(check => { + const at = `${pack.id}.${check.id}`; + const { feature } = scenarioFeature(check, at); + const ids = check.criteria ?? feature.criteria.map(criterion => criterion.id); + return ids.map(id => { + const criterion = feature.criteria.find(candidate => candidate.id === id); + assert(criterion, `${at} must select ${id} from ${check.source}`); + return criterion; + }); + }); +} + +test('every feature pack states one whole product request and one interface it can add to an app', () => { + const requirementPaths = new Set(); + const contractPaths = new Set(); + for (const { pack } of featurePacks) { + const at = pack.id; + assert.equal(pack.moduleType, 'feature', at); + assert.equal(pack.task.requirements.length, 1, `${at} must state one product request`); + const ownedContracts = pack.task.contracts.filter(fragment => fragment.id !== 'ecommerce.orders.data'); + assert.equal(ownedContracts.length, 1, `${at} must state one application interface`); + const [requirement] = pack.task.requirements; + const [contract] = ownedContracts; + assert(requirement && contract); + // Dependency mode adds features to an existing app, so every feature + // must compose as an upgrade, and its interface must travel with it. + assert(requirement.modes?.includes('upgrade'), `${at} must compose as an upgrade`); + for (const fragment of pack.task.contracts) { + assert.deepEqual(fragment.modes, requirement.modes, `${at} interface modes must match its request`); + if (fragment.id === 'ecommerce.orders.data') assert.equal(fragment.path, 'contracts/order-data.md'); + } + for (const fragment of [requirement, ...pack.task.contracts]) { + assert.equal(fragment.from, undefined, `${at} must not slice ${fragment.path}`); + assert.equal(fragment.until, undefined, `${at} must not slice ${fragment.path}`); + } + assert.equal(requirementPaths.has(requirement.path), false, + `${requirement.path} is shared by two feature packs`); + assert.equal(contractPaths.has(contract.path), false, + `${contract.path} is shared by two feature packs`); + requirementPaths.add(requirement.path); + contractPaths.add(contract.path); + } +}); + +test('feature requests are implementation-neutral and never name the testing interface', () => { + for (const { pack } of featurePacks) { + const [requirement] = pack.task.requirements; + assert(requirement); + assert.doesNotMatch(fragmentText(requirement), + /framework|ORM|database|websocket|endpoint|\broutes?\b|reducer|testid|MongoDB|PostgreSQL|SpacetimeDB/i, + `${pack.id} ${requirement.path}`); + } +}); + +test('shopping criteria in one scenario never share a product, so state cannot leak between them', () => { + const [quantity, checkout] = requiredPack('ecommerce.feature.cart').checks.length + ? selectedCriteria(requiredPack('ecommerce.feature.cart')) + .concat(selectedCriteria(requiredPack('ecommerce.feature.checkout'))) + : []; + assert(quantity && checkout, 'cart and checkout must each select a criterion'); + const product = (criterion: CompiledCriterion): string => { + const add = criterion.steps.find(step => step.do === 'click' && step.testid === 'add-to-cart'); + assert(add && typeof add.in?.contains === 'string', `${criterion.id} must add a named product`); + return add.in.contains; + }; + assert.notEqual(product(quantity), product(checkout)); +}); + +test('every replayed request names a declared actor whose request it replays', () => { + // A replay captured from an actor the scenario never declared records + // nothing and degrades to inconclusive, which reads as a pass on its own. + // Replaying one's own request is legitimate (idempotency); an undeclared + // source is not. + const graded = new Set(definition.nodes.flatMap(node => node.gradingChecks.map(check => check.id))); + let replays = 0; + for (const pack of packs.values()) { + for (const check of pack.checks) { + const prefix = `${pack.stableId ?? pack.id}.${check.stableId ?? check.id}.`; + if (![...graded].some(id => id.startsWith(prefix))) continue; + const { feature } = scenarioFeature(check, `${pack.id}.${check.id}`); + const actors = feature.actors ?? []; + for (const criterion of feature.criteria) { + for (const step of criterion.steps.filter(candidate => candidate.do === 'replayAs')) { + replays += 1; + assert(typeof step.from === 'string' && actors.includes(step.from), + `${check.source} ${criterion.id} replays a request from undeclared actor ${String(step.from)}`); + assert(typeof step.actor === 'string' && actors.includes(step.actor), + `${check.source} ${criterion.id} replays as undeclared actor ${String(step.actor)}`); + } + } + } + } + assert(replays > 0, 'the catalog must grade at least one replayed request'); +}); + +test('refund accounting proves one persisted effect after a same-staff replay', () => { + const refund = requiredPack('ecommerce.spec.transactional-integrity'); + const accounting = selectedCriteria(refund).find(criterion => criterion.id === '615b'); + assert(accounting); + const replay = accounting.steps.findIndex(step => step.do === 'replayAs'); + assert(replay >= 0); + assert.equal(accounting.steps[replay]?.actor, 'staff'); + assert.equal(accounting.steps[replay]?.from, 'staff'); + assert.equal(accounting.steps[replay + 1]?.do, 'expectReplayCompleted'); + assert.equal(accounting.steps.some(step => step.do === 'expectReplayRejected'), false); + const fresh = accounting.steps.findIndex(step => step.do === 'freshClient'); + assert(fresh > replay); + const observations = accounting.steps.slice(fresh + 1); + assert(observations.some(step => step.do === 'signIn' && step.actor === 'owner-fresh')); + for (const testid of ['support-refund-total', 'order-refund-total']) { + assert(observations.some(step => step.do === 'expectNumber' && step.actor === 'owner-fresh' + && step.testid === testid && step.relativeTo === 'paid-total' && step.plus === 0)); + } + // Refund records are counted inside each order, where the contract places them. + assert(observations.some(step => step.do === 'expectElementCount' + && step.testid === 'refund-entry' && step.in?.contains === 'Keyboard' && step.equals === 1)); + assert(observations.some(step => step.do === 'expectElementCount' + && step.testid === 'refund-entry' && step.in?.contains === 'Mouse' && step.equals === 0)); + const access = selectedCriteria(requiredPack('ecommerce.spec.access-control')).find(criterion => criterion.id === '615c'); + assert(access?.steps.some(step => step.do === 'expectActionOutcome' + && step.outcome === 'refused')); +}); + +// Each check has authored reporting metadata; browser transport is not a UI category. +test('every current progression check has a category independent of its operational role', () => { + const checks = definition.nodes.flatMap(node => node.gradingChecks); + assert.ok(checks.length); + assert.ok(checks.every(check => ['feature', 'production', 'interface'].includes(check.category ?? ''))); + assert.ok(checks.some(check => check.role === 'feature' && check.category === 'production')); +}); diff --git a/tools/stack-bench/tests/progression-grade-bundle.test.ts b/tools/stack-bench/tests/progression-grade-bundle.test.ts new file mode 100644 index 00000000000..9e415a7ba02 --- /dev/null +++ b/tools/stack-bench/tests/progression-grade-bundle.test.ts @@ -0,0 +1,290 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { createCheckEvidence } from '../src/evidence/check-evidence.js'; +import type { CheckEvidence, CheckEvidenceStatus } from '../src/evidence/check-evidence.js'; +import { createArtifact, emptyArtifactIdentities } from '../src/evidence/artifacts.js'; +import { gradeBundleToProgressionResult } + from '../src/progression/grade-bundle-result.js'; + +interface BundleFixture { + observation: string; + source: { sha256: string }; + selection: { + sha256: string; + checks: Array<{ stableKey: string; points: number }>; + attemptedChecks: string[]; + reportedChecks: string[]; + notRun: Array<{ stableKey: string; reason: string }>; + }; + outcome?: { kind: string; reason: string; phase?: string }; + totals: { score: number; max: number; regression: { score: number; max: number } | null }; + suites: { application: { features: Array<{ + id: number; + cleanupEvidence?: { status: string }; + setupEvidence: CheckEvidence; + criteria: Array<{ stableKey: string; points: number; evidence: CheckEvidence }>; + }> } }; +} + +const evidence = (status: CheckEvidenceStatus): CheckEvidence => createCheckEvidence({ + status, + code: status === 'passed' ? 'completed' : 'test_result', + phase: 'assertion', + startedAtMs: 1, + completedAtMs: 2, +}); +const setupEvidence = () => createCheckEvidence({ status: 'passed', code: 'completed', + phase: 'setup', startedAtMs: 1, completedAtMs: 2 }); +const sourceSha256 = 'c'.repeat(64); +const recipeIdentity = { id: 'ecommerce-l1', sha256: 'd'.repeat(64) }; +const progressionIdentity = { id: 'dependency-graph', contentSha256: '8'.repeat(64) }; +const featureCatalogIdentity = { id: 'catalog', contentSha256: '7'.repeat(64) }; +const owner = { schemaVersion: 1, + campaign: { id: 'campaign', version: '1.0.0', sha256: 'e'.repeat(64) }, + attempt: { id: 'campaign-r1', track: 'ecommerce', stack: 'postgres', + agentAdapter: 'claude-code', model: 'test-model', conditionSha256: 'f'.repeat(64) }, + workspace: { appDirectory: 'app' } }; +const runArtifact = () => createArtifact({ kind: 'benchmark_run', id: 'run-1', + attempt: { id: 'run-1', parentId: owner.attempt.id }, + identities: emptyArtifactIdentities({ agentAdapter: { id: owner.attempt.agentAdapter }, + stackAdapter: { id: owner.attempt.stack } }), + payload: { backend: owner.attempt.stack, model: owner.attempt.model, + condition: { contentSha256: owner.attempt.conditionSha256 }, + featureCatalog: featureCatalogIdentity, dependencyPolicy: progressionIdentity, + progressionOwner: { schemaVersion: owner.schemaVersion, + campaign: owner.campaign, attempt: owner.attempt } }, +}); + +const action = () => ({ + type: 'repair', + prompt: { nodeIds: ['accounts', 'catalog'] }, + grading: { + nodeIds: ['accounts', 'catalog'], + checks: [ + { id: 'check.accounts', nodeId: 'accounts', points: 1 }, + { id: 'check.catalog', nodeId: 'catalog', points: 2 }, + ], + }, +}); + +const bundle = (): BundleFixture => ({ + observation: 'scored', + source: { sha256: sourceSha256 }, + selection: { + sha256: 'a'.repeat(64), + checks: [ + { stableKey: 'check.accounts', points: 1 }, + { stableKey: 'check.catalog', points: 2 }, + ], + attemptedChecks: ['check.accounts', 'check.catalog'], + reportedChecks: ['check.accounts', 'check.catalog'], + notRun: [], + }, + totals: { score: 1, max: 3, regression: null }, + suites: { application: { features: [{ id: 1, setupEvidence: setupEvidence(), criteria: [ + { stableKey: 'check.accounts', points: 1, evidence: evidence('passed') }, + { stableKey: 'check.catalog', points: 2, evidence: evidence('failed') }, + ] }] } }, +}); + +const artifact = (payload = bundle(), id = 'grade-1') => createArtifact({ + kind: 'grade_bundle', id, + attempt: { id, parentId: 'run-1' }, + identities: emptyArtifactIdentities({ recipe: recipeIdentity, + stackAdapter: { id: owner.attempt.stack } }), + payload, +}); + +const conversion = { owner, runArtifact: runArtifact(), featureCatalogIdentity, + dependencyPolicyIdentity: progressionIdentity, + sourceSha256, recipeIdentity, + selectionSha256: 'a'.repeat(64) }; + +test('grade bundle conversion preserves exact node ownership and measured outcomes', () => { + assert.deepEqual(gradeBundleToProgressionResult(artifact(), action(), conversion), { + attemptId: 'grade-1', + runId: 'run-1', + sourceSha256, + selectionSha256: 'a'.repeat(64), + outcome: 'conclusive', + nodes: [ + { id: 'accounts', checks: [{ id: 'check.accounts', outcome: 'pass' }] }, + { id: 'catalog', checks: [{ id: 'check.catalog', outcome: 'fail' }] }, + ], + }); +}); + +test('grade bundle conversion rejects incomplete, duplicate, or changed grading scope', () => { + const missing = bundle(); + missing.selection.reportedChecks.pop(); + assert.throws(() => gradeBundleToProgressionResult(artifact(missing, 'missing'), + action(), conversion), /complete progression evidence/); + const duplicate = bundle(); + duplicate.suites.application.features[0]!.criteria.push( + structuredClone(duplicate.suites.application.features[0]!.criteria[0]!)); + assert.throws(() => gradeBundleToProgressionResult(artifact(duplicate, 'duplicate'), + action(), conversion), /repeats check/); + const points = bundle(); + points.selection.checks[0]!.points = 2; + assert.throws(() => gradeBundleToProgressionResult(artifact(points, 'points'), + action(), conversion), /points.*do not match/); + assert.throws(() => gradeBundleToProgressionResult(artifact(), action(), { + ...conversion, selectionSha256: 'b'.repeat(64), + }), /selection identity/); + assert.throws(() => gradeBundleToProgressionResult(artifact(), action(), { + ...conversion, selectionSha256: undefined, + }), /selection identity is required/); + const totals = bundle(); + totals.totals.score = 2; + assert.throws(() => gradeBundleToProgressionResult(artifact(totals, 'totals'), + action(), conversion), /totals do not match/); +}); + +test('a measured app prerequisite failure remains blocked through grade conversion', () => { + const input = bundle(); + input.suites.application.features[0]!.criteria[1]!.evidence = createCheckEvidence({ + status: 'blocked', phase: 'setup', code: 'application_failure', + summary: 'Blocked by a failed prerequisite: stock did not change', startedAtMs: 1, completedAtMs: 2, + }); + const result = gradeBundleToProgressionResult(artifact(input), action(), conversion); + assert.equal(result.outcome, 'conclusive'); + if (result.outcome === 'conclusive') assert.equal(result.nodes[1]!.checks[0]!.outcome, 'blocked'); +}); + +test('grade bundle conversion binds the artifact owner, source, stack, recipe, and nodes', () => { + const stale = artifact(); + stale.attempt.parentId = 'different-run'; + assert.throws(() => gradeBundleToProgressionResult(stale, action(), conversion), + /does not belong/); + assert.throws(() => gradeBundleToProgressionResult(artifact(), action(), { + ...conversion, sourceSha256: '9'.repeat(64), + }), /source identity/); + const foreignRun = runArtifact(); + foreignRun.attempt.parentId = 'different-attempt'; + assert.throws(() => gradeBundleToProgressionResult(artifact(), action(), { + ...conversion, runArtifact: foreignRun, + }), /owned progression benchmark run/); + const oldCampaign = runArtifact(); + oldCampaign.payload.progressionOwner.campaign.sha256 = '7'.repeat(64); + assert.throws(() => gradeBundleToProgressionResult(artifact(), action(), { + ...conversion, runArtifact: oldCampaign, + }), /benchmark owner/); + const badNodes = action(); + badNodes.grading.nodeIds.push('accounts'); + assert.throws(() => gradeBundleToProgressionResult(artifact(), badNodes, conversion), + /node ownership/); + const foreign = action(); + foreign.grading.checks[0]!.nodeId = 'foreign'; + assert.throws(() => gradeBundleToProgressionResult(artifact(), foreign, conversion), + /node ownership/); +}); + +test('typed grader failures do not consume a progression repair as a zero score', () => { + const failed = bundle(); + failed.outcome = { kind: 'harness_failure', reason: 'browser worker stopped' }; + assert.deepEqual(gradeBundleToProgressionResult(artifact(failed, 'harness'), + action(), conversion), { attemptId: 'harness', runId: 'run-1', sourceSha256, + selectionSha256: 'a'.repeat(64), + outcome: 'inconclusive', category: 'harness_failure', + reason: 'browser worker stopped' }); +}); + +test('partial application aborts and completed scores cannot hide measurement or cleanup failures', () => { + for (const failure of ['inconclusive', 'harness_failure', 'cleanup'] as const) { + const partial = bundle(); + partial.outcome = { kind: 'app_failure', phase: 'application-start', reason: 'restart failed' }; + partial.selection.attemptedChecks = ['check.accounts']; + partial.selection.reportedChecks = ['check.accounts']; + partial.selection.notRun = [{ stableKey: 'check.catalog', reason: 'restart failed' }]; + partial.totals.score = 0; + const feature = partial.suites.application.features[0]!; + feature.criteria.pop(); + if (failure === 'cleanup') feature.cleanupEvidence = { status: 'harness_failure' }; + else feature.criteria[0]!.evidence = evidence(failure); + const result = gradeBundleToProgressionResult(artifact(partial), action(), conversion); + assert.equal(result.outcome, 'inconclusive'); + if (result.outcome === 'inconclusive') assert.equal(result.category, + failure === 'inconclusive' ? 'inconclusive_evidence' : 'harness_failure'); + } + const complete = bundle(); + complete.suites.application.features[0]!.cleanupEvidence = { status: 'harness_failure' }; + assert.equal(gradeBundleToProgressionResult(artifact(complete), action(), conversion).outcome, + 'inconclusive'); + const mixed = bundle(); + mixed.outcome = { kind: 'app_failure', phase: 'grading', reason: 'catalog failed' }; + mixed.suites.application.features[0]!.criteria[0]!.evidence = evidence('inconclusive'); + mixed.suites.application.features[0]!.criteria[1]!.evidence = evidence('failed'); + mixed.totals.score = 0; + const unmeasured = gradeBundleToProgressionResult(artifact(mixed, 'mixed'), action(), conversion); + assert.equal(unmeasured.outcome, 'inconclusive'); + if (unmeasured.outcome === 'inconclusive') assert.equal(unmeasured.category, 'inconclusive_evidence'); +}); + +test('completed progression evidence outranks a stale application failure', () => { + const completed = bundle(); + completed.outcome = { kind: 'app_failure', phase: 'grading', reason: 'stale summary' }; + const result = gradeBundleToProgressionResult(artifact(completed, 'completed'), + action(), conversion); + assert.equal(result.outcome, 'conclusive'); + if (result.outcome !== 'conclusive') throw new Error('expected a conclusive result'); + assert.equal(result.applicationFailure, undefined); + assert.deepEqual(result.nodes, [ + { id: 'accounts', checks: [{ id: 'check.accounts', outcome: 'pass' }] }, + { id: 'catalog', checks: [{ id: 'check.catalog', outcome: 'fail' }] }, + ]); +}); + +test('a typed application abort charges current work but not earlier regression guards', () => { + const failed = bundle(); + failed.outcome = { kind: 'app_failure', phase: 'application-start', + reason: 'the generated application did not start' }; + failed.selection.attemptedChecks = []; + failed.selection.reportedChecks = []; + failed.selection.notRun = failed.selection.checks.map(check => ({ + stableKey: check.stableKey, reason: failed.outcome!.reason, + })); + failed.totals = { score: 0, max: 3, regression: null }; + const selected = action(); + selected.prompt.nodeIds = ['catalog']; + const result = gradeBundleToProgressionResult(artifact(failed, 'app-failure'), + selected, conversion); + assert.equal(result.outcome, 'conclusive'); + if (result.outcome !== 'conclusive') throw new Error('expected a conclusive result'); + assert.deepEqual(result.applicationFailure, { + phase: 'application-start', reason: 'the generated application did not start', + }); + assert.deepEqual(result.nodes, [ + { id: 'accounts', checks: [{ id: 'check.accounts', outcome: 'not-run' }] }, + { id: 'catalog', checks: [{ id: 'check.catalog', outcome: 'fail' }] }, + ]); + const earlierFailure = structuredClone(failed); + earlierFailure.selection.attemptedChecks = ['check.accounts']; + earlierFailure.selection.reportedChecks = ['check.accounts']; + earlierFailure.selection.notRun = [{ stableKey: 'check.catalog', reason: failed.outcome!.reason }]; + earlierFailure.suites.application.features[0]!.criteria[0]!.evidence = evidence('failed'); + const measured = gradeBundleToProgressionResult(artifact(earlierFailure, 'earlier-failure'), selected, conversion); + if (measured.outcome !== 'conclusive') throw new Error('expected a conclusive result'); + assert.deepEqual(measured.nodes[0], { id: 'accounts', checks: [{ id: 'check.accounts', outcome: 'fail' }] }); + const incomplete = structuredClone(failed); + incomplete.selection.notRun.pop(); + assert.throws(() => gradeBundleToProgressionResult(artifact(incomplete, 'bad-abort'), + action(), conversion), /application abort is incomplete/); + const crashed = bundle(); + crashed.outcome = { kind: 'app_failure', phase: 'application-readiness', + reason: 'application did not become ready after database reset' }; + crashed.selection.attemptedChecks = ['check.accounts']; + crashed.selection.reportedChecks = ['check.accounts']; + crashed.selection.notRun = [{ stableKey: 'check.catalog', reason: crashed.outcome.reason }]; + crashed.suites.application.features[0]!.criteria.pop(); + crashed.totals = { score: 1, max: 3, regression: null }; + const passedFirst = gradeBundleToProgressionResult(artifact(crashed, 'partial-abort'), action(), conversion); + if (passedFirst.outcome !== 'conclusive') throw new Error('expected a conclusive result'); + assert.deepEqual(passedFirst.nodes, [ + { id: 'accounts', checks: [{ id: 'check.accounts', outcome: 'pass' }] }, + { id: 'catalog', checks: [{ id: 'check.catalog', outcome: 'fail' }] }, + ]); + crashed.totals.score = 0; + assert.throws(() => gradeBundleToProgressionResult(artifact(crashed, 'zeroed-abort'), + action(), conversion), /application abort is incomplete/); +}); diff --git a/tools/stack-bench/tests/progression-graph.contract.ts b/tools/stack-bench/tests/progression-graph.contract.ts new file mode 100644 index 00000000000..521309ffb25 --- /dev/null +++ b/tools/stack-bench/tests/progression-graph.contract.ts @@ -0,0 +1,48 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { + compileProgressionGraph, + renderProgressionGraphHtml, + type ProgressionGraph, +} from '../src/progression/progression-graph.js'; + +const projectRoot = STACK_BENCH_ROOT; +const trackRoot = join(projectRoot, 'tracks', 'ecommerce'); +const definitionPath = join(trackRoot, 'progression', 'ecommerce.json'); + +test('the dependency graph page is generated from the ecommerce definition', () => { + const htmlPath = join(projectRoot, 'docs', 'dependency-graph.html'); + const html = readFileSync(htmlPath, 'utf8'); + const graph = compileProgressionGraph(definitionPath, { trackRoot }); + assert(graph.nodes.length > 0); + assert(graph.levels >= 3); + assert(graph.nodes.every(node => node.description.length > 0)); + assert.deepEqual(graph.nodes.filter(node => node.level === 1).map(node => node.id), + ['accounts', 'catalog', 'staff-access', 'support-intake']); + assert.equal(renderProgressionGraphHtml(html, graph), html, + 'run npm run graph after changing the progression definition'); +}); + +test('graph rendering rejects HTML without generated-data markers', () => { + const graph = compileProgressionGraph(definitionPath, { trackRoot }); + assert.throws(() => renderProgressionGraphHtml('', graph), + /has no generated-data markers/); +}); + +test('graph rendering escapes less-than signs in embedded JSON', () => { + const graph: ProgressionGraph = { + schemaVersion: 1, + definition: { id: 'example.graph', contentSha256: 'a'.repeat(64) }, + levels: 1, + questlines: [{ id: 'example', title: 'Example '); + const capability = { defaultWithin: 2000, testId: stableElementSelector, + scopedUser: (name: string) => name, sleep: async () => {} }; + const result = await executeAction(ACTION_REGISTRY, 'ensureSignedIn', + { do: 'ensureSignedIn', actor: 'a', name: 'alice' }, + { capabilities: { actors: { get: () => ({ page, name: 'a', + loc: (id: string) => page.locator(stableElementSelector(id)).first() }) }, + 'browser-interaction': capability } }); + assert.equal(result.status, 'passed', result.summary ?? undefined); + for (const restoredUser of ['alice', 'someone-else']) { + await page.setContent(``); + const restored = await executeAction(ACTION_REGISTRY, 'ensureSignedIn', + { do: 'ensureSignedIn', actor: 'a', name: 'alice' }, + { capabilities: { actors: { get: () => ({ page, name: 'a', + loc: (id: string) => page.locator(stableElementSelector(id)).first() }) }, + 'browser-interaction': capability } }); + assert.equal(restored.status, restoredUser === 'alice' ? 'passed' : 'failed', restored.summary ?? undefined); + if (restoredUser !== 'alice') assert.match(restored.summary ?? '', /different account/); + } + await page.setContent(`alice +
+ +
`); + const explicit = await executeAction(ACTION_REGISTRY, 'signIn', + { do: 'signIn', actor: 'a', name: 'alice', password: 'wrong-password', expectFailure: true }, + { capabilities: { actors: { get: () => ({ page, name: 'a', + loc: (id: string) => page.locator(stableElementSelector(id)).first() }) }, + 'browser-interaction': capability } }); + assert.equal(explicit.status, 'passed', explicit.summary ?? undefined); + assert.equal(await page.locator('form').getAttribute('data-password'), 'wrong-password'); + } finally { await browser.close(); } +}); + +test('reservation restart readback rejects returned stock with a lost cart entry', async () => { + const source = join(STACK_BENCH_ROOT, 'tracks/ecommerce/scenarios/03-deferred-durability.json'); + const feature = compileScenarioDefinition(JSON.parse(readFileSync(source, 'utf8')), { source }) + .features.find(feature => feature.id === 314)!; + const browser = await chromium.launch({ headless: true }); + try { + for (const retained of [true, false]) { + const page = await browser.newPage(); + // The pending reservation holds one unit until the expiry wait returns it. + let expired = false; + await page.route('http://reservation.test/', route => route.fulfill({ contentType: 'text/html', + body: `durable-reservation +
Desk Lamp${expired ? 100 : 99}
+ ` })); + await page.goto('http://reservation.test/'); + const actor = { page, loc: (id: string, options?: { contains?: string; scope?: { testid: string; contains?: string | RegExp } }) => { + const root = options?.scope ? page.locator(stableElementSelector(options.scope.testid), + { hasText: options.scope.contains }).first() : page; + return root.locator(stableElementSelector(id), { hasText: options?.contains }).first(); + } }; + const capability = { defaultWithin: 300, expand: (value: string) => value, scopedUser: (value: string) => value, + recorded: new Map([['before', 100], ['pending-314', performance.now()], ['pending-314-accepted', performance.now()]]), + testId: stableElementSelector, sleep: async () => {} }; + const clock = { sleep: async () => { expired = true; } }; + let failed: string | undefined; + for (const step of feature.criteria[0]!.steps) { + const input = { ...step, ...(['expect', 'expectNumber'].includes(step.do) ? { within: 300 } : {}), + ...('settleMs' in step ? { settleMs: 0 } : {}) }; + const result = await executeAction(ACTION_REGISTRY, step.do, input, + { capabilities: { actors: { get: () => actor }, 'browser-interaction': capability, + 'browser-observation': capability, clock } }); + if (result.status !== 'passed') { failed = String(step.testid ?? step.do); break; } + } + assert.equal(failed, retained ? undefined : 'cart-item-expired'); + await page.close(); + } + } finally { await browser.close(); } +}); + +test('order status probes reject negative labels that contain the expected status', async () => { + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage(); + const actor = { page, loc: () => page.locator('#order-status') }; + const capability = { defaultWithin: 300, expand: (value: string) => value, + testId: stableElementSelector, sleep: async () => {} }; + for (const [file, status, misleading] of [ + ['02-fulfilment-ship.json', 'shipped', 'not shipped'], + ['02-order-cancellation-history.json', 'cancelled', 'cancellation failed'], + ['03-order-delivery.json', 'delivered', 'not delivered'], + ]) { + const source = join(STACK_BENCH_ROOT, 'tracks/ecommerce/scenarios', file!); + const definition = compileScenarioDefinition(JSON.parse(readFileSync(source, 'utf8')), { source }); + const step = definition.features.flatMap(feature => feature.criteria.flatMap(criterion => criterion.steps)) + .find(step => step.testid === 'order-status' && step.statusText === status)!; + assert(step); + for (const text of [status, ` ${status} `, status!.toUpperCase(), misleading]) { + await page.setContent(`${text}`); + const result = await executeAction(ACTION_REGISTRY, step.do, { ...step, within: 300 }, + { capabilities: { actors: { get: () => actor }, 'browser-observation': capability } }); + assert.equal(result.status, text?.trim().toLowerCase() === status ? 'passed' : 'failed', result.summary ?? undefined); + } + } + } finally { await browser.close(); } +}); + +test('restock reload observations work on persistent pages and reopened panels without hiding pending work', async () => { + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage(); + const source = join(STACK_BENCH_ROOT, 'tracks/ecommerce/scenarios/03-server-time.json'); + const feature = compileScenarioDefinition(JSON.parse(readFileSync(source, 'utf8')), { source }).features[0]!; + const criterion = feature.criteria[0]!; + const entry = criterion.steps.findIndex(step => step.testid === 'admin-link'); + assert(entry >= 0, 'the measured post-reload path must contain the declared admin entry'); + const steps = criterion.steps.slice(entry); + const reorderSource = join(STACK_BENCH_ROOT, 'tracks/ecommerce/scenarios/progression-automatic-reorder.json'); + const reorder = compileScenarioDefinition(JSON.parse(readFileSync(reorderSource, 'utf8')), + { source: reorderSource }).features[0]!; + const reorderEntry = reorder.setup.filter(step => step.do === 'click' && step.actor === 'admin'); + const actor = { page, loc: (id: string, options?: { contains?: string }) => { + const locator = page.locator(stableElementSelector(id)); + return options?.contains ? locator.filter({ hasText: options.contains }) : locator; + } }; + const capability = { defaultWithin: 500, expand: (value: string) => value, + testId: stableElementSelector, sleep: async () => new Promise(resolve => setTimeout(resolve, 1)) }; + for (const layout of ['page', 'panel', 'nested', 'broken']) { + await page.route('http://restock.test/', route => route.fulfill({ contentType: 'text/html', + body: `${layout === 'page' ? '' : ''} +
+ ${layout === 'nested' || layout === 'broken' ? '' : ''} +
+ +
Espresso Machine
+
+
` })); + await page.goto('http://restock.test/'); + // Automatic reorder observes the same pending list and must also reach nested tools. + if (layout === 'nested') { + for (const step of reorderEntry) { + const result = await executeAction(ACTION_REGISTRY, step.do, step, + { capabilities: { actors: { get: () => actor }, 'browser-interaction': capability } }); + assert.equal(result.status, 'passed', result.summary ?? undefined); + } + assert.equal(await page.locator('#pending-restock-item').isVisible(), true); + } + await page.reload(); + const results = []; + for (const step of steps) results.push(await executeAction(ACTION_REGISTRY, step.do, step, + { capabilities: { actors: { get: () => actor }, + 'browser-interaction': capability, 'browser-observation': capability } })); + assert.equal(results[0]!.status, 'passed', 'optional entry also permits an absent self-link'); + assert.equal(results[1]!.status, 'passed', 'optional in-area entry supports both inline and separate tools'); + assert.equal(results[2]!.status, layout === 'broken' ? 'failed' : 'passed'); + if (layout !== 'broken') { + assert.equal(results[3]!.status, 'failed', 'visible pending work must fail absence'); + await page.locator('#pending-restock-item').evaluate(element => element.remove()); + const absent = steps[3]!; + assert.equal((await executeAction(ACTION_REGISTRY, absent.do, absent, + { capabilities: { actors: { get: () => actor }, 'browser-observation': capability } })).status, 'passed'); + } + await page.unroute('http://restock.test/'); + } + } finally { await browser.close(); } +}); + +test('submission evidence waits for delayed saves and blocks rejected or unconfirmed saves', async () => { + const browser = await chromium.launch({ headless: true }); + try { + for (const kind of ['shipping', 'stock-alert'] as const) for (const mode of ['delayed', 'rejected', 'unconfirmed'] as const) { + const source = join(STACK_BENCH_ROOT, 'tracks/ecommerce/scenarios', + kind === 'shipping' ? '02-fulfilment-ship.json' : 'progression-stock-alert-delivery.json'); + const feature = compileScenarioDefinition(JSON.parse(readFileSync(source, 'utf8')), { source }).features[0]!; + const steps = kind === 'shipping' ? feature.criteria[0]!.steps : feature.setup; + const submit = steps.findIndex(step => step.do === 'click' && step.testid === (kind === 'shipping' ? 'ship-submit' : 'stock-alert')); + const control = kind === 'shipping' ? 'fulfilment-panel' : 'item-card'; + const label = kind === 'shipping' ? 'Keyboard' : 'Air Purifier'; + const page = await browser.newPage(); + let finished = false; + try { + await page.route('http://submission.test/**', async route => { + if (route.request().method() === 'POST') { + if (mode === 'delayed') await new Promise(resolve => setTimeout(resolve, 3000)); + finished = true; + await route.fulfill({ status: mode === 'rejected' ? 403 : 200, body: '{}' }); + } else { + await route.fulfill({ contentType: 'text/html', body: ` +
${label} +
${label}
+
+ ` }); + } + }); + await page.goto('http://submission.test/'); + // A previous successful submission must not satisfy this new request. + await page.locator('section').evaluate(element => { element.dataset.submitState = 'succeeded'; }); + const actor = { page, loc: (id: string) => page.locator(stableElementSelector(id)) }; + const capability = { defaultWithin: 1000, expand: (value: string) => value, + testId: stableElementSelector, sleep: async (ms: number) => { await new Promise(resolve => setTimeout(resolve, ms)); } }; + const capabilities = { actors: { get: () => actor }, 'browser-interaction': capability, + 'browser-observation': capability }; + assert.equal((await executeAction(ACTION_REGISTRY, 'click', steps[submit]!, { capabilities })).status, 'passed'); + assert.equal(steps[submit]!.settleMs, undefined, 'no fixed write buffer'); + const observation = { ...steps[submit + 1]!, within: mode === 'delayed' ? 5000 : 300 }; + assert.equal(observation.do, 'expect'); + assert.equal(steps[submit + 1]!.attribute, 'data-submit-state'); + const result = await executeAction(ACTION_REGISTRY, 'expect', observation, { capabilities }); + assert.equal(result.status, mode === 'delayed' ? 'passed' : 'failed', result.summary ?? undefined); + assert(finished); + if (mode === 'delayed') { + assert.equal(await page.locator('button').count(), 0, 'receipt persists after button removal'); + await page.reload(); + } + } finally { await page.close(); } + } + } finally { await browser.close(); } +}); + +test('low-stock live observations stay open while another client restocks', async () => { + const source = join(STACK_BENCH_ROOT, 'tracks/ecommerce/scenarios/02-low-stock.json'); + const feature = compileScenarioDefinition(JSON.parse(readFileSync(source, 'utf8')), { source }).features[0]!; + assert(feature.actors); + const steps = feature.criteria.find(criterion => criterion.id === '5a')!.steps; + const browser = await chromium.launch({ headless: true }); + try { + for (const live of [true, false]) { + const pages: Map = new Map(await Promise.all(feature.actors.map(async name => [name, await browser.newPage()] as const))); + const watcher = pages.get('admin')!; + let stock = 3; + for (const [name, page] of pages) { + await page.exposeFunction('changeStock', async (quantity: number) => { + stock += quantity; + if (live) await watcher.locator('#low-stock-item').evaluate((row, total) => { + (row as HTMLElement).hidden = total > 10; + }, stock); + }); + await page.setContent(`
+ +
+ +
+ +
+
Air Purifier East
+
+

Low stock

Air Purifier
+
+
Air Purifier
+ `); + } + const capability = { defaultWithin: 500, expand: (value: string) => value, + scopedUser: (value: string) => value, testId: stableElementSelector, + sleep: async () => { await new Promise(resolve => setTimeout(resolve, 1)); } }; + const actors = { get: (name: string) => { + const page = pages.get(name)!; + return { page, loc: (id: string, options?: { contains?: string; scope?: { testid: string; contains?: string | RegExp } }) => { + const root = options?.scope ? page.locator(stableElementSelector(options.scope.testid), { hasText: options.scope.contains }).first() : page; + return root.locator(stableElementSelector(id), { hasText: options?.contains }).first(); + } }; + } }; + let failed: string | undefined; + for (const step of steps) { + if (step.actor === 'admin') assert(['expect', 'waitUntilAbsent'].includes(step.do), 'the observer must not navigate or reload'); + const input = ['click', 'expect', 'waitUntilAbsent'].includes(step.do) ? { ...step, within: 500 } : step; + const result = await executeAction(ACTION_REGISTRY, step.do, input, { capabilities: { + actors, 'browser-interaction': capability, 'browser-observation': capability, + } }); + if (result.status !== 'passed') { failed = step.do; break; } + assert.equal(await watcher.locator('#low-stock').isVisible(), true); + } + assert.equal(failed, live ? undefined : 'waitUntilAbsent'); + assert.equal(stock, live ? 10 : 11); + await Promise.all([...pages.values()].map(page => page.close())); + } + } finally { await browser.close(); } +}); diff --git a/tools/stack-bench/tests/repair-cli.test.ts b/tools/stack-bench/tests/repair-cli.test.ts new file mode 100644 index 00000000000..93275071c3f --- /dev/null +++ b/tools/stack-bench/tests/repair-cli.test.ts @@ -0,0 +1,25 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { parseRepairArgs } from '../commands/repair-cli.js'; + +test('repair CLI rejects unbounded, duplicate, and ambiguous requests', () => { + const accepted = [ + ['status', './run', '--level', '2'], + ['grant', './run', '--level', '2', '--repairs', '4', '--max-budget-usd', '25', '--timeout-minutes', '90'], + ]; + for (const args of accepted) { + assert.doesNotThrow(() => parseRepairArgs(['node', 'repair-cli.js', ...args])); + } + const invalid = [ + ['grant', './run', '--level', '1'], + ['grant', './run', '--level', '1', '--repairs', '0'], + ['grant', './run', '--level', '1', '--repairs', '1.5'], + ['grant', './run', '--level', '1', '--level', '2', '--repairs', '4'], + ['grant', './run', '--level', '1', '--repairs', '4', '--timeout-minutes', '0'], + ['status', './run'], + ]; + for (const args of invalid) { + assert.throws(() => parseRepairArgs(['node', 'repair-cli.js', ...args])); + } +}); diff --git a/tools/stack-bench/tests/repair-grant.test.ts b/tools/stack-bench/tests/repair-grant.test.ts new file mode 100644 index 00000000000..5a67761bb17 --- /dev/null +++ b/tools/stack-bench/tests/repair-grant.test.ts @@ -0,0 +1,174 @@ +import assert from 'node:assert/strict'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { emptyArtifactIdentities, writeRunJson } from '../src/evidence/artifacts.js'; +import { compareRepairBaseline, createRepairGrant, inspectRepairParent } + from '../src/runtime/repair-grant.js'; +import type { RepairOutcome, RepairSelection } + from '../src/runtime/repair-grant.js'; +import type { RunRepairRecord } from '../src/evidence/benchmark-run.js'; +import { preserveLevelCheckpoint } from '../src/runtime/source-checkpoint.js'; + +interface FixtureSelection extends RepairSelection { + sha256: string; + scoredPoints: number; + recipe: { id: string }; +} + +interface FixtureOverrides { + productionQuality?: boolean; + repair?: RunRepairRecord; + outcome?: RepairOutcome; + selection?: FixtureSelection; + identities?: unknown; + mode?: { id: string; version: string }; +} + +function parentFixture(root: string, overrides: FixtureOverrides = {}) { + const app = join(root, 'app'); + mkdirSync(app, { recursive: true }); + writeFileSync(join(app, 'app.js'), 'export const broken = true;\n'); + const id = 'parent-run'; + const repair = overrides.repair ?? { status: 'budget-exhausted', limit: 3, + used: 3, stopReason: 'budget-exhausted' }; + const outcome = overrides.outcome ?? { kind: 'app_failure', phase: 'grading', reason: null, + appFailures: ['feature/failure'], inconclusive: [], harnessFailures: [] }; + const selection = overrides.selection ?? { sha256: 'c'.repeat(64), scoredPoints: 2, + recipe: { id: 'ecommerce.sequential-l1' } }; + const identities = overrides.identities ?? emptyArtifactIdentities({ + agentAdapter: { id: 'deterministic', version: '1.0.0', sha256: 'b'.repeat(64) }, + stackAdapter: { id: 'stub', version: '1.0.0' }, + }); + const checkpoint = preserveLevelCheckpoint({ appDir: app, outputDir: root, runId: id, + identities, track: 'loop', backend: 'stub', level: 1, repair, outcome, + selectionSha256: selection.sha256 }); + const level = { level: 1, graded: true, score: 1, max: 2, repairs: repair.used, + repair, outcome, selection, checkpoint, buildCostUsd: 2, repairCostUsd: 0.5, durationSec: 60 }; + const downstream = { level: 2, graded: true, score: 4, max: 4, repairs: 0, + repair: { status: 'not-needed', limit: 3, used: 0, stopReason: 'not-needed' }, + outcome: { kind: 'passed', phase: 'grading', reason: null, appFailures: [], inconclusive: [], + harnessFailures: [] }, buildCostUsd: 99, repairCostUsd: 0, durationSec: 100 }; + writeRunJson(join(root, 'run.json'), { + id, + kind: 'benchmark_run', + startedAt: '2026-08-16T12:00:00.000Z', + completedAt: '2026-08-16T12:01:00.000Z', + identities, + track: 'loop', backend: 'stub', model: 'deterministic', guidance: 'prescribed', + condition: null, selectionRequest: { packs: [], checks: [] }, skills: [], + ...(overrides.productionQuality === undefined ? {} : { productionQuality: overrides.productionQuality }), + ...(overrides.mode ? { mode: overrides.mode } : {}), + runtime: { buildImage: 'test-build-image', url: 'http://localhost:1234' }, + backendLease: { runIndex: 0 }, + levels: [level, downstream], contaminated: false, + totals: { score: 5, max: 6, repairs: repair.used, costUsd: 101.5, durationSec: 160 }, + outcome: { kind: outcome.kind, levels: { 1: outcome, 2: downstream.outcome } }, + }); + return { app, level, checkpoint }; +} + +test('a finite grant is derived only from the exact exhausted parent checkpoint', () => { + for (const productionQuality of [undefined, true]) { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-repair-grant-')); + try { + const fixture = parentFixture(root, { productionQuality }); + const resolved = createRepairGrant(root, { level: 1, repairs: 4 }); + assert.equal(resolved.parent.id, 'parent-run'); + assert.equal(resolved.sourcePath, join(root, fixture.checkpoint.directory)); + assert.equal(resolved.grant.repairsGranted, 4); + assert.equal(resolved.grant.cumulativeRepairsBefore, 3); + assert.equal(resolved.grant.cumulativeRepairsAfter, 3); + assert.equal(resolved.configuration.agentAdapter, 'deterministic'); + assert.equal(resolved.configuration.recipe, 'ecommerce.sequential-l1'); + assert.equal(resolved.configuration.runIndex, 0); + assert.equal(resolved.configuration.url, 'http://localhost:1234'); + assert.equal(resolved.configuration.productionQuality, productionQuality); + assert.equal(resolved.grant.cumulativeCostBeforeUsd, 101.5); + assert.equal(resolved.grant.cumulativeDurationBeforeSec, 160); + assert.deepEqual(resolved.grant.downstreamLevelsToRerun, [2]); + } finally { rmSync(root, { recursive: true, force: true }); } + } +}); + +test('a finite grant can continue an explicitly paused checkpoint', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-repair-paused-')); + try { + for (const stopReason of ['repeated-findings', 'no-source-change']) { + rmSync(root, { recursive: true, force: true }); + parentFixture(root, { repair: { status: 'incomplete', limit: 10, + used: 4, stopReason } }); + const resolved = createRepairGrant(root, { level: 1, repairs: 3 }); + assert.equal(resolved.grant.repairsGranted, 3); + assert.equal(resolved.grant.cumulativeRepairsBefore, 4); + } + rmSync(root, { recursive: true, force: true }); + parentFixture(root, { repair: { status: 'incomplete', limit: 3, + used: 3, stopReason: 'no-source-change' } }); + assert.doesNotThrow(() => inspectRepairParent(root, 1), 'a no-source-change pause can use the final round'); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('repair grants reject incomplete evidence, remaining budget, and changed source bytes', () => { + const incompleteRoot = mkdtempSync(join(tmpdir(), 'stack-bench-repair-incomplete-')); + const remainingRoot = mkdtempSync(join(tmpdir(), 'stack-bench-repair-remaining-')); + const changedRoot = mkdtempSync(join(tmpdir(), 'stack-bench-repair-changed-')); + const boundedRoot = mkdtempSync(join(tmpdir(), 'stack-bench-repair-rounds-')); + try { + parentFixture(incompleteRoot, { outcome: { kind: 'app_failure', phase: 'grading', reason: null, + appFailures: ['failure'], inconclusive: ['missing'], harnessFailures: [] } }); + assert.throws(() => inspectRepairParent(incompleteRoot, 1), /complete conclusive measurement/); + + parentFixture(remainingRoot, { repair: { status: 'incomplete', limit: 3, + used: 2, stopReason: 'agent-session-failure' } }); + assert.throws(() => inspectRepairParent(remainingRoot, 1), /did not exhaust or pause/); + + const changed = parentFixture(changedRoot); + writeFileSync(join(changedRoot, changed.checkpoint.directory, 'app.js'), + 'export const broken = false;\n'); + assert.throws(() => inspectRepairParent(changedRoot, 1), /source bytes do not match/); + + parentFixture(boundedRoot); + for (const repairs of [0, 1.5]) { + assert.throws(() => createRepairGrant(boundedRoot, { level: 1, repairs }), /positive safe integer/); + } + } finally { + rmSync(boundedRoot, { recursive: true, force: true }); + rmSync(incompleteRoot, { recursive: true, force: true }); + rmSync(remainingRoot, { recursive: true, force: true }); + rmSync(changedRoot, { recursive: true, force: true }); + } +}); + +test('generic repair grants reject dependency campaigns', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-repair-dependency-')); + try { + parentFixture(root, { mode: { id: 'dependency', version: '3.2.0' } }); + assert.throws(() => createRepairGrant(root, { level: 1, repairs: 2 }), + /do not support dependency campaigns/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('a continuation baseline must reproduce score, scope, and exact failed criteria', () => { + const parent = { score: 4, max: 6, selection: { sha256: 'a'.repeat(64) }, outcome: { + kind: 'app_failure', appFailures: ['suite/b', 'suite/a'], inconclusive: [], harnessFailures: [], + } }; + const exact = compareRepairBaseline(parent, { score: 4, max: 6, + selectionSha256: 'a'.repeat(64), sourceSha256: 'b'.repeat(64), + expectedSourceSha256: 'b'.repeat(64), outcome: { kind: 'app_failure', + appFailures: ['suite/a', 'suite/b'], inconclusive: [], harnessFailures: [] } }); + assert.deepEqual(exact, { reproduced: true, mismatches: [] }); + const changed = compareRepairBaseline(parent, { score: 4, max: 6, + selectionSha256: 'a'.repeat(64), sourceSha256: 'b'.repeat(64), + expectedSourceSha256: 'b'.repeat(64), outcome: { kind: 'app_failure', + appFailures: ['suite/a'], inconclusive: ['suite/b'], harnessFailures: [] } }); + assert.equal(changed.reproduced, false); + assert.deepEqual(changed.mismatches, ['measurement', 'failed criteria']); + const changedSource = compareRepairBaseline(parent, { score: 4, max: 6, + selectionSha256: 'a'.repeat(64), sourceSha256: 'c'.repeat(64), + expectedSourceSha256: 'b'.repeat(64), outcome: { kind: 'app_failure', + appFailures: ['suite/a', 'suite/b'], inconclusive: [], harnessFailures: [] } }); + assert.deepEqual(changedSource, { reproduced: false, mismatches: ['source'] }); +}); diff --git a/tools/stack-bench/tests/repair-source.integration.ts b/tools/stack-bench/tests/repair-source.integration.ts new file mode 100644 index 00000000000..17fa5adde7b --- /dev/null +++ b/tools/stack-bench/tests/repair-source.integration.ts @@ -0,0 +1,130 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { chmodSync, mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createServer } from 'node:net'; +import { setTimeout as delay } from 'node:timers/promises'; +import test from 'node:test'; +import { createBackendLease, writeBackendLease } from '../src/runtime/backend-lease.js'; +import { controlAppServer } from '../src/runtime/backend-control.js'; +import { restoreRepairSource } from '../src/runtime/source-materialization.js'; +import { snapshotAppSource } from '../src/runtime/source-snapshot.js'; +import { resetMutationDatabase } from '../grader/mutation-test.js'; + +const docker = (args: string[]): string => execFileSync('docker', args, + { encoding: 'utf8', stdio: 'pipe', timeout: 120_000, windowsHide: true }).trim(); +const freePort = async (): Promise => { + const server = createServer(); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + assert(address && typeof address !== 'string'); + await new Promise(resolve => server.close(() => resolve())); + return address.port; +}; + +test('rejected schema repair restores accepted source and a fresh spacetime dev watcher', { + skip: process.env.STACK_BENCH_REPAIR_SOURCE_DOCKER !== '1', timeout: 240_000, +}, async () => { + const image = process.env.STACK_BENCH_REPAIR_BUILD_IMAGE; + const deps = process.env.STACK_BENCH_REPAIR_DEPS_VOLUME; + assert(image && deps, 'set immutable build image and release dependency volume'); + const root = mkdtempSync(join(tmpdir(), 'stack-bench-repair-')); + const app = join(root, 'app'); + const accepted = join(root, 'accepted'); + const name = `stack-bench-repair-proof-${process.pid}-${Date.now()}`; + const port = await freePort(); + const backendPort = await freePort(); + const source = join(app, 'spacetimedb', 'src', 'index.ts'); + mkdirSync(join(app, 'spacetimedb', 'src'), { recursive: true }); + writeFileSync(join(app, 'spacetimedb', 'package.json'), JSON.stringify({ + type: 'module', dependencies: { spacetimedb: 'file:/deps/spacetimedb.tgz' }, + })); + writeFileSync(join(app, 'spacetimedb', 'tsconfig.json'), '{}'); + const moduleSource = (type: string, value: string): string => + `import { schema, table, t } from 'spacetimedb/server';\n` + + `const db = schema({ proof: table({ public: true }, { value: t.${type}() }) });\n` + + `export default db; export const seed = db.reducer(ctx => { if (ctx.db.proof.count() === 0n) ctx.db.proof.insert({ value: ${value} }); });\n`; + writeFileSync(source, moduleSource('string', "'accepted'")); + writeFileSync(join(app, 'server.cjs'), `require('http').createServer((q,s)=>s.end('ready')).listen(${port},'0.0.0.0');`); + writeFileSync(join(app, 'start.sh'), `#!/bin/bash\nset -eu\n` + + `(cd /app/spacetimedb && npm ci --ignore-scripts --no-audit --no-fund)\n` + + `exec /deps/spacetimedb-cli dev repair-proof --no-config --project-path /app --module-path /app/spacetimedb ` + + `--module-bindings-path client/src/module_bindings --client-lang typescript -s http://127.0.0.1:3299 -y ` + + `--run '/deps/spacetimedb-cli call repair-proof seed -s http://127.0.0.1:3299 -y && node /app/server.cjs'\n`); + let id: string | undefined; + const previous = { path: process.env.STACK_BENCH_LEASE, token: process.env.STACK_BENCH_LEASE_TOKEN }; + try { + id = docker(['run', '-d', '--name', name, '--init', '--pull=never', + '-p', `127.0.0.1:${port}:${port}`, '-p', `127.0.0.1:${backendPort}:3299`, + '--mount', `type=bind,source=${app},target=/app`, + '--mount', `type=volume,source=${deps},target=/deps,readonly`, + '--entrypoint', 'sh', image, '-c', + 'mkdir -p /run/application /home/developer; chown 10001:10001 /home/developer; ' + + 'openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out /tmp/key.pem; ' + + 'openssl ec -in /tmp/key.pem -pubout -out /tmp/public.pem; ' + + 'exec /deps/spacetimedb-standalone start --listen-addr 0.0.0.0:3299 --data-dir /tmp/backend ' + + '--jwt-priv-key-path /tmp/key.pem --jwt-pub-key-path /tmp/public.pem']); + for (let attempt = 0; ; attempt++) { + try { docker(['exec', id, 'curl', '-fsS', 'http://127.0.0.1:3299/v1/ping']); break; } + catch (error) { if (attempt === 40) throw error; await delay(250); } + } + docker(['exec', '-w', '/app/spacetimedb', id, 'npm', 'install', '--ignore-scripts', '--no-audit', '--no-fund']); + const lease = createBackendLease({ runId: name, backend: 'spacetime', track: 'chat', runIndex: 0, + serverUri: `http://127.0.0.1:${backendPort}`, module: 'repair-proof', dataDir: join(root, 'data') }); + lease.state = 'active'; + lease.resources.buildContainer = { name, id, image, running: true, owned: true, networkMode: 'bridge', + resourceLimits: { cpuCount: 2, memoryBytes: 2147483648, memorySwapBytes: 2147483648, pids: 512 } }; + const path = join(root, 'lease.json'); + writeBackendLease(path, lease); + process.env.STACK_BENCH_LEASE = path; + process.env.STACK_BENCH_LEASE_TOKEN = lease.ownershipToken; + const spec = { backend: 'spacetime', app, port, probe: '/' }; + const cli = (...args: string[]): string => docker(['exec', '--user', '10001:10001', + '-e', 'HOME=/home/developer', id!, '/deps/spacetimedb-cli', ...args]); + const publish = (...args: string[]): string => cli('publish', 'repair-proof', + '--module-path', '/app/spacetimedb', '-s', 'http://127.0.0.1:3299', '-y', ...args); + const query = (): string => cli('sql', 'repair-proof', '-s', 'http://127.0.0.1:3299', 'select * from proof'); + await controlAppServer(spec, 'start'); + assert.match(query(), /accepted/); + await resetMutationDatabase({ backend: 'spacetime', app, track: 'chat', + reseedOnReset: true, restartSpec: spec }, null); + assert.match(query(), /accepted/, 'isolated reset must run startup seeding outside init'); + await controlAppServer(spec, 'stop'); + chmodSync(source, 0o660); + chmodSync(join(app, 'spacetimedb', 'package.json'), 0o660); + snapshotAppSource(app, accepted); + writeFileSync(source, moduleSource('u32', '7')); + publish('--delete-data'); + await controlAppServer(spec, 'start'); + const watchers = (): string => docker(['exec', id!, 'sh', '-c', "pgrep -f '^/deps/spacetimedb-cli dev ' || true"]); + const rejectedWatcher = watchers(); + assert.match(rejectedWatcher, /^\d+$/); + await controlAppServer(spec, 'stop'); + assert.equal(watchers(), '', 'stop must terminate the rejected dev watcher'); + writeFileSync(source, moduleSource('string', "'accepted'")); + assert.throws(() => publish(), error => { + const failure = error as { stderr?: string; stdout?: string }; + return /migration|schema|breaking/i.test(`${failure.stderr}\n${failure.stdout}`); + }, 'source-only rollback must reproduce the incompatible-schema refusal'); + writeFileSync(source, moduleSource('u32', '7')); + await controlAppServer(spec, 'start'); + await restoreRepairSource(accepted, app, spec); + assert.match(query(), /accepted/, 'grading can read the accepted schema and seed after recovery'); + assert.match(watchers(), /^\d+$/); + assert.notEqual(watchers(), rejectedWatcher); + assert.equal((await fetch(`http://127.0.0.1:${port}/`)).status, 200); + await controlAppServer(spec, 'stop'); + assert.equal(watchers(), '', 'final stop must leave no dev watcher'); + } catch (error) { + if (id) console.error(docker(['logs', '--tail', '12', id])); + throw error; + } finally { + if (previous.path === undefined) delete process.env.STACK_BENCH_LEASE; + else process.env.STACK_BENCH_LEASE = previous.path; + if (previous.token === undefined) delete process.env.STACK_BENCH_LEASE_TOKEN; + else process.env.STACK_BENCH_LEASE_TOKEN = previous.token; + if (id) docker(['rm', '-f', id]); + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/tools/stack-bench/tests/repeat-form-write.integration.ts b/tools/stack-bench/tests/repeat-form-write.integration.ts new file mode 100644 index 00000000000..28df6c06b49 --- /dev/null +++ b/tools/stack-bench/tests/repeat-form-write.integration.ts @@ -0,0 +1,101 @@ +import assert from 'node:assert/strict'; +import { createServer } from 'node:http'; +import { once } from 'node:events'; +import { mkdirSync, writeFileSync } from 'node:fs'; +import test from 'node:test'; +import { chromium } from 'playwright'; +import { Actor } from '../grader/grade.js'; +import { ACTION_REGISTRY } from '../src/actions/action-catalog.js'; +import { executeAction } from '../src/actions/action-contract.js'; +import { classifyResponseContract } from '../src/actions/named-action-runtime.js'; + +// Failure cases, before implementation: partial/duplicate matches, unconfirmed +// responses, non-JSON bodies, foreign origins and absent capture use the form. +// Rejected or lost replay responses MUST NOT trigger another form submission. +// JSON quoting, current cookies and untouched business fields must survive replay. +// Two UI writes must differ only in the name. Generated IDs/nonces disable replay. +test('repeat form write selects UI before sending, never after an uncertain write', async () => { + let mode = '', calls = 0; + const rows: unknown[] = [], observations: unknown[] = []; + const server = createServer(async (req, res) => { + if (req.url === '/api/products') { + const parts = []; for await (const part of req) parts.push(part); + const body = JSON.parse(Buffer.concat(parts).toString()); calls++; + assert.equal(req.headers.cookie, 'session=current'); + if (body.name.startsWith('New') && mode === 'refused') { res.writeHead(403).end(); return; } + rows.push(body); + if (mode === 'lost' && body.name.startsWith('New')) { res.destroy(); return; } + res.setHeader('Content-Type', 'application/json'); res.end(mode.startsWith('convex-') + ? JSON.stringify({ status: 'success', value: null }) : '{}'); + } else { + res.setHeader('Content-Type', 'text/html'); + res.setHeader('Set-Cookie', 'session=current; Path=/'); + res.end(``); + } + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const browser = await chromium.launch(); + let result = 'failed'; + try { + for (mode of ['accepted', 'convex-http', 'convex-query', 'missing', 'partial', 'duplicate', 'duplicate-value', 'non-json', 'unconfirmed', 'foreign', 'changing-argument', 'refused', 'lost']) { + calls = 0; rows.length = 0; + const context = await browser.newContext(); + try { + const page = await context.newPage(), actor = new Actor('admin', page, context); + await actor.ready; await page.goto(url); + for (const name of ['Original product', 'Control product']) { + await page.locator('input').fill(name); + const finished = page.waitForEvent('requestfinished', req => req.url().endsWith('/api/products')); + await page.locator('button').click(); await finished; + } + // Let the actor's complete-response observer finish before selection. + await page.waitForTimeout(50); + if (mode === 'missing') actor.writes.length = 0; + if (mode === 'duplicate') actor.writes.push({ ...actor.writes[0]! }); + if (mode === 'partial') actor.writes[0]!.body!.name = 'Original product plus'; + if (mode === 'duplicate-value') actor.writes[0]!.body!.category = 'Original product'; + if (mode === 'non-json') actor.writes[0]!.headers['content-type'] = 'application/x-www-form-urlencoded'; + if (mode === 'unconfirmed') Object.assign(actor.writes[0]!, { confirmed: false }); + if (mode === 'foreign') actor.writes[0]!.url = 'https://foreign.invalid/api/products'; + const interaction = { defaultWithin: 1000, expand: (s: string) => s, sleep: async () => {} }; + const outcome = await executeAction(ACTION_REGISTRY, 'repeatFormWrite', { + do: 'repeatFormWrite', actor: 'admin', match: 'Original product', replacement: 'New "product" $1', + control: 'Control product', + fields: [{ testid: 'name', text: 'New "product" $1' }], submit: 'save', + }, { capabilities: { actors: { get: () => actor }, 'browser-interaction': interaction, + 'transport-observation': interaction, 'named-actions': { fetch, + classifyResponse: (request: { url: string }, response: { status: number; text: string }) => + classifyResponseContract({ ...request, responseContract: mode === 'convex-http' ? 'convex-mutation' + : mode === 'convex-query' ? 'convex-query' : 'http' }, response), + } } }); + await page.waitForTimeout(50); + const replay = ['accepted', 'convex-http', 'refused', 'lost'].includes(mode); + assert.equal(await page.evaluate(() => (window as unknown as { formSubmits: number }).formSubmits), replay ? 2 : 3, mode); + assert.equal(calls, 3, `${mode}: a replay must never retry`); + assert.equal(outcome.status, mode === 'refused' ? 'failed' : mode === 'lost' ? 'inconclusive' : 'passed', JSON.stringify(outcome)); + assert.deepEqual(rows, [ + { name: 'Original product', price: 1.25, category: 'Volume', ...mode === 'changing-argument' ? { nonce: 1 } : {} }, + { name: 'Control product', price: 1.25, category: 'Volume', ...mode === 'changing-argument' ? { nonce: 2 } : {} }, + ...mode === 'refused' ? [] : [{ name: 'New "product" $1', price: 1.25, category: 'Volume', ...mode === 'changing-argument' ? { nonce: 3 } : {} }], + ]); + observations.push({ mode, outcome, requests: calls }); + } finally { await context.close(); } + } + result = 'passed'; + } finally { + await browser.close(); server.closeAllConnections(); await new Promise(resolve => server.close(() => resolve())); + if (process.env.STACK_BENCH_REPLAY_EVIDENCE) { + mkdirSync(process.env.STACK_BENCH_REPLAY_EVIDENCE, { recursive: true }); + writeFileSync(`${process.env.STACK_BENCH_REPLAY_EVIDENCE}/form-receipt.json`, JSON.stringify({ result, observations, + rerun: 'node --test dist/tests/repeat-form-write.integration.js' }, null, 2)); + } + } +}); diff --git a/tools/stack-bench/tests/report-bugs.test.ts b/tools/stack-bench/tests/report-bugs.test.ts new file mode 100644 index 00000000000..08d1437b176 --- /dev/null +++ b/tools/stack-bench/tests/report-bugs.test.ts @@ -0,0 +1,576 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { writeArtifact } from '../src/evidence/artifacts.js'; +import { createCheckEvidence } from '../src/evidence/check-evidence.js'; +import { finding } from '../src/actions/action-findings.js'; +import type { ActionEvidence } from '../src/actions/action-contract.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { privateGradingDirectory } from '../src/evidence/repair-evidence.js'; + +const CLI = join(STACK_BENCH_ROOT, 'dist', 'commands', 'report-bugs.js'); + +type EvidenceStatus = 'passed' | 'failed' | 'blocked' | 'inconclusive' | 'harness_failure'; + +interface WriteGradeOptions { + grading?: string; + feature?: string; + points?: number; + criterion?: string; + stableKey?: string; + file?: string; + url?: string; + consoleErrors?: string[]; + statedBy?: string; + desc?: string; + evidence?: ReturnType; + setupEvidence?: ReturnType; +} + +function writeGrade(app: string, status: EvidenceStatus, summary: string, + { grading = privateGradingDirectory(app), feature = 'Accounts', points = 1, + criterion = 'owner', stableKey = criterion, file = 'grading-features.json', + url = 'http://app', consoleErrors = [], statedBy, desc, evidence: suppliedEvidence, + setupEvidence: suppliedSetup }: + WriteGradeOptions = {}): void { + mkdirSync(app, { recursive: true }); + mkdirSync(grading, { recursive: true }); + const setupEvidence = suppliedSetup ?? (suppliedEvidence?.phase === 'setup' ? suppliedEvidence + : createCheckEvidence({ status: 'passed', code: 'completed', phase: 'setup', + startedAtMs: 1, completedAtMs: 2 })); + const evidence = suppliedEvidence ?? createCheckEvidence({ status, + code: status === 'passed' ? 'completed' : 'test_result', phase: 'assertion', summary, + startedAtMs: 3, completedAtMs: 4 }); + writeArtifact(join(grading, file), { + kind: 'grade', id: 'repair-selection-grade', identities: {}, + payload: { + url, + total: status === 'passed' ? points : 0, + max: status === 'inconclusive' || status === 'harness_failure' ? 0 : points, + features: [{ id: 1, name: feature, score: status === 'passed' ? points : 0, + max: status === 'inconclusive' || status === 'harness_failure' ? 0 : points, + setupEvidence, consoleErrors, criteria: [{ id: criterion, stableKey, + desc: desc ?? statedBy ?? `Expected ${criterion}`, ...(statedBy ? { statedBy } : {}), points, evidence }] }], + }, + }); +} + +// The agent owns the app directory; a planted link must not redirect a controller write. +test('repair report replaces a planted symlink instead of writing through it', t => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-repair-symlink-')); + try { + const app = join(root, 'app'); + writeGrade(app, 'failed', 'cart total was wrong'); + const victim = join(root, 'victim'); + writeFileSync(victim, 'controller secret'); + try { symlinkSync(victim, join(app, 'BUG_REPORT.md')); } + catch { t.skip('this host cannot create file symlinks'); return; } + const reported = spawnSync(process.execPath, [CLI, '--app', app], { encoding: 'utf8' }); + assert.equal(reported.status, 0, reported.stderr); + assert.equal(readFileSync(victim, 'utf8'), 'controller secret'); + assert.ok(lstatSync(join(app, 'BUG_REPORT.md')).isFile()); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('repair report selection follows typed evidence even when prose claims the opposite', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-repair-selection-')); + try { + const harnessApp = join(root, 'harness'); + writeGrade(harnessApp, 'harness_failure', 'FAILED: the generated app is definitely broken'); + const skipped = spawnSync(process.execPath, [CLI, '--app', harnessApp], { encoding: 'utf8' }); + assert.equal(skipped.status, 3, skipped.stderr); + assert.equal(existsSync(join(harnessApp, 'BUG_REPORT.md')), false); + + const failedApp = join(root, 'failed'); + writeGrade(failedApp, 'failed', 'INCONCLUSIVE: this wording must not suppress repair'); + const reported = spawnSync(process.execPath, [CLI, '--app', failedApp], { encoding: 'utf8' }); + assert.equal(reported.status, 0, reported.stderr); + const repair = readFileSync(join(failedApp, 'BUG_REPORT.md'), 'utf8'); + assert.match(repair, /Actual:\*\* a failure was recorded without a detailed observation/); + assert.doesNotMatch(repair, /INCONCLUSIVE: this wording/); + for (const status of ['passed', 'inconclusive', 'harness_failure'] as const) { + const app = join(root, `setup-${status}`); + const copied = createCheckEvidence({ status: 'failed', code: 'test_result', phase: 'setup', + startedAtMs: 1, completedAtMs: 2 }); + const setup = createCheckEvidence({ status, code: 'test_result', phase: 'setup', + startedAtMs: 1, completedAtMs: 2 }); + writeGrade(app, 'failed', 'outer status cannot make setup repairable', { evidence: copied, setupEvidence: setup }); + const result = spawnSync(process.execPath, [CLI, '--app', app], { encoding: 'utf8' }); + assert.equal(result.status, 3, result.stderr); + assert.equal(existsSync(join(app, 'BUG_REPORT.md')), false); + } + const app = join(root, 'unmeasured-finding'); + writeGrade(app, 'failed', 'status cannot make an inconclusive finding repairable', { + evidence: createCheckEvidence({ status: 'failed', code: 'test_result', phase: 'assertion', + finding: finding('no-backend-control', { target: 'backend-runtime' }), startedAtMs: 1, completedAtMs: 2 }), + }); + const result = spawnSync(process.execPath, [CLI, '--app', app], { encoding: 'utf8' }); + assert.equal(result.status, 3, result.stderr); + assert.equal(existsSync(join(app, 'BUG_REPORT.md')), false); + const zeroPoint = join(root, 'zero-point'); + writeGrade(zeroPoint, 'failed', 'candidate behavior failed', + { feature: 'Candidate concurrency check', points: 0 }); + const zero = spawnSync(process.execPath, [CLI, '--app', zeroPoint], { encoding: 'utf8' }); + assert.equal(zero.status, 3, zero.stderr); + assert.equal(existsSync(join(zeroPoint, 'BUG_REPORT.md')), false); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('repair feedback includes actionable runtime evidence without private artifact names', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-repair-diagnostics-')); + try { + const app = join(root, 'app'); + const total = finding('number-mismatch', { control: 'cart-total', observed: 9, expected: { equals: 12 } }); + const evidence = createCheckEvidence({ + status: 'failed', code: 'test_result', phase: 'assertion', actor: 'buyer', + summary: 'cart total was wrong', observation: { value: 9 }, expected: { value: 12 }, + finding: total, startedAtMs: 3, completedAtMs: 4, + actions: [{ actor: 'buyer', evidence: { + schemaVersion: 2, action: { id: 'expectNumber', version: '1.0.0' }, + status: 'failed', type: 'browser-number', code: 'application_failure', phase: 'execute', + summary: 'cart total was wrong', finding: total, observation: { value: 9 }, expected: { value: 12 }, + retryable: false, timing: { startedAtMs: 3, completedAtMs: 4, durationMs: 1, + deadlineMs: 5_000 }, attachments: [], sensitivity: [], + } }], + attachments: [{ kind: 'screenshot', ref: 'failure-buyer.png' }], + }); + writeGrade(app, 'failed', 'cart total was wrong', { + feature: 'Cart', criterion: 'total', stableKey: 'private.check.cart.total', + desc: 'the cart total equals the sum of its lines, with a 5 percent discount', + statedBy: 'the cart survives a reload', + url: 'http://app/cart', consoleErrors: ['POST /api/cart returned HTTP 500'], evidence, + }); + + const reported = spawnSync(process.execPath, [CLI, '--app', app], { encoding: 'utf8' }); + assert.equal(reported.status, 0, reported.stderr); + const repair = readFileSync(join(app, 'BUG_REPORT.md'), 'utf8'); + assert.match(repair, /Actor\/session:\*\* buyer/); + assert.match(repair, /Expected:\*\* the cart total equals the sum of its lines/); + assert.match(repair, /Actual:\*\* the cart-total control reads 9, expected exactly 12/); + assert.match(repair, /5 percent discount/); + assert.doesNotMatch(repair, /the cart survives a reload/); + assert.match(readFileSync(join(privateGradingDirectory(app), 'grading-features.json'), 'utf8'), /"equals": 12/); + assert.doesNotMatch(repair, /cart total was wrong/); + assert.doesNotMatch(repair, /Application URL|http:\/\//); + assert.doesNotMatch(repair, /failure-buyer\.png/); + assert.match(repair, /Console or network errors:[\s\S]*HTTP 500/); + assert.doesNotMatch(repair, /expect number|private\.check|Stack Bench|grader|criterion/i); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('failed reload reports its transport error without claiming later checks ran or leaking diagnostics', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-repair-reload-')); + try { + for (const [code, expected] of [['ERR_CONNECTION_REFUSED', 'the browser request failed (ERR_CONNECTION_REFUSED)'], + ['ERR_PRIVATE_DIAGNOSTIC', 'the page did not behave as required']]) { + const app = join(root, code!); + const detail = `page.reload: net::${code} at http://user:password@app/private-probe?token=secret\nPRIVATE_STACK`; + const failure = finding('page-error', { detail }); + const evidence = createCheckEvidence({ status: 'failed', code: 'test_result', phase: 'assertion', + actor: 'owner', summary: detail, finding: failure, startedAtMs: 1, completedAtMs: 2, + actions: [{ actor: 'owner', evidence: { + schemaVersion: 2, action: { id: 'reload', version: '1.0.0' }, status: 'failed', + type: 'browser-interaction-evidence', code: 'application_failure', phase: 'execute', + summary: detail, finding: failure, observation: null, expected: null, retryable: false, + timing: { startedAtMs: 1, completedAtMs: 2, durationMs: 1, deadlineMs: 5000 }, + attachments: [], sensitivity: [], + } }], + }); + writeGrade(app, 'failed', detail, { evidence, desc: 'saved history survives reload and a fresh login' }); + const reported = spawnSync(process.execPath, [CLI, '--app', app], { encoding: 'utf8' }); + assert.equal(reported.status, 0, reported.stderr); + const report = readFileSync(join(app, 'BUG_REPORT.md'), 'utf8'); + assert(report.includes(`**Actual:** ${expected}`)); + assert.match(report, /Failed action:\*\* Reload the page/); + assert.match(report, /sequence stopped at this action; later behavior was not observed/); + assert.doesNotMatch(report, /Completed lifecycle actions|page reloaded|PRIVATE_|private-probe|password|secret|http:\/\//); + } + const app = join(root, 'choice'); + const detail = `locator.selectOption: Timeout 5000ms exceeded.\n` + + `waiting for locator('[data-testid="notification-frequency"]')`; + const choice = finding('choice-missing', { control: 'notification-frequency', detail }); + const evidence = createCheckEvidence({ + status: 'failed', code: 'test_result', phase: 'assertion', summary: detail, + finding: choice, observation: detail, startedAtMs: 3, completedAtMs: 4, + actions: [{ actor: 'owner', evidence: { + schemaVersion: 2, action: { id: 'fill', version: '1.0.0' }, + status: 'failed', type: 'browser-interaction-evidence', code: 'application_failure', + phase: 'execute', summary: detail, finding: choice, observation: null, expected: null, + retryable: false, timing: { startedAtMs: 3, completedAtMs: 4, + durationMs: 1, deadlineMs: 60_000 }, attachments: [], sensitivity: [], + } }], + }); + writeGrade(app, 'failed', detail, { evidence }); + + const selected = spawnSync(process.execPath, [CLI, '--app', app], { encoding: 'utf8' }); + assert.equal(selected.status, 0, selected.stderr); + const choiceReport = readFileSync(join(app, 'BUG_REPORT.md'), 'utf8'); + assert.match(choiceReport, /Actual:\*\* the notification-frequency control did not offer the required choice/); + assert.match(choiceReport, /Failed action:\*\* Select the requested choice/); + assert.doesNotMatch(choiceReport, /locator|selectOption|data-(?:role|testid)|Timeout|5000ms|http:\/\//); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('repair feedback refuses internal evaluation language', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-repair-disclosure-')); + try { + const app = join(root, 'app'); + writeGrade(app, 'failed', 'x', { statedBy: 'the Stack Bench test failed' }); + const reported = spawnSync(process.execPath, [CLI, '--app', app], { encoding: 'utf8' }); + assert.equal(reported.status, 2); + assert.match(reported.stderr, /contains internal language/); + assert.equal(existsSync(join(app, 'BUG_REPORT.md')), false); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('repair context identifies early control and value failures without claiming later durability was observed', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-repair-context-')); + try { + for (const missing of [finding('control-missing', { control: 'cart-item', filtered: true }), + finding('value-mismatch', { control: 'staff-role-select', observed: 'staff', expected: 'inventory' }), + finding('number-mismatch', { control: 'item-stock', observed: 100, expected: { equals: 99 } })]) { + const action = (id: string, observation: unknown, failed = false): { actor: string; evidence: ActionEvidence } => ({ + actor: 'shopper', evidence: { + schemaVersion: 2, action: { id, version: '1.0.0' }, status: failed ? 'failed' : 'passed', + type: 'test-evidence', code: failed ? 'application_failure' : 'completed', phase: 'execute', + summary: 'PRIVATE_DIAGNOSTIC', finding: failed ? missing : null, + observation, expected: { internal: 'PRIVATE_EXPECTATION' }, retryable: false, + timing: { startedAtMs: 1, completedAtMs: 2, durationMs: 1, deadlineMs: 100 }, + attachments: [], sensitivity: [], + }, + }); + for (const restarted of [false, true]) { + const app = join(root, restarted ? 'after' : 'before'); + const actions = ['add-to-cart', 'cart-toggle', 'checkout-submit', 'catalog-link', 'add-to-cart', 'cart-toggle'] + .map(control => action('click', { clicked: control })); + actions.unshift(action('callAction', { action: 'submitReview', accepted: true, status: 201, + body: 'PRIVATE_BODY', url: 'http://PRIVATE_URL' })); + actions.push(action('click', { clicked: false, testid: 'SKIPPED_CONTROL' }), + action('fill', { value: 'PRIVATE_PASSWORD' }), action('runScript', { script: 'PRIVATE_SCRIPT' })); + if (restarted) actions.push(action('reload', { reloaded: true }), + action('stopAppServer', { operation: 'stop' }), action('startAppServer', { operation: 'start' }), + action('restartBackend', { operation: 'restart' })); + actions.push(action('expect', null, true)); + // A later record must not be reported as a completed action before the failure. + actions.push(action('restartBackend', { operation: 'restart' })); + actions.push(action('callAction', { action: 'FUTURE_STRANGER_REQUEST', accepted: false, status: 403 })); + const evidence = createCheckEvidence({ status: 'failed', code: 'application_failure', + phase: 'assertion', actor: 'shopper', finding: missing, actions, + startedAtMs: 1, completedAtMs: 2 }); + writeGrade(app, 'failed', 'missing item', { evidence, feature: 'Checkout', + desc: 'cart and order history survive a page reload and runtime restart' }); + const result = spawnSync(process.execPath, [CLI, '--app', app], { encoding: 'utf8' }); + assert.equal(result.status, 0, result.stderr); + const report = readFileSync(join(app, 'BUG_REPORT.md'), 'utf8'); + assert.match(report, /Expected:\*\* cart and order history survive a page reload and runtime restart/); + assert.match(report, /shopper: checkout-submit → shopper: catalog-link → shopper: add-to-cart → shopper: cart-toggle/); + assert.match(report, /shopper: submitReview returned HTTP 201/); + assert.match(report, /The sequence stopped at this (control|value check); later behavior was not observed/); + if (restarted) assert.match(report, /Completed lifecycle actions: page reloaded; application server stopped; application server started; database runtime restarted/); + else assert.doesNotMatch(report, /Completed lifecycle actions/); + assert.doesNotMatch(report, /PRIVATE_|FUTURE_STRANGER_REQUEST|SKIPPED_CONTROL|lost|use a transaction|implement|retry policy/i); + } + } + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('copied setup failures use original observations once and retain distinct setup failures', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-repair-setup-dedup-')); + try { + const copied = createCheckEvidence({ status: 'failed', code: 'application_failure', + phase: 'setup', startedAtMs: 1, completedAtMs: 2 }); + const setup = createCheckEvidence({ status: 'failed', code: 'application_failure', + phase: 'setup', actor: 'buyer', startedAtMs: 1, completedAtMs: 2, + finding: finding('number-mismatch', { control: 'item-stock', observed: 100, + expected: { equals: 99 }, scopeText: 'Bluetooth Speaker' }) }); + for (const criterion of ['authentication', 'ownership']) { + writeGrade(root, 'failed', 'later behavior did not run', { criterion, + file: `grading-${criterion}.json`, evidence: copied, setupEvidence: setup, + statedBy: 'unauthorized purchases must be refused', + consoleErrors: ['POST /api/buy returned 500', 'POST /api/buy returned 500'] }); + } + writeGrade(root, 'failed', 'different setup failure', { criterion: 'other', + file: 'grading-other.json', evidence: copied, + setupEvidence: createCheckEvidence({ ...setup, + startedAtMs: 3, completedAtMs: 4, + finding: finding('control-missing', { control: 'orders-toggle', filtered: false }) }), + statedBy: 'orders must survive a restart' }); + const reported = spawnSync(process.execPath, [CLI, '--app', root], { encoding: 'utf8' }); + assert.equal(reported.status, 0, reported.stderr); + const report = readFileSync(join(root, 'BUG_REPORT.md'), 'utf8'); + assert.equal((report.match(/### Bug /g) ?? []).length, 2); + assert.equal((report.match(/item-stock control reads 100, expected exactly 99/g) ?? []).length, 1); + assert.match(report, /orders-toggle control did not appear/); + assert.match(report, /entry matching "Bluetooth Speaker"/); + assert.equal((report.match(/Setup stopped before the named behavior was reached/g) ?? []).length, 2); + assert.equal((report.match(/POST \/api\/buy returned 500/g) ?? []).length, 1); + assert.doesNotMatch(report, /unauthorized purchases|orders must survive|Expected:\*\*/); + + const blocked = join(root, 'blocked'); + const blockedSetup = createCheckEvidence({ status: 'failed', code: 'application_failure', + phase: 'setup', startedAtMs: 1, completedAtMs: 2, + finding: finding('control-missing', { control: 'item-stock', filtered: false }) }); + writeGrade(blocked, 'blocked', 'setup failed', { evidence: { ...blockedSetup, status: 'blocked' }, + setupEvidence: blockedSetup, statedBy: 'the server refuses an unauthenticated purchase' }); + const blockedResult = spawnSync(process.execPath, [CLI, '--app', blocked], { encoding: 'utf8' }); + assert.equal(blockedResult.status, 0, blockedResult.stderr); + const blockedReport = readFileSync(join(blocked, 'BUG_REPORT.md'), 'utf8'); + assert.match(blockedReport, /item-stock control did not appear/); + assert.match(blockedReport, /Setup stopped before the named behavior was reached/); + assert.doesNotMatch(blockedReport, /Expected:|unauthenticated purchase/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('dependency repair feedback contains only checks selected for that feature', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-repair-check-selection-')); + try { + const app = join(root, 'app'); + writeGrade(app, 'failed', 'account ownership failed', { + criterion: 'owner', stableKey: 'check.accounts.owner', + file: 'grading-accounts.json', feature: 'Accounts', + }); + writeGrade(app, 'failed', 'catalog search failed', { + criterion: 'search', stableKey: 'check.catalog.search', + file: 'grading-catalog.json', feature: 'Catalog', + }); + const reported = spawnSync(process.execPath, [CLI, '--app', app, + '--checks-json', JSON.stringify(['check.accounts.owner']), + '--history-json', JSON.stringify([{ round: 1, result: 'incomplete', + remainingFailures: ['check.accounts.owner', 'check.catalog.search'] }])], + { encoding: 'utf8' }); + assert.equal(reported.status, 0, reported.stderr); + const repair = readFileSync(join(app, 'BUG_REPORT.md'), 'utf8'); + assert.match(repair, /Accounts|account ownership failed/); + assert.doesNotMatch(repair, /Catalog|catalog search failed/); + assert.doesNotMatch(repair, /check\.catalog\.search/); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('repair feedback includes failures caused by the rejected prior repair', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-repair-regression-')); + try { + const app = join(root, 'app'); + writeGrade(app, 'failed', 'checkout still fails', { + feature: 'Checkout', criterion: 'checkout', stableKey: 'check.checkout', + }); + const regression = join(root, 'regression.md'); + writeFileSync(regression, [ + '## Behavior', '', + '### Bug 1: Accounts', '', '**Expected:** the owner keeps access', '', + '**Actual:** the owner was signed out', '', + ].join('\n')); + + const reported = spawnSync(process.execPath, [CLI, '--app', app, + '--prior-regression', regression], { encoding: 'utf8' }); + assert.equal(reported.status, 0, reported.stderr); + const repair = readFileSync(join(app, 'BUG_REPORT.md'), 'utf8'); + assert.match(repair, /Actual:\*\* a failure was recorded without a detailed observation/); + assert.match(repair, /Previous repair regression/); + assert.match(repair, /Accounts/); + assert.match(repair, /owner was signed out/); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('expected failures enter repairs while observed-only failures stay isolated', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-treatment-repair-')); + try { + const app = join(root, 'app'); + const grading = join(root, 'isolated-grading'); + writeGrade(app, 'failed', 'durability was expected but state was lost', + { grading, feature: 'State durability' }); + writeGrade(app, 'failed', 'observed-only failure must not enter repair', { + grading: join(root, 'run', 'first-build-l1-observed'), feature: 'Observed behavior', + }); + + const reported = spawnSync(process.execPath, [CLI, '--app', app, '--results', grading], + { encoding: 'utf8' }); + assert.equal(reported.status, 0, reported.stderr); + const repair = readFileSync(join(app, 'BUG_REPORT.md'), 'utf8'); + assert.match(repair, /State durability/); + assert.match(repair, /Expected:\*\* Expected owner/); + assert.doesNotMatch(repair, /observed-only failure must not enter repair|Observed behavior/); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('application setup failures become actionable repair feedback without criterion results', () => { + for (const [phase, reason, expected] of [ + ['application-restart', 'database reset failed: server/package.json has no dev or start script', + /must provide \/app\/start\.sh/], + ['database-provenance', 'app did not write its marker to the database supplied for this run: marker not found', + /must use the postgres database and connection supplied for this run/], + ] as const) { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-setup-repair-')); + try { + const app = join(root, 'app'); + const grading = privateGradingDirectory(app); + mkdirSync(app, { recursive: true }); + mkdirSync(grading, { recursive: true }); + writeArtifact(join(grading, 'bundle.json'), { + kind: 'grade_bundle', id: 'setup-failure-bundle', identities: {}, + payload: { + definitionSchemaVersion: 1, recipeRelease: null, calibration: null, + label: 'postgres-l1', track: 'ecommerce', backend: 'postgres', url: 'http://app', + app, level: 1, observation: 'scored', suites: {}, + totals: { score: 0, max: 58, dirty: false, contractPass: null, regression: null }, + error: reason, + outcome: { kind: 'app_failure', phase, reason, appFailures: [phase] }, + selection: null, + }, + }); + const reported = spawnSync(process.execPath, [CLI, '--app', app], { encoding: 'utf8' }); + assert.equal(reported.status, 0, reported.stderr); + const repair = readFileSync(join(app, 'BUG_REPORT.md'), 'utf8'); + assert.match(repair, /Application setup/); + assert.match(repair, expected); + assert(repair.includes(reason.slice(-24)), repair); + } finally { + rmSync(root, { recursive: true, force: true }); + } + } +}); + +test('contract feedback reports the clean-state observation without claiming the element exists', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-contract-repair-')); + try { + const app = join(root, 'app'); + const grading = privateGradingDirectory(app); + mkdirSync(app, { recursive: true }); + mkdirSync(grading, { recursive: true }); + writeArtifact(join(grading, 'contract-lint.json'), { + kind: 'contract_lint', id: 'contract-repair-lint', identities: {}, + payload: { + label: 'spacetime-l1', url: 'http://app', level: 1, pass: false, + counts: { lintable: 2, pass: 0, fail: 1, blocked: 1, scenario: 1 }, + results: [ + { id: 'review-average', status: 'FAIL', + detail: 'no element matching [data-testid="review-average"] became visible — expected: the item average as a number' }, + { id: 'cart-panel', status: 'BLOCKED', detail: 'earlier core flow step failed' }, + ], + }, + }); + + const reported = spawnSync(process.execPath, [CLI, '--app', app], { encoding: 'utf8' }); + assert.equal(reported.status, 0, reported.stderr); + const repair = readFileSync(join(app, 'BUG_REPORT.md'), 'utf8'); + assert.match(repair, /clean application state/); + assert.match(repair, /review-average/); + assert.match(repair, /no element matching/); + assert.doesNotMatch(repair, /These elements exist/); + assert.doesNotMatch(repair, /cart-panel/); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('dependency repair feedback contains only interfaces selected for that feature', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-repair-control-selection-')); + try { + const app = join(root, 'app'); + const grading = privateGradingDirectory(app); + mkdirSync(app, { recursive: true }); + mkdirSync(grading, { recursive: true }); + writeArtifact(join(grading, 'contract-lint.json'), { + kind: 'contract_lint', id: 'contract-repair-lint', identities: {}, + payload: { + label: 'postgres-l1', url: 'http://app', level: 1, pass: false, + counts: { lintable: 2, pass: 0, fail: 2, blocked: 0, scenario: 1 }, + results: [ + { id: 'account-menu', status: 'FAIL', detail: 'account menu missing' }, + { id: 'catalog-search', status: 'FAIL', detail: 'catalog search missing' }, + ], + }, + }); + const reported = spawnSync(process.execPath, [CLI, '--app', app, + '--checks-json', JSON.stringify(['check.accounts.owner']), + '--controls-json', JSON.stringify(['account-menu'])], { encoding: 'utf8' }); + assert.equal(reported.status, 0, reported.stderr); + const repair = readFileSync(join(app, 'BUG_REPORT.md'), 'utf8'); + assert.match(repair, /account-menu/); + assert.doesNotMatch(repair, /catalog-search/); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('repair feedback states clean authority without exposing scoring history', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-repair-history-')); + try { + const app = join(root, 'app'); + writeGrade(app, 'failed', 'the owner check still failed'); + const history = [ + { round: 1, beforeScore: 4, beforeMax: 6, afterScore: 4, + afterMax: 6, result: 'kept with no score gain', remainingFailures: ['accounts/owner'] }, + { round: 2, beforeScore: 4, beforeMax: 6, afterScore: 4, + afterMax: 6, result: 'kept with no score gain', remainingFailures: ['accounts/owner'] }, + ]; + const archive = join(privateGradingDirectory(app), 'records', 'bug-report-round2.md'); + const reported = spawnSync(process.execPath, + [CLI, '--app', app, '--history-json', JSON.stringify(history), '--archive', archive], + { encoding: 'utf8' }); + assert.equal(reported.status, 0, reported.stderr); + const repair = readFileSync(join(app, 'BUG_REPORT.md'), 'utf8'); + assert.match(repair, /clean database reset and a fresh/); + assert.match(repair, /Earlier work/); + assert.match(repair, /Preserve earlier fixes/); + assert.doesNotMatch(repair, /Earlier changes did not fix/); + assert.doesNotMatch(repair, /Round|4\/6|accounts\/owner|score/i); + assert.match(repair, /existing local[\s\n]+state/); + assert.equal(readFileSync(archive, 'utf8'), repair); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('repair feedback uses behavioral expectations and findings without implementation advice', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-repair-sources-')); + try { + const app = join(root, 'app'); + const delivered = finding('message-delivered', { actor: 'other' }); + const evidence = createCheckEvidence({ + status: 'failed', code: 'test_result', phase: 'assertion', actor: 'other', + summary: '"support-secret-619" was delivered to other', finding: delivered, + observation: 'other received support-secret-619 in the ticket list', + expected: 'no message containing support-secret-619 reaches other', + startedAtMs: 3, completedAtMs: 4, + actions: [{ actor: 'other', evidence: { + schemaVersion: 2, action: { id: 'expectNotReceived', version: '1.0.0' }, + status: 'failed', type: 'transport-evidence', code: 'application_failure', phase: 'execute', + summary: '"support-secret-619" was delivered to other', finding: delivered, + observation: 'other received support-secret-619', expected: null, + retryable: false, timing: { startedAtMs: 3, completedAtMs: 4, durationMs: 1, + deadlineMs: 5_000 }, attachments: [], sensitivity: [], + } }], + }); + writeGrade(app, 'failed', 'unused', { feature: 'Customer support history', criterion: '612b', + statedBy: 'a support message is visible only to its customer and to staff', + consoleErrors: ['POST /api/support returned 500'], evidence }); + + const reported = spawnSync(process.execPath, [CLI, '--app', app], { encoding: 'utf8' }); + assert.equal(reported.status, 0, reported.stderr); + const repair = readFileSync(join(app, 'BUG_REPORT.md'), 'utf8'); + assert.doesNotMatch(repair, /support-secret-619|ticket list|was delivered to other"/); + assert.match(repair, /Expected:\*\* a support message is visible only to its customer and to staff/); + assert.match(repair, /Actual:\*\* private data was delivered to unauthorized actor other/); + assert.match(repair, /Console or network errors:[\s\S]*POST \/api\/support returned 500/); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/tools/stack-bench/tests/reset-backend.integration.ts b/tools/stack-bench/tests/reset-backend.integration.ts new file mode 100644 index 00000000000..3cf77593193 --- /dev/null +++ b/tools/stack-bench/tests/reset-backend.integration.ts @@ -0,0 +1,88 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import test from 'node:test'; +import { setTimeout as delay } from 'node:timers/promises'; +import { DATABASE_IMAGES } from '../src/stacks/database-containers.js'; + +import type { TextCommandOptions } from '../src/runtime/command-executor.js'; +import { proveMongoDbUse } from '../src/stacks/backends/mongodb-operations.js'; +import { provePostgresUse, resetPostgres } from '../src/stacks/backends/postgres-operations.js'; +import { POSTGRES_APPLICATION_IDENTITY } from '../src/stacks/hosted-database-identity.js'; + +const POSTGRES_USER = POSTGRES_APPLICATION_IDENTITY.user; +const docker = (args: readonly string[], options: Partial = {}): string => + execFileSync('docker', args, { encoding: 'utf8', stdio: 'pipe', timeout: 120_000, ...options }); + +test('runtime database provenance works against the Docker services', () => { + const suffix = `${process.pid}_${Date.now()}`; + const marker = `application-proof-${suffix}`; + const postgresDatabase = `application_proof_${suffix}`; + const mongoDatabase = `application_proof_${suffix}`; + const container = (name: string): { name: string; id: string } => ({ name, + id: docker(['inspect', '--format', '{{.Id}}', name]).trim() }); + try { + docker(['exec', 'stack-bench-postgres', 'psql', '-U', POSTGRES_USER, '-d', 'postgres', + '-v', 'ON_ERROR_STOP=1', '-c', `CREATE DATABASE ${postgresDatabase} OWNER ${POSTGRES_USER};`]); + docker(['exec', 'stack-bench-postgres', 'psql', '-U', POSTGRES_USER, '-d', postgresDatabase, + '-v', 'ON_ERROR_STOP=1', '-c', + `CREATE TABLE account (username text NOT NULL); INSERT INTO account VALUES ('${marker}');`]); + docker(['exec', 'stack-bench-mongodb', 'mongosh', mongoDatabase, '--quiet', '--eval', + `db.account.insertOne({ username: ${JSON.stringify(marker)} })`]); + + assert.equal(provePostgresUse({ lease: { resources: { database: postgresDatabase, + container: container('stack-bench-postgres') } }, marker }).ok, true); + assert.equal(proveMongoDbUse({ lease: { resources: { database: mongoDatabase, + container: container('stack-bench-mongodb') } }, marker }).ok, true); + } finally { + try { + docker(['exec', 'stack-bench-postgres', 'psql', '-U', POSTGRES_USER, '-d', 'postgres', + '-v', 'ON_ERROR_STOP=1', '-c', `DROP DATABASE IF EXISTS ${postgresDatabase} WITH (FORCE);`]); + } catch { /* preserve the test failure */ } + try { + docker(['exec', 'stack-bench-mongodb', 'mongosh', mongoDatabase, '--quiet', '--eval', + 'db.dropDatabase()']); + } catch { /* preserve the test failure */ } + } +}); + +test('PostgreSQL reset removes migrated structures and preserves neighboring databases', { timeout: 60_000 }, async () => { + const container = 'stack-bench-reset-regression-' + process.pid; + const id = docker(['run', '--pull=never', '--rm', '-d', '--name', container, + '--network', 'none', '--memory', '384m', '-e', 'POSTGRES_HOST_AUTH_METHOD=trust', DATABASE_IMAGES.postgres]).trim(); + const database = (name: string, sql: string, user = 'postgres'): string => + docker(['exec', '-i', id, 'psql', '-U', user, '-d', name, '-v', 'ON_ERROR_STOP=1', '-At'], { input: sql }).trim(); + try { + for (let attempt = 0; ; attempt++) { + try { database('postgres', 'SELECT 1'); break; } + catch (error) { if (attempt >= 60) throw error; await delay(250); } + } + database('postgres', 'CREATE USER appuser; CREATE DATABASE reset_target OWNER appuser; CREATE DATABASE neighbor;'); + database('neighbor', 'CREATE TABLE evidence(value integer); INSERT INTO evidence VALUES(42);'); + const lease = { resources: { database: 'reset_target', container: { name: container, id }, + network: { name: 'owned', id, namespaceContainerId: id, hostAddresses: [], services: [], + firewallSha256: null, firewallInstalledAt: null } } }; + const initialize = () => database('reset_target', ` +CREATE TABLE warehouses(name text PRIMARY KEY); +CREATE TABLE stock(warehouse text REFERENCES warehouses, quantity integer); +CREATE TABLE migrations(name text PRIMARY KEY); +INSERT INTO warehouses VALUES('East'); INSERT INTO stock VALUES('East', 7); +INSERT INTO migrations VALUES('initial'); +CREATE TABLE warehouse(id text PRIMARY KEY); +INSERT INTO warehouse SELECT name FROM warehouses; +ALTER TABLE stock ADD COLUMN warehouse_id text REFERENCES warehouse; +CREATE FUNCTION sync_stock() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN NEW.warehouse_id=NEW.warehouse; RETURN NEW; END $$; +CREATE TRIGGER sync_stock BEFORE INSERT ON stock FOR EACH ROW EXECUTE FUNCTION sync_stock(); +CREATE SCHEMA migration_meta; CREATE TABLE migration_meta.version(n integer); INSERT INTO migration_meta.version VALUES(2); +`, 'appuser'); + initialize(); + // The former reset erased the history but left the later trigger active. + database('reset_target', 'TRUNCATE migrations, stock, warehouse, warehouses CASCADE;'); + assert.throws(() => database('reset_target', "INSERT INTO warehouses VALUES('East'); INSERT INTO stock(warehouse,quantity) VALUES('East',7);"), /foreign key/); + resetPostgres({ lease }); + assert.equal(database('reset_target', "SELECT to_regclass('public.stock') IS NULL AND to_regnamespace('migration_meta') IS NULL;"), 't'); + initialize(); + assert.equal(database('reset_target', 'SELECT quantity FROM stock;'), '7'); + assert.equal(database('neighbor', 'SELECT value FROM evidence;'), '42'); + assert.equal(database('postgres', "SELECT rolcreatedb OR rolsuper FROM pg_roles WHERE rolname='appuser';"), 'f'); + } finally { docker(['rm', '-f', id]); } +}); diff --git a/tools/stack-bench/tests/reset-backend.test.ts b/tools/stack-bench/tests/reset-backend.test.ts new file mode 100644 index 00000000000..c3ba41e3b41 --- /dev/null +++ b/tools/stack-bench/tests/reset-backend.test.ts @@ -0,0 +1,422 @@ +import assert from 'node:assert/strict'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { runInNewContext } from 'node:vm'; +import { createBackendLease, writeBackendLease } from '../src/runtime/backend-lease.js'; +import { resetBackend, resetRepairBackend } from '../src/stacks/backend-reset.js'; +import { prepareMongoDbDatabase, proveMongoDbUse, resetMongoDb } from '../src/stacks/backends/mongodb-operations.js'; +import { attemptDatabaseIdentity } from '../src/stacks/hosted-database-identity.js'; +import { provePostgresUse, resetPostgres } from '../src/stacks/backends/postgres-operations.js'; +import { proveSpacetimeUse } from '../src/stacks/backends/spacetime-operations.js'; +import { containerReachableSpacetimeUri } from '../src/runtime/spacetime-target.js'; +import { GeneratedAppLayoutError, resolveSpacetimeModuleLayout } + from '../src/runtime/spacetime-layout.js'; +import type { TextCommandExecutor, TextCommandOptions } from '../src/runtime/command-executor.js'; +import type { BackendLeaseContainer } from '../src/runtime/backend-lease.js'; +import { compiledEntrypoint } from '../src/package-root.js'; + +interface CommandCall { + command: string; + args: readonly string[]; + options: TextCommandOptions; +} + +interface ArgvCall { + argv: string[]; + options: TextCommandOptions; +} + +function writeModule(directory: string): void { + mkdirSync(join(directory, 'src'), { recursive: true }); + writeFileSync(join(directory, 'package.json'), JSON.stringify({ + dependencies: { spacetimedb: 'file:/deps/bindings-typescript' }, + })); + writeFileSync(join(directory, 'src', 'index.ts'), + "import { schema } from 'spacetimedb/server';\nexport default schema({});\n"); +} + +test('Spacetime container targets follow their recorded network topology', () => { + const bridgeLease = { resources: { serverUri: 'http://127.0.0.1:3310', + buildContainer: null } }; + const hostLease: { resources: { serverUri: string; + buildContainer: Pick } } = { + resources: { serverUri: 'http://127.0.0.1:3310', + buildContainer: { networkMode: 'host' } }, + }; + assert.equal(containerReachableSpacetimeUri(bridgeLease), 'http://host.docker.internal:3310'); + assert.equal(containerReachableSpacetimeUri(hostLease), 'http://127.0.0.1:3310'); + assert.throws(() => containerReachableSpacetimeUri(bridgeLease, 'unknown'), + /unsupported build container network mode/); +}); + +test('the Node reset entrypoint refuses without an authenticated lease', () => { + const env = { ...process.env }; + delete env.STACK_BENCH_LEASE; + delete env.STACK_BENCH_LEASE_TOKEN; + assert.throws(() => execFileSync(process.execPath, + [compiledEntrypoint('commands', 'reset-backend.js'), 'mongodb', '.'], { env, stdio: 'pipe' }), + /STACK_BENCH_LEASE is required/); +}); + +test('the reset entrypoint reports a generated layout separately from a harness failure', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-reset-layout-exit-')); + const app = join(root, 'app'); + const leasePath = join(root, 'lease.json'); + mkdirSync(app); + const lease = createBackendLease({ runId: 'layout-exit', backend: 'spacetime', track: 'ecommerce', + runIndex: 0, serverUri: 'http://127.0.0.1:3310', module: 'app-ecom-run0', + dataDir: join(root, 'data') }); + lease.state = 'active'; + writeBackendLease(leasePath, lease); + try { + const result = spawnSync(process.execPath, + [compiledEntrypoint('commands', 'reset-backend.js'), 'spacetime', app], { + encoding: 'utf8', + env: { ...process.env, STACK_BENCH_LEASE: leasePath, + STACK_BENCH_LEASE_TOKEN: lease.ownershipToken }, + }); + assert.equal(result.status, 10); + assert.match(result.stderr, /^GENERATED_APP_LAYOUT:/); + assert.doesNotMatch(result.stderr, /at file:/); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('PostgreSQL reset replaces the leased database with controller authority', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-postgres-reset-')); + const path = join(root, 'lease.json'); + const prior = { path: process.env.STACK_BENCH_LEASE, token: process.env.STACK_BENCH_LEASE_TOKEN }; + const lease = { resources: { database: 'app_ecom_run0', + container: { name: 'postgres-service', id: 'a'.repeat(64) }, + network: { name: 'owned', id: 'b'.repeat(64), namespaceContainerId: 'c'.repeat(64), + hostAddresses: [], services: [], firewallSha256: null, firewallInstalledAt: null } } }; + const leased = createBackendLease({ runId: 'repair-reset', backend: 'postgres', track: 'ecommerce', + runIndex: 0, database: lease.resources.database, container: lease.resources.container }); + leased.state = 'active'; + writeBackendLease(path, leased); + process.env.STACK_BENCH_LEASE = path; + process.env.STACK_BENCH_LEASE_TOKEN = leased.ownershipToken; + const resets: Array<(exec: TextCommandExecutor) => unknown> = [ + exec => resetPostgres({ lease, exec }), + exec => resetRepairBackend({ backend: 'postgres', app: root, exec }), + ]; + try { + for (const reset of resets) { + const commands: string[][] = []; + reset((_command, args) => { + commands.push([...args]); + return args[0] === 'inspect' ? lease.resources.container.id : ''; + }); + assert(commands.some(args => args.includes('dropdb') && args.at(-1) === 'app_ecom_run0')); + assert(commands.some(args => args.includes('createdb') && args.at(-1) === 'app_ecom_run0')); + assert.throws(() => reset((_command, args) => { assert.equal(args[0], 'inspect'); return 'foreign'; }), + /changed after lease creation/); + } + for (const database of ['postgres', 'template0', 'template1']) { + assert.throws(() => resetPostgres({ lease: { resources: { ...lease.resources, database } }, + exec: () => { throw new Error('must not execute'); } }), /maintenance database/); + } + } finally { + if (prior.path === undefined) delete process.env.STACK_BENCH_LEASE; + else process.env.STACK_BENCH_LEASE = prior.path; + if (prior.token === undefined) delete process.env.STACK_BENCH_LEASE_TOKEN; + else process.env.STACK_BENCH_LEASE_TOKEN = prior.token; + rmSync(root, { recursive: true, force: true }); + } +}); + +test('PostgreSQL runtime provenance finds an application marker in the exact leased database', () => { + const lease = { resources: { database: 'app_ecom_run0', + container: { name: 'postgres-service', id: 'a'.repeat(64) } } }; + const calls: CommandCall[] = []; + const exec = (command: string, args: readonly string[], options: TextCommandOptions): string => { + calls.push({ command, args, options }); + if (args[0] === 'inspect') return `${lease.resources.container.id}\n`; + return 'public.account.username\n'; + }; + + assert.deepEqual(provePostgresUse({ lease, marker: "ann'proof", exec }), { + ok: true, + verified: true, + matches: 1, + reason: 'the application marker exists in the leased PostgreSQL database', + }); + const proof = calls.at(-1); + assert(proof); + assert.deepEqual(proof.args.slice(0, 3), ['exec', '-i', 'a'.repeat(64)]); + assert.equal(proof.args.includes('ON_ERROR_STOP=1'), true); + assert.match(proof.options.input ?? '', /FROM information_schema\.columns/); + assert.match(proof.options.input ?? '', /ann''proof/); + assert.match(proof.options.input ?? '', /ORDER BY table_name, ordinal_position\n\\gexec/); + + const emptyExec = (_command: string, args: readonly string[], _options: TextCommandOptions): string => + args[0] === 'inspect' + ? `${lease.resources.container.id}\n` : ''; + assert.equal(provePostgresUse({ lease, marker: 'missing', exec: emptyExec }).ok, false); + assert.throws(() => provePostgresUse({ lease, marker: '', exec }), + /requires a non-empty application marker/); +}); + +test('MongoDB destructive resets use trusted authority while provenance keeps application privileges', () => { + const lease = { ownershipToken: 'mongodb-reset-authority', resources: { + database: 'app_ecom_run0', container: { name: 'mongodb-service', id: 'b'.repeat(64) }, + network: { name: 'attempt', id: 'a'.repeat(64), namespaceContainerId: 'b'.repeat(64), + hostAddresses: [], services: [], firewallSha256: null, firewallInstalledAt: null }, + } }; + const calls: CommandCall[] = []; + const exec = (command: string, args: readonly string[], options: TextCommandOptions): string => { + calls.push({ command, args, options }); + return args[0] === 'inspect' ? lease.resources.container.id : '0'; + }; + resetMongoDb({ lease, exec }); + prepareMongoDbDatabase({ lease, exec, name: lease.resources.database, + expectedName: lease.resources.database, wipe: true }); + const resets = calls.filter(call => call.args.includes('db.dropDatabase()')); + assert.equal(resets.length, 2); + const identity = attemptDatabaseIdentity(lease.ownershipToken); + for (const { args } of resets) { + assert.deepEqual(args.slice(0, 4), ['exec', lease.resources.container.id, 'mongosh', lease.resources.database]); + assert.equal(args[args.indexOf('--username') + 1], 'admin'); + assert.equal(args[args.indexOf('--password') + 1], identity.adminPassword); + assert.equal(args[args.indexOf('--authenticationDatabase') + 1], 'admin'); + } + proveMongoDbUse({ lease, marker: 'proof', exec }); + const proof = calls.at(-1)!.args; + assert.equal(proof[proof.indexOf('--username') + 1], 'appuser'); + assert.equal(proof[proof.indexOf('--password') + 1], identity.password); + assert.equal(proof.includes(identity.adminPassword), false); + assert.throws(() => resetMongoDb({ lease, exec: (_command, args) => { + assert.equal(args[0], 'inspect'); return 'replacement'; + } }), /changed after lease creation/); + assert.throws(() => prepareMongoDbDatabase({ lease, exec: () => assert.fail('wrong target must not run'), + name: 'foreign', expectedName: 'foreign', wipe: true }), /does not match/); +}); + +test('MongoDB runtime provenance finds an application marker in the exact leased database', () => { + const lease = { resources: { database: 'app_ecom_run0', + container: { name: 'mongodb-service', id: 'b'.repeat(64) } } }; + const calls: CommandCall[] = []; + const execWith = (output: string) => + (command: string, args: readonly string[], options: TextCommandOptions): string => { + calls.push({ command, args, options }); + return args[0] === 'inspect' ? `${lease.resources.container.id}\n` : output; + }; + + assert.deepEqual(proveMongoDbUse({ lease, marker: 'ann-proof', exec: execWith('2\n') }), { + ok: true, + verified: true, + matches: 2, + reason: 'the application marker exists in the leased MongoDB database', + }); + const script = calls.at(-1)?.args.at(-1); + assert(script); + assert.match(script, /const marker = "ann-proof"/); + assert.match(script, /containsMarker/); + for (const found of [true, false]) { + let count = -1; + runInNewContext(script, { + db: { + getCollectionNames: () => ['system.views', 'accounts'], + getCollection: (name: string) => { + assert.equal(name, 'accounts'); + const rows = [{ username: found ? 'ann-proof' : 'someone-else' }]; + return { find: () => ({ hasNext: () => rows.length > 0, next: () => rows.shift() }) }; + }, + }, + print: (value: number) => { count = value; }, + }); + assert.equal(count, found ? 1 : 0); + } + assert.equal(proveMongoDbUse({ lease, marker: 'missing', exec: execWith('0\n') }).ok, false); + assert.throws(() => proveMongoDbUse({ lease, marker: 'proof', exec: execWith('not-a-count\n') }), + /invalid count/); + assert.throws(() => proveMongoDbUse({ lease, marker: '', exec: execWith('0\n') }), + /requires a non-empty application marker/); +}); + +test('SpacetimeDB runtime provenance finds an application marker in the exact leased module', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-spacetime-provenance-')); + const leasePath = join(root, 'lease.json'); + const lease = createBackendLease({ runId: 'spacetime-proof', backend: 'spacetime', + track: 'ecommerce', runIndex: 0, module: 'app_ecom_run0', + serverUri: 'http://127.0.0.1:3210', dataDir: root }); + lease.state = 'active'; + lease.resources.buildContainer = { name: 'build', id: 'c'.repeat(64), image: 'stack-bench:test', + owned: true, running: true, networkMode: 'host', + resourceLimits: { cpuCount: 1, memoryBytes: 1, memorySwapBytes: 1, pids: 1 } }; + writeBackendLease(leasePath, lease); + const previousPath = process.env.STACK_BENCH_LEASE; + const previousToken = process.env.STACK_BENCH_LEASE_TOKEN; + process.env.STACK_BENCH_LEASE = leasePath; + process.env.STACK_BENCH_LEASE_TOKEN = lease.ownershipToken; + const schema = { sections: [ + { Typespace: { types: [{ Product: { elements: [ + { name: { some: 'username' }, algebraic_type: { String: {} } }, + { name: { some: 'id' }, algebraic_type: { U64: {} } }, + ] } }] } }, + { Tables: [{ source_name: 'accountSource', product_type_ref: 0 }] }, + { ExplicitNames: { entries: [{ Table: { source_name: 'accountSource', canonical_name: 'account' } }] } }, + ] }; + const exec = (_command: string, args: readonly string[]): string => { + if (args[0] === 'inspect') return `${lease.resources.buildContainer?.id}\n`; + if (args.includes('describe')) return JSON.stringify(schema); + assert.match(args.at(-1)!, /from account where/); + return args.at(-1)?.includes("'proof'") ? ' username\n----------\n "proof"\n' : ''; + }; + const proofLease = { resources: { module: 'app_ecom_run0', serverUri: 'http://127.0.0.1:3210' } }; + try { + assert.deepEqual(proveSpacetimeUse({ lease: proofLease, marker: 'proof', exec }), { + ok: true, + verified: true, + matches: 1, + reason: 'the application marker exists in the leased SpacetimeDB module', + }); + assert.equal(proveSpacetimeUse({ lease: proofLease, marker: 'missing', exec }).ok, false); + assert.throws(() => proveSpacetimeUse({ lease: proofLease, marker: '', exec }), + /requires a non-empty application marker/); + schema.sections[2]!.ExplicitNames!.entries.push({ Table: { source_name: 'accountSource', canonical_name: 'duplicate' } }); + assert.throws(() => proveSpacetimeUse({ lease: proofLease, marker: 'proof', exec }), /invalid table name/); + } finally { + if (previousPath === undefined) delete process.env.STACK_BENCH_LEASE; + else process.env.STACK_BENCH_LEASE = previousPath; + if (previousToken === undefined) delete process.env.STACK_BENCH_LEASE_TOKEN; + else process.env.STACK_BENCH_LEASE_TOKEN = previousToken; + rmSync(root, { recursive: true, force: true }); + } +}); + +test('Spacetime reset publishes inside the exact leased build container', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-reset-node-')); + const app = join(root, 'app'); + const leasePath = join(root, 'lease.json'); + writeModule(join(app, 'backend', 'spacetimedb')); + const lease = createBackendLease({ runId: 'reset-test', backend: 'spacetime', track: 'ecommerce', + runIndex: 0, serverUri: 'http://127.0.0.1:3310', module: 'app-ecom-run0', + dataDir: join(root, 'data') }); + lease.state = 'active'; + lease.resources.buildContainer = { name: 'leased-build', id: 'a'.repeat(64), running: true, + owned: true, image: `sha256:${'b'.repeat(64)}`, + resourceLimits: { cpuCount: 2, memoryBytes: 1024, memorySwapBytes: 1024, pids: 32 } }; + writeBackendLease(leasePath, lease); + const previousLease = process.env.STACK_BENCH_LEASE; + const previousToken = process.env.STACK_BENCH_LEASE_TOKEN; + process.env.STACK_BENCH_LEASE = leasePath; + process.env.STACK_BENCH_LEASE_TOKEN = lease.ownershipToken; + const buildContainer = lease.resources.buildContainer; + assert(buildContainer); + const calls: ArgvCall[] = []; + const exec = (command: string, args: readonly string[], options: TextCommandOptions): string => { + calls.push({ argv: [command, ...args], options }); + if (args[0] === 'inspect') return `${buildContainer.id}\n`; + return ''; + }; + try { + resetBackend({ backend: 'spacetime', app, exec }); + const publish = calls.find(call => call.argv.includes('publish')); + assert(publish); + assert.deepEqual(publish.argv.slice(0, 8), ['docker', 'exec', '--user', '10001:10001', + '-e', 'HOME=/home/developer', '-e', 'USER=developer']); + assert.equal(publish.argv[publish.argv.indexOf('-w') + 1], '/app/backend/spacetimedb'); + assert.ok(publish.argv.includes(buildContainer.id)); + assert.ok(publish.argv.includes('/deps/spacetimedb-cli')); + assert.ok(publish.argv.includes('app-ecom-run0')); + assert.ok(publish.argv.includes('http://host.docker.internal:3310')); + assert.equal(publish.argv.includes(join(app, 'backend', 'spacetimedb')), false); + const declared = join(root, 'declared'); + writeModule(join(declared, 'server', 'spacetimedb')); + writeFileSync(join(declared, 'server', 'spacetime.json'), JSON.stringify({ + server: 'bench', 'module-path': './spacetimedb', + })); + calls.length = 0; + resetBackend({ backend: 'spacetime', app: declared, exec }); + const declaredPublish = calls.find(call => call.argv.includes('publish')); + assert(declaredPublish); + assert.equal(declaredPublish.argv[declaredPublish.argv.indexOf('-w') + 1], '/app/server/spacetimedb'); + assert.equal(declaredPublish.argv.filter(value => value === '/app/server/spacetimedb').length, 2); + calls.length = 0; + resetRepairBackend({ backend: 'spacetime', app, exec }); + const removal = calls.find(call => call.argv.includes('delete')); + assert(removal, 'repair reset must remove the rejected schema before clean startup'); + assert.ok(removal.argv.includes(buildContainer.id)); + assert.ok(removal.argv.includes('10001:10001')); + assert.ok(removal.argv.includes('app-ecom-run0')); + assert.ok(removal.argv.includes('http://host.docker.internal:3310')); + assert.equal(calls.some(call => call.argv.includes('publish')), false, + 'repair reset cannot compile before accepted startup installs dependencies'); + assert.throws(() => resetRepairBackend({ backend: 'spacetime', app, + exec: (_command, args) => args[0] === 'inspect' ? 'wrong-container' : '' }), /changed after lease creation/); + assert.throws(() => resetRepairBackend({ backend: 'spacetime', app, + exec: (_command, args) => { + if (args[0] === 'inspect') return buildContainer.id; + throw new Error('permission denied'); + } }), /could not delete prior module/); + assert.doesNotThrow(() => resetRepairBackend({ backend: 'spacetime', app, + exec: (_command, args) => { + if (args[0] === 'inspect') return buildContainer.id; + throw new Error('404 Not Found'); + } }), 'an already absent rejected module is ready for accepted startup'); + + } finally { + if (previousLease === undefined) delete process.env.STACK_BENCH_LEASE; + else process.env.STACK_BENCH_LEASE = previousLease; + if (previousToken === undefined) delete process.env.STACK_BENCH_LEASE_TOKEN; + else process.env.STACK_BENCH_LEASE_TOKEN = previousToken; + rmSync(root, { recursive: true, force: true }); + } +}); + +test('Spacetime root target takes precedence over nested scaffolds and stays inside the mounted app', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-layout-container-path-')); + try { + writeModule(join(root, 'backend', 'spacetimedb')); + writeModule(join(root, 'backend', 'spacetimedb', 'spacetimedb')); + writeFileSync(join(root, 'backend', 'spacetimedb', 'spacetime.json'), JSON.stringify({ + 'module-path': './spacetimedb', + })); + writeFileSync(join(root, 'spacetime.json'), JSON.stringify({ + 'module-path': '/app/backend/spacetimedb', + })); + const layout = resolveSpacetimeModuleLayout(root); + assert.equal(layout.moduleDirectory, 'backend/spacetimedb'); + assert.equal(layout.containerPath, '/app/backend/spacetimedb'); + assert.equal(layout.source, 'spacetime.json'); + // A valid native entry can re-export code outside src. Discovery syntax + // must not reject it. Missing native files and escaping targets still fail. + const module = join(root, 'backend', 'spacetimedb'); + writeFileSync(join(module, 'index.ts'), "import { schema } from 'spacetimedb/server';\nexport default schema({});\n"); + writeFileSync(join(module, 'src', 'index.ts'), "export { default } from '../index';\n"); + assert.equal(resolveSpacetimeModuleLayout(root).moduleDirectory, 'backend/spacetimedb'); + rmSync(join(module, 'src', 'index.ts')); + assert.throws(() => resolveSpacetimeModuleLayout(root), GeneratedAppLayoutError); + writeModule(module); + rmSync(join(module, 'package.json')); + assert.throws(() => resolveSpacetimeModuleLayout(root), GeneratedAppLayoutError); + writeModule(module); + writeFileSync(join(root, 'spacetime.json'), JSON.stringify({ 'module-path': '../outside' })); + assert.throws(() => resolveSpacetimeModuleLayout(root), /escapes the application/); + writeFileSync(join(root, 'spacetime.json'), JSON.stringify({ 'module-path': 'missing' })); + assert.throws(() => resolveSpacetimeModuleLayout(root), /module directory is missing/); + const empty = join(root, 'empty-app'); + mkdirSync(empty); + assert.throws(() => resolveSpacetimeModuleLayout(empty), GeneratedAppLayoutError); + symlinkSync(module, join(empty, 'outside'), 'junction'); + writeFileSync(join(empty, 'spacetime.json'), JSON.stringify({ 'module-path': 'outside' })); + assert.throws(() => resolveSpacetimeModuleLayout(empty), GeneratedAppLayoutError); + const linkedEntryModule = join(empty, 'module'); + mkdirSync(linkedEntryModule); + writeFileSync(join(linkedEntryModule, 'package.json'), '{}'); + symlinkSync(join(module, 'src'), join(linkedEntryModule, 'src'), 'junction'); + writeFileSync(join(empty, 'spacetime.json'), JSON.stringify({ 'module-path': 'module' })); + assert.throws(() => resolveSpacetimeModuleLayout(empty), /resolves outside the application/); + const ambiguous = join(root, 'ambiguous-app'); + writeModule(join(ambiguous, 'one')); + writeModule(join(ambiguous, 'two')); + assert.throws(() => resolveSpacetimeModuleLayout(ambiguous), /multiple SpacetimeDB module directories/); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/tools/stack-bench/tests/resource-lock-transactions.test.ts b/tools/stack-bench/tests/resource-lock-transactions.test.ts new file mode 100644 index 00000000000..c8836204289 --- /dev/null +++ b/tools/stack-bench/tests/resource-lock-transactions.test.ts @@ -0,0 +1,186 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, readdirSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { createBackendLease, runnerCapacity, claimBackendResources, + claimBackendResourcesWhenAvailable, releaseResourceLocks } from '../src/runtime/backend-lease.js'; +import { hostResourceWaitReason, resourceLockDescriptors, resourceLockTransaction } from '../src/runtime/resource-lock-worker.js'; +import { ATTEMPT_CONTAINER_LIMIT_TOTALS, ATTEMPT_STARTUP_MEMORY_BYTES } from '../src/composition/product-config.js'; +import { SUPABASE_IDENTITY } from '../src/stacks/backends/supabase-identity.js'; + +const SUPABASE_STARTUP_MEMORY_BYTES = SUPABASE_IDENTITY.startupMemoryBytes!; + +test('dynamic startup reservation counts the memory envelope once per worker across backends', () => { + const root = mkdtempSync(join(tmpdir(), 'host-capacity-')); + const first = createBackendLease({ runId: 'first', backend: 'stub', track: 'loop', runIndex: 0 }); + const next = createBackendLease({ runId: 'next', backend: 'stub', track: 'loop', runIndex: 1 }); + const start = Date.now(); + const keys = ['slot:loop:postgres:run0', 'slot:loop:mongodb:run0', 'slot:loop:spacetime:run0']; + const memoryBytes = ATTEMPT_STARTUP_MEMORY_BYTES; + try { + const request = { root, lease: first, keys, operation: 'acquire' as const, capacity: null }; + const locks = resourceLockTransaction(request, (count, memory) => { + assert.equal(count, 1); assert.equal(memory, memoryBytes); return null; + }, start); + assert.equal(JSON.parse(readFileSync(locks[0]!.path, 'utf8')).startupMemoryBytes, memoryBytes); + resourceLockTransaction({ root, lease: next, keys: ['slot:loop:postgres:run1', 'port:5001'], + operation: 'acquire', capacity: null }, (count, memory) => { + assert.equal(count, 2); assert.equal(memory, 2 * memoryBytes); return null; + }, start + 1); + const later = createBackendLease({ runId: 'later', backend: 'stub', track: 'loop', runIndex: 2 }); + resourceLockTransaction({ root, lease: later, keys: ['slot:loop:postgres:run2'], + operation: 'acquire', capacity: null }, (count, memory) => { + assert.equal(count, 1); assert.equal(memory, memoryBytes); return null; + }, start + 60_002); + resourceLockTransaction({ ...request, operation: 'release' }); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('a stack that declares a larger startup reservation holds it for its slot', () => { + const root = mkdtempSync(join(tmpdir(), 'host-capacity-')); + const first = createBackendLease({ runId: 'first', backend: 'stub', track: 'loop', runIndex: 0 }); + const next = createBackendLease({ runId: 'next', backend: 'stub', track: 'loop', runIndex: 1 }); + const start = Date.now(); + try { + assert(SUPABASE_STARTUP_MEMORY_BYTES > ATTEMPT_STARTUP_MEMORY_BYTES); + const locks = resourceLockTransaction({ root, lease: first, keys: ['slot:loop:postgres:run0', 'slot:loop:supabase:run0'], + operation: 'acquire', capacity: null }, (count, memory) => { + assert.equal(count, 1); assert.equal(memory, SUPABASE_STARTUP_MEMORY_BYTES); return null; + }, start); + const recorded = Object.fromEntries(locks.map(lock => [lock.key, + JSON.parse(readFileSync(lock.path, 'utf8')).startupMemoryBytes])); + assert.deepEqual(recorded, { 'slot:loop:postgres:run0': ATTEMPT_STARTUP_MEMORY_BYTES, + 'slot:loop:supabase:run0': SUPABASE_STARTUP_MEMORY_BYTES }); + resourceLockTransaction({ root, lease: next, keys: ['slot:loop:convex:run1', 'port:5001'], + operation: 'acquire', capacity: null }, (count, memory) => { + assert.equal(count, 2); assert.equal(memory, SUPABASE_STARTUP_MEMORY_BYTES + ATTEMPT_STARTUP_MEMORY_BYTES); + return null; + }, start + 1); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('host admission counts a campaign reservation once per index across backends', () => { + const root = mkdtempSync(join(tmpdir(), 'host-capacity-')); + const first = createBackendLease({ runId: 'first', backend: 'stub', track: 'loop', runIndex: 0 }); + const next = createBackendLease({ runId: 'next', backend: 'stub', track: 'loop', runIndex: 1 }); + const keys = ['slot:loop:postgres:run0', 'slot:loop:mongodb:run0', 'slot:loop:spacetime:run0']; + try { + resourceLockTransaction({ root, lease: first, keys, operation: 'acquire', capacity: 1 }); + resourceLockTransaction({ root, lease: first, keys, operation: 'acquire', capacity: 1 }); + assert.throws(() => resourceLockTransaction({ root, lease: next, + keys: ['slot:loop:postgres:run1', 'port:5999'], operation: 'acquire', capacity: 1 }), /host capacity unavailable/); + assert.equal(readdirSync(root).length, 3, 'no partial claim when host capacity is exhausted'); + resourceLockTransaction({ root, lease: first, keys, operation: 'release' }); + resourceLockTransaction({ root, lease: next, keys: ['slot:loop:postgres:run1'], operation: 'acquire', capacity: 1 }); + assert.equal(runnerCapacity({}), null); + assert.equal(runnerCapacity({ STACK_BENCH_RUNNER_CAPACITY: '9' }), 9); + assert.equal(runnerCapacity({ STACK_BENCH_RUNNER_CAPACITY: 'dynamic' }), null); + assert.throws(() => runnerCapacity({ STACK_BENCH_RUNNER_CAPACITY: '0' }), /positive safe integer/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('dynamic admission uses host pressure and retains exclusive resource ownership without a count quota', () => { + const root = mkdtempSync(join(tmpdir(), 'host-unlimited-')); + try { + const capacity = runnerCapacity({ STACK_BENCH_RUNNER_CAPACITY: 'dynamic' }); + assert.equal(hostResourceWaitReason(46 * 1024 ** 3, 29 * 1024 ** 3, 32, 7), null); + assert.match(hostResourceWaitReason(46 * 1024 ** 3, 3 * 1024 ** 3, 32, 7)!, /GiB available/); + assert.match(hostResourceWaitReason(46 * 1024 ** 3, 29 * 1024 ** 3, 32, 33)!, /CPU load/); + assert.match(hostResourceWaitReason(46 * 1024 ** 3, 29 * 1024 ** 3, 32, 7, 10)!, /GiB required/); + // Twelve launches fit a 46 GiB host with 40 GiB free; the summed container caps allowed three. + assert.equal(hostResourceWaitReason(46 * 1024 ** 3, 40 * 1024 ** 3, 32, 7, 12), null); + assert(ATTEMPT_STARTUP_MEMORY_BYTES < ATTEMPT_CONTAINER_LIMIT_TOTALS.memoryBytes); + assert.throws(() => hostResourceWaitReason(NaN, 0, 32, 0), /valid host resource/); + const start = Date.now(); + for (let index = 0; index < 12; index += 1) { + const lease = createBackendLease({ runId: `run-${index}`, backend: 'stub', track: 'loop', runIndex: index }); + resourceLockTransaction({ root, lease, capacity, operation: 'acquire', + keys: [`slot:loop:stub:run${index}`, `port:${5000 + index}`] }, count => { + assert.equal(count, 1, 'old live claims do not impose a fixed count limit'); + return null; + }, start + index * 60_001); + } + const extra = createBackendLease({ runId: 'extra', backend: 'stub', track: 'loop', runIndex: 12 }); + assert.throws(() => resourceLockTransaction({ root, lease: extra, capacity, + operation: 'acquire', keys: ['port:5000'] }), /already leased/); + assert.equal(readdirSync(root).length, 24); + const request = { root, lease: extra, capacity, operation: 'acquire' as const, + keys: ['slot:loop:stub:run12', 'port:5012'] }; + assert.throws(() => resourceLockTransaction(request, () => 'host capacity unavailable: memory pressure'), + /memory pressure/); + assert.equal(readdirSync(root).length, 24, 'pressure cannot leave partial claims'); + resourceLockTransaction(request, count => { assert.equal(count, 2); return null; }, start + 11 * 60_001); + resourceLockTransaction(request, () => { throw new Error('existing claim must not reacquire capacity'); }); + resourceLockTransaction({ ...request, operation: 'release' }, () => { throw new Error('cleanup must stay available'); }); + assert.equal(readdirSync(root).length, 24); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('port and workspace claims remain exclusive across acquisition and intent recovery', () => { + const root = mkdtempSync(join(tmpdir(), 'lock-resources-')); + const campaign = createBackendLease({ runId: 'campaign', backend: 'stub', track: 'loop', runIndex: 0 }); + const standalone = createBackendLease({ runId: 'standalone', backend: 'stub', track: 'loop', runIndex: 1 }); + const workspaceKey = `workspace:${join(root, 'app')}`; + const campaignKeys = ['port:4000', workspaceKey]; + const keys = ['port:4001']; + try { + const reserved = resourceLockTransaction({ root, lease: campaign, keys: campaignKeys, operation: 'acquire' }); + const before = reserved.map(lock => readFileSync(lock.path, 'utf8')); + for (const conflict of ['port:4000', workspaceKey]) { + assert.throws(() => resourceLockTransaction({ root, lease: standalone, + keys: [...keys, conflict], operation: 'acquire' }), /already leased/); + assert.equal(readdirSync(root).length, 2, 'conflict must not acquire the free port'); + } + const acquired = resourceLockTransaction({ root, lease: standalone, keys, operation: 'acquire' }); + assert.deepEqual(acquired.map(lock => lock.key), keys); + assert.deepEqual(resourceLockTransaction({ root, lease: standalone, keys, operation: 'acquire' }), acquired); + const intent = resourceLockDescriptors(root, [...keys, ...campaignKeys, 'port:4002']); + resourceLockTransaction({ root, lease: standalone, keys: intent.map(lock => lock.key), operation: 'release-intent' }); + assert.deepEqual(reserved.map(lock => readFileSync(lock.path, 'utf8')), before); + resourceLockTransaction({ root, lease: campaign, keys, operation: 'acquire' }); + resourceLockTransaction({ root, lease: campaign, keys: [...keys, ...campaignKeys], operation: 'release' }); + assert.equal(readdirSync(root).length, 0); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('a stale release cannot remove a replacement claim, even temporarily', () => { + const root = mkdtempSync(join(tmpdir(), 'lock-transaction-')); + const first = createBackendLease({ runId: 'first', backend: 'stub', track: 'loop', runIndex: 0 }); + const next = createBackendLease({ runId: 'next', backend: 'stub', track: 'loop', runIndex: 0 }); + const transaction = { root, keys: ['slot'], lease: first }; + try { + resourceLockTransaction({ ...transaction, operation: 'acquire' }); + resourceLockTransaction({ ...transaction, operation: 'release' }); + const [lock] = resourceLockTransaction({ ...transaction, lease: next, operation: 'acquire' }); + assert(lock); + const before = readFileSync(lock.path, 'utf8'); + assert.throws(() => resourceLockTransaction({ ...transaction, operation: 'release' }), + /no longer belongs/); + assert.equal(readFileSync(lock.path, 'utf8'), before); + assert.throws(() => resourceLockTransaction({ ...transaction, operation: 'acquire' }), + /already leased by next/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('standalone admission waits for capacity but rejects ownership conflicts', { skip: process.platform !== 'linux' }, async () => { + const root = mkdtempSync(join(tmpdir(), 'standalone-capacity-')); + const first = createBackendLease({ runId: 'first', backend: 'stub', track: 'loop', runIndex: 0 }); + const next = createBackendLease({ runId: 'next', backend: 'stub', track: 'loop', runIndex: 1 }); + const firstPath = join(root, 'first.json'), nextPath = join(root, 'next.json'); + let timer: ReturnType | undefined; + try { + claimBackendResources(firstPath, first, { root, keys: ['slot:loop:postgres:run0', 'port:5999'], capacity: 1 }); + await assert.rejects(claimBackendResourcesWhenAvailable(nextPath, next, + { root, keys: ['port:5999'], capacity: 2 }), /already leased/); + timer = setTimeout(() => releaseResourceLocks(first), 25); + await claimBackendResourcesWhenAvailable(nextPath, next, + { root, keys: ['slot:loop:postgres:run1'], capacity: 1 }); + assert.deepEqual(next.resources.locks.map(lock => lock.key), ['slot:loop:postgres:run1']); + } finally { + clearTimeout(timer); + releaseResourceLocks(first); + releaseResourceLocks(next); + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/tools/stack-bench/tests/response-loss-action.test.ts b/tools/stack-bench/tests/response-loss-action.test.ts new file mode 100644 index 00000000000..08ecfdec93b --- /dev/null +++ b/tools/stack-bench/tests/response-loss-action.test.ts @@ -0,0 +1,47 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { performance } from 'node:perf_hooks'; +import { executeAction } from '../src/actions/action-contract.js'; +import { ACTION_REGISTRY } from '../src/actions/action-catalog.js'; +import type { CheckoutState } from '../src/stacks/checkout-state.js'; + +test('lost-reply action requires observed commit and a clean gate, and always restores delivery', async t => { + let now = 0; + t.mock.method(performance, 'now', () => now); + const before: CheckoutState = { accountId: 'a', itemId: 'i', priceMinor: 100, cart: [], reservations: [], + orders: [], payments: [], refunds: [], stock: [{ itemId: 'i', warehouseId: 'w', quantity: 10 }], + orphanOrderLines: 0, orphanAllocations: 0, orphanRefunds: 0 }; + const prepared = structuredClone(before); prepared.cart = [{ itemId: 'i', quantity: 1 }]; + const after = structuredClone(before); after.stock[0]!.quantity--; + after.orders.push({ id: 'o', accountId: 'a', totalMinor: 100, refundedMinor: 0, status: 'pending', + lines: [{ itemId: 'i', quantity: 1, priceMinor: 100, allocations: [{ warehouseId: 'w', quantity: 1 }] }] }); + const snapshot = (state: CheckoutState) => ({ account: 'a', item: 'i', state, scope: 'orders', + storage: { kind: 'order-data', cart: true, warehouses: true }, schemaSha256: { schema: 'fixed' }, + catalog: [{ itemId: 'i', name: 'i', priceMinor: 100 }] }); + for (const mode of ['valid', 'reader-error', 'gate-error', 'overflow', 'schema-change', 'no-drop', 'no-commit']) { + let armed = false, finished = false, clicked = false; + const gate = { + arm() { armed = true; }, async finish() { finished = true; }, + evidence: () => ({ state: finished ? 'finished' : 'armed', truncated: mode === 'overflow', + errors: mode === 'gate-error' ? ['interception failed'] : [], + events: mode === 'no-drop' ? [] : [{ kind: 'http-request' }, { kind: 'http-response' }] }), + }; + const result = await executeAction(ACTION_REGISTRY, 'loseCheckoutResponse', { + do: 'loseCheckoutResponse', actor: 'a', before: 'before', prepared: 'prepared', quantity: 1, + }, { capabilities: { + actors: { get: () => ({ loc: () => ({ click: async () => { assert(armed); clicked = true; } }) }) }, + 'response-loss': { get: () => gate }, clock: { sleep: async () => { now += 10_000; } }, + 'database-read': { checkoutSnapshots: new Map([['before', snapshot(before)], ['prepared', snapshot(prepared)]]), + getCheckoutState: () => { + if (mode === 'reader-error') throw new Error('reader unavailable'); + const value = snapshot(mode === 'no-commit' ? prepared : after); + if (mode === 'schema-change') value.schemaSha256.schema = 'changed'; + return value; + } }, + } }); + assert(clicked && finished, mode); + assert.equal(result.status, mode === 'valid' ? 'passed' + : ['no-drop', 'no-commit'].includes(mode) ? 'inconclusive' : 'harness_failure', mode); + assert.equal((result.observation as { fault: { state: string } }).fault.state, 'finished'); + } +}); diff --git a/tools/stack-bench/tests/response-loss.integration.ts b/tools/stack-bench/tests/response-loss.integration.ts new file mode 100644 index 00000000000..d7a89e24342 --- /dev/null +++ b/tools/stack-bench/tests/response-loss.integration.ts @@ -0,0 +1,330 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { createServer } from 'node:http'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { once } from 'node:events'; +import { setTimeout as delay } from 'node:timers/promises'; +import test from 'node:test'; +import { chromium } from 'playwright'; +import { installResponseLoss } from '../grader/response-loss.js'; +import { installAuthWebSocketCapture, withWriteCompletion } from '../src/actions/auth-request-patch.js'; +import { installSpacetimeWriteCapture } from '../src/stacks/backends/spacetime-browser-session.js'; +import { createBackendLease, writeBackendLease } from '../src/runtime/backend-lease.js'; +import { ActionInconclusive } from '../src/actions/action-contract.js'; + +// Both observers must operate on one connection, including after reload. A +// committed write with a suppressed reply must not satisfy browser completion. +test('write capture and response loss compose on every native transport', { timeout: 120_000 }, async t => { + const codec = await import(new URL('../src/stacks/spacetime-wire-codec.js', import.meta.url).href); + const { wsServer } = createRequire(import.meta.url)('playwright-core/lib/utilsBundle'); + const encode = (type: { serialize(writer: unknown, value: unknown): void }, value: unknown) => { + const writer = new codec.BinaryWriter(128); type.serialize(writer, value); return Buffer.from(writer.getBuffer()); + }; + const server = createServer((req, res) => { + if (req.url?.includes('/schema')) res.end(JSON.stringify({ reducers: [] })); + else res.end('

checkout fixture

'); + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const sockets = new wsServer({ server }); + const commits = new Set(), forwarded: string[] = []; + sockets.on('connection', (socket: { on(event: string, fn: (raw: Buffer) => void): void; + send(data: string | Buffer): void }, req: { url: string; headers: Record }) => { + const address = new URL(req.url, url), kind = address.searchParams.get('kind')!, trial = address.searchParams.get('trial')!; + if (kind === 'spacetime') socket.send(Buffer.concat([Buffer.from([0]), encode(codec.ServerMessage, { + tag: 'InitialConnection', value: { identity: { __identity__: 1n }, connectionId: { __connection_id__: 1n }, token: 'fixture' }, + })])); + else if (kind === 'socketio') socket.send(`0${JSON.stringify({ sid: trial, upgrades: [], pingInterval: 25000, pingTimeout: 20000 })}`); + else socket.send('{"type":"ready"}'); + socket.on('message', raw => { + if (kind === 'socketio' && String(raw) === '41') return; + const message = kind === 'spacetime' ? codec.ClientMessage.deserialize(new codec.BinaryReader(raw)) + : kind === 'convex' ? JSON.parse(String(raw)) : JSON.parse(String(raw).slice(2)); + const id = kind === 'spacetime' ? message.value.requestId : message.requestId; + const cart = kind === 'spacetime' ? message.value.args[0] : kind === 'convex' ? message.args[0].cart : message.cart; + const key = `${trial}:${cart}`, accepted = !commits.has(key); + commits.add(key); forwarded.push(createHash('sha256').update(raw).digest('hex')); + const reply = kind === 'spacetime' ? Buffer.concat([Buffer.from([0]), encode(codec.ServerMessage, { + tag: 'ReducerResult', value: { requestId: id, timestamp: { __timestamp_micros_since_unix_epoch__: 1n }, + result: { tag: accepted ? 'OkEmpty' : 'Err', ...(accepted ? {} : { value: 'consumed' }) } }, + })]) : kind === 'convex' ? JSON.stringify({ type: 'MutationResponse', requestId: id, success: accepted, result: null }) + : accepted ? `40${JSON.stringify({ sid: trial })}` : '44{"message":"consumed"}'; + setTimeout(() => socket.send(reply), 100); + }); + }); + const dir = mkdtempSync(join(tmpdir(), 'socket-loss-composition-')), leasePath = join(dir, 'lease.json'); + const prior = { lease: process.env.STACK_BENCH_LEASE, token: process.env.STACK_BENCH_LEASE_TOKEN }; + const lease = createBackendLease({ runId: 'socket-loss-composition', backend: 'spacetime', track: 'ecommerce', + runIndex: 0, serverUri: url, module: 'shop', dataDir: join(dir, 'data') }); + lease.state = 'active'; writeBackendLease(leasePath, lease); + process.env.STACK_BENCH_LEASE = leasePath; process.env.STACK_BENCH_LEASE_TOKEN = lease.ownershipToken; + const browser = await chromium.launch({ headless: true }); + const evidence: object[] = []; + try { + for (const kind of ['spacetime', 'convex', 'socketio']) for (const gateFirst of [false, true]) + for (const reload of [false, true]) await t.test(`${kind}, gate first=${gateFirst}, reload=${reload}`, async () => { + const trial = `${kind}-${gateFirst}-${reload}`, context = await browser.newContext(), page = await context.newPage(); + let gate: Awaited> | undefined; + try { + if (gateFirst) gate = await installResponseLoss(context); + await installAuthWebSocketCapture(page); + if (kind === 'spacetime') await installSpacetimeWriteCapture(page); + if (!gateFirst) gate = await installResponseLoss(context); + await page.goto(url); + const connect = () => page.evaluate(async ({ url, kind, trial }) => { + const path = kind === 'spacetime' ? '/proxy/v1/database/shop/subscribe' + : kind === 'convex' ? '/proxy/api/1.0.0/sync' : '/socket.io/'; + const socket = new WebSocket(`${url.replace('http:', 'ws:')}${path}?kind=${kind}&trial=${trial}` + + (kind === 'socketio' ? '&EIO=4&transport=websocket' : ''), kind === 'spacetime' ? 'v3.bsatn.spacetimedb' : []); + socket.binaryType = 'arraybuffer'; Object.assign(window, { socket, replies: 0 }); + await new Promise(resolve => socket.addEventListener('message', () => resolve(), { once: true })); + socket.addEventListener('message', () => { (window as unknown as { replies: number }).replies++; }); + }, { url, kind, trial }); + await connect(); + if (reload) { await page.reload(); await connect(); } + const send = async (id: number, cart: number) => { + const packet = kind === 'spacetime' ? encode(codec.ClientMessage, { tag: 'CallReducer', + value: { reducer: 'checkout', requestId: id, flags: 0, args: new Uint8Array([cart]) } }) + : kind === 'convex' ? JSON.stringify({ type: 'Mutation', requestId: id, udfPath: 'shop:checkout', args: [{ cart }] }) + : `40${JSON.stringify({ requestId: id, cart })}`; + await page.evaluate(({ packet, kind, id }) => { + const socket = (window as unknown as { socket: WebSocket }).socket; + if (kind === 'socketio' && id === 2) socket.send('41'); + socket.send(typeof packet === 'string' ? packet : new Uint8Array(packet)); + }, { packet: typeof packet === 'string' ? packet : [...packet], kind, id }); + return createHash('sha256').update(packet).digest('hex'); + }; + const setup = await withWriteCompletion(page, () => send(1, 1)); + assert.equal(setup.writes.length, 1, 'native capture must still observe setup with the fault installed'); + assert.equal(await page.evaluate(() => (window as unknown as { replies: number }).replies), 1); + gate!.arm(); + const loss = withWriteCompletion(page, () => send(2, 2)).then(() => 'complete', error => { + assert(error instanceof ActionInconclusive); return 'inconclusive'; + }); + for (let n = 0; n < 100 && !gate!.evidence().events.some(e => e.kind === 'ws-drop'); n++) await delay(10); + assert(commits.has(`${trial}:2`), 'independent server state must show the committed operation'); + assert(gate!.evidence().events.some(e => e.kind === 'ws-drop'), 'fault must observe and suppress the native reply'); + const sent = gate!.evidence().events.find(e => e.kind === 'ws-send' && e.sha256 === forwarded.at(-1)); + assert(sent, 'the fault must record the exact bytes forwarded to the server'); + assert.equal(await page.evaluate(() => (window as unknown as { replies: number }).replies), 1); + assert.equal(await Promise.race([loss, delay(50).then(() => 'waiting')]), 'waiting', + 'a suppressed reply must not resolve capture as complete'); + await gate!.finish(); assert.equal(await loss, 'inconclusive'); + await connect(); + const before = commits.size; + const retry = await withWriteCompletion(page, () => send(3, 2)); + assert.equal(retry.writes.length, 1); assert.equal(commits.size, before, 'retry cannot duplicate the effect'); + assert.deepEqual(gate!.evidence().errors, []); assert.equal(gate!.evidence().truncated, false); + evidence.push({ kind, gateFirst, reload, setupWrites: setup.writes.length, + lostReply: 'inconclusive', fault: gate!.evidence(), duplicateEffect: false }); + } finally { await gate?.finish(); await context.close(); } + }); + } finally { + await browser.close(); for (const socket of sockets.clients) socket.terminate(); sockets.close(); + server.closeAllConnections(); await new Promise(resolve => server.close(() => resolve())); + if (prior.lease === undefined) delete process.env.STACK_BENCH_LEASE; else process.env.STACK_BENCH_LEASE = prior.lease; + if (prior.token === undefined) delete process.env.STACK_BENCH_LEASE_TOKEN; else process.env.STACK_BENCH_LEASE_TOKEN = prior.token; + rmSync(dir, { recursive: true, force: true }); + if (process.env.STACK_BENCH_SOCKET_COMPOSITION_EVIDENCE) writeFileSync(process.env.STACK_BENCH_SOCKET_COMPOSITION_EVIDENCE, + JSON.stringify({ command: 'node --test --test-name-pattern="write capture and response loss compose" dist/tests/response-loss.integration.js', + cases: evidence, cleanup: 'browser, server and temporary lease closed' }, null, 2) + '\n'); + } +}); + +test('a lost browser HTTP reply keeps its committed effect and does not change the write', async t => { + // A lost reply must not apply its Set-Cookie header, even after cleanup or retry. + // A normally delivered reply must apply it; the duplicate-write refusal sets no cookie. + const evidence: { loss: boolean; reload: boolean; cookieBeforeFinish: string | null; cookieAfterFinish: string | null; + cookieAfterRetry: string | null; retryCookie: string | null; retryStatus: number }[] = []; + t.after(() => { + if (process.env.STACK_BENCH_RESPONSE_LOSS_EVIDENCE) + writeFileSync(process.env.STACK_BENCH_RESPONSE_LOSS_EVIDENCE, JSON.stringify(evidence, null, 2) + '\n'); + }); + const carts = new Set(); + const retryCookies = new Map(); + const server = createServer((req, res) => { + if (req.url === '/checkout') { + let body = ''; + req.on('data', chunk => { body += chunk; }); + req.on('end', () => { + assert.equal(req.headers['x-application-header'], 'retained'); + assert((req.headers.cookie ?? '').split('; ').includes('original-session=actor-original'), + 'The intercepted request must retain its original HttpOnly session cookie'); + const cart = JSON.parse(body).cart as string; + if (carts.has(cart)) { + retryCookies.set(cart, req.headers.cookie ?? null); + res.writeHead(409); res.end('cart already consumed'); return; + } + carts.add(cart); + res.setHeader('Set-Cookie', `checkout-session=${cart}; Path=/; HttpOnly; SameSite=Lax`); + res.end(JSON.stringify({ order: carts.size })); + }); + } else if (req.url === '/state') res.end(JSON.stringify([...carts])); + else res.end('

ready

'); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(() => { server.closeAllConnections(); server.close(); }); + const address = server.address(); assert(address && typeof address !== 'string'); + const url = `http://127.0.0.1:${address.port}`; + const browser = await chromium.launch({ headless: true }); t.after(() => browser.close()); + for (const { lose, reload } of [{ lose: false, reload: false }, { lose: true, reload: false }, + { lose: true, reload: true }]) await t.test(`loss=${lose}, reload=${reload}`, async () => { + const context = await browser.newContext(); + await context.addCookies([{ name: 'original-session', value: 'actor-original', url, httpOnly: true, sameSite: 'Lax' }]); + const gate = await installResponseLoss(context); + try { + const page = await context.newPage(); await page.goto(url); + // A normal positive checkout may reload this page before the later lost-reply probe. + if (reload) await page.reload(); + if (lose) gate.arm(); + const cart = `cart-${lose}-${reload}`; + await page.evaluate(cart => { + document.querySelector('#result')!.textContent = 'pending'; + void fetch('/checkout', { method: 'POST', headers: { 'x-application-header': 'retained' }, + body: JSON.stringify({ cart }) }).then(async response => { + document.querySelector('#result')!.textContent = await response.text(); + }).catch(() => { document.querySelector('#result')!.textContent = 'unknown'; }); + }, cart); + for (let n = 0; n < 200; n++) { + if (lose ? gate.evidence().events.some(e => e.kind === 'http-response') + : (await page.locator('#result').innerText()).includes('order')) break; + await delay(10); + } + assert((await (await fetch(`${url}/state`)).json()).includes(cart)); + assert.equal(gate.evidence().events.filter(e => e.kind === 'http-response').length, lose ? 1 : 0); + if (lose) { + assert.equal(await page.locator('#result').innerText(), 'pending'); + const request = gate.evidence().events.find(e => e.kind === 'http-request')!; + assert.equal(request.sha256, createHash('sha256').update(JSON.stringify({ cart })).digest('hex')); + } + const sessionCookie = async () => (await context.cookies(url)).find(cookie => cookie.name === 'checkout-session')?.value ?? null; + const cookieBeforeFinish = await sessionCookie(); + await gate.finish(); + if (lose) await page.waitForFunction(() => document.querySelector('#result')!.textContent === 'unknown'); + const cookieAfterFinish = await sessionCookie(); + const before = carts.size; + const retry = await page.evaluate(async cart => (await fetch('/checkout', { + method: 'POST', headers: { 'x-application-header': 'retained' }, body: JSON.stringify({ cart }), + })).status, cart); + assert.equal(retry, 409); assert.equal(carts.size, before); + const observed = { loss: lose, reload, cookieBeforeFinish, cookieAfterFinish, cookieAfterRetry: await sessionCookie(), + retryCookie: retryCookies.get(cart) ?? null, retryStatus: retry }; + evidence.push(observed); + assert.deepEqual([observed.cookieBeforeFinish, observed.cookieAfterFinish, observed.cookieAfterRetry], + [lose ? null : cart, lose ? null : cart, lose ? null : cart], 'Only a delivered reply may update the browser session cookie'); + assert.deepEqual((observed.retryCookie ?? '').split('; ').sort(), + (lose ? ['original-session=actor-original'] : ['original-session=actor-original', `checkout-session=${cart}`]).sort(), + 'Retry must not send a session cookie from the lost reply'); + assert.deepEqual(gate.evidence().errors, []); assert.equal(gate.evidence().truncated, false); + assert.throws(() => gate.arm(), /only once/); + } finally { await gate.finish(); await context.close(); } + }); +}); + +test('native text and binary WebSocket replies are dropped without changing requests or subprotocol', async t => { + // A verified dev socket must survive the fault. A root-path application socket + // with a different token, or a failed client-module lookup, must not be exempt. + const received: Buffer[] = []; + let lookupFails = false; + const server = createServer((req, res) => { + if (req.url === '/@vite/client') { + res.writeHead(lookupFails ? 404 : 200, { 'Content-Type': 'text/javascript' }); + res.end('const wsToken = "verified-dev";'); + } else res.end('

ready

'); + }); + server.on('upgrade', (req, socket) => { + assert.equal(req.headers['sec-websocket-protocol'], 'gate.test'); + const accept = createHash('sha1').update(`${req.headers['sec-websocket-key']}258EAFA5-E914-47DA-95CA-C5AB0DC85B11`).digest('base64'); + socket.write(`HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: ${accept}\r\nSec-WebSocket-Protocol: gate.test\r\n\r\n`); + let pending = Buffer.alloc(0); + // ponytail: this fixture accepts only small, unfragmented frames; use a real + // protocol server if a future control needs extended lengths or fragmentation. + socket.on('data', chunk => { + pending = Buffer.concat([pending, chunk]); + while (pending.length >= 2) { + const opcode = pending[0]! & 15, length = pending[1]! & 127; + assert(length < 126); assert(pending[1]! & 128); assert(pending[0]! & 128); + if (pending.length < 6 + length) return; + const data = Buffer.from(pending.subarray(6, 6 + length)); + for (let i = 0; i < length; i++) data[i] = data[i]! ^ pending[2 + i % 4]!; + pending = pending.subarray(6 + length); + if (opcode === 8) { socket.end(); return; } + assert(opcode === 1 || opcode === 2); + received.push(data); + socket.write(Buffer.concat([Buffer.from([128 | opcode, data.length]), data])); + } + }); + socket.on('error', () => {}); + t.after(() => socket.destroy()); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(() => { server.closeAllConnections(); server.close(); }); + const address = server.address(); assert(address && typeof address !== 'string'); + const url = `http://127.0.0.1:${address.port}`, wsUrl = `ws://127.0.0.1:${address.port}`; + const browser = await chromium.launch({ headless: true }); t.after(() => browser.close()); + const cases = [ + { binary: false, lose: false, failedLookup: false, reload: false }, + { binary: false, lose: true, failedLookup: false, reload: false }, + { binary: true, lose: false, failedLookup: false, reload: false }, + { binary: true, lose: true, failedLookup: false, reload: false }, + { binary: false, lose: true, failedLookup: true, reload: false }, + { binary: false, lose: true, failedLookup: false, reload: true }, + { binary: true, lose: true, failedLookup: false, reload: true }, + ]; + for (const { binary, lose, failedLookup, reload } of cases) await t.test(`binary=${binary}, loss=${lose}, lookup failure=${failedLookup}, reload=${reload}`, async () => { + lookupFails = failedLookup; + const context = await browser.newContext(); + const gate = await installResponseLoss(context); + try { + const page = await context.newPage(); await page.goto(url); + if (reload) { + // Close an intercepted old application connection, then establish the new one below. + await page.evaluate(async wsUrl => { + const socket = new WebSocket(`${wsUrl}/?token=application-session`, 'gate.test'); + await new Promise(resolve => { socket.onopen = () => resolve(); }); + }, wsUrl); + await page.reload(); + } + await page.evaluate(async wsUrl => { + const socket = new WebSocket(`${wsUrl}/?token=application-session`, 'gate.test'); socket.binaryType = 'arraybuffer'; + const dev = new WebSocket(`${wsUrl}/?token=verified-dev`, 'gate.test'); + const devReceived: string[] = []; + Object.assign(window, { socket, dev, devReceived }); + dev.onmessage = event => devReceived.push(event.data); + socket.onmessage = event => { document.querySelector('#result')!.textContent = + typeof event.data === 'string' ? event.data : [...new Uint8Array(event.data)].join(','); }; + await Promise.all([socket, dev].map(ws => new Promise(resolve => { ws.onopen = () => resolve(); }))); + }, wsUrl); + if (lose) gate.arm(); + const count = received.length; + await page.evaluate(binary => { + const socket = (window as unknown as { socket: WebSocket }).socket; + socket.send(binary ? new Uint8Array([0, 128, 255, 42]) : 'checkout'); + }, binary); + for (let n = 0; n < 200; n++) { + if (lose ? gate.evidence().events.some(e => e.kind === 'ws-drop') + : (await page.locator('#result').innerText()) !== 'ready') break; + await delay(10); + } + assert.equal(received.length, count + 1); + assert.deepEqual(received[count], binary ? Buffer.from([0, 128, 255, 42]) : Buffer.from('checkout')); + assert.equal(gate.evidence().events.filter(e => e.kind === 'ws-drop').length, lose ? 1 : 0); + assert.equal(await page.locator('#result').innerText(), lose ? 'ready' : binary ? '0,128,255,42' : 'checkout'); + await page.evaluate(() => (window as unknown as { dev: WebSocket }).dev.send('tooling-update')); + for (let n = 0; n < 200; n++) { + if (failedLookup && lose ? gate.evidence().events.filter(e => e.kind === 'ws-drop').length === 2 + : await page.evaluate(() => (window as unknown as { devReceived: string[] }).devReceived.length === 1)) break; + await delay(10); + } + assert.deepEqual(await page.evaluate(() => (window as unknown as { devReceived: string[] }).devReceived), + failedLookup && lose ? [] : ['tooling-update'], 'Only positively verified tooling may bypass response loss'); + await gate.finish(); + await page.waitForFunction(expected => (window as unknown as { dev: WebSocket }).dev.readyState === expected, + failedLookup ? 3 : 1, { timeout: 1000 }); + assert.deepEqual(gate.evidence().errors, []); assert.equal(gate.evidence().truncated, false); + } finally { await gate.finish(); await context.close(); } + }); +}); diff --git a/tools/stack-bench/tests/review-owner.integration.ts b/tools/stack-bench/tests/review-owner.integration.ts new file mode 100644 index 00000000000..f632ee87bf2 --- /dev/null +++ b/tools/stack-bench/tests/review-owner.integration.ts @@ -0,0 +1,169 @@ +import assert from 'node:assert/strict'; +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { createServer } from 'node:http'; +import { dirname, join } from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import test from 'node:test'; +import { chromium } from 'playwright'; +import { ACTION_REGISTRY } from '../src/actions/action-catalog.js'; +import { executeAction } from '../src/actions/action-contract.js'; +import { createNamedActionsCapability } from '../src/actions/named-action-runtime.js'; +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; + +test('review eligibility sends the claimed username with the nonbuyer credentials', async t => { + const browser = await chromium.launch({ headless: true }); + t.after(() => browser.close()); + const context = await browser.newContext(); + await context.route('http://app.test/**', route => route.fulfill({ contentType: 'text/html', + body: `
Keyboard
` })); + const page = await context.newPage(); + await page.goto('http://app.test/'); + const actors = new Map(['owner', 'stranger'].map(name => [name, { + name, page, context, record() {}, + writes: ['http://app.test/api/session', 'http://native.test/v1/database/shop/call/sign_in'] + .map(url => ({ url, headers: { authorization: `Bearer ${name}` } })), + loc: () => page.locator('[data-role="item-card"]'), + }])); + const scenario = compileScenarioDefinition(JSON.parse(readFileSync(join(STACK_BENCH_ROOT, + 'tracks/ecommerce/scenarios/progression-review-access.json'), 'utf8'))); + const steps = scenario.features[0]!.criteria[0]!.steps; + const calls = steps.filter(step => step.do === 'callAction'); + assert.equal(calls.length, 3); + for (const backend of ['postgres', 'mongodb', 'spacetime']) { + const requests: Array<{ url: string; body: string; authorization: string | undefined }> = []; + const capabilities = { + actors: { get: (name: string) => actors.get(name) }, + 'transport-observation': { defaultWithin: 1000, sleep: async () => {}, + verification: { verified() {}, unverified() {} } }, + 'named-actions': createNamedActionsCapability({ backend, url: 'http://app.test', + spacetime: { uri: 'http://native.test', mod: 'shop' }, + lastCalls: { get: () => null, set() {} }, sleep: async () => {}, + fetchImpl: async (url, options) => { + requests.push({ url, body: options.body!, authorization: options.headers?.authorization }); + return { status: 200, ok: true, text: async () => '' }; + } }), + }; + for (const step of calls) { + const result = await executeAction(ACTION_REGISTRY, step.do, step, { capabilities }); + assert.equal(result.status, 'passed', result.summary ?? step.do); + } + assert.deepEqual(requests.map(request => request.authorization), + ['Bearer owner', 'Bearer stranger', 'Bearer stranger']); + const attack = requests[2]!; + if (backend === 'spacetime') { + assert.equal(attack.url, 'http://native.test/v1/database/shop/call/submit_review'); + assert.equal(attack.body, '[9007199254740993,5,"Forged buyer review","claim-review-owner"]'); + assert.equal(requests[0]!.body, '[9007199254740993,5,"Original buyer review"]'); + } else { + assert.equal(attack.url, 'http://app.test/api/items/9007199254740993/reviews'); + assert.deepEqual(JSON.parse(attack.body), { + rating: 5, comment: 'Forged buyer review', username: 'claim-review-owner', + }); + assert.deepEqual(JSON.parse(requests[0]!.body), { rating: 5, comment: 'Original buyer review' }); + } + } +}); + +test('named action inputs wait for browser initialization without sending incomplete requests', async t => { + // A row can render before its input arrives. Invalid, missing, late, or + // cancelled input must still send no request, including concurrent groups. + const evidence: unknown[] = []; + t.after(() => { + const file = process.env.STACK_BENCH_NAMED_INPUT_EVIDENCE; + if (file) { + mkdirSync(dirname(file), { recursive: true }); + writeFileSync(file, JSON.stringify({ + rerun: 'STACK_BENCH_NAMED_INPUT_EVIDENCE= node --test --test-name-pattern="named action inputs wait" dist/tests/review-owner.integration.js', + fixture: 'Browser row renders before a held HTTP input response; registered actions use real HTTP writes.', + evidence, + }, null, 2) + '\n'); + } + }); + const browser = await chromium.launch({ headless: true }); + t.after(() => browser.close()); + for (const mode of ['primary', 'override', 'concurrent', 'alongside', 'missing', 'malformed', 'late', 'cancelled']) { + await t.test(mode, async () => { + let release!: () => void; + const held = new Promise(resolve => { release = resolve; }); + let started!: () => void; + const hydrating = new Promise(resolve => { started = resolve; }); + const requests: Array<{ path: string; authorization: string | undefined }> = []; + const server = createServer((request, response) => { + if (request.url === '/hydrate') { + started(); + void held.then(() => { response.setHeader('Content-Type', 'application/json'); response.end('{}'); }); + } else if (request.method === 'POST') { + requests.push({ path: request.url!, authorization: request.headers.authorization }); + response.setHeader('Content-Type', 'application/json'); response.end('{}'); + } else { + response.setHeader('Content-Type', 'text/html'); + response.end(`
Desk Lamp
+
Keyboard
`); + } + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); assert(address && typeof address !== 'string'); + const url = `http://127.0.0.1:${address.port}`; + const context = await browser.newContext(); + try { + const page = await context.newPage(); + await page.goto(url, { waitUntil: 'domcontentloaded' }); + await hydrating; + const actors = new Map(['owner', 'other'].map(name => [name, { + name, page, context, record() {}, writes: [{ url, headers: { authorization: `Bearer ${name}` } }], + loc: (id: string) => page.locator(`[data-role="${id}"]`), + }])); + const sleep = (ms: number, signal: AbortSignal) => delay(ms, undefined, { signal }); + const capabilities = { + actors: { get: (name: string) => actors.get(name) }, + 'transport-observation': { defaultWithin: 300, sleep, verification: { verified() {}, unverified() {} } }, + 'named-actions': createNamedActionsCapability({ backend: 'mongodb', url, + lastCalls: { get: () => null, set() {} }, sleep }), + }; + const namedAction = { id: 'cancel', path: '/api/orders/:id/cancel', reducer: 'cancel_order', args: [0], + params: [{ name: 'orderId', in: 'path', placeholder: ':id', wireType: 'u64' }] }; + const input = { testid: 'order-item', contains: 'Desk Lamp', attribute: 'data-cancel-input', + ...(mode === 'override' ? { overrides: { orderId: { + actor: 'owner', testid: 'other-order', attribute: 'data-entity-id', + } } } : {}) }; + const group = { action: 'cancel', namedAction, input, from: 'owner', actors: ['other'], requests: 1, requestTimeoutMs: 300 }; + const step = mode === 'alongside' + ? { do: 'callConcurrently', action: 'cancel', actors: ['other'], requests: 1, settleMs: 0, + namedAction: { ...namedAction, params: [], args: [], path: '/control' }, alongside: [group] } + : mode === 'concurrent' ? { do: 'callConcurrently', ...group, settleMs: 0 } + : { do: 'callAction', actor: 'other', from: 'owner', action: 'cancel', namedAction, input, settleMs: 0 }; + const controller = new AbortController(); + const pending = executeAction(ACTION_REGISTRY, step.do, step, { capabilities, signal: controller.signal }); + await delay(75); + if (mode === 'cancelled') controller.abort('fixture cancellation'); + if (mode !== 'late' && mode !== 'cancelled') release(); + const result = await pending; + if (mode === 'late') { release(); await delay(30); } + const expected = ['primary', 'override', 'concurrent', 'alongside'].includes(mode) ? 'passed' + : mode === 'cancelled' ? 'inconclusive' : 'failed'; + evidence.push({ mode, expected, result, requests }); + assert.equal(result.status, expected, result.summary ?? mode); + if (expected === 'passed') { + const byPath = (a: { path: string }, b: { path: string }) => a.path.localeCompare(b.path); + assert.deepEqual(requests.toSorted(byPath), [ + ...(mode === 'alongside' ? [{ path: '/control', authorization: 'Bearer other' }] : []), + { path: '/api/orders/9007199254740993/cancel', authorization: 'Bearer other' }, + ].toSorted(byPath)); + } else { + assert.equal(requests.length, 0, 'unready input must never dispatch a write'); + if (mode !== 'cancelled') assert.equal(result.finding?.kind, + mode === 'malformed' ? 'interface-invalid' : 'interface-missing'); + } + } finally { + release(); await context.close(); server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + } + }); + } +}); diff --git a/tools/stack-bench/tests/run-checkpoints.test.ts b/tools/stack-bench/tests/run-checkpoints.test.ts new file mode 100644 index 00000000000..f34841aacac --- /dev/null +++ b/tools/stack-bench/tests/run-checkpoints.test.ts @@ -0,0 +1,75 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { createCheckEvidence } from '../src/evidence/check-evidence.js'; +import { checkpointChecks, checkpointSessions, completionCurve, recordRunCheckpoint } from '../src/evidence/run-checkpoints.js'; +import type { CheckpointRun } from '../src/evidence/run-checkpoints.js'; + +const selected = [{ stableKey: 'feature.a', points: 9 }, { stableKey: 'feature.b', points: 1 }, + { stableKey: 'control', points: 0 }]; +const run = (): CheckpointRun => ({ condition: { requested: { levels: [{ selection: { scoredChecks: selected } }] } } }); +const session = (costUsd: number, exact = true) => ({ costUsd, costComplete: true, + costReceipts: [{ receipt: { costUsd, exact, complete: true, reconciled: true, error: null } }] }); +function measure(target: CheckpointRun, passed: string[], costUsd: number, accepted = true, exact = true, + blocked: string[] = []) { + const sequence = (target.checkpoints?.length ?? 0) + 1; + return recordRunCheckpoint(target, { phase: sequence === 1 ? 'first-build' : 'repair', level: 1, + accepted, extraSessions: [session(costUsd, exact)], sourceSha256: String(sequence).repeat(64), + evidence: { path: `grades/${sequence}/bundle.json`, sha256: 'a'.repeat(64) }, + bundle: { selection: { sha256: 'b'.repeat(64), reportedChecks: selected.map(check => check.stableKey) }, + suites: { app: { features: [{ criteria: selected.map(check => ({ ...check, + evidence: blocked.includes(check.stableKey) + ? createCheckEvidence({ status: 'blocked', phase: 'setup', code: 'application_failure', + startedAtMs: 1, completedAtMs: 2 }) + : createCheckEvidence({ status: passed.includes(check.stableKey) ? 'passed' : 'failed', + code: passed.includes(check.stableKey) ? 'completed' : 'application_assertion', + phase: 'assertion', startedAtMs: 1, completedAtMs: 2 }) })) }] } } } }); +} + +test('check completion has a fixed denominator, excludes zero-point controls, and counts no weighted points', () => { + const point = measure(run(), ['feature.a'], 2); + assert.deepEqual(point.completion, { selected: 2, passed: 1, failed: 1, blocked: 0, unmeasured: 0, rate: 0.5 }); + assert.deepEqual(point.cost, { status: 'exact', costUsd: 2 }); + const blocked = measure(run(), ['feature.a'], 2, true, true, ['feature.b']); + assert.deepEqual(blocked.completion, { selected: 2, passed: 1, failed: 0, blocked: 1, unmeasured: 0, rate: 0.5 }); +}); + +test('cost curves use measured checkpoints, preserve regressions, bounds, and targets not reached', () => { + const target = run(); + measure(target, ['feature.a'], 2); + measure(target, [], 3, false); + measure(target, ['feature.a', 'feature.b'], 4, true, false); + const curve = completionCurve(target.checkpoints!, [1, 2, 3, 4], [0.5, 1]); + assert.deepEqual(curve.completionAtSpend.map(point => point.completion?.rate ?? null), [null, 0.5, 0.5, 1]); + assert.deepEqual(curve.costToCompletion.map(point => point.cost), [ + { status: 'exact', costUsd: 2 }, { status: 'upper-bound', costUsd: 4 }]); + assert.equal(curve.checkpoints[1]?.completion.rate, 0, 'rejected regression remains visible'); + const partial = completionCurve(target.checkpoints!.slice(0, 1), [], [1]); + assert.equal(partial.costToCompletion[0]?.status, 'not-reached'); + assert.equal(partial.costToCompletion[0]?.cost.costUsd, null); + assert.equal(completionCurve([], [2], [1]).costToCompletion[0]?.status, 'unmeasured'); +}); + +test('session extraction excludes inherited levels and preserves distinct equal-cost paid sessions', () => { + const first = session(2); + const second = session(2); + const target = { levels: [{ level: 1, buildSessions: [session(9)] }, + { level: 2, buildSessions: [first] }], progressionResume: { inheritedLevels: [1] } }; + assert.deepEqual(checkpointSessions(target, [first, second]), [first, second]); + const resumed = { ...run(), ...target }; + assert.equal(measure(resumed, ['feature.a'], 1).cost.status, 'unknown', 'missing prior proof stays unknown'); +}); + + +test('checkpoint replay excludes diagnostic checks and preserves accepted state across rejected grades', () => { + const target = run(); + measure(target, ['feature.a'], 1); + measure(target, [], 2, false); + const previous = target.checkpoints![0]!.checks; + const checks = checkpointChecks(selected.map(check => ({ id: check.stableKey, points: check.points })), + [...previous, { id: 'outside.scope', status: 'passed' }], { suites: {}, selection: { reportedChecks: [] } }); + assert.deepEqual(checks, [{ id: 'feature.a', status: 'passed' }, { id: 'feature.b', status: 'failed' }]); + const checkpoint = recordRunCheckpoint(target, { level: 1, phase: 'repair', + sourceSha256: 'a'.repeat(64), evidence: { path: 'grade.json', sha256: 'b'.repeat(64) }, + bundle: { suites: {}, selection: { sha256: 'c'.repeat(64), reportedChecks: [] } } }); + assert.equal(checkpoint.completion.rate, 0.5); +}); diff --git a/tools/stack-bench/tests/run-resource-selection.test.ts b/tools/stack-bench/tests/run-resource-selection.test.ts new file mode 100644 index 00000000000..ccae64cb96e --- /dev/null +++ b/tools/stack-bench/tests/run-resource-selection.test.ts @@ -0,0 +1,86 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { loadTrack, portsFor } from '../src/composition/tracks.js'; +import { runResourceLockKeys } from '../src/runtime/backend-lease.js'; +import { selectRunResources } from '../src/runtime/run-resource-selection.js'; + +test('resource selection skips exclusions, held locks and occupied ports across every requested stack', async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-selection-')); + try { + const track = loadTrack('ecommerce'); + const held = 'slot:ecommerce:mongodb:run1'; + writeFileSync(join(root, `${createHash('sha256').update(held).digest('hex')}.lock.json`), '{}'); + const busy = portsFor(track, 'postgres', 2).express; + const backends = ['postgres', 'mongodb']; + const result = await selectRunResources({ track, backends, count: 2, + serverUri: () => null, excludedRunIndices: [0], + env: { STACK_BENCH_RESOURCE_LOCK_DIR: root }, probePort: port => ({ free: port !== busy }) }); + assert.deepEqual(result.runIndices, [3, 4]); + assert.deepEqual(result.keys, result.runIndices.flatMap(runIndex => backends.flatMap(backend => + runResourceLockKeys({ track: track.name, backend, runIndex, ports: portsFor(track, backend, runIndex) })))); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('resource selection probes the database listener and keeps selected groups disjoint', async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-selection-')); + try { + const result = await selectRunResources({ track: loadTrack('ecommerce'), backends: ['spacetime'], count: 2, + // Runs 1 and 2 cannot both own the same database listener. + serverUri: index => `http://127.0.0.1:${index < 3 ? 18000 + Math.min(index, 1) : 18000 + index}`, + env: { STACK_BENCH_RESOURCE_LOCK_DIR: root }, probePort: port => ({ free: port !== 18000 }) }); + assert.deepEqual(result.runIndices, [1, 3]); + } finally { rmSync(root, { recursive: true, force: true }); } + const mixedRoot = mkdtempSync(join(tmpdir(), 'stack-bench-selection-')); + try { + const mixed = await selectRunResources({ track: loadTrack('ecommerce'), + backends: ['spacetime', 'convex'], count: 1, + serverUri: (index, backend) => `http://127.0.0.1:${(backend === 'convex' ? 19000 : 18000) + index}`, + env: { STACK_BENCH_RESOURCE_LOCK_DIR: mixedRoot }, probePort: port => ({ free: port !== 19000 }) }); + assert.deepEqual(mixed.runIndices, [1]); + assert(mixed.keys.includes('port:18001')); + assert(mixed.keys.includes('port:19001')); + } finally { rmSync(mixedRoot, { recursive: true, force: true }); } +}); + +test('resource selection aborts before probing on cancellation', async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-selection-')); + try { + const input = { track: loadTrack('ecommerce'), backends: ['postgres'], count: 1, + serverUri: () => null, env: { STACK_BENCH_RESOURCE_LOCK_DIR: root }, + probePort: () => ({ free: true }) }; + await assert.rejects(selectRunResources({ ...input, signal: AbortSignal.abort(), + probePort: () => { throw new Error('must not probe'); } }), { name: 'AbortError' }); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('appliance admission skips ports Docker cannot publish and stops on other Docker errors', { + skip: process.platform !== 'linux' && 'uses a POSIX docker stub', +}, async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-selection-')); + const bin = mkdtempSync(join(tmpdir(), 'stack-bench-docker-')); + try { + const track = loadTrack('ecommerce'); + const busy = portsFor(track, 'postgres', 0).vite; + // Docker Desktop reports host ports the Linux loopback probe cannot see. + writeFileSync(join(bin, 'docker'), [ + '#!/bin/sh', + '[ -n "$DOCKER_STUB_ERROR" ] && { echo "$DOCKER_STUB_ERROR" >&2; exit 125; }', + `case " $* " in *"127.0.0.1:${busy}:${busy}"*) echo "port is already allocated" >&2; exit 125;; esac`, + 'exit 0', '', + ].join('\n'), { mode: 0o755 }); + const env = { STACK_BENCH_RESOURCE_LOCK_DIR: root, STACK_BENCH_APPLIANCE: '1', + STACK_BENCH_CONTROLLER_IMAGE_ID: `sha256:${'a'.repeat(64)}`, PATH: `${bin}:${process.env.PATH}` }; + const input = { track, backends: ['postgres'], count: 1, serverUri: () => null, + probePort: () => ({ free: true }) }; + assert.deepEqual((await selectRunResources({ ...input, env })).runIndices, [1]); + await assert.rejects(selectRunResources({ ...input, env: { ...env, DOCKER_STUB_ERROR: 'Unable to find image' } }), + /Command failed/); + } finally { + rmSync(root, { recursive: true, force: true }); + rmSync(bin, { recursive: true, force: true }); + } +}); diff --git a/tools/stack-bench/tests/run-session-artifacts.test.ts b/tools/stack-bench/tests/run-session-artifacts.test.ts new file mode 100644 index 00000000000..ff499b410ce --- /dev/null +++ b/tools/stack-bench/tests/run-session-artifacts.test.ts @@ -0,0 +1,125 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { runSessionRecord } from '../src/evidence/benchmark-run.js'; +import { readRunJson, writeRunJson } from '../src/evidence/artifacts.js'; +import type { AgentCostReceiptEntry, ValidatedAgentResult } + from '../src/agents/agent-result-contract.js'; +import type { RunSessionRecord } from '../src/evidence/benchmark-run.js'; + +function receipt(): AgentCostReceiptEntry[] { + return [{ invocation: 1, receipt: { schemaVersion: 3, source: 'credential-broker', exact: true, estimatedRequests: 0, + estimatedByReason: { 'no-usage': 0, 'response-aborted': 0, 'upstream-error': 0 }, + model: 'test-model', maxBudgetUsd: 10, costUsd: 1, cliCostUsd: 1, + calculatedCostUsd: 1, usage: { input: 1, output: 1, cacheWrite5m: 0, + cacheWrite1h: 0, cacheRead: 0 }, pricingRates: { input: 1, output: 1, + cacheWrite5m: 1, cacheWrite1h: 1, cacheRead: 1 }, complete: true, + reconciled: true, error: null } }]; +} + +function session(marker: string, + { billable = true, costComplete = true }: { billable?: boolean; costComplete?: boolean } = {}, +): ValidatedAgentResult { + return { appDir: '/app', mode: 'build', level: 1, backend: 'stub', track: 'loop', + model: 'test-model', guidance: null, ok: true, sessionId: marker, + costUsd: billable ? 1 : 0, durationMs: 10, + usage: { input: 1, output: 1, cacheWrite: 0, cacheRead: 0 }, + costReceipts: billable ? receipt() : [], + setup: { resources: { buildContainerMemory: { + currentBytes: 100, peakBytes: 200, limitBytes: 400, + oomEvents: 0, oomKillEvents: 0, + }, buildContainerPids: { current: 8, peak: 512, limit: 512, limitEvents: 129 }, + memoryProbeError: null } }, tokens: 2, outputTokens: 1, turns: 1, + promptBytes: 20, thinking: null, transcript: { kind: 'test', id: marker }, + provenance: null, providerMetadata: null, costComplete }; +} + +type SessionLocation = 'buildSessions' | 'repairSessions'; + +interface SessionArtifactLevel { + buildSessions?: RunSessionRecord[]; + repairSessions?: RunSessionRecord[]; + resumedRepair?: Record; + outcome?: { kind: string; phase?: string }; +} + +interface SessionCase { + name: string; + marker: string; + billable?: boolean; + costComplete?: boolean; + round?: number; + level: (source: ValidatedAgentResult) => SessionArtifactLevel; + location: SessionLocation; +} + +const cases: readonly SessionCase[] = [ + { name: 'successful build', marker: 'build', + level: source => ({ buildSessions: [runSessionRecord(source)] }), + location: 'buildSessions' }, + { name: 'failed build', marker: 'failed-build', costComplete: false, + level: source => ({ outcome: { kind: 'harness_failure' }, + buildSessions: [runSessionRecord(source)] }), + location: 'buildSessions' }, + { name: 'normal repair', marker: 'repair', round: 1, + level: source => ({ repairSessions: [runSessionRecord(source, 1)] }), + location: 'repairSessions' }, + { name: 'resumed repair', marker: 'resumed-repair', round: 4, + level: source => ({ resumedRepair: {}, repairSessions: [runSessionRecord(source, 4)] }), + location: 'repairSessions' }, + { name: 'model-free early exit', marker: 'model-free', billable: false, + level: source => ({ outcome: { kind: 'ungraded', phase: 'reference-mutation-only' }, + buildSessions: [runSessionRecord(source)] }), + location: 'buildSessions' }, +]; + +interface StoredSession { + costReceipts: unknown[]; + costComplete: boolean; + round?: number; + resources: unknown; +} + +function object(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function storedSession(run: unknown, location: SessionLocation): StoredSession { + if (!object(run) || !Array.isArray(run.levels) || !object(run.levels[0])) { + throw new Error('run artifact is missing its first level'); + } + const level = run.levels[0]; + const sessions = level[location]; + const candidate = Array.isArray(sessions) ? sessions[0] : undefined; + if (!object(candidate) || !Array.isArray(candidate.costReceipts) + || typeof candidate.costComplete !== 'boolean') { + throw new Error(`run artifact is missing ${location} cost data`); + } + const round = typeof candidate.round === 'number' ? candidate.round : undefined; + return { costReceipts: candidate.costReceipts, costComplete: candidate.costComplete, + resources: candidate.resources, + ...(round === undefined ? {} : { round }) }; +} + +for (const item of cases) { + test(`${item.name} keeps its cost receipts in run.json`, () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-session-artifact-')); + try { + const path = join(root, 'run.json'); + const source = session(item.marker, { billable: item.billable !== false, + costComplete: item.costComplete !== false }); + writeRunJson(path, { id: `run-${item.marker}`, levels: [item.level(source)] }); + const stored = storedSession(readRunJson(path, `run-${item.marker}`), item.location); + assert.deepEqual(stored.costReceipts, source.costReceipts); + assert.equal(Object.hasOwn(stored, 'costReceipts'), true); + assert.equal(stored.costComplete, source.costComplete); + assert.deepEqual(stored.resources, source.setup.resources); + if (item.round !== undefined) assert.equal(stored.round, item.round); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +} diff --git a/tools/stack-bench/tests/run-suite.test.ts b/tools/stack-bench/tests/run-suite.test.ts new file mode 100644 index 00000000000..a54f62a0d34 --- /dev/null +++ b/tools/stack-bench/tests/run-suite.test.ts @@ -0,0 +1,502 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { createBoundRecipeTaskRequest, selectScenarioChecks } from '../src/composition/recipe-selection.js'; +import { isModularRecipeTaskRequest } from '../src/composition/recipe-selection.js'; +import { requireRecipeRelease as resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { attachRegressionScope, childFailureDetail, clearPreviousGradeOutputs, findMutationBackups, selectObservationScope, + applicationFailureTotals, codeMetrics, resetFailureOutcome, suitesForRecipe, + checkRuntimeDatabaseProvenance, databaseProvenanceFailure, + contractLintArgv, databaseLeaseForGrading, databaseNameForGrading, runGraderChild, + verifyApplicationProbe, waitForApplicationProbe, closeSuiteBrowser, preserveStartFailure, suiteMayRetry } + from '../commands/run-suite.js'; +import { loadTrack } from '../src/composition/tracks.js'; +import { GENERATED_APP_LAYOUT_EXIT_CODE } from '../src/stacks/backend-reset.js'; +import { createBackendLease } from '../src/runtime/backend-lease.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; +import { readArtifactPayload } from '../src/evidence/artifacts.js'; +import { createCheckEvidence } from '../src/evidence/check-evidence.js'; + +// Recovery must not erase product failures, ignore cleanup, or retry unknown errors. +test('suite recovery requires exclusively passed or explicitly retryable inconclusive evidence', () => { + const criterion = (status: 'passed' | 'failed' | 'blocked' | 'inconclusive' | 'harness_failure', retryable = false) => ({ + id: status, evidence: createCheckEvidence({ status, retryable, code: 'test_result', + phase: status === 'blocked' ? 'setup' : 'assertion', startedAtMs: 0, completedAtMs: 1 }), + }); + const grade = (...criteria: ReturnType[]) => ({ total: 0, max: 1, + features: [{ name: 'test', criteria }] }); + assert.equal(suiteMayRetry(grade(criterion('inconclusive', true))), true); + assert.equal(suiteMayRetry(grade(criterion('passed'), criterion('inconclusive', true))), true); + for (const status of ['failed', 'blocked', 'harness_failure', 'inconclusive'] as const) { + assert.equal(suiteMayRetry(grade(criterion(status), criterion('inconclusive', true))), false, status); + } + assert.equal(suiteMayRetry(grade()), false); + assert.equal(suiteMayRetry(grade(criterion('passed'))), false); + const cleanup = grade(criterion('inconclusive', true)); + assert.equal(suiteMayRetry({ ...cleanup, features: [{ ...cleanup.features[0]!, + cleanupEvidence: { status: 'harness_failure', failures: [{ stage: 'context-close' }] } }] }), false); + assert.equal(suiteMayRetry({ total: 0, max: 1, features: [{ name: 'missing', criteria: [{ id: 'missing' }] }] }), false); +}); + +const ECOMMERCE = join(STACK_BENCH_ROOT, 'tracks', 'ecommerce'); + +test('application startup failures retain useful logs with credentials removed', () => { + const output = mkdtempSync(join(tmpdir(), 'startup-failure-log-')); + try { + preserveStartFailure({ startLog: 'early cause\npassword=do-not-publish\nlast line' }, output); + const log = readFileSync(join(output, 'application-start.log'), 'utf8'); + assert.match(log, /early cause/); + assert.match(log, /last line/); + assert.doesNotMatch(log, /do-not-publish/); + } finally { rmSync(output, { recursive: true, force: true }); } +}); + +test('browser shutdown failure replaces a previously written app outcome before returning', async () => { + const bundle = { error: 'app restart failed', + outcome: { kind: 'app_failure', phase: 'application-start', reason: 'app restart failed' } }; + let saved = structuredClone(bundle); + const failure = new Error('browser control disconnected'); + await assert.rejects(closeSuiteBrowser({ close: async () => { throw failure; } }, bundle, + () => { saved = structuredClone(bundle); }), error => error === failure); + assert.equal(saved.outcome.kind, 'harness_failure'); + assert.equal(saved.outcome.phase, 'grading-cleanup'); + assert.match(saved.outcome.reason, /browser control disconnected/); + await closeSuiteBrowser({ close: async () => {} }, bundle, + () => assert.fail('successful cleanup must not replace evidence')); +}); + +function sequentialL2Track() { + const temp = mkdtempSync(join(tmpdir(), 'stack-bench-sequential-l2-')); + const root = join(temp, 'ecommerce'); + cpSync(ECOMMERCE, root, { recursive: true }); + return { temp, track: { ...loadTrack('ecommerce'), dir: root } }; +} + +test('a targeted staff repair writes explicit empty lint evidence without falling back to level contracts', () => { + const out = mkdtempSync(join(tmpdir(), 'stack-bench-empty-lint-')); + try { + for (const level of [2, 3]) { + const binding = resolveRecipeRelease(loadTrack('ecommerce'), level, 'ecommerce.progression-catalog'); + const selected = createBoundRecipeTaskRequest(binding, { + featureIds: ['ecommerce.progression.staff-access'], + taskMode: 'upgrade', + }); + assert(selected.selection.checks.length > 0, 'scenario grading must still select staff checks'); + const argv = contractLintArgv({ + url: 'http://127.0.0.1:1', level: String(level), track: 'ecommerce', label: 'staff-repair', + out, bundleArtifactId: 'staff-repair', + }, selected); + assert(argv.includes('--selected-hooks')); + assert.equal(argv.includes('--hook'), false); + const result = spawnSync(process.execPath, argv, { encoding: 'utf8' }); + assert.equal(result.status, 0, result.stderr || result.stdout); + const report = readArtifactPayload<{ selectedHooks: string[]; pass: boolean; results: unknown[] }>( + join(out, 'contract-lint.json'), { expectedKind: 'contract_lint' }); + assert.deepEqual(report.selectedHooks, []); + assert.equal(report.pass, true); + assert.deepEqual(report.results, []); + + const unscoped = spawnSync(process.execPath, argv.filter(value => value !== '--selected-hooks'), + { encoding: 'utf8' }); + assert.equal(unscoped.status, 2); + assert.match(unscoped.stderr, /No contracts found/); + } + } finally { rmSync(out, { recursive: true, force: true }); } +}); + +test('code metrics count each package manifest once when server code is at the app root', () => { + const temp = mkdtempSync(join(tmpdir(), 'stack-bench-code-metrics-')); + try { + mkdirSync(join(temp, 'client'), { recursive: true }); + writeFileSync(join(temp, 'index.js'), 'export const app = true;\n'); + writeFileSync(join(temp, 'package.json'), JSON.stringify({ + dependencies: { express: '1', mongodb: '1' }, + })); + writeFileSync(join(temp, 'client', 'package.json'), JSON.stringify({ + dependencies: { react: '1' }, + })); + + assert.equal(codeMetrics({ app: temp, backend: 'mongodb' }).runtimeDeps, 3); + } finally { + rmSync(temp, { recursive: true, force: true }); + } +}); + +test('mutation backup scanning ignores volatile build caches and tolerates their removal', () => { + const temp = mkdtempSync(join(tmpdir(), 'stack-bench-mutation-scan-')); + try { + mkdirSync(join(temp, 'server', 'src'), { recursive: true }); + mkdirSync(join(temp, 'client', 'node_modules', '.vite', 'deps_temp'), { recursive: true }); + const backup = join(temp, 'server', 'src', 'index.ts.mutation-backup'); + writeFileSync(backup, 'source'); + writeFileSync(join(temp, 'client', 'node_modules', '.vite', 'deps_temp', + 'cache.mutation-backup'), 'generated'); + assert.deepEqual(findMutationBackups(temp), [backup]); + + const disappearing = Object.assign(new Error('directory disappeared'), { code: 'ENOENT' }); + const directory = { name: 'source', isDirectory: () => true, isFile: () => false }; + assert.deepEqual(findMutationBackups('/app', { + readDir: dir => { + if (dir === '/app') return [directory]; + throw disappearing; + }, + }), []); + } finally { rmSync(temp, { recursive: true, force: true }); } +}); + +test('grader child diagnostics retain the cause instead of only trailing stack frames', () => { + const stderr = [ + 'Error: check evidence action is malformed', + ' at validateCheckEvidence (check-evidence.mjs:1:1)', + ' at buildCheckEvidence (grade.mjs:2:2)', + ' at gradeFeature (grade.mjs:3:3)', + ' at async main (grade.mjs:4:4)', + '', + 'Node.js v24.18.1', + ].join('\n'); + const detail = childFailureDetail({ stderr, message: 'command failed' }); + assert.match(detail, /^Error: check evidence action is malformed \|/); + assert.match(detail, /gradeFeature/); + assert.doesNotMatch(detail, /validateCheckEvidence/); + + const rejection = [ + 'node:internal/process/promises:394', + ' triggerUncaughtException(err, true /* fromPromise */);', + ' ^', + '', + 'browserContext.close: Target page, context or browser has been closed', + ' at closeAll (grade.mjs:596:21)', + 'Node.js v24.18.1', + ].join('\n'); + assert.match(childFailureDetail({ stderr: rejection }), + /^browserContext\.close: Target page, context or browser has been closed/); + + const processDetail = childFailureDetail({ + stderr: 'hosted application port 6301 still has a listener', + message: 'Command failed: docker exec generated-app sh -lc ', + }); + assert.equal(processDetail, 'hosted application port 6301 still has a listener'); +}); + +test('grader subprocesses run asynchronously and retain redacted output', async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-grader-child-')); + try { + const result = await runGraderChild(['--eval', + "console.log('starting account-create'); console.error('ANTHROPIC_API_KEY=should-not-leak'); process.exit(1)"], + root, 'account-create'); + assert(result.failure); + assert.match(result.stderr, /\[redacted credential\]/); + assert.doesNotMatch(result.stderr, /should-not-leak/); + assert.equal(readFileSync(join(root, result.stdoutName), 'utf8'), 'starting account-create\n'); + assert.equal(readFileSync(join(root, result.stderrName), 'utf8'), '[redacted credential]\n'); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('database grading uses the exact container from the authenticated run lease', () => { + const calls: Array<{ path: string; + expected?: { token?: string; backend?: string; active?: boolean } }> = []; + const lease = databaseLeaseForGrading('mongodb', { + STACK_BENCH_LEASE: 'private/lease.json', + STACK_BENCH_LEASE_TOKEN: 'secret-token', + }, { + readLease: (path, expected) => { + calls.push({ path, expected }); + return createBackendLease({ runId: 'grading-test', backend: 'mongodb', track: 'ecommerce', + runIndex: 0, database: 'app_ecommerce_run0', container: { + name: 'stack-bench-mongodb', id: 'mongodb-container-id' } }); + }, + }); + assert.equal(lease?.resources.container?.name, 'stack-bench-mongodb'); + assert.equal(lease?.resources.container?.id, 'mongodb-container-id'); + assert.deepEqual(calls, [{ path: 'private/lease.json', + expected: { token: 'secret-token', backend: 'mongodb', active: true } }]); + assert.throws(() => databaseLeaseForGrading('postgres', { + STACK_BENCH_LEASE: 'private/lease.json', + }), /both lease path and lease token/); + assert.equal(databaseLeaseForGrading('spacetime', {}), null); + + const env = { STACK_BENCH_LEASE: 'private/lease.json', STACK_BENCH_LEASE_TOKEN: 'secret-token' }; + const convex = databaseLeaseForGrading('convex', env, { readLease: (_path, expected) => { + assert.deepEqual(expected, { token: 'secret-token', backend: 'convex', active: true }); + const native = createBackendLease({ runId: 'grading-convex', backend: 'convex', track: 'ecommerce', runIndex: 0, + serverUri: 'http://127.0.0.1:14310' }); + native.resources.container = { name: 'owned-convex', id: 'a'.repeat(64), owned: true }; + return native; + } }); + assert.equal(convex?.resources.serverUri, 'http://127.0.0.1:14310'); + assert.equal(convex?.resources.container?.id, 'a'.repeat(64)); + + const spacetimeLease = () => createBackendLease({ runId: 'grading-test', backend: 'spacetime', + track: 'ecommerce', runIndex: 0, module: 'app_ecommerce_run0', + serverUri: 'http://127.0.0.1:3210', dataDir: join(tmpdir(), 'stack-bench-spacetime-test') }); + const spacetime = databaseLeaseForGrading('spacetime', env, { readLease: spacetimeLease }); + assert.equal(spacetime?.resources.module, 'app_ecommerce_run0'); + assert.equal(spacetime?.resources.serverUri, 'http://127.0.0.1:3210'); + assert.throws(() => databaseLeaseForGrading('spacetime', env, { readLease: () => { + const incomplete = spacetimeLease(); + incomplete.resources.module = null; + return incomplete; + } }), /no complete module target/); +}); + +test('database grading uses the exact database from the authenticated run lease', () => { + const track = { slug: 'ecom' }; + assert.equal(databaseNameForGrading(track, 3), 'app_ecom_run3'); + assert.equal(databaseNameForGrading(track, 3, { + resources: { database: 'leased_database' }, + }), 'leased_database'); + assert.throws(() => databaseNameForGrading(track, 3, { resources: {} }), + /active database lease has no database name/); +}); + +test('runtime database proof requires an authenticated lease', () => { + assert.deepEqual(checkRuntimeDatabaseProvenance({ backend: 'spacetime' }), { + ok: null, + verified: false, + reason: 'standalone grading has no authenticated database lease', + }); + assert.deepEqual(databaseProvenanceFailure(new Error('docker command failed')), { + kind: 'harness_failure', + phase: 'database-provenance', + reason: 'runtime database provenance failed: docker command failed', + }); +}); + +test('generated layout and restart defects are repairable app failures, not harness failures', () => { + const application = Object.assign(new Error('server/package.json has no dev or start script'), + { code: 'generated_app_not_restartable' }); + assert.deepEqual(resetFailureOutcome(application), + { kind: 'app_failure', phase: 'application-restart', + appFailures: ['application-restart'] }); + assert.deepEqual(resetFailureOutcome(new Error('database container disappeared')), + { kind: 'harness_failure', phase: 'database-reset' }); + assert.deepEqual(resetFailureOutcome({ status: GENERATED_APP_LAYOUT_EXIT_CODE }), + { kind: 'app_failure', phase: 'application-layout', + appFailures: ['application-layout'] }); +}); + +test('application readiness returns as soon as the public app responds', async () => { + const observed: number[] = []; + const waits: number[] = []; + const result = await waitForApplicationProbe('http://app', { + attempts: 5, + intervalMs: 25, + probe: async () => { + observed.push(observed.length + 1); + return observed.length < 3 + ? { ok: false, detail: 'not ready' } + : { ok: true, detail: null }; + }, + sleepImpl: async ms => { waits.push(ms); }, + }); + assert.deepEqual(result, { ok: true, detail: null }); + assert.deepEqual(observed, [1, 2, 3]); + assert.deepEqual(waits, [25, 25]); + + let calls = 0; + const exhausted = await waitForApplicationProbe('http://app', { + attempts: 2, intervalMs: 0, + probe: async () => { calls += 1; return { ok: false, detail: 'not ready' }; }, + sleepImpl: async () => {}, + }); + assert.deepEqual(exhausted, { ok: false, detail: 'not ready' }); + assert.equal(calls, 2); +}); + +test('an application setup failure receives the exact current and inherited denominators', () => { + const selection = { checks: [ + { executionId: 'current', points: 3 }, + { executionId: 'inherited', points: 2 }, + { executionId: 'control', points: 0 }, + ] }; + const suites = [{ id: 'current' }, { id: 'inherited', inherited: true }, { id: 'control' }]; + assert.deepEqual(applicationFailureTotals(selection, suites, new Set()), { score: 0, max: 3, dirty: false, + contractPass: null, regression: { score: 0, max: 2 } }); + const measured = { checks: selection.checks.map((check, index) => ({ ...check, stableKey: `check-${index}` })) }; + assert.equal(applicationFailureTotals(measured, suites, new Set(['check-0', 'check-1'])).score, 3, + 'passes measured before the abort count; inherited regression guards stay unscored'); +}); + +test('a new grade removes every prior grade output but keeps operator records', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-grade-cleanup-')); + try { + mkdirSync(join(root, 'failure-media'), { recursive: true }); + mkdirSync(join(root, 'database-provenance'), { recursive: true }); + writeFileSync(join(root, 'bundle.json'), 'old'); + writeFileSync(join(root, 'grading-features.json'), 'old'); + writeFileSync(join(root, 'grader-features.stdout.log'), 'old'); + writeFileSync(join(root, 'grader-features.stderr.log'), 'old'); + writeFileSync(join(root, 'application-start.log'), 'old startup failure'); + writeFileSync(join(root, 'grading-account-create@L1.json'), 'stale earlier level'); + writeFileSync(join(root, 'grader-account-create@L1.stdout.log'), 'stale earlier level'); + writeFileSync(join(root, 'operator-notes.txt'), 'keep'); + clearPreviousGradeOutputs(root); + assert.equal(existsSync(join(root, 'bundle.json')), false); + assert.equal(existsSync(join(root, 'grading-features.json')), false); + assert.equal(existsSync(join(root, 'grader-features.stdout.log')), false); + assert.equal(existsSync(join(root, 'grader-features.stderr.log')), false); + assert.equal(existsSync(join(root, 'application-start.log')), false); + assert.equal(existsSync(join(root, 'grading-account-create@L1.json')), false); + assert.equal(existsSync(join(root, 'grader-account-create@L1.stdout.log')), false); + assert.equal(existsSync(join(root, 'failure-media')), false); + assert.equal(existsSync(join(root, 'database-provenance')), false); + assert.equal(readFileSync(join(root, 'operator-notes.txt'), 'utf8'), 'keep'); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('observed-only scope is modular, disjoint, and contributes no score', () => { + const binding = resolveRecipeRelease(loadTrack('ecommerce'), 1, + 'ecommerce.sequential-l1'); + const selected = createBoundRecipeTaskRequest(binding, { + featureIds: ['ecommerce.feature.accounts'], + observedSpecifications: ['ecommerce.spec.state-durability'], + }); + assert(isModularRecipeTaskRequest(selected)); + const scored = selectObservationScope(selected, 'scored'); + const observed = selectObservationScope(selected, 'observed'); + assert(scored); + assert(observed); + assert(observed.observedPoints !== undefined); + assert.deepEqual(observed.checks, selected.selection.observedChecks); + assert.equal(observed.scoredPoints, 0); + assert(observed.observedPoints > 0); + let measurementPoints = 0; + for (const source of new Set(observed.checks.map(check => check.source))) { + assert(source); + const checks = observed.checks.filter(check => check.source === source); + const scenario = compileScenarioDefinition(JSON.parse(readFileSync(join(ECOMMERCE, source), 'utf8'))); + const measured = selectScenarioChecks(scenario, { checks }, checks.map(check => check.stableKey)); + measurementPoints += measured.features.flatMap(feature => feature.criteria) + .reduce((sum, criterion) => sum + criterion.points, 0); + } + assert.equal(measurementPoints, observed.observedPoints); + assert.equal(observed.checks.some(check => scored.checks.includes(check)), false); + assert.throws(() => selectObservationScope( + createBoundRecipeTaskRequest(binding, { featureIds: ['ecommerce.feature.accounts'] }), + 'observed'), /scope is empty/); +}); + +test('recipe weights govern scenario grading with and without explicit check filters', () => { + const binding = resolveRecipeRelease(loadTrack('ecommerce'), 1, 'ecommerce.sequential-l1'); + let total = 0; + let overridden = 0; + let zeroPoint = 0; + for (const execution of binding.plan.execution) { + const checks = binding.release.checkCatalog.filter(check => check.executionId === execution.id); + const scenario = compileScenarioDefinition(JSON.parse(readFileSync(join(ECOMMERCE, execution.source), 'utf8'))); + const original = structuredClone(scenario); + for (const keys of [[], checks.map(check => check.stableKey)]) { + const selected = selectScenarioChecks(scenario, { checks }, keys); + for (const check of checks) { + const criterion = selected.features.find(feature => feature.id === check.featureId)?.criteria + .find(candidate => candidate.id === check.criterionId); + assert(criterion); + assert.equal(criterion.points, check.points, check.stableKey); + } + assert.deepEqual(scenario, original, 'grading weights must not change the source definition'); + } + assert.equal(selectScenarioChecks(scenario, null).features, scenario.features); + for (const check of checks) { + total += check.points; + const source = scenario.features.find(feature => feature.id === check.featureId)?.criteria + .find(criterion => criterion.id === check.criterionId); + assert(source); + if (check.points !== source.points) overridden += 1; + if (check.points === 0) zeroPoint += 1; + } + } + assert.equal(total, 57); + assert(overridden > 0, 'the regression must exercise a recipe weight override'); + assert(zeroPoint > 0, 'zero-point controls must retain zero weight'); +}); + +test('recipe execution keeps inherited suites out of the current-level score', () => { + const { temp, track } = sequentialL2Track(); + try { + const binding = resolveRecipeRelease(track, 2, 'ecommerce.sequential-l2'); + const suites = suitesForRecipe(track, binding); + const inherited = suites.filter(suite => suite.inherited); + assert.equal(inherited.length, 37); + assert(inherited.every(suite => suite.fromLevel === 1)); + assert.equal(suites.filter(suite => !suite.inherited).length, 11); + } finally { rmSync(temp, { recursive: true, force: true }); } +}); + +test('L2 grading rechecks the exact selected L1 score without adding it to L2 points', () => { + const { temp, track } = sequentialL2Track(); + try { + const l1 = resolveRecipeRelease(track, 1, 'ecommerce.sequential-l1'); + const l2 = resolveRecipeRelease(track, 2, 'ecommerce.sequential-l2'); + const prior = createBoundRecipeTaskRequest(l1, { + featureIds: ['ecommerce.feature.accounts', 'ecommerce.feature.cart-checkout', + 'ecommerce.feature.catalog', 'ecommerce.feature.purchasing', + 'ecommerce.feature.reviews', 'ecommerce.feature.warehouse-admin'], + expectedSpecifications: ['ecommerce.spec.access-control', + 'ecommerce.spec.concurrency-safety', + 'ecommerce.spec.external-data-sync', 'ecommerce.spec.live-state', + 'ecommerce.spec.state-durability', + 'ecommerce.spec.transactional-integrity'], + }); + const current = createBoundRecipeTaskRequest(l2, { + featureIds: ['ecommerce.inventory-operations-features', + 'ecommerce.operations-access-features', 'ecommerce.returns-pricing-features'], + expectedSpecifications: ['ecommerce.inventory-operations-specifications', + 'ecommerce.operations-access-specifications', + 'ecommerce.returns-pricing-specifications'], + dependencyExpansion: 'exact', + }); + assert(isModularRecipeTaskRequest(prior)); + assert(isModularRecipeTaskRequest(current)); + const scope = attachRegressionScope(current.selection, l2, suitesForRecipe(track, l2), + prior.selection.scoredChecks.map(check => check.stableKey)); + assert(scope); + assert(scope.regressionChecks); + assert(scope.evaluationSha256); + assert.equal(scope.scoredPoints, current.selection.scoredPoints); + assert.equal(scope.regressionPoints, prior.selection.scoredPoints); + assert.equal(scope.regressionChecks.length, prior.selection.scoredChecks.length); + assert.equal(scope.checks.length, + current.selection.scoredChecks.length + prior.selection.scoredChecks.length); + assert.match(scope.evaluationSha256, /^[a-f0-9]{64}$/); + } finally { rmSync(temp, { recursive: true, force: true }); } +}); + +test('a readiness probe timeout is recorded as a timeout, not a refusal', async () => { + let requests = 0; + assert.deepEqual(await verifyApplicationProbe('http://app', { + fetchImpl: async () => { requests += 1; return { ok: true, status: 200 }; }, + }), { ok: true, detail: null }); + assert.equal(requests, 1); + assert.deepEqual(await verifyApplicationProbe('http://app', { + fetchImpl: async () => ({ ok: false, status: 503 }), + }), { ok: false, detail: 'application returned HTTP 503' }); + const probe = (error: Error) => verifyApplicationProbe('http://app', { fetchImpl: async () => { throw error; } }); + const timedOut = await probe(Object.assign(new Error('The operation was aborted due to timeout'), { name: 'TimeoutError' })); + assert.equal(timedOut.ok, false); + assert.equal(timedOut.timedOut, true); + const refused = await probe(new TypeError('fetch failed')); + assert.equal(refused.ok, false); + assert.equal(refused.timedOut, undefined); +}); + +test('code metrics read only regular files the app owns', t => { + const temp = mkdtempSync(join(tmpdir(), 'stack-bench-code-metrics-link-')); + try { + const app = join(temp, 'app'); + mkdirSync(app); + writeFileSync(join(app, 'index.js'), 'export const app = true;\n'); + writeFileSync(join(temp, 'outside.txt'), 'a\nb\nc\nd\ne\nf\n'); + try { symlinkSync(join(temp, 'outside.txt'), join(app, 'planted.js')); } + catch { t.skip('this host cannot create file symlinks'); return; } + const metrics = codeMetrics({ app, backend: 'mongodb' }); + assert.equal(metrics.totalFiles, 1); + assert.equal(metrics.totalLoc, 2); + } finally { + rmSync(temp, { recursive: true, force: true }); + } +}); diff --git a/tools/stack-bench/tests/runner-environment.test.ts b/tools/stack-bench/tests/runner-environment.test.ts new file mode 100644 index 00000000000..aa45cc4f14d --- /dev/null +++ b/tools/stack-bench/tests/runner-environment.test.ts @@ -0,0 +1,74 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { controllerRunner, missingRunnerObservation, runnerEnvironmentIdentity, RUNNER_OBSERVATION_FIELDS } + from '../src/runtime/runner-environment.js'; + +test('appliance controllers record Docker daemon observations', () => { + assert.deepEqual(controllerRunner({ env: {}, platform: 'win32', architecture: 'x64', hostname: 'dev-box' }), { + schemaVersion: 1, + mode: 'local-controller', + platform: 'win32', + architecture: 'x64', + hostname: 'dev-box', + }); + const runner = controllerRunner({ + env: { STACK_BENCH_APPLIANCE: '1' }, + platform: 'linux', + architecture: 'x64', + hostname: 'runner-1', + dockerInfo: { + ServerVersion: '29.1.2', + OSType: 'linux', + Architecture: 'x86_64', + KernelVersion: '6.8.0-test', + NCPU: 8, + MemTotal: 16_000_000_000, + ContainersRunning: 4, + }, + }); + assert.deepEqual(runner, { + schemaVersion: 1, + mode: 'appliance', + platform: 'linux', + architecture: 'x64', + hostname: 'runner-1', + dockerEngineVersion: '29.1.2', + dockerOs: 'linux', + dockerArchitecture: 'x86_64', + kernelVersion: '6.8.0-test', + cpuCount: 8, + memoryBytes: 16_000_000_000, + containersRunning: 4, + packageRegistry: null, + }); + assert.deepEqual(missingRunnerObservation(runner), []); + assert.deepEqual(missingRunnerObservation(null), RUNNER_OBSERVATION_FIELDS); + const { containersRunning: count, ...identity } = runner; + identity.memoryBytes = 15_259 * 1_048_576; + assert.equal(count, 4); + assert.deepEqual(runnerEnvironmentIdentity(runner), identity); + assert.deepEqual(runnerEnvironmentIdentity({ ...runner, containersRunning: 9 }), identity); + assert.deepEqual(runnerEnvironmentIdentity({ ...runner, extraConfiguration: 'preserved' }), + { ...identity, extraConfiguration: 'preserved' }); + assert.equal(runner.containersRunning, 4); + assert.equal(runner.memoryBytes, 16_000_000_000); +}); + +test('runner identity tolerates page-scale memory drift but retains capacity and configuration changes', () => { + const runner = { memoryBytes: 49_232_842_752, cpuCount: 32, kernelVersion: 'same' }; + assert.deepEqual(runnerEnvironmentIdentity(runner), + runnerEnvironmentIdentity({ ...runner, memoryBytes: 49_232_826_368 })); + for (const change of [{ memoryBytes: runner.memoryBytes - 1_048_576 }, { cpuCount: 16 }, + { kernelVersion: 'changed' }, { memoryBytes: 'invalid' }, { memoryBytes: 0 }]) { + assert.notDeepEqual(runnerEnvironmentIdentity(runner), runnerEnvironmentIdentity({ ...runner, ...change })); + } + assert.equal(runner.memoryBytes, 49_232_842_752); +}); + +test('appliance controllers reject incomplete Docker observations', () => { + assert.throws(() => controllerRunner({ + env: { STACK_BENCH_APPLIANCE: '1' }, + dockerInfo: {}, + }), /Docker daemon inspection did not return ServerVersion/); +}); diff --git a/tools/stack-bench/tests/runtime-action-executors.test.ts b/tools/stack-bench/tests/runtime-action-executors.test.ts new file mode 100644 index 00000000000..181a4ecce69 --- /dev/null +++ b/tools/stack-bench/tests/runtime-action-executors.test.ts @@ -0,0 +1,761 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import test from 'node:test'; +import { runInNewContext } from 'node:vm'; + +import { ACTION_REGISTRY } from '../src/actions/action-catalog.js'; +import { ActionApplicationFailure, ActionInconclusive, executeAction } from '../src/actions/action-contract.js'; +import { + createDatabaseWriteCapability, + createDatabaseReadCapability, + createLifecycleCapability, + databaseWriteFailureDetail, +} from '../src/actions/runtime-action-executors.js'; +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; +import { inspectBuildContainer } from '../src/stacks/hosted-lifecycle.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; + +type UnknownRecord = Record; + +interface ServiceOverrides { + readonly applicationLifecycle?: unknown; + readonly backendLifecycle?: unknown; + readonly browser?: unknown; + readonly clock?: unknown; + readonly concurrency?: unknown; + readonly databaseWrite?: unknown; +} + +const sleep = async (_milliseconds?: number, _signal?: AbortSignal): Promise => {}; +const restartSpec = { backend: 'stub', app: '.', port: 7000, probe: '' }; + +function services( + actors: ReadonlyMap = new Map(), + overrides: ServiceOverrides = {}, +): Record { + return { + actors: { get: (name: string) => actors.get(name) }, + 'application-lifecycle': overrides.applicationLifecycle + ?? createLifecycleCapability({ target: 'app-server', sleep, control: async () => {} }), + 'backend-lifecycle': overrides.backendLifecycle + ?? createLifecycleCapability({ target: 'backend-runtime', sleep, control: async () => {} }), + 'browser-interaction': overrides.browser ?? { + clients: { open: async () => {}, fresh: async () => 'a-fresh' }, + sleep, + }, + clock: overrides.clock ?? { sleep }, + concurrency: overrides.concurrency ?? { + defaultWithin: 5000, + dispatch: async () => null, + expand: (value: string | undefined) => value, + sleep, + testId: (id: string) => `[data-testid="${id}"]`, + }, + 'database-write': overrides.databaseWrite ?? { setStock: async (input: unknown) => input }, + }; +} + +async function run(input: UnknownRecord, capabilities: Record) { + const action = String(input.do); + return executeAction(ACTION_REGISTRY, action, input, { + capabilities, + }); +} + +function observation(result: { readonly observation: unknown }): UnknownRecord { + assert(result.observation !== null && typeof result.observation === 'object'); + return result.observation as UnknownRecord; +} + +test('stock observations compare authoritative quantities and cannot use a missing baseline', async () => { + const recorded = new Map(); + let quantity = 20; + const capabilities = { + clock: { sleep }, + 'browser-observation': { recorded }, + 'database-read': { getStock: async () => ({ backend: 'postgres', item: 'Keyboard', quantity }) }, + }; + assert.equal((await run({ do: 'dbExpectStock', item: 'Keyboard', relativeTo: 'before' }, capabilities)).status, + 'inconclusive'); + assert.equal((await run({ do: 'dbRecordStock', item: 'Keyboard', as: 'before' }, capabilities)).status, 'passed'); + quantity = 19; + assert.equal((await run({ do: 'dbExpectStock', item: 'Keyboard', relativeTo: 'before', plus: -1 }, capabilities)).status, + 'passed'); + quantity = 18; + const failed = await run({ do: 'dbExpectStock', item: 'Keyboard', relativeTo: 'before', plus: -1 }, capabilities); + assert.equal(failed.status, 'failed'); + assert.match(failed.summary ?? '', /stored stock for Keyboard/); + assert.equal((await run({ do: 'dbExpectStock', item: 'Keyboard', equals: 18 }, capabilities)).status, 'passed'); + for (const [lower, upper, status] of [[18, 19, 'passed'], [17, 18, 'passed'], + [19, 20, 'failed'], [16, 17, 'failed']] as const) { + assert.equal((await run({ do: 'dbExpectStock', item: 'Keyboard', atLeast: lower, atMost: upper }, capabilities)).status, status); + } + const disabled = { ...capabilities, 'database-read': createDatabaseReadCapability({ + backend: 'postgres', skip: true, expand: value => value, + }) }; + assert.equal((await run({ do: 'dbExpectStock', item: 'Keyboard', equals: 18 }, disabled)).status, 'inconclusive'); +}); + +test('contention checks reject negative warehouse stock even when the total is correct', async () => { + for (const [file, criterionId, good, bad] of [ + ['01-last-unit.json', '201a', [0, 0], [1, -1]], + ['01-restock-race.json', '202a', [51, 51], [103, -1]], + ] as const) { + const source = join(STACK_BENCH_ROOT, 'tracks/ecommerce/scenarios', file); + const criterion = compileScenarioDefinition(JSON.parse(readFileSync(source, 'utf8')), { source }) + .features.flatMap(feature => feature.criteria).find(check => check.id === criterionId)!; + const steps = criterion.steps.filter(step => step.do === 'dbExpectStock' && step.warehouse); + assert.equal(steps.length, 2); + assert.equal(good[0] + good[1], bad[0] + bad[1]); + for (const quantities of [good, bad]) { + const results = await Promise.all(steps.map(step => run(step, { + clock: { sleep }, 'browser-observation': { recorded: new Map() }, + 'database-read': { getStock: async ({ warehouse }: { warehouse: string }) => ({ + quantity: quantities[warehouse === 'East' ? 0 : 1], + }) }, + }))); + assert.equal(results.every(result => result.status === 'passed'), quantities === good); + if (quantities === bad) assert(results.some(result => result.status === 'failed')); + } + } +}); + +test('timed observations use the original origin and a missed window is unmeasured', async () => { + const recorded = new Map(); + const waits: number[] = []; + const capabilities = { + actors: { get: () => ({}) }, + 'browser-observation': { recorded }, + clock: { sleep: async (ms: number) => { waits.push(ms); } }, + }; + assert.equal((await run({ do: 'expectElapsed', since: 'start', atMost: 1000 }, capabilities)).status, 'inconclusive'); + assert.equal((await run({ do: 'recordTime', as: 'start' }, capabilities)).status, 'passed'); + assert.equal((await run({ do: 'expectElapsed', since: 'start', atMost: 1000 }, capabilities)).status, 'passed'); + recorded.set('start', performance.now() - 2000); + const missed = await run({ do: 'expectElapsed', since: 'start', atMost: 1000 }, capabilities); + assert.equal(missed.status, 'inconclusive'); + assert.match(missed.summary ?? '', /timing observation window/); + assert.equal((await run({ do: 'wait', actor: 'a', ms: 5000, since: 'start' }, capabilities)).status, 'passed'); + assert(waits[0] !== undefined && waits[0] > 2000 && waits[0] <= 3000); + assert.equal((await run({ do: 'wait', actor: 'a', ms: 1000, since: 'start' }, capabilities)).status, 'passed'); + assert.equal(waits[1], 0); + const missing = await run({ do: 'wait', actor: 'missing', ms: 1 }, { ...capabilities, actors: { get: () => undefined } }); + assert.equal(missing.status, 'harness_failure'); + assert.equal(missing.code, 'unclassified_exception'); + assert.equal(missing.summary, 'harness did not create actor "missing"'); + assert.equal(waits.length, 2); +}); + +test('transfer conservation rejects moving the wrong product despite correct warehouse totals', async () => { + const source = join(STACK_BENCH_ROOT, 'tracks/ecommerce/scenarios/02-strengthened.json'); + const criterion = compileScenarioDefinition(JSON.parse(readFileSync(source, 'utf8')), { source }) + .features.find(feature => feature.id === 202)!.criteria[0]; + assert(criterion); + for (const wrongProduct of [false, true]) { + const quantities = new Map([['East', 50], ['West', 50]]); + const capabilities = { clock: { sleep }, 'browser-observation': { recorded: new Map() }, + 'database-read': { getStock: async (input: { item: string; warehouse: string }) => ({ + item: input.item, quantity: quantities.get(input.warehouse), + }) } }; + for (const step of criterion.steps.filter(step => step.do === 'dbRecordStock')) { + assert.equal((await run(step, capabilities)).status, 'passed'); + } + if (!wrongProduct) { quantities.set('East', 33); quantities.set('West', 67); } + const checks: readonly UnknownRecord[] = criterion.steps.filter(step => step.do === 'dbExpectStock'); + assert.equal(checks.length, 2); + for (const step of checks) { + assert.equal((await run({ ...step, within: 1 }, capabilities)).status, wrongProduct ? 'failed' : 'passed'); + } + } +}); + +test('bounded stock reads observe deferred writes and reject missing, repeated, or inaccessible writes', async t => { + let now = 0; + t.mock.method(Date, 'now', () => now); + for (const [values, status] of [[[100, 105], 'passed'], [[100], 'failed'], [[110], 'failed']] as const) { + let reads = 0; + const result = await run({ do: 'dbExpectStock', item: 'Keyboard', equals: 105, within: 1000 }, { + 'browser-observation': { recorded: new Map() }, + 'database-read': { getStock: async () => ({ quantity: values[Math.min(reads++, values.length - 1)] }) }, + clock: { sleep: async (ms: number) => { now += ms; } }, + }); + assert.equal(result.status, status); + } + const result = await run({ do: 'dbExpectStock', item: 'Keyboard', equals: 105, within: 20 }, { + 'browser-observation': { recorded: new Map() }, + 'database-read': { getStock: async () => { throw new Error('database unavailable'); } }, + clock: { sleep }, + }); + assert.equal(result.status, 'harness_failure'); + for (const mode of ['late-read', 'late-retry', 'late-wake']) await t.test(mode, async () => { + let reads = 0; + const result = await run({ do: 'dbExpectStock', item: 'Keyboard', equals: 105, within: 1000 }, { + 'browser-observation': { recorded: new Map() }, + 'database-read': { getStock: async () => { + reads++; + if (mode === 'late-read' || (mode === 'late-retry' && reads > 1)) now += 1001; + return { quantity: mode === 'late-read' || reads > 1 ? 105 : 100 }; + } }, + clock: { sleep: async () => { now += mode === 'late-retry' ? 250 : 1001; } }, + }); + assert.equal(result.status, mode === 'late-wake' ? 'failed' : 'passed', mode); + assert.equal(reads, mode === 'late-retry' ? 2 : 1); + }); +}); + +test('compiled restock race waits for the stored total and rejects missing effects', async t => { + let now = 0; + t.mock.method(Date, 'now', () => now); + const source = join(STACK_BENCH_ROOT, 'tracks/ecommerce/scenarios/01-restock-race.json'); + const scenario = compileScenarioDefinition(JSON.parse(readFileSync(source, 'utf8')), { source }); + const step = scenario.features.flatMap(feature => feature.criteria).find(check => check.id === '202a')!.steps + .find(step => step.do === 'dbExpectStock' && !step.warehouse)!; + for (const mode of ['delayed', 'missing', 'slow-missing', 'slow-repeated', 'late-reader-error']) await t.test(mode, async () => { + now = 0; + const readStarts: number[] = []; + const result = await run(step, { + 'browser-observation': { recorded: new Map([['stored-before-rush', 100]]) }, + 'database-read': { getStock: async () => { + readStarts.push(now); + if (mode.startsWith('slow-') || mode === 'late-reader-error') now += 450; + if (mode === 'late-reader-error' && now > 10000) throw new Error('database unavailable'); + return { quantity: mode === 'slow-repeated' ? 104 : now >= 6500 && mode === 'delayed' ? 102 : 100 }; + } }, + clock: { sleep: async (ms: number) => { now += ms; } }, + }); + assert.equal(result.status, mode === 'delayed' ? 'passed' : mode === 'late-reader-error' ? 'harness_failure' : 'failed', mode); + assert(readStarts.every(start => start < 10000), 'no stock read starts after the retry deadline'); + if (mode.startsWith('slow-')) { + assert(now > 10000, 'the final native read completes after the retry deadline'); + assert.equal(result.finding?.kind, 'number-mismatch'); + } + }); +}); + +test('cancellation probes reject a refund to the wrong warehouse even when total stock is restored', async () => { + const source = join(STACK_BENCH_ROOT, 'tracks/ecommerce/scenarios/02-self-contained.json'); + const feature = compileScenarioDefinition(JSON.parse(readFileSync(source, 'utf8')), { source }) + .features.find(feature => feature.id === 202)!; + for (const criterion of feature.criteria) { + const stock = new Map([['East', 50], ['West', 50]]); + const capabilities = { clock: { sleep }, 'browser-observation': { recorded: new Map() }, + 'database-read': { getStock: async (input: { item: string; warehouse?: string }) => ({ + backend: 'postgres', item: input.item, + quantity: input.warehouse ? stock.get(input.warehouse)! : [...stock.values()].reduce((a, b) => a + b, 0), + }) } }; + const records = criterion.steps.filter(step => step.do === 'dbRecordStock'); + const checks = criterion.steps.filter(step => step.do === 'dbExpectStock' && step.plus === 0); + for (const step of records) assert.equal((await run(step, capabilities)).status, 'passed'); + for (const step of checks) assert.equal((await run(step, capabilities)).status, 'passed'); + stock.set('East', 49); stock.set('West', 51); + const results = await Promise.all(checks.map(step => run(step, capabilities))); + assert.equal(results.filter(result => result.status === 'failed').length, 2); + } +}); + +test('optional diagnostics keep fixed fresh-account schedules and cannot pass by rejecting all work', () => { + const load = (file: string) => { + const path = join(STACK_BENCH_ROOT, 'tracks/ecommerce/scenarios', file); + return compileScenarioDefinition(JSON.parse(readFileSync(path, 'utf8')), { source: path }); + }; + for (const [file, cohorts] of [['diagnostic-checkout-contention.json', 12], ['diagnostic-purchase-contention.json', 24]] as const) { + const scenario = load(file); + const accounts = scenario.features.flatMap(feature => feature.setup.filter(step => step.do === 'signUp').map(step => step.name)); + assert.equal(new Set(accounts).size, cohorts); + const widths: unknown[] = []; + for (const feature of scenario.features) { + const account = feature.setup.find(step => step.do === 'signUp')!.name; + const initial = feature.setup.findIndex(step => step.do === 'dbRecordCheckout'); + if (file === 'diagnostic-checkout-contention.json') { + assert(feature.setup.some(step => step.do === 'dbRecordStock')); + assert(initial < feature.setup.findIndex(step => step.do === 'click')); + assert.equal(feature.setup[initial]!.account, `{user:${account}}`); + } else { + assert(feature.setup.some(step => step.do === 'dbSetStock' && step.quantity === 128)); + assert.equal(feature.setup.filter(step => step.do === 'dbRecordCheckout').length, 2); + } + for (const criterion of feature.criteria) { + assert.equal(criterion.points, 0, 'diagnostics must not add scored ladder credit'); + const call = criterion.steps.find(step => step.do === 'callConcurrently')!; + widths.push(call.requests); + if (file === 'diagnostic-checkout-contention.json') { + assert.equal(criterion.steps[0]!.do, 'dbRecordCheckout'); + assert.equal(criterion.steps[0]!.account, `{user:${account}}`); + assert(criterion.steps.some(step => step.do === 'dbExpectCheckout' && step.quantity === 1)); + assert(criterion.steps.some(step => step.do === 'expectCallOutcomes')); + assert(criterion.steps.some(step => step.do === 'dbExpectStock' && step.plus === -1)); + assert(criterion.steps.some(step => step.do === 'expect' && step.testid === 'order-item' && step.count === 1)); + } else { + // Purchases require every affordable request and its stored effect. + assert.equal(call.action, 'buy'); + assert(criterion.steps.some(step => step.do === 'expectCallOutcomes' && step.accepted === call.requests)); + assert(criterion.steps.some(step => step.do === 'dbExpectPurchases' && step.purchases === call.requests)); + } + } + } + assert.deepEqual(widths, [1, 1, 1, 4, 4, 4, 16, 16, 16, 64, 64, 64]); + } + // Scarce and mixed diagnostics keep their fixed schedules and mandatory stored progress. + for (const [file, cohorts, calls] of [['diagnostic-scarce-stock.json', 12, 255], ['diagnostic-restock-contention.json', 80, 1000]] as const) { + const scenario = load(file); + assert.equal(scenario.features.length, cohorts); + let count = 0; + for (const feature of scenario.features) { + const criterion = feature.criteria[0]!; + assert.equal(criterion.points, 0); + const call = criterion.steps[0]!; + const restock = (call.alongside as Array<{ requests: number }> | undefined)?.[0]; + count += Number(call.requests) + (restock?.requests ?? 0); + assert.equal(criterion.steps[1]!.do, 'dbExpectPurchases'); + assert.equal(criterion.steps[1]!.purchases, restock ? call.requests : Math.min(3, Number(call.requests))); + if (restock) assert(feature.setup.some(step => step.do === 'signIn' && step.actor === 'admin')); + } + assert.equal(count, calls); + } +}); + +test('race preserves branch ordering while overlapping branches through registered dispatch', async () => { + const events: string[] = []; + const capability = services(new Map(), { concurrency: { + defaultWithin: 5000, + expand: (value: string | undefined) => value, + sleep, + testId: (id: string) => id, + dispatch: async (step: UnknownRecord) => { + events.push(`start-${String(step.actor)}-${String(step.ms)}`); + if (step.ms === 1) await new Promise(resolve => setImmediate(resolve)); + events.push(`end-${String(step.actor)}-${String(step.ms)}`); + }, + } }); + const result = await run({ do: 'race', settleMs: 0, branches: [ + [{ do: 'wait', actor: 'a', ms: 1 }, { do: 'wait', actor: 'a', ms: 2 }], + [{ do: 'wait', actor: 'b', ms: 3 }], + ] }, capability); + assert.equal(result.status, 'passed'); + assert(events.indexOf('start-b-3') < events.indexOf('start-a-2')); + assert(events.indexOf('end-a-1') < events.indexOf('start-a-2')); + + const nestedFailure = new Error('nested action failed'); + Object.defineProperty(nestedFailure, 'actionEvidence', { value: { + status: 'failed', summary: 'nested application mismatch', + } }); + const failed = await run({ do: 'race', settleMs: 0, branches: [ + [{ do: 'wait', actor: 'a', ms: 1 }], [{ do: 'wait', actor: 'b', ms: 1 }], + ] }, services(new Map(), { concurrency: { + defaultWithin: 5000, expand: (value: string | undefined) => value, sleep, + testId: (id: string) => id, + dispatch: async () => { throw nestedFailure; }, + } })); + assert.equal(failed.status, 'failed'); + assert.equal(failed.summary, 'nested application mismatch'); +}); + +test('concurrent branches drain and preserve uncertainty over an earlier app failure', async () => { + for (const later of [null, new ActionInconclusive('unmeasured'), new Error('browser protocol failed')]) { + let drained = false; + const result = await run({ do: 'race', settleMs: 0, branches: [ + [{ do: 'wait', actor: 'a', ms: 1 }], [{ do: 'wait', actor: 'b', ms: 1 }], + ] }, services(new Map(), { concurrency: { + sleep, + dispatch: async (step: UnknownRecord) => { + if (step.actor === 'a') throw new ActionApplicationFailure('app failed'); + await new Promise(resolve => setImmediate(resolve)); + drained = true; + if (later) throw later; + }, + } })); + assert.equal(drained, true, 'no branch may keep changing the app after the action returns'); + assert.equal(result.status, later instanceof ActionInconclusive ? 'inconclusive' + : later ? 'harness_failure' : 'failed'); + } +}); + +test('concurrent replay refuses to invent contention when fewer than two writes exist', async () => { + const actor = { writes: [], lastWrites: {}, lastWrite: null }; + const result = await run({ do: 'replayConcurrently', actors: ['a', 'b'], + settleMs: 0 }, services(new Map([['a', actor], ['b', actor]]))); + assert.equal(result.status, 'inconclusive'); + assert.match(result.summary ?? '', /never contended/); +}); + +test('lifecycle operations distinguish missing control, unsafe refusal, and success', async () => { + const missing = await run({ do: 'restartBackend', settleMs: 0 }, services()); + assert.equal(missing.status, 'inconclusive'); + assert.match(missing.summary ?? '', /no control over the database runtime/); + + const refusedError = Object.assign(new Error('refused'), { status: 3 }); + const refusedCapability = createLifecycleCapability({ restartSpec, + target: 'backend-runtime', sleep, + control: async () => { throw refusedError; } }); + const refused = await run({ do: 'restartBackend', settleMs: 0 }, + services(new Map(), { backendLifecycle: refusedCapability })); + assert.equal(refused.status, 'inconclusive'); + assert.match(refused.summary ?? '', /was refused on this host/); + + const calls: Array = []; + const operated: Array<'restart' | 'start' | 'stop'> = []; + const successful = createLifecycleCapability({ restartSpec, + target: 'backend-runtime', sleep, + control: async (spec, mode) => { calls.push([spec, mode]); }, + onOperated: mode => operated.push(mode) }); + const passed = await run({ do: 'restartBackend', settleMs: 0 }, + services(new Map(), { backendLifecycle: successful })); + assert.equal(passed.status, 'passed'); + assert.deepEqual(calls, [[restartSpec, 'restart']]); + assert.deepEqual(operated, ['restart']); +}); + +test('a lifecycle operation reports the state it left only when the control completed', async () => { + const operated: Array<'restart' | 'start' | 'stop'> = []; + const failing = createLifecycleCapability({ restartSpec, target: 'app-server', sleep, + control: async () => { throw Object.assign(new Error('exit 1'), { status: 1, stdout: 'port busy' }); }, + onOperated: mode => operated.push(mode) }); + const failed = await run({ do: 'stopAppServer', settleMs: 0 }, + services(new Map(), { applicationLifecycle: failing })); + assert.equal(failed.status, 'failed'); + assert.equal(operated.length, 0); + + const working = createLifecycleCapability({ restartSpec, target: 'app-server', sleep, + control: async () => {}, onOperated: mode => operated.push(mode) }); + const capabilities = services(new Map(), { applicationLifecycle: working }); + assert.equal((await run({ do: 'stopAppServer', settleMs: 0 }, capabilities)).status, 'passed'); + assert.equal((await run({ do: 'startAppServer', settleMs: 0 }, capabilities)).status, 'passed'); + assert.deepEqual(operated, ['stop', 'start']); +}); + +test('lifecycle control failures are app failures only when the app is at fault', async () => { + const outcome = async (target: 'app-server' | 'backend-runtime', error: Error) => { + const capability = createLifecycleCapability({ restartSpec, target, sleep, + control: async () => { throw error; } }); + return run({ do: target === 'app-server' ? 'startAppServer' : 'restartBackend', settleMs: 0 }, + services(new Map(), target === 'app-server' + ? { applicationLifecycle: capability } : { backendLifecycle: capability })); + }; + const appFault = () => Object.assign(new Error('application did not start'), + { code: 'generated_app_not_restartable' }); + const dockerTimeout = () => Object.assign(new Error('docker exec timed out'), { code: 'ETIMEDOUT' }); + + const appStart = await outcome('app-server', appFault()); + assert.equal(appStart.status, 'failed'); + assert.match(appStart.summary ?? '', /application server could not start/); + assert.equal((await outcome('backend-runtime', appFault())).status, 'failed'); + assert.equal((await outcome('app-server', dockerTimeout())).status, 'harness_failure'); + assert.equal((await outcome('backend-runtime', dockerTimeout())).status, 'harness_failure'); + assert.equal((await outcome('backend-runtime', + new Error('SPACETIME_BIN is unavailable: '))).status, 'harness_failure'); + // A replaced container is lost ownership, not an application that cannot start. + const lease = { resources: { buildContainer: { owned: true, name: 'owned-app', id: 'a'.repeat(64) } } }; + const lostOwnership = (() => { + try { inspectBuildContainer(lease as never, (() => 'b'.repeat(64)) as never); } + catch (error) { return error as Error; } + throw new Error('a replaced container was accepted'); + })(); + assert.equal((await outcome('app-server', lostOwnership)).status, 'harness_failure'); +}); + +test('direct PostgreSQL stock writes quote names and require exactly one updated row', async () => { + const waits: number[] = []; + let stockSql = ''; + let stockCommands = 0; + const databaseLease = { resources: { database: 'bench', + container: { name: 'leased-postgres', id: 'postgres-id' } } }; + const capability = createDatabaseWriteCapability({ + backend: 'postgres', + databaseLease, + expand: value => value, + exec: (_command, args, options) => { + if (args[0] === 'inspect') return 'postgres-id\n'; + if (++stockCommands === 1) { + stockSql = options.input ?? ''; + return 'UPDATE 1\n'; + } + return '{"items":1,"namedWarehouses":1,"warehouses":1,"quantities":[7]}\n'; + }, + }); + const passed = await run({ do: 'dbSetStock', item: "Kid's Keyboard", warehouse: 'Main', + quantity: 7, settleMs: 17 }, { + ...services(new Map(), { databaseWrite: capability, clock: { + sleep: async (ms: number) => { waits.push(ms); }, + } }), + }); + assert.equal(passed.status, 'passed'); + assert.equal(stockCommands, 2, 'the update requires an independent stored quantity read'); + assert.match(stockSql, /Kid''s Keyboard/); + assert.deepEqual(waits, [17]); + + const missed = (counts: Record) => createDatabaseWriteCapability({ backend: 'postgres', databaseLease, + expand: value => value, exec: (_command, args) => { + if (args[0] === 'inspect') return 'postgres-id\n'; + return `UPDATE 0\n${JSON.stringify(counts)}\n`; + } }); + const failed = await run({ do: 'dbSetStock', item: 'Missing', warehouse: 'Main', + quantity: 7, settleMs: 0 }, services(new Map(), { databaseWrite: missed({ items: 1, warehouses: 1, stocks: 0 }) })); + assert.equal(failed.status, 'failed'); + assert.equal(failed.finding?.kind, 'stock-interface-missing'); + assert.match(String(failed.finding?.fields.detail), /could not locate one relational stock row/); + assert.equal(failed.finding?.fields.missingRow, 'stock'); + for (const [counts, row] of [[{ items: 0, warehouses: 1, stocks: 0 }, 'item'], + [{ items: 1, warehouses: 0, stocks: 0 }, 'warehouse']] as const) { + const result = await run({ do: 'dbSetStock', item: 'Missing', warehouse: 'Main', quantity: 7, settleMs: 0 }, + services(new Map(), { databaseWrite: missed(counts) })); + assert.equal(result.finding?.kind, 'stock-interface-missing'); + assert.equal(result.finding?.fields.missingRow, row); + } + // Duplicate names are an invalid interface, as they are for the stock read. + const ambiguous = await run({ do: 'dbSetStock', item: 'Keyboard', warehouse: 'Main', quantity: 7, settleMs: 0 }, + services(new Map(), { databaseWrite: missed({ items: 2, warehouses: 1, stocks: 1 }) })); + assert.equal(ambiguous.status, 'failed'); + assert.equal(ambiguous.finding?.kind, 'interface-invalid'); + assert.equal(ambiguous.finding?.fields.action, 'set stock'); + assert.match(String(ambiguous.finding?.fields.detail), /ambiguous/); +}); + +test('a database without the stock interface is an application failure that keeps its diagnostic', async () => { + const error = Object.assign(new Error('direct stock correction requires singular collections ' + + '`item`, `warehouse`, and `stock`'), { stdout: 'MISSING\n' }); + assert.match(databaseWriteFailureDetail(error), /singular collections/); + assert.match(databaseWriteFailureDetail(error), /MISSING/); + + const capability = createDatabaseWriteCapability({ backend: 'mongodb', + databaseLease: { resources: { database: 'bench', + container: { name: 'leased-mongodb', id: 'mongodb-id' } } }, + expand: value => value, exec: (_command, args) => { + if (args[0] === 'inspect') return 'mongodb-id\n'; + throw error; + } }); + const result = await run({ do: 'dbSetStock', item: 'Desk Lamp', warehouse: 'East', + quantity: 5, settleMs: 0 }, services(new Map(), { databaseWrite: capability })); + // The contract names the stock tables; an app without them has failed the + // interface, and the writer's diagnostic travels as detail. + assert.equal(result.status, 'failed'); + assert.equal(result.finding?.kind, 'stock-interface-missing'); + assert.match(String(result.finding?.fields.detail), /singular collections/); + assert.match(String(result.finding?.fields.detail), /MISSING/); + for (const row of ['ITEM', 'WAREHOUSE'] as const) { + const missing = createDatabaseWriteCapability({ backend: 'mongodb', + databaseLease: { resources: { database: 'bench', container: { name: 'leased-mongodb', id: 'mongodb-id' } } }, + expand: value => value, exec: (_command, args) => { + if (args[0] === 'inspect') return 'mongodb-id\n'; + throw Object.assign(new Error('missing row'), { stdout: `MISSING_${row}\n` }); + } }); + const result = await run({ do: 'dbSetStock', item: 'Desk Lamp', warehouse: 'East', quantity: 5, settleMs: 0 }, + services(new Map(), { databaseWrite: missing })); + assert.equal(result.finding?.kind, 'stock-interface-missing'); + assert.equal(result.finding?.fields.missingRow, row.toLowerCase()); + assert.equal(result.finding?.fields.item, 'Desk Lamp'); + assert.equal(result.finding?.fields.warehouse, 'East'); + } + +}); + +test('direct MongoDB writes preserve IDs and reject ambiguous stock rows', async () => { + class ObjectId { + readonly _bsontype = 'ObjectId'; + constructor(readonly value: string) {} + toHexString(): string { return this.value; } + } + const hex = '0123456789abcdef01234567'; + const bson = new ObjectId(hex); + const equal = (a: unknown, b: unknown): boolean => a instanceof ObjectId && b instanceof ObjectId + ? a.value === b.value : a === b; + for (const [parent, reference] of [[0, 0], [bson, hex], [hex, bson], [bson, bson]] as const) { + for (const duplicates of ['', 'stock', 'parent']) { + const calls: Array = []; + const stock = { _id: 'stock-id', item_id: reference, warehouse_id: 7, quantity: 1 }; + const cursor = (rows: unknown[]) => ({ limit: () => ({ toArray: () => rows }) }); + const capability = createDatabaseWriteCapability({ + backend: 'mongodb', + databaseLease: { resources: { database: 'bench', + container: { name: 'leased-mongodb', id: 'mongodb-id' } } }, + expand: value => value, + exec: (command, args) => { + calls.push([command, args]); + if (args[0] === 'inspect') return 'mongodb-id\n'; + let output = ''; + runInNewContext(args.at(-1)!, { + ObjectId, + db: { + getCollectionNames: () => ['item', 'warehouse', 'stock'], + item: { find: () => cursor(Array(duplicates === 'parent' ? 2 : 1) + .fill({ id: parent, _id: 'generated-item-object-id' })) }, + warehouse: { find: () => cursor([{ id: stock.warehouse_id, _id: 'generated-warehouse-object-id' }]) }, + stock: { + find: (query: { item_id: { $in: unknown[] }; warehouse_id: { $in: unknown[] } }) => { + const matched = query.item_id.$in.some(id => equal(id, stock.item_id)) + && query.warehouse_id.$in.includes(stock.warehouse_id); + return cursor(matched ? (duplicates === 'stock' ? [stock, stock] : [stock]) : []); + }, + updateOne: (query: { _id: string }, update: { $set: { quantity: number } }) => { + assert.equal(query._id, stock._id); + stock.quantity = update.$set.quantity; + return { matchedCount: 1 }; + }, + }, + }, + quit: () => { throw Object.assign(new Error('refused'), { stdout: output }); }, + print: (value: string) => { output += value + '\n'; }, + }); + return output; + }, + }); + const result = await run({ do: 'dbSetStock', item: 'Desk Lamp', warehouse: 'East', + quantity: 5, settleMs: 0 }, services(new Map(), { databaseWrite: capability })); + // Ambiguity is the application's invalid interface, graded as on every other stack. + assert.equal(result.status, duplicates ? 'failed' : 'passed'); + assert.equal(result.finding?.kind, duplicates ? 'interface-invalid' : undefined); + assert.equal(stock.quantity, duplicates ? 1 : 5); + assert.deepEqual(calls.find(([, args]) => args.includes('mongosh'))?.[1].slice(0, 2), + ['exec', 'mongodb-id']); + } + } +}); + +test('MongoDB auth and command failures cannot be scored as missing stock data', async () => { + const databaseLease = { resources: { database: 'bench', + container: { name: 'leased-mongodb', id: 'mongodb-id' } } }; + for (const error of [ + Object.assign(new Error('mongosh --password SENTINEL_PRIVATE_PASSWORD exited 1'), + { stderr: 'MongoServerError: Command find requires authentication' }), + Object.assign(new Error('mongosh exited 1'), + { stderr: 'MongoServerError: Authentication failed' }), + Object.assign(new Error('docker exec exited 126'), + { stderr: 'OCI runtime exec failed: executable file not found in $PATH' }), + Object.assign(new Error('docker timed out'), { code: 'ETIMEDOUT' }), + ]) { + const capability = createDatabaseWriteCapability({ backend: 'mongodb', databaseLease, + expand: value => value, exec: (_command, args) => { + if (args[0] === 'inspect') return 'mongodb-id\n'; + throw error; + } }); + const result = await run({ do: 'dbSetStock', item: 'Desk Lamp', warehouse: 'East', + quantity: 5, settleMs: 0 }, services(new Map(), { databaseWrite: capability })); + assert.equal(result.status, 'harness_failure', error.message); + assert.equal(result.finding, null); + assert.match(result.summary ?? '', /authentication|Authentication|executable file|timed out/); + assert.doesNotMatch(JSON.stringify(result), /SENTINEL_PRIVATE_PASSWORD/); + } + for (const [output, status] of [['NOMATCH\n', 'failed'], ['unexpected output\n', 'harness_failure']]) { + const capability = createDatabaseWriteCapability({ backend: 'mongodb', databaseLease, + expand: value => value, exec: (_command, args) => args[0] === 'inspect' ? 'mongodb-id\n' : output! }); + const result = await run({ do: 'dbSetStock', item: 'Desk Lamp', warehouse: 'East', + quantity: 5, settleMs: 0 }, services(new Map(), { databaseWrite: capability })); + assert.equal(result.status, status); + } +}); + +test('direct database writes fail as harness errors without lease authority', async () => { + const capability = createDatabaseWriteCapability({ + backend: 'postgres', expand: value => value, exec: () => 'UPDATE 1\n', + }); + const result = await run({ do: 'dbSetStock', item: 'Desk Lamp', warehouse: 'East', + quantity: 5, settleMs: 0 }, services(new Map(), { databaseWrite: capability })); + assert.equal(result.status, 'harness_failure'); + assert.match(result.summary ?? '', /authenticated backend lease/); +}); + +test('the null control can skip direct database writes', async () => { + const capability = createDatabaseWriteCapability({ + backend: 'postgres', skip: true, expand: value => value, + exec: () => { throw new Error('must not execute'); }, + }); + const result = await run({ do: 'dbSetStock', item: 'Desk Lamp', warehouse: 'East', + quantity: 5, settleMs: 0 }, services(new Map(), { databaseWrite: capability })); + assert.equal(result.status, 'passed'); +}); + +const interruption = { interrupt: async () => ({ closed: 1, open: [] }), restore: async () => {} }; + +test('offline lifecycle preserves settling time and verifies browser network state', async () => { + const offlineStates: boolean[] = []; + const waits: number[] = []; + let browserOnline = true; + const actor = { networkInterruption: interruption, page: { url: () => 'http://app/', + evaluate: async () => browserOnline, + context: () => ({ + setOffline: async (value: boolean) => { + offlineStates.push(value); + browserOnline = !value; + }, + }) } }; + const capabilities = services(new Map([['a', actor]]), { browser: { + clients: { open: async () => {}, fresh: async () => 'a-fresh' }, + sleep: async (ms: number) => { waits.push(ms); }, + } }); + const disconnected = await run({ do: 'setOffline', actor: 'a', offline: true, settleMs: 10 }, + capabilities); + const reconnected = await run({ do: 'setOffline', actor: 'a', offline: false, settleMs: 20 }, + capabilities); + assert.equal(disconnected.status, 'passed'); + assert.equal(observation(disconnected).browserOnline, false); + assert.equal(reconnected.status, 'passed'); + assert.equal(observation(reconnected).browserOnline, true); + assert.deepEqual(offlineStates, [true, false]); + assert.deepEqual(waits, [10, 20]); +}); + +test('offline lifecycle fails closed when browser network state does not change', async () => { + const actor = { networkInterruption: interruption, page: { url: () => 'http://app/', evaluate: async () => true, + context: () => ({ setOffline: async () => {} }) } }; + const result = await run({ do: 'setOffline', actor: 'a', offline: true, settleMs: 1 }, + services(new Map([['a', actor]]))); + assert.equal(result.status, 'harness_failure'); + assert.match(result.summary ?? '', /navigator\.onLine remained true/); +}); + +test('an application that reloads itself onto the browser error page while offline fails', async () => { + let url = 'http://app/'; + const actor = { networkInterruption: interruption, page: { url: () => url, evaluate: async () => true, + context: () => ({ setOffline: async () => { url = 'chrome-error://chromewebdata/'; } }) } }; + const result = await run({ do: 'setOffline', actor: 'a', offline: true, settleMs: 1 }, + services(new Map([['a', actor]]))); + assert.equal(result.status, 'failed'); + assert.equal(result.finding?.kind, 'page-error'); +}); + +test('a crashed page during a concurrency barrier or click remains a harness failure', async () => { + for (const phase of ['barrier', 'click']) { + const closed = () => { throw new Error('Target page, context or browser has been closed'); }; + const actor = { loc: () => ({ + waitFor: async () => { if (phase === 'barrier') closed(); }, + isEnabled: async () => true, + click: async () => closed(), + }) }; + const result = await run({ do: 'clickConcurrently', actors: ['a', 'b'], + testid: 'buy', settleMs: 0 }, services(new Map([['a', actor], ['b', actor]]))); + assert.equal(result.status, 'harness_failure', phase); + assert.equal(result.code, 'unclassified_exception', phase); + } +}); + +test('concurrent click timeouts on a starved host are retryable and inconclusive', async () => { + const gib = 1024 ** 3; + const idle = { memTotalBytes: 32 * gib, memAvailableBytes: 20 * gib, cpuCount: 8, load: 2 }; + const timeout = () => Object.assign(new Error('locator timed out'), { name: 'TimeoutError' }); + for (const phase of ['barrier', 'click']) { + const actor = { loc: () => ({ + waitFor: async () => { if (phase === 'barrier') throw timeout(); }, + isEnabled: async () => true, + click: async () => { throw timeout(); }, + count: async () => 1, + }) }; + for (const [pressure, status] of [ + [idle, 'failed'], + [{ ...idle, load: 8 }, 'inconclusive'], + [{ ...idle, memAvailableBytes: 3 * gib }, 'inconclusive'], + ] as const) { + const result = await run({ do: 'clickConcurrently', actors: ['a', 'b'], testid: 'buy', settleMs: 0 }, + services(new Map([['a', actor], ['b', actor]]), { concurrency: { + defaultWithin: 5000, dispatch: async () => null, expand: (value: string | undefined) => value, + hostPressure: () => pressure, sleep, testId: (id: string) => id, + } })); + assert.equal(result.status, status, phase); + assert.equal(result.retryable, status === 'inconclusive', phase); + assert.equal(result.finding?.kind ?? null, status === 'failed' + ? (phase === 'barrier' ? 'control-not-ready' : 'clicks-failed') : null, phase); + assert.deepEqual(observation(result).hostPressure, pressure, phase); + assert.deepEqual(observation(result).attached, + [{ actor: 'a', attached: true }, { actor: 'b', attached: true }], phase); + } + } +}); diff --git a/tools/stack-bench/tests/saved-diagnostic.test.ts b/tools/stack-bench/tests/saved-diagnostic.test.ts new file mode 100644 index 00000000000..4afa5b0cc3e --- /dev/null +++ b/tools/stack-bench/tests/saved-diagnostic.test.ts @@ -0,0 +1,111 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { appendFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test, { type TestContext } from 'node:test'; +import { writeArtifact } from '../src/evidence/artifacts.js'; +import { hashAppSource } from '../src/runtime/source-snapshot.js'; +import { inspectSavedDiagnostic } from '../src/runtime/saved-diagnostic.js'; + +const digest = (path: string) => createHash('sha256').update(readFileSync(path)).digest('hex'); +function fixture(t: TestContext) { + const root = mkdtempSync(join(tmpdir(), 'saved-diagnostic-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const app = join(root, 'source'), runPath = join(root, 'run.json'), checkpointPath = join(root, 'checkpoint.json'); + mkdirSync(app); writeFileSync(join(app, 'server.js'), 'accepted source'); + const sourceSha256 = hashAppSource(app).sha256, selectionSha256 = 'a'.repeat(64); + const checkpoint = { backend: 'postgres', track: 'ecommerce', level: 3, source: { sha256: sourceSha256 }, + selection: { sha256: selectionSha256, checks: [{ stableKey: 'ecommerce.checkout.complete' }] } }; + const declaration = { sequence: 1, phase: 'final', level: 3, accepted: true, excluded: false, workNodeIds: [], + sourceSha256, selectionSha256, evidence: { path: 'checkpoint.json', sha256: '0'.repeat(64) }, + cost: { status: 'exact', costUsd: 1 }, executionCost: { status: 'exact', costUsd: 1 }, + completion: { selected: 1, passed: 1, failed: 0, blocked: 0, unmeasured: 0, rate: 1 }, + checks: [{ id: 'ecommerce.checkout.complete', status: 'passed' }] }; + const run = { backend: 'postgres', track: 'ecommerce', contaminated: false, + runtime: { buildImage: `sha256:${'b'.repeat(64)}` }, backendLease: { runIndex: 1, + resources: { serverUri: null as string | null, module: null as string | null, database: 'app_ecom_run1' as string | null, + container: undefined as { image: string } | undefined } }, + condition: { guidance: { credentialAliases: { 'role:admin': 'admin' } } }, checkpoints: [declaration] }; + const readerPath = join(root, 'reader.json'); + writeFileSync(readerPath, JSON.stringify({ sourceSha256, sql: 'SELECT 1' })); + const input = { run: 'run.json', runSha256: '', checkpoint: 1, source: 'source', + reader: { path: 'reader.json', sha256: digest(readerPath) } }; + const save = (completedAt: string | null = '2026-09-15T12:01:00.000Z') => { + writeArtifact(checkpointPath, { kind: 'grade_bundle', id: 'accepted-grade', payload: checkpoint }); + declaration.evidence.sha256 = digest(checkpointPath); + writeArtifact(runPath, { kind: 'benchmark_run', id: 'saved-run', payload: run, + timestamps: { startedAt: '2026-09-15T12:00:00.000Z', completedAt } }); + input.runSha256 = digest(runPath); + }; + save(); + return { root, app, runPath, checkpointPath, readerPath, sourceSha256, checkpoint, declaration, run, input, save }; +} + +test('saved diagnostic binds accepted final source, original build and run index through real artifact envelopes', t => { + const f = fixture(t), result = inspectSavedDiagnostic(f.input, f.root); + assert.equal(result.sourceSha256, f.sourceSha256); + assert.equal(result.source, f.app); assert.equal(result.run, f.runPath); + assert.equal(result.reader.path, f.readerPath); assert.equal(result.reader.sha256, digest(f.readerPath)); + assert.equal(result.buildImage, f.run.runtime.buildImage); assert.equal(result.runIndex, 1); + assert.equal(result.selectionSha256, f.declaration.selectionSha256); + assert.deepEqual(result.credentialAliases, { 'role:admin': 'admin' }); +}); + +test('saved diagnostics retain native target identity and reject a cross-stack checkpoint', t => { + const f = fixture(t); + f.run.backend = f.checkpoint.backend = 'spacetime'; + f.run.backendLease.resources = { serverUri: 'http://127.0.0.1:3211', module: 'original-module', database: null, + container: { image: `sha256:${'c'.repeat(64)}` } }; + f.save(); + const result = inspectSavedDiagnostic(f.input, f.root); + assert.equal(result.serverUri, 'http://127.0.0.1:3211'); assert.equal(result.module, 'original-module'); + assert.equal(result.backendImage, f.run.backendLease.resources.container!.image); + f.checkpoint.backend = 'mongodb'; f.save(); + assert.throws(() => inspectSavedDiagnostic(f.input, f.root), /mismatch/); + f.checkpoint.backend = 'spacetime'; f.run.backendLease.resources.serverUri = 'http://remote:3211'; f.save(); + assert.throws(() => inspectSavedDiagnostic(f.input, f.root), /local SpacetimeDB/); + f.run.backendLease.resources.serverUri = 'http://127.0.0.1:3211'; + f.run.backendLease.resources.container = undefined; f.save(); + assert.throws(() => inspectSavedDiagnostic(f.input, f.root), /original backend image/); +}); + +test('saved diagnostic rejects changed run, checkpoint, source and reader bytes', t => { + for (const target of ['run', 'checkpoint', 'source', 'reader'] as const) { + const f = fixture(t); + const path = { run: f.runPath, checkpoint: f.checkpointPath, + source: join(f.app, 'server.js'), reader: f.readerPath }[target]; + appendFileSync(path, ' '); + assert.throws(() => inspectSavedDiagnostic(f.input, f.root), /changed|mismatch|does not match/); + } + const f = fixture(t); + writeFileSync(f.readerPath, JSON.stringify({ sourceSha256: 'c'.repeat(64), sql: 'SELECT 1' })); + f.input.reader.sha256 = digest(f.readerPath); + assert.throws(() => inspectSavedDiagnostic(f.input, f.root), /reader does not match/); +}); + +test('saved diagnostic requires a completed uncontaminated run and an unambiguous final accepted L3 checkpoint', t => { + for (const mode of ['running', 'contaminated', 'rejected', 'excluded', 'earlier', 'duplicate', 'wrong-level']) { + const f = fixture(t); + if (mode === 'contaminated') f.run.contaminated = true; + if (mode === 'rejected') f.declaration.accepted = false; + if (mode === 'excluded') f.declaration.excluded = true; + if (mode === 'earlier') f.run.checkpoints.push({ ...f.declaration, sequence: 2 }); + if (mode === 'duplicate') f.run.checkpoints.push({ ...f.declaration }); + if (mode === 'wrong-level') f.declaration.level = 2; + f.save(mode === 'running' ? null : undefined); + assert.throws(() => inspectSavedDiagnostic(f.input, f.root), /./, mode); + } +}); + +test('saved diagnostic rejects source or selection substitution and payment/reservation selections', t => { + for (const mode of ['source', 'selection', 'duplicate', 'payment-records', 'payment-deduplication', 'reservation']) { + const f = fixture(t); + if (mode === 'source') f.checkpoint.source.sha256 = 'c'.repeat(64); + else if (mode === 'selection') f.checkpoint.selection.sha256 = 'c'.repeat(64); + else if (mode === 'duplicate') f.checkpoint.selection.checks.push({ ...f.checkpoint.selection.checks[0]! }); + else f.checkpoint.selection.checks = [{ stableKey: `ecommerce.spec.${mode}.check` }]; + f.save(); + assert.throws(() => inspectSavedDiagnostic(f.input, f.root), /mismatch|order-only/, mode); + } +}); diff --git a/tools/stack-bench/tests/scenario-contract-check.contract.ts b/tools/stack-bench/tests/scenario-contract-check.contract.ts new file mode 100644 index 00000000000..48a604f1824 --- /dev/null +++ b/tools/stack-bench/tests/scenario-contract-check.contract.ts @@ -0,0 +1,38 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import test from 'node:test'; +import { STACK_BENCH_ROOT, compiledEntrypoint } from '../src/package-root.js'; +import { referencedInterfaceValues } from '../commands/check-scenarios.js'; + +test('interface checks include nested action inputs and replay targets', () => { + const step = { do: 'parallel', branches: [[ + { do: 'callAction', input: { testid: 'item-card', attribute: 'data-buy-input' } }, + { do: 'replayAs', namedTarget: { testid: 'pending-restock-item', attribute: 'data-entity-id' } }, + ]] }; + assert.deepEqual(referencedInterfaceValues(step, 'testid'), ['item-card', 'pending-restock-item']); + assert.deepEqual(referencedInterfaceValues(step, 'attribute'), ['data-buy-input', 'data-entity-id']); +}); + +function checkScenarioArgs(args: readonly string[]): void { + const result = spawnSync(process.execPath, [compiledEntrypoint('commands', + 'check-scenarios.js'), ...args], { + cwd: STACK_BENCH_ROOT, + encoding: 'utf8', + }); + assert.equal(result.status, 0, result.stdout + result.stderr); + assert.match(result.stdout, /0 errors; 0 warnings/); +} + +test('the progression scenarios use only their selected application interfaces', () => { + checkScenarioArgs(['--track', 'ecommerce', '--recipe', 'progression-catalog.json']); +}); + +for (const recipe of [ + 'sequential-l1.json', + 'sequential-l2.json', + 'sequential-l3.json', +]) { + test(`${recipe} scenarios match the product request and application interface`, () => { + checkScenarioArgs(['--track', 'ecommerce', '--recipe', recipe]); + }); +} diff --git a/tools/stack-bench/tests/scoring.test.ts b/tools/stack-bench/tests/scoring.test.ts new file mode 100644 index 00000000000..b742e0aa0fd --- /dev/null +++ b/tools/stack-bench/tests/scoring.test.ts @@ -0,0 +1,127 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { compareCriterionEvidence, formatRepairProgress } from '../src/evidence/scoring.js'; +import { + createCheckEvidence, + type CheckEvidenceStatus, +} from '../src/evidence/check-evidence.js'; + +const typedCriterion = (id: string, status: CheckEvidenceStatus, summary: string | null) => { + const evidence = createCheckEvidence({ status, code: 'test_result', phase: 'assertion', summary, + startedAtMs: 1, completedAtMs: 2 }); + return { id, stableKey: id, points: 1, evidence }; +}; +const criterion = ( + id: string, + passed: boolean, + options: { inconclusive?: boolean; points?: number } = {}, +) => ({ + ...typedCriterion(id, options.inconclusive ? 'inconclusive' : passed ? 'passed' : 'failed', null), + points: options.points ?? 1, +}); +const bundle = ( + criteria: ReturnType[], + suite = 'features', + featureId: string | number = 1, +) => ({ + suites: { [suite]: { features: [{ id: featureId, name: 'Display name', criteria }] } }, +}); + +test('a newly measurable failure does not make unchanged shared criteria regress', () => { + const before = bundle([ + criterion('stable', true), + criterion('new', false, { inconclusive: true }), + ]); + const after = bundle([ + criterion('stable', true), + criterion('new', false), + ]); + + const result = compareCriterionEvidence(before, after); + assert.equal(result.before, 1); + assert.equal(result.after, 1); + assert.deepEqual(result.newlyConclusive, ['features/1/new']); + assert.deepEqual(result.lostEvidence, []); + assert.equal(formatRepairProgress(result, + { before: 1, beforeMax: 1, after: 1, afterMax: 2 }), + 'no change among 1 criterion measured in both rounds (1/1 points); 1 previously unavailable criterion became measurable; overall 1/1 -> 1/2'); +}); + +test('unchanged repair evidence is reported without hiding the overall scores', () => { + const result = compareCriterionEvidence( + bundle([criterion('a', true)]), + bundle([criterion('a', true)]), + ); + assert.equal(formatRepairProgress(result, + { before: 1, beforeMax: 1, after: 1, afterMax: 1 }), + 'no improvement among 1 criterion measured in both rounds (1/1 points); overall 1/1 -> 1/1'); +}); + +test('pass or fail to inconclusive is reported as lost evidence', () => { + for (const passedBefore of [true, false]) { + const result = compareCriterionEvidence( + bundle([criterion('a', passedBefore)]), + bundle([criterion('a', false, { inconclusive: true })]), + ); + assert.deepEqual(result.lostEvidence, ['features/1/a']); + } +}); + +test('a real regression is compared on stable evidence', () => { + const result = compareCriterionEvidence( + bundle([criterion('a', true)]), + bundle([criterion('a', false)]), + ); + assert.equal(result.before, 1); + assert.equal(result.after, 0); + assert.deepEqual(result.lostEvidence, []); + assert.deepEqual(result.regressions, ['a']); +}); + +test('suite identity prevents unrelated criteria from colliding', () => { + const before = { + suites: { + features: bundle([criterion('same', true)]).suites.features, + invariants: bundle([criterion('same', false)]).suites.features, + }, + }; + const after = { + suites: { + features: bundle([criterion('same', false)]).suites.features, + invariants: bundle([criterion('same', true)]).suites.features, + }, + }; + const result = compareCriterionEvidence(before, after); + assert.equal(result.count, 2); + assert.equal(result.before, 1); + assert.equal(result.after, 1); +}); + +test('rubric point changes invalidate the comparison', () => { + const result = compareCriterionEvidence( + bundle([criterion('a', true)]), + bundle([criterion('a', true, { points: 2 })]), + ); + assert.deepEqual(result.definitionChanges, [ + { key: 'features/1/a', before: 1, after: 2 }, + ]); +}); + +test('typed scoring never reparses a misleading summary', () => { + const result = compareCriterionEvidence( + bundle([typedCriterion('a', 'passed', null)]), + bundle([typedCriterion('a', 'failed', 'INCONCLUSIVE: merely presentation text')]), + ); + assert.equal(result.count, 1); + assert.equal(result.before, 1); + assert.equal(result.after, 0); + assert.deepEqual(result.lostEvidence, []); +}); + +test('repair comparison rejects invalid rubric points', () => { + assert.throws(() => compareCriterionEvidence( + bundle([criterion('a', true, { points: -1 })]), + bundle([criterion('a', true)]), + ), /points must be a non-negative integer/); +}); diff --git a/tools/stack-bench/tests/selected-grade.integration.ts b/tools/stack-bench/tests/selected-grade.integration.ts new file mode 100644 index 00000000000..65260600663 --- /dev/null +++ b/tools/stack-bench/tests/selected-grade.integration.ts @@ -0,0 +1,536 @@ +import assert from 'node:assert/strict'; +import { execFile, spawn } from 'node:child_process'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createServer } from 'node:http'; +import test from 'node:test'; + +import { readGradeArtifactPayload } from '../src/evidence/artifacts.js'; +import { requireRecipeRelease as resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { resolveRecipeSelection } from '../src/composition/recipe-selection.js'; +import { loadTrack } from '../src/composition/tracks.js'; +import { compiledEntrypoint, STACK_BENCH_ROOT } from '../src/package-root.js'; + +const GRADER = compiledEntrypoint('grader', 'grade.js'); + +test('order ownership measures state and useful access across refusal formats', async t => { + // Failure cases: a safe conflict or private 404 is a refusal; writing then + // refusing, refusing every caller, a missing route, and a server crash are not + // valid ownership protection. Exercise the authored check, not a copied probe. + const root = join(STACK_BENCH_ROOT, 'results', 'diagnostics', `ownership-refusal-${Date.now()}`); + mkdirSync(root, { recursive: true }); + t.diagnostic(`Evidence: ${root}; repeat: node --test --test-name-pattern="order ownership measures" dist/tests/selected-grade.integration.js`); + const track = loadTrack('ecommerce'); + const binding = resolveRecipeRelease(track, 2, { id: 'ecommerce.progression-catalog' }); + const check = binding.release.checkCatalog.find(c => c.criterionId === '204a'); + assert(check?.source); + const selection = resolveRecipeSelection(binding.release, { checkKeys: [check.stableKey] }); + for (const fixture of [ + { name: 'safe-forbidden', status: 403, expected: 'passed' }, + { name: 'safe-conflict', status: 409, expected: 'passed' }, + { name: 'safe-private-not-found', status: 404, expected: 'passed' }, + { name: 'write-then-refuse', status: 409, expected: 'failed' }, + { name: 'reject-all', status: 403, expected: 'failed' }, + { name: 'missing-route', status: 404, expected: 'failed' }, + { name: 'server-error', status: 500, expected: 'failed' }, + ]) await t.test(fixture.name, async () => { + const app = join(root, fixture.name); mkdirSync(app); + const orders: Array<{ id: number; item: string; owner: string; status: string }> = []; + const requests: Array<{ caller: string; order: number; status: number }> = []; + const html = ` + + +
+ ${['Desk Lamp', 'Keyboard'].map((item, id) => `
${item}
`).join('')} +
+ `; + const server = createServer((req, res) => { + const caller = decodeURIComponent((req.headers.cookie ?? '').replace(/^user=/, '')); + res.setHeader('cache-control', 'no-store'); + if (req.url === '/session') { + let name = ''; req.on('data', chunk => { name += chunk; }); + req.on('end', () => { res.setHeader('set-cookie', `user=${encodeURIComponent(name)}; Path=/`); res.end('{}'); }); + } else if (req.url === '/orders') { + res.setHeader('content-type', 'application/json'); res.end(JSON.stringify(orders.filter(o => o.owner === caller))); + } else if (req.url?.endsWith('/buy')) { + const id = Number(req.url.split('/')[3]); + orders.push({ id: orders.length + 1, item: ['Desk Lamp', 'Keyboard'][id]!, owner: caller, status: 'pending' }); + res.setHeader('content-type', 'application/json'); res.end('{}'); + } else if (req.url?.endsWith('/cancel')) { + const order = orders.find(o => o.id === Number(req.url!.split('/')[3])); + const allowed = order?.owner === caller && !['reject-all', 'missing-route'].includes(fixture.name); + if (order && (allowed || fixture.name === 'write-then-refuse')) order.status = 'cancelled'; + res.statusCode = allowed ? 200 : fixture.status; + requests.push({ caller, order: order?.id ?? 0, status: res.statusCode }); + res.setHeader('content-type', 'application/json'); res.end('{}'); + } else { res.setHeader('content-type', 'text/html'); res.end(html); } + }); + writeFileSync(join(app, 'index.html'), html); + writeFileSync(join(app, 'expected.json'), JSON.stringify(fixture)); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + try { + const address = server.address(); assert(address && typeof address !== 'string'); + const out = join(app, 'grade.json'); + const execution = await run(GRADER, ['--url', `http://127.0.0.1:${address.port}`, '--level', '2', '--track', 'ecommerce', + '--backend', 'postgres', '--app', app, '--spec', join(track.dir, check.source!), '--out', out, + '--recipe', binding.release.id, '--selected-check', check.stableKey, + '--expected-recipe-sha256', binding.release.contentSha256, '--selection-sha256', selection.sha256]); + writeFileSync(join(app, 'stdout.log'), execution.stdout); writeFileSync(join(app, 'stderr.log'), execution.stderr); + writeFileSync(join(app, 'effects.json'), JSON.stringify({ orders, requests }, null, 2)); + const criterion = first(first(readGradeArtifactPayload(out).features).criteria); + assert.equal(criterion.evidence.status, fixture.expected); + assert(requests.length > 0, 'the check must reach the cancel endpoint'); + if (fixture.expected === 'passed') assert.equal(requests.length, 4, 'both caller attacks and both owner controls must execute'); + if (fixture.name === 'write-then-refuse') assert.equal(criterion.evidence.finding?.kind, 'value-mismatch'); + } catch (error) { + writeFileSync(join(app, 'failure.txt'), String(error)); throw error; + } finally { + server.closeAllConnections(); await new Promise(resolve => server.close(() => resolve())); + writeFileSync(join(app, 'cleanup.json'), JSON.stringify({ serverClosed: !server.listening })); + } + }); +}); + +const first = (values: readonly Value[]): Value => { + const value = values[0]; + assert(value); + return value; +}; +const record = (value: unknown): value is Record => + value !== null && typeof value === 'object' && !Array.isArray(value); +const actionIds = (actions: ReadonlyArray<{ evidence: unknown }>): string[] => actions.map(entry => { + if (!record(entry.evidence) || !record(entry.evidence.action) + || typeof entry.evidence.action.id !== 'string') { + throw new Error('grade action evidence has no action id'); + } + return entry.evidence.action.id; +}); + +function startBlankApp(html: string = '', catalogHtml?: string) { + const source = ` + import { createServer } from 'node:http'; + const server = createServer((request, response) => { + response.writeHead(200, { 'content-type': 'text/html; charset=utf-8' }); + response.end(request.url === '/catalog' ? ${JSON.stringify(catalogHtml ?? html)} : ${JSON.stringify(html)}); + }); + server.listen(0, '127.0.0.1', () => console.log(server.address().port)); + process.on('SIGTERM', () => server.close(() => process.exit(0))); + `; + const child = spawn(process.execPath, ['--input-type=module', '--eval', source], { + stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true, + }); + const port = new Promise((resolve, reject) => { + const deadline = setTimeout(() => reject(new Error('blank app did not start')), 10_000); + child.once('error', reject); + child.stdout.once('data', data => { + clearTimeout(deadline); + resolve(Number(data.toString().trim())); + }); + }); + return { child, port }; +} + +function run(file: string, argv: readonly string[]): Promise<{ stdout: string; stderr: string }> { + return new Promise((resolve, reject) => { + execFile(process.execPath, [file, ...argv], { + encoding: 'utf8', timeout: 60_000, maxBuffer: 8 * 1024 * 1024, + }, (error, stdout, stderr) => { + if (error) reject(new Error(`${error.message}\n${stdout}\n${stderr}`)); + else resolve({ stdout, stderr }); + }); + }); +} + +test('the live grader executes and reports exactly one selected stable check', async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-selected-grade-')); + const app = join(root, 'app'); + const out = join(root, 'grade.json'); + mkdirSync(app, { recursive: true }); + const server = startBlankApp(); + try { + const port = await server.port; + const track = loadTrack('ecommerce'); + const binding = resolveRecipeRelease(track, 1); + const check = binding.release.checkCatalog.find(candidate => candidate.criterionId === '2a'); + assert(check); + assert(check.source); + const selection = resolveRecipeSelection(binding.release, { checkKeys: [check.stableKey] }); + await run(GRADER, [ + '--url', `http://127.0.0.1:${port}`, + '--level', '1', '--track', 'ecommerce', '--backend', 'postgres', '--app', app, + '--spec', join(track.dir, check.source), '--out', out, + '--expected-recipe-sha256', binding.release.contentSha256, + '--selected-check', check.stableKey, '--selection-sha256', selection.sha256, + ]); + const report = readGradeArtifactPayload(out); + assert(report.selection); + assert.equal(report.selection.sha256, selection.sha256); + assert.deepEqual(report.selection.checks.map(item => item.stableKey), [check.stableKey]); + assert.equal(report.features.length, 1); + const feature = first(report.features); + const criterion = first(feature.criteria); + assert.deepEqual(feature.criteria.map(item => item.stableKey), [check.stableKey]); + assert.equal(feature.setupEvidence.status, 'passed'); + assert.equal(criterion.evidence.status, 'failed'); + assert.equal(criterion.evidence.phase, 'assertion'); + assert(criterion.evidence.actions.length > 0, 'the selected check must execute'); + assert.equal(report.max, check.points); + } finally { + server.child.kill('SIGTERM'); + rmSync(root, { recursive: true, force: true }); + } +}); + +test('catalog values are checked on the catalog page, including through navigation', async t => { + // Failure cases: a landing page must not hide a working catalog from the + // grader; navigation must not excuse a missing item, wrong price, or dead link. + const parent = join(STACK_BENCH_ROOT, 'results', 'diagnostics'); + mkdirSync(parent, { recursive: true }); + const root = mkdtempSync(join(parent, 'catalog-navigation-')); + t.diagnostic(`Evidence: ${root}; repeat: node --test --test-name-pattern="catalog values are checked" dist/tests/selected-grade.integration.js`); + const track = loadTrack('ecommerce'); + const binding = resolveRecipeRelease(track, 1, { id: 'ecommerce.progression-catalog' }); + const check = binding.release.checkCatalog.find(c => c.stableKey === 'ecommerce.feature.catalog.catalog-values.2a'); + assert(check?.source); + const selection = resolveRecipeSelection(binding.release, { checkKeys: [check.stableKey] }); + for (const fixture of [ + { name: 'initial-catalog', linked: false, item: 'Air Purifier', price: 189, opens: true, expected: 'passed' }, + { name: 'linked-catalog', linked: true, item: 'Air Purifier', price: 189, opens: true, expected: 'passed' }, + { name: 'wrong-price', linked: true, item: 'Air Purifier', price: 1.89, opens: true, expected: 'failed' }, + { name: 'missing-item', linked: true, item: 'Desk Lamp', price: 189, opens: true, expected: 'failed' }, + { name: 'dead-link', linked: true, item: 'Air Purifier', price: 189, opens: false, expected: 'failed' }, + ]) await t.test(fixture.name, async () => { + const app = join(root, fixture.name); + mkdirSync(app); + const catalog = `
+ ${fixture.item}${fixture.price} + 100
`; + const html = fixture.linked ? `

Storefront

Shop` : catalog; + writeFileSync(join(app, 'index.html'), html); + writeFileSync(join(app, 'catalog.html'), catalog); + writeFileSync(join(app, 'expected.json'), JSON.stringify(fixture, null, 2)); + const server = startBlankApp(html, catalog); + const out = join(app, 'grade.json'); + try { + const port = await server.port; + const execution = await run(GRADER, ['--url', `http://127.0.0.1:${port}`, '--level', '1', '--track', 'ecommerce', + '--backend', 'postgres', '--app', app, '--spec', join(track.dir, check.source!), '--out', out, + '--recipe', binding.release.id, '--selected-check', check.stableKey, + '--expected-recipe-sha256', binding.release.contentSha256, '--selection-sha256', selection.sha256]); + writeFileSync(join(app, 'stdout.log'), execution.stdout); + writeFileSync(join(app, 'stderr.log'), execution.stderr); + const report = readGradeArtifactPayload(out); + assert.equal(first(first(report.features).criteria).evidence.status, fixture.expected); + assert.equal(report.max, 1); + } finally { + server.child.kill('SIGTERM'); + } + }); +}); + +test('purchase observers reach a separate catalog before the write and stay there', async t => { + // A working linked catalog must pass. A dead link must fail. A stale observer + // must fail even if returning to the catalog would show the updated stock. + const root = mkdtempSync(join(STACK_BENCH_ROOT, 'results', 'diagnostics', 'purchase-navigation-')); + t.diagnostic(`Evidence: ${root}; repeat: node --test --test-name-pattern="purchase observers" dist/tests/selected-grade.integration.js`); + const track = loadTrack('ecommerce'); + const binding = resolveRecipeRelease(track, 2, { id: 'ecommerce.progression-catalog' }); + const check = binding.release.checkCatalog.find(c => c.stableKey === 'ecommerce.spec.live-state.purchase-stock.3b'); + assert(check?.source); + const selection = resolveRecipeSelection(binding.release, { checkKeys: [check.stableKey] }); + for (const fixture of [ + { name: 'root', linked: false, opens: true, live: true, expected: 'passed' }, + { name: 'linked', linked: true, opens: true, live: true, expected: 'passed' }, + { name: 'dead-link', linked: true, opens: false, live: true, expected: 'failed' }, + { name: 'stale-observer', linked: true, opens: true, live: false, expected: 'failed' }, + { name: 'late-first-read', linked: true, opens: true, live: true, delayGuest: true, expected: 'failed' }, + ]) await t.test(fixture.name, async () => { + const app = join(root, fixture.name); + mkdirSync(app); + let stock = 100, navigationsAfterPurchase = 0, purchases = 0; + const pendingReads: Array<() => void> = []; + const delayGuest = 'delayGuest' in fixture; + const catalog = `
+ Espresso Machine100 +
`; + const html = (inCatalog: boolean) => ` + + + Shop + ${inCatalog ? catalog : '

Home

'} + `; + const server = createServer((request, response) => { + if (request.url?.startsWith('/stock')) { + if (request.method === 'POST') { stock--; purchases++; pendingReads.splice(0).forEach(read => read()); } + response.writeHead(200, { 'content-type': 'text/plain', 'cache-control': 'no-store' }); + if (request.url === '/stock?wait=1' && !purchases) { + pendingReads.push(() => response.end(String(stock))); + return; + } + response.end(String(stock)); + return; + } + if (purchases) navigationsAfterPurchase++; + response.writeHead(200, { 'content-type': 'text/html' }); + response.end(html(!fixture.linked || request.url === '/catalog')); + }); + writeFileSync(join(app, 'index.html'), html(!fixture.linked)); + writeFileSync(join(app, 'catalog.html'), html(true)); + writeFileSync(join(app, 'expected.json'), JSON.stringify(fixture, null, 2)); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + try { + const address = server.address(); + assert(address && typeof address !== 'string'); + const out = join(app, 'grade.json'); + const execution = await run(GRADER, ['--url', `http://127.0.0.1:${address.port}`, '--level', '2', + '--track', 'ecommerce', '--backend', 'postgres', '--app', app, + '--spec', join(track.dir, check.source!), '--out', out, '--recipe', binding.release.id, + '--selected-check', check.stableKey, '--expected-recipe-sha256', binding.release.contentSha256, + '--selection-sha256', selection.sha256]); + writeFileSync(join(app, 'stdout.log'), execution.stdout); + writeFileSync(join(app, 'stderr.log'), execution.stderr); + writeFileSync(join(app, 'effects.json'), JSON.stringify({ purchases, stock, navigationsAfterPurchase })); + const report = readGradeArtifactPayload(out); + assert.equal(first(first(report.features).criteria).evidence.status, fixture.expected); + if (fixture.opens) assert.equal(purchases, delayGuest ? 0 : 1, 'observe initial data before the purchase'); + assert.equal(navigationsAfterPurchase, 0, 'observers must not reload to see the write'); + } finally { + server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + } + }); +}); + +test('cart checks reopen catalog after authentication and reload without refreshing live observers', async t => { + // Authentication and reload can both return a state-based app to its home + // screen. Persistence must still be measured; stale live views must fail. + const root = mkdtempSync(join(STACK_BENCH_ROOT, 'results', 'diagnostics', 'cart-navigation-')); + t.diagnostic(`Evidence: ${root}; repeat: node --test --test-name-pattern="cart checks reopen" dist/tests/selected-grade.integration.js`); + const track = loadTrack('ecommerce'); + const binding = resolveRecipeRelease(track, 2, { id: 'ecommerce.progression-catalog' }); + for (const fixture of [{ id: '4b', live: true }, { id: '4c', live: true }, { id: '4c', live: false }]) { + await t.test(`${fixture.id}-${fixture.live ? 'live' : 'stale'}`, async () => { + const check = binding.release.checkCatalog.find(c => c.criterionId === fixture.id && c.source === 'scenarios/01-cart.json'); + assert(check?.source); + const selection = resolveRecipeSelection(binding.release, { checkKeys: [check.stableKey] }); + const app = join(root, `${fixture.id}-${fixture.live}`); + mkdirSync(app); + let item = '', writes = 0; + const html = `

Home

+ + + + + `; + const server = createServer((request, response) => { + if (request.url === '/cart') { + response.setHeader('content-type', 'text/plain'); + response.setHeader('cache-control', 'no-store'); + if (request.method === 'POST') { + let body = ''; request.on('data', chunk => { body += chunk; }); + request.on('end', () => { item = body; writes++; response.end(item); }); + } else response.end(item); + } else { response.setHeader('content-type', 'text/html'); response.end(html); } + }); + writeFileSync(join(app, 'index.html'), html); + writeFileSync(join(app, 'expected.json'), JSON.stringify(fixture)); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + try { + const address = server.address(); assert(address && typeof address !== 'string'); + const out = join(app, 'grade.json'); + const execution = await run(GRADER, ['--url', `http://127.0.0.1:${address.port}`, '--level', '2', '--track', 'ecommerce', + '--backend', 'postgres', '--app', app, '--spec', join(track.dir, check.source), '--out', out, + '--recipe', binding.release.id, '--selected-check', check.stableKey, + '--expected-recipe-sha256', binding.release.contentSha256, '--selection-sha256', selection.sha256]); + writeFileSync(join(app, 'stdout.log'), execution.stdout); + writeFileSync(join(app, 'stderr.log'), execution.stderr); + writeFileSync(join(app, 'effects.json'), JSON.stringify({ item, writes })); + const report = readGradeArtifactPayload(out); + assert.equal(first(first(report.features).criteria).evidence.status, fixture.live ? 'passed' : 'failed'); + assert.equal(writes, 1, 'the cart observation must follow an actual add'); + } finally { + server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + } + }); + } +}); + +test('setup can wait for app readiness without relaxing scored checks', async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-ready-grade-')); + const out = join(root, 'grade.json'); + const spec = join(root, 'scenario.json'); + writeFileSync(spec, JSON.stringify({ + schemaVersion: 1, + level: 1, + features: [{ + id: 1, + name: 'delayed app readiness', + actors: ['a'], + setup: [{ do: 'signUp', actor: 'a', name: 'Alice' }], + criteria: [ + { id: '1a', desc: 'setup completed', points: 1, + steps: [{ do: 'expect', actor: 'a', testid: 'current-user' }] }, + { id: '1b', desc: 'scored checks keep the normal deadline', points: 1, + steps: [ + { do: 'click', actor: 'a', testid: 'slow-action' }, + { do: 'expect', actor: 'a', testid: 'slow-result' }, + ] }, + ], + }], + })); + const server = startBlankApp(` +
+ + `); + try { + const port = await server.port; + await run(GRADER, ['--url', `http://127.0.0.1:${port}`, '--level', '1', + '--spec', spec, '--out', out]); + const report = readGradeArtifactPayload(out); + const feature = first(report.features); + assert.equal(feature.setupEvidence.status, 'passed'); + assert.deepEqual(actionIds(feature.setupEvidence.actions), ['signUp']); + assert.equal(report.total, 1); + assert.equal(report.max, 2); + assert.equal(first(feature.criteria).evidence.status, 'passed'); + assert.deepEqual(actionIds(first(feature.criteria).evidence.actions), ['expect']); + const failedCriterion = feature.criteria[1]; + assert(failedCriterion); + assert.equal(failedCriterion.evidence.status, 'failed'); + } finally { + server.child.kill('SIGTERM'); + rmSync(root, { recursive: true, force: true }); + } +}); + +test('an inconclusive check keeps the recipe denominator fixed', async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-fixed-denominator-')); + const app = join(root, 'app'); + const out = join(root, 'grade.json'); + const spec = join(root, 'scenario.json'); + mkdirSync(app, { recursive: true }); + writeFileSync(spec, JSON.stringify({ + schemaVersion: 1, + level: 1, + features: [{ + id: 1, + name: 'unsupported action', + actors: ['a'], + setup: [], + criteria: [{ id: '1a', desc: 'the declared point remains in the contract', points: 3, + steps: [{ do: 'callAction', actor: 'a', action: 'not-declared', + input: { testid: 'action-input', attribute: 'data-input' } }] }], + }], + })); + const server = startBlankApp('
'); + try { + const port = await server.port; + await run(GRADER, ['--url', `http://127.0.0.1:${port}`, '--level', '1', + '--backend', 'postgres', '--app', app, '--spec', spec, '--out', out]); + const report = readGradeArtifactPayload(out); + assert.equal(report.total, 0); + assert.equal(report.max, 3); + const feature = first(report.features); + assert.equal(feature.max, 3); + assert.equal(first(feature.criteria).evidence.status, 'inconclusive'); + assert.equal(first(report.inconclusive).points, 3); + } finally { + server.child.kill('SIGTERM'); + rmSync(root, { recursive: true, force: true }); + } +}); + +test('standalone zero-point diagnostics reach execution without a recipe binding', async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-standalone-diagnostic-')); + const spec = join(root, 'scenario.json'); + const out = join(root, 'grade.json'); + writeFileSync(spec, JSON.stringify({ schemaVersion: 1, track: 'ecommerce', level: 2, + features: [{ id: 9000, name: 'Standalone diagnostic', actors: ['a'], setup: [], + criteria: [{ id: '9000a', category: 'production', points: 0, + desc: 'local entry point control', + steps: [{ do: 'expect', actor: 'a', testid: 'diagnostic-ready', within: 1000 }] }], + }], + })); + const server = startBlankApp('
ready
'); + try { + const port = await server.port; + await run(GRADER, ['--url', `http://127.0.0.1:${port}`, '--level', '2', '--spec', spec, '--out', out]); + const report = readGradeArtifactPayload(out); + assert.equal(report.recipeRelease, null); + assert.equal(report.selection, null); + assert.equal(report.max, 0); + assert.equal(report.total, 0); + assert.equal(report.features.length, 1); + const criterion = first(first(report.features).criteria); + assert.equal(criterion.evidence.status, 'passed'); + assert.equal(criterion.evidence.phase, 'assertion'); + assert.deepEqual(actionIds(criterion.evidence.actions), ['expect']); + } finally { + server.child.kill('SIGTERM'); + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/tools/stack-bench/tests/semantic-version.test.ts b/tools/stack-bench/tests/semantic-version.test.ts new file mode 100644 index 00000000000..9a5d1ac9ee4 --- /dev/null +++ b/tools/stack-bench/tests/semantic-version.test.ts @@ -0,0 +1,11 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { isExactSemanticVersion } from '../src/semantic-version.js'; + +test('accepts exact semantic versions used by runtime contracts', () => { + assert.equal(isExactSemanticVersion('1.2.3'), true); + assert.equal(isExactSemanticVersion('1.2.3-rc.1+build.7'), true); + assert.equal(isExactSemanticVersion('v1.2.3'), false); + assert.equal(isExactSemanticVersion('^1.2.3'), false); +}); diff --git a/tools/stack-bench/tests/session-metrics.test.ts b/tools/stack-bench/tests/session-metrics.test.ts new file mode 100644 index 00000000000..30f68dd0bcf --- /dev/null +++ b/tools/stack-bench/tests/session-metrics.test.ts @@ -0,0 +1,47 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { summarizeSessions } from '../src/evidence/session-metrics.js'; + +test('session totals include build and every fix without treating missing metrics as data', () => { + const totals = summarizeSessions([ + { costUsd: 1.11111, tokens: 100, outputTokens: 10, turns: 2, durationMs: 50, + promptBytes: 20, usage: { input: 10, output: 20, cacheWrite: 30, cacheRead: 40 }, + providerThrottle: { waits: 2, waitedMs: 20 }, + thinking: { blocks: 2, signatureBytes: 200 } }, + { costUsd: 0.22222, tokens: 50, outputTokens: 5, turns: 1, durationMs: 25, + promptBytes: 10, usage: { input: 5, output: 10, cacheWrite: 15, cacheRead: 20 } }, + { costUsd: 0.1 }, + ]); + + assert.deepEqual(totals, { + sessions: 3, + costUsd: 1.43333, + tokens: 150, + outputTokens: 15, + turns: 3, + durationMs: 75, + activeDurationMs: 55, + providerThrottle: { waits: 2, waitedMs: 20 }, + promptBytes: 30, + usage: { input: 15, output: 30, cacheWrite: 45, cacheRead: 60 }, + thinking: { blocks: 2, signatureBytes: 200, sessions: 1 }, + }); +}); + +test('empty session totals are explicit zeroes', () => { + assert.deepEqual(summarizeSessions([]), { + sessions: 0, costUsd: 0, tokens: 0, outputTokens: 0, turns: 0, + durationMs: 0, activeDurationMs: 0, promptBytes: 0, + providerThrottle: { waits: 0, waitedMs: 0 }, + usage: { input: 0, output: 0, cacheWrite: 0, cacheRead: 0 }, + thinking: null, + }); +}); + +test('session totals reject impossible metrics', () => { + assert.throws(() => summarizeSessions([{ costUsd: -1 }]), /costUsd/); + assert.throws(() => summarizeSessions([{ tokens: 1.5 }]), /tokens/); + assert.throws(() => summarizeSessions([{ + durationMs: 10, providerThrottle: { waitedMs: 11 }, + }]), /exceeds total session duration/); +}); diff --git a/tools/stack-bench/tests/shipping-accounting.integration.ts b/tools/stack-bench/tests/shipping-accounting.integration.ts new file mode 100644 index 00000000000..c7ccfa4a633 --- /dev/null +++ b/tools/stack-bench/tests/shipping-accounting.integration.ts @@ -0,0 +1,89 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import test from 'node:test'; +import { chromium } from 'playwright'; +import { ACTION_REGISTRY } from '../src/actions/action-catalog.js'; +import { executeAction } from '../src/actions/action-contract.js'; +import { createNamedActionsCapability } from '../src/actions/actor-transport-action-executors.js'; +import { stableElementSelector } from '../src/actions/element-selector.js'; +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; + +test('shipping accounting accepts a correct sale and rejects a missing sale or duplicate shipping charges', async () => { + const source = join(STACK_BENCH_ROOT, 'tracks/ecommerce/scenarios/progression-shipping-accounting.json'); + const steps = compileScenarioDefinition(JSON.parse(readFileSync(source, 'utf8')), { source }) + .features[0]!.criteria[0]!.steps; + const browser = await chromium.launch({ headless: true }); + try { + for (const defect of ['none', 'missing-sale', 'stock-twice', 'revenue-twice']) { + const context = await browser.newContext(); + try { + let east = 70, west = 30, revenue = 0, status = ''; + // The fixture supplies signed-in actors; the actual measured actions run below. + const actors = new Map(); + for (const name of ['customer', 'admin', 'staff']) { + const page = await context.newPage(); + await page.exposeFunction('buy', () => { + status = 'pending'; + if (defect !== 'missing-sale') { east--; revenue += 89; } + }); + await page.route('http://shipping-accounting.test/**', route => route.fulfill({ contentType: 'text/html', body: ` + ${name === 'customer' ? 'shipping-accounting' : name} + +
${revenue}
+
Keyboard
+ + ${status ? `
Keyboard${status}
` : ''} + ` })); + await page.goto('http://shipping-accounting.test/'); + actors.set(name, { name, page, context, record: () => {}, + writes: [{ url: 'http://shipping-accounting.test/', headers: { authorization: `Bearer ${name}` } }], + loc: (id: string, options?: { + contains?: string; scope?: { testid: string; contains?: string }; + }) => { + const root = options?.scope ? page.locator(stableElementSelector(options.scope.testid)) + .filter({ hasText: options.scope.contains }).first() : page; + const loc = root.locator(stableElementSelector(id)).filter({ visible: true }); + return (options?.contains ? loc.filter({ hasText: options.contains }) : loc).first(); + } }); + } + const sleep = async () => new Promise(resolve => setTimeout(resolve, 20)); + const capability = { recorded: new Map(), defaultWithin: 500, + expand: (value: string) => value, scopedUser: (value: string) => value, + testId: stableElementSelector, sleep }; + // Staff ships through the declared named action with its own credentials. + const named = createNamedActionsCapability({ backend: 'postgres', url: 'http://shipping-accounting.test', + lastCalls: { get: () => null, set: () => {} }, sleep, + fetchImpl: async (_url, options) => { + assert.equal(options.headers?.authorization, 'Bearer staff'); + assert.deepEqual(JSON.parse(String(options.body)), { orderId: '1' }); + status = 'shipped'; + if (defect === 'stock-twice') west--; + if (defect === 'revenue-twice') revenue += 89; + return { status: 200, ok: true, text: async () => '{}' }; + } }); + const transport = { defaultWithin: 500, expand: (value: string) => value, sleep, + verification: { unverified: () => {}, verified: () => {} } }; + let failed = null; + for (const step of steps) { + const result = await executeAction(ACTION_REGISTRY, step.do, + Object.hasOwn(step, 'within') ? { ...step, within: 500 } : step, { capabilities: { + actors: { get: (name: string) => actors.get(name) }, 'browser-interaction': capability, + 'browser-observation': capability, clock: { sleep }, 'named-actions': named, 'transport-observation': transport, + 'database-read': { getStock: async (input: { item: string; warehouse?: string }) => ({ + backend: 'postgres', item: input.item, quantity: input.warehouse === 'East' ? east : input.warehouse === 'West' ? west : east + west, + }) }, + } }); + if (result.status !== 'passed') { failed = { step, result }; break; } + } + if (defect === 'none') assert.equal(failed, null, failed?.result.summary ?? undefined); + else { + assert(failed, defect); + assert.equal(failed.step.do, defect === 'revenue-twice' ? 'expectNumber' : 'dbExpectStock', failed.result.summary ?? undefined); + assert.equal(failed.result.status, 'failed'); + } + } finally { await context.close(); } + } + } finally { await browser.close(); } +}); diff --git a/tools/stack-bench/tests/signin-reopen.integration.ts b/tools/stack-bench/tests/signin-reopen.integration.ts new file mode 100644 index 00000000000..f4619736e2d --- /dev/null +++ b/tools/stack-bench/tests/signin-reopen.integration.ts @@ -0,0 +1,123 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { chromium } from 'playwright'; + +import { ACTION_REGISTRY } from '../src/actions/action-catalog.js'; +import { executeAction } from '../src/actions/action-contract.js'; + +// A header toggle that only opens and closes the account panel closes it when a +// failed sign-up left it open on sign-up; reopening shows sign-in. A modal left on +// sign-up covers the toggle until its overlay-close dismisses it, and may drop the +// toggle's ID while it is open. A toggle that opens slowly from a closed panel must +// still be clicked only once. +test('sign-in reopens a toggle that closed a panel left on sign-up', async () => { + const browser = await chromium.launch({ headless: true }); + try { + for (const layout of ['left-on-signup', 'modal-on-signup', 'modal-drops-toggle', 'slow-open']) { + const page = await browser.newPage(); + await page.setContent(`
`); + const actor = { page, loc: (id: string) => page.locator(`#${id}`) }; + const capabilities = { actors: { get: () => actor }, 'browser-interaction': { + defaultWithin: 4000, scopedUser: (user: string) => user, testId: (id: string) => `#${id}`, + sleep: (ms: number) => page.waitForTimeout(ms), + } }; + const result = await executeAction(ACTION_REGISTRY, 'signIn', + { do: 'signIn', actor: 'probe', name: 'customer', exact: true, password: 'secret' }, { capabilities }); + assert.equal(result.status, 'passed', `${layout}: ${JSON.stringify(result.summary)}`); + assert.equal(await page.locator('#current-user').innerText(), 'customer'); + assert.equal(await page.evaluate(() => (window as unknown as { toggleClicks: number }).toggleClicks), + layout === 'left-on-signup' ? 2 : 1, layout); + if (layout.startsWith('modal')) assert.equal(await page.locator('#overlay-close').count(), 0); + await page.close(); + } + } finally { await browser.close(); } +}); + +// Sign-in belongs on the page the application's address opens while signed out; a +// signed-out page elsewhere in the app may offer none. Sign-in opens that page. +test('sign-in opens the application address when the current page has no sign-in', async () => { + const { createServer } = await import('node:http'); + const server = createServer((_request, response) => { + response.setHeader('content-type', 'text/html'); + response.end(`
`); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const applicationUrl = `http://127.0.0.1:${(server.address() as { port: number }).port}/`; + const browser = await chromium.launch({ headless: true }); + try { + for (const withAddress of [true, false]) { + const page = await browser.newPage(); + await page.goto(`${applicationUrl}#/orders`); + const actor = { page, loc: (id: string) => page.locator(`#${id}`) }; + const capabilities = { actors: { get: () => actor }, 'browser-interaction': { + defaultWithin: 2000, scopedUser: (user: string) => user, testId: (id: string) => `#${id}`, + sleep: (ms: number) => page.waitForTimeout(ms), ...(withAddress ? { applicationUrl } : {}), + } }; + const result = await executeAction(ACTION_REGISTRY, 'signIn', + { do: 'signIn', actor: 'probe', name: 'customer', exact: true, password: 'secret' }, { capabilities }); + assert.equal(result.status, withAddress ? 'passed' : 'failed', JSON.stringify(result.summary)); + if (withAddress) assert.equal(await page.locator('#current-user').innerText(), 'customer'); + await page.close(); + } + } finally { + await browser.close(); + await new Promise(resolve => server.close(resolve)); + } +}); + +// A forced click presses a position, not an element. When a sticky header covers the +// control there, the simultaneous clicks must still reach the control itself. +test('simultaneous clicks reach a control that a sticky header covers', async () => { + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage({ viewport: { width: 800, height: 600 } }); + await page.setContent(`
Store
+
+ `); + await page.evaluate(() => scrollTo(0, (document.querySelector('#add') as HTMLElement).offsetTop - 40)); + const actor = { page, loc: (id: string) => page.locator(`#${id}`) }; + const capabilities = { actors: { get: () => actor }, concurrency: { + defaultWithin: 4000, dispatch: async () => null, expand: (value: string | undefined) => value, + sleep: (ms: number) => page.waitForTimeout(ms), testId: (id: string) => `#${id}`, + hostPressure: () => null, + } }; + const result = await executeAction(ACTION_REGISTRY, 'clickConcurrently', + { do: 'clickConcurrently', actors: ['a', 'b'], testid: 'add', settleMs: 0 }, { capabilities }); + assert.equal(result.status, 'passed', JSON.stringify(result.summary)); + assert.equal(await page.evaluate(() => (window as unknown as { adds: number }).adds), 2); + } finally { await browser.close(); } +}); diff --git a/tools/stack-bench/tests/signup-claims.integration.ts b/tools/stack-bench/tests/signup-claims.integration.ts new file mode 100644 index 00000000000..590bf5dc726 --- /dev/null +++ b/tools/stack-bench/tests/signup-claims.integration.ts @@ -0,0 +1,175 @@ +import assert from 'node:assert/strict'; +import { once } from 'node:events'; +import { createServer } from 'node:http'; +import { writeFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import test from 'node:test'; +import { chromium } from 'playwright'; +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; +import { gradeFeature } from '../grader/grade.js'; +import { classifyMutationResult, indexMutationReport } from '../src/evidence/mutation-analysis.js'; + +// Failure cases: a correct two-write registration must pass; a claim accepted by +// the second write must fail even if the first write is safe; refusing every +// registration must fail; an unreached second target must remain inconclusive. +test('signup claims grade each registration write through a fresh browser session', async t => { + let mode = '', stock = 0, registrations = 0; + const accounts = new Map(); + const calls: { path: string; claim: unknown; status: number }[] = []; + const evidence: unknown[] = []; + let baselineGrade: Awaited> | undefined; + const server = createServer(async (req, res) => { + const path = new URL(req.url!, 'http://fixture.test').pathname; + let raw = ''; for await (const chunk of req) raw += String(chunk); + const body = raw ? JSON.parse(raw) as Record : {}; + const cookies = Object.fromEntries((req.headers.cookie ?? '').split(';').filter(Boolean) + .map(item => item.trim().split('=').map(decodeURIComponent))); + const send = (status: number, value: unknown) => { + calls.push({ path, claim: body.role ?? null, status }); + res.writeHead(status, { 'Content-Type': 'application/json' }); res.end(JSON.stringify(value)); + }; + if (path === '/reconnect-delay') { + // The reference UI can show the account before its new socket connects. + setTimeout(() => send(200, { ok: true }), 750); + return; + } + if (path === '/profile') { + registrations++; + if (mode === 'reject-all' || mode === 'unreached' && registrations === 3) return send(403, { error: 'Registration refused' }); + res.setHeader('Set-Cookie', `pending=${encodeURIComponent(String(body.username))}; HttpOnly; Path=/`); + return send(200, { ok: true }); + } + if (path === '/finish') { + assert(cookies.pending, 'registration must first send its ordinary profile request'); + accounts.set(cookies.pending, mode === 'unsafe-finalizer' && body.role === 'admin' ? 'admin' : 'customer'); + res.setHeader('Set-Cookie', `session=${encodeURIComponent(cookies.pending)}; HttpOnly; Path=/`); + return send(200, { user: cookies.pending }); + } + if (path === '/login') { + const user = String(body.username); + if (!accounts.has(user)) return send(403, { error: 'Unknown account' }); + res.setHeader('Set-Cookie', `session=${encodeURIComponent(user)}; HttpOnly; Path=/`); + return send(200, { user }); + } + if (path === '/restock') { + const allowed = accounts.get(cookies.session ?? '') === 'admin'; + if (allowed) stock++; + return send(allowed ? 200 : 403, allowed ? { ok: true } : { error: 'Administrator required' }); + } + res.writeHead(200, { 'Content-Type': 'text/html' }); + res.end(`${stock}
`); + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + const { wsServer } = createRequire(import.meta.url)('playwright-core/lib/utilsBundle'); + const sockets = new wsServer({ server }); + sockets.on('connection', (socket: { send(value: string): void; on(event: string, callback: (value: Buffer) => void): void }) => { + socket.send('0'+JSON.stringify({sid: 'fixture', upgrades: [], pingInterval: 25000, pingTimeout: 20000})); + socket.on('message', value => { + const frame=String(value); if(!frame.startsWith('40'))return; + const claims=frame.length>2 ? JSON.parse(frame.slice(2)) : {}; + if((mode==='startup-unsafe'||mode==='early-user-unsafe')&&claims.role==='admin'&&accounts.has(claims.user))accounts.set(claims.user,'admin'); + calls.push({path:'/socket-connect',claim:claims.role??null,status:200}); + socket.send('40'+JSON.stringify({sid:'namespace'})); + }); + }); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const browser = await chromium.launch({ headless: true }); + try { + for (const [variant, expected] of [['correct', 'passed'], ['unsafe-finalizer', 'failed'], + ['reject-all', 'failed'], ['client-reject-all', 'failed'], ['unreached', 'inconclusive'], + ['missing-signout', 'failed'], ['startup-safe', 'passed'], ['startup-unsafe', 'failed'], + ['early-user-safe', 'passed'], ['early-user-unsafe', 'failed']]) await t.test(variant!, async () => { + mode = variant!; stock = 0; registrations = 0; calls.length = 0; accounts.clear(); accounts.set('admin', 'admin'); + const action = { id: 'restock', path: '/restock', method: 'POST', reducer: 'restock', args: [], params: [] }; + const definition = compileScenarioDefinition({ schemaVersion: 1, track: 'ecommerce', level: 1, + name: 'Signup write observation', features: [{ id: 103, name: 'Signup claims', actors: ['admin', 'claimant'], + setup: [{ do: 'signIn', actor: 'admin', name: 'admin', password: 'fixture-password', exact: true }, + ...(mode.startsWith('startup-') || mode.startsWith('early-user-') + ? [{ do:'expect', actor:'claimant', testid:'connection-ready' }] : [])], + criteria: [{ id: '103b', category: 'production', desc: 'Signup claims do not grant stock authority', points: 1, + steps: [ + { do: 'callAction', actor: 'admin', action: 'restock', namedAction: action, settleMs: 0 }, + { do: 'expectActionOutcome', actor: 'admin', outcome: 'accepted' }, + { do: 'probeSignupClaims', actor: 'claimant', name: 'claimant', fields: { role: 'admin' }, branches: [[ + { do: 'callAction', actor: 'claimant', action: 'restock', namedAction: action, authentication: 'optional', settleMs: 0 }, + { do: 'reload', actor: 'admin', settleMs: 0 }, + { do: 'expectNumber', actor: 'admin', testid: 'stock', equals: 1, within: 200 }, + { do: 'expectActionOutcome', actor: 'claimant', outcome: 'application-refused', routeProvenBy: 'admin' }, + ]] }, + ] }] }] }); + const grade = await gradeFeature(browser, definition.features[0]!, + { url, level: 1, headed: false, selectedCheckKeys: [], nullControl: false }, + { url, runId: `signup-${variant}`, roomName: name => name, actions: [], spacetime: null, + backend: 'postgres', nullControl: false, defaultWithin: 200 }); + evidence.push({ variant, grade, calls: [...calls], stock, registrations }); + assert.equal(grade.criteria[0]!.evidence.status, expected, JSON.stringify(evidence.at(-1))); + if (variant === 'correct') baselineGrade = grade; + if (variant === 'unsafe-finalizer' || variant === 'missing-signout') { + assert(baselineGrade); + const report = { features: [grade] }; + const classification = classifyMutationResult({ features: [baselineGrade] }, report, { targets: ['103:103b'] }); + assert.equal(indexMutationReport(report).criteria.get('103:103b')?.failedAction, + variant === 'unsafe-finalizer' ? 'expectNumber' : 'click'); + assert.equal(classification.status, variant === 'unsafe-finalizer' ? 'CAUGHT' : 'CAUGHT_OFF_ASSERTION'); + } + if (variant === 'unsafe-finalizer') { + assert(calls.some(call => call.path === '/finish' && call.claim === 'admin')); + assert.equal(stock, 2, 'the grader must observe the unauthorized stored effect'); + } + if (variant === 'correct') assert.equal(registrations, 3, 'ordinary registration plus one fresh account per write'); + if (variant === 'startup-safe' || variant === 'early-user-safe') { + assert.equal(registrations, 4, 'registration and reconnect remain three measured writes'); + } + if (variant === 'startup-unsafe' || variant === 'early-user-unsafe') { + assert(calls.some(call=>call.path==='/socket-connect'&&call.claim==='admin')); + assert.equal(stock, 2, 'authority-bearing reconnect after signup must still be measured'); + } + }); + } finally { + await browser.close(); sockets.close(); server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + if (process.env.STACK_BENCH_SIGNUP_CLAIMS_EVIDENCE) writeFileSync(process.env.STACK_BENCH_SIGNUP_CLAIMS_EVIDENCE, + `${JSON.stringify({ schemaVersion: 1, modelCalls: 0, evidence }, null, 2)}\n`); + } +}); diff --git a/tools/stack-bench/tests/snapshots/definitions/all-actions.snapshot.json b/tools/stack-bench/tests/snapshots/definitions/all-actions.snapshot.json new file mode 100644 index 00000000000..02244029bcd --- /dev/null +++ b/tools/stack-bench/tests/snapshots/definitions/all-actions.snapshot.json @@ -0,0 +1,527 @@ +{ + "features": [ + { + "actors": [ + "a", + "b" + ], + "criteria": [ + { + "desc": "Compiles one representative of every registered action", + "id": "all-actions", + "points": 0, + "steps": [ + { + "actor": "a", + "branches": [ + [ + { + "actor": "a", + "do": "expect", + "testid": "current-user" + } + ] + ], + "do": "probeSignupClaims", + "fields": { + "role": "admin" + }, + "name": "claimant" + }, + { + "actor": "a", + "do": "probePasswordBoundary", + "impostorActor": "b", + "name": "boundary", + "password": "long-password-A", + "returningActor": "b", + "wrongPassword": "long-password-B" + }, + { + "actor": "a", + "do": "armScriptCanary" + }, + { + "actor": "a", + "do": "expectNoScriptExecution" + }, + { + "actor": "a", + "do": "confirmCheckout" + }, + { + "actor": "a", + "do": "prepareResponseLoss" + }, + { + "actor": "a", + "before": "before", + "do": "loseCheckoutResponse", + "prepared": "prepared", + "quantity": 1 + }, + { + "actor": "a", + "as": "crash", + "before": "before", + "do": "crashCheckout", + "offsetMs": 0, + "prepared": "prepared", + "quantity": 1, + "requests": 1, + "target": "database" + }, + { + "do": "expectCrashCheckout", + "from": "crash", + "verdict": "atomicity" + }, + { + "account": "a", + "as": "before", + "do": "dbRecordCheckout", + "item": "Keyboard" + }, + { + "before": "before", + "do": "dbExpectCheckout", + "prepared": "prepared", + "quantity": 1 + }, + { + "actor": "a", + "before": "before", + "do": "dbExpectOperation", + "operation": "buy", + "otherBefore": "other" + }, + { + "before": "before", + "do": "dbExpectCancellation" + }, + { + "before": { + "a": "before" + }, + "do": "dbExpectPurchases", + "purchases": 1 + }, + { + "before": "before", + "do": "dbExpectNoPurchase" + }, + { + "actor": "a", + "before": "before", + "do": "dbExpectPurchase", + "stockBefore": "stock" + }, + { + "action": "buy", + "actor": "a", + "authentication": "none", + "do": "callAction", + "input": { + "attribute": "data-action-input", + "contains": "item", + "testid": "item-card" + }, + "settleMs": 1 + }, + { + "action": "buy", + "actors": [ + "a", + "b" + ], + "do": "callConcurrently", + "settleMs": 1 + }, + { + "actor": "a", + "do": "clearInput" + }, + { + "actor": "a", + "do": "click", + "testid": "button" + }, + { + "actors": [ + "a", + "b" + ], + "do": "clickConcurrently", + "settleMs": 1, + "testid": "button" + }, + { + "actor": "a", + "do": "closeClient" + }, + { + "actor": "a", + "do": "createRoom", + "room": "room" + }, + { + "do": "dbSetStock", + "item": "item", + "quantity": 1, + "settleMs": 1, + "warehouse": "warehouse" + }, + { + "as": "stored-before", + "do": "dbRecordStock", + "item": "item" + }, + { + "do": "dbExpectStock", + "item": "item", + "plus": 0, + "relativeTo": "stored-before" + }, + { + "actor": "a", + "do": "ensureSignedIn", + "name": "alice" + }, + { + "actor": "a", + "do": "enterRoom", + "room": "room" + }, + { + "actor": "a", + "do": "expect", + "testid": "status" + }, + { + "actor": "a", + "do": "expectActionOutcome", + "outcome": "refused" + }, + { + "actors": [ + "a", + "b" + ], + "contains": "ok", + "do": "expectActorsWith", + "equals": 2, + "maxEach": 1, + "testid": "status" + }, + { + "actors": [ + "a", + "b" + ], + "do": "expectAgreement", + "testid": "status" + }, + { + "actor": "a", + "count": 1, + "do": "expectAllPresent", + "prefix": "message", + "within": 1 + }, + { + "accepted": 1, + "do": "expectCallOutcomes" + }, + { + "actor": "a", + "contains": "item", + "do": "expectElementCount", + "equals": 1, + "testid": "row", + "within": 1 + }, + { + "actor": "a", + "do": "expectForgeryRejected" + }, + { + "actor": "a", + "contains": "secret", + "do": "expectNotReceived" + }, + { + "actor": "a", + "as": "request", + "do": "markDelivery", + "idleMs": 1, + "phase": "start" + }, + { + "actor": "b", + "as": "request", + "do": "expectNotDelivered", + "from": "a", + "within": 1 + }, + { + "actor": "a", + "do": "expectNumber", + "equals": 1, + "testid": "count" + }, + { + "actor": "a", + "do": "expectWarehouseInventory", + "items": [ + { + "name": "item", + "stock": { + "East": 1, + "West": 0 + } + } + ], + "within": 1 + }, + { + "actors": [ + "a", + "b" + ], + "do": "expectOrderMatches", + "prefix": "message" + }, + { + "actor": "a", + "contains": "hello", + "do": "expectReceived", + "within": 1 + }, + { + "actor": "a", + "do": "expectReplayCompleted" + }, + { + "actor": "a", + "do": "expectReplayRejected" + }, + { + "actor": "a", + "do": "expectSequence", + "equals": [ + "one", + "two" + ], + "testid": "row" + }, + { + "actor": "a", + "do": "expectStable", + "intervalMs": 1, + "samples": 2, + "testid": "count" + }, + { + "actor": "a", + "do": "expectUnavailable", + "testid": "admin" + }, + { + "actor": "a", + "do": "fill", + "testid": "input", + "text": "value" + }, + { + "actor": "a", + "do": "forgeWrite", + "fromActor": "b", + "settleMs": 1 + }, + { + "actor": "a", + "do": "freshClient" + }, + { + "actor": "a", + "do": "openClient" + }, + { + "actor": "a", + "do": "openItem", + "item": "Keyboard" + }, + { + "actor": "a", + "do": "pressKey", + "key": "Enter" + }, + { + "branches": [ + [ + { + "actor": "a", + "do": "wait", + "ms": 1 + } + ], + [ + { + "actor": "b", + "do": "wait", + "ms": 1 + } + ] + ], + "do": "race", + "settleMs": 1 + }, + { + "actor": "a", + "as": "before", + "do": "recordNumber", + "testid": "count" + }, + { + "as": "started", + "do": "recordTime" + }, + { + "atMost": 1000, + "do": "expectElapsed", + "since": "started" + }, + { + "actor": "a", + "do": "reload", + "settleMs": 1 + }, + { + "actor": "a", + "control": "Item 1", + "do": "repeatFormWrite", + "fields": [ + { + "testid": "name", + "text": "{value}" + } + ], + "match": "Item 2", + "replacement": "{value}", + "settleMs": 1, + "submit": "save" + }, + { + "actor": "a", + "do": "replayAs", + "from": "b", + "match": "write" + }, + { + "actors": [ + "a", + "b" + ], + "do": "replayConcurrently", + "settleMs": 1 + }, + { + "do": "restartBackend", + "settleMs": 1 + }, + { + "args": [], + "do": "runScript", + "script": "fixture.mjs" + }, + { + "actor": "a", + "do": "send", + "text": "hello" + }, + { + "delayMs": 1, + "do": "sendConcurrently", + "senders": [ + { + "actor": "a", + "count": 1, + "prefix": "message-a" + }, + { + "actor": "b", + "count": 1, + "prefix": "message-b" + } + ] + }, + { + "actor": "a", + "count": 1, + "delayMs": 1, + "do": "sendMany", + "prefix": "message" + }, + { + "actor": "a", + "do": "setOffline", + "offline": true + }, + { + "actor": "a", + "do": "signIn", + "name": "alice" + }, + { + "actor": "a", + "do": "signUp", + "name": "alice" + }, + { + "do": "startAppServer" + }, + { + "do": "stopAppServer" + }, + { + "actor": "a", + "do": "typeInto", + "text": "value" + }, + { + "actor": "a", + "do": "wait", + "ms": 1 + }, + { + "actor": "a", + "do": "waitUntilAbsent", + "testid": "status" + }, + { + "before": [ + "before-a", + "before-b" + ], + "do": "dbExpectPurchaseCount", + "purchasesEach": 500 + }, + { + "before": "catalog-before", + "do": "dbExpectCatalogItem", + "name": "New product", + "priceMinor": 125 + } + ] + } + ], + "id": 1, + "max": 0, + "name": "Every registered action", + "setup": [] + } + ], + "level": 1, + "name": "Action language fixture", + "schemaVersion": 1 +} diff --git a/tools/stack-bench/tests/snapshots/definitions/chat.snapshot.json b/tools/stack-bench/tests/snapshots/definitions/chat.snapshot.json new file mode 100644 index 00000000000..43ca4d2009c --- /dev/null +++ b/tools/stack-bench/tests/snapshots/definitions/chat.snapshot.json @@ -0,0 +1,3680 @@ +{ + "definitionSchemaVersion": 1, + "levels": [ + { + "level": 1, + "suites": [ + { + "id": "features", + "inherit": "none", + "scenario": "scenarios/01-basic-chat.json" + }, + { + "id": "invariants", + "inherit": "all-higher-levels", + "scenario": "scenarios/01-invariants.json" + }, + { + "id": "delivery", + "inherit": "none", + "scenario": "scenarios/01-delivery.json" + }, + { + "id": "systems", + "inherit": "all-higher-levels", + "scenario": "scenarios/01-systems.json" + } + ] + }, + { + "level": 2, + "suites": [ + { + "id": "features", + "inherit": "none", + "scenario": "scenarios/02-features.json" + }, + { + "id": "invariants", + "inherit": "all-higher-levels", + "scenario": "scenarios/02-invariants.json" + }, + { + "id": "systems", + "inherit": "all-higher-levels", + "scenario": "scenarios/02-systems.json" + } + ] + } + ], + "manifest": { + "plannedThrough": 5, + "portOffset": 0, + "reseedOnReset": true, + "restartProbe": "/api/rooms", + "schemaVersion": 1, + "slug": "", + "suites": { + "1": [ + { + "id": "features", + "inherit": "none", + "spec": "scenarios/01-basic-chat.json" + }, + { + "id": "invariants", + "inherit": "all-higher-levels", + "spec": "scenarios/01-invariants.json" + }, + { + "id": "delivery", + "inherit": "none", + "spec": "scenarios/01-delivery.json" + }, + { + "id": "systems", + "inherit": "all-higher-levels", + "spec": "scenarios/01-systems.json" + } + ], + "2": [ + { + "id": "features", + "inherit": "none", + "spec": "scenarios/02-features.json" + }, + { + "id": "invariants", + "inherit": "all-higher-levels", + "spec": "scenarios/02-invariants.json" + }, + { + "id": "systems", + "inherit": "all-higher-levels", + "spec": "scenarios/02-systems.json" + } + ] + }, + "title": "Chat", + "validatedThrough": 0 + }, + "scenarios": [ + { + "definition": { + "features": [ + { + "actors": [ + "alice", + "bob" + ], + "criteria": [ + { + "desc": "Creating an account signs the user in and shows their username", + "id": "signup-signs-in", + "points": 1, + "steps": [ + { + "actor": "alice", + "contains": "Alice", + "do": "expect", + "testid": "current-user", + "within": 6000 + } + ] + }, + { + "desc": "Signing up with a taken username is rejected with a visible error, and must not sign the visitor in as the existing account", + "id": "duplicate-username-rejected", + "points": 1, + "steps": [ + { + "actor": "bob", + "do": "signUp", + "expectFailure": true, + "name": "Alice" + }, + { + "actor": "bob", + "do": "expect", + "testid": "auth-error", + "within": 6000 + }, + { + "absent": true, + "actor": "bob", + "do": "expect", + "testid": "current-user", + "within": 3000 + } + ] + }, + { + "desc": "A user can sign out and sign back in with the same credentials", + "id": "signout-and-signin", + "points": 1, + "steps": [ + { + "actor": "alice", + "do": "click", + "testid": "signout" + }, + { + "actor": "alice", + "do": "waitUntilAbsent", + "testid": "current-user", + "within": 6000 + }, + { + "actor": "alice", + "do": "signIn", + "name": "Alice" + }, + { + "actor": "alice", + "contains": "Alice", + "do": "expect", + "testid": "current-user", + "within": 6000 + } + ] + } + ], + "id": 1, + "name": "Accounts", + "setup": [ + { + "actor": "alice", + "do": "signUp", + "name": "Alice" + } + ] + }, + { + "actors": [ + "alice", + "bob" + ], + "criteria": [ + { + "desc": "A sent message appears in the sender's own message list", + "id": "own-message-appears", + "points": 1, + "steps": [ + { + "actor": "alice", + "do": "send", + "text": "hello from alice" + }, + { + "actor": "alice", + "contains": "hello from alice", + "do": "expect", + "testid": "message-item", + "within": 5000 + } + ] + }, + { + "desc": "A second user in the same room receives messages live, with no reload", + "id": "cross-user-realtime", + "points": 1, + "steps": [ + { + "actor": "bob", + "do": "enterRoom", + "room": "general" + }, + { + "actor": "bob", + "contains": "hello from alice", + "do": "expect", + "testid": "message-item", + "within": 5000 + }, + { + "actor": "bob", + "do": "send", + "text": "hello from bob" + }, + { + "actor": "alice", + "contains": "hello from bob", + "do": "expect", + "testid": "message-item", + "within": 3000 + } + ] + }, + { + "desc": "Online users lists both participants; leaving a room works", + "id": "presence-and-leave", + "points": 1, + "steps": [ + { + "actor": "alice", + "contains": "Bob", + "do": "expect", + "testid": "online-users", + "within": 5000 + }, + { + "actor": "bob", + "do": "click", + "testid": "leave-room" + }, + { + "actor": "bob", + "do": "waitUntilAbsent", + "testid": "message-input", + "within": 5000 + } + ] + } + ], + "id": 2, + "name": "Basic Chat", + "setup": [ + { + "actor": "alice", + "do": "signUp", + "name": "Alice" + }, + { + "actor": "bob", + "do": "signUp", + "name": "Bob" + }, + { + "actor": "alice", + "do": "createRoom", + "room": "general" + }, + { + "actor": "alice", + "do": "enterRoom", + "room": "general" + } + ] + }, + { + "actors": [ + "alice", + "bob", + "carol" + ], + "criteria": [ + { + "desc": "Typing by another user in the same room shows an indicator live", + "id": "indicator-appears", + "points": 1, + "steps": [ + { + "actor": "bob", + "do": "typeInto", + "text": "composing a reply" + }, + { + "actor": "alice", + "contains": "typing", + "do": "expect", + "testid": "typing-indicator", + "within": 4000 + } + ] + }, + { + "desc": "Indicator disappears within 6s once typing stops", + "id": "indicator-expires", + "points": 1, + "steps": [ + { + "actor": "bob", + "do": "clearInput" + }, + { + "actor": "alice", + "contains": "typing", + "do": "waitUntilAbsent", + "testid": "typing-indicator", + "within": 6500 + } + ] + }, + { + "desc": "Typing is scoped to the room — a user in another room sees nothing", + "id": "scoped-to-room", + "points": 1, + "steps": [ + { + "actor": "bob", + "do": "typeInto", + "text": "still typing here" + }, + { + "actor": "alice", + "contains": "typing", + "do": "expect", + "testid": "typing-indicator", + "within": 4000 + }, + { + "absent": true, + "actor": "carol", + "contains": "typing", + "do": "expect", + "testid": "typing-indicator", + "within": 2000 + } + ] + } + ], + "id": 3, + "name": "Typing Indicators", + "setup": [ + { + "actor": "alice", + "do": "signUp", + "name": "Alice" + }, + { + "actor": "bob", + "do": "signUp", + "name": "Bob" + }, + { + "actor": "carol", + "do": "signUp", + "name": "Carol" + }, + { + "actor": "alice", + "do": "createRoom", + "room": "typing-a" + }, + { + "actor": "carol", + "do": "createRoom", + "room": "typing-b" + }, + { + "actor": "alice", + "do": "enterRoom", + "room": "typing-a" + }, + { + "actor": "bob", + "do": "enterRoom", + "room": "typing-a" + }, + { + "actor": "carol", + "do": "enterRoom", + "room": "typing-b" + } + ] + }, + { + "actors": [ + "alice", + "bob" + ], + "criteria": [ + { + "desc": "Receipt appears for the sender once another user views the message", + "id": "receipt-appears", + "points": 1, + "steps": [ + { + "actor": "bob", + "do": "enterRoom", + "room": "receipts" + }, + { + "actor": "alice", + "do": "expect", + "testid": "read-receipt", + "within": 6000 + } + ] + }, + { + "desc": "Receipt names the user who read it", + "id": "receipt-names-reader", + "points": 1, + "steps": [ + { + "actor": "alice", + "contains": "Bob", + "do": "expect", + "testid": "read-receipt", + "within": 6000 + } + ] + }, + { + "desc": "The sender is not listed as a reader of their own message", + "id": "sender-excluded", + "points": 1, + "steps": [ + { + "actor": "alice", + "do": "expect", + "notContains": "Alice", + "testid": "read-receipt", + "within": 6000 + } + ] + } + ], + "id": 4, + "name": "Read Receipts", + "setup": [ + { + "actor": "alice", + "do": "signUp", + "name": "Alice" + }, + { + "actor": "bob", + "do": "signUp", + "name": "Bob" + }, + { + "actor": "alice", + "do": "createRoom", + "room": "receipts" + }, + { + "actor": "alice", + "do": "enterRoom", + "room": "receipts" + }, + { + "actor": "alice", + "do": "send", + "text": "did you see this" + } + ] + }, + { + "actors": [ + "alice", + "bob" + ], + "criteria": [ + { + "desc": "A badge appears live on a room with unread messages", + "id": "badge-appears", + "points": 1, + "steps": [ + { + "actor": "alice", + "do": "send", + "text": "unread one" + }, + { + "actor": "bob", + "do": "expect", + "in": { + "contains": "{room:unread-main}", + "testid": "room-item" + }, + "testid": "unread-badge", + "within": 5000 + } + ] + }, + { + "desc": "The badge reflects the number of unread messages (paced above the app's send rate limit)", + "id": "badge-counts", + "points": 1, + "steps": [ + { + "actor": "alice", + "do": "wait", + "ms": 1400 + }, + { + "actor": "alice", + "do": "send", + "text": "unread two" + }, + { + "actor": "bob", + "contains": "2", + "do": "expect", + "in": { + "contains": "{room:unread-main}", + "testid": "room-item" + }, + "testid": "unread-badge", + "within": 5000 + } + ] + }, + { + "desc": "The badge clears when the room is opened", + "id": "badge-clears", + "points": 1, + "steps": [ + { + "actor": "bob", + "do": "enterRoom", + "room": "unread-main" + }, + { + "actor": "bob", + "do": "waitUntilAbsent", + "in": { + "contains": "{room:unread-main}", + "testid": "room-item" + }, + "testid": "unread-badge", + "within": 5000 + } + ] + } + ], + "id": 5, + "name": "Unread Message Counts", + "setup": [ + { + "actor": "alice", + "do": "signUp", + "name": "Alice" + }, + { + "actor": "bob", + "do": "signUp", + "name": "Bob" + }, + { + "actor": "alice", + "do": "createRoom", + "room": "unread-main" + }, + { + "actor": "bob", + "do": "createRoom", + "room": "unread-side" + }, + { + "actor": "alice", + "do": "enterRoom", + "room": "unread-main" + }, + { + "actor": "bob", + "do": "enterRoom", + "room": "unread-main" + }, + { + "actor": "bob", + "do": "enterRoom", + "room": "unread-side" + } + ] + }, + { + "actors": [ + "alice", + "bob" + ], + "criteria": [ + { + "desc": "Someone joining a room appears in its member panel without anyone reloading", + "id": "members-appear-live", + "points": 1, + "steps": [ + { + "actor": "bob", + "do": "enterRoom", + "room": "team" + }, + { + "actor": "alice", + "contains": "Bob", + "do": "expect", + "testid": "member-item", + "within": 8000 + } + ] + }, + { + "desc": "The room's creator is marked as its owner", + "id": "owner-is-shown", + "points": 1, + "steps": [ + { + "actor": "alice", + "do": "expect", + "testid": "member-owner", + "within": 6000 + } + ] + }, + { + "desc": "Removing a member takes them out of the panel live, for everyone still in the room", + "id": "removed-member-disappears", + "points": 1, + "steps": [ + { + "actor": "alice", + "do": "click", + "in": { + "contains": "Bob", + "testid": "member-item" + }, + "testid": "member-remove" + }, + { + "actor": "alice", + "contains": "Bob", + "do": "waitUntilAbsent", + "testid": "member-item", + "within": 8000 + } + ] + } + ], + "id": 6, + "name": "Room Members and Roles", + "setup": [ + { + "actor": "alice", + "do": "signUp", + "name": "Alice" + }, + { + "actor": "bob", + "do": "signUp", + "name": "Bob" + }, + { + "actor": "alice", + "do": "createRoom", + "room": "team" + }, + { + "actor": "alice", + "do": "enterRoom", + "room": "team" + } + ] + }, + { + "actors": [ + "alice", + "bob" + ], + "criteria": [ + { + "desc": "A user can open a direct conversation with another user", + "id": "dm-can-be-started", + "points": 1, + "steps": [ + { + "actor": "alice", + "do": "click", + "in": { + "contains": "Bob", + "testid": "online-user" + }, + "testid": "dm-start" + }, + { + "actor": "alice", + "contains": "Bob", + "do": "expect", + "testid": "dm-item", + "within": 8000 + } + ] + }, + { + "desc": "A direct message reaches the other participant live, with no reload", + "id": "dm-delivers-live", + "points": 1, + "steps": [ + { + "actor": "alice", + "do": "click", + "in": { + "contains": "Bob", + "testid": "dm-list" + }, + "testid": "dm-item" + }, + { + "actor": "bob", + "contains": "Alice", + "do": "click", + "testid": "dm-item" + }, + { + "actor": "alice", + "do": "send", + "text": "a private word" + }, + { + "actor": "bob", + "contains": "a private word", + "do": "expect", + "testid": "message-item", + "within": 10000 + } + ] + }, + { + "desc": "A direct conversation and its messages survive a reload", + "id": "dm-persists", + "points": 1, + "steps": [ + { + "actor": "bob", + "do": "reload", + "settleMs": 3500 + }, + { + "actor": "bob", + "contains": "Alice", + "do": "expect", + "testid": "dm-item", + "within": 10000 + }, + { + "actor": "bob", + "contains": "Alice", + "do": "click", + "testid": "dm-item" + }, + { + "actor": "bob", + "contains": "a private word", + "do": "expect", + "testid": "message-item", + "within": 10000 + } + ] + } + ], + "id": 7, + "name": "Direct Messages", + "setup": [ + { + "actor": "alice", + "do": "signUp", + "name": "Alice" + }, + { + "actor": "bob", + "do": "signUp", + "name": "Bob" + }, + { + "actor": "alice", + "do": "wait", + "ms": 2000 + } + ] + }, + { + "actors": [ + "alice", + "bob" + ], + "criteria": [ + { + "desc": "The author of a message is offered edit and delete on it", + "id": "author-can-edit-and-delete", + "points": 1, + "steps": [ + { + "actor": "alice", + "do": "expect", + "in": { + "contains": "a message of my own", + "testid": "message-item" + }, + "testid": "message-edit", + "within": 6000 + }, + { + "actor": "alice", + "do": "expect", + "in": { + "contains": "a message of my own", + "testid": "message-item" + }, + "testid": "message-delete", + "within": 6000 + } + ] + }, + { + "desc": "Nobody else is offered edit or delete on that message", + "id": "others-are-not", + "points": 1, + "steps": [ + { + "actor": "bob", + "contains": "a message of my own", + "do": "expect", + "testid": "message-item", + "within": 8000 + }, + { + "absent": true, + "actor": "bob", + "do": "expect", + "in": { + "contains": "a message of my own", + "testid": "message-item" + }, + "testid": "message-edit", + "within": 5000 + }, + { + "absent": true, + "actor": "bob", + "do": "expect", + "in": { + "contains": "a message of my own", + "testid": "message-item" + }, + "testid": "message-delete", + "within": 5000 + } + ] + } + ], + "id": 8, + "name": "Message Ownership", + "setup": [ + { + "actor": "alice", + "do": "signUp", + "name": "Alice" + }, + { + "actor": "bob", + "do": "signUp", + "name": "Bob" + }, + { + "actor": "alice", + "do": "createRoom", + "room": "owned" + }, + { + "actor": "alice", + "do": "enterRoom", + "room": "owned" + }, + { + "actor": "bob", + "do": "enterRoom", + "room": "owned" + }, + { + "actor": "alice", + "do": "send", + "text": "a message of my own" + } + ] + } + ], + "level": 1, + "note": "Level 1 is sequential's basic chat with accounts added. Accounts replace the display-name stub; typing indicators, read receipts and unread counts are sequential's, unchanged apart from signing up rather than naming yourself.", + "schemaVersion": 1 + }, + "path": "scenarios/01-basic-chat.json" + }, + { + "definition": { + "features": [ + { + "actors": [ + "alice", + "bob" + ], + "criteria": [ + { + "desc": "Under a concurrent write storm from two clients, the sender sees every message exactly once — no drops, no double-render from optimistic insert plus broadcast echo", + "id": "no-loss-no-duplication-sender", + "points": 1, + "steps": [ + { + "delayMs": 1400, + "do": "sendConcurrently", + "senders": [ + { + "actor": "alice", + "count": 6, + "prefix": "AA" + }, + { + "actor": "bob", + "count": 6, + "prefix": "BB" + } + ] + }, + { + "actor": "alice", + "count": 6, + "do": "expectAllPresent", + "prefix": "AA", + "within": 20000 + } + ] + }, + { + "desc": "The receiving client also observes every message from the other sender exactly once", + "id": "no-loss-no-duplication-receiver", + "points": 1, + "steps": [ + { + "actor": "alice", + "count": 6, + "do": "expectAllPresent", + "prefix": "BB", + "within": 20000 + }, + { + "actor": "bob", + "count": 6, + "do": "expectAllPresent", + "prefix": "AA", + "within": 20000 + } + ] + }, + { + "desc": "Both clients observe the interleaved messages in the SAME order. Non-atomic broadcast lets concurrent senders interleave differently per client.", + "id": "consistent-order-across-clients", + "points": 1, + "steps": [ + { + "actors": [ + "alice", + "bob" + ], + "do": "expectOrderMatches", + "prefix": "(?:AA|BB)" + } + ] + } + ], + "id": 201, + "max": 3, + "name": "Delivery Integrity", + "setup": [ + { + "actor": "alice", + "do": "signUp", + "name": "Alice" + }, + { + "actor": "bob", + "do": "signUp", + "name": "Bob" + }, + { + "actor": "alice", + "do": "createRoom", + "room": "storm" + }, + { + "actor": "alice", + "do": "enterRoom", + "room": "storm" + }, + { + "actor": "bob", + "do": "enterRoom", + "room": "storm" + } + ] + }, + { + "actors": [ + "alice", + "bob" + ], + "criteria": [ + { + "desc": "A client that loses its connection and regains it must catch up on everything it missed, WITHOUT a reload.", + "id": "resync-after-disconnect", + "points": 1, + "steps": [ + { + "actor": "alice", + "do": "setOffline", + "offline": true, + "settleMs": 1500 + }, + { + "actor": "bob", + "count": 3, + "delayMs": 1400, + "do": "sendMany", + "prefix": "OFFLINE" + }, + { + "actor": "alice", + "do": "setOffline", + "offline": false, + "settleMs": 3000 + }, + { + "actor": "alice", + "count": 3, + "do": "expectAllPresent", + "prefix": "OFFLINE", + "within": 15000 + } + ] + }, + { + "desc": "Catching up must not double-deliver messages the client already had (resubscribe replaying history on top of existing state)", + "id": "no-duplication-after-resync", + "points": 1, + "steps": [ + { + "actor": "bob", + "count": 2, + "delayMs": 1400, + "do": "sendMany", + "prefix": "AFTER" + }, + { + "actor": "alice", + "count": 2, + "do": "expectAllPresent", + "prefix": "AFTER", + "within": 10000 + }, + { + "actor": "alice", + "count": 3, + "do": "expectAllPresent", + "prefix": "OFFLINE", + "within": 5000 + }, + { + "actor": "alice", + "count": 2, + "do": "expectAllPresent", + "prefix": "BEFORE", + "within": 10000 + } + ] + } + ], + "id": 202, + "max": 2, + "name": "Connection Resilience", + "setup": [ + { + "actor": "alice", + "do": "signUp", + "name": "Alice" + }, + { + "actor": "bob", + "do": "signUp", + "name": "Bob" + }, + { + "actor": "alice", + "do": "createRoom", + "room": "resilience" + }, + { + "actor": "alice", + "do": "enterRoom", + "room": "resilience" + }, + { + "actor": "bob", + "do": "enterRoom", + "room": "resilience" + }, + { + "actor": "bob", + "count": 2, + "delayMs": 1400, + "do": "sendMany", + "prefix": "BEFORE" + }, + { + "actor": "alice", + "count": 2, + "do": "expectAllPresent", + "prefix": "BEFORE", + "within": 10000 + } + ] + } + ], + "level": 1, + "note": "Delivery and resilience invariants — properties of the transport, not of any feature. These are the ones a human grader cannot test at all (you cannot generate concurrency or simulate a network drop by clicking), and where the asymmetry is structural: Express stacks broadcast with io.emit() AFTER awaiting the write, so delivery is not atomic with the commit; a SpacetimeDB subscription update IS the commit. Senders pace themselves above the apps' rate limits (observed: 400ms SpacetimeDB, 350ms Mongo) — the spec requires spam prevention, so a 0-delay burst is correctly rejected and would test nothing. Concurrency comes from multiple senders overlapping, not from one sender bursting. Actors sign up with credentials, as level 1 requires.", + "schemaVersion": 1 + }, + "path": "scenarios/01-delivery.json" + }, + { + "definition": { + "features": [ + { + "actors": [ + "victim", + "impostor" + ], + "criteria": [ + { + "desc": "Knowing a username must not grant access to that account. Signing up with a taken username, or signing in with the wrong password, must be rejected — not silently resolved to the existing account.", + "id": "username-is-not-a-credential", + "points": 1, + "steps": [ + { + "actor": "impostor", + "do": "signUp", + "expectFailure": true, + "name": "Victim" + }, + { + "absent": true, + "actor": "impostor", + "contains": "Victim", + "do": "expect", + "testid": "current-user", + "within": 5000 + } + ] + }, + { + "desc": "Signing in with a wrong password is rejected and grants nothing", + "id": "wrong-password-rejected", + "points": 1, + "steps": [ + { + "actor": "impostor", + "do": "signIn", + "expectFailure": true, + "name": "Victim", + "password": "not-the-password" + }, + { + "actor": "impostor", + "do": "expect", + "testid": "auth-error", + "within": 6000 + }, + { + "absent": true, + "actor": "impostor", + "contains": "victim private message", + "do": "expect", + "testid": "message-item", + "within": 4000 + } + ] + } + ], + "id": 101, + "max": 2, + "name": "Account Integrity", + "setup": [ + { + "actor": "victim", + "do": "signUp", + "name": "Victim" + }, + { + "actor": "victim", + "do": "createRoom", + "room": "private-notes" + }, + { + "actor": "victim", + "do": "enterRoom", + "room": "private-notes" + }, + { + "actor": "victim", + "do": "send", + "text": "victim private message" + } + ] + }, + { + "actors": [ + "alice", + "bob" + ], + "criteria": [ + { + "desc": "Reloading keeps the user signed in as the same account", + "id": "session-survives-reload", + "points": 1, + "steps": [ + { + "actor": "alice", + "do": "reload", + "settleMs": 3000 + }, + { + "actor": "alice", + "contains": "Alice", + "do": "expect", + "testid": "current-user", + "within": 8000 + } + ] + }, + { + "desc": "A user who loses and regains their connection while sitting on the page is still signed in as the same account. The connection drops and re-establishes on its own; re-identifying the client here would orphan everything it owns.", + "id": "session-survives-reconnect", + "points": 1, + "steps": [ + { + "actor": "alice", + "do": "setOffline", + "offline": true, + "settleMs": 3000 + }, + { + "actor": "alice", + "do": "setOffline", + "offline": false, + "settleMs": 6000 + }, + { + "actor": "alice", + "contains": "Alice", + "do": "expect", + "testid": "current-user", + "within": 20000 + } + ] + }, + { + "desc": "After reconnecting, the room owner can still remove a member, and that member loses access.", + "id": "own-content-still-owned-after-reconnect", + "points": 1, + "steps": [ + { + "actor": "alice", + "do": "enterRoom", + "room": "mine" + }, + { + "actor": "alice", + "contains": "alice message", + "do": "expect", + "testid": "message-item", + "within": 15000 + }, + { + "actor": "alice", + "do": "click", + "in": { + "contains": "Bob", + "testid": "member-item" + }, + "testid": "member-remove" + }, + { + "actor": "alice", + "contains": "Bob", + "do": "waitUntilAbsent", + "testid": "member-item", + "within": 10000 + }, + { + "actor": "bob", + "do": "waitUntilAbsent", + "testid": "message-input", + "within": 10000 + } + ] + } + ], + "id": 102, + "max": 3, + "name": "Session Durability", + "note": "Session durability is tested by dropping the client's connection, not by restarting the backend. Both force the same reconnect, but only one needs a server the harness owns — and restarting a shared host takes unrelated databases down with it. Backend restarts belong where they test storage rather than session handling (level 4).", + "setup": [ + { + "actor": "alice", + "do": "signUp", + "name": "Alice" + }, + { + "actor": "bob", + "do": "signUp", + "name": "Bob" + }, + { + "actor": "alice", + "do": "createRoom", + "room": "mine" + }, + { + "actor": "alice", + "do": "enterRoom", + "room": "mine" + }, + { + "actor": "alice", + "do": "send", + "text": "alice message" + }, + { + "actor": "bob", + "do": "enterRoom", + "room": "mine" + } + ] + }, + { + "actors": [ + "alice", + "bob" + ], + "criteria": [ + { + "desc": "A message sent in one room never appears in another", + "id": "messages-scoped-to-room", + "points": 1, + "steps": [ + { + "absent": true, + "actor": "bob", + "contains": "message for room a", + "do": "expect", + "testid": "message-item", + "within": 5000 + } + ] + }, + { + "desc": "A room renders exactly once in the list — an optimistic insert plus a broadcast echo duplicates it", + "id": "no-duplicate-rooms", + "points": 1, + "steps": [ + { + "actor": "alice", + "contains": "{room:room-a}", + "count": 1, + "do": "expect", + "testid": "room-item", + "within": 5000 + } + ] + } + ], + "id": 103, + "max": 2, + "name": "Data Isolation", + "setup": [ + { + "actor": "alice", + "do": "signUp", + "name": "Alice" + }, + { + "actor": "bob", + "do": "signUp", + "name": "Bob" + }, + { + "actor": "alice", + "do": "createRoom", + "room": "room-a" + }, + { + "actor": "bob", + "do": "createRoom", + "room": "room-b" + }, + { + "actor": "alice", + "do": "enterRoom", + "room": "room-a" + }, + { + "actor": "bob", + "do": "enterRoom", + "room": "room-b" + }, + { + "actor": "alice", + "do": "send", + "text": "message for room a" + } + ] + }, + { + "actors": [ + "alice", + "bob" + ], + "criteria": [ + { + "desc": "Your own message must never count as unread for you. (historical: a sender's own badge went to 1 on SpacetimeDB, twice)", + "id": "own-messages-not-unread", + "points": 1, + "steps": [ + { + "actor": "alice", + "do": "send", + "text": "alice speaking" + }, + { + "actor": "alice", + "do": "wait", + "ms": 1400 + }, + { + "actor": "alice", + "do": "enterRoom", + "room": "side" + }, + { + "absent": true, + "actor": "alice", + "do": "expect", + "in": { + "contains": "{room:main}", + "testid": "room-item" + }, + "testid": "unread-badge", + "within": 5000 + } + ] + }, + { + "desc": "Once a room is open its badge stays clear while new messages arrive in it — you are looking at them. (historical: badges that never cleared)", + "id": "unread-does-not-resurrect", + "points": 1, + "steps": [ + { + "actor": "alice", + "do": "enterRoom", + "room": "main" + }, + { + "actor": "alice", + "do": "wait", + "ms": 1400 + }, + { + "actor": "bob", + "do": "send", + "text": "bob speaking" + }, + { + "actor": "alice", + "contains": "bob speaking", + "do": "expect", + "testid": "message-item", + "within": 6000 + }, + { + "absent": true, + "actor": "alice", + "do": "expect", + "in": { + "contains": "{room:main}", + "testid": "room-item" + }, + "testid": "unread-badge", + "within": 4000 + } + ] + }, + { + "desc": "Receipts stay live for later messages, not just the first one. (historical: 'Seen by' that only appeared after leaving and re-entering the room)", + "id": "receipts-keep-updating", + "points": 1, + "steps": [ + { + "actor": "alice", + "do": "wait", + "ms": 1400 + }, + { + "actor": "alice", + "do": "send", + "text": "second message from alice" + }, + { + "actor": "bob", + "contains": "second message from alice", + "do": "expect", + "testid": "message-item", + "within": 6000 + }, + { + "actor": "alice", + "contains": "Bob", + "do": "expect", + "in": { + "contains": "second message from alice", + "testid": "message-item" + }, + "testid": "read-receipt", + "within": 10000 + } + ] + } + ], + "id": 104, + "max": 3, + "name": "Per-User State Integrity", + "note": "Unread counts and read receipts are per-user state derived from other people's actions. The historical grading data has more defects here than anywhere else in level 1.", + "setup": [ + { + "actor": "alice", + "do": "signUp", + "name": "Alice" + }, + { + "actor": "bob", + "do": "signUp", + "name": "Bob" + }, + { + "actor": "alice", + "do": "createRoom", + "room": "main" + }, + { + "actor": "alice", + "do": "createRoom", + "room": "side" + }, + { + "actor": "alice", + "do": "enterRoom", + "room": "main" + }, + { + "actor": "bob", + "do": "enterRoom", + "room": "main" + } + ] + }, + { + "actors": [ + "alice", + "bob", + "carol" + ], + "criteria": [ + { + "desc": "The 'Seen by' list must not change while nothing is happening. (historical: names flipping order continuously from an unstable sort)", + "id": "receipt-text-does-not-flicker", + "points": 1, + "steps": [ + { + "actor": "alice", + "contains": "Bob", + "do": "expect", + "testid": "read-receipt", + "within": 10000 + }, + { + "actor": "alice", + "contains": "Carol", + "do": "expect", + "testid": "read-receipt", + "within": 10000 + }, + { + "actor": "alice", + "do": "expectStable", + "intervalMs": 700, + "samples": 4, + "testid": "read-receipt" + } + ] + }, + { + "desc": "A user who signs out disappears from the online list. (historical: presence lists stale or stuck, six runs)", + "id": "presence-drops-signed-out-user", + "points": 1, + "steps": [ + { + "actor": "alice", + "contains": "Bob", + "do": "expect", + "testid": "online-users", + "within": 8000 + }, + { + "actor": "bob", + "do": "click", + "testid": "signout" + }, + { + "actor": "alice", + "contains": "Bob", + "do": "waitUntilAbsent", + "testid": "online-users", + "within": 12000 + } + ] + } + ], + "id": 105, + "max": 2, + "name": "Presence and Display Stability", + "note": "Presence was wrong in six of the historically graded runs, and 'Seen by' lists were seen flickering between orderings. Neither is visible to a single assertion.", + "setup": [ + { + "actor": "alice", + "do": "signUp", + "name": "Alice" + }, + { + "actor": "bob", + "do": "signUp", + "name": "Bob" + }, + { + "actor": "carol", + "do": "signUp", + "name": "Carol" + }, + { + "actor": "alice", + "do": "createRoom", + "room": "lobby" + }, + { + "actor": "alice", + "do": "enterRoom", + "room": "lobby" + }, + { + "actor": "bob", + "do": "enterRoom", + "room": "lobby" + }, + { + "actor": "carol", + "do": "enterRoom", + "room": "lobby" + }, + { + "actor": "alice", + "do": "send", + "text": "who is here" + } + ] + }, + { + "actors": [ + "alice", + "bob" + ], + "criteria": [ + { + "desc": "A message the server refused for exceeding the rate limit must not stay visible to its sender. An optimistic insert without rollback leaves the sender believing they said something nobody received.", + "id": "rejected-message-does-not-linger", + "points": 1, + "steps": [ + { + "actor": "alice", + "do": "send", + "text": "first of a burst" + }, + { + "actor": "alice", + "do": "send", + "text": "REJECTED BY RATE LIMIT" + }, + { + "actor": "alice", + "do": "wait", + "ms": 4000 + }, + { + "absent": true, + "actor": "bob", + "contains": "REJECTED BY RATE LIMIT", + "do": "expect", + "testid": "message-item", + "within": 4000 + }, + { + "absent": true, + "actor": "alice", + "contains": "REJECTED BY RATE LIMIT", + "do": "expect", + "testid": "message-item", + "within": 4000 + } + ] + }, + { + "desc": "A brand-new browser sees what the acting client claims. Anything visible only to the client that performed the action never reached the server.", + "id": "a-fresh-client-sees-the-same-thing", + "points": 1, + "steps": [ + { + "actor": "alice", + "do": "wait", + "ms": 1500 + }, + { + "actor": "alice", + "do": "send", + "text": "durable statement" + }, + { + "actor": "alice", + "contains": "durable statement", + "do": "expect", + "testid": "message-item", + "within": 6000 + }, + { + "actor": "bob", + "do": "freshClient" + }, + { + "actor": "bob-fresh", + "do": "signUp", + "name": "Observer" + }, + { + "actor": "bob-fresh", + "do": "enterRoom", + "room": "truth" + }, + { + "actor": "bob-fresh", + "contains": "durable statement", + "do": "expect", + "testid": "message-item", + "within": 10000 + } + ] + } + ], + "id": 107, + "max": 2, + "name": "Server-Side Truth", + "note": "Whatever the acting client shows must actually exist on the server. Optimistic inserts that were never confirmed, and counts incremented only in one browser, are invisible to any assertion made from that same browser.", + "setup": [ + { + "actor": "alice", + "do": "signUp", + "name": "Alice" + }, + { + "actor": "bob", + "do": "signUp", + "name": "Bob" + }, + { + "actor": "alice", + "do": "createRoom", + "room": "truth" + }, + { + "actor": "alice", + "do": "enterRoom", + "room": "truth" + }, + { + "actor": "bob", + "do": "enterRoom", + "room": "truth" + } + ] + }, + { + "actors": [ + "alice", + "bob", + "snoop" + ], + "criteria": [ + { + "desc": "A third party does not see the conversation in their list", + "id": "outsider-cannot-see-the-conversation", + "points": 1, + "steps": [ + { + "absent": true, + "actor": "snoop", + "contains": "Alice", + "do": "expect", + "testid": "dm-item", + "within": 6000 + } + ] + }, + { + "desc": "A third party never sees the messages", + "id": "outsider-cannot-read-the-messages", + "points": 1, + "steps": [ + { + "absent": true, + "actor": "snoop", + "contains": "SECRET between alice and bob", + "do": "expect", + "testid": "message-item", + "within": 6000 + } + ] + }, + { + "desc": "A third party's client is never SENT the private message. Hiding it in the interface is not privacy — anyone with developer tools can read what their own browser received.", + "id": "outsider-never-receives-it-over-the-wire", + "points": 1, + "steps": [ + { + "actor": "snoop", + "contains": "a public message in the lobby", + "do": "expectReceived", + "within": 8000 + }, + { + "actor": "bob", + "do": "click", + "in": { + "contains": "Alice", + "testid": "dm-list" + }, + "testid": "dm-item" + }, + { + "actor": "bob", + "do": "wait", + "ms": 2000 + }, + { + "actor": "alice", + "do": "send", + "text": "SECOND SECRET for bob only" + }, + { + "actor": "alice", + "do": "wait", + "ms": 4000 + }, + { + "actor": "bob", + "contains": "SECOND SECRET for bob only", + "do": "expectReceived", + "within": 8000 + }, + { + "actor": "snoop", + "contains": "SECOND SECRET for bob only", + "do": "expectNotReceived" + } + ] + } + ], + "id": 109, + "max": 3, + "name": "Direct Message Privacy", + "note": "A direct conversation is the first data in level 1 that must not reach someone. The wire check is the one that matters: an app that broadcasts widely and filters in the browser looks perfect on screen while the data is already in every client.", + "setup": [ + { + "actor": "alice", + "do": "signUp", + "name": "Alice" + }, + { + "actor": "bob", + "do": "signUp", + "name": "Bob" + }, + { + "actor": "snoop", + "do": "signUp", + "name": "Snoop" + }, + { + "actor": "alice", + "do": "createRoom", + "room": "lobby" + }, + { + "actor": "alice", + "do": "enterRoom", + "room": "lobby" + }, + { + "actor": "snoop", + "do": "enterRoom", + "room": "lobby" + }, + { + "actor": "alice", + "do": "send", + "text": "a public message in the lobby" + }, + { + "actor": "alice", + "do": "wait", + "ms": 2500 + }, + { + "actor": "alice", + "do": "click", + "in": { + "contains": "Bob", + "testid": "online-user" + }, + "testid": "dm-start" + }, + { + "actor": "alice", + "do": "wait", + "ms": 2000 + }, + { + "actor": "alice", + "do": "click", + "in": { + "contains": "Bob", + "testid": "dm-list" + }, + "testid": "dm-item" + }, + { + "actor": "alice", + "do": "send", + "text": "SECRET between alice and bob" + } + ] + }, + { + "actors": [ + "owner", + "member", + "target", + "outsider" + ], + "criteria": [ + { + "desc": "Someone who never joined the room cannot read its history", + "id": "outsider-cannot-read-the-room", + "points": 1, + "steps": [ + { + "absent": true, + "actor": "outsider", + "contains": "owner message", + "do": "expect", + "testid": "message-item", + "within": 6000 + } + ] + }, + { + "desc": "Someone who never joined cannot post to the room, even by re-issuing a member's own send request", + "id": "outsider-cannot-post-to-the-room", + "points": 1, + "steps": [ + { + "actor": "outsider", + "do": "replayAs", + "from": "owner", + "match": "owner message", + "swap": { + "find": "owner message", + "with": "posted by an outsider" + } + }, + { + "actor": "outsider", + "do": "expectReplayRejected" + }, + { + "absent": true, + "actor": "owner", + "contains": "posted by an outsider", + "do": "expect", + "testid": "message-item", + "within": 6000 + } + ] + }, + { + "desc": "A removed user stops receiving the room's messages immediately, without reloading. Revoking access has to stop the stream, not just stop the interface drawing it.", + "id": "removed-member-stops-receiving", + "points": 1, + "steps": [ + { + "actor": "target", + "contains": "owner message", + "do": "expectReceived", + "within": 8000 + }, + { + "actor": "owner", + "do": "click", + "in": { + "contains": "Target", + "testid": "member-item" + }, + "testid": "member-remove" + }, + { + "actor": "owner", + "do": "wait", + "ms": 3000 + }, + { + "actor": "owner", + "do": "send", + "text": "after the removal" + }, + { + "actor": "owner", + "do": "wait", + "ms": 4000 + }, + { + "actor": "target", + "contains": "after the removal", + "do": "expectNotReceived" + } + ] + }, + { + "desc": "A member who is not an owner cannot remove anyone — not through the interface, and not by re-issuing the owner's own removal request against a different person", + "id": "only-an-owner-can-remove-a-member", + "points": 1, + "steps": [ + { + "absent": true, + "actor": "member", + "do": "expect", + "testid": "member-remove", + "within": 5000 + }, + { + "actor": "member", + "do": "replayAs", + "from": "owner", + "match": "member", + "swap": { + "find": "{user:Target}", + "with": "{user:Owner}" + } + }, + { + "actor": "member", + "do": "expectReplayRejected" + }, + { + "actor": "owner", + "contains": "Owner", + "do": "expect", + "testid": "member-item", + "within": 6000 + } + ] + } + ], + "id": 108, + "max": 4, + "name": "Authorization", + "note": "HTTP replays test authenticated server refusal when a matching request can be captured. Without an observable replay path, the criterion is inconclusive. DOM-only criteria establish interface behavior, not server authorization.", + "setup": [ + { + "actor": "owner", + "do": "signUp", + "name": "Owner" + }, + { + "actor": "member", + "do": "signUp", + "name": "Member" + }, + { + "actor": "target", + "do": "signUp", + "name": "Target" + }, + { + "actor": "outsider", + "do": "signUp", + "name": "Outsider" + }, + { + "actor": "owner", + "do": "createRoom", + "room": "guarded" + }, + { + "actor": "owner", + "do": "enterRoom", + "room": "guarded" + }, + { + "actor": "member", + "do": "enterRoom", + "room": "guarded" + }, + { + "actor": "target", + "do": "enterRoom", + "room": "guarded" + }, + { + "actor": "owner", + "do": "send", + "text": "owner message" + }, + { + "actor": "owner", + "do": "wait", + "ms": 2500 + } + ] + } + ], + "level": 1, + "note": "Cross-cutting properties, scored on a separate axis from features. A feature-only score cannot see these: an app can implement every listed feature and still let one user take over another's account. Each invariant is its own feature with fresh contexts, so one root defect cannot cascade into several apparent problems.", + "schemaVersion": 1, + "track": "stack-bench" + }, + "path": "scenarios/01-invariants.json" + }, + { + "definition": { + "features": [ + { + "actors": [ + "alice", + "bob", + "auditor" + ], + "criteria": [ + { + "desc": "a back-office script purges SysBob's messages directly in the database; a client with the room open sees them disappear without touching the page", + "id": "901a", + "note": "WITHHELD until proven against a mutant (backoffice-writes-via-server) and a real failing build. The script is the app's own scripts/backoffice.mjs, required by the spec to write to the database without the web server's help — moderation, compliance takedowns and data-retention jobs all look exactly like this.", + "points": 0, + "steps": [ + { + "args": [ + "purge-user", + "{user:SysBob}" + ], + "do": "runScript", + "script": "scripts/backoffice.mjs", + "settleMs": 4000 + }, + { + "actor": "alice", + "contains": "purge-me", + "do": "expectElementCount", + "equals": 0, + "testid": "message-item", + "within": 10000 + } + ] + }, + { + "desc": "a customer arriving after the purge never sees the messages — the rows are gone from the database, not merely from one client's memory", + "id": "901b", + "note": "WITHHELD — promoted together with 901a. A fresh actor rather than a reload, so this cannot be confused by session-persistence behaviour, which invariant 105 scores separately.", + "points": 0, + "steps": [ + { + "actor": "auditor", + "do": "signUp", + "name": "SysAuditor" + }, + { + "actor": "auditor", + "do": "enterRoom", + "room": "sysroom" + }, + { + "actor": "auditor", + "contains": "purge-me", + "do": "expectElementCount", + "equals": 0, + "testid": "message-item", + "within": 10000 + } + ] + }, + { + "desc": "a purge that lands while the app server is down is still reflected to an open client once the server returns — the deploy window loses nothing", + "id": "901c", + "note": "WITHHELD — same promotion rule, plus this is the anti-cheat phase: a script that talks to the app server fails here outright. NOTIFY-style event relays also fail unless the app refetches on reconnect, because a notification fired while nobody listened is gone forever.", + "points": 0, + "steps": [ + { + "actor": "alice", + "do": "send", + "text": "deploy-window-msg" + }, + { + "actor": "auditor", + "contains": "deploy-window-msg", + "do": "expect", + "testid": "message-item", + "within": 8000 + }, + { + "do": "stopAppServer" + }, + { + "args": [ + "purge-user", + "{user:SysAlice}" + ], + "do": "runScript", + "script": "scripts/backoffice.mjs" + }, + { + "do": "startAppServer" + }, + { + "actor": "auditor", + "contains": "deploy-window-msg", + "do": "expectElementCount", + "equals": 0, + "testid": "message-item", + "within": 20000 + } + ] + } + ], + "id": 901, + "name": "A write from outside the app is visible without reload", + "setup": [ + { + "actor": "alice", + "do": "signUp", + "name": "SysAlice" + }, + { + "actor": "bob", + "do": "signUp", + "name": "SysBob" + }, + { + "actor": "alice", + "do": "createRoom", + "room": "sysroom" + }, + { + "actor": "alice", + "do": "enterRoom", + "room": "sysroom" + }, + { + "actor": "bob", + "do": "enterRoom", + "room": "sysroom" + }, + { + "actor": "bob", + "do": "send", + "text": "purge-me-alpha" + }, + { + "actor": "bob", + "do": "wait", + "ms": 1400 + }, + { + "actor": "bob", + "do": "send", + "text": "purge-me-beta" + }, + { + "actor": "alice", + "contains": "purge-me-beta", + "do": "expect", + "testid": "message-item", + "within": 8000 + } + ] + }, + { + "actors": [ + "host", + "watcher", + "joiner" + ], + "criteria": [ + { + "desc": "one user enters the room — fetching its member list — at the same moment another joins: the fetcher ends with the joiner exactly once, not zero (stale fetch) and not twice (fetch merged with the join broadcast)", + "id": "902a", + "note": "WITHHELD until proven against a mutant (fetch-then-append-dupe) and a real failing build. The race window is real; INCONCLUSIVE on a missed race, never a false FAIL.", + "points": 0, + "steps": [ + { + "branches": [ + [ + { + "actor": "watcher", + "do": "enterRoom", + "room": "raceroom" + } + ], + [ + { + "actor": "joiner", + "do": "enterRoom", + "room": "raceroom" + } + ] + ], + "do": "race", + "settleMs": 4000 + }, + { + "actor": "watcher", + "contains": "RaceJoiner", + "do": "expectElementCount", + "equals": 1, + "testid": "member-item", + "within": 10000 + }, + { + "actor": "watcher", + "contains": "RaceHost", + "do": "expectElementCount", + "equals": 1, + "testid": "member-item", + "within": 10000 + } + ] + } + ], + "id": 902, + "name": "A list being opened while its contents change", + "setup": [ + { + "actor": "host", + "do": "signUp", + "name": "RaceHost" + }, + { + "actor": "watcher", + "do": "signUp", + "name": "RaceWatcher" + }, + { + "actor": "joiner", + "do": "signUp", + "name": "RaceJoiner" + }, + { + "actor": "host", + "do": "createRoom", + "room": "raceroom" + }, + { + "actor": "host", + "do": "enterRoom", + "room": "raceroom" + } + ] + } + ], + "level": 1, + "note": "Properties of the system, not of any one feature: what happens when the database changes underneath the app, and when two things happen at the same time. A broadcast layer only announces writes that went through it — a moderation script that deletes rows directly leaves every open client showing messages that no longer exist. Every criterion is withheld at 0 points until it demonstrably fails a real build AND its mutant is CAUGHT, the same promotion rule as contention.", + "schemaVersion": 1 + }, + "path": "scenarios/01-systems.json" + }, + { + "definition": { + "features": [ + { + "actors": [ + "owner", + "outsider" + ], + "criteria": [ + { + "desc": "the owner sees the private badge on their room", + "id": "201a", + "points": 1, + "steps": [ + { + "actor": "owner", + "do": "expect", + "testid": "room-private-badge", + "within": 8000 + } + ] + }, + { + "desc": "a private room does not appear in a non-member's room list at all", + "id": "201b", + "points": 2, + "steps": [ + { + "absent": true, + "actor": "outsider", + "contains": "{room:vault}", + "do": "expect", + "testid": "room-item", + "within": 6000 + } + ] + } + ], + "id": 201, + "name": "Private Rooms", + "setup": [ + { + "actor": "owner", + "do": "signUp", + "name": "Pria" + }, + { + "actor": "outsider", + "do": "signUp", + "name": "Oleg" + }, + { + "actor": "owner", + "do": "createRoom", + "private": true, + "room": "vault" + }, + { + "actor": "owner", + "do": "enterRoom", + "room": "vault" + }, + { + "actor": "owner", + "do": "send", + "text": "vault-secret-note" + } + ] + }, + { + "actors": [ + "owner", + "guest" + ], + "criteria": [ + { + "desc": "an invited user gains access and can read the room's history", + "id": "202a", + "points": 2, + "steps": [ + { + "actor": "owner", + "do": "fill", + "testid": "invite-input", + "text": "{user:Gina}" + }, + { + "actor": "owner", + "do": "click", + "settleMs": 1200, + "testid": "invite-submit" + }, + { + "actor": "guest", + "do": "enterRoom", + "room": "committee" + }, + { + "actor": "guest", + "contains": "pre-invite-history", + "do": "expect", + "testid": "message-item", + "within": 10000 + } + ] + }, + { + "desc": "removal takes effect live: the removed member stops receiving without reloading", + "id": "202b", + "points": 2, + "steps": [ + { + "actor": "owner", + "do": "click", + "in": { + "contains": "Gina", + "testid": "member-item" + }, + "settleMs": 1500, + "testid": "member-remove" + }, + { + "actor": "owner", + "do": "send", + "text": "post-removal-note" + }, + { + "actor": "owner", + "do": "wait", + "ms": 2500 + }, + { + "absent": true, + "actor": "guest", + "contains": "post-removal-note", + "do": "expect", + "testid": "message-item", + "within": 4000 + } + ] + } + ], + "id": 202, + "name": "Invitation and Removal", + "setup": [ + { + "actor": "owner", + "do": "signUp", + "name": "Held" + }, + { + "actor": "guest", + "do": "signUp", + "name": "Gina" + }, + { + "actor": "owner", + "do": "createRoom", + "private": true, + "room": "committee" + }, + { + "actor": "owner", + "do": "enterRoom", + "room": "committee" + }, + { + "actor": "owner", + "do": "send", + "text": "pre-invite-history" + } + ] + }, + { + "actors": [ + "subject", + "viewer" + ], + "criteria": [ + { + "desc": "clicking a username opens that person's profile with their name and bio", + "id": "203a", + "points": 1, + "steps": [ + { + "actor": "viewer", + "contains": "Sana", + "do": "click", + "settleMs": 1200, + "testid": "member-item" + }, + { + "actor": "viewer", + "contains": "Sana", + "do": "expect", + "testid": "profile-name", + "within": 8000 + }, + { + "actor": "viewer", + "contains": "first-bio-text", + "do": "expect", + "testid": "profile-bio", + "within": 8000 + } + ] + }, + { + "desc": "an open profile panel is live: a bio edit appears to a viewer without reopening", + "id": "203b", + "points": 2, + "steps": [ + { + "actor": "subject", + "do": "click", + "ifAvailable": true, + "testid": "profile-close" + }, + { + "actor": "subject", + "contains": "Sana", + "do": "click", + "testid": "member-item" + }, + { + "actor": "subject", + "do": "fill", + "testid": "bio-input", + "text": "second-bio-text" + }, + { + "actor": "subject", + "do": "click", + "testid": "bio-save" + }, + { + "actor": "viewer", + "contains": "second-bio-text", + "do": "expect", + "testid": "profile-bio", + "within": 10000 + } + ] + }, + { + "desc": "the panel closes, and reopening shows current data", + "id": "203c", + "points": 1, + "steps": [ + { + "actor": "viewer", + "do": "click", + "settleMs": 800, + "testid": "profile-close" + }, + { + "actor": "viewer", + "do": "waitUntilAbsent", + "testid": "profile-bio", + "within": 4000 + }, + { + "actor": "viewer", + "contains": "Sana", + "do": "click", + "settleMs": 1200, + "testid": "member-item" + }, + { + "actor": "viewer", + "contains": "second-bio-text", + "do": "expect", + "testid": "profile-bio", + "within": 8000 + } + ] + } + ], + "id": 203, + "name": "Profiles", + "setup": [ + { + "actor": "subject", + "do": "signUp", + "name": "Sana" + }, + { + "actor": "viewer", + "do": "signUp", + "name": "Vero" + }, + { + "actor": "subject", + "do": "createRoom", + "room": "profroom" + }, + { + "actor": "subject", + "do": "enterRoom", + "room": "profroom" + }, + { + "actor": "viewer", + "do": "enterRoom", + "room": "profroom" + }, + { + "actor": "subject", + "do": "click", + "ifAvailable": true, + "testid": "profile-close" + }, + { + "actor": "subject", + "contains": "Sana", + "do": "click", + "testid": "member-item" + }, + { + "actor": "subject", + "do": "fill", + "testid": "bio-input", + "text": "first-bio-text" + }, + { + "actor": "subject", + "do": "click", + "settleMs": 1200, + "testid": "bio-save" + } + ] + }, + { + "actors": [ + "asker", + "asked", + "third" + ], + "criteria": [ + { + "desc": "a request reaches the recipient as pending, and accepting puts each user in the other's friends list — live, no reloads anywhere", + "id": "204a", + "points": 2, + "steps": [ + { + "actor": "asker", + "contains": "Freya", + "do": "click", + "settleMs": 1200, + "testid": "member-item" + }, + { + "actor": "asker", + "do": "click", + "settleMs": 1200, + "testid": "friend-request" + }, + { + "actor": "asker", + "do": "click", + "ifAvailable": true, + "testid": "profile-close" + }, + { + "actor": "asked", + "contains": "Finn", + "do": "expect", + "testid": "friend-pending", + "within": 10000 + }, + { + "actor": "asked", + "do": "click", + "in": { + "contains": "Finn", + "testid": "friend-pending" + }, + "settleMs": 1500, + "testid": "friend-accept" + }, + { + "actor": "asker", + "contains": "Freya", + "do": "expect", + "testid": "friend-item", + "within": 10000 + }, + { + "actor": "asked", + "contains": "Finn", + "do": "expect", + "testid": "friend-item", + "within": 10000 + } + ] + }, + { + "desc": "a declined request creates no friendship on either side", + "id": "204b", + "points": 1, + "steps": [ + { + "actor": "third", + "contains": "Freya", + "do": "click", + "settleMs": 1200, + "testid": "member-item" + }, + { + "actor": "third", + "do": "click", + "settleMs": 1200, + "testid": "friend-request" + }, + { + "actor": "third", + "do": "click", + "ifAvailable": true, + "testid": "profile-close" + }, + { + "actor": "asked", + "do": "click", + "in": { + "contains": "Falk", + "testid": "friend-pending" + }, + "settleMs": 1500, + "testid": "friend-decline" + }, + { + "absent": true, + "actor": "third", + "contains": "Freya", + "do": "expect", + "testid": "friend-item", + "within": 5000 + }, + { + "absent": true, + "actor": "asked", + "contains": "Falk", + "do": "expect", + "testid": "friend-item", + "within": 5000 + } + ] + }, + { + "desc": "unfriending removes the row live from BOTH lists", + "id": "204c", + "points": 2, + "steps": [ + { + "actor": "asker", + "do": "click", + "in": { + "contains": "Freya", + "testid": "friend-item" + }, + "settleMs": 1500, + "testid": "unfriend" + }, + { + "actor": "asker", + "contains": "Freya", + "do": "waitUntilAbsent", + "testid": "friend-item", + "within": 8000 + }, + { + "actor": "asked", + "contains": "Finn", + "do": "waitUntilAbsent", + "testid": "friend-item", + "within": 8000 + } + ] + } + ], + "id": 204, + "name": "Friends", + "setup": [ + { + "actor": "asker", + "do": "signUp", + "name": "Finn" + }, + { + "actor": "asked", + "do": "signUp", + "name": "Freya" + }, + { + "actor": "third", + "do": "signUp", + "name": "Falk" + }, + { + "actor": "asker", + "do": "createRoom", + "room": "friendroom" + }, + { + "actor": "asker", + "do": "enterRoom", + "room": "friendroom" + }, + { + "actor": "asked", + "do": "enterRoom", + "room": "friendroom" + }, + { + "actor": "third", + "do": "enterRoom", + "room": "friendroom" + } + ] + }, + { + "actors": [ + "watcher", + "flapper" + ], + "criteria": [ + { + "desc": "a friend's presence flips to offline live when they leave, and back when they return", + "id": "205a", + "points": 2, + "steps": [ + { + "actor": "watcher", + "contains": "online", + "do": "expect", + "in": { + "contains": "Flip", + "testid": "friend-item" + }, + "testid": "friend-presence", + "within": 10000 + }, + { + "actor": "flapper", + "do": "closeClient" + }, + { + "actor": "watcher", + "contains": "offline", + "do": "expect", + "in": { + "contains": "Flip", + "testid": "friend-item" + }, + "testid": "friend-presence", + "within": 15000 + }, + { + "actor": "flapper", + "do": "openClient" + }, + { + "actor": "flapper", + "do": "ensureSignedIn", + "name": "Flip" + }, + { + "actor": "watcher", + "contains": "online", + "do": "expect", + "in": { + "contains": "Flip", + "testid": "friend-item" + }, + "testid": "friend-presence", + "within": 15000 + } + ] + } + ], + "id": 205, + "name": "Friend Presence", + "setup": [ + { + "actor": "watcher", + "do": "signUp", + "name": "Wava" + }, + { + "actor": "flapper", + "do": "signUp", + "name": "Flip" + }, + { + "actor": "watcher", + "do": "createRoom", + "room": "presroom" + }, + { + "actor": "watcher", + "do": "enterRoom", + "room": "presroom" + }, + { + "actor": "flapper", + "do": "enterRoom", + "room": "presroom" + }, + { + "actor": "watcher", + "contains": "Flip", + "do": "click", + "settleMs": 1200, + "testid": "member-item" + }, + { + "actor": "watcher", + "do": "click", + "settleMs": 1200, + "testid": "friend-request" + }, + { + "actor": "watcher", + "do": "click", + "ifAvailable": true, + "testid": "profile-close" + }, + { + "actor": "flapper", + "do": "click", + "in": { + "contains": "Wava", + "testid": "friend-pending" + }, + "settleMs": 1500, + "testid": "friend-accept" + } + ] + } + ], + "level": 2, + "note": "Level 2 features: access control and the social layer. Levels are cumulative — these run against the same app as level 1's suites, after the upgrade.", + "schemaVersion": 1 + }, + "path": "scenarios/02-features.json" + }, + { + "definition": { + "features": [ + { + "actors": [ + "owner", + "member", + "outsider" + ], + "criteria": [ + { + "desc": "a private message reaches a member and is never sent over the wire to a non-member — regardless of what any page renders", + "id": "211a", + "points": 2, + "steps": [ + { + "actor": "owner", + "do": "send", + "text": "sealed-room-secret" + }, + { + "actor": "owner", + "do": "wait", + "ms": 4000 + }, + { + "actor": "member", + "contains": "sealed-room-secret", + "do": "expectReceived", + "within": 8000 + }, + { + "actor": "outsider", + "contains": "sealed-room-secret", + "do": "expectNotReceived" + } + ] + }, + { + "desc": "a non-member's post into a private room, issued as a raw request, is rejected and appears to nobody", + "id": "211b", + "points": 2, + "steps": [ + { + "actor": "outsider", + "do": "replayAs", + "from": "owner", + "match": "sealed-room-secret", + "swap": { + "find": "sealed-room-secret", + "with": "smuggled-outsider-post" + } + }, + { + "actor": "outsider", + "do": "expectReplayRejected" + }, + { + "absent": true, + "actor": "member", + "contains": "smuggled-outsider-post", + "do": "expect", + "testid": "message-item", + "within": 5000 + } + ] + } + ], + "id": 211, + "name": "Private Room Privacy", + "setup": [ + { + "actor": "owner", + "do": "signUp", + "name": "Opal" + }, + { + "actor": "member", + "do": "signUp", + "name": "Mia" + }, + { + "actor": "outsider", + "do": "signUp", + "name": "Xeno" + }, + { + "actor": "owner", + "do": "createRoom", + "private": true, + "room": "sealed" + }, + { + "actor": "owner", + "do": "enterRoom", + "room": "sealed" + }, + { + "actor": "owner", + "do": "fill", + "testid": "invite-input", + "text": "{user:Mia}" + }, + { + "actor": "owner", + "do": "click", + "settleMs": 1500, + "testid": "invite-submit" + }, + { + "actor": "member", + "do": "enterRoom", + "room": "sealed" + } + ] + }, + { + "actors": [ + "a", + "b" + ], + "criteria": [ + { + "desc": "after an accept, each side holds exactly one row for the other — never two, never a half-friendship", + "id": "212a", + "points": 1, + "steps": [ + { + "actor": "a", + "contains": "Ben", + "do": "expectElementCount", + "equals": 1, + "testid": "friend-item", + "within": 10000 + }, + { + "actor": "b", + "contains": "Ada", + "do": "expectElementCount", + "equals": 1, + "testid": "friend-item", + "within": 10000 + } + ] + }, + { + "desc": "replaying the accept a second time as a raw request creates no second friendship", + "id": "212b", + "points": 2, + "steps": [ + { + "actor": "b", + "do": "replayAs", + "from": "b", + "match": "friend" + }, + { + "actor": "b", + "do": "expectReplayCompleted" + }, + { + "actor": "b", + "do": "wait", + "ms": 2500 + }, + { + "actor": "a", + "contains": "Ben", + "do": "expectElementCount", + "equals": 1, + "testid": "friend-item", + "within": 8000 + }, + { + "actor": "b", + "contains": "Ada", + "do": "expectElementCount", + "equals": 1, + "testid": "friend-item", + "within": 8000 + } + ] + } + ], + "id": 212, + "name": "Friendship Is One Relationship", + "setup": [ + { + "actor": "a", + "do": "signUp", + "name": "Ada" + }, + { + "actor": "b", + "do": "signUp", + "name": "Ben" + }, + { + "actor": "a", + "do": "createRoom", + "room": "symroom" + }, + { + "actor": "a", + "do": "enterRoom", + "room": "symroom" + }, + { + "actor": "b", + "do": "enterRoom", + "room": "symroom" + }, + { + "actor": "a", + "contains": "Ben", + "do": "click", + "settleMs": 1200, + "testid": "member-item" + }, + { + "actor": "a", + "do": "click", + "settleMs": 1200, + "testid": "friend-request" + }, + { + "actor": "a", + "do": "click", + "ifAvailable": true, + "testid": "profile-close" + }, + { + "actor": "b", + "do": "click", + "in": { + "contains": "Ada", + "testid": "friend-pending" + }, + "settleMs": 1500, + "testid": "friend-accept" + } + ] + }, + { + "actors": [ + "victim", + "attacker" + ], + "criteria": [ + { + "desc": "another user's raw request to edit someone's bio is rejected, and the bio is unchanged for everyone", + "id": "213a", + "points": 2, + "steps": [ + { + "actor": "attacker", + "do": "replayAs", + "from": "victim", + "match": "authentic-bio-text", + "swap": { + "find": "authentic-bio-text", + "with": "defaced-bio-text" + } + }, + { + "actor": "attacker", + "do": "expectReplayRejected" + }, + { + "actor": "attacker", + "contains": "Vik", + "do": "click", + "settleMs": 1200, + "testid": "member-item" + }, + { + "actor": "attacker", + "contains": "authentic-bio-text", + "do": "expect", + "testid": "profile-bio", + "within": 8000 + }, + { + "absent": true, + "actor": "attacker", + "contains": "defaced-bio-text", + "do": "expect", + "testid": "profile-bio", + "within": 4000 + } + ] + } + ], + "id": 213, + "name": "A Profile Belongs to Its Owner", + "setup": [ + { + "actor": "victim", + "do": "signUp", + "name": "Vik" + }, + { + "actor": "attacker", + "do": "signUp", + "name": "Axl" + }, + { + "actor": "victim", + "do": "createRoom", + "room": "bioroom" + }, + { + "actor": "victim", + "do": "enterRoom", + "room": "bioroom" + }, + { + "actor": "attacker", + "do": "enterRoom", + "room": "bioroom" + }, + { + "actor": "victim", + "do": "click", + "ifAvailable": true, + "testid": "profile-close" + }, + { + "actor": "victim", + "contains": "Vik", + "do": "click", + "testid": "member-item" + }, + { + "actor": "victim", + "do": "fill", + "testid": "bio-input", + "text": "authentic-bio-text" + }, + { + "actor": "victim", + "do": "click", + "settleMs": 1500, + "testid": "bio-save" + } + ] + } + ], + "level": 2, + "note": "What level 2 must hold true regardless of UI: privacy is a property of the wire, not of what a page chooses to render; a friendship is one relationship; a profile belongs to its owner. Three-way verdicts follow level 1's replay conventions.", + "schemaVersion": 1 + }, + "path": "scenarios/02-invariants.json" + }, + { + "definition": { + "features": [ + { + "actors": [ + "subject", + "viewer" + ], + "criteria": [ + { + "desc": "a profile panel opened at the same moment the bio is saved converges on the NEW bio — the panel's subscription outlives its opening fetch", + "id": "921a", + "note": "WITHHELD until proven against a mutant (fetch-on-open-no-subscribe) and a real failing build. A fetch-on-open panel that races the save shows bio-before-race forever; a subscribed panel converges.", + "points": 0, + "steps": [ + { + "branches": [ + [ + { + "actor": "viewer", + "contains": "Zed", + "do": "click", + "testid": "member-item" + } + ], + [ + { + "actor": "subject", + "do": "click", + "testid": "bio-save" + } + ] + ], + "do": "race", + "settleMs": 4000 + }, + { + "actor": "viewer", + "contains": "bio-after-race", + "do": "expect", + "testid": "profile-bio", + "within": 10000 + } + ] + } + ], + "id": 921, + "name": "A panel opened at the moment its contents change", + "setup": [ + { + "actor": "subject", + "do": "signUp", + "name": "Zed" + }, + { + "actor": "viewer", + "do": "signUp", + "name": "Yara" + }, + { + "actor": "subject", + "do": "createRoom", + "room": "panelrace" + }, + { + "actor": "subject", + "do": "enterRoom", + "room": "panelrace" + }, + { + "actor": "viewer", + "do": "enterRoom", + "room": "panelrace" + }, + { + "actor": "subject", + "do": "click", + "ifAvailable": true, + "testid": "profile-close" + }, + { + "actor": "subject", + "contains": "Zed", + "do": "click", + "testid": "member-item" + }, + { + "actor": "subject", + "do": "fill", + "testid": "bio-input", + "text": "bio-before-race" + }, + { + "actor": "subject", + "do": "click", + "settleMs": 1500, + "testid": "bio-save" + }, + { + "actor": "subject", + "do": "click", + "ifAvailable": true, + "testid": "profile-close" + }, + { + "actor": "subject", + "contains": "Zed", + "do": "click", + "testid": "member-item" + }, + { + "actor": "subject", + "do": "fill", + "testid": "bio-input", + "text": "bio-after-race" + } + ] + }, + { + "actors": [ + "asker", + "asked" + ], + "criteria": [ + { + "desc": "the accept lands while the asker's client is mid-refresh: the new friend appears exactly once — not zero (stale fetch), not twice (fetch merged with the accept broadcast)", + "id": "922a", + "note": "WITHHELD until proven against a mutant (fetch-then-append-dupe) and a real failing build. The reload is what forces the asker's client into a full refetch for the accept to race.", + "points": 0, + "steps": [ + { + "branches": [ + [ + { + "actor": "asker", + "do": "reload", + "settleMs": 500 + } + ], + [ + { + "actor": "asked", + "do": "click", + "in": { + "contains": "Quin", + "testid": "friend-pending" + }, + "testid": "friend-accept" + } + ] + ], + "do": "race", + "settleMs": 5000 + }, + { + "actor": "asker", + "do": "ensureSignedIn", + "name": "Quin" + }, + { + "actor": "asker", + "contains": "Rune", + "do": "expectElementCount", + "equals": 1, + "testid": "friend-item", + "within": 12000 + }, + { + "actor": "asked", + "contains": "Quin", + "do": "expectElementCount", + "equals": 1, + "testid": "friend-item", + "within": 12000 + } + ] + } + ], + "id": 922, + "name": "A friendship formed while the list showing it is fetched", + "setup": [ + { + "actor": "asker", + "do": "signUp", + "name": "Quin" + }, + { + "actor": "asked", + "do": "signUp", + "name": "Rune" + }, + { + "actor": "asker", + "do": "createRoom", + "room": "listrace" + }, + { + "actor": "asker", + "do": "enterRoom", + "room": "listrace" + }, + { + "actor": "asked", + "do": "enterRoom", + "room": "listrace" + }, + { + "actor": "asker", + "contains": "Rune", + "do": "click", + "settleMs": 1200, + "testid": "member-item" + }, + { + "actor": "asker", + "do": "click", + "settleMs": 1500, + "testid": "friend-request" + }, + { + "actor": "asker", + "do": "click", + "ifAvailable": true, + "testid": "profile-close" + } + ] + } + ], + "level": 2, + "note": "The subscription-lifecycle races level 2's surfaces make possible: a panel that opens at the moment its contents change, and a relationship that forms while the list showing it is being fetched. Fetch-on-open architectures pass every sequential test and fail exactly here. Every criterion is withheld at 0 points until it demonstrably fails a real build AND its mutant is CAUGHT — same promotion rule as contention.", + "schemaVersion": 1 + }, + "path": "scenarios/02-systems.json" + } + ], + "track": "chat" +} diff --git a/tools/stack-bench/tests/snapshots/definitions/ecommerce.snapshot.json b/tools/stack-bench/tests/snapshots/definitions/ecommerce.snapshot.json new file mode 100644 index 00000000000..b109dbb4093 --- /dev/null +++ b/tools/stack-bench/tests/snapshots/definitions/ecommerce.snapshot.json @@ -0,0 +1,6376 @@ +{ + "definitionSchemaVersion": 1, + "levels": [ + { + "level": 1, + "suites": [ + { + "id": "features", + "inherit": "none", + "scenario": "scenarios/01-features.json" + }, + { + "id": "invariants", + "inherit": "all-higher-levels", + "scenario": "scenarios/01-invariants.json" + }, + { + "id": "contention", + "inherit": "all-higher-levels", + "scenario": "scenarios/01-contention.json" + }, + { + "id": "systems", + "inherit": "all-higher-levels", + "scenario": "scenarios/01-systems.json" + } + ] + }, + { + "level": 2, + "suites": [ + { + "id": "features", + "inherit": "none", + "scenario": "scenarios/02-features.json" + }, + { + "id": "invariants", + "inherit": "all-higher-levels", + "scenario": "scenarios/02-invariants.json" + }, + { + "id": "contention", + "inherit": "all-higher-levels", + "scenario": "scenarios/01-contention.json" + } + ] + }, + { + "level": 3, + "suites": [ + { + "id": "features", + "inherit": "none", + "scenario": "scenarios/03-features.json" + }, + { + "id": "invariants", + "inherit": "all-higher-levels", + "scenario": "scenarios/03-invariants.json" + }, + { + "id": "systems", + "inherit": "all-higher-levels", + "scenario": "scenarios/01-systems.json" + } + ] + } + ], + "manifest": { + "actions": [ + { + "args": [ + 0 + ], + "id": "buy", + "params": [ + { + "in": "body", + "name": "itemId", + "wireType": "u64" + } + ], + "path": "/api/items/00000000-0000-0000-0000-000000000000/buy", + "reducer": "buy_now" + }, + { + "args": [ + 0, + 1 + ], + "id": "addToCart", + "params": [ + { + "in": "body", + "name": "itemId", + "wireType": "u64" + }, + { + "in": "body", + "name": "quantity" + } + ], + "path": "/api/cart", + "reducer": "add_to_cart" + }, + { + "args": [], + "id": "checkout", + "path": "/api/checkout", + "reducer": "checkout" + }, + { + "args": [ + 0, + 0, + 1 + ], + "id": "restock", + "params": [ + { + "in": "body", + "name": "itemId", + "wireType": "u64" + }, + { + "in": "body", + "name": "warehouseId", + "wireType": "u64" + }, + { + "in": "body", + "name": "quantity" + } + ], + "path": "/api/admin/restock", + "reducer": "admin_restock" + }, + { + "args": [ + "", + "" + ], + "id": "signUp", + "params": [ + { + "in": "body", + "name": "username" + }, + { + "in": "body", + "name": "password" + } + ], + "path": "/api/auth/signup", + "reducer": "sign_up" + }, + { + "args": [ + "", + "" + ], + "id": "signIn", + "path": "/api/auth/signin", + "reducer": "sign_in" + } + ], + "databaseProvenance": { + "browserAction": "signUp" + }, + "plannedThrough": 6, + "portOffset": 300, + "reseedOnReset": true, + "restartProbe": "/api/items", + "schemaVersion": 1, + "slug": "ecom", + "suites": { + "1": [ + { + "id": "features", + "inherit": "none", + "spec": "scenarios/01-features.json" + }, + { + "id": "invariants", + "inherit": "all-higher-levels", + "spec": "scenarios/01-invariants.json" + }, + { + "id": "contention", + "inherit": "all-higher-levels", + "spec": "scenarios/01-contention.json" + }, + { + "id": "systems", + "inherit": "all-higher-levels", + "spec": "scenarios/01-systems.json" + } + ], + "2": [ + { + "id": "features", + "inherit": "none", + "spec": "scenarios/02-features.json" + }, + { + "id": "invariants", + "inherit": "all-higher-levels", + "spec": "scenarios/02-invariants.json" + }, + { + "id": "contention", + "inherit": "all-higher-levels", + "spec": "scenarios/01-contention.json" + } + ], + "3": [ + { + "id": "features", + "inherit": "none", + "spec": "scenarios/03-features.json" + }, + { + "id": "invariants", + "inherit": "all-higher-levels", + "spec": "scenarios/03-invariants.json" + }, + { + "id": "systems", + "inherit": "all-higher-levels", + "spec": "scenarios/01-systems.json" + } + ] + }, + "title": "Shop", + "validatedThrough": 0 + }, + "scenarios": [ + { + "definition": { + "features": [ + { + "actors": [ + "a", + "b", + "c", + "d", + "e", + "f" + ], + "criteria": [ + { + "desc": "six customers buy the last three units at once: stock lands on exactly zero, never below", + "id": "201a", + "points": 1, + "provenBy": "Candidate backend-specific mutants are declared for this criterion and must pass the live Docker mutation gate before this oracle is called production-validated. PostgreSQL and MongoDB remove an explicit concurrency control; SpacetimeDB omits inventory reservation because reducer transactions make the equivalent interleaving defect inexpressible.", + "statedBy": "stock may never go negative, and two customers must never both get the last unit", + "steps": [ + { + "actors": [ + "a", + "b", + "c", + "d", + "e", + "f" + ], + "do": "clickConcurrently", + "in": { + "contains": "Mirrorless Camera", + "testid": "item-card" + }, + "settleMs": 6000, + "testid": "buy-now" + }, + { + "actor": "a", + "do": "expectNumber", + "equals": 0, + "in": { + "contains": "Mirrorless Camera", + "testid": "item-card" + }, + "testid": "item-stock" + }, + { + "actors": [ + "a", + "b", + "c", + "d", + "e", + "f" + ], + "do": "expectAgreement", + "in": { + "contains": "Mirrorless Camera", + "testid": "item-card" + }, + "numeric": true, + "testid": "item-stock" + } + ] + }, + { + "desc": "exactly three of the six customers end up with an order, and none is charged twice", + "id": "201b", + "points": 0, + "statedBy": "two customers must never both get the last unit", + "steps": [ + { + "actors": [ + "a", + "b", + "c", + "d", + "e", + "f" + ], + "do": "clickConcurrently", + "settleMs": 2500, + "testid": "orders-toggle" + }, + { + "actors": [ + "a", + "b", + "c", + "d", + "e", + "f" + ], + "contains": "Mirrorless Camera", + "do": "expectActorsWith", + "equals": 3, + "maxEach": 1, + "testid": "order-item" + } + ], + "withheld": "rewritten to count the population instead of one participant: the old phrasing asked actor a for at most one order, which passes when nobody got one. Stays at 0 points until it kills a mutant on a real build." + }, + { + "desc": "revenue counts three sales, not six", + "id": "201c", + "points": 0, + "statedBy": "stock must never leave a warehouse without a corresponding order", + "steps": [], + "withheld": "needs an admin actor; wire once 201a has demonstrably failed somewhere" + } + ], + "id": 201, + "name": "The last unit is sold once", + "setup": [ + { + "actor": "a", + "do": "signUp", + "name": "c1" + }, + { + "actor": "b", + "do": "signUp", + "name": "c2" + }, + { + "actor": "c", + "do": "signUp", + "name": "c3" + }, + { + "actor": "d", + "do": "signUp", + "name": "c4" + }, + { + "actor": "e", + "do": "signUp", + "name": "c5" + }, + { + "actor": "f", + "do": "signUp", + "name": "c6" + }, + { + "actor": "a", + "do": "fill", + "enter": true, + "settleMs": 1200, + "testid": "search-input", + "text": "Mirrorless Camera" + }, + { + "actor": "b", + "do": "fill", + "enter": true, + "settleMs": 1200, + "testid": "search-input", + "text": "Mirrorless Camera" + }, + { + "actor": "c", + "do": "fill", + "enter": true, + "settleMs": 1200, + "testid": "search-input", + "text": "Mirrorless Camera" + }, + { + "actor": "d", + "do": "fill", + "enter": true, + "settleMs": 1200, + "testid": "search-input", + "text": "Mirrorless Camera" + }, + { + "actor": "e", + "do": "fill", + "enter": true, + "settleMs": 1200, + "testid": "search-input", + "text": "Mirrorless Camera" + }, + { + "actor": "f", + "do": "fill", + "enter": true, + "settleMs": 1200, + "testid": "search-input", + "text": "Mirrorless Camera" + } + ] + }, + { + "actors": [ + "admin", + "a", + "b", + "c" + ], + "criteria": [ + { + "desc": "an uncontended restock of five is stored by the server and shows on the storefront", + "id": "202-control", + "points": 0, + "steps": [ + { + "actor": "admin", + "do": "click", + "testid": "admin-link" + }, + { + "actor": "admin", + "as": "warehouse-before", + "do": "recordNumber", + "in": { + "contains": "Bluetooth Speaker", + "testid": "admin-location-row" + }, + "testid": "admin-location-qty" + }, + { + "actor": "a", + "as": "storefront-before", + "do": "recordNumber", + "in": { + "contains": "Bluetooth Speaker", + "testid": "item-card" + }, + "testid": "item-stock" + }, + { + "actor": "admin", + "do": "fill", + "in": { + "contains": "Bluetooth Speaker", + "testid": "admin-location-row" + }, + "testid": "restock-input", + "text": "5" + }, + { + "actor": "admin", + "do": "click", + "in": { + "contains": "Bluetooth Speaker", + "testid": "admin-location-row" + }, + "testid": "restock-submit" + }, + { + "actor": "admin", + "do": "wait", + "ms": 4000 + }, + { + "actor": "admin", + "do": "expectNumber", + "in": { + "contains": "Bluetooth Speaker", + "testid": "admin-location-row" + }, + "plus": 5, + "relativeTo": "warehouse-before", + "testid": "admin-location-qty", + "within": 8000 + }, + { + "actor": "a", + "do": "pressKey", + "key": "Escape", + "settleMs": 800 + }, + { + "actor": "a", + "do": "expectNumber", + "in": { + "contains": "Bluetooth Speaker", + "testid": "item-card" + }, + "plus": 5, + "relativeTo": "storefront-before", + "testid": "item-stock", + "within": 8000 + } + ], + "withheld": "diagnostic: establishes that a restock is applied and reaches the storefront" + }, + { + "desc": "three purchases and a restock of five during the rush leave stock at start + 2", + "id": "202a", + "points": 0, + "statedBy": "a restock in one warehouse raises the storefront number live", + "steps": [ + { + "actor": "a", + "as": "rush-before", + "do": "recordNumber", + "in": { + "contains": "Bluetooth Speaker", + "testid": "item-card" + }, + "testid": "item-stock" + }, + { + "actors": [ + "a", + "b", + "c" + ], + "do": "clickConcurrently", + "in": { + "contains": "Bluetooth Speaker", + "testid": "item-card" + }, + "settleMs": 500, + "testid": "buy-now" + }, + { + "actor": "admin", + "do": "fill", + "in": { + "contains": "Bluetooth Speaker", + "testid": "admin-location-row" + }, + "testid": "restock-input", + "text": "5" + }, + { + "actor": "admin", + "do": "click", + "in": { + "contains": "Bluetooth Speaker", + "testid": "admin-location-row" + }, + "testid": "restock-submit" + }, + { + "actor": "admin", + "do": "wait", + "ms": 6000 + }, + { + "actor": "a", + "do": "pressKey", + "key": "Escape", + "settleMs": 800 + }, + { + "actor": "a", + "do": "expectNumber", + "in": { + "contains": "Bluetooth Speaker", + "testid": "item-card" + }, + "plus": 2, + "relativeTo": "rush-before", + "testid": "item-stock", + "within": 10000 + } + ], + "withheld": "A candidate cross-feature mutation now omits purchase inventory reservation and directly makes this read start + 5 instead of start + 2. Keep at zero until that exact target passes the repeated live Docker mutation gate." + } + ], + "id": 202, + "name": "A restock during a rush is not lost", + "note": "The rush assertion only means something if a restock works at all, so the control runs first and has to move the number. Without it a restock that never reached the server and a restock lost to a race look identical: both leave stock at start - purchases. The control also separates the two, by reading what the server stored (admin-location-qty) before what the buyer sees (item-stock). Totals are asserted as CHANGES rather than absolute values, because earlier features buy from the same catalogue.", + "setup": [ + { + "actor": "admin", + "do": "signIn", + "exact": true, + "name": "admin", + "password": "stackbench-admin-2026" + }, + { + "actor": "a", + "do": "signUp", + "name": "r1" + }, + { + "actor": "b", + "do": "signUp", + "name": "r2" + }, + { + "actor": "c", + "do": "signUp", + "name": "r3" + } + ] + }, + { + "actors": [ + "tab1", + "tab2", + "filler" + ], + "criteria": [ + { + "desc": "the same item added from two tabs at once becomes one line of two", + "id": "203a", + "points": 0, + "statedBy": "raises its quantity rather than adding a second line", + "steps": [ + { + "actors": [ + "tab1", + "tab2" + ], + "do": "clickConcurrently", + "in": { + "contains": "Gaming Mouse", + "testid": "item-card" + }, + "settleMs": 4000, + "testid": "add-to-cart" + }, + { + "actor": "tab1", + "do": "click", + "testid": "cart-toggle" + }, + { + "actor": "tab1", + "contains": "Gaming Mouse", + "count": 1, + "do": "expect", + "testid": "cart-item" + }, + { + "actor": "tab1", + "do": "expectNumber", + "equals": 2, + "in": { + "contains": "Gaming Mouse", + "testid": "cart-item" + }, + "testid": "cart-quantity" + } + ], + "withheld": "runs clean (2/2) but no mutant has been attempted against it yet." + }, + { + "desc": "checking the same cart out twice at once produces one order", + "id": "203b", + "note": "replayConcurrently replays each actor's LAST POST, so a racing tab must not touch the cart after its checkout is captured — that would overwrite the captured request with an add-to-cart. A third session on the same account fills the cart instead, leaving tab1 and tab2 holding nothing but their checkouts. A checkout carries no JSON body, so it never lands in lastWrites — the replay must select it by URL with the match option, or it silently races the last add-to-cart instead.", + "points": 1, + "provenBy": "checkout-not-idempotent (postgres-ecom-l1.json): stripping FOR UPDATE from the cart read at checkout made two replayed checkout requests each create an order — the criterion failed on its own assertion (expected exactly 1 order containing Keyboard, found 2). The unmutated app passes, refusing the second checkout with a 400.", + "statedBy": "checking out twice must not produce two orders", + "steps": [ + { + "actor": "tab1", + "do": "click", + "testid": "checkout-submit" + }, + { + "actor": "tab1", + "do": "wait", + "ms": 2500 + }, + { + "actor": "tab2", + "do": "fill", + "enter": true, + "settleMs": 1500, + "testid": "search-input", + "text": "Webcam" + }, + { + "actor": "tab2", + "do": "click", + "in": { + "contains": "Webcam", + "testid": "item-card" + }, + "testid": "add-to-cart" + }, + { + "actor": "tab2", + "do": "wait", + "ms": 1200 + }, + { + "actor": "tab2", + "do": "click", + "testid": "cart-toggle" + }, + { + "actor": "tab2", + "do": "click", + "testid": "checkout-submit" + }, + { + "actor": "tab2", + "do": "wait", + "ms": 2500 + }, + { + "actor": "filler", + "do": "fill", + "enter": true, + "settleMs": 1500, + "testid": "search-input", + "text": "Keyboard" + }, + { + "actor": "filler", + "do": "click", + "in": { + "contains": "Keyboard", + "testid": "item-card" + }, + "testid": "add-to-cart" + }, + { + "actor": "filler", + "do": "wait", + "ms": 2000 + }, + { + "action": "checkout", + "actors": [ + "tab1", + "tab2" + ], + "do": "callConcurrently", + "settleMs": 5000 + }, + { + "accepted": 1, + "do": "expectCallOutcomes" + }, + { + "actor": "filler", + "do": "click", + "ifAvailable": true, + "testid": "overlay-close", + "unlessVisible": "order-item", + "within": 1000 + }, + { + "actor": "filler", + "do": "click", + "testid": "orders-toggle" + }, + { + "actor": "filler", + "contains": "Keyboard", + "count": 1, + "do": "expect", + "testid": "order-item" + } + ], + "withheld": "uses the spec's named `checkout` action so it can be issued on either stack. It previously replayed a captured HTTP write, which SpacetimeDB never produces — that client writes over WebSocket — so the criterion reported INCONCLUSIVE there and the two backends were scored out of different totals. Points retained: it has already failed a mutant on the stacks that could run it." + } + ], + "id": 203, + "name": "One cart, two tabs, one checkout", + "note": "Both tabs reach the item by search rather than trusting the storefront: earlier features in this suite buy things, which re-ranks the top ten. Each tab opens its OWN cart — panel state belongs to a browser context, not to the feature.", + "setup": [ + { + "actor": "tab1", + "do": "signUp", + "name": "twin" + }, + { + "actor": "tab2", + "do": "signIn", + "name": "twin" + }, + { + "actor": "filler", + "do": "signIn", + "name": "twin" + }, + { + "actor": "tab1", + "do": "fill", + "enter": true, + "settleMs": 1500, + "testid": "search-input", + "text": "Gaming Mouse" + }, + { + "actor": "tab2", + "do": "fill", + "enter": true, + "settleMs": 1500, + "testid": "search-input", + "text": "Gaming Mouse" + } + ] + } + ], + "level": 1, + "name": "Storefront and warehouse — contention (WITHHELD)", + "note": "Candidate checks score zero until qualification proves the reference passes and the declared mutation fails only its own criterion. Setup failure, inconclusive evidence, a different failed criterion, or collateral regressions do not qualify.", + "schemaVersion": 1, + "track": "ecommerce", + "writeUrlPattern": "\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses" + }, + "path": "scenarios/01-contention.json" + }, + { + "definition": { + "features": [ + { + "actors": [ + "shopper", + "impostor", + "password-owner", + "password-returning", + "password-impostor", + "password-ordinary", + "password-ordinary-returning", + "password-ordinary-impostor" + ], + "criteria": [ + { + "desc": "a visitor can create an account and is signed in as it", + "id": "1a", + "points": 1, + "statedBy": "a visitor can create an account with a username and password", + "steps": [ + { + "actor": "shopper", + "contains": "ann", + "do": "expect", + "testid": "current-user", + "within": 6000 + } + ] + }, + { + "desc": "a taken username is refused and does not sign the visitor in as the existing account", + "id": "1b", + "points": 1, + "statedBy": "signing up with a taken username fails with a visible error and\nmust never sign the visitor in as the existing account", + "steps": [ + { + "actor": "impostor", + "do": "signUp", + "expectFailure": true, + "name": "ann", + "password": "different-pw" + }, + { + "actor": "impostor", + "do": "expect", + "testid": "auth-error", + "within": 6000 + }, + { + "actor": "impostor", + "application": true, + "do": "reload", + "settleMs": 1000 + }, + { + "absent": true, + "actor": "impostor", + "do": "expect", + "testid": "current-user" + } + ] + }, + { + "desc": "the complete accepted password is required to sign in", + "id": "1c", + "points": 1, + "statedBy": "signing in with a wrong password fails with a visible error", + "steps": [ + { + "actor": "password-ordinary", + "do": "signUp", + "name": "password-ordinary", + "password": "ordinary-password-A" + }, + { + "actor": "password-ordinary", + "contains": "password-ordinary", + "do": "expect", + "testid": "current-user", + "within": 6000 + }, + { + "actor": "password-ordinary-returning", + "awaitSignedIn": false, + "do": "signIn", + "name": "password-ordinary", + "password": "ordinary-password-A" + }, + { + "actor": "password-ordinary-returning", + "contains": "password-ordinary", + "do": "expect", + "testid": "current-user", + "within": 10000 + }, + { + "actor": "password-ordinary-impostor", + "do": "signIn", + "expectFailure": true, + "name": "password-ordinary", + "password": "ordinary-password-B" + }, + { + "actor": "password-ordinary-impostor", + "do": "expect", + "testid": "auth-error", + "within": 6000 + }, + { + "actor": "password-ordinary-impostor", + "application": true, + "do": "reload", + "settleMs": 1000 + }, + { + "absent": true, + "actor": "password-ordinary-impostor", + "do": "expect", + "testid": "current-user" + }, + { + "actor": "password-owner", + "do": "probePasswordBoundary", + "impostorActor": "password-impostor", + "name": "password-boundary", + "password": "界界界界界界界界界界界界界界界界界界界界界界界界-A", + "returningActor": "password-returning", + "wrongPassword": "界界界界界界界界界界界界界界界界界界界界界界界界-B" + } + ] + }, + { + "desc": "signing out and back in returns the same account", + "id": "1d", + "points": 1, + "statedBy": "a signed-in user can sign out, returning to the signed-out state", + "steps": [ + { + "actor": "shopper", + "do": "click", + "testid": "current-user", + "unlessVisible": "signout" + }, + { + "actor": "shopper", + "do": "click", + "testid": "signout" + }, + { + "actor": "shopper", + "do": "waitUntilAbsent", + "testid": "current-user", + "within": 6000 + }, + { + "actor": "shopper", + "do": "signIn", + "name": "ann" + }, + { + "actor": "shopper", + "contains": "ann", + "do": "expect", + "testid": "current-user", + "within": 6000 + } + ] + }, + { + "desc": "the session survives a reload", + "id": "1e", + "points": 1, + "statedBy": "a signed-in session persists across a page reload", + "steps": [ + { + "actor": "shopper", + "do": "reload", + "settleMs": 4000 + }, + { + "actor": "shopper", + "contains": "ann", + "do": "expect", + "testid": "current-user", + "within": 6000 + } + ] + } + ], + "id": 1, + "name": "Accounts", + "note": "The refusals are attempted by a second, signed-out actor: a client that is already signed in has no sign-up form to submit, so re-using one actor would test nothing.", + "setup": [ + { + "actor": "shopper", + "do": "signUp", + "name": "ann" + } + ] + }, + { + "actors": [ + "buyer", + "visitor" + ], + "criteria": [ + { + "desc": "a signed-out visitor sees the catalogue with prices and stock", + "id": "2a", + "points": 1, + "statedBy": "every item shows its name, price and current stock", + "steps": [ + { + "actor": "visitor", + "contains": "Air Purifier", + "do": "expect", + "testid": "item-card" + }, + { + "actor": "visitor", + "do": "expect", + "in": { + "contains": "Air Purifier", + "testid": "item-card" + }, + "testid": "item-price" + }, + { + "actor": "visitor", + "atLeast": 1, + "do": "expectNumber", + "in": { + "contains": "Air Purifier", + "testid": "item-card" + }, + "testid": "item-stock" + } + ] + }, + { + "desc": "the storefront shows ten items, alphabetically while nothing has been bought", + "id": "2b", + "points": 1, + "statedBy": "the storefront shows the 10 most-purchased items", + "steps": [ + { + "actor": "visitor", + "count": 10, + "do": "expect", + "testid": "item-card" + }, + { + "actor": "visitor", + "contains": "Air Purifier", + "do": "expect", + "testid": "item-card" + }, + { + "actor": "visitor", + "contains": "Laptop Stand", + "do": "expect", + "testid": "item-card" + }, + { + "absent": true, + "actor": "visitor", + "contains": "Mirrorless Camera", + "do": "expect", + "testid": "item-card" + } + ] + }, + { + "desc": "a purchase moves the bought item to the front of the ranking, live", + "id": "2c", + "points": 1, + "statedBy": "when anyone buys something, every\nvisitor's storefront updates without a reload", + "steps": [ + { + "actor": "buyer", + "do": "click", + "in": { + "contains": "Coffee Grinder", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "visitor", + "do": "wait", + "ms": 2500 + }, + { + "actor": "visitor", + "contains": "Coffee Grinder", + "do": "expect", + "in": { + "testid": "item-list" + }, + "testid": "item-name" + }, + { + "actors": [ + "buyer", + "visitor" + ], + "do": "expectAgreement", + "in": { + "contains": "Coffee Grinder", + "testid": "item-card" + }, + "numeric": true, + "testid": "item-stock" + } + ] + }, + { + "desc": "search finds an item that is not on the storefront", + "id": "2d", + "points": 1, + "statedBy": "search covers the whole catalogue, not just the items currently on the storefront", + "steps": [ + { + "actor": "visitor", + "do": "fill", + "enter": true, + "settleMs": 1500, + "testid": "search-input", + "text": "Mirrorless" + }, + { + "actor": "visitor", + "contains": "Mirrorless Camera", + "do": "expect", + "in": { + "testid": "search-results" + }, + "testid": "item-card" + } + ] + } + ], + "id": 2, + "name": "Storefront is public and live", + "setup": [ + { + "actor": "buyer", + "do": "signUp", + "name": "dov" + } + ] + }, + { + "actors": [ + "buyer", + "watcher", + "visitor" + ], + "criteria": [ + { + "desc": "a signed-out visitor is offered no way to buy", + "id": "3a", + "points": 1, + "statedBy": "a signed-out visitor sees no buy or\nadd-to-cart action", + "steps": [ + { + "absent": true, + "actor": "visitor", + "do": "expect", + "testid": "buy-now" + }, + { + "absent": true, + "actor": "visitor", + "do": "expect", + "testid": "add-to-cart" + } + ] + }, + { + "desc": "buying reduces the stock every other client sees, without a reload", + "id": "3b", + "points": 2, + "statedBy": "buying an item reduces its stock by one for everyone", + "steps": [ + { + "actor": "watcher", + "do": "expectNumber", + "equals": 100, + "in": { + "contains": "Espresso Machine", + "testid": "item-card" + }, + "testid": "item-stock" + }, + { + "actor": "buyer", + "do": "click", + "in": { + "contains": "Espresso Machine", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "watcher", + "do": "expectNumber", + "equals": 99, + "in": { + "contains": "Espresso Machine", + "testid": "item-card" + }, + "testid": "item-stock" + }, + { + "actor": "visitor", + "do": "expectNumber", + "equals": 99, + "in": { + "contains": "Espresso Machine", + "testid": "item-card" + }, + "testid": "item-stock" + } + ] + }, + { + "desc": "the purchase is recorded in the buyer's order history at the price paid", + "id": "3c", + "points": 1, + "statedBy": "it creates an order for the buyer,\nvisible in their order history", + "steps": [ + { + "actor": "buyer", + "do": "click", + "in": { + "contains": "Espresso Machine", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "buyer", + "do": "click", + "ifAvailable": true, + "testid": "overlay-close", + "unlessVisible": "order-item", + "within": 1000 + }, + { + "actor": "buyer", + "do": "click", + "testid": "orders-toggle" + }, + { + "actor": "buyer", + "contains": "Espresso Machine", + "do": "expect", + "testid": "order-item" + }, + { + "actor": "buyer", + "atLeast": 449, + "do": "expectNumber", + "in": { + "contains": "Espresso Machine", + "testid": "order-item" + }, + "testid": "order-total" + }, + { + "actor": "buyer", + "do": "pressKey", + "key": "Escape", + "settleMs": 800 + } + ] + }, + { + "desc": "an item sells out visibly, and a checkout exceeding stock is refused and changes nothing", + "id": "3d", + "points": 2, + "statedBy": "an item with zero stock cannot be bought", + "steps": [ + { + "actor": "buyer", + "do": "fill", + "enter": true, + "settleMs": 1500, + "testid": "search-input", + "text": "Mirrorless Camera" + }, + { + "actor": "buyer", + "do": "click", + "in": { + "contains": "Mirrorless Camera", + "testid": "item-card" + }, + "testid": "add-to-cart" + }, + { + "actor": "buyer", + "do": "wait", + "ms": 800 + }, + { + "actor": "buyer", + "do": "click", + "in": { + "contains": "Mirrorless Camera", + "testid": "item-card" + }, + "testid": "add-to-cart" + }, + { + "actor": "buyer", + "do": "wait", + "ms": 800 + }, + { + "actor": "buyer", + "do": "click", + "in": { + "contains": "Mirrorless Camera", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "buyer", + "do": "wait", + "ms": 800 + }, + { + "actor": "buyer", + "do": "click", + "in": { + "contains": "Mirrorless Camera", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "buyer", + "do": "wait", + "ms": 800 + }, + { + "actor": "buyer", + "do": "click", + "in": { + "contains": "Mirrorless Camera", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "buyer", + "do": "expectNumber", + "equals": 0, + "in": { + "contains": "Mirrorless Camera", + "testid": "item-card" + }, + "testid": "item-stock" + }, + { + "actor": "buyer", + "do": "expect", + "in": { + "contains": "Mirrorless Camera", + "testid": "item-card" + }, + "testid": "out-of-stock" + }, + { + "actor": "buyer", + "do": "click", + "testid": "cart-toggle" + }, + { + "actor": "buyer", + "do": "click", + "testid": "checkout-submit" + }, + { + "actor": "buyer", + "do": "expect", + "testid": "buy-error", + "within": 6000 + }, + { + "actor": "buyer", + "contains": "Mirrorless Camera", + "do": "expect", + "testid": "cart-item" + }, + { + "actor": "buyer", + "do": "pressKey", + "key": "Escape", + "settleMs": 800 + }, + { + "actor": "buyer", + "do": "expectNumber", + "equals": 0, + "in": { + "contains": "Mirrorless Camera", + "testid": "item-card" + }, + "testid": "item-stock" + } + ] + } + ], + "id": 3, + "name": "Buying", + "setup": [ + { + "actor": "buyer", + "do": "signUp", + "name": "eli" + }, + { + "actor": "watcher", + "do": "signUp", + "name": "fay" + } + ] + }, + { + "actors": [ + "tab1", + "tab2" + ], + "criteria": [ + { + "desc": "adding the same item twice raises its quantity instead of adding a second line", + "id": "4a", + "points": 1, + "statedBy": "adding an item that is already in the cart raises its quantity", + "steps": [ + { + "actor": "tab1", + "do": "click", + "in": { + "contains": "Headphones", + "testid": "item-card" + }, + "testid": "add-to-cart" + }, + { + "actor": "tab1", + "do": "wait", + "ms": 800 + }, + { + "actor": "tab1", + "do": "click", + "in": { + "contains": "Headphones", + "testid": "item-card" + }, + "testid": "add-to-cart" + }, + { + "actor": "tab1", + "do": "click", + "testid": "cart-toggle" + }, + { + "actor": "tab1", + "contains": "Headphones", + "count": 1, + "do": "expect", + "testid": "cart-item" + }, + { + "actor": "tab1", + "do": "expectNumber", + "equals": 2, + "in": { + "contains": "Headphones", + "testid": "cart-item" + }, + "testid": "cart-quantity" + }, + { + "actor": "tab1", + "do": "pressKey", + "key": "Escape", + "settleMs": 800 + } + ] + }, + { + "desc": "the cart survives a reload", + "id": "4b", + "points": 1, + "statedBy": "the cart survives a reload", + "steps": [ + { + "actor": "tab1", + "do": "fill", + "enter": true, + "settleMs": 1500, + "testid": "search-input", + "text": "Laptop Stand" + }, + { + "actor": "tab1", + "do": "click", + "in": { + "contains": "Laptop Stand", + "testid": "item-card" + }, + "testid": "add-to-cart" + }, + { + "actor": "tab1", + "do": "click", + "testid": "cart-toggle" + }, + { + "actor": "tab1", + "contains": "Laptop Stand", + "do": "expect", + "testid": "cart-item", + "within": 10000 + }, + { + "actor": "tab1", + "do": "reload", + "settleMs": 4000 + }, + { + "actor": "tab1", + "do": "ensureSignedIn", + "name": "gus", + "readyTestid": "current-user" + }, + { + "actor": "tab1", + "do": "click", + "testid": "cart-toggle" + }, + { + "actor": "tab1", + "contains": "Laptop Stand", + "do": "expect", + "testid": "cart-item" + }, + { + "actor": "tab1", + "do": "pressKey", + "key": "Escape", + "settleMs": 800 + } + ] + }, + { + "desc": "the same account signed in elsewhere sees one cart, live", + "id": "4c", + "points": 2, + "statedBy": "the same account signed\nin twice sees one cart, and a change made in one place appears in the other without a\nreload", + "steps": [ + { + "actor": "tab2", + "do": "signIn", + "name": "gus" + }, + { + "actor": "tab2", + "do": "click", + "testid": "cart-toggle" + }, + { + "actor": "tab2", + "contains": "Laptop Stand", + "do": "expect", + "testid": "cart-item" + }, + { + "actor": "tab1", + "do": "fill", + "enter": true, + "settleMs": 1500, + "testid": "search-input", + "text": "Induction Cooktop" + }, + { + "actor": "tab1", + "do": "click", + "in": { + "contains": "Induction Cooktop", + "testid": "item-card" + }, + "testid": "add-to-cart" + }, + { + "actor": "tab2", + "contains": "Induction Cooktop", + "do": "expect", + "testid": "cart-item", + "within": 10000 + } + ] + }, + { + "desc": "checkout creates one order, reduces stock and empties the cart", + "id": "4d", + "points": 2, + "statedBy": "checkout turns the cart into one order, reduces stock for every line, and\nempties the cart", + "steps": [ + { + "actor": "tab1", + "do": "click", + "testid": "cart-toggle" + }, + { + "actor": "tab1", + "do": "click", + "testid": "checkout-submit" + }, + { + "actor": "tab1", + "do": "wait", + "ms": 2000 + }, + { + "actor": "tab1", + "do": "expect", + "testid": "empty-cart" + }, + { + "actor": "tab1", + "do": "pressKey", + "key": "Escape", + "settleMs": 800 + }, + { + "actor": "tab1", + "do": "fill", + "enter": true, + "settleMs": 1200, + "testid": "search-input", + "text": "" + }, + { + "actor": "tab1", + "do": "expectNumber", + "equals": 98, + "in": { + "contains": "Headphones", + "testid": "item-card" + }, + "testid": "item-stock" + }, + { + "actor": "tab1", + "do": "click", + "ifAvailable": true, + "testid": "overlay-close", + "unlessVisible": "order-item", + "within": 1000 + }, + { + "actor": "tab1", + "do": "click", + "testid": "orders-toggle" + }, + { + "actor": "tab1", + "contains": "Headphones", + "do": "expect", + "testid": "order-item" + } + ] + } + ], + "id": 4, + "name": "Cart belongs to the account", + "setup": [ + { + "actor": "tab1", + "do": "signUp", + "name": "gus" + } + ] + }, + { + "actors": [ + "author", + "other", + "visitor" + ], + "criteria": [ + { + "desc": "a customer can review an item and everyone sees it, signed out included", + "id": "6a", + "points": 2, + "statedBy": "reviews are visible to everyone, including signed-out visitors", + "steps": [ + { + "actor": "author", + "do": "fill", + "enter": true, + "settleMs": 1500, + "testid": "search-input", + "text": "Induction Cooktop" + }, + { + "actor": "author", + "do": "openItem", + "item": "Induction Cooktop" + }, + { + "actor": "author", + "do": "click", + "ifAvailable": true, + "testid": "review-toggle", + "unlessVisible": "review-rating" + }, + { + "actor": "author", + "do": "fill", + "testid": "review-rating", + "text": "4" + }, + { + "actor": "author", + "do": "fill", + "testid": "review-input", + "text": "solid mold" + }, + { + "actor": "author", + "do": "click", + "testid": "review-submit" + }, + { + "actor": "author", + "contains": "solid mold", + "do": "expect", + "testid": "review-item" + }, + { + "actor": "visitor", + "do": "openItem", + "item": "Induction Cooktop" + }, + { + "actor": "visitor", + "contains": "solid mold", + "do": "expect", + "testid": "review-item", + "within": 10000 + } + ] + }, + { + "desc": "one review per customer per item — a second does not add a duplicate", + "id": "6b", + "points": 1, + "statedBy": "one review per customer per item", + "steps": [ + { + "actor": "author", + "do": "click", + "ifAvailable": true, + "testid": "review-toggle", + "unlessVisible": "review-rating" + }, + { + "actor": "author", + "do": "fill", + "testid": "review-rating", + "text": "2" + }, + { + "actor": "author", + "do": "fill", + "testid": "review-input", + "text": "changed my mind" + }, + { + "actor": "author", + "do": "click", + "testid": "review-submit" + }, + { + "actor": "author", + "do": "wait", + "ms": 2000 + }, + { + "absent": true, + "actor": "author", + "contains": "solid mold", + "do": "expect", + "testid": "review-item" + } + ] + }, + { + "desc": "the average rating reflects both reviewers, live", + "id": "6c", + "points": 1, + "statedBy": "each item shows its average rating, which updates live as reviews arrive", + "steps": [ + { + "actor": "other", + "do": "fill", + "enter": true, + "settleMs": 1500, + "testid": "search-input", + "text": "Induction Cooktop" + }, + { + "actor": "other", + "do": "openItem", + "item": "Induction Cooktop" + }, + { + "actor": "other", + "do": "click", + "ifAvailable": true, + "testid": "review-toggle", + "unlessVisible": "review-rating" + }, + { + "actor": "other", + "do": "fill", + "testid": "review-rating", + "text": "4" + }, + { + "actor": "other", + "do": "fill", + "testid": "review-input", + "text": "does the job" + }, + { + "actor": "other", + "do": "click", + "testid": "review-submit" + }, + { + "actor": "other", + "do": "expectNumber", + "equals": 3, + "testid": "review-average", + "within": 10000 + }, + { + "actors": [ + "author", + "other" + ], + "do": "expectAgreement", + "numeric": true, + "testid": "review-average" + } + ] + } + ], + "id": 6, + "name": "Reviews", + "note": "Both reviewers buy the item in setup: reviews are gated on a purchase, so an unpurchased reviewer would be refused by a compliant app.", + "setup": [ + { + "actor": "author", + "do": "signUp", + "name": "hal" + }, + { + "actor": "other", + "do": "signUp", + "name": "ivy" + }, + { + "actor": "author", + "do": "click", + "in": { + "contains": "Induction Cooktop", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "author", + "do": "wait", + "ms": 800 + }, + { + "actor": "other", + "do": "click", + "in": { + "contains": "Induction Cooktop", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "other", + "do": "wait", + "ms": 800 + } + ] + }, + { + "actors": [ + "admin", + "authorized", + "customer", + "visitor" + ], + "criteria": [ + { + "desc": "the admin area is for admins only", + "id": "7a", + "points": 1, + "statedBy": "admin accounts see an admin area; customers do not", + "steps": [ + { + "actor": "authorized", + "do": "signIn", + "exact": true, + "name": "admin", + "password": "stackbench-admin-2026" + }, + { + "actor": "authorized", + "do": "click", + "testid": "admin-link" + }, + { + "actor": "authorized", + "do": "expect", + "testid": "admin-item-row" + }, + { + "actor": "customer", + "do": "click", + "ifAvailable": true, + "settleMs": 1500, + "testid": "admin-link" + }, + { + "absent": true, + "actor": "customer", + "do": "expect", + "testid": "admin-item-row" + } + ] + }, + { + "desc": "admin lists every item, every warehouse, and what each warehouse holds", + "id": "7b", + "points": 1, + "statedBy": "admin lists every item with its stock, every warehouse, and the stock of\neach item in each warehouse", + "steps": [ + { + "actor": "admin", + "do": "click", + "testid": "admin-link" + }, + { + "actor": "admin", + "count": 13, + "do": "expect", + "testid": "admin-item-row" + }, + { + "actor": "admin", + "contains": "East", + "do": "expect", + "testid": "admin-warehouse-item" + }, + { + "actor": "admin", + "contains": "West", + "do": "expect", + "testid": "admin-warehouse-item" + }, + { + "actor": "admin", + "do": "expectNumber", + "equals": 100, + "in": { + "contains": "Gaming Mouse", + "testid": "admin-item-row" + }, + "testid": "admin-stock" + } + ] + }, + { + "desc": "the storefront stock is the sum across warehouses, and a restock raises it live", + "id": "7c", + "points": 2, + "statedBy": "an item's stock on the storefront is the sum of that item's units across all\nwarehouses", + "steps": [ + { + "actor": "visitor", + "do": "pressKey", + "key": "Escape", + "settleMs": 800 + }, + { + "actor": "visitor", + "do": "fill", + "enter": true, + "settleMs": 1500, + "testid": "search-input", + "text": "Gaming Mouse" + }, + { + "actor": "visitor", + "do": "expectNumber", + "equals": 100, + "in": { + "contains": "Gaming Mouse", + "testid": "item-card" + }, + "testid": "item-stock" + }, + { + "actor": "admin", + "do": "fill", + "in": { + "contains": "Gaming Mouse", + "testid": "admin-location-row" + }, + "testid": "restock-input", + "text": "25" + }, + { + "actor": "admin", + "do": "click", + "in": { + "contains": "Gaming Mouse", + "testid": "admin-location-row" + }, + "testid": "restock-submit" + }, + { + "actor": "visitor", + "do": "expectNumber", + "equals": 125, + "in": { + "contains": "Gaming Mouse", + "testid": "item-card" + }, + "testid": "item-stock", + "within": 10000 + }, + { + "actor": "admin", + "do": "expectNumber", + "equals": 125, + "in": { + "contains": "Gaming Mouse", + "testid": "admin-item-row" + }, + "testid": "admin-stock" + } + ] + } + ], + "id": 7, + "name": "Admin and warehouses", + "setup": [ + { + "actor": "customer", + "do": "signUp", + "name": "jon" + }, + { + "actor": "admin", + "do": "signIn", + "exact": true, + "name": "admin", + "password": "stackbench-admin-2026" + } + ] + } + ], + "level": 1, + "name": "Storefront and warehouse — features", + "note": "Each feature is given its own seeded items where it mutates stock, because features share one database and a purchase in one would otherwise move another's numbers. Items reserved per feature: F2 Coffee Grinder, F3 Espresso Machine + Mirrorless Camera, F4 Headphones + Laptop Stand, F6 Induction Cooktop, F7 Gaming Mouse. Panels are closed after use and items are reached via search, because the storefront shows only the top ten and earlier features change that ranking.", + "schemaVersion": 1, + "track": "ecommerce", + "writeUrlPattern": "\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses" + }, + "path": "scenarios/01-features.json" + }, + { + "definition": { + "features": [ + { + "actors": [ + "buyer", + "guest" + ], + "criteria": [ + { + "desc": "a purchase request replayed without a session creates no order and moves no stock", + "id": "101a", + "points": 2, + "statedBy": "the server must refuse a purchase request that arrives without a valid session", + "steps": [ + { + "actor": "buyer", + "do": "expectNumber", + "equals": 100, + "in": { + "contains": "Bluetooth Speaker", + "testid": "item-card" + }, + "testid": "item-stock" + }, + { + "actor": "buyer", + "do": "click", + "in": { + "contains": "Bluetooth Speaker", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "buyer", + "do": "wait", + "ms": 1500 + }, + { + "actor": "guest", + "do": "replayAs", + "from": "buyer", + "match": "buy", + "settleMs": 2500 + }, + { + "actor": "guest", + "do": "expectReplayRejected" + }, + { + "actor": "buyer", + "do": "expectNumber", + "equals": 99, + "in": { + "contains": "Bluetooth Speaker", + "testid": "item-card" + }, + "testid": "item-stock" + } + ] + } + ], + "id": 101, + "name": "Purchase requires an account", + "setup": [ + { + "actor": "buyer", + "do": "signUp", + "name": "kim" + } + ] + }, + { + "actors": [ + "victim", + "attacker" + ], + "criteria": [ + { + "desc": "an order cannot be placed in someone else's name by editing the request", + "id": "102a", + "points": 2, + "statedBy": "knowing someone's username must never grant access to it", + "steps": [ + { + "actor": "victim", + "do": "click", + "in": { + "contains": "Coffee Grinder", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "attacker", + "do": "click", + "in": { + "contains": "Desk Lamp", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "attacker", + "do": "wait", + "ms": 1500 + }, + { + "actor": "attacker", + "do": "forgeWrite", + "fromActor": "victim", + "settleMs": 2500 + }, + { + "actor": "attacker", + "do": "expectForgeryRejected" + }, + { + "actor": "victim", + "do": "click", + "ifAvailable": true, + "testid": "overlay-close", + "unlessVisible": "order-item", + "within": 1000 + }, + { + "actor": "victim", + "do": "click", + "testid": "orders-toggle" + }, + { + "actor": "victim", + "contains": "Coffee Grinder", + "count": 1, + "do": "expect", + "testid": "order-item" + }, + { + "absent": true, + "actor": "victim", + "contains": "Desk Lamp", + "do": "expect", + "testid": "order-item" + } + ] + } + ], + "id": 102, + "name": "Purchases are attributed to whoever made them", + "setup": [ + { + "actor": "victim", + "do": "signUp", + "name": "lee" + }, + { + "actor": "attacker", + "do": "signUp", + "name": "mel" + } + ] + }, + { + "actors": [ + "admin", + "customer" + ], + "criteria": [ + { + "desc": "a restock replayed by a customer is refused and changes no stock", + "id": "103a", + "points": 2, + "statedBy": "the server must refuse admin actions from a non-admin account", + "steps": [ + { + "actor": "admin", + "do": "click", + "testid": "admin-link" + }, + { + "actor": "admin", + "do": "fill", + "in": { + "contains": "Air Purifier", + "testid": "admin-location-row" + }, + "testid": "restock-input", + "text": "10" + }, + { + "actor": "admin", + "do": "click", + "in": { + "contains": "Air Purifier", + "testid": "admin-location-row" + }, + "testid": "restock-submit" + }, + { + "actor": "admin", + "do": "wait", + "ms": 2000 + }, + { + "actor": "customer", + "do": "pressKey", + "key": "Escape", + "settleMs": 800 + }, + { + "actor": "customer", + "as": "purifier-after-admin-restock", + "do": "recordNumber", + "in": { + "contains": "Air Purifier", + "testid": "item-card" + }, + "testid": "item-stock" + }, + { + "actor": "customer", + "do": "replayAs", + "from": "admin", + "match": "restock", + "settleMs": 2500 + }, + { + "actor": "customer", + "do": "expectReplayRejected" + }, + { + "actor": "customer", + "do": "expectNumber", + "in": { + "contains": "Air Purifier", + "testid": "item-card" + }, + "plus": 0, + "relativeTo": "purifier-after-admin-restock", + "testid": "item-stock" + } + ] + } + ], + "id": 103, + "name": "Stock is administered by admins only", + "setup": [ + { + "actor": "admin", + "do": "signIn", + "exact": true, + "name": "admin", + "password": "stackbench-admin-2026" + }, + { + "actor": "customer", + "do": "signUp", + "name": "ned" + } + ] + }, + { + "actors": [ + "buyer" + ], + "criteria": [ + { + "desc": "a purchase request with the price edited does not sell the item for less", + "id": "104a", + "points": 2, + "statedBy": "the price at the time of purchase", + "steps": [ + { + "actor": "buyer", + "do": "click", + "in": { + "contains": "Espresso Machine", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "buyer", + "do": "wait", + "ms": 1500 + }, + { + "actor": "buyer", + "do": "replayAs", + "from": "buyer", + "match": "buy", + "settleMs": 2500, + "swap": { + "find": "449", + "with": "1" + } + }, + { + "actor": "buyer", + "do": "expectReplayRejected" + }, + { + "actor": "buyer", + "do": "click", + "ifAvailable": true, + "testid": "overlay-close", + "unlessVisible": "order-item", + "within": 1000 + }, + { + "actor": "buyer", + "do": "click", + "testid": "orders-toggle" + }, + { + "actor": "buyer", + "atLeast": 449, + "do": "expectNumber", + "in": { + "contains": "Espresso Machine", + "testid": "order-item" + }, + "testid": "order-total" + } + ] + } + ], + "id": 104, + "name": "The price is the store's to set", + "setup": [ + { + "actor": "buyer", + "do": "signUp", + "name": "oli" + } + ] + }, + { + "actors": [ + "shopper" + ], + "criteria": [ + { + "desc": "cart and order history survive a reload as the same account", + "id": "105a", + "points": 1, + "statedBy": "persists across a page reload", + "steps": [ + { + "actor": "shopper", + "do": "click", + "in": { + "contains": "Keyboard", + "testid": "item-card" + }, + "testid": "add-to-cart" + }, + { + "actor": "shopper", + "do": "click", + "in": { + "contains": "Headphones", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "shopper", + "do": "reload", + "settleMs": 4000 + }, + { + "actor": "shopper", + "contains": "pat", + "do": "expect", + "testid": "current-user" + }, + { + "actor": "shopper", + "do": "click", + "testid": "cart-toggle" + }, + { + "actor": "shopper", + "contains": "Keyboard", + "do": "expect", + "testid": "cart-item" + }, + { + "actor": "shopper", + "do": "click", + "ifAvailable": true, + "testid": "overlay-close", + "unlessVisible": "order-item", + "within": 1000 + }, + { + "actor": "shopper", + "do": "click", + "testid": "orders-toggle" + }, + { + "actor": "shopper", + "contains": "Headphones", + "do": "expect", + "testid": "order-item" + } + ] + }, + { + "desc": "the same account survives the connection dropping and coming back", + "id": "105b", + "points": 1, + "statedBy": "across the connection dropping and re-establishing", + "steps": [ + { + "actor": "shopper", + "do": "setOffline", + "offline": true, + "settleMs": 3000 + }, + { + "actor": "shopper", + "do": "setOffline", + "offline": false, + "settleMs": 6000 + }, + { + "actor": "shopper", + "contains": "pat", + "do": "expect", + "testid": "current-user" + }, + { + "actor": "shopper", + "do": "click", + "testid": "cart-toggle" + }, + { + "actor": "shopper", + "contains": "Keyboard", + "do": "expect", + "testid": "cart-item" + } + ] + } + ], + "id": 105, + "name": "An account keeps what belongs to it", + "setup": [ + { + "actor": "shopper", + "do": "signUp", + "name": "pat" + } + ] + }, + { + "actors": [ + "one", + "two" + ], + "criteria": [ + { + "desc": "a customer's order history contains only their own orders", + "id": "106a", + "points": 1, + "statedBy": "a customer sees only their own orders", + "steps": [ + { + "actor": "one", + "do": "click", + "in": { + "contains": "Coffee Grinder", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "one", + "do": "wait", + "ms": 2000 + }, + { + "actor": "two", + "do": "click", + "ifAvailable": true, + "testid": "overlay-close", + "unlessVisible": "order-item", + "within": 1000 + }, + { + "actor": "two", + "do": "click", + "testid": "orders-toggle" + }, + { + "absent": true, + "actor": "two", + "contains": "Coffee Grinder", + "do": "expect", + "testid": "order-item" + } + ] + } + ], + "id": 106, + "name": "One customer's orders are not another's", + "setup": [ + { + "actor": "one", + "do": "signUp", + "name": "quin" + }, + { + "actor": "two", + "do": "signUp", + "name": "ros" + } + ] + }, + { + "actors": [ + "buyer", + "admin" + ], + "criteria": [ + { + "desc": "revenue rises by exactly what was bought", + "id": "107a", + "points": 1, + "statedBy": "the sum of all order totals", + "steps": [ + { + "actor": "admin", + "do": "click", + "testid": "admin-link" + }, + { + "actor": "admin", + "as": "revenue-before", + "do": "recordNumber", + "testid": "admin-revenue" + }, + { + "actor": "admin", + "as": "brick-before", + "do": "recordNumber", + "in": { + "contains": "Laptop Stand", + "testid": "admin-item-row" + }, + "testid": "admin-stock" + }, + { + "actor": "buyer", + "do": "pressKey", + "key": "Escape", + "settleMs": 800 + }, + { + "actor": "buyer", + "do": "click", + "in": { + "contains": "Laptop Stand", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "buyer", + "do": "wait", + "ms": 1000 + }, + { + "actor": "buyer", + "do": "click", + "in": { + "contains": "Laptop Stand", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "buyer", + "do": "wait", + "ms": 2000 + }, + { + "actor": "admin", + "do": "expectNumber", + "plus": 58, + "relativeTo": "revenue-before", + "testid": "admin-revenue", + "within": 10000 + } + ] + }, + { + "desc": "what the store sold is what left the warehouses, and a fresh client agrees", + "id": "107b", + "points": 1, + "statedBy": "must always equal what the orders themselves add up to", + "steps": [ + { + "actor": "admin", + "do": "expectNumber", + "in": { + "contains": "Laptop Stand", + "testid": "admin-item-row" + }, + "plus": -2, + "relativeTo": "brick-before", + "testid": "admin-stock" + }, + { + "actor": "buyer", + "do": "freshClient" + }, + { + "actor": "buyer-fresh", + "do": "expectNumber", + "in": { + "contains": "Laptop Stand", + "testid": "item-card" + }, + "plus": -2, + "relativeTo": "brick-before", + "testid": "item-stock" + } + ] + } + ], + "id": 107, + "name": "The books balance", + "setup": [ + { + "actor": "buyer", + "do": "signUp", + "name": "sam" + }, + { + "actor": "admin", + "do": "signIn", + "exact": true, + "name": "admin", + "password": "stackbench-admin-2026" + } + ] + }, + { + "actors": [ + "owner", + "stranger" + ], + "criteria": [ + { + "desc": "someone who never bought the item cannot review it", + "id": "108a", + "points": 2, + "statedBy": "refuses a review from someone who has never ordered the item", + "steps": [ + { + "actor": "stranger", + "do": "fill", + "enter": true, + "settleMs": 1500, + "testid": "search-input", + "text": "Webcam" + }, + { + "actor": "stranger", + "do": "openItem", + "item": "Webcam" + }, + { + "actor": "stranger", + "do": "click", + "ifAvailable": true, + "testid": "review-toggle", + "unlessVisible": "review-rating" + }, + { + "actor": "stranger", + "do": "fill", + "testid": "review-rating", + "text": "5" + }, + { + "actor": "stranger", + "do": "fill", + "testid": "review-input", + "text": "never bought this" + }, + { + "actor": "stranger", + "do": "click", + "testid": "review-submit" + }, + { + "actor": "stranger", + "do": "expect", + "testid": "review-error", + "within": 6000 + }, + { + "absent": true, + "actor": "stranger", + "contains": "never bought this", + "do": "expect", + "testid": "review-item" + } + ] + }, + { + "desc": "buying the item earns the right to review it", + "id": "108b", + "points": 1, + "statedBy": "a review of an item they have bought", + "steps": [ + { + "actor": "owner", + "do": "fill", + "enter": true, + "settleMs": 1500, + "testid": "search-input", + "text": "Webcam" + }, + { + "actor": "owner", + "do": "click", + "in": { + "contains": "Webcam", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "owner", + "do": "wait", + "ms": 2000 + }, + { + "actor": "owner", + "do": "openItem", + "item": "Webcam" + }, + { + "actor": "owner", + "do": "click", + "ifAvailable": true, + "testid": "review-toggle", + "unlessVisible": "review-rating" + }, + { + "actor": "owner", + "do": "fill", + "testid": "review-rating", + "text": "4" + }, + { + "actor": "owner", + "do": "fill", + "testid": "review-input", + "text": "bought and used it" + }, + { + "actor": "owner", + "do": "click", + "testid": "review-submit" + }, + { + "actor": "owner", + "contains": "bought and used it", + "do": "expect", + "testid": "review-item", + "within": 8000 + } + ] + } + ], + "id": 108, + "name": "A review is a claim about a purchase", + "setup": [ + { + "actor": "owner", + "do": "signUp", + "name": "tam" + }, + { + "actor": "stranger", + "do": "signUp", + "name": "uma" + } + ] + }, + { + "actors": [ + "owner", + "stranger" + ], + "criteria": [ + { + "desc": "one customer’s cart request replayed by another leaves the owner’s cart untouched", + "id": "109a", + "note": "A cart add request that contains only an item and quantity is legitimately reusable by another authenticated customer: the server applies it to the caller's own cart. The ownership assertion below is the security boundary; requiring every replay to return non-2xx incorrectly rejects identity-derived APIs.", + "points": 2, + "statedBy": "nobody else can read or change it", + "steps": [ + { + "actor": "owner", + "do": "click", + "in": { + "contains": "Coffee Grinder", + "testid": "item-card" + }, + "testid": "add-to-cart" + }, + { + "actor": "owner", + "do": "wait", + "ms": 1500 + }, + { + "actor": "stranger", + "do": "replayAs", + "from": "owner", + "match": "cart", + "settleMs": 2500 + }, + { + "actor": "owner", + "do": "click", + "testid": "cart-toggle" + }, + { + "actor": "owner", + "do": "expectNumber", + "equals": 1, + "in": { + "contains": "Coffee Grinder", + "testid": "cart-item" + }, + "testid": "cart-quantity" + } + ] + }, + { + "desc": "a quantity below one is refused rather than credited", + "id": "109b", + "points": 1, + "statedBy": "a request carrying a negative quantity is refused and changes nothing", + "steps": [ + { + "actor": "owner", + "do": "replayAs", + "from": "owner", + "match": "cart", + "settleMs": 2500, + "swap": { + "find": "\"quantity\":1", + "with": "\"quantity\":-3" + } + }, + { + "actor": "owner", + "atLeast": 0, + "do": "expectNumber", + "testid": "cart-total" + }, + { + "actor": "owner", + "atLeast": 1, + "do": "expectNumber", + "in": { + "contains": "Coffee Grinder", + "testid": "cart-item" + }, + "testid": "cart-quantity" + } + ] + } + ], + "id": 109, + "name": "A cart is nobody else’s business", + "setup": [ + { + "actor": "owner", + "do": "signUp", + "name": "vic" + }, + { + "actor": "stranger", + "do": "signUp", + "name": "wes" + } + ] + } + ], + "level": 1, + "name": "Storefront and warehouse — invariants", + "note": "Cross-cutting properties, scored separately from features because an app can implement every feature and still let anyone spend anyone else's money. Each is stated as an outcome a customer would recognise, not as a protocol detail — a backend must not lose points for being harder to inspect, nor gain them for it. 108 and 109 cover what a customer may assert about things that are not theirs: a rating is a claim about a purchase, and a cart belongs to exactly one account.", + "schemaVersion": 1, + "track": "ecommerce", + "writeUrlPattern": "\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses" + }, + "path": "scenarios/01-invariants.json" + }, + { + "definition": { + "features": [ + { + "actors": [ + "viewer" + ], + "criteria": [ + { + "desc": "a back-office script sets Desk Lamp's East stock to 5, directly in the database; an anonymous viewer with the storefront already open sees 50 without touching the page", + "id": "901a", + "note": "WITHHELD until proven against a mutant (backoffice-writes-via-server) and a real failing build. The script is the app's own scripts/backoffice.mjs, required by the spec to write to the database without the web server's help — the way a cron job or another team's service would.", + "points": 0, + "statedBy": "The storefront reflects current database values after changes made outside the application server.", + "steps": [ + { + "do": "dbSetStock", + "item": "Desk Lamp", + "quantity": 5, + "settleMs": 4000, + "warehouse": "East" + }, + { + "actor": "viewer", + "do": "expectNumber", + "equals": 50, + "in": { + "contains": "Desk Lamp", + "testid": "item-card" + }, + "testid": "item-stock", + "within": 10000 + } + ] + }, + { + "desc": "the out-of-band change survives a reload — the page did not merely patch its own memory", + "id": "901b", + "note": "WITHHELD — promoted together with 901a.", + "points": 0, + "statedBy": "the app must remain correct - and its open pages current", + "steps": [ + { + "actor": "viewer", + "do": "reload", + "settleMs": 3000 + }, + { + "actor": "viewer", + "do": "expectNumber", + "equals": 50, + "in": { + "contains": "Desk Lamp", + "testid": "item-card" + }, + "testid": "item-stock", + "within": 10000 + } + ] + }, + { + "desc": "the client, not the server, misses the window: a correction lands while this browser is disconnected, and the page is current again once it reconnects", + "id": "901d", + "points": 1, + "steps": [ + { + "actor": "viewer", + "do": "setOffline", + "offline": true + }, + { + "do": "dbSetStock", + "item": "Desk Lamp", + "quantity": 7, + "settleMs": 4000, + "warehouse": "East" + }, + { + "actor": "viewer", + "do": "setOffline", + "offline": false + }, + { + "actor": "viewer", + "do": "expectNumber", + "equals": 17, + "in": { + "contains": "Desk Lamp", + "testid": "item-card" + }, + "testid": "item-stock", + "within": 20000 + } + ], + "withheld": "WITHHELD until it has failed one real build and passed another, the same gate 901c cleared. Distinct from 901c: there the SERVER was down, so nothing was listening at all. Here the server stays up and sees the write; it is this CLIENT that was disconnected. An app that resumes its stream on reconnect without asking for current state stays wrong, and a dropped connection is far more common than a deploy — a closed laptop, a tunnel, a flaky phone." + } + ], + "id": 901, + "name": "A write from outside the app is visible without reload", + "setup": [ + { + "actor": "viewer", + "do": "expectNumber", + "equals": 100, + "in": { + "contains": "Desk Lamp", + "testid": "item-card" + }, + "testid": "item-stock" + } + ] + }, + { + "actors": [ + "reader", + "reviewer" + ], + "criteria": [ + { + "desc": "one customer opens the Keyboard's reviews at the same moment another posts one: the opener ends with that review exactly once — not zero (stale fetch), not twice (fetch merged with its own live echo)", + "id": "902a", + "note": "WITHHELD until proven against a mutant (fetch-then-append-dupe) and a real failing build. The race window is real and the criterion may need repetition to be reliable; INCONCLUSIVE on a missed race, never a false FAIL.", + "points": 0, + "steps": [ + { + "branches": [ + [ + { + "actor": "reader", + "do": "openItem", + "item": "Keyboard" + } + ], + [ + { + "actor": "reviewer", + "do": "click", + "ifAvailable": true, + "testid": "review-toggle", + "unlessVisible": "review-rating" + }, + { + "actor": "reviewer", + "do": "fill", + "testid": "review-input", + "text": "race-review-kbd" + }, + { + "actor": "reviewer", + "do": "click", + "testid": "review-submit" + } + ] + ], + "do": "race", + "settleMs": 4000 + }, + { + "actor": "reader", + "contains": "race-review-kbd", + "do": "expectElementCount", + "equals": 1, + "testid": "review-item", + "within": 10000 + } + ] + } + ], + "id": 902, + "name": "A list being opened while its contents change", + "setup": [ + { + "actor": "reviewer", + "do": "signUp", + "name": "raceR" + }, + { + "actor": "reader", + "do": "signUp", + "name": "raceL" + }, + { + "actor": "reviewer", + "do": "click", + "in": { + "contains": "Keyboard", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "reviewer", + "do": "wait", + "ms": 800 + }, + { + "actor": "reviewer", + "do": "openItem", + "item": "Keyboard", + "settleMs": 1200 + }, + { + "actor": "reviewer", + "do": "click", + "ifAvailable": true, + "testid": "review-toggle", + "unlessVisible": "review-rating" + }, + { + "actor": "reviewer", + "do": "fill", + "testid": "review-rating", + "text": "5" + } + ] + } + ], + "level": 1, + "name": "Storefront and warehouse — systems", + "note": "Properties of the system, not of any one feature: what happens when the database changes underneath the app, and when two things happen at the same time. These are the failures that appear in production and never in a demo — a broadcast layer only announces writes that went through it, and a fetch-then-merge list drops or duplicates whatever moved while it was fetching. Every criterion is withheld at 0 points until it demonstrably fails a real build AND its mutant is CAUGHT, the same promotion rule as contention.", + "schemaVersion": 1, + "track": "ecommerce", + "writeUrlPattern": "\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses" + }, + "path": "scenarios/01-systems.json" + }, + { + "definition": { + "features": [ + { + "actors": [ + "customer", + "staff", + "admin" + ], + "criteria": [ + { + "desc": "an order placed by a customer appears in the staff queue without a reload", + "id": "1a", + "points": 2, + "statedBy": "appears in the fulfilment queue in the order it was placed", + "steps": [ + { + "actor": "staff", + "do": "click", + "testid": "staff-link" + }, + { + "actor": "staff", + "as": "depth-before", + "do": "recordNumber", + "testid": "queue-depth" + }, + { + "actor": "customer", + "do": "fill", + "enter": true, + "settleMs": 1500, + "testid": "search-input", + "text": "Desk Lamp" + }, + { + "actor": "customer", + "do": "click", + "in": { + "contains": "Desk Lamp", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "staff", + "contains": "Desk Lamp", + "do": "expect", + "testid": "queue-item", + "within": 10000 + }, + { + "actor": "staff", + "do": "expectNumber", + "plus": 1, + "relativeTo": "depth-before", + "testid": "queue-depth" + } + ] + }, + { + "desc": "the queue names the warehouse the order will ship from", + "id": "1b", + "points": 1, + "statedBy": "which warehouse each will ship from", + "steps": [ + { + "actor": "staff", + "do": "expect", + "in": { + "contains": "Desk Lamp", + "testid": "queue-item" + }, + "testid": "queue-warehouse" + } + ] + }, + { + "desc": "shipping removes the order from the queue for staff and marks it shipped for the customer, live", + "id": "1c", + "points": 2, + "statedBy": "it leaves the queue for everyone watching the queue, and the customer's order history shows it as shipped, live", + "steps": [ + { + "actor": "staff", + "do": "click", + "in": { + "contains": "Desk Lamp", + "testid": "queue-item" + }, + "testid": "ship-submit" + }, + { + "absent": true, + "actor": "staff", + "contains": "Desk Lamp", + "do": "expect", + "testid": "queue-item", + "within": 10000 + }, + { + "actor": "customer", + "do": "click", + "ifAvailable": true, + "testid": "overlay-close", + "unlessVisible": "order-item", + "within": 1000 + }, + { + "actor": "customer", + "do": "click", + "testid": "orders-toggle" + }, + { + "actor": "customer", + "do": "expect", + "ignoreCase": true, + "in": { + "contains": "Desk Lamp", + "testid": "order-item" + }, + "testid": "order-status", + "value": "shipped", + "within": 10000 + } + ] + }, + { + "desc": "the fulfilment area is for staff and admins, not customers", + "id": "1d", + "points": 1, + "statedBy": "a staff account signs in like anyone else but sees the fulfilment area instead of the admin area", + "steps": [ + { + "actor": "admin", + "do": "click", + "testid": "staff-link" + }, + { + "actor": "admin", + "do": "expect", + "testid": "fulfilment-panel" + }, + { + "actor": "customer", + "do": "click", + "ifAvailable": true, + "settleMs": 1500, + "testid": "staff-link" + }, + { + "absent": true, + "actor": "customer", + "do": "expect", + "testid": "fulfilment-panel" + } + ] + }, + { + "desc": "the server refuses a ship request from a customer, not just the button", + "id": "1e", + "points": 2, + "statedBy": "staff cannot change prices or create items; admins can do everything staff can", + "steps": [ + { + "actor": "customer", + "do": "fill", + "enter": true, + "settleMs": 1500, + "testid": "search-input", + "text": "Webcam" + }, + { + "actor": "customer", + "do": "click", + "in": { + "contains": "Webcam", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "customer", + "do": "wait", + "ms": 2500 + }, + { + "actor": "customer", + "do": "click", + "ifAvailable": true, + "testid": "overlay-close", + "unlessVisible": "order-item", + "within": 1000 + }, + { + "actor": "customer", + "do": "click", + "testid": "orders-toggle" + }, + { + "actor": "customer", + "do": "replayAs", + "from": "staff", + "match": "ship", + "namedAction": { + "args": [ + 0 + ], + "id": "ship", + "path": "/api/fulfilment/ship", + "reducer": "ship_order" + }, + "namedTarget": { + "attribute": "data-entity-id", + "contains": "Webcam", + "testid": "order-item", + "valueType": "number" + }, + "settleMs": 2500, + "swap": { + "find": "Desk Lamp", + "with": "Webcam" + } + }, + { + "actor": "customer", + "do": "expectReplayRejected" + }, + { + "actor": "customer", + "do": "expect", + "ignoreCase": true, + "in": { + "contains": "Webcam", + "testid": "order-item" + }, + "testid": "order-status", + "value": "pending" + } + ] + } + ], + "id": 1, + "name": "Fulfilment queue", + "setup": [ + { + "actor": "customer", + "do": "signUp", + "name": "fq" + }, + { + "actor": "staff", + "do": "signIn", + "exact": true, + "name": "staff", + "password": "stackbench-staff-2026" + }, + { + "actor": "admin", + "do": "signIn", + "exact": true, + "name": "admin", + "password": "stackbench-admin-2026" + } + ] + }, + { + "actors": [ + "admin", + "staff", + "visitor" + ], + "criteria": [ + { + "desc": "a transfer moves stock without creating or destroying any", + "id": "2a", + "points": 3, + "statedBy": "a transfer moves stock, it does not create or destroy it", + "steps": [ + { + "actor": "visitor", + "as": "total-before", + "do": "recordNumber", + "in": { + "contains": "Headphones", + "testid": "item-card" + }, + "testid": "item-stock" + }, + { + "actor": "admin", + "as": "east-before", + "do": "recordNumber", + "in": { + "contains": "Headphones", + "testid": "admin-location-row" + }, + "testid": "admin-location-qty" + }, + { + "actor": "admin", + "as": "west-before", + "do": "recordNumber", + "in": { + "contains": "West", + "testid": "admin-warehouse-item" + }, + "testid": "warehouse-total" + }, + { + "actor": "admin", + "do": "fill", + "in": { + "contains": "Headphones", + "testid": "admin-item-row" + }, + "testid": "transfer-from", + "text": "East" + }, + { + "actor": "admin", + "do": "fill", + "in": { + "contains": "Headphones", + "testid": "admin-item-row" + }, + "testid": "transfer-to", + "text": "West" + }, + { + "actor": "admin", + "do": "fill", + "in": { + "contains": "Headphones", + "testid": "admin-item-row" + }, + "testid": "transfer-qty", + "text": "10" + }, + { + "actor": "admin", + "do": "click", + "in": { + "contains": "Headphones", + "testid": "admin-item-row" + }, + "testid": "transfer-submit" + }, + { + "actor": "admin", + "do": "wait", + "ms": 2500 + }, + { + "actor": "admin", + "do": "expectNumber", + "in": { + "contains": "Headphones", + "testid": "admin-location-row" + }, + "plus": -10, + "relativeTo": "east-before", + "testid": "admin-location-qty" + }, + { + "actor": "visitor", + "do": "expectNumber", + "in": { + "contains": "Headphones", + "testid": "item-card" + }, + "plus": 0, + "relativeTo": "total-before", + "testid": "item-stock" + } + ] + }, + { + "desc": "the warehouse totals move live as stock is transferred", + "id": "2b", + "points": 2, + "statedBy": "the per-warehouse numbers staff and admins see both move at once", + "steps": [ + { + "actor": "admin", + "do": "expectNumber", + "in": { + "contains": "West", + "testid": "admin-warehouse-item" + }, + "plus": 10, + "relativeTo": "west-before", + "testid": "warehouse-total", + "within": 10000 + } + ] + }, + { + "desc": "a transfer that would overdraw a warehouse is refused and changes nothing", + "id": "2c", + "points": 2, + "statedBy": "a transfer that would leave a warehouse short is refused and changes nothing", + "steps": [ + { + "actor": "admin", + "as": "total-guard", + "do": "recordNumber", + "in": { + "contains": "Headphones", + "testid": "admin-item-row" + }, + "testid": "admin-stock" + }, + { + "actor": "admin", + "do": "fill", + "in": { + "contains": "Headphones", + "testid": "admin-item-row" + }, + "testid": "transfer-from", + "text": "East" + }, + { + "actor": "admin", + "do": "fill", + "in": { + "contains": "Headphones", + "testid": "admin-item-row" + }, + "testid": "transfer-to", + "text": "West" + }, + { + "actor": "admin", + "do": "fill", + "in": { + "contains": "Headphones", + "testid": "admin-item-row" + }, + "testid": "transfer-qty", + "text": "99999" + }, + { + "actor": "admin", + "do": "click", + "in": { + "contains": "Headphones", + "testid": "admin-item-row" + }, + "testid": "transfer-submit" + }, + { + "actor": "admin", + "do": "expect", + "testid": "order-error", + "within": 6000 + }, + { + "actor": "admin", + "do": "expectNumber", + "in": { + "contains": "Headphones", + "testid": "admin-item-row" + }, + "plus": 0, + "relativeTo": "total-guard", + "testid": "admin-stock" + } + ] + } + ], + "id": 2, + "name": "Moving stock between warehouses", + "setup": [ + { + "actor": "admin", + "do": "signIn", + "exact": true, + "name": "admin", + "password": "stackbench-admin-2026" + }, + { + "actor": "staff", + "do": "signIn", + "exact": true, + "name": "staff", + "password": "stackbench-staff-2026" + }, + { + "actor": "admin", + "do": "click", + "testid": "admin-link" + } + ] + }, + { + "actors": [ + "customer", + "staff", + "admin" + ], + "criteria": [ + { + "desc": "cancelling a pending order returns its stock, empties it from the queue and lowers revenue", + "id": "3a", + "points": 3, + "statedBy": "the stock goes back to the warehouse it came from, the order leaves the fulfilment queue, and revenue falls", + "steps": [ + { + "as": "stored-before-cancel-3a", + "do": "dbRecordStock", + "item": "Coffee Grinder" + }, + { + "actor": "admin", + "as": "rev-before", + "do": "recordNumber", + "testid": "admin-revenue" + }, + { + "actor": "customer", + "do": "fill", + "enter": true, + "settleMs": 1500, + "testid": "search-input", + "text": "Coffee Grinder" + }, + { + "actor": "customer", + "as": "stock-before", + "do": "recordNumber", + "in": { + "contains": "Coffee Grinder", + "testid": "item-card" + }, + "testid": "item-stock" + }, + { + "actor": "customer", + "do": "click", + "in": { + "contains": "Coffee Grinder", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "customer", + "do": "wait", + "ms": 2500 + }, + { + "actor": "customer", + "do": "click", + "ifAvailable": true, + "testid": "overlay-close", + "unlessVisible": "order-item", + "within": 1000 + }, + { + "actor": "customer", + "do": "click", + "testid": "orders-toggle" + }, + { + "actor": "customer", + "contains": "Coffee Grinder", + "count": 1, + "do": "expect", + "testid": "order-item", + "within": 10000 + }, + { + "do": "dbExpectStock", + "item": "Coffee Grinder", + "plus": -1, + "relativeTo": "stored-before-cancel-3a" + }, + { + "actor": "staff", + "contains": "Coffee Grinder", + "count": 1, + "do": "expect", + "testid": "queue-item", + "within": 10000 + }, + { + "actor": "admin", + "do": "expectNumber", + "plus": 64, + "relativeTo": "rev-before", + "testid": "admin-revenue", + "within": 10000 + }, + { + "actor": "customer", + "do": "click", + "in": { + "contains": "Coffee Grinder", + "testid": "order-item" + }, + "testid": "cancel-order" + }, + { + "actor": "customer", + "do": "wait", + "ms": 2500 + }, + { + "absent": true, + "actor": "staff", + "contains": "Coffee Grinder", + "do": "expect", + "testid": "queue-item", + "within": 10000 + }, + { + "actor": "admin", + "do": "expectNumber", + "plus": 0, + "relativeTo": "rev-before", + "testid": "admin-revenue", + "within": 10000 + }, + { + "actor": "customer", + "do": "pressKey", + "key": "Escape" + }, + { + "actor": "customer", + "do": "expectNumber", + "in": { + "contains": "Coffee Grinder", + "testid": "item-card" + }, + "plus": 0, + "relativeTo": "stock-before", + "testid": "item-stock" + }, + { + "do": "dbExpectStock", + "item": "Coffee Grinder", + "plus": 0, + "relativeTo": "stored-before-cancel-3a" + } + ] + }, + { + "desc": "a cancelled order is shown as cancelled in the customer’s history", + "id": "3b", + "points": 1, + "statedBy": "a customer can cancel an order that has not shipped", + "steps": [ + { + "actor": "customer", + "do": "click", + "ifAvailable": true, + "testid": "overlay-close", + "unlessVisible": "order-item", + "within": 1000 + }, + { + "actor": "customer", + "do": "click", + "settleMs": 1200, + "testid": "orders-toggle" + }, + { + "actor": "customer", + "do": "expect", + "ignoreCase": true, + "in": { + "contains": "Coffee Grinder", + "testid": "order-item" + }, + "testid": "order-status", + "value": "cancelled", + "within": 8000 + } + ] + }, + { + "desc": "returning an item from a shipped order brings the stock back and lowers revenue", + "id": "3c", + "points": 3, + "statedBy": "the stock comes back, revenue falls by what was paid for it, and the order shows the item as returned", + "steps": [ + { + "actor": "customer", + "do": "fill", + "enter": true, + "settleMs": 1500, + "testid": "search-input", + "text": "Keyboard" + }, + { + "actor": "customer", + "as": "kb-before", + "do": "recordNumber", + "in": { + "contains": "Keyboard", + "testid": "item-card" + }, + "testid": "item-stock" + }, + { + "actor": "admin", + "as": "rev2-before", + "do": "recordNumber", + "testid": "admin-revenue" + }, + { + "actor": "customer", + "do": "click", + "in": { + "contains": "Keyboard", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "customer", + "do": "wait", + "ms": 2500 + }, + { + "actor": "staff", + "do": "click", + "in": { + "contains": "Keyboard", + "testid": "queue-item" + }, + "testid": "ship-submit" + }, + { + "actor": "staff", + "do": "wait", + "ms": 2500 + }, + { + "actor": "customer", + "do": "click", + "ifAvailable": true, + "testid": "overlay-close", + "unlessVisible": "order-item", + "within": 1000 + }, + { + "actor": "customer", + "do": "click", + "testid": "orders-toggle" + }, + { + "actor": "customer", + "do": "click", + "in": { + "contains": "Keyboard", + "testid": "order-item" + }, + "testid": "return-item" + }, + { + "actor": "customer", + "do": "wait", + "ms": 2500 + }, + { + "actor": "admin", + "do": "expectNumber", + "plus": 0, + "relativeTo": "rev2-before", + "testid": "admin-revenue", + "within": 10000 + }, + { + "actor": "customer", + "do": "pressKey", + "key": "Escape" + }, + { + "actor": "customer", + "do": "expectNumber", + "in": { + "contains": "Keyboard", + "testid": "item-card" + }, + "plus": 0, + "relativeTo": "kb-before", + "testid": "item-stock" + } + ] + } + ], + "id": 3, + "name": "Cancelling and returning", + "setup": [ + { + "actor": "customer", + "do": "signUp", + "name": "cr" + }, + { + "actor": "staff", + "do": "signIn", + "exact": true, + "name": "staff", + "password": "stackbench-staff-2026" + }, + { + "actor": "admin", + "do": "signIn", + "exact": true, + "name": "admin", + "password": "stackbench-admin-2026" + }, + { + "actor": "admin", + "do": "click", + "testid": "admin-link" + }, + { + "actor": "staff", + "do": "click", + "testid": "staff-link" + } + ] + }, + { + "actors": [ + "admin", + "customer", + "visitor" + ], + "criteria": [ + { + "desc": "a customer's receipt keeps the price that was paid after the price changes", + "id": "4a", + "points": 3, + "statedBy": "past orders keep the price that was paid", + "steps": [ + { + "actor": "customer", + "do": "fill", + "enter": true, + "settleMs": 1500, + "testid": "search-input", + "text": "Air Purifier" + }, + { + "actor": "customer", + "do": "click", + "in": { + "contains": "Air Purifier", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "customer", + "do": "wait", + "ms": 2500 + }, + { + "actor": "admin", + "do": "fill", + "in": { + "contains": "Air Purifier", + "testid": "admin-item-row" + }, + "testid": "price-input", + "text": "1.00" + }, + { + "actor": "admin", + "do": "click", + "in": { + "contains": "Air Purifier", + "testid": "admin-item-row" + }, + "testid": "price-submit" + }, + { + "actor": "admin", + "do": "wait", + "ms": 2500 + }, + { + "actor": "customer", + "do": "click", + "ifAvailable": true, + "testid": "overlay-close", + "unlessVisible": "order-item", + "within": 1000 + }, + { + "actor": "customer", + "do": "click", + "testid": "orders-toggle" + }, + { + "actor": "customer", + "atLeast": 189, + "do": "expectNumber", + "in": { + "contains": "Air Purifier", + "testid": "order-item" + }, + "testid": "order-total" + } + ] + }, + { + "desc": "the new price reaches a signed-out visitor without a reload", + "id": "4b", + "points": 2, + "statedBy": "the storefront shows the new price immediately, to everyone", + "steps": [ + { + "actor": "visitor", + "do": "fill", + "enter": true, + "settleMs": 1500, + "testid": "search-input", + "text": "Air Purifier" + }, + { + "actor": "visitor", + "do": "expectNumber", + "equals": 1, + "in": { + "contains": "Air Purifier", + "testid": "item-card" + }, + "testid": "item-price", + "within": 10000 + } + ] + }, + { + "desc": "a price change updates an open cart and checkout charges the new price", + "id": "4c", + "points": 3, + "statedBy": "an item in someone's cart at the moment the price changes is charged the new price at checkout, and the cart total they are looking at updates to match", + "steps": [ + { + "actor": "customer", + "do": "pressKey", + "key": "Escape" + }, + { + "actor": "customer", + "do": "fill", + "enter": true, + "settleMs": 1500, + "testid": "search-input", + "text": "Desk Lamp" + }, + { + "actor": "customer", + "do": "click", + "in": { + "contains": "Desk Lamp", + "testid": "item-card" + }, + "testid": "add-to-cart" + }, + { + "actor": "customer", + "do": "click", + "testid": "cart-toggle" + }, + { + "actor": "customer", + "do": "expectNumber", + "equals": 42, + "testid": "cart-total" + }, + { + "actor": "admin", + "do": "fill", + "in": { + "contains": "Desk Lamp", + "testid": "admin-item-row" + }, + "testid": "price-input", + "text": "52.00" + }, + { + "actor": "admin", + "do": "click", + "in": { + "contains": "Desk Lamp", + "testid": "admin-item-row" + }, + "testid": "price-submit" + }, + { + "actor": "customer", + "do": "expectNumber", + "equals": 52, + "testid": "cart-total", + "within": 10000 + }, + { + "actor": "customer", + "do": "click", + "testid": "checkout-submit" + }, + { + "actor": "customer", + "do": "click", + "ifAvailable": true, + "testid": "overlay-close", + "unlessVisible": "order-item", + "within": 1000 + }, + { + "actor": "customer", + "do": "click", + "settleMs": 1500, + "testid": "orders-toggle" + }, + { + "actor": "customer", + "do": "expectNumber", + "equals": 52, + "in": { + "contains": "Desk Lamp", + "testid": "order-item" + }, + "testid": "order-total" + } + ] + } + ], + "id": 4, + "name": "Prices change, history does not", + "setup": [ + { + "actor": "admin", + "do": "signIn", + "exact": true, + "name": "admin", + "password": "stackbench-admin-2026" + }, + { + "actor": "customer", + "do": "signUp", + "name": "pr" + }, + { + "actor": "admin", + "do": "click", + "testid": "admin-link" + } + ] + }, + { + "actors": [ + "admin", + "customer", + "visitor" + ], + "criteria": [ + { + "desc": "an item falling to ten units or fewer joins the low-stock list, live", + "id": "5a", + "points": 3, + "statedBy": "items enter and leave this list as stock moves, sells, is restocked, cancelled or returned", + "steps": [ + { + "actor": "admin", + "contains": "Mirrorless Camera", + "do": "expect", + "testid": "low-stock-item", + "within": 8000 + }, + { + "actor": "admin", + "do": "fill", + "in": { + "contains": "Mirrorless Camera", + "testid": "admin-location-row" + }, + "testid": "restock-input", + "text": "8" + }, + { + "actor": "admin", + "do": "click", + "in": { + "contains": "Mirrorless Camera", + "testid": "admin-location-row" + }, + "testid": "restock-submit" + }, + { + "absent": true, + "actor": "admin", + "contains": "Mirrorless Camera", + "do": "expect", + "testid": "low-stock-item", + "within": 10000 + }, + { + "actor": "customer", + "do": "fill", + "enter": true, + "settleMs": 1500, + "testid": "search-input", + "text": "Mirrorless Camera" + }, + { + "actor": "customer", + "do": "click", + "in": { + "contains": "Mirrorless Camera", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "admin", + "contains": "Mirrorless Camera", + "do": "expect", + "testid": "low-stock-item", + "within": 10000 + } + ] + }, + { + "desc": "a purchase moves that category's units and revenue, live", + "id": "5b", + "points": 3, + "statedBy": "for each\ncategory, how many units have been sold and the revenue they earned, live", + "steps": [ + { + "actor": "admin", + "do": "click", + "ifAvailable": true, + "testid": "sales-link", + "unlessVisible": "category-row" + }, + { + "actor": "admin", + "as": "audio-units", + "do": "recordNumber", + "in": { + "contains": "Audio", + "testid": "category-row" + }, + "testid": "category-units" + }, + { + "actor": "admin", + "as": "audio-rev", + "do": "recordNumber", + "in": { + "contains": "Audio", + "testid": "category-row" + }, + "testid": "category-revenue" + }, + { + "actor": "customer", + "do": "fill", + "enter": true, + "settleMs": 1500, + "testid": "search-input", + "text": "Bluetooth Speaker" + }, + { + "actor": "customer", + "do": "click", + "in": { + "contains": "Bluetooth Speaker", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "admin", + "do": "expectNumber", + "in": { + "contains": "Audio", + "testid": "category-row" + }, + "plus": 1, + "relativeTo": "audio-units", + "testid": "category-units", + "within": 10000 + }, + { + "actor": "admin", + "do": "expectNumber", + "in": { + "contains": "Audio", + "testid": "category-row" + }, + "plus": 79.5, + "relativeTo": "audio-rev", + "testid": "category-revenue", + "within": 10000 + } + ] + }, + { + "desc": "buying from a category makes that category's other items recommended", + "id": "5c", + "points": 2, + "statedBy": "the items from categories\nthey have bought from, most-purchased first, excluding items already in\ntheir cart", + "steps": [ + { + "actor": "customer", + "do": "fill", + "enter": true, + "settleMs": 1500, + "testid": "search-input", + "text": "Bluetooth Speaker" + }, + { + "actor": "customer", + "do": "click", + "in": { + "contains": "Bluetooth Speaker", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "customer", + "do": "pressKey", + "key": "Escape" + }, + { + "actor": "customer", + "contains": "Headphones", + "do": "expect", + "testid": "recommended-item", + "within": 10000 + }, + { + "actor": "customer", + "do": "fill", + "enter": true, + "settleMs": 1500, + "testid": "search-input", + "text": "Headphones" + }, + { + "actor": "customer", + "do": "click", + "in": { + "contains": "Headphones", + "testid": "item-card" + }, + "testid": "add-to-cart" + }, + { + "actor": "customer", + "do": "pressKey", + "key": "Escape" + }, + { + "absent": true, + "actor": "customer", + "contains": "Headphones", + "do": "expect", + "testid": "recommended-item", + "within": 10000 + } + ] + }, + { + "desc": "a signed-out visitor sees best sellers in the recommendations slot", + "id": "5d", + "points": 1, + "statedBy": "for a signed-out visitor this is simply the best sellers", + "steps": [ + { + "actor": "visitor", + "contains": "Bluetooth Speaker", + "do": "expect", + "testid": "recommended-item", + "within": 10000 + } + ] + } + ], + "id": 5, + "name": "The operational views", + "setup": [ + { + "actor": "admin", + "do": "signIn", + "exact": true, + "name": "admin", + "password": "stackbench-admin-2026" + }, + { + "actor": "customer", + "do": "signUp", + "name": "ops" + }, + { + "actor": "admin", + "do": "click", + "testid": "admin-link" + }, + { + "actor": "admin", + "do": "click", + "ifAvailable": true, + "testid": "low-stock-link", + "unlessVisible": "low-stock-item", + "within": 10000 + } + ] + } + ], + "level": 2, + "name": "Running the business — features", + "note": "The level's thesis is that eight actions each touch several of ten live views. These features are therefore written as ONE action watched by SEVERAL viewers at once, not as one action checked once — a backend that updates the view the actor is looking at and forgets the others fails on the watcher, not on the actor. Items are reached by search because earlier features re-rank the storefront. Panels are closed with Escape, which the spec requires, rather than by clicking an opener again.", + "schemaVersion": 1, + "track": "ecommerce", + "writeUrlPattern": "\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses|\\/transfer|\\/cancel|\\/return|\\/ship|\\/price|\\/fulfil" + }, + "path": "scenarios/02-features.json" + }, + { + "definition": { + "features": [ + { + "actors": [ + "admin", + "customer" + ], + "criteria": [ + { + "desc": "a customer replaying an admin's transfer moves no stock", + "id": "201a", + "points": 2, + "statedBy": "an admin can transfer a number of units of an item from one warehouse to another", + "steps": [ + { + "actor": "admin", + "do": "fill", + "in": { + "contains": "Gaming Mouse", + "testid": "admin-item-row" + }, + "testid": "transfer-from", + "text": "East" + }, + { + "actor": "admin", + "do": "fill", + "in": { + "contains": "Gaming Mouse", + "testid": "admin-item-row" + }, + "testid": "transfer-to", + "text": "West" + }, + { + "actor": "admin", + "do": "fill", + "in": { + "contains": "Gaming Mouse", + "testid": "admin-item-row" + }, + "testid": "transfer-qty", + "text": "5" + }, + { + "actor": "admin", + "do": "click", + "in": { + "contains": "Gaming Mouse", + "testid": "admin-item-row" + }, + "testid": "transfer-submit" + }, + { + "actor": "admin", + "do": "wait", + "ms": 2500 + }, + { + "actor": "customer", + "as": "after-admin-move", + "do": "recordNumber", + "in": { + "contains": "Gaming Mouse", + "testid": "item-card" + }, + "testid": "item-stock" + }, + { + "actor": "customer", + "do": "replayAs", + "from": "admin", + "match": "transfer", + "settleMs": 2500 + }, + { + "actor": "customer", + "do": "expectReplayRejected" + }, + { + "actor": "customer", + "do": "expectNumber", + "in": { + "contains": "Gaming Mouse", + "testid": "item-card" + }, + "plus": 0, + "relativeTo": "after-admin-move", + "testid": "item-stock" + } + ] + }, + { + "desc": "a customer cannot set a price by replaying an admin's change", + "id": "201b", + "points": 2, + "statedBy": "an admin can change an item's price", + "steps": [ + { + "actor": "admin", + "do": "fill", + "in": { + "contains": "Gaming Mouse", + "testid": "admin-item-row" + }, + "testid": "price-input", + "text": "77.00" + }, + { + "actor": "admin", + "do": "click", + "in": { + "contains": "Gaming Mouse", + "testid": "admin-item-row" + }, + "testid": "price-submit" + }, + { + "actor": "admin", + "do": "wait", + "ms": 2500 + }, + { + "actor": "customer", + "do": "replayAs", + "from": "admin", + "match": "price", + "settleMs": 2500, + "swap": { + "find": "77", + "with": "1" + } + }, + { + "actor": "customer", + "do": "expectReplayRejected" + }, + { + "actor": "customer", + "atLeast": 77, + "do": "expectNumber", + "in": { + "contains": "Gaming Mouse", + "testid": "item-card" + }, + "testid": "item-price" + } + ] + }, + { + "desc": "a customer cannot ship their own order", + "id": "201c", + "note": "DOM diagnostic: checks the ship control is absent for a customer. The SERVER-side refusal of a customer's ship request is now criterion 1e, which replays a real ship as the customer and requires the server to reject it. Kept at 0 points so hiding the button is never mistaken for enforcement.", + "points": 0, + "statedBy": "staff cannot change prices or create items; admins can do everything staff can", + "steps": [ + { + "actor": "customer", + "do": "pressKey", + "key": "Escape" + }, + { + "actor": "customer", + "do": "click", + "in": { + "contains": "Laptop Stand", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "customer", + "do": "wait", + "ms": 2500 + }, + { + "actor": "customer", + "do": "click", + "ifAvailable": true, + "testid": "overlay-close", + "unlessVisible": "order-item", + "within": 1000 + }, + { + "actor": "customer", + "do": "click", + "testid": "orders-toggle", + "unlessVisible": "order-item" + }, + { + "absent": true, + "actor": "customer", + "do": "expect", + "testid": "ship-submit" + }, + { + "actor": "customer", + "do": "expect", + "ignoreCase": true, + "in": { + "contains": "Laptop Stand", + "testid": "order-item" + }, + "testid": "order-status", + "value": "pending" + } + ] + } + ], + "id": 201, + "name": "Operating the store requires being an operator", + "setup": [ + { + "actor": "admin", + "do": "signIn", + "exact": true, + "name": "admin", + "password": "stackbench-admin-2026" + }, + { + "actor": "customer", + "do": "signUp", + "name": "notstaff" + }, + { + "actor": "admin", + "do": "click", + "testid": "admin-link" + } + ] + }, + { + "actors": [ + "admin", + "customer" + ], + "criteria": [ + { + "desc": "a transfer changes where stock is, never how much there is", + "id": "202a", + "points": 2, + "statedBy": "a transfer moves stock, it does not create or destroy it", + "steps": [ + { + "actor": "admin", + "as": "espresso-total", + "do": "recordNumber", + "in": { + "contains": "Espresso Machine", + "testid": "admin-item-row" + }, + "testid": "admin-stock" + }, + { + "actor": "admin", + "do": "fill", + "in": { + "contains": "Espresso Machine", + "testid": "admin-item-row" + }, + "testid": "transfer-from", + "text": "East" + }, + { + "actor": "admin", + "do": "fill", + "in": { + "contains": "Espresso Machine", + "testid": "admin-item-row" + }, + "testid": "transfer-to", + "text": "West" + }, + { + "actor": "admin", + "do": "fill", + "in": { + "contains": "Espresso Machine", + "testid": "admin-item-row" + }, + "testid": "transfer-qty", + "text": "17" + }, + { + "actor": "admin", + "do": "click", + "in": { + "contains": "Espresso Machine", + "testid": "admin-item-row" + }, + "testid": "transfer-submit" + }, + { + "actor": "admin", + "do": "wait", + "ms": 3000 + }, + { + "actor": "admin", + "do": "expectNumber", + "in": { + "contains": "Espresso Machine", + "testid": "admin-item-row" + }, + "plus": 0, + "relativeTo": "espresso-total", + "testid": "admin-stock" + } + ] + }, + { + "desc": "a sale and its cancellation leave the shelf exactly as they found it", + "id": "202b", + "points": 2, + "statedBy": "the stock goes back to the warehouse it came from", + "steps": [ + { + "actor": "customer", + "as": "cooktop-before", + "do": "recordNumber", + "in": { + "contains": "Induction Cooktop", + "testid": "item-card" + }, + "testid": "item-stock" + }, + { + "actor": "customer", + "do": "click", + "in": { + "contains": "Induction Cooktop", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "customer", + "do": "wait", + "ms": 2000 + }, + { + "actor": "customer", + "do": "expectNumber", + "in": { + "contains": "Induction Cooktop", + "testid": "item-card" + }, + "plus": -1, + "relativeTo": "cooktop-before", + "testid": "item-stock" + }, + { + "actor": "customer", + "do": "click", + "ifAvailable": true, + "testid": "overlay-close", + "unlessVisible": "order-item", + "within": 1000 + }, + { + "actor": "customer", + "do": "click", + "testid": "orders-toggle", + "unlessVisible": "order-item" + }, + { + "actor": "customer", + "do": "click", + "in": { + "contains": "Induction Cooktop", + "testid": "order-item" + }, + "testid": "cancel-order" + }, + { + "actor": "customer", + "do": "wait", + "ms": 2500 + }, + { + "actor": "customer", + "do": "pressKey", + "key": "Escape" + }, + { + "actor": "customer", + "do": "expectNumber", + "in": { + "contains": "Induction Cooktop", + "testid": "item-card" + }, + "plus": 0, + "relativeTo": "cooktop-before", + "testid": "item-stock" + } + ] + }, + { + "desc": "a fresh client sees the same totals, so nothing was only corrected in one browser", + "id": "202c", + "points": 1, + "statedBy": "every one of these numbers is the same for every person looking at it", + "steps": [ + { + "actor": "customer", + "do": "freshClient" + }, + { + "actor": "customer-fresh", + "do": "expectNumber", + "in": { + "contains": "Induction Cooktop", + "testid": "item-card" + }, + "plus": 0, + "relativeTo": "cooktop-before", + "testid": "item-stock" + } + ] + }, + { + "desc": "a transfer racing a purchase loses no unit between the rows: total = before − sold, exactly, however the two interleave", + "id": "202d", + "note": "WITHHELD until proven against a mutant (transfer-read-modify-write) and a real failing build. A non-transactional transfer reads both rows, computes, writes both — a purchase landing between the read and the write is erased. Sequential tests cannot see this; 202a passes either way.", + "points": 0, + "statedBy": "a transfer moves stock, it does not create or destroy it", + "steps": [ + { + "actor": "admin", + "as": "hp-before", + "do": "recordNumber", + "in": { + "contains": "Headphones", + "testid": "admin-item-row" + }, + "testid": "admin-stock" + }, + { + "branches": [ + [ + { + "actor": "admin", + "do": "fill", + "in": { + "contains": "Headphones", + "testid": "admin-item-row" + }, + "testid": "transfer-from", + "text": "East" + }, + { + "actor": "admin", + "do": "fill", + "in": { + "contains": "Headphones", + "testid": "admin-item-row" + }, + "testid": "transfer-to", + "text": "West" + }, + { + "actor": "admin", + "do": "fill", + "in": { + "contains": "Headphones", + "testid": "admin-item-row" + }, + "testid": "transfer-qty", + "text": "25" + }, + { + "actor": "admin", + "do": "click", + "in": { + "contains": "Headphones", + "testid": "admin-item-row" + }, + "testid": "transfer-submit" + } + ], + [ + { + "actor": "customer", + "do": "click", + "in": { + "contains": "Headphones", + "testid": "item-card" + }, + "testid": "buy-now" + } + ] + ], + "do": "race", + "settleMs": 5000 + }, + { + "actor": "admin", + "do": "expectNumber", + "in": { + "contains": "Headphones", + "testid": "admin-item-row" + }, + "plus": -1, + "relativeTo": "hp-before", + "testid": "admin-stock", + "within": 12000 + } + ] + } + ], + "id": 202, + "name": "Stock is conserved however it moves", + "setup": [ + { + "actor": "admin", + "do": "signIn", + "exact": true, + "name": "admin", + "password": "stackbench-admin-2026" + }, + { + "actor": "customer", + "do": "signUp", + "name": "conserve" + }, + { + "actor": "admin", + "do": "click", + "testid": "admin-link" + } + ] + }, + { + "actors": [ + "admin", + "customer", + "customer2" + ], + "criteria": [ + { + "category": "production", + "desc": "concurrent cancellation restores original stock and the booked amount once, while revenue returns to its prior value", + "id": "203a", + "note": "Draft: native order, refund and original warehouse allocation evidence supplements the retained revenue assertions. Four calls from two sessions of the owner must cancel once; a repeated call may deliberately refuse or succeed without extra effects. Qualification pending.", + "points": 3, + "statedBy": "revenue always equals the sum of orders that are still standing", + "steps": [ + { + "actor": "admin", + "as": "rev-start", + "do": "recordNumber", + "testid": "admin-revenue" + }, + { + "actor": "customer", + "do": "click", + "in": { + "contains": "Desk Lamp", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "customer", + "do": "wait", + "ms": 2500 + }, + { + "actor": "admin", + "do": "reload", + "settleMs": 1000 + }, + { + "actor": "admin", + "do": "ensureSignedIn", + "exact": true, + "name": "admin", + "password": "stackbench-admin-2026", + "readyTestid": "current-user" + }, + { + "actor": "admin", + "do": "click", + "ifAvailable": true, + "testid": "overlay-close", + "within": 1000 + }, + { + "actor": "admin", + "do": "click", + "ifAvailable": true, + "testid": "current-user", + "unlessVisible": [ + "admin-link", + "admin-revenue" + ] + }, + { + "actor": "admin", + "do": "click", + "testid": "admin-link", + "unlessVisible": "admin-revenue" + }, + { + "actor": "admin", + "do": "expectNumber", + "plus": 42, + "relativeTo": "rev-start", + "testid": "admin-revenue", + "within": 10000 + }, + { + "actor": "customer", + "do": "click", + "ifAvailable": true, + "testid": "overlay-close", + "unlessVisible": "order-item", + "within": 1000 + }, + { + "actor": "customer", + "do": "click", + "ifAvailable": true, + "testid": "current-user", + "unlessVisible": [ + "orders-toggle", + "order-item" + ] + }, + { + "actor": "customer", + "do": "click", + "testid": "orders-toggle", + "unlessVisible": "order-item" + }, + { + "actor": "customer", + "do": "expect", + "ignoreCase": true, + "in": { + "contains": "Desk Lamp", + "testid": "order-item" + }, + "statusText": "pending", + "testid": "order-status", + "within": 10000 + }, + { + "account": "{user:books}", + "as": "cancel-before", + "do": "dbRecordCheckout", + "item": "Desk Lamp", + "storage": { + "cart": false, + "kind": "order-data", + "warehouses": true + } + }, + { + "action": "cancel", + "actors": [ + "customer", + "customer2" + ], + "do": "callConcurrently", + "from": "customer", + "input": { + "attribute": "data-cancel-input", + "contains": "Desk Lamp", + "testid": "order-item" + }, + "namedAction": { + "args": [ + 0 + ], + "id": "cancel", + "params": [ + { + "in": "path", + "name": "orderId", + "placeholder": ":id", + "wireType": "u64" + } + ], + "path": "/api/orders/:id/cancel", + "reducer": "cancel_order" + }, + "requests": 4, + "settleMs": 3000 + }, + { + "do": "expectCallOutcomes" + }, + { + "before": "cancel-before", + "do": "dbExpectCancellation" + }, + { + "actor": "admin", + "do": "reload", + "settleMs": 1000 + }, + { + "actor": "admin", + "do": "ensureSignedIn", + "exact": true, + "name": "admin", + "password": "stackbench-admin-2026", + "readyTestid": "current-user" + }, + { + "actor": "admin", + "do": "click", + "ifAvailable": true, + "testid": "overlay-close", + "within": 1000 + }, + { + "actor": "admin", + "do": "click", + "ifAvailable": true, + "testid": "current-user", + "unlessVisible": [ + "admin-link", + "admin-revenue" + ] + }, + { + "actor": "admin", + "do": "click", + "testid": "admin-link", + "unlessVisible": "admin-revenue" + }, + { + "actor": "admin", + "do": "expectNumber", + "plus": 0, + "relativeTo": "rev-start", + "testid": "admin-revenue", + "within": 10000 + } + ] + }, + { + "category": "production", + "desc": "a price change does not rewrite revenue already earned", + "id": "203b", + "points": 2, + "statedBy": "changing a price never alters the history, the revenue already recorded", + "steps": [ + { + "actor": "admin", + "as": "history-revenue-before-sale", + "do": "recordNumber", + "testid": "admin-revenue" + }, + { + "actor": "customer", + "do": "pressKey", + "key": "Escape" + }, + { + "actor": "customer", + "do": "click", + "in": { + "contains": "Bluetooth Speaker", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "customer", + "do": "wait", + "ms": 2500 + }, + { + "actor": "admin", + "do": "reload", + "settleMs": 1000 + }, + { + "actor": "admin", + "do": "ensureSignedIn", + "exact": true, + "name": "admin", + "password": "stackbench-admin-2026", + "readyTestid": "current-user" + }, + { + "actor": "admin", + "do": "click", + "ifAvailable": true, + "testid": "overlay-close", + "within": 1000 + }, + { + "actor": "admin", + "do": "click", + "testid": "admin-link", + "unlessVisible": "admin-revenue" + }, + { + "actor": "admin", + "do": "expectNumber", + "plus": 79.5, + "relativeTo": "history-revenue-before-sale", + "testid": "admin-revenue", + "within": 10000 + }, + { + "actor": "customer", + "do": "click", + "ifAvailable": true, + "testid": "overlay-close", + "unlessVisible": "order-item", + "within": 1000 + }, + { + "actor": "customer", + "do": "click", + "testid": "orders-toggle", + "unlessVisible": "order-item" + }, + { + "actor": "customer", + "contains": "Bluetooth Speaker", + "count": 1, + "do": "expect", + "testid": "order-item", + "within": 10000 + }, + { + "actor": "admin", + "as": "rev-after-sale", + "do": "recordNumber", + "testid": "admin-revenue" + }, + { + "actor": "admin", + "do": "fill", + "in": { + "contains": "Bluetooth Speaker", + "testid": "admin-item-row" + }, + "testid": "price-input", + "text": "5.00" + }, + { + "actor": "admin", + "do": "click", + "in": { + "contains": "Bluetooth Speaker", + "testid": "admin-item-row" + }, + "testid": "price-submit" + }, + { + "actor": "admin", + "do": "wait", + "ms": 3000 + }, + { + "actor": "admin", + "do": "reload", + "settleMs": 2000 + }, + { + "actor": "admin", + "do": "ensureSignedIn", + "exact": true, + "name": "admin", + "password": "stackbench-admin-2026", + "readyTestid": "current-user" + }, + { + "actor": "admin", + "do": "click", + "ifAvailable": true, + "testid": "admin-link", + "within": 1000 + }, + { + "actor": "admin", + "do": "expectNumber", + "equals": 5, + "in": { + "contains": "Bluetooth Speaker", + "testid": "admin-item-row" + }, + "testid": "price-input", + "within": 10000 + }, + { + "actor": "admin", + "do": "reload", + "settleMs": 1000 + }, + { + "actor": "admin", + "do": "ensureSignedIn", + "exact": true, + "name": "admin", + "password": "stackbench-admin-2026", + "readyTestid": "current-user" + }, + { + "actor": "admin", + "do": "click", + "ifAvailable": true, + "testid": "overlay-close", + "within": 1000 + }, + { + "actor": "admin", + "do": "click", + "testid": "admin-link", + "unlessVisible": "admin-revenue" + }, + { + "actor": "admin", + "do": "expectNumber", + "plus": 0, + "relativeTo": "rev-after-sale", + "testid": "admin-revenue" + } + ] + } + ], + "id": 203, + "name": "The books still balance once money can flow backwards", + "setup": [ + { + "actor": "admin", + "do": "signIn", + "exact": true, + "name": "admin", + "password": "stackbench-admin-2026" + }, + { + "actor": "customer", + "do": "signUp", + "name": "books" + }, + { + "actor": "customer2", + "do": "signIn", + "name": "books" + }, + { + "actor": "admin", + "do": "click", + "ifAvailable": true, + "testid": "current-user", + "unlessVisible": [ + "admin-link", + "admin-panel" + ] + }, + { + "actor": "admin", + "do": "click", + "testid": "admin-link", + "unlessVisible": "admin-panel" + }, + { + "actor": "customer", + "do": "click", + "ifAvailable": true, + "testid": "catalog-link", + "unlessVisible": "item-list" + } + ] + }, + { + "actors": [ + "one", + "two" + ], + "criteria": [ + { + "desc": "one customer cannot cancel another's order", + "id": "204a", + "note": "WITHHELD: the current replay happens after the owner's cancellation, so a state-only rejection can masquerade as an ownership check. Restore points only after the harness can replay a captured cancellation against a still-pending order and a cross-account authorization mutant is caught.", + "points": 0, + "statedBy": "a customer can cancel an order that has not shipped", + "steps": [ + { + "actor": "one", + "do": "click", + "in": { + "contains": "Laptop Stand", + "testid": "item-card" + }, + "testid": "buy-now" + }, + { + "actor": "one", + "do": "wait", + "ms": 2500 + }, + { + "actor": "one", + "do": "click", + "ifAvailable": true, + "testid": "overlay-close", + "unlessVisible": "order-item", + "within": 1000 + }, + { + "actor": "one", + "do": "click", + "testid": "orders-toggle" + }, + { + "actor": "one", + "do": "click", + "in": { + "contains": "Laptop Stand", + "testid": "order-item" + }, + "testid": "cancel-order" + }, + { + "actor": "one", + "do": "wait", + "ms": 2000 + }, + { + "actor": "two", + "do": "replayAs", + "from": "one", + "match": "cancel", + "settleMs": 2500 + }, + { + "actor": "two", + "do": "expectReplayRejected" + }, + { + "actor": "two", + "do": "click", + "ifAvailable": true, + "testid": "overlay-close", + "unlessVisible": "order-item", + "within": 1000 + }, + { + "actor": "two", + "do": "click", + "testid": "orders-toggle" + }, + { + "absent": true, + "actor": "two", + "contains": "Laptop Stand", + "do": "expect", + "testid": "order-item" + } + ] + } + ], + "id": 204, + "name": "An order belongs to the person who placed it", + "setup": [ + { + "actor": "one", + "do": "signUp", + "name": "owner" + }, + { + "actor": "two", + "do": "signUp", + "name": "other" + } + ] + } + ], + "level": 2, + "name": "Running the business — invariants", + "note": "Draft: revised observations require matching reference and defect-control qualification.", + "schemaVersion": 1, + "track": "ecommerce", + "writeUrlPattern": "\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses|\\/transfer|\\/cancel|\\/return|\\/ship|\\/price|\\/fulfil" + }, + "path": "scenarios/02-invariants.json" + }, + { + "definition": { + "features": [ + { + "actors": [ + "shopper", + "watcher" + ], + "criteria": [ + { + "desc": "adding to the cart takes a unit off the shelf immediately, for a signed-out viewer who never touched the page", + "id": "301a", + "points": 2, + "statedBy": "storefront stock drops immediately, for everyone", + "steps": [ + { + "actor": "shopper", + "do": "click", + "in": { + "contains": "Desk Lamp", + "testid": "item-card" + }, + "settleMs": 1500, + "testid": "add-to-cart" + }, + { + "actor": "watcher", + "do": "expectNumber", + "in": { + "contains": "Desk Lamp", + "testid": "item-card" + }, + "plus": -1, + "relativeTo": "lamp-before", + "testid": "item-stock", + "within": 10000 + } + ] + }, + { + "desc": "the cart line shows the reservation counting down", + "id": "301b", + "points": 1, + "statedBy": "the cart line shows the reservation's remaining time, live", + "steps": [ + { + "actor": "shopper", + "do": "click", + "settleMs": 1200, + "testid": "cart-toggle" + }, + { + "actor": "shopper", + "do": "expect", + "in": { + "contains": "Desk Lamp", + "testid": "cart-item" + }, + "testid": "cart-reservation-timer", + "within": 8000 + } + ] + }, + { + "desc": "left alone, the reservation expires on its own: the unit returns to the shelf for the watcher without a reload, and the line is marked expired rather than vanishing", + "id": "301c", + "note": "waits out the spec's 90-second window plus slack; slow by nature.", + "points": 3, + "statedBy": "the reservation expires on its own and the unit returns to\nstorefront stock - visibly, without anyone reloading", + "steps": [ + { + "actor": "shopper", + "do": "wait", + "ms": 100000 + }, + { + "actor": "watcher", + "do": "expectNumber", + "in": { + "contains": "Desk Lamp", + "testid": "item-card" + }, + "plus": 0, + "relativeTo": "lamp-before", + "testid": "item-stock", + "within": 30000 + }, + { + "actor": "shopper", + "do": "expect", + "in": { + "contains": "Desk Lamp", + "testid": "cart-item" + }, + "testid": "cart-item-expired", + "within": 15000 + } + ] + } + ], + "id": 301, + "name": "A reservation holds stock and then releases it", + "setup": [ + { + "actor": "shopper", + "do": "signUp", + "name": "resa" + }, + { + "actor": "watcher", + "as": "lamp-before", + "do": "recordNumber", + "in": { + "contains": "Desk Lamp", + "testid": "item-card" + }, + "testid": "item-stock" + } + ] + }, + { + "actors": [ + "admin", + "watcher" + ], + "criteria": [ + { + "desc": "a scheduled restock appears as pending with time remaining, and has not yet changed stock", + "id": "302a", + "points": 2, + "statedBy": "the pending restocks list shows each one with its remaining time, live", + "steps": [ + { + "actor": "admin", + "do": "fill", + "testid": "schedule-restock-item", + "text": "Webcam" + }, + { + "actor": "admin", + "do": "fill", + "testid": "schedule-restock-warehouse", + "text": "East" + }, + { + "actor": "admin", + "do": "fill", + "testid": "schedule-restock-qty", + "text": "7" + }, + { + "actor": "admin", + "do": "fill", + "testid": "schedule-restock-delay", + "text": "45" + }, + { + "actor": "admin", + "do": "click", + "settleMs": 2000, + "testid": "schedule-restock-submit" + }, + { + "actor": "admin", + "count": 1, + "do": "expect", + "testid": "pending-restock-item", + "within": 8000 + }, + { + "actor": "admin", + "do": "expect", + "in": { + "testid": "pending-restock-item" + }, + "testid": "pending-restock-remaining", + "within": 8000 + }, + { + "actor": "watcher", + "do": "expectNumber", + "in": { + "contains": "Webcam", + "testid": "item-card" + }, + "plus": 0, + "relativeTo": "webcam-before", + "testid": "item-stock" + } + ] + }, + { + "desc": "when the delay elapses it applies itself: stock rises for a viewer who did nothing, and it leaves the pending list", + "id": "302b", + "points": 3, + "statedBy": "the entry leaves the pending list, and it appears in the stock ledger", + "steps": [ + { + "actor": "admin", + "do": "wait", + "ms": 55000 + }, + { + "actor": "watcher", + "do": "expectNumber", + "in": { + "contains": "Webcam", + "testid": "item-card" + }, + "plus": 7, + "relativeTo": "webcam-before", + "testid": "item-stock", + "within": 30000 + }, + { + "absent": true, + "actor": "admin", + "do": "expect", + "testid": "pending-restock-item", + "within": 15000 + } + ] + }, + { + "desc": "a cancelled restock never applies", + "id": "302c", + "points": 2, + "statedBy": "an admin can cancel a pending restock before it fires; cancelled restocks\nnever apply", + "steps": [ + { + "actor": "admin", + "do": "fill", + "testid": "schedule-restock-item", + "text": "Keyboard" + }, + { + "actor": "admin", + "do": "fill", + "testid": "schedule-restock-warehouse", + "text": "West" + }, + { + "actor": "admin", + "do": "fill", + "testid": "schedule-restock-qty", + "text": "9" + }, + { + "actor": "admin", + "do": "fill", + "testid": "schedule-restock-delay", + "text": "40" + }, + { + "actor": "admin", + "do": "click", + "settleMs": 2000, + "testid": "schedule-restock-submit" + }, + { + "actor": "watcher", + "as": "kbd-before", + "do": "recordNumber", + "in": { + "contains": "Keyboard", + "testid": "item-card" + }, + "testid": "item-stock" + }, + { + "actor": "admin", + "count": 1, + "do": "expect", + "testid": "pending-restock-item" + }, + { + "actor": "admin", + "do": "click", + "in": { + "testid": "pending-restock-item" + }, + "settleMs": 2000, + "testid": "pending-restock-cancel" + }, + { + "actor": "admin", + "do": "wait", + "ms": 50000 + }, + { + "actor": "watcher", + "do": "expectNumber", + "in": { + "contains": "Keyboard", + "testid": "item-card" + }, + "plus": 0, + "relativeTo": "kbd-before", + "testid": "item-stock", + "within": 10000 + } + ] + } + ], + "id": 302, + "name": "A restock scheduled for later applies itself", + "setup": [ + { + "actor": "admin", + "do": "signIn", + "exact": true, + "name": "admin", + "password": "stackbench-admin-2026" + }, + { + "actor": "admin", + "do": "click", + "settleMs": 1500, + "testid": "admin-link" + }, + { + "actor": "admin", + "do": "click", + "ifAvailable": true, + "testid": "restocks-link" + }, + { + "actor": "watcher", + "as": "webcam-before", + "do": "recordNumber", + "in": { + "contains": "Webcam", + "testid": "item-card" + }, + "testid": "item-stock" + } + ] + } + ], + "level": 3, + "name": "Work that happens later — features", + "note": "Deferred work observed from the outside: a reservation that releases itself, a restock that fires on a timer, an order that advances with nobody watching. Every wait here is real time — these criteria are slow by nature and the timings come from the spec (90s reservation, 60s transitions), not from guesses about implementation.", + "schemaVersion": 1, + "track": "ecommerce", + "writeUrlPattern": "\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses|\\/schedule" + }, + "path": "scenarios/03-features.json" + }, + { + "definition": { + "features": [ + { + "actors": [ + "admin", + "watcher" + ], + "criteria": [ + { + "desc": "a restock scheduled before a restart still fires after it, once, at its own time — an in-memory timer does not survive and must not be how this was built", + "id": "311a", + "points": 4, + "statedBy": "work scheduled\nbefore a restart fires after it, once, at the right time", + "steps": [ + { + "do": "restartBackend", + "settleMs": 15000 + }, + { + "actor": "watcher", + "do": "reload", + "settleMs": 5000 + }, + { + "actor": "watcher", + "do": "expectNumber", + "in": { + "contains": "Air Purifier", + "testid": "item-card" + }, + "plus": 5, + "relativeTo": "air-before", + "testid": "item-stock", + "within": 90000 + } + ] + }, + { + "desc": "it fired exactly once — the restart did not replay it", + "id": "311b", + "points": 3, + "statedBy": "must never double-apply\na scheduled restock or advance an order twice", + "steps": [ + { + "actor": "watcher", + "do": "wait", + "ms": 20000 + }, + { + "actor": "watcher", + "do": "expectNumber", + "in": { + "contains": "Air Purifier", + "testid": "item-card" + }, + "plus": 5, + "relativeTo": "air-before", + "testid": "item-stock" + } + ] + } + ], + "id": 311, + "name": "Scheduled work survives a restart", + "setup": [ + { + "actor": "admin", + "do": "signIn", + "exact": true, + "name": "admin", + "password": "stackbench-admin-2026" + }, + { + "actor": "admin", + "do": "click", + "settleMs": 1500, + "testid": "admin-link" + }, + { + "actor": "admin", + "do": "click", + "ifAvailable": true, + "testid": "restocks-link" + }, + { + "actor": "watcher", + "as": "air-before", + "do": "recordNumber", + "in": { + "contains": "Air Purifier", + "testid": "item-card" + }, + "testid": "item-stock" + }, + { + "actor": "admin", + "do": "fill", + "testid": "schedule-restock-item", + "text": "Air Purifier" + }, + { + "actor": "admin", + "do": "fill", + "testid": "schedule-restock-warehouse", + "text": "East" + }, + { + "actor": "admin", + "do": "fill", + "testid": "schedule-restock-qty", + "text": "5" + }, + { + "actor": "admin", + "do": "fill", + "testid": "schedule-restock-delay", + "text": "70" + }, + { + "actor": "admin", + "do": "click", + "settleMs": 2000, + "testid": "schedule-restock-submit" + } + ] + }, + { + "actors": [ + "admin", + "watcher" + ], + "criteria": [ + { + "desc": "a restock due in three minutes has not applied thirty seconds after a restart — re-reading pending work must not run it", + "id": "312a", + "points": 3, + "statedBy": "a restart must not cause pending work to run\nimmediately", + "steps": [ + { + "do": "restartBackend", + "settleMs": 15000 + }, + { + "actor": "watcher", + "do": "reload", + "settleMs": 5000 + }, + { + "actor": "watcher", + "do": "wait", + "ms": 25000 + }, + { + "actor": "watcher", + "do": "expectNumber", + "in": { + "contains": "Espresso Machine", + "testid": "item-card" + }, + "plus": 0, + "relativeTo": "esp-before", + "testid": "item-stock" + } + ] + } + ], + "id": 312, + "name": "A restart does not make pending work fire early", + "setup": [ + { + "actor": "admin", + "do": "signIn", + "exact": true, + "name": "admin", + "password": "stackbench-admin-2026" + }, + { + "actor": "admin", + "do": "click", + "settleMs": 1500, + "testid": "admin-link" + }, + { + "actor": "admin", + "do": "click", + "ifAvailable": true, + "testid": "restocks-link" + }, + { + "actor": "watcher", + "as": "esp-before", + "do": "recordNumber", + "in": { + "contains": "Espresso Machine", + "testid": "item-card" + }, + "testid": "item-stock" + }, + { + "actor": "admin", + "do": "fill", + "testid": "schedule-restock-item", + "text": "Espresso Machine" + }, + { + "actor": "admin", + "do": "fill", + "testid": "schedule-restock-warehouse", + "text": "West" + }, + { + "actor": "admin", + "do": "fill", + "testid": "schedule-restock-qty", + "text": "4" + }, + { + "actor": "admin", + "do": "fill", + "testid": "schedule-restock-delay", + "text": "180" + }, + { + "actor": "admin", + "do": "click", + "settleMs": 2000, + "testid": "schedule-restock-submit" + } + ] + }, + { + "actors": [ + "shopper", + "admin" + ], + "criteria": [ + { + "desc": "a reservation taken and left to expire returns exactly what it took, counted in the admin's own totals", + "id": "313a", + "points": 3, + "statedBy": "units are never created or destroyed by the\npassage of time", + "steps": [ + { + "actor": "shopper", + "do": "click", + "in": { + "contains": "Bluetooth Speaker", + "testid": "item-card" + }, + "settleMs": 1500, + "testid": "add-to-cart" + }, + { + "actor": "admin", + "do": "expectNumber", + "in": { + "contains": "Bluetooth Speaker", + "testid": "admin-item-row" + }, + "plus": -1, + "relativeTo": "spk-before", + "testid": "admin-stock", + "within": 10000 + }, + { + "actor": "shopper", + "do": "wait", + "ms": 100000 + }, + { + "actor": "admin", + "do": "expectNumber", + "in": { + "contains": "Bluetooth Speaker", + "testid": "admin-item-row" + }, + "plus": 0, + "relativeTo": "spk-before", + "testid": "admin-stock", + "within": 30000 + } + ] + } + ], + "id": 313, + "name": "Time does not create or destroy stock", + "setup": [ + { + "actor": "shopper", + "do": "signUp", + "name": "consv" + }, + { + "actor": "admin", + "do": "signIn", + "exact": true, + "name": "admin", + "password": "stackbench-admin-2026" + }, + { + "actor": "admin", + "do": "click", + "settleMs": 1500, + "testid": "admin-link" + }, + { + "actor": "admin", + "as": "spk-before", + "do": "recordNumber", + "in": { + "contains": "Bluetooth Speaker", + "testid": "admin-item-row" + }, + "testid": "admin-stock" + } + ] + } + ], + "level": 3, + "name": "Work that happens later — invariants", + "note": "What must hold about deferred work regardless of how it is implemented: it survives a restart, it fires once, it does not fire early, and time never creates or destroys stock. This is the level where in-memory timers stop being an implementation detail and become a defect.", + "schemaVersion": 1, + "track": "ecommerce", + "writeUrlPattern": "\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses|\\/schedule" + }, + "path": "scenarios/03-invariants.json" + } + ], + "track": "ecommerce" +} diff --git a/tools/stack-bench/tests/snapshots/definitions/loop.snapshot.json b/tools/stack-bench/tests/snapshots/definitions/loop.snapshot.json new file mode 100644 index 00000000000..9e4aa01a677 --- /dev/null +++ b/tools/stack-bench/tests/snapshots/definitions/loop.snapshot.json @@ -0,0 +1,71 @@ +{ + "definitionSchemaVersion": 1, + "levels": [ + { + "level": 1, + "suites": [ + { + "id": "features", + "inherit": "none", + "scenario": "scenarios/01-loop.json" + } + ] + } + ], + "manifest": { + "internal": true, + "plannedThrough": 1, + "portOffset": 600, + "restartProbe": "/", + "schemaVersion": 1, + "slug": "loop", + "suites": { + "1": [ + { + "id": "features", + "inherit": "none", + "spec": "scenarios/01-loop.json" + } + ] + }, + "title": "Loop Fixture", + "validatedThrough": 1 + }, + "scenarios": [ + { + "definition": { + "features": [ + { + "actors": [ + "viewer" + ], + "criteria": [ + { + "desc": "The application title is visible", + "id": "title-visible", + "points": 1, + "statedBy": "Fixture Chat", + "steps": [ + { + "actor": "viewer", + "contains": "Fixture Chat", + "do": "expect", + "testid": "app-title", + "within": 1000 + } + ] + } + ], + "id": 1, + "name": "Application title", + "setup": [] + } + ], + "level": 1, + "schemaVersion": 1 + }, + "path": "scenarios/01-loop.json" + } + ], + "track": "loop" +} diff --git a/tools/stack-bench/tests/source-checkpoint.test.ts b/tools/stack-bench/tests/source-checkpoint.test.ts new file mode 100644 index 00000000000..5adb6c3a963 --- /dev/null +++ b/tools/stack-bench/tests/source-checkpoint.test.ts @@ -0,0 +1,258 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { fileURLToPath } from 'node:url'; + +import { createArtifact, readArtifact, writeArtifact, writeRunJson } from '../src/evidence/artifacts.js'; +import { preserveLevelCheckpoint } from '../src/runtime/source-checkpoint.js'; +import { hashAppSource } from '../src/runtime/source-snapshot.js'; +import { inspectGradeSource } from '../commands/bench.js'; + +interface CheckpointPayload extends Record { + source: { sha256: string; directory: string }; +} + +test('dependency diagnostics bind the rejected first-build source and its own check scope', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-dependency-replay-')); + const output = join(root, 'execution'); + const sourcePath = join(output, 'first-build-l2'); + const bundlePath = join(output, 'first-build-l2-grading', 'bundle.json'); + try { + mkdirSync(sourcePath, { recursive: true }); + mkdirSync(join(output, 'source')); + writeFileSync(join(sourcePath, 'app.js'), 'export const depth = 2;\n'); + writeFileSync(join(output, 'source', 'app.js'), 'export const depth = 1;\n'); + const source = hashAppSource(sourcePath); + const check = 'ecommerce.spec.access-control.warehouse-write-boundary.103b'; + const selection = { schemaVersion: 3, sha256: 'a'.repeat(64), + recipe: { id: 'ecommerce.progression-catalog', sha256: 'd'.repeat(64) }, + requested: { features: ['ecommerce.feature.warehouse-admin'], checks: [check], + specifications: { requested: [], expected: ['ecommerce.spec.access-control'], observed: [] }, + dependencyExpansion: 'exact' }, + scoredChecks: [{ stableKey: check, points: 2 }], observedChecks: [] }; + const run = { + id: 'dependency-parent', startedAt: '2026-09-06T12:00:00.000Z', + completedAt: '2026-09-06T12:01:00.000Z', contaminated: false, + track: 'ecommerce', backend: 'postgres', model: 'test', mode: { id: 'dependency' }, + identities: { agentAdapter: { id: 'claude-code' } }, + backendLease: { runIndex: 2, resources: { serverUri: null as string | null } }, + runtime: { buildImage: 'test-image' }, + condition: { guidance: { credentialAliases: { staff: 'saved-staff' } }, requested: { + levels: [{ level: 2, recipe: { id: selection.recipe.id, contentSha256: selection.recipe.sha256 }, + selection: { ...selection, sha256: 'e'.repeat(64), scoredChecks: [ + ...selection.scoredChecks, { stableKey: 'never-attempted', points: 1 }] }, + task: { mode: 'upgrade', contractSha256: 'b'.repeat(64), requirementSha256: 'c'.repeat(64) } }], + } }, + levels: [{ level: 2, selection, checkpoint: null, repairs: 0, + firstBuild: { source: { sha256: source.sha256, files: source.files.length }, + outcome: { kind: 'inconclusive' } }, outcome: { kind: 'inconclusive' } }], + }; + const save = () => writeRunJson(join(output, 'run.json'), run); + save(); + writeArtifact(bundlePath, { kind: 'grade_bundle', id: 'dependency-parent-grade-l2', + attempt: { id: 'dependency-parent-grade-l2', parentId: run.id }, + identities: { recipe: selection.recipe }, + payload: { backend: 'postgres', track: 'ecommerce', level: 2, observation: 'scored', + source: { sha256: source.sha256 }, selection, + recipeRelease: { id: selection.recipe.id, contentSha256: selection.recipe.sha256 }, + outcome: { kind: 'inconclusive' } } }); + const before = readFileSync(join(output, 'run.json')); + const replay = inspectGradeSource(output, { level: 2, checkKeys: [check] }); + assert.equal(replay.sourcePath, sourcePath); + assert.equal(replay.source.sha256, source.sha256); + assert.equal(replay.serverUri, null); + assert.deepEqual(replay.aliases, { staff: 'saved-staff' }); + assert.deepEqual(readFileSync(join(output, 'run.json')), before); + const summarized = JSON.parse(before.toString()); + summarized.payload.levels[0].selection = { sha256: selection.sha256 }; + writeFileSync(join(output, 'run.json'), JSON.stringify(summarized)); + assert.equal(inspectGradeSource(output, { level: 2, checkKeys: [check] }).source.sha256, source.sha256); + summarized.payload.levels[0].selection.sha256 = 'f'.repeat(64); + writeFileSync(join(output, 'run.json'), JSON.stringify(summarized)); + assert.throws(() => inspectGradeSource(output, { level: 2, checkKeys: [check] }), /match/); + writeFileSync(join(output, 'run.json'), before); + const pending = JSON.parse(before.toString()); + pending.timestamps.completedAt = null; + writeFileSync(join(output, 'run.json'), JSON.stringify(pending)); + assert.throws(() => inspectGradeSource(output, { level: 2, checkKeys: [check] }), /recovered/); + const recovery = { schemaVersion: 1, runId: run.id, backend: run.backend, status: 'clean', reason: null, + cleanup: { succeeded: true, retained: false }, + resources: { backendState: 'released', listenerProcesses: [], locks: [] }, instructions: [] }; + const saveRecovery = (parentId: string) => writeArtifact(join(output, 'recovery.json'), { + kind: 'recovery', id: 'recovery', attempt: { id: 'recovery', parentId }, payload: recovery, + }); + saveRecovery('wrong-parent'); + assert.throws(() => inspectGradeSource(output, { level: 2, checkKeys: [check] }), /cleanup/); + saveRecovery(run.id); + assert.equal(inspectGradeSource(output, { level: 2, checkKeys: [check] }).parent.timestamps.completedAt, null); + recovery.cleanup.retained = true; + recovery.status = 'retained'; + saveRecovery(run.id); + assert.throws(() => inspectGradeSource(output, { level: 2, checkKeys: [check] }), /cleanup/); + writeFileSync(join(output, 'run.json'), before); + assert.throws(() => inspectGradeSource(output), /grade-level/); + assert.throws(() => inspectGradeSource(output, { level: 2 }), /check/); + assert.throws(() => inspectGradeSource(output, { level: 3, checkKeys: [check] }), /level|depth|candidate/); + assert.throws(() => inspectGradeSource(output, { level: 2, checkKeys: ['never-attempted'] }), /scope|check/); + assert.equal(inspectGradeSource(output, { level: 2, checkKeys: ['added-since-the-run'] }).source.sha256, + source.sha256); + run.levels.push(run.levels[0]!); + save(); + assert.throws(() => inspectGradeSource(output, { level: 2, checkKeys: [check] }), /unique|ambiguous|exactly one/); + run.levels.pop(); + run.contaminated = true; + save(); + assert.throws(() => inspectGradeSource(output, { level: 2, checkKeys: [check] }), /contaminat/); + run.contaminated = false; + save(); + const grade = JSON.parse(readFileSync(bundlePath, 'utf8')); + grade.attempt.parentId = 'other-run'; + writeFileSync(bundlePath, JSON.stringify(grade)); + assert.throws(() => inspectGradeSource(output, { level: 2, checkKeys: [check] }), /parent|match/); + grade.attempt.parentId = run.id; + run.backend = 'spacetime'; + run.backendLease.runIndex = 7; + run.backendLease.resources.serverUri = 'http://127.0.0.1:3217'; + grade.payload.backend = 'spacetime'; + writeFileSync(bundlePath, JSON.stringify(grade)); + save(); + const spacetimeReplay = inspectGradeSource(output, { level: 2, checkKeys: [check] }); + assert.equal(spacetimeReplay.serverUri, 'http://127.0.0.1:3217'); + assert.equal(spacetimeReplay.parent.payload.backendLease.runIndex, 7); + run.backend = 'convex'; + run.backendLease.resources.serverUri = 'http://127.0.0.1:13217'; + grade.payload.backend = 'convex'; + writeFileSync(bundlePath, JSON.stringify(grade)); + save(); + assert.equal(inspectGradeSource(output, { level: 2, checkKeys: [check] }).serverUri, 'http://127.0.0.1:13217'); + for (const uri of [null, 'http://remote.example:3217', 'https://127.0.0.1:3217', 'http://127.0.0.1']) { + run.backendLease.resources.serverUri = uri; + save(); + const rejected = spawnSync(process.execPath, + [fileURLToPath(new URL('../commands/bench.js', import.meta.url)), '--grade-from', output, + '--grade-level', '2', '--out', join(root, 'regrade'), '--check', check], + { encoding: 'utf8' }); + assert.equal(rejected.status, 1); + assert.match(rejected.stderr, /serverUri/); + assert.doesNotMatch(rejected.stderr, /preflight|Docker|docker/); + } + run.backendLease.resources.serverUri = 'http://127.0.0.1:3217'; + save(); + writeFileSync(join(sourcePath, 'app.js'), 'changed candidate'); + assert.throws(() => inspectGradeSource(output, { level: 2, checkKeys: [check] }), /source|match/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('a level checkpoint preserves only source and binds it to the parent run', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-level-checkpoint-')); + const app = join(root, 'app'); + const output = join(root, 'result'); + try { + mkdirSync(join(app, 'src'), { recursive: true }); + writeFileSync(join(app, 'src', 'app.ts'), 'export const answer = 42;\n'); + mkdirSync(join(app, 'node_modules', 'dep'), { recursive: true }); + writeFileSync(join(app, 'node_modules', 'dep', 'index.js'), 'not source\n'); + writeFileSync(join(app, 'BUG_REPORT.md'), 'private repair evidence\n'); + + const checkpoint = preserveLevelCheckpoint({ + appDir: app, + outputDir: output, + runId: 'run-parent', + track: 'ecommerce', + backend: 'postgres', + level: 1, + repair: { status: 'budget-exhausted', limit: 3, used: 3, + stopReason: 'budget-exhausted' }, + outcome: { kind: 'app_failure', failed: 1 }, + selectionSha256: 'a'.repeat(64), + }); + + const artifact = readArtifact(join(output, checkpoint.artifact), + { expectedKind: 'source_checkpoint', expectedId: 'run-parent-l1-checkpoint' }); + assert.equal(artifact.attempt.parentId, 'run-parent'); + assert.equal(artifact.payload.source.sha256, checkpoint.sha256); + assert.equal(artifact.payload.source.directory, 'level-l1-source'); + assert.equal(existsSync(join(output, checkpoint.directory, 'src', 'app.ts')), true); + assert.equal(existsSync(join(output, checkpoint.directory, 'node_modules')), false); + assert.equal(existsSync(join(output, checkpoint.directory, 'BUG_REPORT.md')), false); + + const selection = { schemaVersion: 3, sha256: 'a'.repeat(64), + requested: { features: [], checks: [] }, scoredChecks: [] }; + writeRunJson(join(output, 'run.json'), { + id: 'run-parent', startedAt: '2026-09-05T12:00:00.000Z', + completedAt: '2026-09-05T12:01:00.000Z', + track: 'ecommerce', backend: 'postgres', model: 'test', mode: { id: 'sequential' }, + identities: { agentAdapter: { id: 'claude-code' } }, + backendLease: { runIndex: 0 }, runtime: { buildImage: 'test-image' }, + condition: { guidance: { credentialAliases: { customer: 'saved-customer' } }, + requested: { levels: [{ level: 1, selection, + task: { contractSha256: 'b'.repeat(64), requirementSha256: 'c'.repeat(64) } }] } }, + levels: [{ level: 1, checkpoint }], + }); + const regrade = inspectGradeSource(output); + assert.equal(regrade.source.sha256, checkpoint.sha256); + assert.deepEqual(regrade.aliases, { customer: 'saved-customer' }); + const outsideScope = spawnSync(process.execPath, + [fileURLToPath(new URL('../commands/bench.js', import.meta.url)), '--grade-from', output, + '--out', join(root, 'regrade'), '--check', 'not-in-the-original-scope'], { encoding: 'utf8' }); + assert.equal(outsideScope.status, 1); + assert.match(outsideScope.stderr, /regrade checks must belong to the original scored scope/); + writeFileSync(join(output, checkpoint.directory, 'src', 'app.ts'), 'changed source'); + assert.throws(() => inspectGradeSource(output), /does not match its parent/); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('source checkpoint artifacts reject paths and repair accounting that cannot be trusted', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-level-checkpoint-invalid-')); + const app = join(root, 'app'); + try { + mkdirSync(app, { recursive: true }); + writeFileSync(join(app, 'app.js'), 'export default true;\n'); + assert.throws(() => createArtifact({ + kind: 'source_checkpoint', + id: 'wrong-path', + payload: { + schemaVersion: 3, + track: 'ecommerce', + backend: 'postgres', + level: 1, + source: { directory: '../outside', sha256: 'a'.repeat(64), files: 1 }, + repair: { status: 'budget-exhausted', limit: 3, used: 3, + stopReason: 'budget-exhausted' }, + outcome: { kind: 'app_failure' }, + selectionSha256: null, + }, + }), /directory does not match its level/); + assert.throws(() => preserveLevelCheckpoint({ + appDir: app, + outputDir: join(root, 'result'), + runId: 'run-parent', + track: 'ecommerce', + backend: 'postgres', + level: 1, + repair: { status: 'budget-exhausted', limit: 3, used: 4, + stopReason: 'budget-exhausted' }, + outcome: { kind: 'app_failure' }, + }), /used exceeds its budget/); + assert.throws(() => preserveLevelCheckpoint({ + appDir: app, + outputDir: join(root, 'feature-repairs'), + runId: 'run-parent', + track: 'ecommerce', + backend: 'postgres', + level: 1, + repair: { status: 'budget-exhausted', limit: 2, used: 2, + stopReason: 'budget-exhausted', nodeRepairs: [ + { nodeId: 'accounts', used: -1, exhaustionReason: null }, + ] }, + outcome: { kind: 'app_failure' }, + }), /nodeRepairs\[0\] is invalid/); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/tools/stack-bench/tests/source-materialization.test.ts b/tools/stack-bench/tests/source-materialization.test.ts new file mode 100644 index 00000000000..9bb20ef8487 --- /dev/null +++ b/tools/stack-bench/tests/source-materialization.test.ts @@ -0,0 +1,265 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import test from 'node:test'; +import { materializeAcceptedSource, restoreRepairSource } from '../src/runtime/source-materialization.js'; +import { assertPlainAppSourceTree, hashAppSource, snapshotAppSource, restoreAppSource } from '../src/runtime/source-snapshot.js'; + +test('accepted startup cannot read rejected files outside the saved source', async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-runtime-rollback-')); + try { + const app = join(root, 'app'), accepted = join(root, 'accepted'); + mkdirSync(app); + writeFileSync(join(app, 'start.sh'), 'node app.cjs\n'); + writeFileSync(join(app, 'app.cjs'), `const fs = require('node:fs'); + const paths = ['config.log', 'config.pid', 'stack-bench/config']; + process.stdout.write(paths.filter(p => fs.existsSync(p)).map(p => fs.readFileSync(p, 'utf8')).join(','));`); + snapshotAppSource(app, accepted); + const hash = hashAppSource(accepted).sha256; + for (const file of ['config.log', 'config.pid', 'stack-bench/config']) { + mkdirSync(dirname(join(app, file)), { recursive: true }); + writeFileSync(join(app, file), 'rejected'); + } + const execute = () => execFileSync(process.execPath, ['app.cjs'], { cwd: app, encoding: 'utf8' }); + assert.equal(execute(), 'rejected,rejected,rejected'); + assert.equal(hashAppSource(app).sha256, hash, 'control: excluded files have no source identity'); + const starts: unknown[] = []; + await materializeAcceptedSource(accepted, app, { backend: 'postgres', app, port: 6573, probe: '' }, + async (_spec, mode, options) => { if (mode === 'start') { starts.push(options); assert.equal(execute(), ''); } }); + assert.equal(execute(), '', 'accepted app must have the same inputs as a clean reconstruction'); + // The app was reset to source without dependencies, so it is not told this is a warm start. + assert.deepEqual(starts, [{ cleanSource: true }]); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('rejected installed dependency changes cannot survive accepted source materialization', { timeout: 60_000 }, async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-dependency-rollback-')); + try { + const app = join(root, 'app'), accepted = join(root, 'accepted'), dependency = join(root, 'dependency'); + mkdirSync(dependency); + mkdirSync(join(app, 'vendor'), { recursive: true }); + writeFileSync(join(dependency, 'package.json'), JSON.stringify({ name: 'rollback-dependency', version: '1.0.0' })); + writeFileSync(join(dependency, 'index.js'), 'module.exports = "accepted";\n'); + const npm = (cwd: string, args: string[]) => execFileSync(process.platform === 'win32' ? process.execPath : 'npm', + [...(process.platform === 'win32' ? [join(dirname(process.execPath), 'node_modules/npm/bin/npm-cli.js')] : []), + ...args, '--offline', '--ignore-scripts', '--no-audit', '--no-fund', '--cache', join(root, 'cache')], + { cwd, encoding: 'utf8', stdio: 'pipe', timeout: 15_000 }); + npm(dependency, ['pack', '--pack-destination', root]); + copyFileSync(join(root, 'rollback-dependency-1.0.0.tgz'), join(app, 'vendor', 'dependency.tgz')); + writeFileSync(join(app, 'package.json'), JSON.stringify({ dependencies: { 'rollback-dependency': 'file:vendor/dependency.tgz' } })); + writeFileSync(join(app, 'server.cjs'), 'process.stdout.write(require("rollback-dependency"));\n'); + writeFileSync(join(app, 'start.sh'), '#!/bin/sh\nset -eu\nnpm ci --offline --ignore-scripts\nnode server.cjs\n'); + npm(app, ['install', '--package-lock-only']); + npm(app, ['ci']); + snapshotAppSource(app, accepted); + const execute = () => execFileSync(process.execPath, [join(app, 'server.cjs')], { encoding: 'utf8' }); + assert.equal(execute(), 'accepted'); + writeFileSync(join(app, 'node_modules', 'rollback-dependency', 'index.js'), 'module.exports = "rejected";\n'); + const abandoned = join(app, 'rejected-layout', 'node_modules'); + mkdirSync(abandoned, { recursive: true }); + writeFileSync(join(abandoned, 'extra.js'), 'rejected'); + assert.equal(hashAppSource(app).sha256, hashAppSource(accepted).sha256); + assert.equal(execute(), 'rejected', 'control must change executed code without changing accepted source'); + const events: string[] = []; + await restoreRepairSource(accepted, app, { backend: 'postgres', app, port: 6573, probe: '' }, async (_spec, mode) => { + events.push(mode ?? 'restart'); + if (mode === 'start') { + assert.equal(existsSync(join(app, 'node_modules')), false); + assert.equal(existsSync(abandoned), false); + npm(app, ['ci']); + assert.equal(execute(), 'accepted'); + } + }, () => { events.push('reset'); }); + assert.deepEqual(events, ['stop', 'reset', 'start']); + assert.equal(execute(), 'accepted'); + assert.equal(hashAppSource(app).sha256, hashAppSource(accepted).sha256); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('clean dependency restore removes nested links without touching external packages', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-dependency-link-')); + try { + const app = join(root, 'app'), accepted = join(root, 'accepted'), outside = join(root, 'outside'); + mkdirSync(join(accepted, 'server'), { recursive: true }); + writeFileSync(join(accepted, 'server', 'index.js'), 'accepted'); + mkdirSync(outside); + writeFileSync(join(outside, 'dependency.js'), 'external'); + for (const location of ['server', 'abandoned']) { + mkdirSync(join(app, location), { recursive: true }); + symlinkSync(outside, join(app, location, 'node_modules'), process.platform === 'win32' ? 'junction' : 'dir'); + } + restoreAppSource(accepted, app, { cleanDependencies: true }); + assert.equal(existsSync(join(app, 'server', 'node_modules')), false); + assert.equal(existsSync(join(app, 'abandoned')), false); + assert.equal(readFileSync(join(outside, 'dependency.js'), 'utf8'), 'external'); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('clean startup regenerates root local environment without changing accepted source', async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-local-env-')); + try { + const app = join(root, 'app'), accepted = join(root, 'accepted'); + mkdirSync(app); + writeFileSync(join(app, 'start.sh'), '#!/bin/sh\n'); + writeFileSync(join(app, 'app.js'), 'export const ready = true;\n'); + writeFileSync(join(app, '.env.local'), 'VITE_CONVEX_URL=stale\n'); + writeFileSync(join(app, '.env'), 'APP_MODE=production\n'); + snapshotAppSource(app, accepted); + assert.equal(existsSync(join(accepted, '.env.local')), false); + const sourceSha256 = hashAppSource(accepted).sha256; + writeFileSync(join(app, '.env.local'), 'VITE_CONVEX_URL=wrong-runtime-value\n'); + await materializeAcceptedSource(accepted, app, { backend: 'convex', app, port: 0, probe: '' }, + async (_spec, mode) => { + if (mode !== 'start') return; + assert.equal(existsSync(join(app, '.env.local')), false); + assert.equal(readFileSync(join(app, '.env'), 'utf8'), 'APP_MODE=production\n'); + writeFileSync(join(app, '.env.local'), 'VITE_CONVEX_URL=current-lease\n'); + }); + assert.equal(hashAppSource(app).sha256, sourceSha256); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('startup may add runtime files but not change or remove accepted source', async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-startup-writes-')); + try { + const app = join(root, 'app'), accepted = join(root, 'accepted'); + mkdirSync(app); + writeFileSync(join(app, 'start.sh'), '#!/bin/sh\n'); + writeFileSync(join(app, 'app.js'), 'export const ready = true;\n'); + // A build marker left in the app directory is runtime state, not source. + mkdirSync(join(app, '.run')); + writeFileSync(join(app, '.run', 'frontend-build'), 'stale marker'); + snapshotAppSource(app, accepted); + assert.equal(existsSync(join(accepted, '.run')), false); + const start = (write: () => void) => materializeAcceptedSource(accepted, app, + { backend: 'spacetime', app, port: 0, probe: '' }, async (_spec, mode) => { if (mode === 'start') write(); }); + await start(() => { + mkdirSync(join(app, '.run'), { recursive: true }); + writeFileSync(join(app, '.run', 'frontend-build'), 'build marker'); + }); + for (const change of [ + () => writeFileSync(join(app, 'app.js'), 'export const ready = false;\n'), + () => rmSync(join(app, 'app.js')), + ]) { + await assert.rejects(start(change), (error: { code?: string; message?: string }) => + error.code === 'generated_app_source_changed' && /: app\.js$/.test(error.message ?? '')); + assert.equal(readFileSync(join(app, 'app.js'), 'utf8'), 'export const ready = true;\n'); + } + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('a start that only rewrites the package entry in its lock file keeps the accepted application', async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-lock-identity-')); + try { + const app = join(root, 'app'), accepted = join(root, 'accepted'); + mkdirSync(app); + writeFileSync(join(app, 'start.sh'), '#!/bin/sh\n'); + const lock = (name: string, version: string, range = '^5.3.0') => `${JSON.stringify({ name, version: '1.0.0', lockfileVersion: 3, + packages: { '': { name, version: '1.0.0', license: 'ISC', dependencies: { express: range } }, + 'node_modules/express': { version, integrity: 'sha512-fixture' } } }, null, 2)}\n`; + writeFileSync(join(app, 'package-lock.json'), lock('app', '5.3.0')); + snapshotAppSource(app, accepted); + const start = (written: string) => materializeAcceptedSource(accepted, app, + { backend: 'postgres', app, port: 0, probe: '' }, + async (_spec, mode) => { if (mode === 'start') writeFileSync(join(app, 'package-lock.json'), written); }); + // npm copies a hand-edited package.json name and ranges into the lock: the same packages are installed. + for (const written of [lock('storefront', '5.3.0'), lock('storefront', '5.3.0', '^5.0.0')]) { + await start(written); + assert.equal(readFileSync(join(app, 'package-lock.json'), 'utf8'), lock('app', '5.3.0'), 'the accepted lock file is back'); + } + // A different pinned dependency, or a lock file that no longer parses, is a different application. + for (const written of [lock('storefront', '5.3.1'), 'not json']) { + await assert.rejects(start(written), (error: { code?: string; message?: string }) => + error.code === 'generated_app_source_changed' && /: package-lock\.json$/.test(error.message ?? '')); + } + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('clean restore discards root local environment even from an older source snapshot', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-old-local-env-')); + try { + const app = join(root, 'app'), accepted = join(root, 'accepted'); + mkdirSync(app); + mkdirSync(accepted); + writeFileSync(join(accepted, 'start.sh'), '#!/bin/sh\n'); + writeFileSync(join(accepted, '.env.local'), 'VITE_CONVEX_URL=old-lease\n'); + writeFileSync(join(app, '.env.local'), 'VITE_CONVEX_URL=stale\n'); + restoreAppSource(accepted, app, { cleanDependencies: true }); + assert.equal(existsSync(join(app, '.env.local')), false); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('an authored root local environment remains bound and survives clean startup', async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-warm-local-env-')); + try { + const app = join(root, 'app'), accepted = join(root, 'accepted'); + mkdirSync(app); + writeFileSync(join(app, 'start.sh'), '#!/bin/sh\n'); + writeFileSync(join(app, '.env.local'), 'VITE_CONVEX_URL=http://lease\nAPP_MODE=authored\n'); + snapshotAppSource(app, accepted); + assert.equal(existsSync(join(accepted, '.env.local')), true); + const sourceSha256 = hashAppSource(accepted).sha256; + await materializeAcceptedSource(accepted, app, + { backend: 'convex', app, port: 0, probe: '' }, async (_spec, mode) => { + if (mode === 'start') assert.equal(readFileSync(join(app, '.env.local'), 'utf8'), + 'VITE_CONVEX_URL=http://lease\nAPP_MODE=authored\n'); + }); + assert.equal(hashAppSource(app).sha256, sourceSha256); + writeFileSync(join(app, '.env.local'), 'VITE_CONVEX_URL=http://lease\nAPP_MODE=changed\n'); + assert.notEqual(hashAppSource(app).sha256, sourceSha256); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('nested local environment, root environment, and authored code remain source-bound', async () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-authored-env-')); + try { + const app = join(root, 'app'), accepted = join(root, 'accepted'); + mkdirSync(join(app, 'client'), { recursive: true }); + writeFileSync(join(app, 'start.sh'), '#!/bin/sh\n'); + writeFileSync(join(app, '.env'), 'APP_MODE=production\n'); + writeFileSync(join(app, 'client', '.env.local'), 'CLIENT_MODE=production\n'); + writeFileSync(join(app, 'client', 'app.js'), 'export const ready = true;\n'); + snapshotAppSource(app, accepted); + for (const [file, value] of [ + ['.env', 'APP_MODE=changed\n'], + ['client/.env.local', 'CLIENT_MODE=changed\n'], + ['client/app.js', 'export const ready = false;\n'], + ] as const) { + await assert.rejects(materializeAcceptedSource(accepted, app, + { backend: 'convex', app, port: 0, probe: '' }, async (_spec, mode) => { + if (mode === 'start') writeFileSync(join(app, file), value); + }), /materialized application source differs from its accepted snapshot/); + } + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('runtime file names do not hide authored directories or links', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-runtime-name-dir-')); + try { + const app = join(root, 'app'); + mkdirSync(app); + writeFileSync(join(app, 'app.js'), 'export const ready = true;\n'); + const original = hashAppSource(app).sha256; + for (const name of ['.env.local', 'cache.tsbuildinfo', 'a.log']) { + const directory = join(app, name); + mkdirSync(directory); + writeFileSync(join(directory, 'logic.js'), 'export const value = 1;\n'); + const accepted = hashAppSource(app).sha256; + assert.notEqual(accepted, original, `${name} directory must belong to source identity`); + writeFileSync(join(directory, 'logic.js'), 'export const value = 2;\n'); + assert.notEqual(hashAppSource(app).sha256, accepted, `${name} executable change must be detected`); + rmSync(directory, { recursive: true, force: true }); + } + const accepted = join(root, 'accepted'); + snapshotAppSource(app, accepted); + const outside = join(root, 'outside.js'); + writeFileSync(outside, 'export const value = 3;\n'); + symlinkSync(outside, join(app, '.env.local'), 'file'); + assert.throws(() => assertPlainAppSourceTree(app), /unsupported filesystem entry.*\.env\.local/); + restoreAppSource(accepted, app, { cleanDependencies: true }); + assert.equal(existsSync(join(app, '.env.local')), false); + assert.equal(readFileSync(outside, 'utf8'), 'export const value = 3;\n'); + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/tools/stack-bench/tests/source-snapshot.test.ts b/tools/stack-bench/tests/source-snapshot.test.ts new file mode 100644 index 00000000000..1b849f0c811 --- /dev/null +++ b/tools/stack-bench/tests/source-snapshot.test.ts @@ -0,0 +1,292 @@ +import assert from 'node:assert/strict'; +import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, + symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { assertAppSourceIdentity, assertPlainAppSourceTree, hashAppSource, resetAppToSource, + restoreAppSource, seedAppSource, snapshotAppSource } from '../src/runtime/source-snapshot.js'; + +const TOOL_CACHES = ['.apt', '.cache', '.debroot', '.libs', '.npm-cache', '.pw-browsers', '.pwcache']; + +const put = (path: string, content: string): void => { + mkdirSync(join(path, '..'), { recursive: true }); + writeFileSync(path, content); +}; + +test('source rollback is layout-independent and preserves watched directories and nested dependencies', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-source-snapshot-')); + const app = join(root, 'app'); + const snapshot = join(root, 'snapshot'); + const client = join(app, 'ui', 'src'); + put(join(client, 'App.tsx'), 'export const value = "good";\n'); + put(join(app, 'functions', 'handler.mjs'), 'export default "original";\n'); + put(join(app, 'backend', 'spacetimedb', 'node_modules', 'dep', 'index.js'), 'dependency\n'); + put(join(app, 'BUG_REPORT.md'), 'current harness report\n'); + put(join(app, 'bug-report-quality.json'), '{}\n'); + put(join(app, '.read-guard-settings.json'), '{}\n'); + put(join(app, '.stack-bench-backend'), 'mongodb\n'); + put(join(app, '.prompt-build-l1.md'), 'prompt\n'); + put(join(app, '.session-fix-l1.json'), '{}\n'); + put(join(app, 'ui', 'dist', 'bundle.js'), 'compiled\n'); + put(join(app, 'ui', 'src', 'module_bindings', 'index.ts'), 'generated\n'); + put(join(app, 'server.log'), 'server output\n'); + put(join(app, '.server.pid'), '100\n'); + put(join(app, 'ui', 'vite.log'), 'client output\n'); + put(join(app, 'src', 'stack-bench', 'runtime.ts'), 'export const owned = true;\n'); + put(join(app, 'src', 'server.txt'), 'model-authored input\n'); + put(join(app, 'ui', 'tsconfig.tsbuildinfo'), 'compiler cache\n'); + for (const directory of TOOL_CACHES) put(join(app, directory, 'tool-artifact'), 'not application source\n'); + + const watchedDirectoryIdentity = statSync(client).ino; + snapshotAppSource(app, snapshot); + + assert.equal(existsSync(join(snapshot, 'backend', 'spacetimedb', 'node_modules')), false); + assert.equal(existsSync(join(snapshot, 'BUG_REPORT.md')), false); + for (const file of ['bug-report-quality.json', '.read-guard-settings.json', + '.stack-bench-backend', '.prompt-build-l1.md', + '.session-fix-l1.json']) { + assert.equal(existsSync(join(snapshot, file)), true); + } + assert.equal(existsSync(join(snapshot, 'ui', 'dist')), false); + assert.equal(existsSync(join(snapshot, 'ui', 'src', 'module_bindings')), true); + assert.equal(existsSync(join(snapshot, 'server.log')), false); + assert.equal(existsSync(join(snapshot, '.server.pid')), false); + // Logs are runtime output at any depth. + assert.equal(existsSync(join(snapshot, 'ui', 'vite.log')), false); + assert.equal(existsSync(join(snapshot, 'src', 'stack-bench', 'runtime.ts')), true); + assert.equal(existsSync(join(snapshot, 'src', 'server.txt')), true); + assert.equal(existsSync(join(snapshot, 'ui', 'tsconfig.tsbuildinfo')), false); + for (const directory of TOOL_CACHES) assert.equal(existsSync(join(snapshot, directory)), false, directory); + + put(join(client, 'App.tsx'), 'export const value = "bad fix";\n'); + put(join(client, 'introduced.ts'), 'remove me\n'); + put(join(app, 'functions', 'handler.mjs'), 'broken\n'); + put(join(app, 'new-layout', 'bad-source.js'), 'remove me\n'); + put(join(app, 'new-layout', 'node_modules', 'installed', 'index.js'), 'keep me\n'); + put(join(app, 'ui', 'dist', 'stale.js'), 'remove me\n'); + put(join(app, 'ui', 'src', 'module_bindings', 'new.ts'), 'stale binding\n'); + put(join(app, 'BUG_REPORT.md'), 'latest harness report\n'); + put(join(app, 'bug-report-quality.json'), '{"latest":true}\n'); + put(join(app, 'server.log'), 'new server output\n'); + put(join(app, '.server.pid'), '200\n'); + put(join(app, 'ui', 'vite.log'), 'new client output\n'); + + restoreAppSource(snapshot, app); + + assert.equal(readFileSync(join(client, 'App.tsx'), 'utf8'), 'export const value = "good";\n'); + assert.equal(readFileSync(join(app, 'functions', 'handler.mjs'), 'utf8'), 'export default "original";\n'); + assert.equal(existsSync(join(client, 'introduced.ts')), false); + assert.equal(existsSync(join(app, 'new-layout', 'bad-source.js')), false); + assert.equal(readFileSync(join(app, 'new-layout', 'node_modules', 'installed', 'index.js'), 'utf8'), 'keep me\n'); + assert.equal(readFileSync(join(app, 'backend', 'spacetimedb', 'node_modules', 'dep', 'index.js'), 'utf8'), 'dependency\n'); + assert.equal(existsSync(join(app, 'ui', 'dist')), false); + assert.equal(existsSync(join(app, 'ui', 'tsconfig.tsbuildinfo')), false); + assert.equal(existsSync(join(app, 'ui', 'src', 'module_bindings')), true); + assert.equal(readFileSync(join(app, 'BUG_REPORT.md'), 'utf8'), 'latest harness report\n'); + assert.equal(readFileSync(join(app, 'bug-report-quality.json'), 'utf8'), '{}\n'); + assert.equal(readFileSync(join(app, 'server.log'), 'utf8'), 'new server output\n'); + assert.equal(readFileSync(join(app, '.server.pid'), 'utf8'), '200\n'); + assert.equal(readFileSync(join(app, 'ui', 'vite.log'), 'utf8'), 'new client output\n'); + assert.equal(statSync(client).ino, watchedDirectoryIdentity, 'restore replaced a watched source directory'); +}); + +test('source seeding copies arbitrary layouts without dependencies or repair evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-source-seed-')); + const source = join(root, 'source'); + const target = join(root, 'target'); + put(join(source, 'unconventional', 'api', 'main.go'), 'package main\n'); + put(join(source, 'unconventional', 'node_modules', 'dep.js'), 'dependency\n'); + put(join(source, '.prompt-build-l1.md'), 'model-authored file\n'); + + seedAppSource(source, target); + + assert.equal(readFileSync(join(target, 'unconventional', 'api', 'main.go'), 'utf8'), 'package main\n'); + assert.equal(existsSync(join(target, 'unconventional', 'node_modules')), false); + assert.equal(readFileSync(join(target, '.prompt-build-l1.md'), 'utf8'), 'model-authored file\n'); + if (process.platform !== 'win32') { + assert.equal(statSync(join(target, 'unconventional')).mode & 0o702, 0o700); + assert.equal(statSync(join(target, 'unconventional', 'api', 'main.go')).mode & 0o602, 0o600); + } +}); + +test('source operations reject overlapping trees before deleting data', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-source-overlap-')); + const app = join(root, 'app'); + try { + put(join(app, 'src', 'app.ts'), 'preserve me\n'); + assert.throws(() => snapshotAppSource(app, app), /must not overlap/); + assert.throws(() => snapshotAppSource(app, join(app, 'snapshot')), /must not overlap/); + assert.throws(() => resetAppToSource(join(app, 'src'), app), /must not overlap/); + assert.equal(readFileSync(join(app, 'src', 'app.ts'), 'utf8'), 'preserve me\n'); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('source copies reject symbolic links before changing permissions', { skip: process.platform === 'win32' }, () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-source-link-')); + const source = join(root, 'source'); + const outside = join(root, 'outside.txt'); + try { + put(outside, 'outside\n'); + mkdirSync(source, { recursive: true }); + symlinkSync(outside, join(source, 'link.txt')); + assert.throws(() => seedAppSource(source, join(root, 'target')), + /unsupported filesystem entry link\.txt/); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('source snapshots ignore nested node_modules symlinks', { skip: process.platform === 'win32' }, () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-source-dependency-link-')); + const app = join(root, 'app'); + const snapshot = join(root, 'snapshot'); + const dependencies = join(root, 'dependencies'); + const dependencyLink = join(app, '_scratch', 'rls-client', 'node_modules'); + try { + put(join(app, 'src', 'app.ts'), 'export const app = true;\n'); + put(join(dependencies, 'package', 'index.js'), 'dependency\n'); + mkdirSync(join(app, '_scratch', 'rls-client'), { recursive: true }); + symlinkSync(dependencies, dependencyLink, 'dir'); + + const identity = hashAppSource(app); + assert.doesNotThrow(() => assertPlainAppSourceTree(app)); + snapshotAppSource(app, snapshot); + assert.equal(existsSync(join(snapshot, '_scratch', 'rls-client', 'node_modules')), false); + restoreAppSource(snapshot, app); + assert.equal(existsSync(dependencyLink), true); + put(join(dependencies, 'package', 'index.js'), 'changed dependency\n'); + assert.equal(hashAppSource(app).sha256, identity.sha256); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('source restore replaces symbolic links without touching their targets', + { skip: process.platform === 'win32' }, () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-restore-link-')); + const snapshot = join(root, 'snapshot'); + const app = join(root, 'app'); + const outside = join(root, 'outside.txt'); + try { + put(join(snapshot, 'src', 'app.ts'), 'restored\n'); + put(outside, 'outside\n'); + mkdirSync(join(app, 'src'), { recursive: true }); + symlinkSync(outside, join(app, 'src', 'app.ts')); + restoreAppSource(snapshot, app); + assert.equal(readFileSync(join(app, 'src', 'app.ts'), 'utf8'), 'restored\n'); + assert.equal(readFileSync(outside, 'utf8'), 'outside\n'); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + +test('clean source reset removes all runtime state and preserves only root git metadata', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-clean-source-reset-')); + const app = join(root, 'app'); + const snapshot = join(root, 'snapshot'); + try { + put(join(app, 'src', 'app.ts'), 'export const value = 1;\n'); + put(join(app, 'src', 'stack-bench', 'runtime.ts'), 'export const owned = true;\n'); + put(join(app, 'src', 'server.txt'), 'model-authored input\n'); + put(join(app, '.git', 'HEAD'), 'before-reset\n'); + snapshotAppSource(app, snapshot); + + put(join(app, 'src', 'app.ts'), 'export const value = 2;\n'); + put(join(app, 'introduced.ts'), 'remove me\n'); + put(join(app, '.git', 'HEAD'), 'preserved\n'); + put(join(app, 'node_modules', 'package', 'index.js'), 'patched dependency\n'); + put(join(app, 'server', 'node_modules', 'package', 'index.js'), 'nested dependency\n'); + put(join(app, 'dist', 'server.js'), 'generated\n'); + put(join(app, 'client', '.vite', 'bundle.js'), 'generated\n'); + put(join(app, 'client', '.cache', 'entry'), 'cached\n'); + put(join(app, 'client', 'src', 'module_bindings', 'index.ts'), 'generated binding\n'); + put(join(app, 'server', 'tsconfig.tsbuildinfo'), 'compiler cache\n'); + put(join(app, 'stack-bench', 'bundle.json'), '{"private":true}\n'); + put(join(app, 'BUG_REPORT.md'), 'repair evidence\n'); + put(join(app, 'server.log'), 'runtime log\n'); + put(join(app, 'client.log'), 'runtime log\n'); + put(join(app, 'vite.log'), 'runtime log\n'); + put(join(app, '.server.pid'), '200\n'); + + resetAppToSource(snapshot, app); + + assert.equal(readFileSync(join(app, 'src', 'app.ts'), 'utf8'), 'export const value = 1;\n'); + assert.equal(readFileSync(join(app, 'src', 'stack-bench', 'runtime.ts'), 'utf8'), + 'export const owned = true;\n'); + assert.equal(readFileSync(join(app, 'src', 'server.txt'), 'utf8'), 'model-authored input\n'); + assert.equal(readFileSync(join(app, '.git', 'HEAD'), 'utf8'), 'preserved\n'); + for (const path of ['introduced.ts', 'node_modules', join('server', 'node_modules'), + 'dist', join('client', '.vite'), join('client', '.cache'), + join('client', 'src', 'module_bindings'), join('server', 'tsconfig.tsbuildinfo'), + 'stack-bench', 'BUG_REPORT.md', 'server.log', 'client.log', 'vite.log', '.server.pid']) { + assert.equal(existsSync(join(app, path)), false, `${path} survived the clean source reset`); + } + if (process.platform !== 'win32') { + chmodSync(join(snapshot, 'src'), 0o555); + chmodSync(join(snapshot, 'src', 'app.ts'), 0o444); + resetAppToSource(snapshot, app); + assert.equal(statSync(join(app, 'src')).mode & 0o702, 0o700); + assert.equal(statSync(join(app, 'src', 'app.ts')).mode & 0o602, 0o600); + } + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('source identity matches preserved bytes and ignores dependencies and harness evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-source-identity-')); + const app = join(root, 'app'); + const snapshot = join(root, 'snapshot'); + try { + put(join(app, 'src', 'app.ts'), 'export const value = 1;\n'); + put(join(app, 'node_modules', 'dep', 'index.js'), 'dependency v1\n'); + put(join(app, 'stack-bench', 'bundle.json'), '{}\n'); + put(join(app, 'client', 'src', 'module_bindings', 'index.ts'), 'generated v1\n'); + put(join(app, 'BUG_REPORT.md'), 'private evidence\n'); + put(join(app, 'bug-report-quality.json'), '{}\n'); + put(join(app, 'server.log'), 'server output v1\n'); + put(join(app, '.server.pid'), '100\n'); + put(join(app, 'client', 'vite.log'), 'client output v1\n'); + put(join(app, 'src', 'stack-bench', 'runtime.ts'), 'export const owned = 1;\n'); + put(join(app, 'src', 'server.txt'), 'model-authored input v1\n'); + put(join(app, '.npm-cache', 'tool-artifact'), 'cache v1\n'); + const first = hashAppSource(app); + assert.deepEqual(assertAppSourceIdentity(app, first.sha256), first); + snapshotAppSource(app, snapshot); + assert.equal(hashAppSource(snapshot).sha256, first.sha256); + put(join(app, 'node_modules', 'dep', 'index.js'), 'dependency v2\n'); + put(join(app, 'stack-bench', 'bundle.json'), '{"changed":true}\n'); + put(join(app, 'client', 'src', 'module_bindings', 'index.ts'), 'generated v2\n'); + put(join(app, 'server.log'), 'server output v2\n'); + put(join(app, '.server.pid'), '200\n'); + put(join(app, 'client', 'vite.log'), 'client output v2\n'); + put(join(app, '.npm-cache', 'tool-artifact'), 'cache v2\n'); + // Log and pid files an application writes at startup, at any depth. + put(join(app, 'app.log'), 'listening\n'); + put(join(app, '.app.pid'), '4242\n'); + put(join(app, '.run', 'frontend.log'), 'ready\n'); + assert.equal(hashAppSource(app).sha256, first.sha256); + // The .run directory is runtime state; other files an app writes still count as source. + put(join(app, '.run', 'frontend-build'), 'marker\n'); + assert.equal(hashAppSource(app).sha256, first.sha256); + put(join(app, 'state', 'frontend.json'), '{}\n'); + assert.notEqual(hashAppSource(app).sha256, first.sha256); + rmSync(join(app, 'state'), { recursive: true }); + put(join(app, 'src', 'stack-bench', 'runtime.ts'), 'export const owned = 2;\n'); + assert.notEqual(hashAppSource(app).sha256, first.sha256); + put(join(app, 'src', 'stack-bench', 'runtime.ts'), 'export const owned = 1;\n'); + put(join(app, 'src', 'server.txt'), 'model-authored input v2\n'); + assert.notEqual(hashAppSource(app).sha256, first.sha256); + put(join(app, 'src', 'server.txt'), 'model-authored input v1\n'); + put(join(app, 'src', 'app.ts'), 'export const value = 2;\n'); + assert.notEqual(hashAppSource(app).sha256, first.sha256); + assert.throws(() => assertAppSourceIdentity(app, first.sha256, 'restored mutation source'), + /restored mutation source hash .* does not match/); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/tools/stack-bench/tests/spacetime-captured-replay.integration.ts b/tools/stack-bench/tests/spacetime-captured-replay.integration.ts new file mode 100644 index 00000000000..18d48ee0807 --- /dev/null +++ b/tools/stack-bench/tests/spacetime-captured-replay.integration.ts @@ -0,0 +1,608 @@ +import assert from 'node:assert/strict'; +import { createServer, type ServerResponse } from 'node:http'; +import { once } from 'node:events'; +import { createRequire } from 'node:module'; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { chromium, type Page } from 'playwright'; +import { Actor } from '../grader/grade.js'; +import { ACTION_REGISTRY } from '../src/actions/action-catalog.js'; +import { executeAction, ActionInconclusive } from '../src/actions/action-contract.js'; +import { withAuthRequestPatch } from '../src/actions/auth-request-patch.js'; +import { beginSpacetimeAuthObservation, confirmSpacetimeSignup, installSpacetimeWriteCapture } from '../src/stacks/backends/spacetime-browser-session.js'; +import { createBackendLease, writeBackendLease } from '../src/runtime/backend-lease.js'; + +// Failure cases specified before implementation: unsupported or ambiguous +// capture, changed non-name arguments, missing commit receipts, wrong target, +// reconnects and unsupported protocol use UI BEFORE sending. Once sent, refusal +// fails; lost/malformed replies, collisions and timeout are inconclusive. None +// may retry through the form. All other values and the live connection survive. +// An app may proxy the leased database through its own origin under any prefix; +// the database layout then comes from the lease, not the app. +test('SpacetimeDB catalog replay preserves the caller and never retries a sent write', async () => { + const codec = await import(new URL('../src/stacks/spacetime-wire-codec.js', import.meta.url).href); + const { wsServer } = createRequire(import.meta.url)('playwright-core/lib/utilsBundle'); + const encode = (type: { serialize(writer: unknown, value: unknown): void }, value: unknown) => { + const writer = new codec.BinaryWriter(256); type.serialize(writer, value); return Buffer.from(writer.getBuffer()); + }; + const strings = codec.AlgebraicType.makeSerializer({ tag: 'String' }); + const readString = codec.AlgebraicType.makeDeserializer({ tag: 'String' }); + const callBytes = (name: string, requestId: number, nonce: string) => { + const writer = new codec.BinaryWriter(128); strings(writer, name); strings(writer, nonce); + return encode(codec.ClientMessage, { tag: 'CallReducer', value: { + reducer: 'catalog_add', requestId, flags: 0, args: writer.getBuffer(), + } }); + }; + let mode = '', calls = 0, connections = 0, page: Page; + const rows: string[] = [], observations: unknown[] = []; + const schema = (res: ServerResponse) => { + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ reducers: [{ name: 'catalog_add', params: { elements: [ + { name: { some: 'title' }, algebraic_type: mode === 'unsupported-schema' ? { Array: { U8: [] } } : { String: [] } }, + { name: { some: 'nonce' }, algebraic_type: { String: [] } }, + ] } }] })); + }; + const database = createServer((req, res) => { + if (new URL(req.url!, 'http://localhost').pathname === '/v1/database/catalog/schema') schema(res); + else res.writeHead(404).end(); + }).listen(0, '127.0.0.1'); + await once(database, 'listening'); + const databaseUrl = `http://127.0.0.1:${(database.address() as { port: number }).port}`; + const server = createServer(async (req, res) => { + const url = new URL(req.url!, 'http://localhost'); + if (url.pathname.endsWith('/schema')) { + if (mode === 'app-proxy') res.writeHead(404).end(); + else schema(res); + } else if (url.pathname === '/encode') { + res.end(callBytes(url.searchParams.get('name')!, Number(url.searchParams.get('id')), url.searchParams.get('nonce')!)); + } else { + res.setHeader('Content-Type', 'text/html'); + res.end(``); + } + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const sockets = new wsServer({ server }); + sockets.on('connection', (socket: { on(event: string, fn: (data: Buffer) => void): void; send(data: Buffer): void; close(): void }) => { + connections++; + const send = (value: unknown) => socket.send(Buffer.concat([Buffer.from([0]), encode(codec.ServerMessage, value)])); + send({ tag: 'InitialConnection', value: { identity: { __identity__: 1n }, connectionId: { __connection_id__: BigInt(connections) }, token: 'fixture-token' } }); + socket.on('message', data => { + const message = codec.ClientMessage.deserialize(new codec.BinaryReader(data)); + if (message.tag !== 'CallReducer') return; + const reader = new codec.BinaryReader(message.value.args), name = readString(reader), nonce = readString(reader); + assert.equal(nonce, mode === 'changing-argument' ? String(calls + 1) : 'unchanged'); calls++; + const native = message.value.requestId > 100; + if (!(native && mode === 'refused')) rows.push(name); + if (native && mode === 'lost') { socket.close(); return; } + if (native && mode === 'timeout') return; + if (native && mode === 'malformed') { socket.send(Buffer.from([0, 255])); return; } + if (native && mode === 'collision' && name !== 'Collision') { + void page.evaluate(bytes => (window as unknown as { socket: WebSocket }).socket.send(new Uint8Array(bytes)), + [...callBytes('Collision', message.value.requestId, 'unchanged')]); + return; + } + if (mode === 'unconfirmed' && name === 'Original') return; + send({ tag: 'ReducerResult', value: { requestId: message.value.requestId, + timestamp: { __timestamp_micros_since_unix_epoch__: 1n }, result: native && mode === 'refused' + ? { tag: 'Err', value: new Uint8Array() } : { tag: 'OkEmpty' } } }); + }); + }); + const root = mkdtempSync(join(tmpdir(), 'stack-bench-native-replay-')); + const leasePath = join(root, 'lease.json'); + const previous = { path: process.env.STACK_BENCH_LEASE, token: process.env.STACK_BENCH_LEASE_TOKEN }; + const browser = await chromium.launch(); + let result = 'failed'; + try { + for (mode of ['accepted', 'app-proxy', 'changing-argument', 'unsupported-schema', 'unsupported-protocol', 'unconfirmed', 'wrong-module', 'reconnect', 'refused', 'lost', 'malformed', 'collision', 'timeout']) { + calls = 0; connections = 0; rows.length = 0; + const lease = createBackendLease({ runId: 'native-replay', backend: 'spacetime', track: 'ecommerce', runIndex: 0, + serverUri: mode === 'app-proxy' ? databaseUrl : url, module: mode === 'wrong-module' ? 'other-module' : 'catalog', dataDir: join(root, 'data') }); + lease.state = 'active'; writeBackendLease(leasePath, lease); + process.env.STACK_BENCH_LEASE = leasePath; process.env.STACK_BENCH_LEASE_TOKEN = lease.ownershipToken; + const context = await browser.newContext(); + try { + page = await context.newPage(); + const actor = new Actor('admin', page, context, false, true); + await actor.ready; await page.goto(url); + await page.waitForFunction(() => (window as unknown as { socket: WebSocket }).socket.readyState === WebSocket.OPEN); + for (const name of ['Original', 'Control']) { + await page.locator('input').fill(name); await page.locator('button').click(); + const deadline = Date.now() + 3000; + while (calls < (name === 'Original' ? 1 : 2)) { + assert(Date.now() < deadline, `${mode}: UI control did not reach the server`); + await new Promise(resolve => setTimeout(resolve, 5)); + } + } + await page.waitForTimeout(50); + if (mode === 'reconnect') { + await page.evaluate(async () => { + const state = window as unknown as { socket: WebSocket }; + const url = state.socket.url; + await new Promise(resolve => { state.socket.addEventListener('close', () => resolve(), { once: true }); state.socket.close(); }); + state.socket = new WebSocket(url, 'v3.bsatn.spacetimedb'); + await new Promise(resolve => state.socket.addEventListener('open', () => resolve(), { once: true })); + }); + } + const interaction = { defaultWithin: 1000, expand: (s: string) => s, sleep: async () => {} }; + const outcome = await executeAction(ACTION_REGISTRY, 'repeatFormWrite', { + do: 'repeatFormWrite', actor: 'admin', match: 'Original', control: 'Control', replacement: 'New "product"', + fields: [{ testid: 'name', text: 'New "product"' }], submit: 'save', + }, { capabilities: { actors: { get: () => actor }, 'browser-interaction': interaction, + 'transport-observation': interaction, 'named-actions': { fetch } } }); + await page.waitForTimeout(50); + const native = ['accepted', 'app-proxy', 'refused', 'lost', 'malformed', 'collision', 'timeout'].includes(mode); + assert.equal(await page.evaluate(() => (window as unknown as { formSubmits: number }).formSubmits), native ? 2 : 3, mode); + assert.equal(calls, mode === 'collision' ? 4 : 3, `${mode}: no retry`); + assert.equal(connections, mode === 'reconnect' ? 2 : 1, `${mode}: no harness-created connection`); + assert.equal(outcome.status, mode === 'refused' ? 'failed' : ['lost', 'malformed', 'collision', 'timeout'].includes(mode) ? 'inconclusive' : 'passed', JSON.stringify(outcome)); + assert.equal(rows.filter(name => name === 'New "product"').length, mode === 'refused' ? 0 : 1, mode); + observations.push({ mode, calls, connections, outcome }); + } finally { await context.close(); } + } + result = 'passed'; + } finally { + await browser.close(); sockets.close(); server.closeAllConnections(); await new Promise(resolve => server.close(() => resolve())); + await new Promise(resolve => database.close(() => resolve())); + if (previous.path === undefined) delete process.env.STACK_BENCH_LEASE; else process.env.STACK_BENCH_LEASE = previous.path; + if (previous.token === undefined) delete process.env.STACK_BENCH_LEASE_TOKEN; else process.env.STACK_BENCH_LEASE_TOKEN = previous.token; + rmSync(root, { recursive: true, force: true }); + if (process.env.STACK_BENCH_REPLAY_EVIDENCE) { + mkdirSync(process.env.STACK_BENCH_REPLAY_EVIDENCE, { recursive: true }); + writeFileSync(join(process.env.STACK_BENCH_REPLAY_EVIDENCE, 'spacetime-receipt.json'), JSON.stringify({ result, observations, + rerun: 'node --test dist/tests/spacetime-captured-replay.integration.js' }, null, 2)); + } + } +}); + +// Failure cases: a receipt cannot hide a later ambiguous call or decode failure. +// Renaming a reducer does not change its credential semantics; supported scalar +// fields, 64-bit integers, optional values, and enums must preserve their values. +// An exact localhost alias is valid; wrong ports, type changes, and nonmatching +// credentials must not earn a receipt. Unknown schemas and wrong lease targets +// remain unmeasured. No probe may replay a second request on another connection. +// A completed HTTP or native login can reload and close its socket. That must +// preserve its receipt; closing before a native reply must remain inconclusive. +// Credentials may go to a procedure instead of a reducer: its Returned status is +// a completed call and InternalError (a thrown refusal) is a refusal. A reducer +// call never matches a procedure declaration of the same name. +test('SpacetimeDB auth receipts remain valid only for one complete credential call', async t => { + const codec = await import(new URL('../src/stacks/spacetime-wire-codec.js', import.meta.url).href); + const { wsServer } = createRequire(import.meta.url)('playwright-core/lib/utilsBundle'); + const encode = (type: { serialize(writer: unknown, value: unknown): void }, value: unknown) => { + const writer = new codec.BinaryWriter(256); type.serialize(writer, value); return Buffer.from(writer.getBuffer()); + }; + let mode = '', connections = 0, procedure = false; + const received: unknown[][] = [], observations: unknown[] = []; + const scalarCase = () => mode === 'scalar-fields'; + const integerCase = () => ['u64-fields', 'type-changing-fields'].includes(mode); + const optionCase = () => mode.startsWith('optional-'); + const types = (): { tag: string; value?: unknown }[] => { + const base = [{ tag: 'String' }, { tag: 'String' }]; + if (mode === 'opaque-salted-signup') return [...base, { tag: 'String' }]; + if (scalarCase()) return [...base, { tag: 'Bool' }, { tag: 'U32' }]; + if (integerCase()) return [...base, { tag: 'U64' }]; + if (optionCase()) return [...base, { tag: 'Sum', value: { variants: [ + { name: 'some', algebraicType: { tag: 'Bool' } }, + { name: 'none', algebraicType: { tag: 'Product', value: { elements: [] } } }, + ] } }]; + if (mode === 'enum-fields') return [...base, { tag: 'Sum', value: { variants: [ + { name: 'Customer', algebraicType: { tag: 'Product', value: { elements: [] } } }, + { name: 'Staff', algebraicType: { tag: 'Product', value: { elements: [] } } }, + ] } }]; + return base; + }; + const values = (user?: string, credential?: string): unknown[] => { + const base = [user ?? (mode === 'nonmatching-credentials' ? 'different-customer' : 'customer'), + credential ?? (['opaque-hash-login', 'opaque-delayed-login', 'opaque-ok-denial', 'opaque-completed-reload'].includes(mode) ? 'hash-wrong' + : mode === 'opaque-hash-signup' || mode === 'opaque-delayed-signup' ? 'hash-claimant' : 'secret')]; + return mode === 'opaque-salted-signup' ? [...base, `salt-${base[0]}`] + : scalarCase() ? [...base, false, 17] : integerCase() ? [...base, 9007199254740993n] + : optionCase() ? [...base, mode === 'optional-some' ? false : undefined] + : mode === 'enum-fields' ? [...base, { tag: 'Customer', value: {} }] : base; + }; + const rawTypes = (): unknown[] => { + const base = [{ String: [] }, { String: [] }]; + if (mode === 'opaque-salted-signup') return [...base, { String: [] }]; + if (scalarCase()) return [...base, { Bool: [] }, { U32: [] }]; + if (integerCase()) return [...base, { U64: [] }]; + if (optionCase()) return [...base, { Sum: { variants: [ + { name: { some: 'some' }, algebraic_type: { Bool: [] } }, + { name: { some: 'none' }, algebraic_type: { Product: { elements: [] } } }, + ] } }]; + if (mode === 'enum-fields') return [...base, { Sum: { variants: [ + { name: { some: 'Customer' }, algebraic_type: { Product: { elements: [] } } }, + { name: { some: 'Staff' }, algebraic_type: { Product: { elements: [] } } }, + ] } }]; + return base; + }; + const evidenceJson = (value: unknown) => JSON.stringify(value, + (_key, entry) => typeof entry === 'bigint' ? { bigint: String(entry) } : entry, 2); + const callBytes = (id: number, user?: string, credential?: string, reducer?: string) => { + const writer = new codec.BinaryWriter(128), args = values(user, credential); + types().forEach((type, index) => codec.AlgebraicType.makeSerializer(type)(writer, args[index])); + const name = reducer ?? (mode === 'renamed' || scalarCase() ? 'register_customer' : 'sign_up'); + return encode(codec.ClientMessage, procedure + ? { tag: 'CallProcedure', value: { procedure: name, requestId: id, flags: 0, args: writer.getBuffer() } } + : { tag: 'CallReducer', value: { reducer: name, requestId: id, flags: 0, args: writer.getBuffer() } }); + }; + const server = createServer((req, res) => { + if (req.url === '/authenticate') { + let body = ''; + req.on('data', data => { body += String(data); }); + req.on('end', () => { + received.push(JSON.parse(body)); + res.setHeader('Content-Type', 'application/json'); res.end('true'); + }); + } else if (req.url?.includes('/schema')) { + res.setHeader('Content-Type', 'application/json'); + const operations = [{ name: mode === 'renamed' || scalarCase() ? 'register_customer' : 'sign_up', + params: { elements: rawTypes().map((raw, index) => ({ name: { some: ['username', 'password', mode === 'opaque-salted-signup' ? 'salt' : integerCase() ? 'nonce' : 'is_admin', 'nonce'][index] }, + algebraic_type: mode === 'unsupported-schema' && index === 1 ? { Array: { U8: [] } } : raw })) } }, + ...(['opaque-hash-signup', 'opaque-delayed-signup'].includes(mode) ? [{ name: 'sign_in', params: { elements: rawTypes().map((raw, index) => ({ + name: { some: ['username', 'password'][index] }, algebraic_type: raw })) } }] : [])]; + res.end(JSON.stringify(procedure || mode === 'reducer-call-to-procedure' + ? { reducers: [], misc_exports: operations.map(operation => ({ Procedure: { ...operation, + return_type: { Product: { elements: [] } } } })) } + : { reducers: operations })); + } else { + res.setHeader('Content-Type', 'text/html'); res.end('Native auth form'); + } + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + const port = (server.address() as { port: number }).port, url = `http://127.0.0.1:${port}`; + const sockets = new wsServer({ server }); + sockets.on('connection', (socket: { on(event: string, fn: (data: Buffer) => void): void; send(data: Buffer): void; close(): void }) => { + connections++; + const send = (value: unknown) => socket.send(Buffer.concat([Buffer.from([0]), encode(codec.ServerMessage, value)])); + send({ tag: 'InitialConnection', value: { identity: { __identity__: 1n }, connectionId: { __connection_id__: BigInt(connections) }, token: 'fixture-token' } }); + socket.on('message', data => { + // Malformed outbound traffic must be stopped by the active native probe. + if (data[0] === 255) { received.push(['malformed']); return; } + const message = codec.ClientMessage.deserialize(new codec.BinaryReader(data)); + if (message.tag !== 'CallReducer' && message.tag !== 'CallProcedure') return; + const reader = new codec.BinaryReader(message.value.args); + received.push(types().map(type => codec.AlgebraicType.makeDeserializer(type)(reader))); + if (mode === 'lost-receipt' || mode === 'opaque-closed-witness' && message.value.requestId === 31) { + socket.close(); return; + } + const args = received.at(-1) as unknown[]; + const opaqueRejected = ['deferred-reject-login', 'deferred-reject-signup'].includes(mode) + || ['opaque-hash-login', 'opaque-delayed-login', 'opaque-completed-reload'].includes(mode) && args[1] !== 'hash-correct' + || ['opaque-hash-signup', 'opaque-delayed-signup', 'opaque-salted-signup'].includes(mode) && args[0] === 'claimant'; + const rejected = mode === 'refused' || opaqueRejected; + send(message.tag === 'CallProcedure' + ? { tag: 'ProcedureResult', value: { requestId: message.value.requestId, + timestamp: { __timestamp_micros_since_unix_epoch__: 1n }, + totalHostExecutionDuration: { __time_duration_micros__: 1n }, + status: rejected ? { tag: 'InternalError', value: 'refused' } : { tag: 'Returned', value: new Uint8Array() } } } + : { tag: 'ReducerResult', value: { requestId: message.value.requestId, + timestamp: { __timestamp_micros_since_unix_epoch__: 1n }, result: rejected + ? { tag: 'Err', value: new Uint8Array() } : { tag: 'OkEmpty' } } }); + if (mode === 'malformed-inbound') socket.send(Buffer.from([0, 255])); + }); + }); + const root = mkdtempSync(join(tmpdir(), 'stack-bench-native-auth-')), leasePath = join(root, 'lease.json'); + const previous = { path: process.env.STACK_BENCH_LEASE, token: process.env.STACK_BENCH_LEASE_TOKEN }; + const browser = await chromium.launch(); + try { + for (mode of ['accepted', 'localhost', 'refused', 'renamed', 'scalar-fields', 'u64-fields', 'optional-some', 'optional-none', 'enum-fields', 'type-changing-fields', 'nonmatching-credentials', 'missing-claim', 'opaque-hash-login', 'opaque-delayed-login', 'opaque-ok-denial', 'opaque-completed-reload', 'opaque-token-competing-call', 'opaque-token-lookalike', 'opaque-token-query', 'opaque-token-body', 'opaque-token-wrong-target', 'opaque-hash-signup', 'opaque-delayed-signup', 'deferred-reject-login', 'deferred-reject-signup', 'opaque-salted-signup', 'opaque-http-signup-profile', 'opaque-ambiguous-witness', 'opaque-unrelated-witness', 'opaque-closed-witness', 'opaque-abandoned-witness', 'reload-after-receipt', 'http-reload', 'lost-receipt', 'reconnected-second', 'second-after-receipt', + 'second-then-submit-error', 'malformed-outbound', 'malformed-inbound', 'wrong-module', 'foreign-target', 'unsupported-protocol', 'unsupported-schema', 'app-proxy', + 'reducer-call-to-procedure', 'procedure:accepted', 'procedure:refused', 'procedure:opaque-hash-login', 'procedure:opaque-hash-signup']) { + procedure = mode.startsWith('procedure:'); + mode = mode.replace(/^procedure:/, ''); + await t.test(`${mode}${procedure ? ' (procedure)' : ''}`, async () => { + received.length = 0; connections = 0; + const lease = createBackendLease({ runId: 'native-auth', backend: 'spacetime', track: 'ecommerce', runIndex: 0, + serverUri: mode === 'foreign-target' ? `http://127.0.0.1:${port + 1}` : url, + module: mode === 'wrong-module' ? 'different' : 'auth', dataDir: join(root, 'data') }); + lease.state = 'active'; writeBackendLease(leasePath, lease); + process.env.STACK_BENCH_LEASE = leasePath; process.env.STACK_BENCH_LEASE_TOKEN = lease.ownershipToken; + const context = await browser.newContext(); + try { + const page = await context.newPage(); await installSpacetimeWriteCapture(page); await page.goto(url); + const socketUrl = (['foreign-target', 'localhost'].includes(mode) ? url.replace('127.0.0.1', 'localhost') : url).replace('http:', 'ws:') + + (mode === 'app-proxy' ? '/proxy' : '') + '/v1/database/auth/subscribe'; + await page.evaluate(async ({ socketUrl, protocol }) => { + const socket = new WebSocket(socketUrl, protocol); socket.binaryType = 'arraybuffer'; + Object.assign(window, { socket, replies: 0 }); + socket.addEventListener('message', () => (window as unknown as { replies: number }).replies++); + await new Promise(resolve => socket.addEventListener('message', resolve, { once: true })); + }, { socketUrl, protocol: mode === 'unsupported-protocol' ? 'unsupported' : 'v3.bsatn.spacetimedb' }); + if (mode === 'deferred-reject-login' || mode === 'deferred-reject-signup') { + const signup = mode === 'deferred-reject-signup'; + await page.evaluate(({ signup, call }) => { + const prefix = signup ? 'signup' : 'signin'; + document.body.innerHTML = `
`; + document.querySelector('form')!.addEventListener('submit', async event => { + event.preventDefault(); + const socket = (window as unknown as { socket: WebSocket }).socket; + const reply = new Promise(resolve => socket.addEventListener('message', resolve, { once: true })); + socket.send(new Uint8Array(call)); await reply; + document.querySelector('#auth-error')!.removeAttribute('hidden'); + }); + }, { signup, call: [...callBytes(31, signup ? 'ordinary' : 'customer')] }); + const actor = { page, loc: (id: string) => page.locator(`#${id}`) }; + const capabilities = { actors: { get: () => actor }, + 'browser-interaction': { defaultWithin: 200, scopedUser: (user: string) => user, + testId: (id: string) => `#${id}`, sleep: (ms: number) => page.waitForTimeout(ms) }, + 'browser-observation': { defaultWithin: 100, expand: (value: string) => value, + testId: (id: string) => `#${id}`, sleep: (ms: number) => page.waitForTimeout(ms) } }; + const user = signup ? 'ordinary' : 'customer'; + const action = signup ? 'signUp' : 'signIn'; + const submitted = await executeAction(ACTION_REGISTRY, action, + { do: action, actor: 'probe', name: user, exact: true, + deferAuthFailureToExpect: true }, { capabilities }); + assert.equal(submitted.status, 'passed', `${mode}: refusal must reach the next assertion`); + const observed = await executeAction(ACTION_REGISTRY, 'expect', + { do: 'expect', actor: 'probe', testid: 'current-user', contains: user, within: 100 }, { capabilities }); + assert.equal(observed.status, 'failed', `${mode}: the explicit session assertion must catch refusal`); + observations.push({ mode, submittedStatus: submitted.status, assertionStatus: observed.status, + nativeCalls: received.length }); + return; + } + if (['opaque-ambiguous-witness', 'opaque-unrelated-witness', 'opaque-closed-witness', 'opaque-abandoned-witness'].includes(mode)) { + const observation = beginSpacetimeAuthObservation(page, 'signin', 'customer'); + const calls = mode === 'opaque-ambiguous-witness' ? [callBytes(31), callBytes(32)] : [callBytes(31)]; + for (const call of calls) await page.evaluate(async ({ bytes, closes }) => { + const socket = (window as unknown as { socket: WebSocket }).socket; + const reply = new Promise(resolve => socket.addEventListener(closes ? 'close' : 'message', resolve, { once: true })); + socket.send(new Uint8Array(bytes)); await reply; + }, { bytes: [...call], closes: mode === 'opaque-closed-witness' }); + observation?.stop(); + if (mode === 'opaque-abandoned-witness') observation?.discard(); + else assert.equal(await observation?.finish(true, mode !== 'opaque-unrelated-witness'), false, + 'an unproved submit must not become an authentication witness'); + if (mode === 'opaque-closed-witness') await page.evaluate(async socketUrl => { + const socket = new WebSocket(socketUrl, 'v3.bsatn.spacetimedb'); + (window as unknown as { socket: WebSocket }).socket = socket; + await new Promise(resolve => socket.addEventListener('message', resolve, { once: true })); + }, socketUrl); + received.length = 0; + } + if (mode === 'opaque-completed-reload') { + for (const [index, credential] of ['hash-correct', 'hash-wrong'].entries()) { + const observation = beginSpacetimeAuthObservation(page, 'signin', 'customer'); + await page.evaluate(async call => { + const socket = (window as unknown as { socket: WebSocket }).socket; + const reply = new Promise(resolve => socket.addEventListener('message', resolve, { once: true })); + socket.send(new Uint8Array(call)); await reply; + }, [...callBytes(index + 31, 'customer', credential)]); + if (index === 0) { + await page.evaluate(async () => { await fetch('/v1/identity/websocket-token', { method: 'POST' }); }); + await page.reload(); + await page.evaluate(async socketUrl => { + const socket = new WebSocket(socketUrl, 'v3.bsatn.spacetimedb'); + (window as unknown as { socket: WebSocket }).socket = socket; + await new Promise(resolve => socket.addEventListener('message', resolve, { once: true })); + }, socketUrl); + } + observation?.stop(); + assert.equal(await observation?.finish(index === 0), true, + 'completed native receipt survives normal navigation before account UI appears'); + } + received.length = 0; + } + if (['opaque-token-competing-call', 'opaque-token-lookalike', 'opaque-token-query', 'opaque-token-body', 'opaque-token-wrong-target'].includes(mode)) { + const observation = beginSpacetimeAuthObservation(page, 'signin', 'customer'); + await page.evaluate(async call => { + const socket = (window as unknown as { socket: WebSocket }).socket; + const reply = new Promise(resolve => socket.addEventListener('message', resolve, { once: true })); + socket.send(new Uint8Array(call)); await reply; + }, [...callBytes(31, 'customer', 'hash-correct')]); + const target = mode === 'opaque-token-competing-call' ? '/v1/database/auth/call/sign_in' + : mode === 'opaque-token-lookalike' ? '/prefix/v1/identity/websocket-token' + : mode === 'opaque-token-query' ? '/v1/identity/websocket-token?extra=1' + : mode === 'opaque-token-wrong-target' ? `http://127.0.0.1:${port + 1}/v1/identity/websocket-token` + : '/v1/identity/websocket-token'; + await page.evaluate(async ({ target, body }) => { + await fetch(target, { method: 'POST', ...(body ? { body: 'probe' } : {}) }).catch(() => {}); + }, { target, body: ['opaque-token-competing-call', 'opaque-token-body'].includes(mode) }); + observation?.stop(); + assert.equal(await observation?.finish(true), false, `${mode}: a competing POST must invalidate the witness`); + observations.push({ mode, rejectedCompetingWrite: true }); + return; + } + if (mode === 'opaque-delayed-login' || mode === 'opaque-ok-denial') { + await page.evaluate(calls => { + document.body.innerHTML = '
'; + let sent = 0; + document.querySelector('form')!.addEventListener('submit', async event => { + event.preventDefault(); + await new Promise(resolve => setTimeout(resolve, 250)); + const socket = (window as unknown as { socket: WebSocket }).socket; + const reply = new Promise(resolve => socket.addEventListener('message', resolve, { once: true })); + socket.send(new Uint8Array(calls[sent++]!)); await reply; + if (sent === 1) { const current = document.querySelector('#current-user')!; current.textContent = 'customer'; current.removeAttribute('hidden'); } + else document.querySelector('#auth-error')!.removeAttribute('hidden'); + }); + }, [[...callBytes(31, 'customer', 'hash-correct')], [...callBytes(32, 'customer', 'hash-wrong')]]); + const actor = { page, loc: (id: string) => page.locator(`#${id}`) }; + const capabilities = { actors: { get: () => actor }, 'browser-interaction': { + defaultWithin: 1500, scopedUser: (user: string) => user, testId: (id: string) => `#${id}`, + sleep: (ms: number) => new Promise(resolve => setTimeout(resolve, ms)), + } }; + const good = await executeAction(ACTION_REGISTRY, 'signIn', + { do: 'signIn', actor: 'probe', name: 'customer', exact: true, password: 'correct', + deferAuthFailureToExpect: true }, { capabilities }); + assert.equal(good.status, 'passed', JSON.stringify(good)); + await page.evaluate(() => document.querySelector('#current-user')!.setAttribute('hidden', '')); + const bad = await executeAction(ACTION_REGISTRY, 'signIn', + { do: 'signIn', actor: 'probe', name: 'customer', exact: true, password: 'wrong', expectFailure: true }, { capabilities }); + assert.equal(bad.status, 'passed', JSON.stringify(bad)); + received.length = 0; + } else if (mode === 'opaque-hash-login' || mode === 'opaque-hash-signup') { + const baselineCalls = mode === 'opaque-hash-login' + ? [{ bytes: callBytes(31, 'customer', 'hash-correct'), kind: 'signin' as const, accepted: true }, + { bytes: callBytes(32, 'customer', 'hash-wrong'), kind: 'signin' as const, accepted: false }] + : [{ bytes: callBytes(31, 'ordinary', 'hash-ordinary'), kind: 'signup' as const, accepted: true }, + { bytes: callBytes(32, 'ordinary', 'hash-ordinary', 'sign_in'), kind: 'signin' as const, accepted: true }]; + for (const { bytes, kind, accepted } of baselineCalls) { + const observation = beginSpacetimeAuthObservation(page, kind, mode === 'opaque-hash-login' ? 'customer' : 'ordinary'); + await page.evaluate(async call => { + const state = window as unknown as { socket: WebSocket }; + const reply = new Promise(resolve => state.socket.addEventListener('message', resolve, { once: true })); + state.socket.send(new Uint8Array(call)); await reply; + }, [...bytes]); + observation?.stop(); + assert.equal(await observation?.finish(accepted), true, `ordinary form and native result must agree: ${evidenceJson(received)}`); + if (kind === 'signin' && accepted) confirmSpacetimeSignup(page, 'ordinary'); + } + received.length = 0; + } + if (mode === 'opaque-delayed-signup') { + await page.evaluate(calls => { + document.body.innerHTML = '
'; + let sent = 0; + for (const form of document.querySelectorAll('form')) form.addEventListener('submit', async event => { + event.preventDefault(); + await new Promise(resolve => setTimeout(resolve, 250)); + const socket = (window as unknown as { socket: WebSocket }).socket; + const reply = new Promise(resolve => socket.addEventListener('message', resolve, { once: true })); + socket.send(new Uint8Array(calls[sent++]!)); await reply; + const current = document.querySelector('#current-user')!; + current.textContent = 'ordinary'; current.removeAttribute('hidden'); + }); + }, [[...callBytes(31, 'ordinary', 'hash-ordinary')], + [...callBytes(32, 'ordinary', 'hash-ordinary', 'sign_in')]]); + const actor = { page, loc: (id: string) => page.locator(`#${id}`) }; + const capabilities = { actors: { get: () => actor }, 'browser-interaction': { + defaultWithin: 1500, scopedUser: (user: string) => user, testId: (id: string) => `#${id}`, + sleep: (ms: number) => new Promise(resolve => setTimeout(resolve, ms)), + } }; + const signup = await executeAction(ACTION_REGISTRY, 'signUp', + { do: 'signUp', actor: 'probe', name: 'ordinary', exact: true, password: 'ordinary-password', + deferAuthFailureToExpect: true }, { capabilities }); + assert.equal(signup.status, 'passed', JSON.stringify(signup)); + await page.evaluate(() => document.querySelector('#current-user')!.setAttribute('hidden', '')); + const signin = await executeAction(ACTION_REGISTRY, 'signIn', + { do: 'signIn', actor: 'probe', name: 'ordinary', exact: true, password: 'ordinary-password' }, { capabilities }); + assert.equal(signin.status, 'passed', JSON.stringify(signin)); + received.length = 0; + } + if (mode === 'opaque-salted-signup') { + const observation = beginSpacetimeAuthObservation(page, 'signup', 'ordinary'); + await page.evaluate(async call => { + const socket = (window as unknown as { socket: WebSocket }).socket; + const reply = new Promise(resolve => socket.addEventListener('message', resolve, { once: true })); + socket.send(new Uint8Array(call)); await reply; + }, [...callBytes(31, 'ordinary', 'hash-ordinary')]); + observation?.stop(); + assert.equal(await observation?.finish(true), true); + await page.evaluate(() => { + document.body.innerHTML = '
'; + document.querySelector('form')!.addEventListener('submit', async event => { + event.preventDefault(); + await fetch('/authenticate', { method: 'POST', headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ username: 'ordinary', password: 'ordinary-password' }) }); + const current = document.querySelector('#current-user')!; + current.textContent = 'ordinary'; current.removeAttribute('hidden'); + }); + }); + const actor = { page, loc: (id: string) => page.locator(`#${id}`) }; + const signin = await executeAction(ACTION_REGISTRY, 'signIn', + { do: 'signIn', actor: 'probe', name: 'ordinary', exact: true, password: 'ordinary-password' }, { + capabilities: { actors: { get: () => actor }, 'browser-interaction': { + defaultWithin: 1500, scopedUser: (user: string) => user, testId: (id: string) => `#${id}`, + } }, + }); + assert.equal(signin.status, 'passed', JSON.stringify(signin)); + received.length = 0; + } + if (mode === 'opaque-http-signup-profile') { + const observation = beginSpacetimeAuthObservation(page, 'signup', 'ordinary'); + await page.evaluate(async call => { + await fetch('/authenticate', { method: 'POST', headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ username: 'ordinary', password: 'hash-ordinary' }) }); + const socket = (window as unknown as { socket: WebSocket }).socket; + const reply = new Promise(resolve => socket.addEventListener('message', resolve, { once: true })); + socket.send(new Uint8Array(call)); await reply; + }, [...callBytes(31, 'ordinary', 'hash-ordinary')]); + observation?.stop(); + assert.equal(await observation?.finish(true), false, + 'a native profile write cannot witness a competing HTTP signup'); + confirmSpacetimeSignup(page, 'ordinary'); + received.length = 0; + } + const patch = mode === 'type-changing-fields' ? { fields: { nonce: 'wrong-type' } } : scalarCase() ? { fields: { isAdmin: true } } : mode === 'missing-claim' ? { fields: { role: 'admin' } } : { password: 'modified' }; + let outcome: unknown; + try { + outcome = await withAuthRequestPatch(page, ['opaque-hash-signup', 'opaque-delayed-signup', 'opaque-salted-signup'].includes(mode) ? 'claimant' : 'customer', + ['opaque-hash-signup', 'opaque-delayed-signup', 'opaque-salted-signup'].includes(mode) ? 'pw-claimant' : ['opaque-hash-login', 'opaque-delayed-login', 'opaque-ok-denial'].includes(mode) ? 'not-the-password' : 'secret', + ['opaque-hash-signup', 'opaque-delayed-signup', 'opaque-salted-signup'].includes(mode) ? { fields: { role: 'admin' } } : patch, async () => { + await page.evaluate(async ({ first, second, mode }) => { + const state = window as unknown as { socket: WebSocket; replies: number }; + if (mode === 'http-reload') { + await fetch('/authenticate', { method: 'POST', headers: { 'content-type': 'application/json' }, + body: JSON.stringify(['customer', 'secret']) }).then(response => response.json()); + } else { + const reply = new Promise(resolve => state.socket.addEventListener(mode === 'lost-receipt' ? 'close' : 'message', resolve, { once: true })); + state.socket.send(new Uint8Array(first)); + if (mode === 'type-changing-fields') await Promise.race([reply, new Promise(resolve => setTimeout(resolve, 500))]); + else await reply; + } + if (mode.startsWith('second-')) state.socket.send(new Uint8Array(second)); + if (mode === 'malformed-outbound') state.socket.send(new Uint8Array([255])); + }, { first: [...callBytes(1, ['opaque-hash-signup', 'opaque-delayed-signup', 'opaque-salted-signup'].includes(mode) ? 'claimant' : undefined)], + second: [...callBytes(2, ['opaque-hash-signup', 'opaque-delayed-signup', 'opaque-salted-signup'].includes(mode) ? 'claimant' : undefined)], mode }); + if (['reload-after-receipt', 'http-reload', 'reconnected-second'].includes(mode)) await page.reload(); + if (mode === 'reconnected-second') await page.evaluate(async ({ socketUrl, second }) => { + const socket = new WebSocket(socketUrl, 'v3.bsatn.spacetimedb'); + Object.assign(window, { socket }); + await new Promise(resolve => socket.addEventListener('message', resolve, { once: true })); + socket.send(new Uint8Array(second)); + }, { socketUrl, second: [...callBytes(2)] }); + // Let native socket events settle while the submission remains active. + await page.waitForTimeout(100); + if (mode === 'second-then-submit-error') throw new Error('Submission did not finish'); + return { submitted: true }; + }, undefined, ['opaque-hash-login', 'opaque-delayed-login', 'opaque-ok-denial', 'opaque-completed-reload', + 'opaque-ambiguous-witness', 'opaque-unrelated-witness', 'opaque-closed-witness', 'opaque-abandoned-witness'].includes(mode) ? 'signin' + : ['opaque-hash-signup', 'opaque-delayed-signup', 'opaque-salted-signup', 'opaque-http-signup-profile'].includes(mode) ? 'signup' : undefined); + } catch (error) { outcome = error; } + const measured = ['accepted', 'localhost', 'refused', 'renamed', 'scalar-fields', 'u64-fields', 'optional-some', 'optional-none', 'enum-fields', 'missing-claim', 'opaque-hash-login', 'opaque-delayed-login', 'opaque-ok-denial', 'opaque-completed-reload', 'opaque-hash-signup', 'opaque-delayed-signup', 'opaque-salted-signup', 'reload-after-receipt', 'http-reload', 'app-proxy'].includes(mode); + observations.push({ mode, measured, connections, received: structuredClone(received), + outcome: outcome instanceof Error ? { name: outcome.name, message: outcome.message } : outcome }); + if (!measured) assert(outcome instanceof ActionInconclusive, `${mode}: ${evidenceJson(observations.at(-1))}`); + else { + assert(!(outcome instanceof Error), `${mode}: ${outcome instanceof Error ? outcome.stack : String(outcome)}`); + const receipt = (outcome as { requestPatch: { transport?: string; success?: boolean; status?: number; absentParameters?: string[] } }).requestPatch; + if (mode === 'http-reload') assert.equal(receipt.status, 200); + else { assert.equal(receipt.transport, 'spacetime-websocket'); assert.equal(receipt.success, + !['refused', 'opaque-hash-login', 'opaque-delayed-login', 'opaque-completed-reload', 'opaque-hash-signup', 'opaque-delayed-signup', 'opaque-salted-signup'].includes(mode)); } + const expected = values(); + if (['opaque-hash-signup', 'opaque-delayed-signup', 'opaque-salted-signup'].includes(mode)) expected[0] = 'claimant'; + if (mode === 'opaque-salted-signup') expected[2] = 'salt-claimant'; + if (scalarCase()) expected[2] = true; + else if (mode !== 'missing-claim' && !['opaque-hash-signup', 'opaque-delayed-signup', 'opaque-salted-signup'].includes(mode)) expected[1] = 'modified'; + assert.deepEqual(received, [expected], 'only the intended credential or claim changes'); + if (mode === 'missing-claim') assert.deepEqual(receipt.absentParameters, ['role']); + } + assert.equal(connections, ['reconnected-second', 'opaque-closed-witness', 'opaque-completed-reload'].includes(mode) ? 2 : 1, + 'only the app can create connections'); + if (mode.startsWith('second-') || mode === 'reconnected-second' || mode === 'malformed-outbound') assert.equal(received.length, 1, 'ambiguous traffic must not be forwarded'); + } finally { await context.close(); } + }); + } + } finally { + await browser.close(); sockets.close(); server.closeAllConnections(); await new Promise(resolve => server.close(() => resolve())); + if (previous.path === undefined) delete process.env.STACK_BENCH_LEASE; else process.env.STACK_BENCH_LEASE = previous.path; + if (previous.token === undefined) delete process.env.STACK_BENCH_LEASE_TOKEN; else process.env.STACK_BENCH_LEASE_TOKEN = previous.token; + rmSync(root, { recursive: true, force: true }); + if (process.env.STACK_BENCH_REPLAY_EVIDENCE) { + mkdirSync(process.env.STACK_BENCH_REPLAY_EVIDENCE, { recursive: true }); + writeFileSync(join(process.env.STACK_BENCH_REPLAY_EVIDENCE, 'spacetime-auth-receipt.json'), evidenceJson({ observations, + rerun: 'node --test --test-name-pattern="SpacetimeDB auth receipts" dist/tests/spacetime-captured-replay.integration.js' })); + } + } +}); diff --git a/tools/stack-bench/tests/spacetime-guidance.contract.ts b/tools/stack-bench/tests/spacetime-guidance.contract.ts new file mode 100644 index 00000000000..048e440e149 --- /dev/null +++ b/tools/stack-bench/tests/spacetime-guidance.contract.ts @@ -0,0 +1,17 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import test from 'node:test'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; + +test('SpacetimeDB guidance uses the repeatable local authentication path', () => { + const guidance = readFileSync(join(STACK_BENCH_ROOT, 'backends', 'spacetime.md'), 'utf8'); + const commands = guidance.split(/\r?\n/).filter(line => line.includes('')); + const stateful = commands.filter(line => /\b(?:publish|dev)\b/.test(line)); + + assert.equal(stateful.length, 2, 'expected one publish and one development command'); + for (const command of stateful) assert.match(command, /--yes\b/, command); + assert.doesNotMatch(guidance, /echo\s+y\s*\|/i); + assert.doesNotMatch(guidance, /--delete-data\b/); + assert.doesNotMatch(guidance, /\s+(?:publish|dev)[^\n]*--anonymous/); +}); diff --git a/tools/stack-bench/tests/spacetime-reducer-arguments.test.ts b/tools/stack-bench/tests/spacetime-reducer-arguments.test.ts new file mode 100644 index 00000000000..fdc602861a7 --- /dev/null +++ b/tools/stack-bench/tests/spacetime-reducer-arguments.test.ts @@ -0,0 +1,48 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { spacetimeNamedActionFetch, withOptionalReducerArguments } + from '../src/stacks/backends/spacetime-reducer-arguments.js'; + +const target = { uri: 'http://127.0.0.1:3000', mod: 'shop' }; +const schema = { typespace: { types: [{ Array: { String: [] } }] }, reducers: [ + { name: 'checkout', params: { elements: [{ algebraic_type: { String: [] } }, { algebraic_type: { Bool: [] } }, + { algebraic_type: { Sum: { variants: [{ name: { some: 'some' }, algebraic_type: { String: [] } }, + { name: { some: 'none' }, algebraic_type: { Product: { elements: [] } } }] } } }, + { algebraic_type: { Ref: 0 } }] } }, + { name: 'checkout_count', params: { elements: [{ algebraic_type: { U32: [] } }] } }, + { name: 'plain', params: { elements: [] } }, +] }; + +function stub() { + const calls: { url: string; body?: string | null }[] = []; + const fetchImpl = async (url: string, options: { body?: string | null }) => { + calls.push({ url, body: options.body }); + return { ok: true, status: 200, text: async () => JSON.stringify(schema) }; + }; + return { calls, fetchImpl }; +} + +test('a reducer called without arguments receives the blank value of each declared parameter', async () => { + const { calls, fetchImpl } = stub(); + assert.equal(await withOptionalReducerArguments(target, 'checkout', '[]', fetchImpl), + JSON.stringify(['', false, { none: [] }, []])); + assert.equal(calls[0]!.url, 'http://127.0.0.1:3000/v1/database/shop/schema?version=9'); + // No blank value exists for a number, and a no-parameter reducer needs none. + assert.equal(await withOptionalReducerArguments(target, 'checkout_count', '[]', fetchImpl), '[]'); + assert.equal(await withOptionalReducerArguments(target, 'plain', '[]', fetchImpl), '[]'); + // Given arguments are the interface's own; they are never looked up or changed. + const before = calls.length; + assert.equal(await withOptionalReducerArguments(target, 'checkout', '["Ada",true]', fetchImpl), '["Ada",true]'); + assert.equal(calls.length, before); +}); + +test('only reducer calls on the leased database are completed', async () => { + const { calls, fetchImpl } = stub(); + const send = spacetimeNamedActionFetch(fetchImpl, target); + await send('http://127.0.0.1:3000/v1/database/shop/call/checkout', { method: 'POST', body: '[]' }); + assert.equal(calls.at(-1)!.body, JSON.stringify(['', false, { none: [] }, []])); + await send('http://127.0.0.1:3000/v1/database/other/call/checkout', { method: 'POST', body: '[]' }); + assert.equal(calls.at(-1)!.body, '[]'); + assert.equal(spacetimeNamedActionFetch(fetchImpl, null), fetchImpl); +}); diff --git a/tools/stack-bench/tests/spacetime-saved-checkout.test.ts b/tools/stack-bench/tests/spacetime-saved-checkout.test.ts new file mode 100644 index 00000000000..a7c409387a7 --- /dev/null +++ b/tools/stack-bench/tests/spacetime-saved-checkout.test.ts @@ -0,0 +1,89 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { hashAppSource } from '../src/runtime/source-snapshot.js'; +import type { TextCommandExecutor } from '../src/runtime/command-executor.js'; +import { getSavedSpacetimeCheckoutState } from '../src/stacks/backends/spacetime-saved-checkout.js'; +import { orderCheckoutDifferences } from '../src/stacks/checkout-state.js'; + +function fixture() { + const root = mkdtempSync(join(tmpdir(), 'saved-spacetime-')), app = join(root, 'app'); + mkdirSync(app); writeFileSync(join(app, 'module.ts'), 'accepted'); + const state = { accountId: '1', itemId: '2', priceMinor: 8900, cart: [], + stock: [{ warehouseId: '3', quantity: 4 }], orders: [], payments: [], reservations: [], + orphanOrderLines: 0, orphanAllocations: 0 }; + const mapping = { sourceSha256: hashAppSource(app).sha256, tables: ['account', 'item', 'stock'], + convert: `(tables, account, item) => ({accountMatches: tables.account.length, itemMatches: tables.item.length, state: ${JSON.stringify(state)}})` }; + const reader = { path: join(root, 'reader.json'), sha256: '' }; + const save = () => { const bytes = JSON.stringify(mapping); writeFileSync(reader.path, bytes); reader.sha256 = createHash('sha256').update(bytes).digest('hex'); }; + save(); + const snapshot = { account: { inserts: [{ id: 1 }], deletes: [] }, item: { inserts: [{ id: 2 }], deletes: [] } }; + return { root, mapping, save, state, snapshot, args: { app, reader, account: 'buyer', item: 'Keyboard', + spacetime: { buildContainer: { id: 'owned-id', name: 'owned-name' }, mod: 'owned-module', containerUri: 'http://127.0.0.1:3211' } } }; +} + +test('saved SpacetimeDB reader collects all tables in one bounded initial native subscription', t => { + const f = fixture(); t.after(() => rmSync(f.root, { recursive: true, force: true })); + let reads = 0; + const exec: TextCommandExecutor = (file, args, options) => { + assert.equal(file, 'docker'); + if (args[0] === 'inspect') { assert.equal(args.at(-1), 'owned-name'); return 'owned-id'; } + reads++; assert(args.includes('owned-id')); assert(args.includes('subscribe')); assert(args.includes('owned-module')); + assert(args.includes('--print-initial-update')); assert(args.includes('--num-updates')); assert(args.includes('0')); + assert.deepEqual(args.slice(-3), ['SELECT * FROM account', 'SELECT * FROM item', 'SELECT * FROM stock']); + assert.equal(options.timeout, 60_000); return JSON.stringify(f.snapshot); + }; + const result = getSavedSpacetimeCheckoutState({ ...f.args, exec }); + assert.equal(reads, 1); assert.deepEqual(result.state, f.state); assert.equal(result.scope, 'orders'); + assert.deepEqual(result.schemaSha256, { source: f.mapping.sourceSha256, reader: f.args.reader.sha256 }); +}); + +test('saved snapshot source, reader and owned container bindings fail closed', t => { + const f = fixture(); t.after(() => rmSync(f.root, { recursive: true, force: true })); + const noExec: TextCommandExecutor = () => { throw new Error('must not execute'); }; + assert.throws(() => getSavedSpacetimeCheckoutState({ ...f.args, reader: { ...f.args.reader, sha256: '0'.repeat(64) }, exec: noExec }), /reader hash mismatch/); + writeFileSync(join(f.args.app, 'module.ts'), 'changed'); + assert.throws(() => getSavedSpacetimeCheckoutState({ ...f.args, exec: noExec }), /source hash mismatch/); + writeFileSync(join(f.args.app, 'module.ts'), 'accepted'); + assert.throws(() => getSavedSpacetimeCheckoutState({ ...f.args, exec: () => 'replacement' }), /changed after lease creation/); +}); + +test('saved reader accepts reconstructed current stock but exposes lost checkout effects', t => { + const f = fixture(); t.after(() => rmSync(f.root, { recursive: true, force: true })); + f.mapping.tables.push('orders'); + f.mapping.convert = `(tables) => ({accountMatches: tables.account.length, itemMatches: tables.item.length, + state: {...${JSON.stringify(f.state)}, stock: tables.stock, orders: tables.orders}})`; + f.save(); + const before = structuredClone(f.state), prepared = { ...structuredClone(before), cart: [{ itemId: '2', quantity: 1 }] }; + const order = { id: 'order', accountId: '1', totalMinor: 8900, refundedMinor: 0, status: 'pending', + lines: [{ itemId: '2', quantity: 1, priceMinor: 8900, allocations: [{ warehouseId: '3', quantity: 1 }] }] }; + // These are the native subscription results after connection hooks finish. + // Restoring the current quantity is valid; restoring the original quantity + // would erase the acknowledged purchase's stock effect. + const read = (stock: typeof before.stock, orders: typeof order[]) => getSavedSpacetimeCheckoutState({ ...f.args, + exec: (_file, args) => args[0] === 'inspect' ? 'owned-id' : JSON.stringify({ ...f.snapshot, + stock: { inserts: stock, deletes: [] }, orders: { inserts: orders, deletes: [] } }) }).state; + const currentStock = [{ warehouseId: '3', quantity: 3 }]; + assert.deepEqual(orderCheckoutDifferences(before, prepared, read(currentStock, [order]), 1), []); + for (const state of [read(before.stock, [order]), read([], [order]), read(currentStock, []), + read(currentStock, [{ ...order, lines: [] }])]) { + assert(orderCheckoutDifferences(before, prepared, state, 1).length > 0); + } +}); + +test('saved snapshot rejects malformed tables, missing entities, invalid canonical state and a stuck converter', t => { + const f = fixture(); t.after(() => rmSync(f.root, { recursive: true, force: true })); + const read = (value: unknown) => getSavedSpacetimeCheckoutState({ ...f.args, + exec: (_file, args) => args[0] === 'inspect' ? 'owned-id' : JSON.stringify(value) }); + for (const value of [{ ...f.snapshot, alien: { inserts: [], deletes: [] } }, + { ...f.snapshot, account: { inserts: [], deletes: [] } }, + { ...f.snapshot, item: { inserts: [{ id: 1 }, { id: 2 }], deletes: [] } }, + { ...f.snapshot, stock: { inserts: [], deletes: [{}] } }]) assert.throws(() => read(value)); + f.mapping.convert = '() => ({accountMatches:1,itemMatches:1,state:{}})'; f.save(); + assert.throws(() => read(f.snapshot)); + f.mapping.convert = '() => { while (true) {} }'; f.save(); + assert.throws(() => read(f.snapshot), /timed out/); +}); diff --git a/tools/stack-bench/tests/stack-adapters.test.ts b/tools/stack-bench/tests/stack-adapters.test.ts new file mode 100644 index 00000000000..19fca3b5915 --- /dev/null +++ b/tools/stack-bench/tests/stack-adapters.test.ts @@ -0,0 +1,155 @@ +import assert from 'node:assert/strict'; +import { resolve } from 'node:path'; +import test from 'node:test'; + +import { createStackAdapterRegistry } from '../src/stacks/stack-adapter-contract.js'; +import { STACK_ADAPTER_REGISTRY } from '../src/stacks/stack-adapters.js'; +import { setSpacetimeStock } from '../src/stacks/backends/spacetime-operations.js'; +import { describesMissingStockInterface } from '../src/stacks/stock-interface.js'; + +test('build plans expose only artifacts owned by the selected stack', () => { + const appDir = resolve('bench', 'run', 'app'); + const repo = resolve('repo'); + const spacetime = STACK_ADAPTER_REGISTRY.get('spacetime').buildContainer.plan({ repo, appDir }); + assert.equal(spacetime.mounts.some(mount => mount.target === '/deps/.spacetimedb-cli'), true); + assert.equal(spacetime.readyFile, '/deps/.ready'); + + const appliance = STACK_ADAPTER_REGISTRY.get('spacetime').buildContainer.plan({ + repo, + appDir, + env: { STACK_BENCH_RELEASE_DEPS_VOLUME: 'stack-bench-release-deps', + STACK_BENCH_APPLIANCE: '1' }, + }); + assert.deepEqual(appliance.requiredPaths, []); + assert.deepEqual(appliance.mounts, [ + { kind: 'volume', source: 'stack-bench-release-deps', target: '/release-deps', readOnly: true }, + ]); + assert.doesNotMatch(appliance.init, /npm install|npm pack/); + + for (const adapter of [STACK_ADAPTER_REGISTRY.get('postgres'), + STACK_ADAPTER_REGISTRY.get('mongodb'), STACK_ADAPTER_REGISTRY.get('stub')]) { + const plan = adapter.buildContainer.plan({ env: {} }); + assert.deepEqual(plan.mounts, [], `${adapter.id} must not receive another stack's artifacts`); + assert.deepEqual(plan.requiredPaths, []); + assert.equal(plan.readyFile, null); + } +}); + +test('a stock write that finds no table, column, or row is the application missing its interface', () => { + for (const detail of [ + 'Error: `stock` does not have a field `quantity`', + 'Error: no such table: stock', + 'table stock is marked private', + 'ERROR: column "quantity" of relation "stock" does not exist', + 'ERROR: relation "stock" does not exist', + 'WARNING: This command is UNSTABLE.\n\nError: `id` is not in scope\n\nCaused by:\n HTTP status client error (400 Bad Request)', + 'Error: `item_id` is not in scope\r\n', + 'Error: `warehouse_id` is not in scope', + 'Table stock not found', + 'relation "stock" does not exist', + 'no such column: quantity', + 'field item_id not found', + 'Error: `stock` is a view; DML on views is not supported', + ]) assert.ok(describesMissingStockInterface(detail), detail); + for (const detail of [ + 'connection refused', 'ETIMEDOUT', 'HTTP status server error (500 Internal Server Error)', + 'HTTP status client error (400 Bad Request)', + 'Error: `unrelated_field` is not in scope', + 'Error: syntax error near `id`', + 'transport failed while running query: `id` is not in scope', + 'OCI runtime exec failed: executable file not found in $PATH', + 'FATAL: role "appuser" does not exist', + 'FATAL: database "bench" does not exist', + 'Error: `catalog` is a view; DML on views is not supported', + ]) assert.equal(describesMissingStockInterface(detail), false, detail); + + const exec = (_command: string, args: readonly string[]): string => { + if (args[0] === 'inspect') return 'leased-build-id'; + const sql = args.at(-1) ?? ''; + if (/select id from item/.test(sql)) return 'id\n---\n1\n'; + if (/select id from warehouse/.test(sql)) return 'id\n---\n2\n'; + throw Object.assign(new Error('exit 1'), { status: 1, + stderr: 'Error: `stock` does not have a field `quantity`\n' }); + }; + assert.throws(() => setSpacetimeStock({ item: 'widget', warehouse: 'east', quantity: 3, + spacetime: { buildContainer: { name: 'leased-build', id: 'leased-build-id' }, mod: 'shop', + containerUri: 'http://host.docker.internal:3000' }, exec }), + (error: unknown) => error instanceof Error && 'stockInterface' in error + && error.stockInterface === true && /does not have a field/.test(error.message)); + + assert.throws(() => setSpacetimeStock({ item: 'widget', warehouse: 'east', quantity: 3, + spacetime: { buildContainer: { name: 'leased-build', id: 'leased-build-id' }, mod: 'shop', + containerUri: 'http://host.docker.internal:3000' }, + exec: (_command, args) => args[0] === 'inspect' ? 'leased-build-id' + : /select id from item/.test(args.at(-1) ?? '') ? 'id\n---\n1\n' : 'id\n---\n' }), + (error: unknown) => error instanceof Error && 'missingRow' in error && error.missingRow === 'warehouse'); + + for (const [stderr, expectedInterface] of [ + ['Error: `id` is not in scope\n', true], + ['Error: `unrelated_field` is not in scope\n', false], + ['connection refused', false], + ] as const) { + assert.throws(() => setSpacetimeStock({ item: 'widget', warehouse: 'east', quantity: 3, + spacetime: { buildContainer: { name: 'leased-build', id: 'leased-build-id' }, mod: 'shop', + containerUri: 'http://host.docker.internal:3000' }, + exec: (_command, args) => { + if (args[0] === 'inspect') return 'leased-build-id'; + throw Object.assign(new Error('exit 1'), { stderr }); + } }), + (error: unknown) => error instanceof Error + && ('stockInterface' in error && error.stockInterface === true) === expectedInterface); + } +}); + +test('Spacetime stock writes separate missing rows from failed writes and malformed evidence', () => { + for (const [rows, expectedInterface] of [ + ['warehouse_id | quantity\n---+---\n', true], + ['invalid output', false], + ['warehouse_id | quantity\n---+---\n2 | 100\n', false], + ] as const) { + let writes = 0; + assert.throws(() => setSpacetimeStock({ item: 'widget', warehouse: 'east', quantity: 3, + spacetime: { buildContainer: { name: 'leased-build', id: 'leased-build-id' }, mod: 'shop', + containerUri: 'http://localhost:3000' }, exec: (_command, args) => { + const sql = args.at(-1) ?? ''; + if (args[0] === 'inspect') return 'leased-build-id'; + if (/select id from item/.test(sql)) return 'id\n---\n1\n'; + if (/select id from warehouse/.test(sql)) return 'id\n---\n2\n'; + if (/select warehouse_id, quantity/.test(sql)) return rows; + if (/^update stock/.test(sql)) { writes++; return ''; } + return 'quantity\n---\n100\n'; + } }), error => error instanceof Error + && ('stockInterface' in error && error.stockInterface === true) === expectedInterface); + assert.equal(writes, rows.includes('2 | 100') ? 1 : 0); + } + + const calls: Array<[string, readonly string[]]> = []; + setSpacetimeStock({ item: 'widget', warehouse: 'east', quantity: 3, + spacetime: { buildContainer: { name: 'leased-build', id: 'leased-build-id' }, mod: 'shop', + containerUri: 'http://host.docker.internal:3000' }, exec: (command, args) => { + calls.push([command, args]); + if (args[0] === 'inspect') return 'leased-build-id'; + const sql = args.at(-1) ?? ''; + if (/select id from item/.test(sql)) return 'id\n---\n1\n'; + if (/select id from warehouse/.test(sql)) return 'id\n---\n2\n'; + if (/select warehouse_id, quantity/.test(sql)) return 'warehouse_id | quantity\n---+---\n2 | 3\n'; + if (/select quantity/.test(sql)) return '3\n'; + return ''; + } }); + assert(calls.some(([, args]) => /^update stock/.test(args.at(-1) ?? ''))); + for (const [command, args] of calls.filter(([, args]) => args[0] !== 'inspect')) { + assert.equal(command, 'docker'); + assert.deepEqual(args.slice(0, 8), ['exec', '--user', '10001:10001', '-e', + 'HOME=/home/developer', '-e', 'USER=developer', 'leased-build-id']); + } +}); + +test('registry rejects unknown, duplicate, and invalid adapter identities', () => { + assert.throws(() => STACK_ADAPTER_REGISTRY.get('new-db'), /unknown stack adapter/); + const identity = { id: 'fake', version: '1.0.0', + lifecycle: { activate: () => undefined } } as const; + assert.equal(createStackAdapterRegistry([identity]).get('fake').version, '1.0.0'); + assert.throws(() => createStackAdapterRegistry([identity, identity]), /duplicate stack adapter/); + assert.throws(() => createStackAdapterRegistry([{ ...identity, version: 'latest' }]), + /version is invalid/); +}); diff --git a/tools/stack-bench/tests/stack-grading-operations.test.ts b/tools/stack-bench/tests/stack-grading-operations.test.ts new file mode 100644 index 00000000000..93504bc0813 --- /dev/null +++ b/tools/stack-bench/tests/stack-grading-operations.test.ts @@ -0,0 +1,77 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { + httpNamedActionRequest, + spacetimeNamedActionRequest, +} from '../src/stacks/stack-grading-operations.js'; + +test('SpacetimeDB requests preserve full unsigned 64-bit values', () => { + const request = spacetimeNamedActionRequest({ + action: { + path: '', + reducer: 'set_stock', + params: [{ name: 'quantity', wireType: 'u64' }], + }, + input: { values: { quantity: '18446744073709551615' } }, + spacetime: { uri: 'http://127.0.0.1:3000', mod: 'shop' }, + }); + + assert.equal(request.url, 'http://127.0.0.1:3000/v1/database/shop/call/set_stock'); + assert.equal(request.body, '[18446744073709551615]'); + for (const quantity of ['18446744073709551616', '-1']) { + assert.throws( + () => spacetimeNamedActionRequest({ + action: { + path: '', + reducer: 'set_stock', + params: [{ name: 'quantity', wireType: 'u64' }], + }, + input: { values: { quantity } }, + spacetime: { uri: 'http://127.0.0.1:3000', mod: 'shop' }, + }), + (error: unknown) => { + assert(error instanceof Error); + assert.equal((error as Error & { code?: string }).code, 'invalid_named_action_input', quantity); + return true; + }, + ); + } +}); + +test('HTTP requests substitute path values and keep other values in the body', () => { + const request = httpNamedActionRequest({ + action: { + path: '/items/:item', + method: 'PATCH', + params: [ + { name: 'item', in: 'path', placeholder: ':item' }, + { name: 'quantity' }, + ], + }, + input: { values: { item: 'camera/one', quantity: 3 } }, + url: 'http://127.0.0.1:4000/', + }); + + assert.deepEqual(request, { + url: 'http://127.0.0.1:4000/items/camera%2Fone', + method: 'PATCH', + body: '{"quantity":3}', + missingNote: 'no route at /items/:item', + }); + + const replay = httpNamedActionRequest({ + action: { + path: '/items/:item/reviews', + params: [ + { name: 'item', in: 'path', placeholder: ':item' }, + { name: 'rating' }, + { name: 'comment' }, + ], + }, + input: { args: [42, 5, 'verified purchase'] }, + url: 'http://127.0.0.1:4000', + }); + assert.equal(replay.url, 'http://127.0.0.1:4000/items/42/reviews'); + assert.deepEqual(JSON.parse(replay.body), { rating: 5, comment: 'verified purchase' }); +}); diff --git a/tools/stack-bench/tests/stack-parity.contract.ts b/tools/stack-bench/tests/stack-parity.contract.ts new file mode 100644 index 00000000000..192457a3c79 --- /dev/null +++ b/tools/stack-bench/tests/stack-parity.contract.ts @@ -0,0 +1,56 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { compileCampaignFile } from '../src/campaigns/campaign-compiler.js'; +import type { CompiledCampaignPlan } from '../src/campaigns/campaign-compiler.js'; + +const reference = join(STACK_BENCH_ROOT, 'appliance', 'campaign.ecommerce-progression-reference.json'); + +// Every level's scored checks, with their points, in a stable order. +function gradedChecks(plan: CompiledCampaignPlan): string[] { + const attempt = plan.attempts[0]; + assert.ok(attempt); + return attempt.condition.requested.levels.map(level => `L${level.level} ` + + (level.selection.scoredChecks ?? []).map(check => `${check.stableKey}=${check.points}`).sort().join(' ')); +} + +// Points available per questline: the denominators of the questline average. +function questlinePoints(plan: CompiledCampaignPlan): string[] { + const catalog = plan.featureCatalog; + assert.ok(catalog); + const nodePoints = new Map(catalog.definition.nodes.map(node => + [node.id, node.gradingChecks.reduce((total, check) => total + check.points, 0)])); + return catalog.definition.questlines.map(questline => `${questline.id}=` + + questline.nodes.reduce((total, id) => total + (nodePoints.get(id) ?? 0), 0)); +} + +test('each stack compiles the reference graph to identical checks, points, and questline denominators', () => { + const source = JSON.parse(readFileSync(reference, 'utf8')) as { + stacks: Array<{ id: string; adapterVersion: string }>; + }; + assert.deepEqual(source.stacks.map(stack => stack.id).sort(), ['mongodb', 'postgres', 'spacetime']); + const dir = mkdtempSync(join(tmpdir(), 'stack-parity-')); + try { + const together = compileCampaignFile(reference); + const alone = source.stacks.map(stack => { + const file = join(dir, `${stack.id}.json`); + writeFileSync(file, JSON.stringify({ ...source, stacks: [stack] })); + const plan = compileCampaignFile(file); + assert.deepEqual(plan.stacks.map(item => item.id), [stack.id]); + return plan; + }); + for (const plan of alone) { + assert.deepEqual(gradedChecks(plan), gradedChecks(together)); + assert.deepEqual(questlinePoints(plan), questlinePoints(together)); + } + const levels = together.attempts[0]!.condition.requested.levels; + assert.equal(levels.length, 6); + assert.ok(levels.every(level => (level.selection.scoredChecks ?? []).length > 0)); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/tools/stack-bench/tests/stack-reference-operations.test.ts b/tools/stack-bench/tests/stack-reference-operations.test.ts new file mode 100644 index 00000000000..0ec844dbe4c --- /dev/null +++ b/tools/stack-bench/tests/stack-reference-operations.test.ts @@ -0,0 +1,144 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; + +import { deployMongoDbReference, deployPostgresReference, deploySpacetimeReference } + from '../src/stacks/stack-reference-operations.js'; +import type { ReferenceDeployInput, ReferenceHelpers } + from '../src/stacks/stack-reference-operations.js'; +import { loadTrack } from '../src/composition/tracks.js'; +import { attemptDatabaseIdentity } from '../src/stacks/hosted-database-identity.js'; + +const unused = () => { throw new Error('not used by this deployment'); }; +const helpersWith = (overrides: Partial): ReferenceHelpers => ({ + phase() {}, docker() {}, startDetached() {}, async waitFor() {}, containerLogs() { return ''; }, + runSync: unused, dbName: unused, moduleName: unused, loadTrack: unused, ...overrides, +}); +// Tests give only the lease fields each deployment reads. +const leaseOf = (resources: object, extra: object = {}) => + ({ resources, ...extra }) as unknown as ReferenceDeployInput['lease']; + +test('PostgreSQL reference starts its schema through the normal startup path', async () => { + const dockerCalls: Array> = []; + const starts: Array> = []; + const commands: Array = []; + const helpers = helpersWith({ + dbName() { return 'app_ecom_run0'; }, + runSync(_label, _command, args) { + commands.push(args); + return args[0] === 'inspect' ? 'container-id\n' : ''; + }, + docker(...args) { dockerCalls.push(args); }, + startDetached(...args) { starts.push(args); }, + }); + await deployPostgresReference({ + args: { backend: 'postgres', track: 'ecommerce', runIndex: 0 }, + metadata: { installDirectories: [], server: { directory: 'server' }, + client: { directory: 'client' } }, + lease: leaseOf({ database: 'app_ecom_run0', + container: { name: 'postgres', id: 'container-id' } }), + track: { slug: 'ecommerce', restartProbe: '/api/items' }, container: 'build-0', + ports: { dbPort: 6532, vite: 6573, express: null }, buildNetworkMode: 'host', helpers, + }); + + assert(commands.some(args => args.includes('dropdb') && args.at(-1) === 'app_ecom_run0')); + assert(commands.some(args => args.includes('createdb') && args.at(-1) === 'app_ecom_run0')); + assert.deepEqual(dockerCalls.map(call => call.slice(0, 4)), [ + ['build-0', '/app/client', 'npm', ['run', 'build']], + ]); + assert.equal(starts[0]?.[4]?.script, 'start'); + const serverPackage = JSON.parse(readFileSync(join(STACK_BENCH_ROOT, + 'reference-apps/ecommerce/postgres/server/package.json'), 'utf8')); + assert.equal(serverPackage.scripts.prestart, undefined, 'restarts must not push over extension data'); + const serverSource = readFileSync(join(STACK_BENCH_ROOT, + 'reference-apps/ecommerce/postgres/server/src/index.ts'), 'utf8'); + const initialize = serverSource.indexOf('await initializeCoreSchema()'); + assert(initialize >= 0 && initialize < serverSource.indexOf('httpServer.listen(')); + assert(serverPackage.devDependencies['drizzle-kit']); + +}); + +test('hosted reference credentials stay in process environment', async () => { + const starts: Array> = []; + const commands: Array = []; + const ownershipToken = 'reference-mongodb-authority'; + const helpers = helpersWith({ + dbName() { return 'app_ecom_run0'; }, + runSync(_label, _command, args) { + commands.push(args); + return args[0] === 'inspect' ? 'container-id\n' : ''; + }, + startDetached(...args) { starts.push(args); }, + }); + await deployMongoDbReference({ + args: { backend: 'mongodb', track: 'ecommerce', runIndex: 0 }, + metadata: { installDirectories: [], server: { directory: 'server' }, + client: { directory: 'client' } }, + lease: leaseOf({ database: 'app_ecom_run0', + container: { name: 'mongodb', id: 'container-id' }, + network: { name: 'attempt', id: 'a'.repeat(64), namespaceContainerId: 'b'.repeat(64), + hostAddresses: [], services: [], firewallSha256: null, firewallInstalledAt: null } }, { ownershipToken }), + track: { slug: 'ecommerce', restartProbe: '' }, container: 'build-0', + ports: { dbPort: 6537, vite: 6723, express: null }, + buildNetworkMode: 'host', helpers, + }); + const applicationStart = starts[0]; + assert(applicationStart); + assert.match(applicationStart[3].DATABASE_URL ?? '', /app_ecom_run0/); + assert.deepEqual(applicationStart.slice(0, 3), ['build-0', '/app', 'reference-application']); + assert.equal(applicationStart[3].PORT, '6723'); + assert.equal(applicationStart[3].JWT_SECRET, 'stack-bench-reference-only-secret-2026'); + assert.deepEqual(applicationStart[4], { script: 'start' }); + const identity = attemptDatabaseIdentity(ownershipToken); + const reset = commands.find(args => args.includes('db.dropDatabase()')); + assert(reset); + assert.deepEqual(reset.slice(0, 4), ['exec', 'container-id', 'mongosh', 'app_ecom_run0']); + assert.equal(reset[reset.indexOf('--username') + 1], 'admin'); + assert.equal(reset[reset.indexOf('--password') + 1], identity.adminPassword); + assert.equal(reset[reset.indexOf('--authenticationDatabase') + 1], 'admin'); + assert(applicationStart[3].DATABASE_URL?.includes(identity.password)); + assert(!JSON.stringify(starts).includes(identity.adminPassword)); +}); + +test('Spacetime reference client uses its assigned Vite port', async () => { + const starts: Array> = []; + const waits: Array> = []; + const helpers = helpersWith({ + loadTrack() { return loadTrack('ecommerce'); }, + moduleName() { return 'ecommerce_42'; }, + startDetached(...args) { starts.push(args); }, + async waitFor(...args) { waits.push(args); }, + }); + + await deploySpacetimeReference({ + args: { backend: 'spacetime', track: 'ecommerce', runIndex: 42 }, + metadata: { + kind: 'spacetime', + installDirectories: [], + moduleDirectory: 'backend/spacetimedb', + bindingsDirectory: 'client/src/module_bindings', + client: { directory: 'client' }, + }, + lease: leaseOf({ + module: 'ecommerce_42', + serverUri: 'ws://host.docker.internal:3315', + }), + track: { slug: 'ecommerce', restartProbe: '' }, + container: 'build-42', + ports: { vite: 6475, express: null, dbPort: null }, + buildNetworkMode: 'bridge', + helpers, + }); + + assert.equal(starts.length, 1); + const clientStart = starts[0]; + const clientWait = waits[0]; + assert(clientStart); + assert(clientWait); + assert.deepEqual(clientStart.slice(0, 3), ['build-42', '/app/client', 'reference-client']); + assert.equal(clientStart[3].VITE_PORT, '6475'); + assert.deepEqual(clientStart[4], { networkVisible: true, port: 6475 }); + assert.equal(clientWait[0], 'http://127.0.0.1:6475'); +}); diff --git a/tools/stack-bench/tests/stack-testability.test.ts b/tools/stack-bench/tests/stack-testability.test.ts new file mode 100644 index 00000000000..afb80171ca9 --- /dev/null +++ b/tools/stack-bench/tests/stack-testability.test.ts @@ -0,0 +1,101 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { GRADING_CAPABILITY_IDS } from '../src/actions/action-contract.js'; +import { describeStackTestabilityProblems, + resolveStackTestability, type TestabilityStack } from '../src/campaigns/stack-testability.js'; +import type { CompiledRecipePlan } from '../src/composition/composition-compiler.js'; +import type { CompiledStep } from '../src/composition/definition-compiler.js'; +import type { TrackAction } from '../src/composition/tracks.js'; + +const boundAction = { id: 'buy', path: '/api/buy', reducer: 'buy_now', args: [0], + params: [{ name: 'itemId', in: 'body' }] }; + +const criteria: Record = { + replay: [{ do: 'replayAs', actor: 'other', from: 'owner', match: 'secret' }], + namedReplay: [{ do: 'replayAs', actor: 'other', from: 'owner', match: 'secret', + namedAction: boundAction }], + routeOnly: [{ do: 'replayAs', actor: 'other', from: 'owner', match: 'secret', + namedAction: { id: 'ship', path: '/api/ship', args: [] } }], + trackCall: [{ do: 'callAction', actor: 'owner', action: 'buy', input: {} }], + unknownCall: [{ do: 'callAction', actor: 'owner', action: 'refund', input: {} }], + stock: [{ do: 'dbSetStock', item: 'Lamp', warehouse: 'East', quantity: 1 }], + raced: [{ do: 'race', branches: [[{ do: 'forgeWrite', actor: 'a', fromActor: 'b' }], + [{ do: 'click', actor: 'a', testid: 'buy-now' }]] }], + browser: [{ do: 'click', actor: 'owner', testid: 'buy-now' }], +}; + +function plan(setup: CompiledStep[] = []): CompiledRecipePlan { + const feature = { id: 1, name: 'feature', setup, + criteria: Object.entries(criteria).map(([id, steps]) => ({ id, desc: id, points: 1, steps })) }; + return { + checks: Object.keys(criteria).map(id => ({ stableKey: `pack.feature.${id}`, packId: 'pack', + checkGroupId: 'group', criterionId: id, role: 'feature', source: 'scenario.json', + featureId: 1, description: id, sourcePoints: 1, points: 1 })), + execution: [{ id: 'exec', source: 'scenario.json', checkGroups: [{ packId: 'pack', + checkGroupId: 'group', role: 'feature', source: 'scenario.json', + feature, actions: [] }] }], + } as unknown as CompiledRecipePlan; +} + +const http = { id: 'postgres', grading: { transport: 'http' as const, capabilities: GRADING_CAPABILITY_IDS } }; +const reducer = { id: 'spacetime', grading: { transport: 'reducer' as const, capabilities: GRADING_CAPABILITY_IDS } }; +const stub = { id: 'stub', grading: { transport: 'http' as const, + capabilities: GRADING_CAPABILITY_IDS.filter(id => id !== 'database-write') } }; +const trackActions = [boundAction] as unknown as TrackAction[]; +const keys = (...ids: string[]) => ids.map(id => `pack.feature.${id}`); + +const resolve = (checkKeys: string[], stacks: TestabilityStack[] = [http, reducer, stub], setup?: CompiledStep[]) => + describeStackTestabilityProblems(resolveStackTestability({ + plan: plan(setup), checkKeys, trackActions, stacks })); + +test('a replay without a named action only resolves where HTTP writes are captured', () => { + assert.deepEqual(resolve(keys('replay')), [ + 'spacetime cannot measure pack.feature.replay: replayAs re-issues a captured HTTP write, ' + + 'which spacetime grading cannot replay; give the step a named action', + ]); + assert.deepEqual(resolve(keys('namedReplay')), []); +}); + +test('named actions need the binding each transport issues', () => { + assert.deepEqual(resolve(keys('routeOnly')), [ + 'spacetime cannot measure pack.feature.routeOnly: replayAs application action ship declares no reducer for spacetime', + ]); + assert.deepEqual(resolve(keys('trackCall')), []); + assert.deepEqual(resolve(keys('unknownCall'), [http]), [ + 'postgres cannot measure pack.feature.unknownCall: callAction names no application action "refund"', + ]); +}); + +test('runtime capabilities come from the action registry and the stack declaration', () => { + assert.deepEqual(resolve(keys('stock')), [ + 'stub cannot measure pack.feature.stock: dbSetStock needs the database-write capability, which stub does not provide', + ]); + assert.deepEqual(resolve(keys('browser')), []); +}); + +test('branches and feature setup count, unselected criteria do not', () => { + assert.deepEqual(resolve(keys('raced'), [reducer]), [ + 'spacetime cannot measure pack.feature.raced: forgeWrite re-issues a captured HTTP write, ' + + 'which spacetime grading cannot replay; give the step a named action', + ]); + assert.deepEqual(resolve(keys('browser'), [stub], criteria.stock), [ + 'stub cannot measure pack.feature.browser: dbSetStock needs the database-write capability, which stub does not provide', + ]); + assert.deepEqual(resolve(keys('browser'), [reducer]), []); + assert.throws(() => resolve(keys('missing')), /selected check pack\.feature\.missing is not in the recipe/); +}); + +test('a stack declares the named-action binding its transport name does not imply', () => { + const platform = { id: 'supabase', grading: { transport: 'http', capabilities: GRADING_CAPABILITY_IDS, + namedActionBinding: 'reducer' as const } }; + assert.deepEqual(resolve(keys('replay', 'raced', 'namedReplay', 'trackCall'), [platform]), []); + assert.deepEqual(resolve(keys('routeOnly'), [platform]), [ + 'supabase cannot measure pack.feature.routeOnly: replayAs application action ship declares no reducer for supabase', + ]); + const undeclared = { id: 'native', grading: { transport: 'supabase' as const, capabilities: GRADING_CAPABILITY_IDS } }; + assert.deepEqual(resolve(keys('replay', 'routeOnly'), [undeclared]), [ + 'native cannot measure pack.feature.replay: replayAs re-issues a captured HTTP write, ' + + 'which native grading cannot replay; give the step a named action', + ]); +}); diff --git a/tools/stack-bench/tests/staff-navigation.contract.ts b/tools/stack-bench/tests/staff-navigation.contract.ts new file mode 100644 index 00000000000..09858d4b9e3 --- /dev/null +++ b/tools/stack-bench/tests/staff-navigation.contract.ts @@ -0,0 +1,59 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import test from 'node:test'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; + +const track = join(STACK_BENCH_ROOT, 'tracks/ecommerce'); +function feature(name: string) { + const source = join(track, 'scenarios', name); + return compileScenarioDefinition(JSON.parse(readFileSync(source, 'utf8')), { source }).features[0]!; +} + +test('access checks observe protected content and retain direct restock authorization', () => { + const staff = feature('progression-staff-access.json'); + for (const actor of ['staff', 'admin']) { + assert(staff.criteria[0]!.steps.some(step => step.do === 'click' && step.actor === actor)); + assert(staff.criteria[0]!.steps.some(step => step.do === 'expect' + && step.actor === actor && step.testid === 'staff-area')); + } + const customerBoundary = staff.criteria.find(criterion => criterion.id === '601b')!; + assert(customerBoundary.steps.some(step => step.do === 'expect' + && step.actor === 'authorized' && step.testid === 'staff-area' && step.absent !== true)); + for (const name of ['progression-staff-access.json', '02-fulfilment-access.json', + '01-warehouse-admin-staff.json', '03-deferred-access.json']) { + const steps = feature(name).criteria.flatMap(criterion => criterion.steps); + assert(!steps.some(step => step.do === 'expect' && step.absent === true + && ['staff-link', 'admin-link', 'admin-panel'].includes(String(step.testid)))); + } + const restock = feature('03-deferred-access.json').criteria[0]!.steps; + assert(restock.some(step => step.do === 'callAction')); + assert(restock.some(step => step.do === 'expectActionOutcome' && step.outcome === 'refused')); + assert(restock.some(step => step.do === 'replayAs')); + assert(restock.some(step => step.do === 'expectReplayRejected')); + for (const name of ['01-features.json', '01-warehouse-admin-staff.json']) { + const source = join(track, 'scenarios', name); + const warehouse = compileScenarioDefinition(JSON.parse(readFileSync(source, 'utf8')), { source }) + .features.find(item => item.id === 7)!; + const boundary = warehouse.criteria.find(item => item.id === '7a')!; + const authorized = name === '01-features.json' ? 'authorized' : 'admin'; + assert(boundary.steps.some(step => step.do === 'expect' && step.actor === authorized + && step.testid === 'admin-item-row' && step.absent !== true)); + assert.equal([...warehouse.setup, ...boundary.steps].filter(step => step.do === 'click' + && step.actor === authorized && step.testid === 'admin-link').length, 1); + if (authorized !== 'admin') { + assert(!boundary.steps.some(step => step.actor === 'admin'), + 'the positive control must not open the next criterion actor\'s panel'); + assert(warehouse.criteria.find(item => item.id === '7b')!.steps.some(step => + step.do === 'click' && step.actor === 'admin' && step.testid === 'admin-link')); + } + } + // Deferred delivery is first observed through its positive completed order. + const integrity = join(track, 'scenarios', '03-deferred-integrity.json'); + const delivery = compileScenarioDefinition(JSON.parse(readFileSync(integrity, 'utf8')), { source: integrity }) + .features.find(item => item.id === 312)!; + const firstObservation = delivery.criteria[0]!.steps.find(step => step.do === 'expect')!; + assert.equal(firstObservation.testid, 'completed-order-item'); + assert.equal(firstObservation.absent, undefined); +}); diff --git a/tools/stack-bench/tests/staff-role-authorization.integration.ts b/tools/stack-bench/tests/staff-role-authorization.integration.ts new file mode 100644 index 00000000000..5b4032ea9b4 --- /dev/null +++ b/tools/stack-bench/tests/staff-role-authorization.integration.ts @@ -0,0 +1,140 @@ +import assert from 'node:assert/strict'; +import { readFileSync, writeFileSync } from 'node:fs'; +import { createServer } from 'node:http'; +import { join } from 'node:path'; +import test from 'node:test'; +import { chromium } from 'playwright'; +import { gradeFeature } from '../grader/grade.js'; +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; + +// A denial caused by self-protection or last-administrator protection is not +// proof that the server checked the caller's current administrator role. +test('role authorization probes distinguish access control from account safeguards', async t => { + const browser = await chromium.launch({ headless: true }); + let mode = 'correct'; + const roles = new Map(); + const sessions = new Map(); + const users = ['admin', 'staff', 'staff2']; + const writes: { actor: string; target: string; role: string; before: string; status: number; reason: string }[] = []; + const reads: Record[] = []; + const evidence: unknown[] = []; + const server = createServer(async (request, response) => { + const path = new URL(request.url!, 'http://fixture.test').pathname; + const session = sessions.get((request.headers.authorization ?? '').replace('Bearer ', '')); + if (path === '/login') { + let text = ''; for await (const chunk of request) text += String(chunk); + const { username } = JSON.parse(text) as { username: string }; + const token = `session-${sessions.size + 1}`; + sessions.set(token, { user: username, roleAtLogin: roles.get(username)! }); + response.writeHead(200, { 'Content-Type': 'application/json' }).end(JSON.stringify({ token, username })); return; + } + if (path === '/state') { + reads.push(Object.fromEntries(roles)); + response.writeHead(200, { 'Content-Type': 'application/json' }).end(JSON.stringify(Object.fromEntries(roles))); return; + } + if (path.startsWith('/api/staff/') && request.method === 'PUT') { + let text = ''; for await (const chunk of request) text += String(chunk); + const target = users[Number(path.split('/')[3]) - 1]!; + const { role } = JSON.parse(text) as { role: string }; + const actor = session?.user ?? ''; + const before = roles.get(target)!; + const currentRole = roles.get(actor); + const authorized = mode === 'no-admin-check-self-guard' + || (mode === 'stale-session-last-admin' ? session?.roleAtLogin : currentRole) === 'admin'; + const reason = !session ? 'no-session' + : target === actor ? 'self-change' + : before === 'admin' && role !== 'admin' && [...roles.values()].filter(value => value === 'admin').length === 1 + ? 'last-administrator' + : !authorized ? 'not-administrator' : 'accepted'; + const status = reason === 'accepted' ? 200 : 403; + if (mode === 'delayed-save') await new Promise(resolve => setTimeout(resolve, 250)); + if (status === 200 && mode !== 'false-save-receipt') roles.set(target, role); + writes.push({ actor, target, role, before, status, reason }); + response.writeHead(status, { 'Content-Type': 'application/json' }) + .end(JSON.stringify(status === 200 ? { ok: true } : { error: reason })); return; + } + response.writeHead(200, { 'Content-Type': 'text/html' }).end(`
`); + }); + try { + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + for (const fixture of [ + { check: '621b', mode: 'correct', status: 'passed' }, + { check: '621b', mode: 'no-admin-check-self-guard', status: 'failed' }, + { check: '621d', mode: 'correct', status: 'passed' }, + { check: '621d', mode: 'stale-session-last-admin', status: 'failed' }, + { check: '621a', mode: 'correct', status: 'passed' }, + { check: '621a', mode: 'delayed-save', status: 'passed' }, + { check: '621a', mode: 'false-save-receipt', status: 'failed' }, + ]) await t.test(`${fixture.check}: ${fixture.mode}`, async () => { + mode = fixture.mode; writes.length = 0; reads.length = 0; sessions.clear(); roles.clear(); + for (const user of users) roles.set(user, user === 'admin' ? 'admin' : 'staff'); + const source = join(STACK_BENCH_ROOT, 'tracks/ecommerce/scenarios/progression-staff-roles.json'); + const definition = JSON.parse(readFileSync(source, 'utf8')); + const selected = definition.features[0]; + selected.criteria = selected.criteria.filter((criterion: { id: string }) => criterion.id === fixture.check); + if (fixture.check === '621a') { + // This browser fixture measures the first persisted read, not database restart. + const restart = selected.criteria[0].steps.findIndex((step: { do: string }) => step.do === 'restartBackend'); + assert.ok(restart > 0); + selected.criteria[0].steps = selected.criteria[0].steps.slice(0, restart); + } + for (const step of [...selected.setup, ...selected.criteria[0].steps]) { + if ('within' in step || ['click', 'fill', 'expect'].includes(step.do)) step.within = 1000; + if ('settleMs' in step) step.settleMs = 0; + } + const feature = compileScenarioDefinition(definition, { source }).features[0]!; + const grade = await gradeFeature(browser, feature, + { url, level: definition.level, headed: false, selectedCheckKeys: [], nullControl: false }, + { runId: `role-authority-${fixture.check}-${mode}`, roomName: name => name, url, actions: [], + spacetime: null, backend: 'postgres', nullControl: false, defaultWithin: 1000 }); + const row = { ...fixture, grade, writes: [...writes], reads: [...reads], roles: Object.fromEntries(roles) }; + evidence.push(row); + assert.equal(grade.criteria[0]!.evidence.status, fixture.status, JSON.stringify(row)); + if (fixture.mode === 'no-admin-check-self-guard' || fixture.mode === 'stale-session-last-admin') { + assert.ok(writes.some(write => write.actor === 'staff' && write.target !== 'staff' && write.status === 200), + 'the deliberate authorization defect must reach another account'); + } + }); + } finally { + await browser.close(); server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + if (process.env.STACK_BENCH_ROLE_AUTHORIZATION_EVIDENCE) { + writeFileSync(process.env.STACK_BENCH_ROLE_AUTHORIZATION_EVIDENCE, JSON.stringify({ + rerun: 'STACK_BENCH_ROLE_AUTHORIZATION_EVIDENCE= node --test dist/tests/staff-role-authorization.integration.js', evidence, + }, null, 2)); + } + } +}); diff --git a/tools/stack-bench/tests/state-volume.test.ts b/tools/stack-bench/tests/state-volume.test.ts new file mode 100644 index 00000000000..f165ed37214 --- /dev/null +++ b/tools/stack-bench/tests/state-volume.test.ts @@ -0,0 +1,149 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, rmSync, statSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import test from 'node:test'; +import { prepareStateVolume, writeStateSecret } from '../appliance/state-volume.js'; +import { DATABASE_IMAGES } from '../src/stacks/database-containers.js'; +import { stackReleaseImages } from '../src/stacks/stack-identities.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { compileCampaignFile } from '../src/campaigns/campaign-compiler.js'; + +test('setup uses the daemon volume path and exact local images without disclosing secrets', () => { + const calls: readonly string[][] = []; + const root = '/var/lib/docker/volumes/stack-bench-state/_data'; + const id = `sha256:${'a'.repeat(64)}`; + const env = prepareStateVolume({}, args => { + (calls as string[][]).push([...args]); + if (args[0] === 'version') return 'linux'; + if (args[0] === 'image') return id; + if (args[0] === 'volume' && args[1] === 'inspect') return root; + return ''; + }); + assert.match(env, new RegExp(`STACK_BENCH_STATE_ROOT=${root}`)); + assert.match(env, new RegExp(`STACK_BENCH_CONTROLLER_IMAGE=${id}`)); + const initialization = calls.find(args => args[0] === 'run'); + assert(initialization?.includes(`type=volume,source=stack-bench-state,target=${root}`)); + assert.doesNotMatch(env, /ANTHROPIC_API_KEY=|CLAUDE_CODE_OAUTH_TOKEN=/); + assert.equal(calls.filter(args => args[0] === 'pull').length, 0, 'installed exact images need no pull'); + assert.throws(() => prepareStateVolume({}, () => 'windows'), /Linux containers/); +}); + +test('setup installs an image-bound paid demo and preserves an existing plan', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-setup-')); + const controller = `sha256:${'a'.repeat(64)}`; + const build = `sha256:${'b'.repeat(64)}`; + let initialization: readonly string[] = []; + try { + prepareStateVolume({}, args => { + if (args[0] === 'version') return 'linux'; + if (args[0] === 'image') return args.at(-1) === 'stack-bench-build:local' ? build : controller; + if (args[0] === 'volume' && args[1] === 'inspect') return '/state'; + if (args[0] === 'run') initialization = args; + return ''; + }); + const script = initialization[initialization.indexOf('-e') + 1]!; + const localScript = script.replaceAll('/opt/stack-bench/appliance', + resolve(STACK_BENCH_ROOT, 'appliance').replaceAll('\\', '/')); + const initialize = (image: string) => execFileSync(process.execPath, + ['-e', localScript, root, image, build]); + initialize(controller); + assert.equal(readFileSync(join(root, 'results', 'plans', 'demo.json'), 'utf8'), + readFileSync(resolve(STACK_BENCH_ROOT, 'appliance', 'campaign.demo.json'), 'utf8')); + const path = join(root, 'results', 'plans', 'paid-l1.json'); + const bytes = readFileSync(path, 'utf8'); + const plan = JSON.parse(bytes); + assert.equal(plan.state, 'frozen'); + assert.equal(plan.runtime.controllerImage, controller); + assert.equal(plan.runtime.buildImage, build); + assert.equal(plan.parallelism, 3); + assert.equal(plan.budgets.maxCostUsdPerAttempt, 10); + const compiled = compileCampaignFile(path); + assert.equal(compiled.state, 'frozen'); + assert.deepEqual(compiled.definition.runtime, plan.runtime); + const presetBytes = new Map(); + for (const id of ['ecommerce-sequential', 'ecommerce-progressive', 'ecommerce-progressive-l3', 'ecommerce-single-build']) { + const presetPath = join(root, 'results/run-presets', `${id}.json`); + presetBytes.set(presetPath, readFileSync(presetPath, 'utf8')); + const preset = JSON.parse(presetBytes.get(presetPath)!); + assert.equal(preset.runtime.controllerImage, controller); + assert.equal(preset.runtime.buildImage, build); + assert.equal(preset.agents[0].adapter, 'claude-code'); + assert.equal(preset.repair.budget.total, 0); + assert.equal(preset.conditions.length, 4); + if (id === 'ecommerce-single-build') assert.equal(preset.mode.workSelection, 'all-at-once'); + if (id === 'ecommerce-progressive' || id === 'ecommerce-progressive-l3') { + assert.deepEqual(preset.mode, { id: 'dependency', workSelection: 'progressive', + retainPriorContracts: true, unchangedFailureLimit: 7 }); + assert.deepEqual(preset.budgets, { attemptTimeoutMinutes: 240, maxCostUsdPerAttempt: 50 }); + } + if (id === 'ecommerce-progressive-l3') { + assert.deepEqual(preset.levels, [1, 2, 3]); + assert.deepEqual(preset.selection.levels.map((selection: { level: number }) => selection.level), [1, 2, 3]); + assert.deepEqual(preset.stacks.map((stack: { id: string }) => stack.id), ['mongodb', 'postgres', 'spacetime', 'convex']); + assert.equal(preset.stacks.at(-1).adapterVersion, '1.0.0'); + assert.equal(preset.parallelism, 4); + } else assert.equal(preset.stacks.length, 3, 'existing three-stack presets keep their scope'); + } + initialize(`sha256:${'c'.repeat(64)}`); + assert.equal(readFileSync(path, 'utf8'), bytes, 'setup must not rebind an existing plan'); + for (const [presetPath, original] of presetBytes) { + assert.equal(readFileSync(presetPath, 'utf8'), original, 'setup must not rebind an existing preset'); + } + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('setup pulls each absent pinned native backend image before state initialization', () => { + const pinned = [...Object.values(DATABASE_IMAGES), ...stackReleaseImages().map(image => image.reference)]; + assert(pinned.some(reference => reference.startsWith('supabase/postgres@sha256:'))); + const missing = new Set(pinned); + const pulls: string[][] = []; + prepareStateVolume({}, args => { + if (args[0] === 'version') return 'linux'; + if (args[0] === 'image') { + if (missing.has(args.at(-1)!)) throw new Error('No such image'); + return `sha256:${'a'.repeat(64)}`; + } + if (args[0] === 'pull') { + pulls.push([...args]); + missing.delete(args.at(-1)!); + } + if (args[0] === 'volume' && args[1] === 'inspect') return '/var/lib/docker/volumes/stack-bench-state/_data'; + if (args[0] === 'run') assert.equal(missing.size, 0); + return ''; + }); + assert.deepEqual(pulls, pinned.map(reference => ['pull', '--platform', 'linux/amd64', reference])); +}); + +test('a failed native image pull stops setup before it initializes shared state', () => { + const calls: readonly string[][] = []; + assert.throws(() => prepareStateVolume({}, args => { + (calls as string[][]).push([...args]); + if (args[0] === 'version') return 'linux'; + if (args[0] === 'image' && args.at(-1) === DATABASE_IMAGES.postgres) throw new Error('No such image'); + if (args[0] === 'image') return `sha256:${'a'.repeat(64)}`; + if (args[0] === 'pull') throw new Error('registry unavailable'); + return ''; + }), /registry unavailable/); + assert.equal(calls.some(args => ['run', 'volume'].includes(args[0]!)), false); +}); + +test('secret input is restricted to named private files and never needs a shell argument value', () => { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-secret-')); + try { + assert.throws(() => writeStateSecret('../other', 'secret', root), /secret name/); + assert.throws(() => writeStateSecret('anthropic_api_key', 'a\nb', root), /one non-empty line/); + assert.throws(() => writeStateSecret('codex_auth', '{invalid', root), /valid JSON/); + writeStateSecret('codex_auth', '{\n "auth_mode": "chatgpt"\n}', root); + assert.equal(readFileSync(join(root, 'secrets', 'codex_auth'), 'utf8'), '{"auth_mode":"chatgpt"}\n'); + writeStateSecret('openrouter_api_key', 'test-openrouter', root); + assert.equal(readFileSync(join(root, 'secrets', 'openrouter_api_key'), 'utf8'), 'test-openrouter\n'); + writeStateSecret('openai_api_key', 'test-openai', root); + assert.equal(readFileSync(join(root, 'secrets', 'openai_api_key'), 'utf8'), 'test-openai\n'); + writeStateSecret('anthropic_api_key', ' test-key\n', root); + const path = join(root, 'secrets', 'anthropic_api_key'); + assert.equal(readFileSync(path, 'utf8'), 'test-key\n'); + if (process.platform !== 'win32') assert.equal(statSync(path).mode & 0o777, 0o600); + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/tools/stack-bench/tests/stock-alert-delivery.integration.ts b/tools/stack-bench/tests/stock-alert-delivery.integration.ts new file mode 100644 index 00000000000..59f2b420788 --- /dev/null +++ b/tools/stack-bench/tests/stock-alert-delivery.integration.ts @@ -0,0 +1,410 @@ +import assert from 'node:assert/strict'; +import { appendFileSync, mkdirSync, readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import test from 'node:test'; +import { chromium, type BrowserContext, type Page } from 'playwright'; +import { ACTION_REGISTRY } from '../src/actions/action-catalog.js'; +import { executeAction, type ActionEvidence } from '../src/actions/action-contract.js'; +import { createNamedActionsCapability } from '../src/actions/actor-transport-action-executors.js'; +import { stableElementSelector } from '../src/actions/element-selector.js'; +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { createServer } from 'node:http'; +import { gradeFeature } from '../grader/grade.js'; + +test('promotion access measures denied management and writes rather than link visibility', async t => { + // Failure first: a visible denied link is valid; a write is forbidden even if its response says 403. + let mode = '', codes: string[] = []; + const requests: Array<{ code: string; authorized: boolean; status: number; committed: boolean }> = []; + const evidence: unknown[] = []; + const server = createServer(async (request, response) => { + if (request.url === '/api/promotions' && request.method === 'POST') { + let body = ''; for await (const chunk of request) body += String(chunk); + const { code } = JSON.parse(body); + const authorized = request.headers.authorization === 'Bearer staff'; + const committed = authorized || mode !== 'denied'; + const status = authorized || mode === 'accepted-write' ? 200 : 403; + if (committed) codes.push(code); + requests.push({ code, authorized, status, committed }); + response.writeHead(status, { 'Content-Type': 'application/json' }); + response.end(JSON.stringify({ ok: status === 200 })); return; + } + response.writeHead(200, { 'Content-Type': 'text/html' }); + response.end(`
`); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const browser = await chromium.launch({ headless: true }); + try { + for (mode of ['denied', 'accepted-write', 'write-then-deny']) await t.test(mode, async () => { + codes = []; requests.length = 0; + const definition = JSON.parse(readFileSync(join(STACK_BENCH_ROOT, + 'tracks/ecommerce/scenarios/progression-promotion-rules.json'), 'utf8')); + const selected = definition.features[0]; + selected.criteria = selected.criteria.filter((criterion: { id: string }) => criterion.id === '620b'); + for (const step of [...selected.setup, ...selected.criteria[0].steps]) { + if ('within' in step) step.within = 700; + if ('settleMs' in step) step.settleMs = 0; + } + const grade = await gradeFeature(browser, compileScenarioDefinition(definition).features[0]!, + { url, level: definition.level, headed: false, selectedCheckKeys: [], nullControl: false }, + { runId: `promotion-access-${mode}`, roomName: name => name, url, actions: [], + spacetime: null, backend: 'postgres', nullControl: false, defaultWithin: 700 }); + const row = { mode, grade, requests: [...requests], codes: [...codes] }; evidence.push(row); + assert.equal(grade.setupEvidence.status, 'passed', JSON.stringify(row)); + assert.equal(grade.criteria[0]!.evidence.status, mode === 'denied' ? 'passed' : 'failed', JSON.stringify(row)); + assert.equal(requests.filter(request => request.authorized && request.code === 'ACCESS10').length, 1); + assert.equal(requests.filter(request => !request.authorized && request.code === 'HACK10').length, 1, + 'the customer replay must reach the real endpoint; a link-only failure is not the boundary test'); + assert.equal(codes.includes('HACK10'), mode !== 'denied'); + if (mode === 'write-then-deny') assert.equal((grade.criteria[0]!.evidence.actions.at(-1)?.evidence as ActionEvidence | undefined)?.action.id, 'expect'); + }); + } finally { + await browser.close(); server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + if (process.env.STACK_BENCH_PROMOTION_ACCESS_EVIDENCE) { + const { writeFileSync } = await import('node:fs'); + writeFileSync(process.env.STACK_BENCH_PROMOTION_ACCESS_EVIDENCE, JSON.stringify({ + rerun: 'STACK_BENCH_PROMOTION_ACCESS_EVIDENCE= node --test --test-name-pattern="promotion access measures" dist/tests/stock-alert-delivery.integration.js', evidence, + }, null, 2)); + } + } +}); + +test('promotion and role access controls reject optimistic writes that disappear on reload', async () => { + const browser = await chromium.launch({ headless: true }); + try { + for (const kind of ['promotion', 'role'] as const) for (const persists of [true, false]) + for (const layout of kind === 'promotion' && persists ? ['inline', 'toggle', 'route'] : ['inline']) + for (const sequential of kind === 'promotion' && persists ? [false, true] : [false]) { + const context = await browser.newContext(); + try { + let saved = kind === 'role' ? 'inventory' : ''; + let writes = 0; + let loads = 0; + await context.route('http://controls.test/**', async route => { + if (route.request().method() === 'POST') { + writes++; + if (persists) saved = route.request().postData()!; + await route.fulfill({ status: 200, body: '' }); + return; + } + loads++; + await route.fulfill({ contentType: 'text/html', body: ` +
+ + +
+ +
${kind === 'promotion' && saved ? `
${saved}
` : ''}
+
+
+ ` }); + }); + const page = await context.newPage(); + await page.goto('http://controls.test/'); + const actor = { name: kind === 'role' ? 'replayAdmin' : 'staff', page, writes: [], + loc: (id: string, options?: { contains?: string; scope?: { testid: string; contains?: string | RegExp } }) => { + const root = options?.scope ? page.locator(stableElementSelector(options.scope.testid), { hasText: options.scope.contains }).first() : page; + return root.locator(stableElementSelector(id), { hasText: options?.contains }).first(); + } }; + const capability = { defaultWithin: 300, recorded: new Map(), + expand: (value: string) => value, scopedUser: (value: string) => value, + testId: stableElementSelector, sleep: async () => {} }; + const customerPage = await context.newPage(); + await customerPage.route('http://controls.test/**', route => route.fulfill({ contentType: 'text/html', + body: 'Customer' })); + await customerPage.goto('http://controls.test/'); + const customer = { name: 'customer', page: customerPage, context, + loc: (id: string) => customerPage.locator(stableElementSelector(id)), + writes: [{ url: 'http://controls.test/write', headers: { authorization: 'Bearer customer' } }] }; + let replayRequests = 0; + const named = createNamedActionsCapability({ actions: [], backend: 'postgres', url: 'http://controls.test', + lastCalls: { get: () => null, set: () => {} }, sleep: async () => {}, now: Date.now, + fetchImpl: async (url, options) => { + assert.equal(url, 'http://controls.test/api/promotions'); + assert.equal(JSON.parse(String(options.body)).code, 'HACK10'); + assert.equal(options.headers?.authorization, 'Bearer customer'); + replayRequests++; + return { status: 403, ok: false, text: async () => '' }; + } }); + const capabilities = { actors: { get: (name: string) => name === 'customer' ? customer : actor }, + 'browser-interaction': capability, 'browser-observation': capability, 'named-actions': named, + 'transport-observation': { ...capability, verification: { verified: () => {}, unverified: () => {} } } }; + const scenario = compileScenarioDefinition(JSON.parse(readFileSync(join(STACK_BENCH_ROOT, + 'tracks/ecommerce/scenarios', kind === 'role' ? 'progression-staff-roles.json' : 'progression-promotion-rules.json'), 'utf8'))); + const criterion = scenario.features[0]!.criteria.find(c => c.id === (kind === 'role' ? '621b' : '620b'))!; + const steps = [...(sequential ? scenario.features[0]!.criteria.find(c => c.id === '620a')!.steps : []), + ...(kind === 'promotion' ? criterion.steps : criterion.steps.slice(0, criterion.steps.findIndex(step => step.actor !== actor.name)))]; + let failure: string | null = null; + let reloads = 0; + let writeReloads = 0; + let unreloadedWrite = false; + for (const step of steps) { + if (step.do === 'reload') { + reloads++; + // The scenario also reloads before the write and after the replay; the reload after the write decides. + if (unreloadedWrite) { + unreloadedWrite = false; + writeReloads++; + await page.waitForFunction(count => performance.getEntriesByType('resource').filter(entry => entry.name.endsWith('/write')).length === count, sequential ? 2 : 1); + assert.equal(writes, sequential ? 2 : 1); + assert.equal(kind === 'role' ? await page.locator('#staff-role-select').inputValue() + : (await page.locator('[data-role="promotion-item"]').innerText()).slice(0, 8), kind === 'role' ? 'staff' : 'ACCESS10'); + } + } + if (step.do === 'click' && ['promotion-submit', 'staff-role-save'].includes(String(step.testid))) unreloadedWrite = true; + const input = ['expect', 'click'].includes(step.do) ? { ...step, within: 300 } : step; + const result = await executeAction(ACTION_REGISTRY, step.do, input, { capabilities }); + if (result.status !== 'passed') { assert.equal(result.status, 'failed', result.summary ?? undefined); failure = String(step.testid); break; } + } + assert.equal(writeReloads, 1, 'the optimistic write must be followed by a reload'); + assert.equal(loads, (layout === 'route' ? 2 : 1) + reloads, 'the control must read the server state after a real reload'); + assert.equal(failure, persists ? null : kind === 'role' ? 'staff-role-select' : 'promotion-item', `${kind}/${layout}/sequential=${sequential}`); + assert.equal(replayRequests, kind === 'promotion' && persists ? 1 : 0); + } finally { await context.close(); } + } + } finally { await browser.close(); } +}); + +test('stock alerts accept fresh load-on-open views and reject missing, premature, late duplicate, or private delivery', async () => { + const browser = await chromium.launch({ headless: true }); + try { + // Vary valid layouts on the positive path; failure cases need only one layout. + for (const [id, mode, layout, expectedFailure] of [ + ['631c', 'transition', 'inline', null], + ['631c', 'transition', 'inline-loading', null], + ['631c', 'transition', 'inline-no-toggle', null], + ['631c', 'transition', 'modal', null], + ['631c', 'transition', 'page', null], + ['631c', 'transition', 'settings-page', null], + ['631c', 'transition', 'settings-toggle', null], + ['631c', 'transition', 'catalog-settings', null], + ['631c', 'transition', 'menu-toggle', null], + ['631c', 'transition', 'menu-settings', null], + ['631c', 'transition', 'inline-menu', null], + ['631c', 'transition', 'menu-broken', 'click/before-restock'], + ['631c', 'transition', 'menu-missing', 'click/before-restock'], + ['631c', 'transition', 'settings-noop', 'click/before-restock'], + ['631c', 'transition', 'no-controls', 'click/before-restock'], + ['631c', 'pending', 'page', 'expect/after-restock'], + ['631c', 'premature', 'page', 'expectElementCount/before-restock'], + ['631a', 'transition', 'inline', null], + ['631a', 'transition', 'modal', null], + ['631a', 'transition', 'page', null], + ['631a', 'transition', 'settings-page', null], + ['631a', 'transition', 'menu-toggle', null], + ['631a', 'pending', 'page', 'expectElementCount/before-restock'], + ['631a', 'duplicate', 'page', 'expectElementCount/after-restock'], + ['631a', 'duplicate', 'settings-page', 'expectElementCount/after-restock'], + ['631b', 'private', 'page', null], + ['631b', 'private', 'settings-page', null], + ['631b', 'private', 'menu-settings', null], + ['631b', 'pending', 'page', 'expect/before-restock'], + ['631b', 'leak', 'page', 'expect/before-restock'], + ['631b', 'leak', 'settings-page', 'expect/before-restock'], + ] as const) { + const contexts: BrowserContext[] = []; + const deliveries = mode === 'premature' || id !== '631c' && mode !== 'pending' ? 1 : 0; + let elapsed = 0; + let deliveryDue = Infinity; + let duplicateDue = Infinity; + let pendingWrite = false; + let restocked = false; + let readsAfterRestock = 0; + let freshClients = 0; + let waitMs = 0; + const navigation = { catalog: 0, settings: 0, toggle: 0, account: 0 }; + const observations: { action: string; actor?: string; control?: string; status: string; summary: string | null }[] = []; + const actors = new Map ReturnType }>(); + try { + const makeActor = async (name: string) => { + const context = await browser.newContext(); + contexts.push(context); + const page = await context.newPage(); + await page.exposeFunction('recordNavigation', (control: keyof typeof navigation) => { navigation[control]++; }); + await page.exposeFunction('readNotifications', async (user: string) => { + assert.equal(pendingWrite, false, 'the driver must await the write before its fresh read'); + if (layout === 'inline-loading') await new Promise(resolve => setTimeout(resolve, 30)); + if (restocked) readsAfterRestock++; + const count = deliveries + (elapsed >= deliveryDue ? 1 : 0) + (elapsed >= duplicateDue ? 1 : 0); + return user === 'stock-subscriber' || mode === 'leak' ? count : 0; + }); + await context.route('http://app.test/**', route => route.fulfill({ contentType: 'text/html', body: `
+ +
+ + + ${layout === 'modal' ? '' : ''} + +
Air Purifier East
+ + ` })); + await page.goto('http://app.test/'); + const actor = { name, page, context, record() {}, writes: [{ url: 'http://app.test/api/session', headers: { authorization: 'Bearer fixture-admin' } }], + loc: (id: string, options?: { contains?: string; scope?: { testid: string; contains?: string | RegExp } }) => { + const root = options?.scope ? page.locator(stableElementSelector(options.scope.testid), + { hasText: options.scope.contains }).first() : page; + return root.locator(stableElementSelector(id), { hasText: options?.contains }).first(); + } }; + actors.set(name, actor); + return name; + }; + await makeActor('subscriber'); + await makeActor('other'); + await makeActor('admin'); + const sleep = async (ms: number) => { elapsed += ms; await new Promise(resolve => setTimeout(resolve, 1)); }; + const capability = { defaultWithin: 500, recorded: new Map(), + expand: (value: string) => value, scopedUser: (value: string) => value, + testId: stableElementSelector, sleep, + clients: { fresh: async (_actor: unknown, name: string) => { + freshClients++; + return makeActor(`${name}-fresh`); + } } }; + const named = createNamedActionsCapability({ actions: [], backend: 'postgres', url: 'http://app.test', + lastCalls: { get: () => null, set: () => {} }, sleep, now: () => elapsed, + fetchImpl: async (url, options) => { + assert.equal(url, 'http://app.test/api/admin/restock'); + assert.deepEqual(JSON.parse(String(options.body)), { itemId: 1, warehouseId: 2, quantity: 1 }); + pendingWrite = true; + await new Promise(resolve => setTimeout(resolve, 30)); + if (mode !== 'pending' && mode !== 'premature') { + if (id === '631c') deliveryDue = elapsed + 5000; + if (mode === 'duplicate') duplicateDue = elapsed + 5000; + } + pendingWrite = false; + restocked = true; + return { status: 200, ok: true, text: async () => '' }; + } }); + const capabilities = { actors: { get: (name: string) => actors.get(name) }, + 'browser-interaction': capability, 'browser-observation': capability, + 'named-actions': named, 'transport-observation': { ...capability, + verification: { verified: () => {} } }, + clock: { sleep: async (ms: number) => { + assert.equal(pendingWrite, false, 'the observation window must start after the write completes'); + waitMs += ms; await sleep(ms); + } } }; + const name = id === '631c' ? 'progression-stock-alert-delivery.json' : 'progression-stock-alerts.json'; + const scenario = compileScenarioDefinition(JSON.parse(readFileSync(join(STACK_BENCH_ROOT, + 'tracks/ecommerce/scenarios', name), 'utf8'))); + const criterion = scenario.features[0]!.criteria.find(criterion => criterion.id === id)!; + let failed: string | null = null; + for (const step of criterion.steps) { + const input = ['expect', 'expectElementCount', 'click'].includes(step.do) + ? { ...step, within: 500 } : step; + const result = await executeAction(ACTION_REGISTRY, step.do, input, { capabilities }); + observations.push({ action: step.do, actor: step.actor, control: step.testid, + status: result.status, summary: result.summary ?? null }); + if (result.status !== 'passed') { + assert.equal(result.status, 'failed', result.summary ?? undefined); + failed = `${step.do}/${restocked ? 'after-restock' : 'before-restock'}`; + break; + } + } + const evidenceFile = process.env.STACK_BENCH_NOTIFICATION_NAVIGATION_EVIDENCE; + if (evidenceFile) { + mkdirSync(dirname(evidenceFile), { recursive: true }); + const screenshot = join(dirname(evidenceFile), `${id}-${mode}-${layout}.png`); + await actors.get('subscriber-fresh')!.page.screenshot({ path: screenshot }); + appendFileSync(evidenceFile, `${JSON.stringify({ id, mode, layout, scenario: name, + expectedFailure, failed, navigation, observations, screenshot })}\n`); + } + assert.equal(failed, expectedFailure, `${id}: ${mode}/${layout}`); + if (['inline', 'inline-loading', 'inline-no-toggle', 'inline-menu', 'modal', 'page'].includes(layout)) { + assert.equal(navigation.catalog, 0, 'a visible notification destination needs no catalog navigation'); + assert.equal(navigation.settings, 0, 'a visible notification destination needs no settings navigation'); + assert.equal(navigation.account, 0, 'a visible feed or opener needs no account navigation'); + } + if (['menu-toggle', 'menu-settings'].includes(layout)) assert.ok(navigation.account > 0, 'reach the feed through the account menu'); + if (layout === 'inline-loading') { + assert.equal(await actors.get('subscriber-fresh')!.page.locator('body') + .getAttribute('data-toggle-clicked'), null, 'an open loading panel must not be toggled closed'); + } + if (restocked) { + assert(readsAfterRestock > 0, 'delivery must be read again, without a push update'); + assert(freshClients >= 2, 'the second observation must use another independent client'); + assert.equal(waitMs, 10000, 'delayed delivery or persistent duplicates are sampled after the authored interval'); + } + } finally { await Promise.all(contexts.map(context => context.close())); } + } + } finally { await browser.close(); } +}); diff --git a/tools/stack-bench/tests/stock-read-backends.test.ts b/tools/stack-bench/tests/stock-read-backends.test.ts new file mode 100644 index 00000000000..726260171dc --- /dev/null +++ b/tools/stack-bench/tests/stock-read-backends.test.ts @@ -0,0 +1,195 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { runInNewContext } from 'node:vm'; +import { getMongoDbStock, setMongoDbStock } from '../src/stacks/backends/mongodb-operations.js'; +import { getSpacetimeStock } from '../src/stacks/backends/spacetime-operations.js'; +import { getSupabaseStock } from '../src/stacks/backends/supabase-operations.js'; +import { supabaseAdapter } from '../src/stacks/backends/supabase-adapter.js'; +import { supabaseExec, supabaseLease } from './helpers/supabase-lease.js'; + +const container = { name: 'leased', id: 'leased-id' }; +const lease = { resources: { container, database: 'app' } }; +const spacetime = { buildContainer: container, mod: 'app', containerUri: 'http://database:3000' }; +const table = (columns: string[], rows: unknown[][]): string => + [columns.join(' | '), columns.map(() => '---').join('+'), ...rows.map(row => row.join(' | '))].join('\n'); +const interfaceFailure = (error: unknown): boolean => error instanceof Error + && 'stockInterface' in error && error.stockInterface === true; + +class ObjectId { + readonly _bsontype = 'ObjectId'; + constructor(readonly value: string) {} + toHexString(): string { return this.value; } +} + +type Document = Record; +function mongoExec(data: Record) { + return (_command: string, args: readonly string[]): string => { + if (args[0] === 'inspect') return container.id; + const equal = (a: unknown, b: unknown): boolean => a instanceof ObjectId && b instanceof ObjectId + ? a.value === b.value : a === b; + const match = (row: Document, query: Document): boolean => Object.entries(query).every(([key, value]) => { + if (key === '$or') return (value as Document[]).some(part => match(row, part)); + if (value && typeof value === 'object' && '$in' in value) { + return (value.$in as unknown[]).some(candidate => equal(row[key], candidate)); + } + return equal(row[key], value); + }); + const db = Object.fromEntries(Object.entries(data).map(([name, rows]) => [name, { + find: (query: Document) => { + const found = rows.filter(row => match(row, query)); + return { toArray: () => found, limit: (count: number) => ({ toArray: () => found.slice(0, count) }) }; + }, + updateOne: (query: Document, update: { $set: Document }) => { + const found = rows.find(row => match(row, query)); + if (found) Object.assign(found, update.$set); + return { matchedCount: found ? 1 : 0 }; + }, + }])); + let output = ''; + const stopped = { code: 0 }; + try { + runInNewContext(args.at(-1)!, { db: { ...db, getCollectionNames: () => Object.keys(data) }, + ObjectId, print: (value: string) => { output += value; }, + quit: (code = 0) => { stopped.code = code; throw stopped; } }); + } catch (error) { if (error !== stopped) throw error; } + // mongosh exits non-zero after quit(1); the child process error carries stdout. + if (stopped.code) throw Object.assign(new Error('mongosh exited 1'), { stdout: output }); + return output; + }; +} + +test('MongoDB stock reads preserve ObjectId/string references, zero and negative values', () => { + const itemId = '0123456789abcdef01234567'; + const exec = mongoExec({ item: [{ _id: new ObjectId(itemId), name: 'Widget' }], + warehouse: [{ _id: 1, name: 'East' }, { _id: 2, name: 'West' }], + stock: [{ item_id: itemId, warehouse_id: 1, quantity: 0 }, + { item_id: new ObjectId(itemId), warehouse_id: 2, quantity: -3 }] }); + assert.equal(getMongoDbStock({ item: 'Widget', warehouse: 'East', lease, exec }).quantity, 0); + assert.equal(getMongoDbStock({ item: 'Widget', lease, exec }).quantity, -3); +}); + +test('MongoDB stock reads and writes use only the declared item_id/warehouse_id fields', () => { + const data = () => ({ item: [{ id: 1, name: 'Widget' }], warehouse: [{ id: 2, name: 'East' }], + stock: [{ _id: 's', itemId: 1, warehouseId: 2, quantity: 3 }] as Document[] }); + const missingStock = (error: unknown): boolean => interfaceFailure(error) + && (error as { missingRow?: unknown }).missingRow === 'stock'; + assert.throws(() => getMongoDbStock({ item: 'Widget', warehouse: 'East', lease, exec: mongoExec(data()) }), missingStock); + assert.throws(() => getMongoDbStock({ item: 'Widget', lease, exec: mongoExec(data()) }), missingStock); + const undeclared = data(); + assert.throws(() => setMongoDbStock({ item: 'Widget', warehouse: 'East', quantity: 9, lease, + exec: mongoExec(undeclared) }), missingStock); + assert.equal(undeclared.stock[0]!.quantity, 3); + // A declared row without its warehouse_id is not read through a camelCase fallback. + const partial = { ...data(), stock: [{ item_id: 1, warehouseId: 2, quantity: 3 }] }; + assert.throws(() => getMongoDbStock({ item: 'Widget', lease, exec: mongoExec(partial) }), + (error: unknown) => interfaceFailure(error) && (error as { stockInterfaceInvalid?: unknown }).stockInterfaceInvalid === true); +}); + +test('MongoDB stock writes refuse ambiguous rows and non-numeric quantities as interface failures', () => { + const valid = () => ({ item: [{ id: 1, name: 'Widget' }], warehouse: [{ id: 2, name: 'East' }], + stock: [{ _id: 's', item_id: 1, warehouse_id: 2, quantity: 3 }] as Document[] }); + const written = valid(); + assert.deepEqual(setMongoDbStock({ item: 'Widget', warehouse: 'East', quantity: 0, lease, exec: mongoExec(written) }), + { backend: 'mongodb', item: 'Widget', warehouse: 'East', quantity: 0 }); + assert.equal(written.stock[0]!.quantity, 0); + const invalid = (error: unknown): boolean => interfaceFailure(error) + && (error as { stockInterfaceInvalid?: unknown }).stockInterfaceInvalid === true; + for (const [defect, data] of Object.entries({ + 'duplicate item': { ...valid(), item: [{ id: 1, name: 'Widget' }, { id: 3, name: 'Widget' }] }, + 'duplicate warehouse': { ...valid(), warehouse: [{ id: 2, name: 'East' }, { id: 4, name: 'East' }] }, + 'duplicate stock': { ...valid(), stock: [...valid().stock, { _id: 't', item_id: 1, warehouse_id: 2, quantity: 1 }] }, + 'absent quantity': { ...valid(), stock: [{ _id: 's', item_id: 1, warehouse_id: 2 }] }, + 'text quantity': { ...valid(), stock: [{ _id: 's', item_id: 1, warehouse_id: 2, quantity: '3' }] }, + 'fractional quantity': { ...valid(), stock: [{ _id: 's', item_id: 1, warehouse_id: 2, quantity: 0.5 }] }, + })) { + const before = JSON.stringify(data.stock); + assert.throws(() => setMongoDbStock({ item: 'Widget', warehouse: 'East', quantity: 9, lease, + exec: mongoExec(data) }), invalid, defect); + assert.equal(JSON.stringify(data.stock), before, `${defect} must not be written`); + } +}); + +test('MongoDB stock reads reject missing, duplicate and invalid data', () => { + const valid = { item: [{ id: 1, name: 'Widget' }], warehouse: [{ id: 2, name: 'East' }], + stock: [{ item_id: 1, warehouse_id: 2, quantity: 3 }] }; + for (const data of [ + { ...valid, item: [] }, { ...valid, item: [...valid.item, ...valid.item] }, + { ...valid, warehouse: [...valid.warehouse, ...valid.warehouse] }, + { ...valid, stock: [] }, { ...valid, stock: [...valid.stock, ...valid.stock] }, + ...[null, '3', 0.5, Number.MAX_SAFE_INTEGER + 1].map(quantity => + ({ ...valid, stock: [{ ...valid.stock[0], quantity }] })), + ]) assert.throws(() => getMongoDbStock({ item: 'Widget', warehouse: 'East', lease, + exec: mongoExec(data) }), interfaceFailure); +}); + +function spacetimeExec(stock: unknown[][], itemRows = [['1']], warehouseRows = [['2'], ['3']]) { + return (_command: string, args: readonly string[]): string => { + if (args[0] === 'inspect') return container.id; + const sql = args.at(-1)!; + if (sql.startsWith('select id from item')) return table(['id'], itemRows); + if (sql.startsWith('select id from warehouse')) { + return table(['id'], sql.includes('where name') ? warehouseRows.slice(0, 1) : warehouseRows); + } + assert.match(sql, /^select warehouse_id, quantity from stock/); + return table(['warehouse_id', 'quantity'], stock); + }; +} + +test('SpacetimeDB stock reads parse complete rows and preserve zero and negative values', () => { + assert.equal(getSpacetimeStock({ item: 'Widget', warehouse: 'East', spacetime, + exec: spacetimeExec([[2, 0]]) }).quantity, 0); + assert.equal(getSpacetimeStock({ item: 'Widget', spacetime, + exec: spacetimeExec([[2, 3], [3, -5]]) }).quantity, -2); +}); + +test('SpacetimeDB stock reads reject missing, duplicate, malformed and unsafe rows', () => { + for (const stock of [[], [[2, 1], [2, 2]], [[99, 1]], [[2, 'NaN']], [[2, 0.5]], + [[2, Number.MAX_SAFE_INTEGER + 1]]]) { + assert.throws(() => getSpacetimeStock({ item: 'Widget', spacetime, + exec: spacetimeExec(stock) }), interfaceFailure); + } + for (const ids of [[], [['1'], ['2']]]) { + assert.throws(() => getSpacetimeStock({ item: 'Widget', spacetime, + exec: spacetimeExec([[2, 1]], ids) }), interfaceFailure); + } + assert.throws(() => getSpacetimeStock({ item: 'Widget', spacetime, + exec: (_command, args) => args[0] === 'inspect' ? container.id : 'id\n999' }), /invalid table header/); +}); + +test('stock readers reject a replaced container before querying', () => { + let calls = 0; + const exec = (_command: string, args: readonly string[]): string => { + calls += 1; + assert.equal(args[0], 'inspect'); + return 'different-id'; + }; + assert.throws(() => getMongoDbStock({ item: 'Widget', lease, exec }), /changed after lease/); + assert.throws(() => getSpacetimeStock({ item: 'Widget', spacetime, exec }), /changed after lease/); + assert.equal(calls, 2); +}); + +test('Supabase stock reads run the relational stock SQL as the privileged role', () => { + const platform = supabaseLease(); + const lease = supabaseAdapter.grading.databaseLease(platform); + const calls: { args: readonly string[] }[] = []; + const read = (output: unknown, warehouse?: string) => getSupabaseStock({ item: "Kid's Keyboard", warehouse, lease, + exec: supabaseExec(platform, sql => { assert.match(sql, /'Kid''s Keyboard'/); return JSON.stringify(output); }, calls) }); + assert.deepEqual(read({ items: 1, warehouses: 2, quantities: [0, -3] }), + { backend: 'supabase', item: "Kid's Keyboard", quantity: -3 }); + const psql = calls.find(call => call.args.includes('psql'))!.args; + assert.deepEqual(psql.slice(psql.indexOf('-U'), psql.indexOf('-U') + 4), ['-U', 'supabase_admin', '-d', 'postgres']); + assert.throws(() => read({ items: 0, namedWarehouses: 1, warehouses: 0, quantities: [] }, 'East'), + (error: unknown) => interfaceFailure(error) && (error as { missingRow?: unknown }).missingRow === 'item'); + const missing = Object.assign(new Error('psql failed'), { stderr: 'ERROR: relation "public.stock" does not exist' }); + assert.throws(() => getSupabaseStock({ item: 'Keyboard', lease, + exec: supabaseExec(platform, () => { throw missing; }) }), interfaceFailure); +}); + +test('Supabase observers refuse a replaced platform container', () => { + const platform = supabaseLease(); + const exec = supabaseExec(platform, () => { throw new Error('must not query'); }); + const replaced: typeof exec = (command, args, options) => args[0] === 'inspect' + ? JSON.stringify({ Id: 'd'.repeat(64), State: { Running: true } }) : exec(command, args, options); + assert.throws(() => getSupabaseStock({ item: 'Keyboard', lease: supabaseAdapter.grading.databaseLease(platform), + exec: replaced }), /changed after lease creation/); +}); diff --git a/tools/stack-bench/tests/stock-read-postgres.test.ts b/tools/stack-bench/tests/stock-read-postgres.test.ts new file mode 100644 index 00000000000..6920a02aa5f --- /dev/null +++ b/tools/stack-bench/tests/stock-read-postgres.test.ts @@ -0,0 +1,76 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { getPostgresStock } from '../src/stacks/backends/postgres-operations.js'; +import type { TextCommandExecutor } from '../src/runtime/command-executor.js'; + +const lease = { resources: { database: 'bench', container: { name: 'leased-pg', id: 'pg-id' } } }; + +test('stock reads use the leased database and preserve zero and negative observations', () => { + for (const quantities of [[0], [-2], [4, -1]]) { + let sql = ''; + const exec: TextCommandExecutor = (_command, args, options) => { + if (args[0] === 'inspect') return 'pg-id'; + assert(args.includes('pg-id')); + assert(args.includes('bench')); + sql = options.input ?? ''; + return JSON.stringify({ items: 1, warehouses: quantities.length, quantities }); + }; + const result = getPostgresStock({ item: "Kid's Keyboard", lease, exec }); + assert.equal(result.quantity, quantities.reduce((sum, n) => sum + n, 0)); + assert.match(sql, /Kid''s Keyboard/); + assert.doesNotMatch(sql, /\b(?:UPDATE|INSERT|DELETE)\b/); + } +}); + +test('stock reads refuse changed ownership and missing, ambiguous, or invalid data', () => { + assert.throws(() => getPostgresStock({ item: 'Keyboard', lease, exec: () => 'replacement-id' }), + /changed after lease/); + for (const [output, expected] of [ + ['', /no result/], + ['{}\n{}', /multiple relational/], + [JSON.stringify({ items: 2, warehouses: 1, quantities: [2] }), /ambiguous/], + [JSON.stringify({ items: 1, warehouses: 1, quantities: [2, 3] }), /ambiguous/], + [JSON.stringify({ items: 1, warehouses: 1, quantities: [null] }), /whole number/], + [JSON.stringify({ items: 1, warehouses: 1, quantities: ['2'] }), /whole number/], + [JSON.stringify({ items: 1, warehouses: 1, quantities: [1.5] }), /whole number/], + [JSON.stringify({ items: 1, warehouses: 2, quantities: [Number.MAX_SAFE_INTEGER, 1] }), /whole number/], + [JSON.stringify({ items: 1, warehouses: 3, quantities: [Number.MAX_SAFE_INTEGER, 2, -2] }), /whole number/], + ] as const) { + assert.throws(() => getPostgresStock({ item: 'Keyboard', lease, + exec: (_command, args) => args[0] === 'inspect' ? 'pg-id' : output }), expected); + } + let sql = ''; + assert.equal(getPostgresStock({ item: 'Keyboard', warehouse: "West's", lease, + exec: (_command, args, options) => { + if (args[0] === 'inspect') return 'pg-id'; + sql = options.input ?? ''; + return '{"items":1,"warehouses":1,"namedWarehouses":1,"quantities":[2]}'; + } }).quantity, 2); + assert.match(sql, /West''s/); +}); + +const stockError = (fields: { missingRow?: string; invalid?: boolean }) => (error: unknown): boolean => + error instanceof Error && 'stockInterface' in error && error.stockInterface === true + && ('missingRow' in error ? error.missingRow : undefined) === fields.missingRow + && ('stockInterfaceInvalid' in error && error.stockInterfaceInvalid === true) === (fields.invalid === true); + +test('stock reads name the missing parent and refuse ambiguous parents without stock', () => { + for (const [counts, expected] of [ + [{ items: 0, namedWarehouses: 1 }, { missingRow: 'item' }], + [{ items: 1, namedWarehouses: 0 }, { missingRow: 'warehouse' }], + [{ items: 2, namedWarehouses: 1 }, { invalid: true }], + [{ items: 1, namedWarehouses: 2 }, { invalid: true }], + [{ items: 1, namedWarehouses: 1 }, { missingRow: 'stock' }], + ] as const) { + assert.throws(() => getPostgresStock({ item: 'Keyboard', warehouse: 'East', lease, + exec: (_command, args) => args[0] === 'inspect' ? 'pg-id' + : JSON.stringify({ ...counts, warehouses: 0, quantities: [] }) }), stockError(expected)); + } + let sql = ''; + assert.throws(() => getPostgresStock({ item: 'Keyboard', lease, exec: (_command, args, options) => { + if (args[0] === 'inspect') return 'pg-id'; + sql = options.input ?? ''; + return JSON.stringify({ items: 1, namedWarehouses: 0, warehouses: 0, quantities: [] }); + } }), stockError({ missingRow: 'warehouse' })); + assert.doesNotMatch(sql, /HAVING/, 'an empty read still reports its parent counts'); +}); diff --git a/tools/stack-bench/tests/stock-read.integration.ts b/tools/stack-bench/tests/stock-read.integration.ts new file mode 100644 index 00000000000..8e873889f45 --- /dev/null +++ b/tools/stack-bench/tests/stock-read.integration.ts @@ -0,0 +1,141 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { setTimeout as delay } from 'node:timers/promises'; +import test from 'node:test'; +import { getPostgresStock, setPostgresStock } from '../src/stacks/backends/postgres-operations.js'; +import { getMongoDbStock } from '../src/stacks/backends/mongodb-operations.js'; +import { writePostgresStock } from '../src/stacks/postgres-sql.js'; + +const enabled = process.env.STACK_BENCH_STOCK_READ_DOCKER === '1'; +const docker = (args: string[], input?: string): string => execFileSync('docker', args, + { encoding: 'utf8', input, stdio: 'pipe', timeout: 30_000, windowsHide: true }).trim(); + +for (const backend of ['postgres', 'mongodb'] as const) { + test(`trusted ${backend} stock reads use real database results`, { + skip: enabled ? false : 'Set STACK_BENCH_STOCK_READ_DOCKER=1 for two isolated database checks', + timeout: 90_000, + }, async () => { + const name = `stack-bench-stock-read-${backend}-${randomUUID()}`; + const postgres = backend === 'postgres'; + const id = docker(['run', '--pull=never', '--rm', '-d', '--name', name, + '--network', 'none', '--cpus', '1', '--memory', '384m', + ...(postgres ? ['-e', 'POSTGRES_HOST_AUTH_METHOD=trust', '-e', 'POSTGRES_USER=appuser', + '-e', 'POSTGRES_DB=bench', 'postgres:16-alpine'] : ['mongo:7'])]); + const lease = { resources: { container: { name, id }, database: 'bench' } }; + const run = (source: string): string => postgres + ? docker(['exec', '-i', id, 'psql', '-U', 'appuser', '-d', 'bench', '-v', 'ON_ERROR_STOP=1', '-At'], source) + : docker(['exec', id, 'mongosh', 'bench', '--quiet', '--eval', source]); + try { + for (let attempt = 0; ; attempt++) { + try { run(postgres ? 'SELECT 1;' : 'db.runCommand({ping:1})'); break; } + catch (error) { if (attempt === 30) throw error; await delay(250); } + } + if (postgres) { + run(`CREATE TABLE item (id integer PRIMARY KEY, name text); +CREATE TABLE warehouse (id integer PRIMARY KEY, name text); +CREATE TABLE stock (item_id integer REFERENCES item, warehouse_id integer REFERENCES warehouse, quantity integer); +CREATE TABLE order_line (item_id integer REFERENCES item, warehouse_id integer REFERENCES warehouse, quantity integer); +INSERT INTO item VALUES (0, 'Kid''s Keyboard'); +INSERT INTO warehouse VALUES (1, 'East'), (2, 'West'); +INSERT INTO stock VALUES (0, 1, 5), (0, 2, -1); +INSERT INTO order_line VALUES (0, 1, 17);`); + assert.equal(getPostgresStock({ item: "Kid's Keyboard", lease }).quantity, 4); + assert.equal(getPostgresStock({ item: "Kid's Keyboard", warehouse: 'West', lease }).quantity, -1); + setPostgresStock({ item: "Kid's Keyboard", warehouse: 'East', quantity: 0, lease }); + assert.equal(getPostgresStock({ item: "Kid's Keyboard", warehouse: 'East', lease }).quantity, 0); + assert.equal(run('SELECT quantity FROM order_line;'), '17', 'stock setup must not alter order history'); + assert.throws(() => getPostgresStock({ item: 'Absent', lease }), /no stock data/); + assert.throws(() => setPostgresStock({ item: 'Absent', warehouse: 'East', quantity: 9, lease }), /required item/); + run("INSERT INTO item VALUES (1, 'Kid''s Keyboard'); INSERT INTO stock VALUES (1, 1, 4);"); + assert.throws(() => getPostgresStock({ item: "Kid's Keyboard", lease }), /ambiguous/); + assert.throws(() => setPostgresStock({ item: "Kid's Keyboard", warehouse: 'East', quantity: 9, lease }), /ambiguous/); + assert.equal(run('SELECT string_agg(quantity::text, \',\' ORDER BY item_id) FROM stock WHERE warehouse_id = 1;'), '0,4', + 'an ambiguous write must change no row'); + run('DELETE FROM stock WHERE item_id = 1;'); + run("DELETE FROM item WHERE id = 1; INSERT INTO warehouse VALUES (3, 'East');"); + assert.throws(() => getPostgresStock({ item: "Kid's Keyboard", warehouse: 'East', lease }), /ambiguous/); + run('DELETE FROM warehouse WHERE id = 3;'); + run('INSERT INTO stock VALUES (0, 1, 2);'); + assert.throws(() => getPostgresStock({ item: "Kid's Keyboard", lease }), /ambiguous/); + // The observable interface requires unambiguous links and committed + // writes, not a particular schema constraint or physical table layout. + // Failure cases: duplicate names/ids/holdings, dangling links, invalid + // quantities, read-only views, and acknowledged writes with no effect. + run('DELETE FROM stock WHERE quantity = 2; ALTER TABLE stock DROP CONSTRAINT stock_warehouse_id_fkey;'); + assert.equal(getPostgresStock({ item: "Kid's Keyboard", lease }).quantity, -1); + setPostgresStock({ item: "Kid's Keyboard", warehouse: 'East', quantity: 9, lease }); + assert.equal(run('SELECT quantity FROM stock WHERE warehouse_id = 1;'), '9'); + for (const quantity of [-1, 1.5, NaN, Infinity]) { + assert.throws(() => setPostgresStock({ item: "Kid's Keyboard", warehouse: 'East', quantity, lease })); + } + assert.equal(run('SELECT quantity FROM stock WHERE warehouse_id = 1;'), '9'); + + run(`ALTER TABLE stock RENAME TO inventory; +ALTER TABLE item RENAME TO products; +ALTER TABLE warehouse RENAME TO locations; +CREATE VIEW item AS SELECT id,name FROM products; +CREATE VIEW warehouse AS SELECT id,name FROM locations; +CREATE VIEW stock AS SELECT item_id,warehouse_id,quantity FROM inventory;`); + assert.equal(getPostgresStock({ item: "Kid's Keyboard", warehouse: 'East', lease }).quantity, 9); + setPostgresStock({ item: "Kid's Keyboard", warehouse: 'East', quantity: 7, lease }); + assert.equal(run('SELECT quantity FROM inventory WHERE warehouse_id = 1;'), '7'); + assert.equal(run('SELECT quantity FROM order_line;'), '17'); + const quietWrite = (quantity: number) => writePostgresStock(source => + docker(['exec', '-i', id, 'psql', '-U', 'appuser', '-d', 'bench', '-v', 'ON_ERROR_STOP=1', '-qAt'], source), + { backend: 'supabase', label: 'Supabase', quiet: true }, { item: "Kid's Keyboard", warehouse: 'East', quantity }); + quietWrite(8); + assert.equal(run('SELECT quantity FROM inventory WHERE warehouse_id = 1;'), '8'); + quietWrite(7); + + const rejectInterface = (error: unknown) => error instanceof Error + && 'stockInterface' in error && error.stockInterface === true; + for (const [table, source] of [['item', 'products'], ['warehouse', 'locations']] as const) { + run(`DROP VIEW ${table}; CREATE VIEW ${table} AS SELECT id,name FROM ${source} +UNION ALL SELECT id,'Different name' FROM ${source} WHERE id=${table === 'item' ? 0 : 1};`); + assert.throws(() => getPostgresStock({ item: "Kid's Keyboard", warehouse: 'East', lease }), rejectInterface); + assert.throws(() => setPostgresStock({ item: "Kid's Keyboard", warehouse: 'East', quantity: 99, lease }), rejectInterface); + assert.equal(run('SELECT quantity FROM inventory WHERE warehouse_id = 1;'), '7'); + run(`DROP VIEW ${table}; CREATE VIEW ${table} AS SELECT id,name FROM ${source};`); + } + run('INSERT INTO inventory VALUES (0, 99, 2);'); + assert.throws(() => getPostgresStock({ item: "Kid's Keyboard", lease }), rejectInterface); + run('DELETE FROM inventory WHERE warehouse_id=99; DROP VIEW stock; CREATE VIEW stock AS SELECT item_id,warehouse_id,quantity::numeric+0.5 AS quantity FROM inventory;'); + assert.throws(() => getPostgresStock({ item: "Kid's Keyboard", lease }), rejectInterface); + run('DROP VIEW stock; CREATE VIEW stock AS SELECT item_id,warehouse_id,sum(quantity)::integer AS quantity FROM inventory GROUP BY item_id,warehouse_id;'); + assert.throws(() => setPostgresStock({ item: "Kid's Keyboard", warehouse: 'East', quantity: 99, lease }), rejectInterface); + assert.throws(() => quietWrite(99), rejectInterface); + assert.equal(run('SELECT quantity FROM inventory WHERE warehouse_id = 1;'), '7'); + run(`DROP VIEW stock; CREATE VIEW stock AS SELECT item_id,warehouse_id,quantity FROM inventory; +CREATE FUNCTION ignore_stock_write() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RETURN NEW; END $$; +CREATE TRIGGER ignore_stock_write INSTEAD OF UPDATE ON stock FOR EACH ROW EXECUTE FUNCTION ignore_stock_write();`); + assert.throws(() => setPostgresStock({ item: "Kid's Keyboard", warehouse: 'East', quantity: 99, lease }), rejectInterface); + assert.throws(() => quietWrite(99), rejectInterface); + assert.equal(run('SELECT quantity FROM inventory WHERE warehouse_id = 1;'), '7'); + run('DROP TRIGGER ignore_stock_write ON stock; DROP VIEW stock;'); + assert.throws(() => getPostgresStock({ item: "Kid's Keyboard", lease }), rejectInterface); + assert.throws(() => setPostgresStock({ item: "Kid's Keyboard", warehouse: 'East', quantity: 1, lease }), rejectInterface); + run('CREATE VIEW stock AS SELECT item_id,warehouse_id,quantity FROM inventory;'); + const unavailable = (_command: string, args: readonly string[]): string => { + if (args[0] === 'inspect') return id; + throw new Error('database connection refused'); + }; + assert.throws(() => getPostgresStock({ item: "Kid's Keyboard", lease, exec: unavailable }), + error => error instanceof Error && !('stockInterface' in error)); + } else { + run(`const itemId = ObjectId('0123456789abcdef01234567'); +db.item.insertOne({_id:itemId, name:"Kid's Keyboard"}); +db.warehouse.insertMany([{id:0,name:'East'},{id:2,name:'West'}]); +db.stock.insertMany([{item_id:itemId.toHexString(),warehouse_id:0,quantity:5}, + {item_id:itemId,warehouse_id:2,quantity:-1}]);`); + assert.equal(getMongoDbStock({ item: "Kid's Keyboard", lease }).quantity, 4); + assert.equal(getMongoDbStock({ item: "Kid's Keyboard", warehouse: 'West', lease }).quantity, -1); + run('db.stock.updateOne({warehouse_id:0},{$set:{quantity:0}})'); + assert.equal(getMongoDbStock({ item: "Kid's Keyboard", warehouse: 'East', lease }).quantity, 0); + assert.throws(() => getMongoDbStock({ item: 'Absent', lease }), /missing/); + run("db.stock.insertOne({item_id:'0123456789abcdef01234567',warehouse_id:0,quantity:2})"); + assert.throws(() => getMongoDbStock({ item: "Kid's Keyboard", lease }), /duplicated/); + } + } finally { docker(['rm', '-f', id]); } + }); +} diff --git a/tools/stack-bench/tests/suite-recovery.integration.ts b/tools/stack-bench/tests/suite-recovery.integration.ts new file mode 100644 index 00000000000..9dc11a9f454 --- /dev/null +++ b/tools/stack-bench/tests/suite-recovery.integration.ts @@ -0,0 +1,118 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { createServer } from 'node:http'; +import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { basename, join } from 'node:path'; +import { promisify } from 'node:util'; +import test from 'node:test'; +import { STACK_BENCH_ROOT, compiledEntrypoint } from '../src/package-root.js'; +import { hashAppSource } from '../src/runtime/source-snapshot.js'; +import { readArtifactPayload } from '../src/evidence/artifacts.js'; +import type { GradeBundlePayload } from '../src/evidence/benchmark-run.js'; +import { classifyBundle } from '../src/evidence/outcomes.js'; + +// Exercise the actual runner, browser, grader processes, evidence writer and +// outcome consumer. The stub has no mutable database; backend resets need their +// existing native lifecycle checks as well. Keep the receipt outside the fixture. +for (const mode of ['persistent', 'once'] as const) test(`isolated recovery preserves other failures with a ${mode} navigation timeout`, { + timeout: 180_000, +}, async () => { + const root = mkdtempSync(join(STACK_BENCH_ROOT, 'tracks', 'recovery-test-')); + const evidence = process.env.STACK_BENCH_RECOVERY_EVIDENCE_DIR + ? join(process.env.STACK_BENCH_RECOVERY_EVIDENCE_DIR, mode) : undefined; + const walkDirectory = join(STACK_BENCH_ROOT, 'dist', 'tracks', basename(root)); + let stallRequests = 0; + let unselectedWrites = 0; + const visits: Record = {}; + const server = createServer((request, response) => { + if (request.url === '/record/unselected' && request.method === 'POST') { + unselectedWrites++; + response.end('written'); return; + } + if (request.url?.startsWith('/record/')) { + const name = request.url.slice('/record/'.length); + visits[name] = (visits[name] ?? 0) + 1; + response.end('ok'); return; + } + if (request.url === '/stall.css') return; + const stall = request.headers.cookie?.includes('stall=1'); + if (stall) stallRequests++; + response.writeHead(200, { 'Content-Type': 'text/html', 'Set-Cookie': 'stall=; Max-Age=0; Path=/' }); + response.end(`${stall && (mode === 'persistent' || stallRequests === 1) ? '' : ''} +

Fixture

ready + ${['good', 'bad', 'stall'].map(name => ``).join('')}`); + }); + const receipt: Record = { result: 'running', + command: 'node --test dist/tests/suite-recovery.integration.js', + note: `${mode} stylesheet stall; two completed suites must not repeat. No model calls.` }; + try { + mkdirSync(walkDirectory, { recursive: true }); + cpSync(join(STACK_BENCH_ROOT, 'dist', 'tracks', 'loop', 'walk.js'), join(walkDirectory, 'walk.js')); + mkdirSync(join(root, 'contracts')); + mkdirSync(join(root, 'scenarios')); + mkdirSync(join(root, 'app')); + writeFileSync(join(root, 'app', 'fixture.txt'), 'immutable recovery fixture'); + writeFileSync(join(root, 'contracts', '01-recovery.json'), JSON.stringify({ level: 1, + hooks: [{ id: 'app-title', element: 'title', stage: 'landing', check: 'visible' }] })); + writeFileSync(join(root, 'track.json'), JSON.stringify({ schemaVersion: 1, title: 'Recovery fixture', + slug: 'recovery-test', internal: true, validatedThrough: 1, plannedThrough: 1, + portOffset: 600, restartProbe: '/', suites: { '1': ['good', 'bad', 'stall'].map(id => ({ + id, inherit: 'none', spec: `scenarios/${id}.json` })) }, + actions: [{ id: 'unselected', path: '/record/unselected', reducer: 'unselected', args: [] }] })); + for (const name of ['good', 'bad', 'stall']) { + writeFileSync(join(root, 'scenarios', `${name}.json`), JSON.stringify({ schemaVersion: 1, level: 1, + features: [{ id: 1, name, actors: ['viewer'], setup: [], criteria: [{ id: name, desc: name, points: 1, + steps: [{ do: 'click', actor: 'viewer', testid: name, settleMs: 100 }, + ...(name === 'stall' ? [{ do: 'reload', actor: 'viewer', settleMs: 0 }] : []), + { do: 'expect', actor: 'viewer', testid: name === 'bad' ? 'missing' : 'ready', within: 100 }] }] }] })); + } + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); assert(address && typeof address !== 'string'); + const source = hashAppSource(join(root, 'app')); + const argv = [compiledEntrypoint('commands', 'run-suite.js'), '--app', join(root, 'app'), + '--url', `http://127.0.0.1:${address.port}`, '--backend', 'stub', '--label', 'recovery-test', + '--track', basename(root), '--out', join(root, 'results'), '--source-sha256', source.sha256, + '--no-media', '--no-reset', '--retry-inconclusive']; + receipt.source = source.sha256; + const result = await promisify(execFile)(process.execPath, argv, + { encoding: 'utf8', timeout: 150_000, maxBuffer: 8 * 1024 * 1024 }); + receipt.stdout = result.stdout; + const bundle = readArtifactPayload(join(root, 'results', 'bundle.json'), { expectedKind: 'grade_bundle' }); + receipt.bundle = bundle; + assert.deepEqual(visits, { good: 1, bad: 1, stall: 2 }); + assert.equal(unselectedWrites, 0, 'unselected actions must not write to the app'); + assert.equal(stallRequests, 2); + assert.equal(classifyBundle(bundle).kind, mode === 'persistent' ? 'inconclusive' : 'app_failure'); + assert.equal(bundle.suites?.good?.total, 1); + assert.equal(bundle.suites?.bad?.total, 0); + assert.equal(bundle.suites?.lint?.pass, true); + assert.equal(bundle.suites?.stall?.total, mode === 'persistent' ? 0 : 1); + assert.equal(bundle.totals?.max, 3); + assert.deepEqual(Object.keys(bundle.suiteRetries ?? {}), ['stall']); + const initial = JSON.parse(readFileSync(join(root, 'results', 'suite-retries', 'stall', 'initial', 'grading-stall.json'), 'utf8')); + assert.equal(initial.payload.features[0].criteria[0].evidence.status, 'inconclusive'); + receipt.result = 'passed'; + } catch (error) { + if (error && typeof error === 'object') { + if ('stdout' in error) receipt.stdout = error.stdout; + if ('stderr' in error) receipt.stderr = error.stderr; + } + receipt.result = 'failed'; receipt.error = String(error); throw error; + } finally { + server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + receipt.visits = visits; + receipt.unselectedWrites = unselectedWrites; + receipt.stallRequests = stallRequests; + if (evidence) { + mkdirSync(evidence, { recursive: true }); + // Preserve raw reports and diagnostics, including a failed execution. + if (existsSync(join(root, 'results'))) cpSync(join(root, 'results'), join(evidence, 'results'), { recursive: true }); + writeFileSync(join(evidence, 'receipt.json'), JSON.stringify(receipt, null, 2)); + } + rmSync(root, { recursive: true, force: true }); + rmSync(walkDirectory, { recursive: true, force: true }); + } +}); diff --git a/tools/stack-bench/tests/supabase-operations.test.ts b/tools/stack-bench/tests/supabase-operations.test.ts new file mode 100644 index 00000000000..979c40bf925 --- /dev/null +++ b/tools/stack-bench/tests/supabase-operations.test.ts @@ -0,0 +1,174 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import test from 'node:test'; +import { bindBrowserRequest } from '../src/actions/named-action-runtime.js'; +import type { Actor } from '../src/actions/actor-action-runtime.js'; +import { proveSupabaseUse, supabaseAuthReadEndpoints, supabaseAuthRequestPatch, supabaseNamedActionRequest, + supabasePassiveSocketFrame, supabaseSignatureRefusal, supabaseWriteEndpoints } from '../src/stacks/backends/supabase-operations.js'; +import { SUPABASE_GATEWAY, SUPABASE_SECRETS, supabaseExec, supabaseLease, + withSupabaseLeaseEnvironment } from './helpers/supabase-lease.js'; + +const restock = { id: 'restock', path: '/api/admin/restock', reducer: 'admin_restock', args: [0, 0, 1], + params: [{ name: 'itemId', in: 'body' as const }, { name: 'warehouseId', in: 'body' as const }, + { name: 'quantity', in: 'body' as const }] }; + +test('Supabase named actions call public functions through the data API with the project key', () => { + const lease = supabaseLease(); + const exec = supabaseExec(lease, () => { throw new Error('named actions never use privileged SQL'); }); + const request = supabaseNamedActionRequest({ action: restock, input: { values: { itemId: '42', warehouseId: 7, quantity: 2 } }, + url: 'http://127.0.0.1:5173', lease, exec })!; + assert.equal(request.url, `${SUPABASE_GATEWAY}/rest/v1/rpc/admin_restock`); + assert.equal(request.method, 'POST'); + assert.deepEqual(JSON.parse(request.body), { itemId: '42', warehouseId: 7, quantity: 2 }); + assert.deepEqual(request.headers, { apikey: SUPABASE_SECRETS.anonKey, Authorization: `Bearer ${SUPABASE_SECRETS.anonKey}` }); + assert.equal(request.missingNote, 'no database function public.admin_restock'); + // Positional declared defaults are named by their parameters; path parameters are ordinary arguments. + const buy = { id: 'buy', path: '/api/items/:id/buy', reducer: 'buy_now', args: [0], + params: [{ name: 'itemId', in: 'path' as const, placeholder: ':id' }] }; + assert.deepEqual(JSON.parse(supabaseNamedActionRequest({ action: buy, input: {}, lease, exec })!.body), { itemId: 0 }); + assert.deepEqual(JSON.parse(supabaseNamedActionRequest({ action: restock, input: { args: [1, 2, 3] }, lease, exec })!.body), + { itemId: 1, warehouseId: 2, quantity: 3 }); + assert.throws(() => supabaseNamedActionRequest({ action: { ...restock, params: [] }, input: { args: [1] }, lease, exec }), + (error: unknown) => (error as { code?: unknown }).code === 'invalid_named_action_input'); + assert.equal(supabaseNamedActionRequest({ action: { id: 'ship', path: '/api/ship' }, lease, exec }), null); + assert.equal(supabaseNamedActionRequest({ action: { ...restock, reducer: 'a/b' }, input: {}, lease, exec })!.url, + `${SUPABASE_GATEWAY}/rest/v1/rpc/a%2Fb`); +}); + +test('platform headers go with every named call and the caller credential replaces the anonymous bearer', () => { + const lease = supabaseLease(); + const request = supabaseNamedActionRequest({ action: restock, input: {}, lease, + exec: supabaseExec(lease, () => '') })!; + const actor = { name: 'staff', page: {}, writes: [] } as unknown as Actor; + const signedIn = bindBrowserRequest(actor, request, { Authorization: 'Bearer staff-token' })(); + assert.deepEqual(signedIn.headers, { apikey: SUPABASE_SECRETS.anonKey, Authorization: 'Bearer staff-token' }); + // A differently cased caller header replaces the platform default instead of joining it. + const lower = bindBrowserRequest(actor, request, {})({ 'Content-Type': 'application/json', authorization: 'Bearer x' }); + assert.deepEqual(lower.headers, { apikey: SUPABASE_SECRETS.anonKey, 'Content-Type': 'application/json', + authorization: 'Bearer x' }); + const anonymous = bindBrowserRequest(actor, request, {})({ 'Content-Type': 'application/json' }); + assert.equal(anonymous.headers.Authorization, `Bearer ${SUPABASE_SECRETS.anonKey}`); + assert.equal(anonymous.body, request.body); + // Requests without platform headers are unchanged. + assert.deepEqual(bindBrowserRequest(actor, { url: 'http://app.test/api', body: '{}' }, { Cookie: 'sid=1' })(), + { headers: { Cookie: 'sid=1' }, body: '{}' }); +}); + +test('named actions resolve the gateway and project key from the authenticated lease', t => { + const lease = supabaseLease(); + withSupabaseLeaseEnvironment(t, lease); + const calls: { args: readonly string[] }[] = []; + const request = supabaseNamedActionRequest({ action: restock, input: {}, exec: supabaseExec(lease, () => '', calls) })!; + assert.equal(request.url, `${SUPABASE_GATEWAY}/rest/v1/rpc/admin_restock`); + assert.deepEqual(calls.map(call => call.args[0]), ['inspect', 'exec'], 'the anchor is verified before its key is read'); +}); + +test('only data API and Edge Function requests to the leased gateway are application writes', () => { + const lease = supabaseLease(); + const endpoints = supabaseWriteEndpoints(lease); + const write = (url: string) => endpoints.some(endpoint => url.startsWith(endpoint)); + assert(write(`${SUPABASE_GATEWAY}/rest/v1/rpc/add_to_cart`)); + assert(write(`${SUPABASE_GATEWAY}/rest/v1/orders?select=*`)); + assert(write(`${SUPABASE_GATEWAY}/functions/v1/checkout`)); + for (const url of [`${SUPABASE_GATEWAY}/auth/v1/token?grant_type=password`, `${SUPABASE_GATEWAY}/storage/v1/object`, + 'http://127.0.0.1:5173/rest/v1/rpc/add_to_cart', 'http://127.0.0.1:13411/rest/v1/rpc/x']) { + assert(!write(url), url); + } +}); + +test('Supabase Auth password requests are found by endpoint whatever credential encoding the application sends', () => { + const patch = supabaseAuthRequestPatch(supabaseLease()); + // The reference sends a hex-encoded username email and a digest of the typed password. + const email = `${Buffer.from('claimant').toString('hex')}@accounts.invalid`; + const digest = createHash('sha256').update('secret').digest('hex'); + const signUp = { email, password: digest, data: { username: 'claimant' }, gotrue_meta_security: {} }; + const fields = JSON.parse('{"role":"admin","__proto__":{"polluted":true}}'); + const changed = patch(`${SUPABASE_GATEWAY}/auth/v1/signup`, signUp, { fields })!; + assert.equal(changed.shape, 'supabase-auth'); + assert.deepEqual(JSON.parse(changed.body), JSON.parse(JSON.stringify({ email, password: digest }).slice(0, -1) + + ',"data":{"username":"claimant","role":"admin","__proto__":{"polluted":true}},"gotrue_meta_security":{},' + + '"role":"admin","__proto__":{"polluted":true}}')); + assert.equal(({} as { polluted?: unknown }).polluted, undefined); + assert.deepEqual(signUp.data, { username: 'claimant' }, 'the application request is not modified in place'); + const signIn = { email, password: digest, gotrue_meta_security: {} }; + const token = `${SUPABASE_GATEWAY}/auth/v1/token?grant_type=password`; + assert.deepEqual(JSON.parse(patch(token, signIn, { fields: { role: 'admin' } })!.body), + { ...signIn, role: 'admin', data: { role: 'admin' } }); + // A query-like password replaces whatever the application derived from the typed one. + assert.deepEqual(JSON.parse(patch(token, signIn, { password: "' OR '1'='1" })!.body), + { ...signIn, password: "' OR '1'='1" }, 'a password change adds no metadata'); + // The reference now sends the typed password itself, up to 72 UTF-8 bytes. + const typed = `${'界'.repeat(23)}x-A`; + assert.equal(Buffer.byteLength(typed), 72); + const raw = { email, password: typed, gotrue_meta_security: {} }; + assert.deepEqual(JSON.parse(patch(`${SUPABASE_GATEWAY}/auth/v1/signup`, raw, { fields: { role: 'admin' } })!.body), + { ...raw, role: 'admin', data: { role: 'admin' } }); + assert.deepEqual(JSON.parse(patch(token, raw, { password: "' OR '1'='1" })!.body), { ...raw, password: "' OR '1'='1" }); + // An application can serve Auth through its own origin, behind a prefix. + for (const url of ['http://127.0.0.1:5173/auth/v1/signup', 'http://localhost:8124/supabase/auth/v1/token?grant_type=password']) { + assert.equal(patch(url, signIn, { fields })!.shape, 'supabase-auth', url); + } + for (const url of [`${SUPABASE_GATEWAY}/auth/v1/token?grant_type=refresh_token`, `${SUPABASE_GATEWAY}/rest/v1/rpc/signup`, + 'http://example.com/auth/v1/signup', 'http://localhost:8124/auth/v1/signup/extra']) { + assert.equal(patch(url, signUp, { fields }), undefined, url); + } + for (const body of [{ email }, { ...signIn, password: 7 }, ['secret'], null]) { + assert.equal(patch(token, body, { fields }), null, JSON.stringify(body)); + } + assert.throws(() => patch(token, { ...signUp, data: ['x'] }, { fields }), /metadata is not an object/); +}); + +test('read-only functions are not sign-up writes, and unmatched arguments refuse only an existing function', () => { + const lease = supabaseLease(); + let sql = ''; + const exec = (output: string) => supabaseExec(lease, input => { sql = input; return output; }); + assert.deepEqual(supabaseAuthReadEndpoints(lease, exec('shop_state\nprogression_state\n')), + [`${SUPABASE_GATEWAY}/rest/v1/rpc/shop_state`, `${SUPABASE_GATEWAY}/rest/v1/rpc/progression_state`]); + assert.match(sql, /having bool_and\(p\.provolatile in \('s', 'i'\)\)/, 'a volatile overload keeps the name a write'); + const unmatched = { status: 404, text: JSON.stringify({ code: 'PGRST202', message: 'Could not find the function' }) }; + const call = { url: `${SUPABASE_GATEWAY}/rest/v1/rpc/checkout` }; + assert.equal(supabaseSignatureRefusal(call, unmatched, { lease, exec: exec('1\n') }), true); + assert.match(sql, /p\.proname = 'checkout'/); + assert.equal(supabaseSignatureRefusal(call, unmatched, { lease, exec: exec('0\n') }), false, 'the function does not exist'); + const unused = { lease, exec: supabaseExec(lease, () => { throw new Error('no lookup'); }) }; + assert.equal(supabaseSignatureRefusal(call, { status: 404, text: '{"code":"PGRST205"}' }, unused), false); + assert.equal(supabaseSignatureRefusal(call, { status: 400, text: unmatched.text }, unused), false); + assert.equal(supabaseSignatureRefusal(call, { status: 404, text: 'Not Found' }, unused), false); + assert.equal(supabaseSignatureRefusal({ url: 'http://127.0.0.1:5173/rest/v1/rpc/checkout' }, unmatched, unused), false); + assert.equal(supabaseSignatureRefusal({ url: `${SUPABASE_GATEWAY}/rest/v1/rpc/a'b` }, unmatched, unused), false); +}); + +test('only Realtime subscription upkeep on the leased gateway is a passive socket frame', () => { + const passive = supabasePassiveSocketFrame(supabaseLease()); + const socket = `${SUPABASE_GATEWAY.replace('http', 'ws')}/realtime/v1/websocket?apikey=key&vsn=2.0.0`; + for (const event of ['phx_join', 'phx_leave', 'heartbeat', 'access_token']) { + assert.equal(passive(socket, JSON.stringify(['1', '2', 'realtime:shop', event, {}])), true, event); + assert.equal(passive(socket, JSON.stringify({ topic: 'realtime:shop', event, payload: {}, ref: '2' })), true, event); + } + for (const event of ['broadcast', 'presence']) { + assert.equal(passive(socket, JSON.stringify(['1', '2', 'realtime:shop', event, {}])), false, event); + } + assert.equal(passive(socket, Buffer.from([3, 0])), false, 'a binary broadcast'); + assert.equal(passive(socket, 'not json'), false); + const heartbeat = JSON.stringify(['1', '2', 'phoenix', 'heartbeat', {}]); + assert.equal(passive('ws://localhost:8124/supabase/realtime/v1/websocket?apikey=key', heartbeat), true, 'proxied Realtime'); + assert.equal(passive('ws://example.com/realtime/v1/websocket', heartbeat), false); + assert.equal(passive('ws://127.0.0.1:5173/socket', heartbeat), false); +}); + +test('Supabase provenance scans public application columns through privileged SQL', () => { + const lease = supabaseLease(); + for (const [output, matches] of [['', 0], ['public.order_account.username\n', 1], + ['public.order_account.username\npublic.profile.name\n', 2]] as const) { + let sql = ''; + const result = proveSupabaseUse({ lease, marker: "sb'marker", + exec: supabaseExec(lease, input => { sql = input; return output; }) }); + assert.equal(result.matches, matches); + assert.equal(result.ok, matches > 0); + assert.equal(result.verified, true); + assert.match(sql, /WHERE table_schema = 'public'/); + assert.match(sql, /'sb''marker'/); + assert.match(sql, /\n\\gexec\n$/); + } + assert.throws(() => proveSupabaseUse({ lease, marker: '', exec: supabaseExec(lease, () => '') }), /non-empty/); +}); diff --git a/tools/stack-bench/tests/supabase-owned-lifecycle.integration.ts b/tools/stack-bench/tests/supabase-owned-lifecycle.integration.ts new file mode 100644 index 00000000000..ca68bd9bd76 --- /dev/null +++ b/tools/stack-bench/tests/supabase-owned-lifecycle.integration.ts @@ -0,0 +1,175 @@ +import assert from 'node:assert/strict'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync, existsSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { backendResourceLockKeys, claimBackendResources, createBackendLease, publicBackendLease, + readBackendLease, resourceLockScope } from '../src/runtime/backend-lease.js'; +import { attemptDocker, requireAttemptNetwork } from '../src/runtime/docker-network.js'; +import { releaseBackendLease } from '../src/runtime/backend-teardown.js'; +import { prepareProcessCrash } from '../src/stacks/process-crash.js'; +import { activateSupabase, recoverSupabase, resetSupabase, supabaseApplicationEnvironment } + from '../src/stacks/backends/supabase-lifecycle.js'; +import { SUPABASE_SERVICE_ROLES, readSupabasePlatformSecrets, supabaseSql } from '../src/stacks/backends/supabase-platform.js'; + +// Run inside the Linux controller with its normal lock directory and Docker socket, +// with the pinned Supabase images present. No paid calls or registry entry. +test('Supabase owned platform: activation, gateway readiness, secrets, reset, crash recovery and exact cleanup', { + skip: process.env.STACK_BENCH_SUPABASE_OWNED_TEST !== '1', timeout: 600_000, +}, async () => { + assert.equal(process.platform, 'linux'); + const root = mkdtempSync(join(tmpdir(), 'stack-bench-supabase-owned-')); + const path = join(root, 'lease.json'); + const app = join(root, 'app'); + mkdirSync(join(app, 'supabase', 'functions', 'hello'), { recursive: true }); + writeFileSync(join(app, 'supabase', 'functions', 'hello', 'index.ts'), + "Deno.serve(() => Response.json({ url: Deno.env.get('SUPABASE_URL') }));\n"); + const ports = { vite: 14409, express: 14411, dbPort: null }; + const lease = createBackendLease({ runId: root.split('/').at(-1)!, backend: 'supabase', track: 'ecommerce', + runIndex: 0, serverUri: 'http://127.0.0.1:14410', database: 'postgres' }); + const evidenceDirectory = process.env.STACK_BENCH_SUPABASE_EVIDENCE_DIR; + const evidence: Record = { result: 'running', ports }; + const save = () => { if (evidenceDirectory) writeFileSync(join(evidenceDirectory, 'supabase-owned-lifecycle.json'), + JSON.stringify(evidence, null, 2)); }; + const owned: string[] = []; + let volumes: string[] = []; + let network: string | undefined; + let failure: unknown; + try { + claimBackendResources(path, lease, { ...resourceLockScope(), keys: backendResourceLockKeys(lease, ports) }); + const started = Date.now(); + activateSupabase({ leasePath: path, leaseToken: lease.ownershipToken, ports, app }); + evidence.activationMs = Date.now() - started; + const active = readBackendLease(path, { token: lease.ownershipToken, active: true }); + evidence.activeLease = publicBackendLease(active); + requireAttemptNetwork(active); + const anchor = active.resources.container!; + network = active.resources.network!.id; + const services = active.resources.serviceContainers!; + assert.deepEqual(Object.keys(services).sort(), [...SUPABASE_SERVICE_ROLES].sort()); + for (const container of [anchor, ...Object.values(services), active.resources.browserContainer!]) owned.push(container.id); + for (const service of Object.values(services)) assert.equal(service.networkMode, `container:${anchor.id}`); + volumes = [anchor.id, services.storage!.id].flatMap(id => JSON.parse(attemptDocker(['inspect', '--format', + '{{json .Mounts}}', id])).filter((mount: { Type: string }) => mount.Type === 'volume') + .map((mount: { Name: string }) => mount.Name)); + assert.equal(volumes.length, 2, 'database and storage data are container-owned anonymous volumes'); + + // Every service answers through the published gateway as the application reaches it. + const secrets = readSupabasePlatformSecrets(active); + const environment = supabaseApplicationEnvironment(active); + assert.equal(environment.SUPABASE_URL, 'http://127.0.0.1:14410'); + const service = { apikey: secrets.serviceRoleKey, Authorization: `Bearer ${secrets.serviceRoleKey}` }; + const anon = { apikey: secrets.anonKey, 'content-type': 'application/json' }; + for (const route of ['/auth/v1/health', '/rest/v1/', '/storage/v1/version', '/functions/v1/_health']) { + assert.equal((await fetch(`${environment.SUPABASE_URL}${route}`, { headers: service })).status, 200, route); + } + const hello = await fetch(`${environment.SUPABASE_URL}/functions/v1/hello`, { headers: service }); + assert.deepEqual(await hello.json(), { url: environment.SUPABASE_URL }); + for (const route of ['/', '/pg/', '/api/mcp']) { + assert.notEqual((await fetch(`${environment.SUPABASE_URL}${route}`, { headers: service })).status, 200, route); + } + + // Secrets stay out of the lease, container definitions and process arguments. + const secretValues = Object.values(secrets); + const leaseText = readFileSync(path, 'utf8'); + const definitions = owned.map(id => attemptDocker(['inspect', '--format', + '{{json .Path}}{{json .Args}}{{json .Config.Cmd}}{{json .Config.Entrypoint}}', id])); + const processes = attemptDocker(['top', anchor.id, '-eo', 'pid,args']); + for (const value of secretValues) { + assert(!leaseText.includes(value) && !definitions.some(text => text.includes(value)) && !processes.includes(value)); + } + // The application's database password cannot reach the platform superuser. + const login = (user: string, password: string) => { + try { + attemptDocker(['exec', '-i', anchor.id, 'sh', '-ec', `PGPASSWORD=$(cat) psql -h 127.0.0.1 -U ${user} -d postgres -Atc 'select 1'`], + password); + return true; + } catch { return false; } + }; + assert.equal(login('postgres', secrets.postgresPassword), true); + assert.equal(login('supabase_admin', secrets.postgresPassword), false); + assert.equal(login('supabase_admin', ''), false); + evidence.secrets = 'absent from the lease, container definitions and process arguments; superuser separated'; + + // Application state of every kind, then a data-level reset. + supabaseSql(active, 'create schema app; create table app.items (id int primary key); insert into app.items values (1); ' + + 'create extension pg_cron; alter publication supabase_realtime add table app.items;\n'); + const signUp = await fetch(`${environment.SUPABASE_URL}/auth/v1/signup`, { method: 'POST', headers: anon, + body: JSON.stringify({ email: 'owned@example.com', password: 'password123' }) }); + assert.equal(signUp.status, 200); + assert.equal((await fetch(`${environment.SUPABASE_URL}/storage/v1/bucket`, { method: 'POST', + headers: { ...service, 'content-type': 'application/json' }, body: JSON.stringify({ name: 'owned' }) })).status, 200); + assert.equal((await fetch(`${environment.SUPABASE_URL}/storage/v1/object/owned/file.txt`, { method: 'POST', + headers: { ...service, 'content-type': 'text/plain' }, body: 'stored' })).status, 200); + const resetStarted = Date.now(); + resetSupabase({ lease: active }); + evidence.resetMs = Date.now() - resetStarted; + assert.equal(supabaseSql(active, "select count(*) from pg_namespace where nspname in ('app', 'cron');\n").trim(), '0'); + assert.equal(supabaseSql(active, 'select (select count(*) from auth.users) + (select count(*) from storage.buckets);\n').trim(), '0'); + assert.equal(attemptDocker(['exec', services.storage!.id, 'find', '/var/lib/storage', '-mindepth', '1']), ''); + assert.equal(supabaseSql(active, "select string_agg(pubname, ',' order by pubname) from pg_publication;\n").trim(), + 'supabase_realtime,supabase_realtime_messages_publication'); + assert.equal(supabaseSql(active, "select string_agg(prrelid::regclass::text, ',') from pg_publication_rel pr " + + "join pg_publication p on p.oid = pr.prpubid where p.pubname = 'supabase_realtime';\n").trim(), + 'stackbench_reset.realtime_canary'); + supabaseSql(active, 'create extension pg_cron;\n'); + const signIn = await fetch(`${environment.SUPABASE_URL}/auth/v1/token?grant_type=password`, { method: 'POST', + headers: anon, body: JSON.stringify({ email: 'owned@example.com', password: 'password123' }) }); + assert.equal(signIn.status, 400, 'a previous account cannot sign in after reset'); + resetSupabase({ lease: active }); + assert.equal(supabaseSql(active, "select count(*) from pg_extension where extname = 'pg_cron';\n").trim(), '0'); + evidence.reset = 'schemas, extensions with their schemas, accounts and stored files removed; Realtime publications kept'; + + // A database process crash leaves the namespace; recovery restarts the database and any service it took down. + await assert.rejects(async () => recoverSupabase({ leasePath: path, leaseToken: lease.ownershipToken, ports }), + /Command failed/, 'recovery refuses a live database'); + const crash = await prepareProcessCrash(active, 'database'); + try { + evidence.crash = await crash.crash(); + assert.equal(attemptDocker(['inspect', '--format', '{{.State.Running}}', anchor.id]), 'true'); + recoverSupabase({ leasePath: path, leaseToken: lease.ownershipToken, ports }); + } finally { await crash.close(); } + assert.equal((await fetch(`${environment.SUPABASE_URL}/rest/v1/`, { headers: service })).status, 200); + for (const container of Object.values(services)) { + assert.equal(attemptDocker(['inspect', '--format', '{{.State.Running}}', container.id]), 'true', container.name); + } + // A scheduled job connects without a password. + supabaseSql(active, "create extension pg_cron;\nselect cron.schedule('owned', '1 seconds', 'select 1');\n"); + const deadline = Date.now() + 15_000; + while (supabaseSql(active, "select count(*) from cron.job_run_details where status = 'succeeded';\n").trim() === '0') { + assert(Date.now() < deadline, 'a pg_cron job runs'); + await new Promise(resolve => setTimeout(resolve, 500)); + } + requireAttemptNetwork(readBackendLease(path)); + evidence.result = 'passed'; + } catch (error) { + evidence.result = 'failed'; + evidence.error = error instanceof Error ? error.message : String(error); + failure = error; + } finally { + try { + if (existsSync(path)) { + const partial = readBackendLease(path); + for (const container of [partial.resources.container, ...Object.values(partial.resources.serviceContainers ?? {}), + partial.resources.browserContainer]) if (container && !owned.includes(container.id)) owned.push(container.id); + network ??= partial.resources.network?.id; + assert.equal(releaseBackendLease(path, lease.ownershipToken), true, 'Exact owned cleanup must succeed'); + const final = readBackendLease(path); + evidence.finalLease = publicBackendLease(final); + assert.equal(final.state, 'released'); + for (const id of owned) assert.throws(() => attemptDocker(['inspect', id]), /no such object/i); + for (const volume of volumes) assert.throws(() => attemptDocker(['volume', 'inspect', volume]), /no such volume/i); + if (network) assert.throws(() => attemptDocker(['network', 'inspect', network!]), /not found|no such network/i); + evidence.cleanup = { containers: owned.length, volumes: volumes.length, network: Boolean(network) }; + } + } catch (error) { + evidence.result = 'failed'; + evidence.cleanupError = error instanceof Error ? error.message : String(error); + failure ??= error; + } finally { + save(); + rmSync(root, { recursive: true, force: true }); + } + } + if (failure) throw failure; +}); diff --git a/tools/stack-bench/tests/supabase-platform.test.ts b/tools/stack-bench/tests/supabase-platform.test.ts new file mode 100644 index 00000000000..840b4be238c --- /dev/null +++ b/tools/stack-bench/tests/supabase-platform.test.ts @@ -0,0 +1,254 @@ +import assert from 'node:assert/strict'; +import { createHmac } from 'node:crypto'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { createBackendLease, readBackendLease, writeBackendLease } from '../src/runtime/backend-lease.js'; +import type { BackendLease } from '../src/runtime/backend-lease.js'; +import type { TextCommandExecutor } from '../src/runtime/command-executor.js'; +import { SUPABASE_IDENTITY, SUPABASE_IMAGES } from '../src/stacks/backends/supabase-identity.js'; +import { SUPABASE_RUNTIME, resetSupabase, startSupabasePlatform, supabaseOrchestratorConfig } + from '../src/stacks/backends/supabase-lifecycle.js'; +import { SUPABASE_CONTAINER_LIMITS, SUPABASE_SERVICE_ROLES, formatSupabaseSecrets, generateSupabaseSecrets, + parseSupabaseSecrets, readSupabasePlatformSecrets, renderSupabaseGateway, supabaseAsset, supabaseGatewayUrl, + supabaseServiceSpecs, supabaseSql } from '../src/stacks/backends/supabase-platform.js'; + +const ANCHOR = 'a'.repeat(64); +const STARTED = '2026-09-25T00:00:00Z'; +const secrets = generateSupabaseSecrets(); +const secretValues = Object.values(secrets); + +function claims(token: string, secret: string): Record { + const [header, body, signature] = token.split('.'); + assert.equal(createHmac('sha256', secret).update(`${header}.${body}`).digest('base64url'), signature); + return JSON.parse(Buffer.from(body!, 'base64url').toString('utf8')); +} + +function activeLease(root: string): { path: string; lease: BackendLease } { + const lease = createBackendLease({ runId: 'supabase-unit', backend: 'supabase', track: 'ecommerce', runIndex: 0, + serverUri: 'http://127.0.0.1:13410', database: 'postgres' }); + lease.resources.container = { name: 'sb-unit-backend', id: ANCHOR, image: `sha256:${'d'.repeat(64)}`, owned: true, + networkMode: 'b'.repeat(64) }; + lease.resources.network = { name: 'sb-unit-network', id: 'b'.repeat(64), namespaceContainerId: ANCHOR, + namespaceStartedAt: STARTED, hostAddresses: ['172.20.0.1'], services: [], firewallSha256: 'f'.repeat(64), + firewallInstalledAt: STARTED }; + const path = join(root, 'lease.json'); + writeBackendLease(path, lease); + return { path, lease }; +} + +// Answers the anchor's namespace identity check; everything else is recorded. +function fakeExec(calls: { args: string[]; input?: string }[], answer: (args: readonly string[]) => string = () => ''): + TextCommandExecutor { + return (file, args, options) => { + assert.equal(file, 'docker'); + calls.push({ args: [...args], ...(options.input === undefined ? {} : { input: options.input }) }); + if (args[0] === 'inspect' && args[2] === '{{json .}}') { + return JSON.stringify({ Id: ANCHOR, State: { Running: true, StartedAt: STARTED } }); + } + return answer(args); + }; +} + +test('platform secrets are fresh HS256 keys that round-trip only in their own format', () => { + assert.deepEqual(claims(secrets.anonKey, secrets.jwtSecret).role, 'anon'); + assert.deepEqual(claims(secrets.serviceRoleKey, secrets.jwtSecret).role, 'service_role'); + assert.equal(secrets.realtimeEncryptionKey.length, 16); + assert.notEqual(secrets.adminPassword, secrets.postgresPassword); + assert.notDeepEqual(generateSupabaseSecrets(), secrets); + const text = formatSupabaseSecrets(secrets); + assert.deepEqual(parseSupabaseSecrets(text), secrets); + assert.throws(() => parseSupabaseSecrets(`${text}${text.split('\n')[0]}\n`), /malformed/); + assert.throws(() => parseSupabaseSecrets(`${text}EXTRA=${'x'.repeat(20)}\n`), /malformed/); + assert.throws(() => parseSupabaseSecrets(text.split('\n').slice(1).join('\n')), /lacks POSTGRES_PASSWORD/); + assert.throws(() => formatSupabaseSecrets({ ...secrets, jwtSecret: "x'; drop" }), /JWT_SECRET is invalid/); +}); + +test('the gateway publishes only application services on the leased port', () => { + const rendered = renderSupabaseGateway({ anonKey: secrets.anonKey, serviceRoleKey: secrets.serviceRoleKey, + gatewayPort: 13417 }); + for (const text of Object.values(rendered)) assert.doesNotMatch(text, /\$\{[A-Z_]+\}|STRICT_DNS|cluster: (studio|meta)\b/); + assert.match(rendered['lds.yaml'], /port_value: 13417/); + assert(rendered['lds.yaml'].includes(secrets.anonKey) && rendered['lds.yaml'].includes(secrets.serviceRoleKey)); + assert.doesNotMatch(rendered['cds.yaml'], /name: (studio|meta)\n/); + assert.deepEqual([...rendered['cds.yaml'].matchAll(/address: (\S+)/g)].map(match => match[1]) + .filter(address => address !== '127.0.0.1'), []); + assert.match(rendered['envoy.yaml'], /path: \/etc\/envoy\/lds\.yaml/); + assert.throws(() => renderSupabaseGateway({ anonKey: 'a', serviceRoleKey: 'b', gatewayPort: 70_000 }), /port/); +}); + +test('service specifications keep secrets out of arguments and apply the platform restrictions', () => { + const specs = supabaseServiceSpecs({ gatewayPort: 13410, sitePort: 7823, app: '/work/app', secrets }); + assert.deepEqual(Object.keys(specs), [...SUPABASE_SERVICE_ROLES]); + for (const [role, spec] of Object.entries(specs)) { + const args = [...spec.args, ...spec.command].join(' '); + for (const value of secretValues) assert(!args.includes(value), `${role} arguments hold a secret`); + for (const name of Object.keys(spec.environment)) assert(spec.args.includes(name), `${role} passes ${name} by name`); + assert.equal(spec.image, SUPABASE_IMAGES[role as keyof typeof SUPABASE_IMAGES]); + assert.deepEqual(spec.limits, SUPABASE_CONTAINER_LIMITS[role as keyof typeof SUPABASE_CONTAINER_LIMITS]); + } + assert.equal(specs.realtime.environment.DB_PASSWORD, secrets.adminPassword); + assert.equal(specs.auth.environment.GOTRUE_DB_DATABASE_URL, `postgres://supabase_auth_admin:${secrets.postgresPassword}@db:5432/postgres`); + assert.deepEqual(specs.realtime.args.slice(0, 4), ['--user', '65534:65534', '--entrypoint', '/bin/bash']); + assert.deepEqual(specs.gateway.args.slice(0, 4), ['--user', '101:101', '--entrypoint', 'envoy']); + assert.deepEqual(specs.functions.args.slice(0, 2), ['-v', '/work/app:/home/deno/app:ro']); + assert.match(specs.functions.files['/home']!['deno/functions/main/index.ts']!, /\/home\/deno\/app\/supabase\/functions\//); + assert.equal(specs.functions.environment.SUPABASE_DB_URL, `postgresql://postgres:${secrets.postgresPassword}@127.0.0.1:5432/postgres`); + assert.deepEqual(specs.storage.args.slice(0, 2), ['-v', '/var/lib/storage']); + assert.match(specs.gateway.files['/etc/envoy']!['lds.yaml']!, /port_value: 13410/); + assert.equal(SUPABASE_CONTAINER_LIMITS.db.memoryBytes, 1024 ** 3); +}); + +test('privileged SQL runs as supabase_admin over the exact anchor socket with the statement on stdin', () => { + const root = mkdtempSync(join(tmpdir(), 'supabase-sql-')); + try { + const { lease } = activeLease(root); + const calls: { args: string[]; input?: string }[] = []; + assert.equal(supabaseSql(lease, 'select 1;', { exec: fakeExec(calls, () => '1\n') }), '1\n'); + assert.deepEqual(calls[1], { args: ['exec', '-i', '--user', 'postgres', ANCHOR, 'psql', '-h', '/var/run/postgresql', + '-U', 'supabase_admin', '-d', 'postgres', '-v', 'ON_ERROR_STOP=1', '-X', '-q', '-At'], input: 'select 1;' }); + const replaced: TextCommandExecutor = () => JSON.stringify({ Id: 'c'.repeat(64), State: { Running: true, StartedAt: STARTED } }); + assert.throws(() => supabaseSql(lease, 'select 1;', { exec: replaced }), /changed after lease creation/); + const restarted: TextCommandExecutor = () => JSON.stringify({ Id: ANCHOR, State: { Running: true, StartedAt: 'later' } }); + assert.throws(() => readSupabasePlatformSecrets(lease, restarted), /stopped or restarted/); + assert.deepEqual(readSupabasePlatformSecrets(lease, fakeExec([], () => formatSupabaseSecrets(secrets))), secrets); + assert.equal(supabaseGatewayUrl(lease), 'http://127.0.0.1:13410'); + assert.throws(() => supabaseSql({ ...lease, backend: 'postgres' }, 'select 1;'), /not a Supabase lease/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('the platform starts in order, reaches readiness through the gateway, then records its reset baseline', () => { + const root = mkdtempSync(join(tmpdir(), 'supabase-start-')); + try { + const { path, lease } = activeLease(root); + const calls: { args: string[]; input?: string; environment?: Record }[] = []; + const ids = new Map(SUPABASE_SERVICE_ROLES.map((role, index) => [role, String(index + 1).repeat(64)])); + const probes: string[] = []; + const docker = (args: string[], input?: string, environment?: Record) => { + calls.push({ args, ...(input === undefined ? {} : { input }), ...(environment ? { environment } : {}) }); + if (args[0] === 'inspect' && args[2] === '{{json .}}') { + return JSON.stringify({ Id: ANCHOR, State: { Running: true, StartedAt: STARTED } }); + } + if (args[0] === 'image') return `sha256:${'e'.repeat(64)}`; + if (args[0] === 'create') return ids.get(/service-([a-z]+)$/.exec(args[args.indexOf('--name') + 1]!)![1] as never)!; + if (args.includes('psql') && input?.includes('supabase_realtime_messages_publication')) return '1\n'; + return ''; + }; + startSupabasePlatform({ leasePath: path, leaseToken: lease.ownershipToken, app: '/work/app', secrets, + ports: { vite: 7823, express: 7901, dbPort: null }, docker, sleep: () => undefined, probes: { + http: (url, headers) => { probes.push(`${url} ${headers.apikey === secrets.serviceRoleKey}`); return 200; }, + realtimeJoin: (url, key) => { probes.push(`join ${url} ${key === secrets.anonKey}`); return true; }, + } }); + for (const call of calls) { + for (const value of secretValues) assert(!call.args.join(' ').includes(value), `secret in ${call.args.join(' ')}`); + } + const kind = (call: typeof calls[number]) => call.args[0] === 'exec' && call.args[1] === '-d' ? 'start-db' + : call.args.includes('pg_isready') ? 'ready-db' + : call.args.includes('psql') ? `sql:${call.input?.includes('stackbench_reset') ? 'baseline' + : call.input?.includes('alter role supabase_admin') ? 'admin' : 'query'}` + : call.args[0] === 'exec' && call.args.at(-1)?.startsWith('/') ? `write:${call.args.at(-1)}` + : call.args[0] === 'create' ? `create:${/service-([a-z]+)$/.exec(call.args[call.args.indexOf('--name') + 1]!)![1]}` + : call.args[0] === 'start' ? 'start' : call.args[0] === 'cp' ? 'cp' : null; + const order = calls.map(kind).filter(Boolean); + assert.deepEqual(order.slice(0, 11), ['write:/run/stack-bench/supabase.env', 'write:/etc/postgresql/pg_hba.conf', + 'write:/docker-entrypoint-initdb.d/init-scripts/98-webhooks.sql', 'write:/docker-entrypoint-initdb.d/init-scripts/99-roles.sql', + 'write:/docker-entrypoint-initdb.d/init-scripts/99-jwt.sql', 'write:/docker-entrypoint-initdb.d/migrations/97-_supabase.sql', + 'write:/docker-entrypoint-initdb.d/migrations/99-realtime.sql', 'write:/docker-entrypoint-initdb.d/migrations/99-logs.sql', + 'start-db', 'ready-db', 'sql:admin']); + assert.deepEqual(order.filter(step => step?.startsWith('create:')), SUPABASE_SERVICE_ROLES.map(role => `create:${role}`)); + assert.equal(order.at(-1), 'sql:baseline'); + const secretFile = calls.find(call => kind(call) === 'write:/run/stack-bench/supabase.env')!; + assert(secretFile.input === formatSupabaseSecrets(secrets), 'secrets reach the anchor on stdin'); + assert.match(secretFile.args.join(' '), /umask 077/); + // pg_cron holds no password, so its job connections use the trusted socket. + assert.match(calls.find(call => kind(call) === 'start-db')!.args.at(-1)!, / -c cron\.host=\/var\/run\/postgresql'/); + const hba = calls.find(call => kind(call) === 'write:/etc/postgresql/pg_hba.conf')!.input!; + assert.match(hba, /host {2}all {2}all {2}127\.0\.0\.1\/32 {2}scram-sha-256/); + assert.doesNotMatch(hba, /127\.0\.0\.1\/32\s+trust/); + const auth = calls.find(call => kind(call) === 'create:auth')!; + assert.equal(auth.environment?.GOTRUE_JWT_SECRET, secrets.jwtSecret); + assert.equal(auth.args[auth.args.indexOf('--network') + 1], `container:${ANCHOR}`); + assert.deepEqual(probes, ['/auth/v1/health', '/rest/v1/', '/storage/v1/version', '/functions/v1/_health'] + .map(route => `http://127.0.0.1:13410${route} true`).concat('join http://127.0.0.1:13410 true')); + const services = readBackendLease(path).resources.serviceContainers!; + assert.deepEqual(Object.keys(services), [...SUPABASE_SERVICE_ROLES]); + assert.equal(services.gateway!.id, ids.get('gateway')); + const baseline = calls.find(call => kind(call) === 'sql:baseline')!.input!; + assert(baseline.startsWith('begin;\n') && baseline.trimEnd().endsWith('commit;')); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('reset empties data through the baseline function and storage files, only on the exact storage service', () => { + const root = mkdtempSync(join(tmpdir(), 'supabase-reset-')); + try { + const { path } = activeLease(root); + const lease = readBackendLease(path); + lease.state = 'active'; + assert.throws(() => resetSupabase({ lease, exec: fakeExec([]) }), /owned storage service/); + const storage = { name: 'sb-unit-service-storage', id: '5'.repeat(64), + image: `sha256:${'e'.repeat(64)}`, owned: true, networkMode: `container:${ANCHOR}` }; + lease.resources.serviceContainers = { storage }; + const calls: { args: string[]; input?: string }[] = []; + const inspect = (running: string) => (args: readonly string[]) => + args[0] !== 'inspect' ? '' : args[2] === '{{.Id}}' ? storage.id : running; + resetSupabase({ lease, exec: fakeExec(calls, inspect('true\n')) }); + assert.deepEqual(calls.map(call => call.input ?? call.args.slice(0, 2).join(' ')), ['inspect --format', + 'inspect --format', 'inspect --format', 'select stackbench_reset.reset();\n', `exec ${storage.id}`]); + assert.match(calls.at(-1)!.args.at(-1)!, /find \/var\/lib\/storage -mindepth 1 -maxdepth 1 -exec rm -rf/); + // A deadlocked reset is rolled back by Postgres and repeated; any other error is not. + const failing = (message: string, times: number) => { + let left = times; + return (args: readonly string[]) => { + if (args.includes('psql') && left-- > 0) throw new Error(`Command failed\nERROR: ${message}`); + return inspect('true\n')(args); + }; + }; + const retried: { args: string[]; input?: string }[] = []; + resetSupabase({ lease, exec: fakeExec(retried, failing('deadlock detected', 2)) }); + assert.equal(retried.filter(call => call.input?.includes('reset()')).length, 3); + assert.throws(() => resetSupabase({ lease, exec: fakeExec([], failing('deadlock detected', 3)) }), /deadlock/); + const other: { args: string[]; input?: string }[] = []; + assert.throws(() => resetSupabase({ lease, exec: fakeExec(other, failing('permission denied', 1)) }), /permission denied/); + assert.equal(other.filter(call => call.input?.includes('reset()')).length, 1); + const replaced: { args: string[]; input?: string }[] = []; + assert.throws(() => resetSupabase({ lease, exec: fakeExec(replaced, () => 'other') }), /changed after lease creation/); + assert(!replaced.some(call => call.input?.includes('reset()')), 'no data is reset for a replaced storage service'); + assert.throws(() => resetSupabase({ lease, exec: fakeExec([], inspect('false\n')) }), /is not running/); + assert.throws(() => resetSupabase({ lease: { ...lease, state: 'released' } }), /active lease/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('the reset baseline leaves extension objects to their extension and restores Realtime publications', () => { + const baseline = supabaseAsset('reset/baseline.sql'); + const namespaceLoop = /select nspname from pg_namespace n where[\s\S]*?loop/.exec(baseline)?.[0]; + assert.match(namespaceLoop ?? '', /not_extension_member, 'pg_%', 'namespace', 'n\.oid', 'pg_namespace', 'n\.oid'/); + for (const catalog of ['pg_namespace', 'pg_class', 'pg_policy', 'pg_trigger', 'pg_publication_rel', 'pg_publication', + 'pg_event_trigger']) assert(baseline.includes(`'${catalog}'`), catalog); + assert.match(baseline, /x\.deptype = ''e''/); + assert.match(baseline, /create publication supabase_realtime;\n\s+alter publication supabase_realtime owner to postgres;/); + assert.match(baseline, /alter publication supabase_realtime add table stackbench_reset\.realtime_canary;\n\s+insert into stackbench_reset\.baseline/); + assert(!/^\s*(begin|commit)\s*;/im.test(baseline), 'activation wraps the baseline in its own transaction'); + // An app that drops and recreates public gets the bootstrap schema back, owner, grants and defaults. + assert.match(baseline, /if not exists \(select 1 from pg_namespace where nspname = 'public'\) then\n\s+execute format\('create schema public authorization %I'/); + assert.match(baseline, /alter default privileges for role %I in schema public grant/); + assert.match(baseline, /insert into stackbench_reset\.baseline select 'namespace', oid from pg_namespace where nspname = 'public';/); +}); + +test('Supabase identity, orchestration and crash boundary are its own', () => { + assert.deepEqual(SUPABASE_IDENTITY.ports, { vite: 7823, express: 7901 }); + assert.equal(supabaseOrchestratorConfig({ env: {} }).lease.serverUri, 'http://127.0.0.1:13410'); + assert.deepEqual(supabaseOrchestratorConfig({ env: { STACK_BENCH_SUPABASE_URI: 'http://127.0.0.1:14000' } }).environment, + { STACK_BENCH_SUPABASE_URI: 'http://127.0.0.1:14000' }); + assert.throws(() => supabaseOrchestratorConfig({ env: { STACK_BENCH_SUPABASE_URI: 'http://example.com:80' } }), /loopback/); + const helpers = { requireString: (value: unknown) => value, loopbackHttpUri: (value: unknown) => new URL(String(value)) }; + assert.throws(() => SUPABASE_IDENTITY.lease.validateResources({ helpers, + resources: { serverUri: 'http://127.0.0.1:13410', database: 'app', container: null } }), /postgres database/); + // Checkout is a database function: the database is the only boundary that can interrupt it. + assert.equal(SUPABASE_RUNTIME.combinedBoundary, true); + assert.equal(SUPABASE_RUNTIME.transactionalRequests, true); + assert.equal(SUPABASE_RUNTIME.drainCommand, undefined); + assert.equal(SUPABASE_RUNTIME.databaseUser, 'postgres'); + assert.equal(SUPABASE_RUNTIME.processRecord, '/run/application/restart-supabase-db.pid'); +}); diff --git a/tools/stack-bench/tests/support-save.integration.ts b/tools/stack-bench/tests/support-save.integration.ts new file mode 100644 index 00000000000..42c643ea9a0 --- /dev/null +++ b/tools/stack-bench/tests/support-save.integration.ts @@ -0,0 +1,266 @@ +import assert from 'node:assert/strict'; +import { once } from 'node:events'; +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { createServer } from 'node:http'; +import { dirname, join } from 'node:path'; +import test from 'node:test'; +import { chromium } from 'playwright'; +import { gradeFeature } from '../grader/grade.js'; +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; + +// The declared submit-state receipt precedes a verification reload. A receipt is +// not proof of a saved value, and unrelated reads are not part of the save. +test('support save completes before verification reload without changing timed reloads', async () => { + let mode = '', stored = '', writes = 0; + let events: { event: string; at: number; value?: string }[] = []; + const evidence: unknown[] = []; + const server = createServer(async (request, response) => { + if (request.url === '/poll') { + events.push({ event: 'poll-start', at: Date.now() }); + response.writeHead(200, { 'Content-Type': 'application/json' }); + response.write(' '); // A legitimate long read stays open until the page closes. + return; + } + if (request.method === 'POST' && request.url === '/save-ticket') { + let body = ''; for await (const chunk of request) body += String(chunk); + const value = (JSON.parse(body) as { assignee: string }).assignee; + writes++; + const firstOfTwo = mode === 'overlapping-writes' && writes === 1; + events.push({ event: 'save-start', at: Date.now(), value }); + if (mode === 'refused') { + events.push({ event: 'save-refused', at: Date.now() }); + response.writeHead(409, { 'Content-Type': 'application/json' }); + response.end('{"error":"refused"}'); return; + } + if (mode !== 'lying-success') { + await new Promise(resolve => setTimeout(resolve, firstOfTwo ? 500 : 2000)); + if (!firstOfTwo) { + stored = value; + events.push({ event: 'save-committed', at: Date.now(), value }); + } + } + response.writeHead(200, { 'Content-Type': 'application/json' }); + response.end('{}'); return; + } + if (request.url !== '/') { response.writeHead(404).end(); return; } + events.push({ event: 'page-load', at: Date.now(), value: stored }); + response.writeHead(200, { 'Content-Type': 'text/html' }); + response.end(`staff +
Missing item + +
+ `); + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const browser = await chromium.launch({ headless: true }); + let result = 'failed'; + try { + const original = JSON.parse(readFileSync(join(STACK_BENCH_ROOT, + 'tracks/ecommerce/scenarios/progression-support-triage.json'), 'utf8')); + for (mode of ['delayed', 'delayed-request-start', 'overlapping-writes', 'stale-success', + 'refused', 'missing-receipt', 'lying-success', 'ordinary-timed-reload']) { + stored = ''; writes = 0; events = []; + const definition = structuredClone(original); + const feature = definition.features[0]; + feature.actors = ['staff']; feature.setup = []; + feature.criteria = feature.criteria.filter((criterion: { id: string }) => criterion.id === '611a'); + if (mode === 'ordinary-timed-reload') { + feature.criteria[0].steps = feature.criteria[0].steps + .filter((step: { attribute?: string }) => step.attribute !== 'data-submit-state') + .map((step: { do: string; testid?: string }) => + step.do === 'click' && step.testid === 'support-update' + ? { do: 'click', actor: 'staff', testid: 'support-update', + in: { testid: 'support-ticket', contains: 'Missing item' }, settleMs: 1500 } + : step); + } + const compiled = compileScenarioDefinition(definition).features[0]!; + const grade = await gradeFeature(browser, compiled, + { url, level: 2, headed: false, selectedCheckKeys: [], nullControl: false }, + { runId: 'support-save-completion', roomName: name => name, url, actions: [], + spacetime: null, backend: 'postgres', nullControl: false, defaultWithin: 4000 }); + evidence.push({ mode, grade, events: [...events], writes, stored, steps: feature.criteria[0].steps }); + assert.equal(grade.setupEvidence.status, 'passed', JSON.stringify(grade)); + const correct = ['delayed', 'delayed-request-start', 'overlapping-writes', 'stale-success'].includes(mode); + assert.equal(writes, mode === 'overlapping-writes' ? 2 : 1, `${mode}: no harness write retry`); + assert.equal(grade.criteria[0]!.evidence.status, correct ? 'passed' : 'failed', + JSON.stringify({ mode, grade, events })); + const reload = events.filter(event => event.event === 'page-load')[1]; + if (correct || mode === 'ordinary-timed-reload') { + const commit = events.find(event => event.event === 'save-committed'); + assert.ok(reload && commit, JSON.stringify(events)); + assert.equal(reload.value, correct ? 'staff' : '', JSON.stringify(events)); + assert.equal(reload.at >= commit.at, correct, JSON.stringify(events)); + } else { + assert.equal(stored, mode === 'missing-receipt' ? 'staff' : '', `${mode}: persisted state`); + if (mode === 'refused' || mode === 'missing-receipt') { + assert.equal(reload, undefined, `${mode}: no reload without the required success receipt`); + } else { + assert.ok(reload, 'a lying receipt must still face the persisted-state check'); + assert.equal(reload.value, '', 'the lying receipt must not manufacture stored state'); + } + } + } + result = 'passed'; + } finally { + await browser.close(); + server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + if (process.env.STACK_BENCH_SUPPORT_SAVE_EVIDENCE) { + const file = process.env.STACK_BENCH_SUPPORT_SAVE_EVIDENCE; + mkdirSync(dirname(file), { recursive: true }); + writeFileSync(file, JSON.stringify({ result, + rerun: 'node --test dist/tests/support-save.integration.js', evidence }, null, 2)); + } + } +}); + +// A valid save may finish after the click returns. An ordinary persistence or +// access check must not reload first and then judge a stale page-load snapshot. +test('ordinary profile, notification and promotion saves finish before verification reload', async t => { + type Event = { event: string; at: number; value?: string; method?: string; body?: string; status?: number }; + let mode = '', kind = '', stored = '', events: Event[] = []; + const evidence: unknown[] = []; + const server = createServer(async (request, response) => { + if (request.method === 'POST' && request.url === '/save') { + let body = ''; for await (const chunk of request) body += String(chunk); + const value = (JSON.parse(body) as { value: string }).value; + events.push({ event: 'save-start', at: Date.now(), value, method: request.method, body }); + response.on('close', () => events.push({ event: 'response-close', at: Date.now(), value })); + response.on('finish', () => events.push({ event: 'response-finished', at: Date.now(), value, status: response.statusCode })); + if (mode === 'refused') { + events.push({ event: 'save-refused', at: Date.now() }); + response.writeHead(409, { 'Content-Type': 'application/json' }); response.end('{}'); return; + } + if (['delayed', 'delayed-request-start', 'stale-success', 'close-after-save'].includes(mode)) { + await new Promise(resolve => setTimeout(resolve, 2000)); + } + // The durable operation is valid even if the harness has closed its page. + if (mode !== 'lying-success') { + stored = value; events.push({ event: 'save-committed', at: Date.now(), value }); + } + response.writeHead(200, { 'Content-Type': 'application/json' }); response.end(JSON.stringify({ saved: value })); return; + } + if (request.url !== '/') { response.writeHead(404).end(); return; } + events.push({ event: 'page-load', at: Date.now(), value: stored }); + response.writeHead(200, { 'Content-Type': 'text/html' }); + const profile = kind === 'profile', notification = kind === 'notification'; + const saveId = profile ? 'profile-save' : notification ? 'notification-save' : 'promotion-submit'; + const receiptId = profile ? 'profile-save-state' : 'notification-save-state'; + const username = profile ? 'profile-private-ownerordinary-save-completionf620' + : notification ? 'notification-ownerordinary-save-completionf630' : 'staff'; + response.end(`${username} +
+ ${profile || notification ? `Save status` : ''} + ${profile ? `
+

${stored}

` + : notification ? `
+ +
` + : ` + + +
${stored ? `
${stored}
` : ''}
`} + `); + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const browser = await chromium.launch({ headless: true }); + try { + for (kind of ['profile', 'notification', 'promotion']) for (mode of kind === 'promotion' + ? ['immediate', 'delayed'] : ['immediate', 'delayed', 'delayed-request-start', 'stale-success', + 'close-after-save', 'refused', 'missing-receipt', 'lying-success']) { + await t.test(`${kind}-${mode}`, async () => { + stored = ''; events = []; + const source = join(STACK_BENCH_ROOT, `tracks/ecommerce/scenarios/progression-${kind === 'profile' + ? 'customer-profile' : kind === 'notification' ? 'notification-preferences' : 'promotion-rules'}.json`); + const definition = JSON.parse(readFileSync(source, 'utf8')); + const feature = definition.features[0], criterion = feature.criteria.find((item: { id: string }) => + item.id === (kind === 'notification' ? '630a' : '620b')); + const actor = kind === 'profile' ? 'privateOwner' : kind === 'notification' ? 'owner' : 'staff'; + const steps = (kind === 'notification' ? [...feature.setup, ...criterion.steps] : criterion.steps) as + { do: string; actor?: string; testid?: string; contains?: string }[]; + const first = steps.findIndex(step => kind === 'notification' + ? step.do === 'click' && step.testid === 'notification-order' + : step.do === 'fill' && step.testid === (kind === 'profile' ? 'profile-name' : 'promotion-code')); + const reloadIndex = steps.findIndex((step, index) => index > first && step.do === 'reload'); + const last = steps.findIndex((step, index) => index > reloadIndex && step.do === 'expect' + && step.testid === (kind === 'profile' ? 'profile-address-summary' + : kind === 'notification' ? 'notification-order' : 'promotion-item')); + assert(first >= 0 && reloadIndex > first && last > reloadIndex, + 'select the authored save, reload and persisted read; retain any completion receipt'); + feature.actors = [actor]; feature.setup = []; + feature.criteria = [{ ...criterion, steps: steps.slice(first, last + 1) }]; + const compiled = compileScenarioDefinition(definition).features[0]!; + const grade = await gradeFeature(browser, compiled, + { url, level: 2, headed: false, selectedCheckKeys: [], nullControl: false }, + { runId: 'ordinary-save-completion', roomName: name => name, url, actions: [], + spacetime: null, backend: 'postgres', nullControl: false, defaultWithin: 4000 }); + // Preserve evidence before the expected red assertion. + evidence.push({ kind, mode, source, steps: feature.criteria[0].steps, grade, stored, events: [...events] }); + assert.equal(grade.setupEvidence.status, 'passed', JSON.stringify(grade)); + const commit = events.find(event => event.event === 'save-committed'); + const reload = events.filter(event => event.event === 'page-load')[1]; + assert.equal(events.filter(event => event.event === 'save-start').length, 1, 'no write retry'); + const correct = !['refused', 'missing-receipt', 'lying-success'].includes(mode); + assert.equal(grade.criteria[0]!.evidence.status, correct ? 'passed' : 'failed', + JSON.stringify({ kind, mode, grade, events })); + if (correct) { + assert(commit && reload, JSON.stringify(events)); + assert(stored, 'the valid app committed the submitted value'); + assert(reload.at >= commit.at, 'ordinary verification must follow the completed save'); + } else if (mode === 'refused' || mode === 'missing-receipt') { + assert.equal(reload, undefined, 'no verification reload before a required success receipt'); + } else { + assert(reload, 'a lying receipt must still face the persisted-state read'); + assert.equal(reload.value, '', 'a receipt must not manufacture durable state'); + } + }); + } + } finally { + await browser.close(); server.closeAllConnections(); await new Promise(resolve => server.close(() => resolve())); + if (process.env.STACK_BENCH_ORDINARY_SAVE_EVIDENCE) { + mkdirSync(dirname(process.env.STACK_BENCH_ORDINARY_SAVE_EVIDENCE), { recursive: true }); + writeFileSync(process.env.STACK_BENCH_ORDINARY_SAVE_EVIDENCE, JSON.stringify({ + rerun: 'node --test --test-name-pattern="ordinary profile, notification and promotion" dist/tests/support-save.integration.js', evidence, + }, null, 2)); + } + } +}); diff --git a/tools/stack-bench/tests/tool-policy.test.ts b/tools/stack-bench/tests/tool-policy.test.ts new file mode 100644 index 00000000000..55be3f9e7f4 --- /dev/null +++ b/tools/stack-bench/tests/tool-policy.test.ts @@ -0,0 +1,122 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { brokerProtocol } from '../container/broker-protocols.js'; +import { CODING_PROVIDERS } from '../container/coding-providers.js'; +import { ANTIGRAVITY_SUBAGENT_TOOLS, ANTIGRAVITY_TOOLS, ANTIGRAVITY_WEB_TOOLS } from '../src/agents/antigravity-protocol.js'; +import { DEFAULT_TOOL_POLICY, toolPolicyFromEnvironment, unavailableToolsReason } from '../src/agents/tool-policy.js'; +import type { ToolPolicy } from '../src/agents/tool-policy.js'; +import { extensionAttemptKey } from '../src/campaigns/campaign-extension.js'; +import type { CampaignAttemptPlan } from '../src/campaigns/campaign-compiler.js'; + +const POLICIES: ToolPolicy[] = [ + { webSearch: true, subagents: false }, { webSearch: false, subagents: false }, + { webSearch: true, subagents: true }, { webSearch: false, subagents: true }, +]; + +test('the default tool policy turns web search and subagents off, as every agent ran in 1.1', () => { + assert.deepEqual(DEFAULT_TOOL_POLICY, { webSearch: false, subagents: false }); + assert.deepEqual(toolPolicyFromEnvironment({}), DEFAULT_TOOL_POLICY); + assert.deepEqual(toolPolicyFromEnvironment({ STACK_BENCH_WEB_SEARCH: 'off', STACK_BENCH_SUBAGENTS: 'on' }), + { webSearch: false, subagents: true }); + assert.throws(() => toolPolicyFromEnvironment({ STACK_BENCH_WEB_SEARCH: 'true' }), /STACK_BENCH_WEB_SEARCH/); +}); + +test('Claude Code keeps --bare and refuses a policy that turns on tools --bare does not have', () => { + const args = (tools: ToolPolicy) => CODING_PROVIDERS.anthropic.args({ model: 'claude-test', effort: 'high', + baseUrl: 'http://broker', resumeSession: null, maxBudgetUsd: null, tools }); + const off = args({ webSearch: false, subagents: false }); + assert(off.includes('--bare')); + assert(!off.some(arg => /tools/i.test(arg)), 'the 1.1 arguments, with no tool flags'); + for (const tools of POLICIES.filter(policy => policy.webSearch || policy.subagents)) { + assert.throws(() => args(tools), /cannot be on/); + } + assert.match(unavailableToolsReason('anthropic', { webSearch: true, subagents: false })!, /webSearch.*--bare/); + for (const provider of ['openai', 'xai', 'google', 'openrouter', null]) { + assert.equal(unavailableToolsReason(provider, { webSearch: true, subagents: false }), null); + } +}); + +test('subagents are refused for every agent until the audit is shown to see their work', () => { + for (const provider of ['anthropic', 'openai', 'xai', 'google', 'openrouter', null]) { + for (const webSearch of [false, true]) { + assert.match(unavailableToolsReason(provider, { webSearch, subagents: true })!, /subagents.*audit/); + } + } +}); + +test('an extension matches a parent planned before campaigns recorded a tool policy', () => { + const attempt = { stack: 'postgres', model: 'gpt-test', agentAdapter: 'codex', repetition: 1, + condition: { id: 'neutral', guidance: { mode: 'neutral' }, repair: null } } as unknown as CampaignAttemptPlan; + assert.equal(extensionAttemptKey(attempt), extensionAttemptKey({ ...attempt, tools: { ...DEFAULT_TOOL_POLICY } })); + assert.notEqual(extensionAttemptKey(attempt), extensionAttemptKey({ ...attempt, tools: { webSearch: true, subagents: false } })); +}); + +test('the broker passes Antigravity its web and subagent functions only when the policy turns them on', () => { + const declared = [...ANTIGRAVITY_TOOLS, ...ANTIGRAVITY_WEB_TOOLS, ...ANTIGRAVITY_SUBAGENT_TOOLS, 'ask_question', + 'browser_subagent', 'schedule_message']; + for (const tools of [undefined, ...POLICIES]) { + const protocol = brokerProtocol({ provider: 'google', mode: 'api-key', credential: 'x'.repeat(16), + sessionToken: 'y'.repeat(16), model: 'test-model', maxOutputTokens: 4096, ...(tools ? { tools } : {}) }); + const body = Buffer.from(JSON.stringify({ contents: [], tools: [{ functionDeclarations: declared.map(name => ({ name })) }] })); + const forwarded = protocol.parseRequest(body, '/v1beta/models/test-model:streamGenerateContent') as { + tools: Array<{ functionDeclarations: Array<{ name: string }> }> }; + assert.deepEqual(forwarded.tools[0]!.functionDeclarations.map(declaration => declaration.name), [...ANTIGRAVITY_TOOLS, + ...(tools?.webSearch ? ANTIGRAVITY_WEB_TOOLS : []), ...(tools?.subagents ? ANTIGRAVITY_SUBAGENT_TOOLS : [])]); + // Google's own search and URL tools are web search too. + const hosted = Buffer.from(JSON.stringify({ contents: [], tools: [{ googleSearch: {} }, { urlContext: {} }, + { functionDeclarations: [{ name: 'view_file' }], googleSearchRetrieval: {} }] })); + const sent = (protocol.parseRequest(hosted, '/v1beta/models/test-model:streamGenerateContent') as { + tools: Array> }).tools; + assert.deepEqual(sent, tools?.webSearch + ? [{ googleSearch: {} }, { urlContext: {} }, { functionDeclarations: [{ name: 'view_file' }], googleSearchRetrieval: {} }] + : [{ functionDeclarations: [{ name: 'view_file' }] }], JSON.stringify(tools)); + } +}); + +test('a declared web search is priced when unused, and each search is counted when used', () => { + const base = { mode: 'api-key' as const, credential: 'x'.repeat(16), sessionToken: 'y'.repeat(16), model: 'test-model', + maxOutputTokens: 4096 }; + const sse = (events: unknown[]) => Buffer.from(events.map(event => `data: ${JSON.stringify(event)}\n\n`).join('')); + + const anthropic = brokerProtocol(base); + assert.deepEqual(anthropic.requestPricing({ tools: [{ type: 'web_search_20250305', name: 'web_search' }] }), + { unpriced: null, requiresUsage: 'server-tool', bounded: false }); + const message = (requests: number) => Buffer.from(JSON.stringify({ usage: { input_tokens: 1, output_tokens: 1, + server_tool_use: { web_search_requests: requests } } })); + assert.equal(anthropic.responseUsage(message(3))?.web_search_calls, 3); + assert.equal(anthropic.responseUsage(message(3))?.unpriced_reason, 'server-tool'); + assert.equal(anthropic.responseUsage(message(0))?.unpriced_reason, null); + + const usage = { input_tokens: 10, output_tokens: 2, input_tokens_details: { cached_tokens: 0 } }; + for (const provider of ['openai', 'xai'] as const) { + const responses = brokerProtocol({ ...base, provider }); + assert.deepEqual(responses.requestPricing({ tools: [{ type: 'function', name: 'shell' }, { type: 'web_search' }] }), + { unpriced: null, requiresUsage: 'hosted-tool', bounded: false }, provider); + assert.equal(responses.requestPricing({ tools: [{ type: 'file_search' }] }).unpriced, 'hosted-tool'); + const searched = responses.responseUsage(sse([ + { type: 'response.output_item.done', item: { id: 'ws_1', type: 'web_search_call', status: 'completed' } }, + { type: 'response.output_item.done', item: { id: 'ws_2', type: 'web_search_call', status: 'completed' } }, + { type: 'response.completed', response: { usage, output: [{ id: 'ws_1', type: 'web_search_call' }, + { id: 'ws_2', type: 'web_search_call' }, { id: 'msg_1', type: 'message' }] } }, + ])); + assert.equal(searched?.web_search_calls, 2, provider); + assert.equal(searched?.unpriced_reason, 'hosted-tool', provider); + const unused = responses.responseUsage(sse([{ type: 'response.completed', response: { usage, output: [] } }])); + assert.equal(unused?.web_search_calls, 0, provider); + assert.equal(unused?.unpriced_reason, null, provider); + } + + const gemini = brokerProtocol({ ...base, provider: 'google' }); + assert.deepEqual(gemini.requestPricing({ tools: [{ googleSearch: {} }] }), + { unpriced: null, requiresUsage: 'hosted-tool', bounded: false }); + assert.equal(gemini.requestPricing({ tools: [{ codeExecution: {} }] }).unpriced, 'hosted-tool'); + const grounded = gemini.responseUsage(sse([ + { candidates: [{ groundingMetadata: { webSearchQueries: ['spacetimedb views', 'spacetimedb procedures'] } }] }, + { usageMetadata: { promptTokenCount: 10, candidatesTokenCount: 2 } }, + ])); + assert.equal(grounded?.web_search_calls, 2); + assert.equal(grounded?.unpriced_reason, 'hosted-tool'); + assert.equal(gemini.responseUsage(sse([{ usageMetadata: { promptTokenCount: 10, candidatesTokenCount: 2 } }])) + ?.unpriced_reason, null); +}); diff --git a/tools/stack-bench/tests/tracks.test.ts b/tools/stack-bench/tests/tracks.test.ts new file mode 100644 index 00000000000..89c79d7211e --- /dev/null +++ b/tools/stack-bench/tests/tracks.test.ts @@ -0,0 +1,40 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { loadTrack, portsFor, RUN_INDEX_CAP, suitesFor } + from '../src/composition/tracks.js'; +import type { TrackSuiteSource } from '../src/composition/tracks.js'; + +test('an undeclared level never falls back to L1 grading', () => { + const chat = loadTrack('chat'); + assert.throws(() => suitesFor(chat, 3), /No scenario suites declared/); +}); + +test('invalid track and run-index inputs throw instead of terminating the process', () => { + assert.throws(() => loadTrack('missing-track'), /Unknown track/); + const track = loadTrack('ecommerce'); + for (const runIndex of [-1, 0.5, RUN_INDEX_CAP + 1]) { + assert.throws(() => portsFor(track, 'mongodb', runIndex), /integer from 0 through/); + } +}); + +test('suite inheritance follows the manifest policy instead of suite-name magic', () => { + const ecommerce = loadTrack('ecommerce'); + const inherited = suitesFor(ecommerce, 3).filter(suite => suite.inherited); + assert.deepEqual(inherited.map(suite => suite.id), [ + 'invariants@L1', 'contention@L1', 'invariants@L2', + ]); + + const synthetic: TrackSuiteSource = { + name: 'synthetic', + dir: ecommerce.dir, + suites: { + 1: [ + { id: 'features', spec: 'feature.json', inherit: 'all-higher-levels' }, + { id: 'invariants', spec: 'invariant.json', inherit: 'none' }, + ], + 2: [{ id: 'next', spec: 'next.json', inherit: 'none' }], + }, + }; + assert.deepEqual(suitesFor(synthetic, 2).map(suite => suite.id), ['next', 'features@L1']); +}); diff --git a/tools/stack-bench/tests/transport-cache.integration.ts b/tools/stack-bench/tests/transport-cache.integration.ts new file mode 100644 index 00000000000..2e647949d48 --- /dev/null +++ b/tools/stack-bench/tests/transport-cache.integration.ts @@ -0,0 +1,502 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { once } from 'node:events'; +import { readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { createServer } from 'node:http'; +import { createRequire } from 'node:module'; +import { createGzip, constants } from 'node:zlib'; +import { dirname, join } from 'node:path'; +import test from 'node:test'; +import { chromium, type CDPSession } from 'playwright'; +import { Actor, gradeFeature } from '../grader/grade.js'; +import { executeAction, type ActionEvidence } from '../src/actions/action-contract.js'; +import { ACTION_REGISTRY } from '../src/actions/action-catalog.js'; +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; +import { startNetworkInterruption } from '../src/actions/network-interruption.js'; + +// Failure cases specified before the fix: decoded cached fonts have no readable +// body; fresh/reopened actors can lose capture settings; cache bypass can leak +// into ordinary checks; and a privacy fix must still detect delivered secrets. +test('privacy actors retain font bodies across reloads without changing ordinary HTTP caching', async t => { + const assets = join(dirname(createRequire(import.meta.url).resolve('playwright-core/package.json')), 'lib/vite/traceViewer'); + const fontName = readdirSync(assets).find(name => name.endsWith('.ttf')); + assert(fontName, 'Use the real font already shipped with the pinned browser tooling'); + const font = readFileSync(join(assets, fontName)); + let leak = false; + let fontRequests = 0; + const server = createServer((req, res) => { + if (req.url === '/font') { + fontRequests++; + // Reproduce the observed font response MIME, with valid decodable bytes. + res.writeHead(200, { 'Content-Type': 'text/html', 'Cache-Control': 'public, max-age=3600' }); + res.end(Buffer.concat([font, Buffer.from(leak ? 'private-font-canary' : '')])); + } else { + res.writeHead(200, { 'Content-Type': 'text/html', 'Cache-Control': 'no-store' }); + res.end(` + waiting`); + } + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + t.after(() => { server.closeAllConnections(); server.close(); }); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const browser = await chromium.launch({ headless: true }); + t.after(() => browser.close()); + const evidence: unknown[] = []; + t.after(() => { + if (process.env.STACK_BENCH_CAPTURE_EVIDENCE) writeFileSync(process.env.STACK_BENCH_CAPTURE_EVIDENCE, + JSON.stringify({ rerun: 'STACK_BENCH_CAPTURE_EVIDENCE= node --test dist/tests/transport-cache.integration.js', + browser: browser.version(), node: process.version, fontSha256: createHash('sha256').update(font).digest('hex'), + executables: ['./transport-cache.integration.js', '../grader/grade.js', '../grader/transport-frames.js'] + .map(path => ({ path, sha256: createHash('sha256').update(readFileSync(new URL(path, import.meta.url))).digest('hex') })), + evidence }, null, 2)); + }); + for (const phase of ['ordinary', 'initial', 'replacement', 'fresh', 'race']) { + for (leak of phase === 'ordinary' ? [false] : [false, true]) { + await t.test(`${phase}, leak=${leak}`, async () => { + fontRequests = 0; + const actor = phase === 'fresh' ? 'reader-fresh' : 'reader'; + const loaded = { do: 'expect', actor, testid: 'loaded', contains: 'decoded', within: 3000 }; + const prepare = phase === 'fresh' ? [{ do: 'freshClient', actor: 'reader' }] + : phase === 'replacement' ? [{ do: 'closeClient', actor }, { do: 'openClient', actor, settleMs: 0 }] : []; + const observation = { do: 'expectNotReceived', actor, contains: 'private-font-canary', within: 100 }; + const feature = compileScenarioDefinition({ schemaVersion: 1, track: 'ecommerce', level: 1, features: [{ + id: 1, name: 'cached response capture', actors: ['reader'], + setup: [{ ...loaded, actor: 'reader' }], + criteria: [{ id: 'capture', desc: 'decoded responses remain observable', points: 1, + steps: [...prepare, loaded, + { do: 'reload', actor, settleMs: 0 }, loaded, + { do: 'reload', actor, settleMs: 0 }, loaded, + ...(phase === 'ordinary' ? [] : phase === 'race' + ? [{ do: 'race', settleMs: 0, branches: [[observation], [loaded]] }] : [observation])], + }], + }] }).features[0]!; + const result = await gradeFeature(browser, feature, + { url, level: 1, headed: false, selectedCheckKeys: [], nullControl: false }, + { runId: 'cache-capture', roomName: name => name, url, actions: [], spacetime: null, + backend: 'postgres', nullControl: false, defaultWithin: 3000 }); + evidence.push({ phase, leak, fontRequests, result }); + assert.equal(result.setupEvidence.status, 'passed'); + assert.equal(result.criteria[0]!.evidence.status, leak ? 'failed' : 'passed', JSON.stringify(result)); + if (phase === 'ordinary') assert.equal(fontRequests, 1, 'Ordinary checks must still use native caching'); + else assert(fontRequests >= 3, 'Privacy observations must capture fresh response bytes'); + assert.equal(result.cleanupEvidence, undefined, 'No cleanup failure'); + assert.equal(browser.contexts().length, 0, 'All actor contexts must be closed'); + }); + } + } +}); + +// A privacy verdict can coincide with an ordinary response still being read. +// Allow bounded completion, detect a late body leak, and never pass a hung body. +test('privacy verdict drains an in-flight body and keeps a stalled response inconclusive', async t => { + let mode = 'clean'; + const server = createServer((req, res) => { + if (req.url === '/private-route?token=private-query') { + const body = mode.endsWith('leak') ? 'private-body-canary' + : mode === 'captured' ? 'x'.repeat(8 * 1024 * 1024 - 32) : 'public'; + const respond = () => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.write('{"value":"'); + if (mode !== 'stalled') setTimeout(() => res.end(`${body}"}`), 400); + }; + if (mode.startsWith('late-')) setTimeout(respond, 150); + else if (mode !== 'headers-stalled') respond(); + } else { + res.writeHead(200, { 'Content-Type': 'text/html' }); + res.end(`${mode === 'captured' ? 'private-body-canary' : ''}${mode.startsWith('late-') || mode === 'headers-stalled' ? 'request started' : 'waiting'}`); + } + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + t.after(() => { server.closeAllConnections(); server.close(); }); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const browser = await chromium.launch({ headless: true }); + t.after(() => browser.close()); + const evidence: unknown[] = []; + t.after(() => { + if (process.env.STACK_BENCH_CAPTURE_EVIDENCE) writeFileSync(`${process.env.STACK_BENCH_CAPTURE_EVIDENCE}.pending.json`, + JSON.stringify({ browser: browser.version(), evidence }, null, 2)); + }); + // Headers can arrive after the observation window; an unanswered started + // request must not establish absence, and a late leak must still fail. + for (mode of ['clean', 'leak', 'captured', 'stalled', 'late-clean', 'late-leak', 'headers-stalled']) { + await t.test(mode, async () => { + // A stalled body must identify its source without logging private paths or queries. + const stderr = t.mock.method(process.stderr, 'write'); + const feature = compileScenarioDefinition({ schemaVersion: 1, track: 'ecommerce', level: 1, features: [{ + id: 1, name: 'response completion', actors: ['reader'], setup: [], + criteria: [{ id: 'capture', desc: 'no private response body', points: 1, steps: [ + { do: 'expect', actor: 'reader', testid: 'ready', contains: mode.startsWith('late-') || mode === 'headers-stalled' ? 'request started' : 'headers received' }, + { do: 'expectNotReceived', actor: 'reader', contains: 'private-body-canary', within: 50 }, + ] }], + }] }).features[0]!; + const result = await gradeFeature(browser, feature, + { url, level: 1, headed: false, selectedCheckKeys: [], nullControl: false }, + { runId: 'body-completion', roomName: name => name, url, actions: [], spacetime: null, + backend: 'postgres', nullControl: false, defaultWithin: 1000 }); + const diagnostics = stderr.mock.calls.map(call => String(call.arguments[0])); + evidence.push({ mode, result, diagnostics }); + assert.equal(result.criteria[0]!.evidence.status, + mode.endsWith('clean') ? 'passed' : mode.endsWith('stalled') ? 'inconclusive' : 'failed', JSON.stringify(result)); + assert.equal(browser.contexts().length, 0); + if (mode === 'headers-stalled') { + // Requests with no response headers must leave bounded, safe diagnostics. + const lines = diagnostics.filter(line => line.startsWith('transport request pending ')); + assert.equal(lines.length, 8, 'Only the first eight outstanding requests are logged'); + for (const line of lines) { + const pending = JSON.parse(line.slice('transport request pending '.length)) as Record; + assert.equal(pending.phase, 'absence'); + assert.equal(pending.method, 'GET'); + assert.equal(pending.resourceType, 'fetch'); + assert.equal(pending.origin, url); + assert.equal(pending.pathSha256, createHash('sha256').update('/private-route').digest('hex')); + assert(typeof pending.elapsedMs === 'number' && pending.elapsedMs >= 0); + assert.deepEqual(Object.keys(pending).sort(), ['elapsedMs', 'method', 'origin', 'pathSha256', 'phase', 'resourceType']); + assert(!/private-route|private-query|private-header|private-body-canary/.test(line)); + } + } + if (mode === 'stalled') { + const pending = stderr.mock.calls.map(call => String(call.arguments[0])) + .find(line => line.startsWith('transport request pending ')); + assert(pending, 'The inconclusive result must identify the pending response'); + assert(pending.includes('"phase":"absence"')); + assert(pending.includes('"resourceType":"fetch"')); + assert(!/private-route|private-query|private-body-canary/.test(pending)); + } + }); + } +}); + +// A harness reload must not cancel a preceding read or write. A GET can start +// before navigation without its headers arriving until after the reload begins. +// Both methods must capture leaks and keep stalled responses inconclusive. +test('privacy reload preserves preceding read and write responses', async t => { + let leak = false, delayedHeaders = false, stalled = false, capturedLeak = false, eventSource = false, eventLeak = false, method = 'PUT'; + let pageLoads = 0; + const server = createServer((req, res) => { + res.setHeader('Cache-Control', 'no-store'); + if (req.url === '/events') { + res.writeHead(200, { 'Content-Type': 'text/event-stream' }); + res.write(`data: ${eventLeak ? 'private-save-canary' : 'public'}\n\n`); + } else if (req.url === '/api/profile?token=private-query') { + const start = () => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.write('{"value":"'); + if (!stalled) setTimeout(() => res.end(`${leak ? 'private-save-canary' : 'public'}"}`), 300); + }; + if (delayedHeaders) setTimeout(start, 150); else start(); + } else { + res.writeHead(200, { 'Content-Type': 'text/html' }); + res.end(`${capturedLeak && pageLoads++ === 0 ? 'private-save-canary' : ''}${eventSource ? "" : ''}`); + } + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + t.after(() => { server.closeAllConnections(); server.close(); }); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const browser = await chromium.launch({ headless: true }); + t.after(() => browser.close()); + const evidence: unknown[] = []; + t.after(() => { + if (process.env.STACK_BENCH_CAPTURE_EVIDENCE) writeFileSync(`${process.env.STACK_BENCH_CAPTURE_EVIDENCE}.reload.json`, + JSON.stringify({ browser: browser.version(), evidence }, null, 2)); + }); + // A captured disclosure is a failure even if another request never finishes. + for (const requestMethod of ['PUT', 'GET']) for (const mode of ['clean', 'leak', 'late-clean', 'late-leak', 'stalled', 'captured-stalled', 'eventsource-clean', 'eventsource-leak']) { + await t.test(`${requestMethod} ${mode}`, async () => { + const stderr = t.mock.method(process.stderr, 'write'); + method = requestMethod; + pageLoads = 0; + eventSource = mode.startsWith('eventsource'); eventLeak = mode === 'eventsource-leak'; + leak = !eventSource && mode.endsWith('leak'); delayedHeaders = mode.startsWith('late'); stalled = mode.endsWith('stalled'); capturedLeak = mode === 'captured-stalled'; + const feature = compileScenarioDefinition({ schemaVersion: 1, track: 'ecommerce', level: 1, features: [{ + id: 1, name: 'save then reload', actors: ['reader'], setup: [], + criteria: [{ id: 'capture', desc: 'the save response is captured before controlled reload', points: 1, steps: [ + { do: 'click', actor: 'reader', testid: 'profile-save' }, + { do: 'reload', actor: 'reader', settleMs: 0 }, + { do: 'expectNotReceived', actor: 'reader', contains: 'private-save-canary', within: 50 }, + ] }], + }] }).features[0]!; + const result = await gradeFeature(browser, feature, + { url, level: 1, headed: false, selectedCheckKeys: [], nullControl: false }, + { runId: 'save-reload', roomName: name => name, url, actions: [], spacetime: null, + backend: 'postgres', nullControl: false, defaultWithin: 1000 }); + const diagnostics = stderr.mock.calls.map(call => String(call.arguments[0])); + evidence.push({ method, mode, result, diagnostics }); + assert.equal(result.criteria[0]!.evidence.status, capturedLeak || leak || eventLeak ? 'failed' : stalled ? 'inconclusive' : 'passed', JSON.stringify(result)); + if (mode === 'stalled') { + const lines = diagnostics.filter(line => line.startsWith('transport request pending ')); + assert.equal(lines.length, 1, 'The navigation deadline must identify the outstanding request'); + const pending = JSON.parse(lines[0]!.slice('transport request pending '.length)) as Record; + assert.equal(pending.phase, 'navigation'); + assert.equal(pending.method, method); + assert.equal(pending.resourceType, 'fetch'); + assert.equal(pending.origin, url); + assert.equal(pending.pathSha256, createHash('sha256').update('/api/profile').digest('hex')); + assert(typeof pending.elapsedMs === 'number' && pending.elapsedMs >= 1000); + assert.deepEqual(Object.keys(pending).sort(), ['elapsedMs', 'method', 'origin', 'pathSha256', 'phase', 'resourceType']); + assert(!/api\/profile|private-query|private-header|private-request-body|private-save-canary/.test(lines[0]!)); + // The interrupted read must retain its cause in both durable evidence layers. + const persisted = JSON.parse(JSON.stringify(result)) as typeof result; + const check = persisted.criteria[0]!.evidence; + assert(check.finding?.kind === 'transport-incomplete', 'The criterion must retain the structured capture finding'); + assert(check.finding.fields.capture); + assert.equal(check.finding.fields.capture.navigationInterrupted, 1); + const action = check.actions.at(-1)!.evidence as ActionEvidence; + assert.deepEqual(action.finding, check.finding, 'Action and criterion must preserve the same cause'); + assert.match(check.summary!, /navigation interruptions: 1/); + assert.match(action.summary!, /navigation interruptions: 1/); + } + assert.equal(browser.contexts().length, 0); + }); + } +}); + +// Failure matrix, before observer changes: normal reload must not erase bytes +// already delivered; split UTF-8 and gzip retain exact markers; a finite-window +// observation keeps a silent live request inconclusive. Detached capture and byte overflow +// remain inconclusive, while an already captured leak takes precedence. +test('privacy observes HTTP delivery through reload during a finite window', async t => { + let mode = ''; + let loads = 0; + const marker = 'private-🔒-marker'; + const wire: { event: string; at: number; bytes?: number }[] = []; + const server = createServer((req, res) => { + res.setHeader('Cache-Control', 'no-store'); + if (req.url === '/poll') { + wire.push({ event: 'request', at: Date.now() }); + res.on('close', () => wire.push({ event: 'close', at: Date.now() })); + if (mode === 'silent') return; + const start = () => { + if (res.destroyed) return; + const gzip = mode === 'gzip-leak'; + res.writeHead(200, { 'Content-Type': 'application/json', ...(gzip ? { 'Content-Encoding': 'gzip' } : {}) }); + const output = gzip ? createGzip() : res; + if (output !== res) output.pipe(res); + const text = (mode.includes('leak') && mode !== 'future-leak') || mode === 'completed' ? marker : 'public'; + const body = Buffer.from(`{"value":"${text}"}`); + if (mode === 'overflow') { + res.end('x'.repeat(8 * 1024 * 1024 + 1)); + wire.push({ event: 'overflow-sent', at: Date.now() }); + } else if (mode === 'completed' || mode.endsWith('capture-lost')) { + res.end(body); + wire.push({ event: 'complete', at: Date.now(), bytes: body.length }); + } else { + // Split inside the four-byte lock character, not just between words. + const split = text === marker ? body.indexOf(Buffer.from('🔒')) + 2 : 8; + output.write(body.subarray(0, split)); + if (gzip) (output as ReturnType).flush(constants.Z_SYNC_FLUSH); + wire.push({ event: 'first-chunk', at: Date.now(), bytes: split }); + setTimeout(() => { + if (res.destroyed) return; + output.write(body.subarray(split)); + if (gzip) (output as ReturnType).flush(constants.Z_SYNC_FLUSH); + wire.push({ event: 'second-chunk', at: Date.now(), bytes: body.length - split }); + }, 25); + setTimeout(() => { if (!res.destroyed) { output.end(); wire.push({ event: 'complete', at: Date.now() }); } }, 350); + } + }; + if (mode === 'zero-headers') setTimeout(start, 200); + else start(); + } else if (req.url === '/later') { + res.setHeader('Content-Type', 'application/json'); + setTimeout(() => { if (!res.destroyed) { res.end(JSON.stringify({ value: marker })); wire.push({ event: 'later-leak', at: Date.now() }); } }, 100); + } else { + loads++; + res.setHeader('Content-Type', 'text/html'); + res.end(`ready${loads > 1 && mode === 'future-leak' ? '' : ''}`); + } + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + t.after(() => { server.closeAllConnections(); server.close(); }); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + // Match the appliance's full Chromium executable, not headless_shell. + const browser = await chromium.launch({ headless: true, executablePath: chromium.executablePath() }); + t.after(() => browser.close()); + const evidence: unknown[] = []; + t.after(() => { + if (process.env.STACK_BENCH_CAPTURE_EVIDENCE) writeFileSync(`${process.env.STACK_BENCH_CAPTURE_EVIDENCE}.finite-http.json`, + JSON.stringify({ browser: browser.version(), executable: chromium.executablePath(), evidence }, null, 2)); + }); + for (mode of ['clean', 'zero-headers', 'split-leak', 'gzip-leak', 'future-leak', 'completed', 'silent', 'capture-lost', 'overflow', 'leak-capture-lost']) { + await t.test(mode, async caseT => { + wire.length = 0; loads = 0; + const sessions: CDPSession[] = []; + const newContext = browser.newContext.bind(browser); + caseT.mock.method(browser, 'newContext', async (...args: Parameters) => { + const context = await newContext(...args); + const newSession = context.newCDPSession.bind(context); + caseT.mock.method(context, 'newCDPSession', async (...sessionArgs: Parameters) => { + const session = await newSession(...sessionArgs); + sessions.push(session); + return session; + }); + const newPage = context.newPage.bind(context); + caseT.mock.method(context, 'newPage', async () => { + const page = await newPage(); + const reload = page.reload.bind(page); + caseT.mock.method(page, 'reload', async (...reloadArgs: Parameters) => { + // A real fetch starts at the boundary of the normal reload action. + // No route is intercepted and no application traffic is suppressed. + const request = page.waitForRequest(`${url}/poll`); + const response = mode === 'zero-headers' ? null : page.waitForResponse(`${url}/poll`); + await page.evaluate(() => { void fetch('/poll').then(r => r.text()).catch(() => {}); }); + await request; + if (response) { + const received = await response; + if (mode === 'completed' || mode.endsWith('capture-lost')) { + await received.finished(); + if (mode === 'completed') await page.waitForTimeout(120); + } + else await page.waitForTimeout(60); + } + wire.push({ event: 'reload', at: Date.now() }); + const result = await reload(...reloadArgs); + if (mode.endsWith('capture-lost')) { + for (const session of sessions) await session.detach(); + wire.push({ event: 'observer-detached', at: Date.now() }); + } + return result; + }); + return page; + }); + return context; + }); + const feature = compileScenarioDefinition({ schemaVersion: 1, track: 'ecommerce', level: 1, features: [{ + id: 1, name: 'finite HTTP privacy window', actors: ['reader'], setup: [], criteria: [{ + id: 'capture', desc: 'all delivered bytes remain observable through navigation', points: 1, steps: [ + { do: 'expect', actor: 'reader', testid: 'loaded', contains: 'ready' }, + ...(mode === 'silent' || mode === 'overflow' ? [{ do: 'click', actor: 'reader', testid: 'poll' }] + : [{ do: 'reload', actor: 'reader', settleMs: 0 }]), + ...(mode === 'overflow' ? [{ do: 'expect', actor: 'reader', testid: 'loaded', contains: 'consumed' }] : []), + { do: 'expectNotReceived', actor: 'reader', contains: marker, within: 250 }, + ], + }], + }] }).features[0]!; + const result = await gradeFeature(browser, feature, + { url, level: 1, headed: false, selectedCheckKeys: [], nullControl: false }, + { runId: 'finite-http', roomName: name => name, url, actions: [], spacetime: null, + backend: 'postgres', nullControl: false, defaultWithin: 1000 }); + evidence.push({ mode, wire: [...wire], result }); + assert.equal(result.setupEvidence.status, 'passed', JSON.stringify(result)); + const expected = mode.includes('leak') || mode === 'completed' ? 'failed' + : mode === 'capture-lost' || mode === 'overflow' || mode === 'silent' ? 'inconclusive' : 'passed'; + assert.equal(result.criteria[0]!.evidence.status, expected, JSON.stringify(evidence.at(-1))); + if (mode === 'split-leak' || mode === 'gzip-leak') { + assert(wire.find(event => event.event === 'second-chunk')!.at <= wire.find(event => event.event === 'reload')!.at, + 'The complete marker must reach the wire before reload'); + } + if (mode === 'future-leak') assert(wire.some(event => event.event === 'later-leak')); + assert.equal(browser.contexts().length, 0, 'All actor contexts must close'); + }); + } +}); + +// An app that keeps reading while a privacy check runs must not make the check inconclusive. +test('a response that arrives while a proxy snapshot is in flight is not reported missing', async t => { + const server = createServer((req, res) => { + if (req.url === '/api/profile') { res.setHeader('Content-Type', 'application/json'); res.end('{"name":"Blake Reed"}'); } + else { res.setHeader('Content-Type', 'text/html'); res.end('profile'); } + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + t.after(() => { server.closeAllConnections(); server.close(); }); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const browser = await chromium.launch({ headless: true, executablePath: chromium.executablePath() }); + const proxy = await startNetworkInterruption(true); + const context = await browser.newContext({ proxy: proxy.proxy }); + const page = await context.newPage(); + // The first snapshot is taken, then held back until a later read has reached the browser. + let snapshots = 0; + const racing = { ...proxy, httpSnapshot: async () => { + const snapshot = await proxy.httpSnapshot(); + if (snapshots++ === 0) { + const read = page.waitForResponse(`${url}/api/profile`); + await page.evaluate(() => fetch('/api/profile').then(response => response.json())); + await (await read).finished(); + await page.waitForTimeout(100); // the observer notes the receipt after reading the server address + } + return snapshot; + } }; + const actor = new Actor('reader', page, context, false, false, false, [], true, racing); + t.after(async () => { await context.close(); await proxy.dispose(); await browser.close(); }); + await actor.ready; + await page.goto(url); + await actor.syncReceived(); + assert(snapshots > 1, 'the late receipt needs a later snapshot'); + assert.equal(actor.wasSent('14 Market Street'), false, 'the capture stays complete'); + assert.equal(actor.wasSent('Blake Reed', false), true, 'the late response is inspected'); +}); + +// Partial bytes may prove a privacy failure, but must not become complete JSON +// for ID discovery or confirm a write for bulk replay before its response ends. +test('HTTP privacy fragments do not become completed source or confirmed writes', async t => { + const marker = 'private-write-marker'; + const json = JSON.stringify({ id: 7, value: marker, padding: 'x'.repeat(2 * 1024 * 1024) }); + let finish: (() => void) | undefined; + let append: ((end: number) => void) | undefined; + const server = createServer((req, res) => { + if (req.url === '/api/write') { + res.writeHead(200, { 'Content-Type': 'application/json' }); + let sent = 100; + res.write(json.slice(0, sent)); + append = end => { res.write(json.slice(sent, end)); sent = end; }; + finish = () => res.end(); + } else { res.setHeader('Content-Type', 'text/html'); res.end('write receipt'); } + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + t.after(() => { server.closeAllConnections(); server.close(); }); + const url = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + const browser = await chromium.launch({ headless: true, executablePath: chromium.executablePath() }); + const proxy = await startNetworkInterruption(true); + const context = await browser.newContext({ proxy: proxy.proxy }); + const page = await context.newPage(); + const actor = new Actor('reader', page, context, false, false, false, [`${url}/api/write`], true, proxy); + t.after(async () => { await context.close(); await proxy.dispose(); await browser.close(); }); + await actor.ready; + await page.goto(url); + const response = page.waitForResponse(`${url}/api/write`); + await page.evaluate(() => { void fetch('/api/write', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: '{"name":"item"}' }).then(r => r.json()); }); + const received = await response; + const observation = await executeAction(ACTION_REGISTRY, 'expectNotReceived', + { do: 'expectNotReceived', actor: 'reader', contains: marker, within: 100 }, { + capabilities: { + actors: { get: (name: string) => name === 'reader' ? actor : undefined }, + 'transport-observation': { + defaultWithin: 1000, expand: (value: string) => value, + sleep: (milliseconds: number) => new Promise(resolve => setTimeout(resolve, milliseconds)), + verification: { structural() {}, unverified() {}, verified() {} }, + }, + }, + }); + const before = { received: [...actor.received], confirmed: actor.writes.at(-1)?.confirmed ?? null, observation }; + assert(finish, 'The real response must have delivered its first bytes'); + assert(append); + // Repeated observations of one growing 2 MiB body must not exhaust the + // capture budget by counting every prefix as a separate response. + for (let end = 256 * 1024; end <= json.length + 256 * 1024; end += 256 * 1024) { + const length = Math.min(end, json.length); + append(length); + const deadline = Date.now() + 3000; + while (!(await proxy.httpSnapshot()).records.some(record => record.body.length === length)) { + assert(Date.now() < deadline, 'The proxy must receive the next real body prefix'); + await page.waitForTimeout(10); + } + await actor.syncReceived(); + } + finish(); + await received.finished(); + const deadline = Date.now() + 1000; + while ((!actor.received.includes(json) || actor.writes.at(-1)?.confirmed !== true) && Date.now() < deadline) await page.waitForTimeout(10); + const after = { received: [...actor.received], confirmed: actor.writes.at(-1)?.confirmed ?? null }; + await actor.syncReceived(); + assert.equal(actor.wasSent('not-in-this-response'), false, 'Growing prefixes must count once against the byte limit'); + if (process.env.STACK_BENCH_CAPTURE_EVIDENCE) writeFileSync(`${process.env.STACK_BENCH_CAPTURE_EVIDENCE}.write-boundary.json`, + JSON.stringify({ browser: browser.version(), before, after }, null, 2)); + assert(!before.received.some(body => body.includes(marker)), 'Partial JSON cannot supply discovered IDs'); + assert.notEqual(before.confirmed, true, 'Partial HTTP success cannot authorize bulk replay'); + assert.equal(observation.status, 'failed', JSON.stringify(observation)); + assert(after.received.includes(json), 'The completed response remains available to existing source consumers'); + assert.equal(after.confirmed, true, 'A complete successful response confirms the captured write'); +}); diff --git a/tools/stack-bench/tests/transport-frames.integration.ts b/tools/stack-bench/tests/transport-frames.integration.ts new file mode 100644 index 00000000000..92c862f9d26 --- /dev/null +++ b/tools/stack-bench/tests/transport-frames.integration.ts @@ -0,0 +1,110 @@ +import assert from 'node:assert/strict'; +import { createServer } from 'node:http'; +import test from 'node:test'; +import { chromium } from 'playwright'; +import { captureResponses, ReceivedTransport } from '../grader/transport-frames.js'; +import { ActionInconclusive } from '../src/actions/action-contract.js'; + +test('HTML and live SSE leaks are captured; an unsupported stream cannot pass', async t => { + const server = createServer((req, res) => { + if (req.url === '/events') { + res.writeHead(200, { 'Content-Type': 'text/event-stream' }); + res.write('data: private-canary\n\n'); + } else { + res.writeHead(200, { 'Content-Type': 'text/html' }); + res.end('transport control
private-html-canary
'); + } + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(() => { server.closeAllConnections(); server.close(); }); + const address = server.address(); + assert(address && typeof address !== 'string'); + const browser = await chromium.launch({ headless: true }); + t.after(() => browser.close()); + const page = await browser.newPage(); + const received = new ReceivedTransport(); + await captureResponses(page, received); + await page.goto(`http://127.0.0.1:${address.port}`); + for (let n = 0; n < 100 && !received.contains('private-html-canary', false); n++) await page.waitForTimeout(20); + assert.equal(received.contains('private-html-canary'), true); + await page.evaluate(() => new Promise(resolve => { + const events = new EventSource('/events'); + events.onmessage = () => resolve(); + })); + // CDP and page events use separate channels. Wait for the actual retained evidence. + for (let n = 0; n < 100 && !received.contains('private-canary', false); n++) await page.waitForTimeout(20); + assert.equal(received.contains('private-canary'), true); + assert.equal(received.contains('not-sent'), false); + await page.evaluate(async () => { await fetch('/events'); }); + for (let n = 0; n < 100 && !received.incomplete; n++) await page.waitForTimeout(20); + assert.throws(() => received.contains('not-sent'), ActionInconclusive); +}); + +test('HTTP privacy capture survives immediate app reload; disabled durability loses evidence', async t => { + const server = createServer((req, res) => { + const url = new URL(req.url ?? '/', 'http://fixture.local'); + res.setHeader('Cache-Control', 'no-store'); + if (url.pathname === '/json') { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ private: `private-marker-${url.searchParams.get('sample')}` })); + } else if (url.pathname === '/empty') { + res.writeHead(204, { 'Content-Type': 'application/json' }); + res.end(); + } else { + res.writeHead(200, { 'Content-Type': 'text/html' }); + res.end('reload capture control'); + } + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(() => { server.closeAllConnections(); server.close(); }); + const address = server.address(); + assert(address && typeof address !== 'string'); + const browser = await chromium.launch({ headless: true }); + t.after(() => browser.close()); + for (const durable of [true, false]) { + for (const kind of ['json', 'empty']) { + await t.test(`${kind}, durable=${durable}`, async () => { + const context = await browser.newContext(); + try { + if (!durable) { + // Negative control changes only the native capture setting, never the app's timing. + const createSession = context.newCDPSession.bind(context); + context.newCDPSession = async target => { + const session = await createSession(target); + const send = session.send.bind(session); + session.send = ((method, params) => method === 'Network.configureDurableMessages' + ? Promise.resolve({}) : send(method, params)) as typeof session.send; + return session; + }; + } + const page = await context.newPage(); + const received = new ReceivedTransport(); + await captureResponses(page, received); + await page.goto(`http://127.0.0.1:${address.port}`); + for (let sample = 0; sample < 10; sample++) { + const navigated = page.waitForEvent('framenavigated'); + await page.evaluate(({ kind, sample }) => { + // Launch the app action without tying the evaluation reply to the destroyed realm. + void (async () => { + const response = await fetch(`/${kind}?sample=${sample}`, { method: 'POST' }); + if (kind === 'json') await response.json(); + else await response.text(); + location.reload(); + })(); + }, { kind, sample }); + await navigated; + await page.waitForLoadState('load'); + for (let n = 0; n < 100 && received.pending; n++) await page.waitForTimeout(10); + if (durable && kind === 'json') assert.equal(received.contains(`private-marker-${sample}`), true); + } + if (durable) assert.equal(received.contains('never-delivered'), false); + else assert.throws(() => received.contains('never-delivered'), error => { + assert(error instanceof ActionInconclusive); + assert.match(error.message, /body-read failures: [1-9]/); + return true; + }); + } finally { await context.close(); } + }); + } + } +}); diff --git a/tools/stack-bench/tests/transport-frames.test.ts b/tools/stack-bench/tests/transport-frames.test.ts new file mode 100644 index 00000000000..28663a921aa --- /dev/null +++ b/tools/stack-bench/tests/transport-frames.test.ts @@ -0,0 +1,160 @@ +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import test from 'node:test'; +import { brotliCompressSync, gzipSync } from 'node:zlib'; +import type { Page } from 'playwright'; + +import { captureResponses, ReceivedTransport, transportFrameText } from '../grader/transport-frames.js'; +import { ActionInconclusive } from '../src/actions/action-contract.js'; +import { findingSchema } from '../src/actions/action-findings.js'; + +const message = 'subscription update with support-secret-619 inline'; + +function captureFailure(received: ReceivedTransport) { + try { received.contains('absent'); } + catch (error) { + assert(error instanceof ActionInconclusive); + const finding = findingSchema.parse(error.details.finding); + assert.equal(finding.kind, 'transport-incomplete'); + if (finding.kind === 'transport-incomplete') return finding.fields.capture; + } + assert.fail('Incomplete capture must remain inconclusive'); +} + +test('compressed SpacetimeDB frames decode to their message text and other frames keep their bytes', () => { + assert.equal(transportFrameText(Buffer.concat([Buffer.from([2]), gzipSync(message)])), message); + assert.equal(transportFrameText(Buffer.concat([Buffer.from([1]), brotliCompressSync(message)])), + message); + assert.equal(transportFrameText(Buffer.concat([Buffer.from([0]), Buffer.from(message)])), + `\u0000${message}`); + // Other frames keep their bytes. + assert.equal(transportFrameText(message), message); + assert.equal(transportFrameText(Buffer.from(message)), message); + const socketIo = Buffer.from('2["chat",{"text":"hi"}]'); + assert.equal(transportFrameText(socketIo), socketIo.toString('utf8')); + const notGzip = Buffer.from([2, 0x41, 0x42]); + assert.equal(transportFrameText(notGzip), notGzip.toString('utf8')); +}); + +test('transport absence cannot pass after truncation, eviction, or an unreadable body', () => { + const received = new ReceivedTransport(); + received.record('x'.repeat(200_001) + 'private-tail'); + assert.equal(received.contains('private-tail'), true); + assert.equal(received.contains('absent'), false); + received.pending = 1; + assert.throws(() => received.contains('absent'), /incomplete/); + assert.equal(captureFailure(received)?.pendingBodies, 1); + assert.equal(received.contains('absent', false), false); + received.pending = 0; + const bounded = new ReceivedTransport(20); + bounded.record('first-secret'); + bounded.record('first-secret'); + assert.equal(bounded.contains('absent'), false, 'Identical received text adds no evidence bytes'); + assert.deepEqual(bounded.chunks, ['first-secret']); + bounded.record('second-secret'); + assert.equal(bounded.contains('second-secret'), true); + assert.throws(() => bounded.contains('first-secret'), /incomplete/); + assert.deepEqual(captureFailure(bounded), { + byteLimit: 1, bodyReadFailures: 0, unsupportedStreams: 0, navigationInterrupted: 0, pendingBodies: 0, retainedBytes: 13, + }); + assert.equal(bounded.deliveriesDropped, true, 'Raw deliveries have their own bound'); + const oversized = new ReceivedTransport(5); + oversized.record('too-large'); + assert.throws(() => oversized.contains('large'), /incomplete/); + assert.equal(captureFailure(oversized)?.byteLimit, 1); + received.incomplete = true; + assert.throws(() => received.contains('absent'), /incomplete/); +}); + +test('capture failures identify body reads, declared body caps, and unsupported streams without payloads', async () => { + const commands: unknown[] = []; + const session = Object.assign(new EventEmitter(), { + send: async (method: string, params?: unknown) => { commands.push({ method, params }); }, + }); + const page = Object.assign(new EventEmitter(), { + context: () => ({ newCDPSession: async () => session }), + }); + const received = new ReceivedTransport(); + await captureResponses(page as unknown as Page, received); + assert.deepEqual(commands, [ + { method: 'Network.enable', params: undefined }, + { method: 'Network.configureDurableMessages', params: { + maxTotalBufferSize: 8 * 1024 * 1024, maxResourceBufferSize: 8 * 1024 * 1024, + } }, + ]); + page.emit('response', { + headers: () => ({ 'content-type': 'application/json' }), + text: async () => { throw new Error('private-token-and-body'); }, + }); + assert.equal(captureFailure(received)?.pendingBodies, 1); + await new Promise(resolve => setImmediate(resolve)); + assert.deepEqual(captureFailure(received), { + byteLimit: 0, bodyReadFailures: 1, unsupportedStreams: 0, navigationInterrupted: 0, pendingBodies: 0, retainedBytes: 0, + }); + page.emit('response', { + headers: () => ({ 'content-type': 'text/html', 'content-length': String(8 * 1024 * 1024 + 1) }), + text: () => { assert.fail('Oversized body must not be read'); }, + }); + session.emit('Network.responseReceived', { response: { mimeType: 'text/event-stream' }, type: 'Fetch' }); + assert.deepEqual(captureFailure(received), { + byteLimit: 1, bodyReadFailures: 1, unsupportedStreams: 1, navigationInterrupted: 0, pendingBodies: 0, retainedBytes: 0, + }); + received.record('observed-secret'); + assert.equal(received.contains('observed-secret'), true, 'A captured leak remains measurable'); + assert.equal(received.contains('absent', false), false, 'Positive receipt probes can still wait'); +}); + +test('privacy capture includes public script, style and error bodies', async () => { + const session = Object.assign(new EventEmitter(), { send: async () => {} }); + const page = Object.assign(new EventEmitter(), { context: () => ({ newCDPSession: async () => session }) }); + const received = new ReceivedTransport(); + await captureResponses(page as unknown as Page, received); + for (const type of ['application/javascript', 'text/javascript; charset=utf-8', 'application/ecmascript', + 'text/css', 'application/json', 'text/html', 'text/plain']) { + page.emit('response', { headers: () => ({ 'content-type': type }), status: () => 500, + text: async () => `private-sentinel-${type}` }); + await new Promise(resolve => setImmediate(resolve)); + assert.equal(received.contains(`private-sentinel-${type}`), true, type); + } + assert.equal(received.contains('absent'), false); +}); + +test('body failure diagnostics use bounded categories without error text or URL secrets', async t => { + const messages: string[] = []; + t.mock.method(process.stderr, 'write', (chunk: string) => { messages.push(String(chunk)); return true; }); + const session = Object.assign(new EventEmitter(), { send: async () => {} }); + const page = Object.assign(new EventEmitter(), { + context: () => ({ newCDPSession: async () => session }), isClosed: () => false, + }); + const received = new ReceivedTransport(); + await captureResponses(page as unknown as Page, received); + const cases: [unknown, string][] = [ + [new Error('Protocol error (Network.getResponseBody): Request content was evicted from inspector cache'), 'body-evicted'], + [new Error('Protocol error (Network.getResponseBody): No data found for resource with given identifier'), 'resource-unavailable'], + [new Error('Target page, context or browser has been closed'), 'target-closed'], + [new Error('response.body: net::ERR_BLOCKED_BY_ORB'), 'blocked-by-orb'], + [new Error('response.body: net::ERR_FAILED'), 'request-failed'], + [new Error('Protocol error (Network.getResponseBody): private-password'), 'protocol-error'], + [Object.assign(new Error('private-password https://private.test/secret'), { name: 'private-error-name' }), 'unknown'], + ['private-password', 'unknown'], + [new Error('another private-password'), 'unknown'], + ]; + for (const [error] of cases) { + page.emit('response', { + headers: () => ({ 'content-type': 'text/html' }), status: () => 200, + url: () => 'https://assets.test/private-password?token=private-password', + request: () => ({ url: () => 'https://assets.test/private-password?token=private-password', + method: () => 'GET', resourceType: () => 'font', failure: () => null }), + text: async () => { throw error; }, + }); + await new Promise(resolve => setImmediate(resolve)); + } + assert.equal(messages.length, 8, 'Diagnostics must remain bounded'); + const records = messages.map(line => JSON.parse(line.slice(line.indexOf('{')))); + assert.deepEqual(records.map(record => record.error), cases.slice(0, 8).map(([, category]) => category)); + assert(records.every(record => record.origin === 'https://assets.test' && /^[a-f0-9]{64}$/.test(record.pathSha256))); + assert.doesNotMatch(messages.join(''), /private-password|private-error-name|private\.test/); + assert.equal(captureFailure(received)?.bodyReadFailures, 9); + received.record('observed-secret'); + assert.equal(received.contains('observed-secret'), true); +}); diff --git a/tools/stack-bench/tests/workspace-permissions.integration.ts b/tools/stack-bench/tests/workspace-permissions.integration.ts new file mode 100644 index 00000000000..ba8127aef60 --- /dev/null +++ b/tools/stack-bench/tests/workspace-permissions.integration.ts @@ -0,0 +1,61 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { randomBytes } from 'node:crypto'; +import test from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { codingContainerWorkspaceHandoffCommands } from '../src/runtime/coding-container-policy.js'; + +test('capability-dropped controller and agent retain access through source restores and handoff', { + skip: process.env.STACK_BENCH_WORKSPACE_PERMISSIONS_TEST !== '1', timeout: 60_000, +}, () => { + const image = process.env.STACK_BENCH_NETWORK_CONTROLLER_IMAGE; + assert.match(image ?? '', /^sha256:[a-f0-9]{64}$/); + const prefix = `stack-bench-permissions-probe-${randomBytes(6).toString('hex')}`; + const docker = (args: string[]): string => { + const result = spawnSync('docker', args, { encoding: 'utf8', timeout: 30_000 }); + assert.equal(result.status, 0, `${args.slice(0, 3).join(' ')}: ${result.stderr || result.error}`); + return result.stdout.trim(); + }; + const volume = docker(['volume', 'create', prefix]); + const state = docker(['volume', 'inspect', '--format', '{{.Mountpoint}}', volume]); + const containers: string[] = []; + const create = (suffix: string, args: string[]): string => { + const id = docker(['create', '--name', `${prefix}-${suffix}`, '--network', 'none', '--read-only', + '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges:true', ...args, + '--entrypoint', 'sleep', image!, 'infinity']); + containers.push(id); + docker(['start', id]); + return id; + }; + try { + const controller = create('controller', ['--mount', `type=volume,source=${volume},target=/proof`, + '--mount', `type=bind,source=${fileURLToPath(new URL('../', import.meta.url))},target=/opt/stack-bench/dist,readonly`]); + const source = `import * as fs from 'node:fs'; + import {seedAppSource,restoreAppSource,resetAppToSource} from './dist/src/runtime/source-snapshot.js';`; + docker(['exec', controller, 'node', '--input-type=module', '-e', source + ` + fs.mkdirSync('/proof/source/src',{recursive:true}); + fs.writeFileSync('/proof/source/src/app.js','original'); + fs.chmodSync('/proof/source/src/app.js',0o444); + fs.chmodSync('/proof/source/src',0o555); + seedAppSource('/proof/source','/proof/app');`]); + const mount = ['--mount', `type=bind,source=${state}/app,target=/app`]; + const handoff = create('handoff', [...mount, '--cap-add', 'CHOWN', '--cap-add', 'FOWNER']); + const agent = create('agent', [...mount, '--user', '10001:10001']); + for (const operation of ['seed', 'restore', 'reset']) { + if (operation !== 'seed') docker(['exec', controller, 'node', '--input-type=module', '-e', source + + `${operation === 'restore' ? 'restoreAppSource' : 'resetAppToSource'}('/proof/source','/proof/app');`]); + for (const command of codingContainerWorkspaceHandoffCommands(0)) docker(['exec', handoff, ...command]); + docker(['exec', agent, 'node', '-e', `const fs=require('node:fs'),assert=require('node:assert/strict'); + assert.equal(fs.readFileSync('/app/src/app.js','utf8'),'original'); + fs.writeFileSync('/app/src/app.js','agent edit'); + fs.writeFileSync('/app/src/new.js','agent addition'); + assert.equal(fs.statSync('/app/src/app.js').mode & 0o007,0);`]); + docker(['exec', controller, 'node', '-e', `const fs=require('node:fs'),assert=require('node:assert/strict'); + assert.equal(fs.readFileSync('/proof/app/src/app.js','utf8'),'agent edit'); + fs.writeFileSync('/proof/app/controller-note','controller can write');`]); + } + } finally { + for (const id of containers.reverse()) docker(['rm', '-f', '--volumes', id]); + docker(['volume', 'rm', volume]); + } +}); diff --git a/tools/stack-bench/tracks/chat/LEVELS.md b/tools/stack-bench/tracks/chat/LEVELS.md new file mode 100644 index 00000000000..9fbdb149f5f --- /dev/null +++ b/tools/stack-bench/tracks/chat/LEVELS.md @@ -0,0 +1,63 @@ +# Chat progression + +The chat track covers rooms, messages, presence, and per-user state. Its L1 and +L2 definitions are available, but it has no active reference fixtures. Do not +present chat results as a qualified cross-stack comparison. + +`track.json` is the source of truth for available suites and the validation +boundary. + +## L1: Chat and accounts + +- account creation and sessions; +- room creation and messages; +- presence and typing state; +- read receipts and unread counts; +- ordering, reconnect, and system invariants. + +Each actor uses a separate browser context so identity and per-user state are +measured independently. + +## L2: Authorization and people + +- room ownership and private rooms; +- membership, invitations, and removal; +- profiles and friend requests; +- presence privacy and authorization. + +## Planned work + +- L3 adds contended counters, bounded capacity, unique claims, and transfers. +- L4 adds scheduled and expiring work. +- L5 adds correctness and efficiency under load. + +Planned work is not launchable or qualified. New scored checks require matching +reference, null-control, and defect-detection evidence before promotion. + +## Current measurement limits + +All chat definitions remain unqualified (`validatedThrough: 0`), with no active +reference fixtures. `01-contention-wip.json` is not selected by `track.json`. +The systems suites are selected but their criteria carry zero points. + +Raw authorization replays depend on captured HTTP requests. The current chat +scenarios have no named reducer fallback. An unavailable replay is inconclusive; +a hidden control or absent DOM row does not prove server authorization. DOM-only +privacy criteria measure display isolation, not absence of data on the wire. + +System purge probes execute the application's own back-office script. They +observe open and fresh clients but do not independently read database rows. +Concurrent client actions do not prove server overlap. A passing burst does not +establish a failure rate or sustained capacity. The inactive contention probes +need cross-stack reference and targeted-defect evidence before promotion. + +Replay completion is checked before crediting unchanged friendship state, but +friendship rows are still observed through the interface, not a trusted stored +relationship read. Receipt stability samples a bounded interval; it does not +prove that the display never flickers. Preserve these limits in any report. + +The rate-limit rollback criterion does not establish the server-arrival interval +of its two UI sends or acknowledge the first delivery. Waiting for delivery +before the second send could exceed the allowed interval on a correct app. +This unresolved precondition blocks qualification of that criterion; changing +it into a different validation test would not qualify server rate limiting. diff --git a/tools/stack-bench/tracks/chat/contracts/01-basic-chat.json b/tools/stack-bench/tracks/chat/contracts/01-basic-chat.json new file mode 100644 index 00000000000..5e31a6302e8 --- /dev/null +++ b/tools/stack-bench/tracks/chat/contracts/01-basic-chat.json @@ -0,0 +1,277 @@ +{ + "level": 1, + "track": "stack-bench", + "hooks": [ + { + "id": "app-title", + "element": "the app's visible title, \"Chat\"", + "stage": "landing", + "check": "visible" + }, + { + "id": "signup-username", + "element": "username input on the sign-up form", + "stage": "landing", + "check": "visible" + }, + { + "id": "signup-password", + "element": "password input on the sign-up form", + "stage": "landing", + "check": "visible" + }, + { + "id": "signup-submit", + "element": "button that creates the account", + "stage": "landing", + "check": "visible" + }, + { + "id": "signin-toggle", + "element": "control that switches from the sign-up form to the sign-in form", + "stage": "landing", + "check": "attached" + }, + { + "id": "signin-username", + "element": "username input on the sign-in form", + "stage": "landing", + "check": "attached", + "revealedBy": "signin-toggle" + }, + { + "id": "signin-password", + "element": "password input on the sign-in form", + "stage": "landing", + "check": "attached", + "revealedBy": "signin-toggle" + }, + { + "id": "signin-submit", + "element": "button that signs the user in with existing credentials", + "stage": "landing", + "check": "attached", + "revealedBy": "signin-toggle" + }, + { + "id": "auth-error", + "element": "error message shown when sign-up or sign-in is rejected (taken username, wrong password)", + "stage": "scenario", + "note": "visible after a rejected sign-up or sign-in; must not appear on success" + }, + { + "id": "current-user", + "element": "element displaying the signed-in account's username", + "stage": "main", + "check": "visible" + }, + { + "id": "signout", + "element": "button that signs the current user out", + "stage": "main", + "check": "attached" + }, + { + "id": "room-create", + "element": "control that starts creating a new room", + "stage": "main", + "check": "visible" + }, + { + "id": "room-name-input", + "element": "text input for the new room's name", + "stage": "main", + "check": "attached", + "revealedBy": "room-create" + }, + { + "id": "room-name-submit", + "element": "button that confirms room creation", + "stage": "main", + "check": "attached", + "revealedBy": "room-create" + }, + { + "id": "room-list", + "element": "container listing the rooms", + "stage": "main", + "check": "attached" + }, + { + "id": "room-item", + "element": "one entry per room inside the room list (repeated testid), containing the room name", + "stage": "main-after-create", + "check": "visible" + }, + { + "id": "online-users", + "element": "container listing the usernames of users currently online", + "stage": "main", + "check": "attached", + "note": "container listing signed-in users" + }, + { + "id": "online-user", + "element": "one entry per signed-in user (repeated testid), containing that user's username", + "stage": "scenario", + "note": "appears while that user is signed in, gone once they sign out" + }, + { + "id": "message-input", + "element": "text input for composing a message; pressing Enter sends it", + "stage": "room", + "check": "visible" + }, + { + "id": "message-list", + "element": "scrollable container holding the room's messages", + "stage": "room", + "check": "attached" + }, + { + "id": "message-item", + "element": "one entry per message (repeated testid), containing the message text and its author's username", + "stage": "room-after-send", + "check": "visible" + }, + { + "id": "leave-room", + "element": "button that leaves the current room", + "stage": "room", + "check": "attached" + }, + { + "id": "send-error", + "element": "error shown when a message is rejected (empty, too long, or rate limited)", + "stage": "scenario", + "note": "visible after a rejected send; must not appear on a successful send" + }, + { + "id": "typing-indicator", + "element": "the \"User is typing...\" text element", + "stage": "scenario", + "note": "visible while another user in the SAME room is typing; gone within 6s of inactivity" + }, + { + "id": "read-receipt", + "element": "the \"Seen by ...\" element under a message, listing the account names that have seen it (never the sender)", + "stage": "scenario", + "note": "appears under a message after another user has viewed it" + }, + { + "id": "unread-badge", + "element": "numeric unread-count badge on a room entry in the sidebar", + "stage": "scenario", + "note": "appears on a room entry that has unread messages for the current user; clears when that room is opened" + }, + { + "id": "react-button", + "element": "control on a message that adds a reaction (an emoji picker trigger or a specific emoji button)", + "stage": "scenario", + "note": "present on each message; clicking it adds the current user's reaction" + }, + { + "id": "reaction-count", + "element": "the numeric count shown beside an emoji on a message", + "stage": "scenario", + "note": "shows how many users reacted with that emoji; repeated per emoji per message" + }, + { + "id": "pin-button", + "element": "control on a message that pins it", + "stage": "scenario", + "note": "present on each message" + }, + { + "id": "pinned-item", + "element": "one entry per pinned message in the room's pinned list (repeated testid)", + "stage": "scenario", + "note": "the room's pinned messages, capped at three" + }, + { + "id": "pin-error", + "element": "error shown when pinning is refused because the room already has three pinned messages", + "stage": "scenario", + "note": "visible after a rejected pin; must not appear on a successful one" + }, + { + "id": "pinned-unpin", + "element": "control on a pinned entry that unpins it", + "stage": "scenario", + "note": "present on each pinned-item" + }, + { + "id": "member-panel", + "element": "container listing the members of the current room", + "stage": "room", + "check": "attached" + }, + { + "id": "member-item", + "element": "one entry per room member (repeated testid), containing that member's username", + "stage": "scenario", + "note": "appears for each member of the current room; gone once they leave or are removed" + }, + { + "id": "member-owner", + "element": "marker on a member entry showing they own the room", + "stage": "scenario", + "note": "present on the owner's entry only" + }, + { + "id": "member-remove", + "element": "control on a member entry that removes them from the room", + "stage": "scenario", + "note": "shown to owners only" + }, + { + "id": "member-promote", + "element": "control on a member entry that promotes them to owner", + "stage": "scenario", + "note": "shown to owners only" + }, + { + "id": "dm-start", + "element": "control on another user's entry that opens a direct conversation with them", + "stage": "scenario", + "note": "present on each other user listed as online, and on each member entry" + }, + { + "id": "dm-list", + "element": "container listing the user's direct conversations", + "stage": "main", + "check": "attached" + }, + { + "id": "dm-item", + "element": "one entry per direct conversation (repeated testid), naming the other participant", + "stage": "scenario", + "note": "appears once a direct conversation exists; visible only to its two participants" + }, + { + "id": "permission-error", + "element": "error shown when the server refuses an action the user is not allowed to take", + "stage": "scenario", + "note": "visible after a refused unpin, removal, promotion, edit or delete" + }, + { + "id": "message-edit", + "element": "control on a message that edits it", + "stage": "scenario", + "note": "shown on the author's own messages" + }, + { + "id": "message-delete", + "element": "control on a message that deletes it", + "stage": "scenario", + "note": "shown on the author's own messages" + }, + { + "id": "edited-marker", + "element": "marker showing a message has been edited", + "stage": "scenario", + "note": "appears on a message after it is edited" + } + ], + "note": "Level 1 is sequential's basic chat with accounts added. Hooks cover the sign-up and sign-in forms, rooms and messages, plus typing indicators, read receipts and unread counts." +} diff --git a/tools/stack-bench/tracks/chat/contracts/02-authorization.json b/tools/stack-bench/tracks/chat/contracts/02-authorization.json new file mode 100644 index 00000000000..5541ebeaf06 --- /dev/null +++ b/tools/stack-bench/tracks/chat/contracts/02-authorization.json @@ -0,0 +1,116 @@ +{ + "level": 2, + "track": "chat", + "note": "The core flow remains L1-only. Level 2 runs as scenarios. Repeated hooks use one element per row with identifying text.", + "hooks": [ + { + "id": "room-private-toggle", + "element": "control at room creation that marks the room private", + "stage": "scenario", + "note": "present on the room creation surface; its state at creation decides visibility" + }, + { + "id": "room-private-badge", + "element": "marker on a room that is private, visible to its members", + "stage": "scenario", + "note": "absent on public rooms" + }, + { + "id": "invite-input", + "element": "input where a room owner enters a username to invite", + "stage": "scenario", + "note": "owner-only, on private rooms" + }, + { + "id": "invite-submit", + "element": "control that sends the invitation", + "stage": "scenario", + "note": "owner-only, on private rooms" + }, + { + "id": "perm-error", + "element": "error shown when an action is refused for lack of permission", + "stage": "scenario", + "note": "visible after a refused action; must not appear on permitted ones" + }, + { + "id": "profile-panel", + "element": "panel showing a user's profile, opened by clicking their username", + "stage": "scenario", + "note": "one panel; contents change with the user being viewed" + }, + { + "id": "profile-name", + "element": "the viewed user's display name inside the profile panel", + "stage": "scenario" + }, + { + "id": "profile-bio", + "element": "the viewed user's bio inside the profile panel", + "stage": "scenario", + "note": "live while the panel is open; an edit by the profile's owner appears without reopening" + }, + { + "id": "profile-close", + "element": "control that closes the profile panel", + "stage": "scenario" + }, + { + "id": "bio-input", + "element": "input where a user edits their own bio", + "stage": "scenario", + "note": "only on the viewer's own profile" + }, + { + "id": "bio-save", + "element": "control that saves the bio edit", + "stage": "scenario", + "note": "only on the viewer's own profile" + }, + { + "id": "friend-request", + "element": "control on another user's profile that sends a friend request", + "stage": "scenario", + "note": "absent on one's own profile and on an existing friend's profile" + }, + { + "id": "friend-pending", + "element": "one entry per incoming friend request (repeated testid), containing the sender's username", + "stage": "scenario", + "note": "available while signed in without opening a menu" + }, + { + "id": "friend-accept", + "element": "control on a pending request that accepts it", + "stage": "scenario" + }, + { + "id": "friend-decline", + "element": "control on a pending request that declines it", + "stage": "scenario" + }, + { + "id": "friend-item", + "element": "one entry per friend in the friends list (repeated testid), containing that friend's username", + "stage": "scenario", + "note": "available while signed in without opening a menu" + }, + { + "id": "friend-presence", + "element": "online/offline indicator inside a friend entry", + "stage": "scenario", + "note": "contains the word 'online' or 'offline'; flips live when the friend signs in or out" + }, + { + "id": "unfriend", + "element": "control inside a friend entry that removes the friendship", + "stage": "scenario" + }, + { + "id": "member-item", + "element": "one current-room member row; clicking the row opens that member's profile", + "stage": "scenario", + "note": "also applies to the current user" + } + ] +} diff --git a/tools/stack-bench/tracks/chat/contracts/appendix-01.md b/tools/stack-bench/tracks/chat/contracts/appendix-01.md new file mode 100644 index 00000000000..58c51401827 --- /dev/null +++ b/tools/stack-bench/tracks/chat/contracts/appendix-01.md @@ -0,0 +1,64 @@ + + +## Application interface + +Use each interface name below as an exact HTML attribute value. Use `id` for a +one-off element. Use `data-role` when the same interface can appear more than +once. These attributes do not prescribe the design, architecture, or +implementation. + +Rules: +- Values are exactly as listed in kebab-case. +- An element that is hidden until a menu/toggle opens still counts, as long as it is in the DOM after its toggle is clicked. +- Do not reuse these names on unrelated elements. + +| Element ID | Element | +|---|---| +| `app-title` | the app's visible title, `Chat` | +| `signup-username` | username input on the sign-up form | +| `signup-password` | password input on the sign-up form | +| `signup-submit` | button that creates the account | +| `signin-toggle` | control that switches from the sign-up form to the sign-in form | +| `signup-toggle` | control that reveals the sign-up form when its inputs are not shown; omit it when a signed-out visitor already sees them | +| `signin-username` | username input on the sign-in form | +| `signin-password` | password input on the sign-in form | +| `signin-submit` | button that signs the user in with existing credentials | +| `auth-error` | visible after a rejected sign-up or sign-in; must not appear on success | +| `current-user` | element displaying the signed-in account's username | +| `signout` | button that signs the current user out | +| `room-create` | control that starts creating a new room | +| `room-name-input` | text input for the new room's name | +| `room-name-submit` | button that confirms room creation | +| `room-list` | container listing the rooms | +| `room-item` | one entry per room inside the room list, containing the room name | +| `online-users` | container listing the usernames of users currently online | +| `online-user` | appears while that user is signed in, gone once they sign out | +| `message-input` | text input for composing a message; pressing Enter sends it | +| `message-list` | scrollable container holding the room's messages | +| `message-item` | one entry per message, containing the message text and its author's username | +| `leave-room` | button that leaves the current room | +| `send-error` | visible after a rejected send; must not appear on a successful send | +| `typing-indicator` | visible while another user in the SAME room is typing; gone within 6s of inactivity | +| `read-receipt` | appears under a message after another user has viewed it | +| `unread-badge` | appears on a room entry that has unread messages for the current user; clears when that room is opened | +| `react-button` | present on each message; clicking it adds the current user's reaction | +| `reaction-count` | shows how many users reacted with that emoji; repeated per emoji per message | +| `pin-button` | present on each message | +| `pinned-item` | the room's pinned messages, capped at three | +| `pin-error` | visible after a rejected pin; must not appear on a successful one | +| `pinned-unpin` | present on each pinned-item | +| `member-panel` | container listing the members of the current room | +| `member-item` | appears for each member of the current room; gone once they leave or are removed | +| `member-owner` | present on the owner's entry only | +| `member-remove` | shown to owners only | +| `member-promote` | shown to owners only | +| `dm-start` | present on each other user listed as online, and on each member entry | +| `dm-list` | container listing the user's direct conversations | +| `dm-item` | appears once a direct conversation exists; visible only to its two participants | +| `permission-error` | visible after a refused unpin, removal, promotion, edit or delete | +| `message-edit` | shown on the author's own messages | +| `message-delete` | shown on the author's own messages | +| `edited-marker` | appears on a message after it is edited | + +Before declaring DEPLOY_COMPLETE, verify the interface by running the contract +linter (command provided in your build instructions) and fix any failures. diff --git a/tools/stack-bench/tracks/chat/contracts/appendix-02.md b/tools/stack-bench/tracks/chat/contracts/appendix-02.md new file mode 100644 index 00000000000..77528dc403f --- /dev/null +++ b/tools/stack-bench/tracks/chat/contracts/appendix-02.md @@ -0,0 +1,84 @@ + + +## Application interface + +Use each interface name below as an exact HTML attribute value. Use `id` for a +one-off element. Use `data-role` when the same interface can appear more than +once. These attributes do not prescribe the design, architecture, or +implementation. + +Rules: +- Values are exactly as listed in kebab-case. +- An element that is hidden until a menu/toggle opens still counts, as long as it is in the DOM after its toggle is clicked. +- Do not reuse these names on unrelated elements. + +| Element ID | Element | +|---|---| +| `app-title` | the app's visible title, `Chat` | +| `signup-username` | username input on the sign-up form | +| `signup-password` | password input on the sign-up form | +| `signup-submit` | button that creates the account | +| `signin-toggle` | control that switches from the sign-up form to the sign-in form | +| `signup-toggle` | control that reveals the sign-up form when its inputs are not shown; omit it when a signed-out visitor already sees them | +| `signin-username` | username input on the sign-in form | +| `signin-password` | password input on the sign-in form | +| `signin-submit` | button that signs the user in with existing credentials | +| `auth-error` | visible after a rejected sign-up or sign-in; must not appear on success | +| `current-user` | element displaying the signed-in account's username | +| `signout` | button that signs the current user out | +| `room-create` | control that starts creating a new room | +| `room-name-input` | text input for the new room's name | +| `room-name-submit` | button that confirms room creation | +| `room-list` | container listing the rooms | +| `room-item` | one entry per room inside the room list, containing the room name | +| `online-users` | container listing the usernames of users currently online | +| `online-user` | appears while that user is signed in, gone once they sign out | +| `message-input` | text input for composing a message; pressing Enter sends it | +| `message-list` | scrollable container holding the room's messages | +| `message-item` | one entry per message, containing the message text and its author's username | +| `leave-room` | button that leaves the current room | +| `send-error` | visible after a rejected send; must not appear on a successful send | +| `typing-indicator` | visible while another user in the SAME room is typing; gone within 6s of inactivity | +| `read-receipt` | appears under a message after another user has viewed it | +| `unread-badge` | appears on a room entry that has unread messages for the current user; clears when that room is opened | +| `react-button` | present on each message; clicking it adds the current user's reaction | +| `reaction-count` | shows how many users reacted with that emoji; repeated per emoji per message | +| `pin-button` | present on each message | +| `pinned-item` | the room's pinned messages, capped at three | +| `pin-error` | visible after a rejected pin; must not appear on a successful one | +| `pinned-unpin` | present on each pinned-item | +| `member-panel` | container listing the members of the current room | +| `member-item` | appears for each member of the current room; gone once they leave or are removed. Clicking the row opens that member's profile. | +| `member-owner` | present on the owner's entry only | +| `member-remove` | shown to owners only | +| `member-promote` | shown to owners only | +| `dm-start` | present on each other user listed as online, and on each member entry | +| `dm-list` | container listing the user's direct conversations | +| `dm-item` | appears once a direct conversation exists; visible only to its two participants | +| `permission-error` | visible after a refused unpin, removal, promotion, edit or delete | +| `message-edit` | shown on the author's own messages | +| `message-delete` | shown on the author's own messages | +| `edited-marker` | appears on a message after it is edited | +| `room-private-toggle` | present on the room creation surface; its state at creation decides visibility | +| `room-private-badge` | absent on public rooms | +| `invite-input` | owner-only, on private rooms | +| `invite-submit` | owner-only, on private rooms | +| `perm-error` | visible after a refused action; must not appear on permitted ones | +| `profile-panel` | one panel; contents change with the user being viewed | +| `profile-name` | the viewed user's display name inside the profile panel | +| `profile-bio` | live while the panel is open — an edit by the profile's owner appears without reopening | +| `profile-close` | control that closes the profile panel | +| `bio-input` | only on the viewer's own profile | +| `bio-save` | only on the viewer's own profile | +| `friend-request` | absent on one's own profile and on an existing friend's profile | +| `friend-pending` | one entry per incoming friend request (repeated testid), containing the sender's username | +| `friend-accept` | control on a pending request that accepts it | +| `friend-decline` | control on a pending request that declines it | +| `friend-item` | one entry per friend in the friends list (repeated testid), containing that friend's username | +| `friend-presence` | contains the word 'online' or 'offline'; flips live when the friend signs in or out | +| `unfriend` | control inside a friend entry that removes the friendship | + +While signed in, keep the friends list and incoming friend requests available without opening a menu. + +Before declaring DEPLOY_COMPLETE, verify the interface by running the contract +linter (command provided in your build instructions) and fix any failures. diff --git a/tools/stack-bench/tracks/chat/prompts/01-basic-chat.md b/tools/stack-bench/tracks/chat/prompts/01-basic-chat.md new file mode 100644 index 00000000000..836c88fe6c9 --- /dev/null +++ b/tools/stack-bench/tracks/chat/prompts/01-basic-chat.md @@ -0,0 +1,163 @@ +# Level 1 — Basic chat + +Create a **real-time chat app**. + + +## UI & Style Guide + +### Layout +- **Sidebar** (left, ~220px fixed): app title/branding, user info with status, room list, online users +- **Main area** (right, flex): room header bar, scrollable message list, input bar pinned to bottom +- **Panels** (right slide-in or overlay): threads, pinned messages, profiles, settings + +### Visual Design +- Dark theme with a blue primary accent +- Background: darkest shade for main bg, slightly lighter for sidebar and cards +- Text: light on dark, muted color for timestamps and secondary info +- Borders: subtle 1px, low contrast against background +- Consistent spacing scale (8/12/16/24px) +- Font: system font stack, clear hierarchy (bold headers, regular body, small muted metadata) +- Rounded corners on inputs, buttons, cards, and message containers + +### Components +- **Messages**: sender name (colored) + timestamp (muted) + text. Group consecutive messages from same sender. Action buttons appear on hover only (which buttons depend on the features below). +- **Inputs**: full-width, rounded, subtle border, placeholder text, focus ring using primary color +- **Buttons**: filled with primary color for main actions, outlined/ghost for secondary. Clear hover and active states. +- **Badges**: small pill-shaped with count, contrasting color (e.g., unread count on rooms) +- **Modals/panels**: slide-in from right with subtle backdrop, or dropdown overlays +- **Status indicators**: small colored dots (green=online, yellow=away, red=DND, grey=offline) +- **Room list**: room names with optional icon prefix (#), active room highlighted, unread badge + +### Interaction & UX +- Show loading/connecting state while backend connects (spinner or skeleton, not blank screen) +- Empty states: helpful text when no rooms, no messages, no results ("Create a room to get started") +- Error feedback: inline error messages or toast notifications, never silent failures +- Smooth transitions: fade/slide for panels, modals, and state changes +- Hover reveals: message action buttons, tooltips on reactions, user profile cards +- Keyboard support: Enter to send messages, Escape to close modals/panels +- Auto-scroll to newest message, with scroll-to-bottom button when scrolled up + +## Features + +### Accounts + +- A visitor can **create an account** with a username and password +- A returning visitor can **sign in** with those credentials +- A signed-in user can **sign out**, returning to the signed-out state +- A signed-in session **persists across a page reload** — a returning user is not asked to + sign in again — and across the connection dropping and re-establishing. The same person + stays the same account throughout, keeping their rooms, messages and authorship. +- Usernames are unique. Signing up with a taken username fails with a visible error and must + never sign the visitor in as the existing account. +- Signing in with a wrong password fails with a visible error. +- A user's account name is shown on their messages. + +Identity is an account, not a display name: knowing someone's username must never grant +access to it. + +### Basic Chat Features + +- Users can create chat rooms and join/leave them +- Users can send messages to rooms they've joined +- Show who's online +- Include reasonable validation (e.g., don't let users spam, enforce sensible limits) + + +### Typing Indicators + +- Show when other users are currently typing in the SAME room (typing must be scoped to room — do not broadcast typing to users in different rooms) +- Typing indicator should automatically expire after a few seconds of inactivity +- Display "User is typing..." or "Multiple users are typing..." in the UI + + +### Read Receipts + +- Track which users have seen which messages +- Display "Seen by X, Y, Z" under messages — only show OTHER users who have seen it, not the sender +- Update read status in real-time as users view messages + + +### Unread Message Counts + +- Show unread message count badges on the room list +- Track last-read position per user per room +- Update counts in real-time as new messages arrive or are read + +### Room Members and Roles + +- Each room shows a **member panel**: everyone who has joined it, with their online state +- The panel updates live as people join, leave, connect and disconnect +- The user who created a room is its **owner**; the panel shows who that is +- An owner can **promote** another member to owner, and can **remove** a member from the room +- Only members can read a room's messages or post to it. A non-member is refused by the + server, not merely prevented by hiding a button. +- **Removal takes effect immediately.** A removed user stops receiving that room's messages + without reloading, loses access to its history, and cannot rejoin a room they were removed + from unless an owner re-adds them. + +### Direct Messages + +- A user can open a **direct conversation** with another user from that user's entry in the + online list or a member panel. It reads and composes like a room. +- A direct conversation is visible only to its two participants. Nobody else can read it — + not through the interface, not through any request the client can make, and it is never + sent to anybody else's browser in the first place. +- Direct messages arrive live, are attributed, and persist like room messages +- Unread counts cover direct conversations too + +### Reactions + +- Users can react to a message with an emoji, and remove their own reaction +- Each message shows each emoji with a **count** of how many users have reacted with it +- A user may react at most once per emoji per message — reacting again does not raise the + count by two +- Counts update live for everyone in the room, and every client shows the same count + +### Pinned Messages + +- Any member of a room can **pin** a message +- Only the user who pinned a message, or an owner of the room, can unpin it +- Pinned messages are listed for the room, visible to everyone in it +- A room may have **at most 3 pinned messages**. An attempt to pin a fourth is rejected with + a visible error and changes nothing. +- Unpinning frees a slot +- The pinned list updates live, and every client in the room shows the same list + +### Ownership + +Every action belongs to somebody, and the server decides who may take it: + +- Only the author of a message can edit or delete it; edited messages show as edited +- Only the pinner or a room owner can unpin +- Only a room owner can promote or remove members +- An attempt by the wrong user is refused with a visible error and changes nothing. Enforce + this on the server — hiding a control in the interface is not enforcement. + +### Back office + +**Your app does not own its data.** Real chat services have other systems +writing to the same database: moderation tooling, compliance takedowns, +data-retention jobs. The app must remain correct — and its open pages current — +when data changes by a path its server never saw. This is a property of your +architecture, not a feature to bolt on. + +As the concrete instance of that property, ship a script at +`scripts/backoffice.mjs` (run with `node`, from the app directory) supporting: + +``` +node scripts/backoffice.mjs purge-user +``` + +It deletes every message that user has sent **by writing to the database +directly** — it must work even when the web server is not running, and it must +not talk to the web server. Every open browser must reflect the purge like any +other change: the messages disappear live, without a reload. + +### Limits + +- Enforce the message rate limit **on the server**: at most one message per second per + account, with a visible error when exceeded +- The limit applies to the account, not to a browser tab — the same account in two tabs + shares one budget +- Limits and presence are properties of the application, not of a running process: they must + still be correct after the backend restarts diff --git a/tools/stack-bench/tracks/chat/prompts/02-authorization.md b/tools/stack-bench/tracks/chat/prompts/02-authorization.md new file mode 100644 index 00000000000..31ee4df1ba2 --- /dev/null +++ b/tools/stack-bench/tracks/chat/prompts/02-authorization.md @@ -0,0 +1,57 @@ +# Level 2 — Authorization and People + +Adds ownership, access control, and a social layer to the app from level 1. +Everything from level 1 must keep working. + +## Features + +### Private rooms + +- A room creator can mark a room **private** at creation time +- A private room's messages are readable only by its members. A non-member must not be able + to read them by any means the app offers — not by entering the room, not by any link, and + not through any request the client can make. +- A private room does not appear in a non-member's room list at all — its existence and its + name are members-only +- Public rooms remain readable by any signed-in user + +### Membership + +- A room creator is its **owner** +- An owner can **invite** another user to a private room; the invitee gains access +- An owner can **remove** a member. Removal takes effect immediately: the removed user stops + receiving that room's messages without reloading, and loses read access to its history. +- A removed user can be re-invited + +### Permissions + +- Only an owner sees owner controls (invite, remove, delete room) +- A non-owner must not be able to perform an owner action, including by issuing the request + directly rather than through the UI +- Only the author of a message can delete it; a deleted message disappears live for everyone +- Attempting an unpermitted action shows a visible error and changes nothing + +### Profiles + +- Every account has a **profile**: display name, a short bio, and a joined-date. + The owner can edit their display name and bio. +- Clicking a username anywhere it appears (a message, the member list) opens that + person's **profile panel** +- A profile panel is **live for as long as it is open**: if the person edits their + bio while someone else is looking at their profile, the viewer sees the new bio + without reopening the panel or reloading +- The panel can be closed; a closed panel is gone — reopening it shows current data + +### Friends + +- A signed-in user can **send a friend request** from a profile panel +- The recipient sees the pending request and can **accept** or **decline**; the + sender sees which of these happened +- Each user has a **friends list** showing their friends and each friend's + **online/offline presence** +- The friends list is live: a request accepted while the list is open appears in + it without a reload, and a friend signing in or out flips their presence for + everyone whose list shows them +- Either side can **unfriend**; the row disappears live from both lists +- Friend state is one relationship, not two: A's list and B's list can never + disagree about whether they are friends diff --git a/tools/stack-bench/tracks/chat/prompts/03-contention.md b/tools/stack-bench/tracks/chat/prompts/03-contention.md new file mode 100644 index 00000000000..1fc2e984119 --- /dev/null +++ b/tools/stack-bench/tracks/chat/prompts/03-contention.md @@ -0,0 +1,34 @@ +# Level 3 — Contended state + +Adds shared state that many users mutate at the same time, on top of levels 1 and 2. +Everything from earlier levels must keep working. + +## Features + +### Reactions + +- A user can add an emoji reaction to a message, and remove their own +- Each message shows each emoji with a **count** of how many users reacted with it +- A user may react at most once per emoji per message — reacting twice does not raise the + count by two +- Counts update live for everyone in the room + +### Polls + +- A user can post a **poll** to a room: a question and two or more options +- Each user may cast **one vote**, and may change it — changing a vote moves the tally, it + does not add to it +- The poll shows per-option vote counts and a total, updating live +- A poll can be closed by its author; votes after closing are rejected + +### Limited-capacity events + +- A user can create an **event** in a room with a fixed number of seats +- Users claim seats until the event is full; further claims are rejected with a visible error +- The number of claimed seats must never exceed the capacity, no matter how many users claim + at the same moment +- A user can release a seat, freeing it for someone else +- Seat counts update live + +Correctness under simultaneous use is the point of this level. Totals must be exact when +many users act at the same instant: no lost updates, no double-counting, no overselling. diff --git a/tools/stack-bench/tracks/chat/prompts/04-deferred.md b/tools/stack-bench/tracks/chat/prompts/04-deferred.md new file mode 100644 index 00000000000..c05db9a3fbd --- /dev/null +++ b/tools/stack-bench/tracks/chat/prompts/04-deferred.md @@ -0,0 +1,31 @@ +# Level 4 — Deferred and expiring work + +Adds work that happens later, on top of levels 1 to 3. Everything from earlier levels must +keep working. + +## Features + +### Scheduled messages + +- A user can compose a message and schedule it to send at a future time, as little as 60 + seconds ahead +- The author sees their pending scheduled messages and can cancel one before it sends +- A scheduled message does not appear in the room before its time +- At its time it appears for everyone in the room, attributed to its author +- **Pending scheduled messages survive a backend restart** and still send at their time + +### Expiring messages + +- A user can send a message that expires after a chosen duration, selectable down to 30 + seconds +- An expiring message shows a countdown or expiry indicator +- When it expires it disappears for everyone, live, without a reload +- Expired content is **actually deleted** from storage, not merely hidden in the interface +- Expiry survives a backend restart: a message due to expire during the restart is gone + afterwards, and one not yet due still expires on time + +### Reminders + +- A user can set a personal reminder on a message for a future time +- At that time the user is notified in the app +- A reminder is visible only to the user who set it, and survives a restart diff --git a/tools/stack-bench/tracks/chat/prompts/05-volume.md b/tools/stack-bench/tracks/chat/prompts/05-volume.md new file mode 100644 index 00000000000..cdc5bc114cb --- /dev/null +++ b/tools/stack-bench/tracks/chat/prompts/05-volume.md @@ -0,0 +1,30 @@ +# Level 5 — Volume + +The app from levels 1 to 4 must stay correct and responsive as data and usage grow. +No new user-facing features; this level is about behaviour at scale. + +## Requirements + +### Data volume + +- A room with **50,000 messages** loads and remains usable +- A user in **200 rooms** sees their room list without noticeable delay +- Message history is paginated or virtualised rather than loaded in full +- Reading a room's recent messages must not get slower as its history grows + +### Concurrency + +- **100 concurrent signed-in clients** across a shared set of rooms remain correct: + every message reaches every client in its room, exactly once, in the same order for all +- Sustained writes continue to be delivered without loss under continuous load + +### Responsiveness + +- A sent message reaches other clients in the same room in **under one second at the 95th + percentile** under the load above +- The interface stays responsive while receiving continuous updates + +### Efficiency + +- Per-message work must not grow with the size of the room's history +- Queries backing the main views use indexes rather than scanning diff --git a/tools/stack-bench/tracks/chat/scenarios/01-basic-chat.json b/tools/stack-bench/tracks/chat/scenarios/01-basic-chat.json new file mode 100644 index 00000000000..b871b489033 --- /dev/null +++ b/tools/stack-bench/tracks/chat/scenarios/01-basic-chat.json @@ -0,0 +1,828 @@ +{ + "schemaVersion": 1, + "level": 1, + "note": "Level 1 is sequential's basic chat with accounts added. Accounts replace the display-name stub; typing indicators, read receipts and unread counts are sequential's, unchanged apart from signing up rather than naming yourself.", + "features": [ + { + "id": 1, + "name": "Accounts", + "actors": [ + "alice", + "bob" + ], + "setup": [ + { + "do": "signUp", + "actor": "alice", + "name": "Alice" + } + ], + "criteria": [ + { + "id": "signup-signs-in", + "points": 1, + "desc": "Creating an account signs the user in and shows their username", + "steps": [ + { + "do": "expect", + "actor": "alice", + "testid": "current-user", + "contains": "Alice", + "within": 6000 + } + ] + }, + { + "id": "duplicate-username-rejected", + "points": 1, + "desc": "Signing up with a taken username is rejected with a visible error, and must not sign the visitor in as the existing account", + "steps": [ + { + "do": "signUp", + "actor": "bob", + "name": "Alice", + "expectFailure": true + }, + { + "do": "expect", + "actor": "bob", + "testid": "auth-error", + "within": 6000 + }, + { + "do": "expect", + "actor": "bob", + "testid": "current-user", + "absent": true, + "within": 3000 + } + ] + }, + { + "id": "signout-and-signin", + "points": 1, + "desc": "A user can sign out and sign back in with the same credentials", + "steps": [ + { + "do": "click", + "actor": "alice", + "testid": "signout" + }, + { + "do": "waitUntilAbsent", + "actor": "alice", + "testid": "current-user", + "within": 6000 + }, + { + "do": "signIn", + "actor": "alice", + "name": "Alice" + }, + { + "do": "expect", + "actor": "alice", + "testid": "current-user", + "contains": "Alice", + "within": 6000 + } + ] + } + ] + }, + { + "id": 2, + "name": "Basic Chat", + "actors": [ + "alice", + "bob" + ], + "setup": [ + { + "do": "signUp", + "actor": "alice", + "name": "Alice" + }, + { + "do": "signUp", + "actor": "bob", + "name": "Bob" + }, + { + "do": "createRoom", + "actor": "alice", + "room": "general" + }, + { + "do": "enterRoom", + "actor": "alice", + "room": "general" + } + ], + "criteria": [ + { + "id": "own-message-appears", + "points": 1, + "desc": "A sent message appears in the sender's own message list", + "steps": [ + { + "do": "send", + "actor": "alice", + "text": "hello from alice" + }, + { + "do": "expect", + "actor": "alice", + "testid": "message-item", + "contains": "hello from alice", + "within": 5000 + } + ] + }, + { + "id": "cross-user-realtime", + "points": 1, + "desc": "A second user in the same room receives messages live, with no reload", + "steps": [ + { + "do": "enterRoom", + "actor": "bob", + "room": "general" + }, + { + "do": "expect", + "actor": "bob", + "testid": "message-item", + "contains": "hello from alice", + "within": 5000 + }, + { + "do": "send", + "actor": "bob", + "text": "hello from bob" + }, + { + "do": "expect", + "actor": "alice", + "testid": "message-item", + "contains": "hello from bob", + "within": 3000 + } + ] + }, + { + "id": "presence-and-leave", + "points": 1, + "desc": "Online users lists both participants; leaving a room works", + "steps": [ + { + "do": "expect", + "actor": "alice", + "testid": "online-users", + "contains": "Bob", + "within": 5000 + }, + { + "do": "click", + "actor": "bob", + "testid": "leave-room" + }, + { + "do": "waitUntilAbsent", + "actor": "bob", + "testid": "message-input", + "within": 5000 + } + ] + } + ] + }, + { + "id": 3, + "name": "Typing Indicators", + "actors": [ + "alice", + "bob", + "carol" + ], + "setup": [ + { + "do": "signUp", + "actor": "alice", + "name": "Alice" + }, + { + "do": "signUp", + "actor": "bob", + "name": "Bob" + }, + { + "do": "signUp", + "actor": "carol", + "name": "Carol" + }, + { + "do": "createRoom", + "actor": "alice", + "room": "typing-a" + }, + { + "do": "createRoom", + "actor": "carol", + "room": "typing-b" + }, + { + "do": "enterRoom", + "actor": "alice", + "room": "typing-a" + }, + { + "do": "enterRoom", + "actor": "bob", + "room": "typing-a" + }, + { + "do": "enterRoom", + "actor": "carol", + "room": "typing-b" + } + ], + "criteria": [ + { + "id": "indicator-appears", + "points": 1, + "desc": "Typing by another user in the same room shows an indicator live", + "steps": [ + { + "do": "typeInto", + "actor": "bob", + "text": "composing a reply" + }, + { + "do": "expect", + "actor": "alice", + "testid": "typing-indicator", + "within": 4000, + "contains": "typing" + } + ] + }, + { + "id": "indicator-expires", + "points": 1, + "desc": "Indicator disappears within 6s once typing stops", + "steps": [ + { + "do": "clearInput", + "actor": "bob" + }, + { + "do": "waitUntilAbsent", + "actor": "alice", + "testid": "typing-indicator", + "within": 6500, + "contains": "typing" + } + ] + }, + { + "id": "scoped-to-room", + "points": 1, + "desc": "Typing is scoped to the room \u2014 a user in another room sees nothing", + "steps": [ + { + "do": "typeInto", + "actor": "bob", + "text": "still typing here" + }, + { + "do": "expect", + "actor": "alice", + "testid": "typing-indicator", + "within": 4000, + "contains": "typing" + }, + { + "do": "expect", + "actor": "carol", + "testid": "typing-indicator", + "absent": true, + "within": 2000, + "contains": "typing" + } + ] + } + ] + }, + { + "id": 4, + "name": "Read Receipts", + "actors": [ + "alice", + "bob" + ], + "setup": [ + { + "do": "signUp", + "actor": "alice", + "name": "Alice" + }, + { + "do": "signUp", + "actor": "bob", + "name": "Bob" + }, + { + "do": "createRoom", + "actor": "alice", + "room": "receipts" + }, + { + "do": "enterRoom", + "actor": "alice", + "room": "receipts" + }, + { + "do": "send", + "actor": "alice", + "text": "did you see this" + } + ], + "criteria": [ + { + "id": "receipt-appears", + "points": 1, + "desc": "Receipt appears for the sender once another user views the message", + "steps": [ + { + "do": "enterRoom", + "actor": "bob", + "room": "receipts" + }, + { + "do": "expect", + "actor": "alice", + "testid": "read-receipt", + "within": 6000 + } + ] + }, + { + "id": "receipt-names-reader", + "points": 1, + "desc": "Receipt names the user who read it", + "steps": [ + { + "do": "expect", + "actor": "alice", + "testid": "read-receipt", + "contains": "Bob", + "within": 6000 + } + ] + }, + { + "id": "sender-excluded", + "points": 1, + "desc": "The sender is not listed as a reader of their own message", + "steps": [ + { + "do": "expect", + "actor": "alice", + "testid": "read-receipt", + "notContains": "Alice", + "within": 6000 + } + ] + } + ] + }, + { + "id": 5, + "name": "Unread Message Counts", + "actors": [ + "alice", + "bob" + ], + "setup": [ + { + "do": "signUp", + "actor": "alice", + "name": "Alice" + }, + { + "do": "signUp", + "actor": "bob", + "name": "Bob" + }, + { + "do": "createRoom", + "actor": "alice", + "room": "unread-main" + }, + { + "do": "createRoom", + "actor": "bob", + "room": "unread-side" + }, + { + "do": "enterRoom", + "actor": "alice", + "room": "unread-main" + }, + { + "do": "enterRoom", + "actor": "bob", + "room": "unread-main" + }, + { + "do": "enterRoom", + "actor": "bob", + "room": "unread-side" + } + ], + "criteria": [ + { + "id": "badge-appears", + "points": 1, + "desc": "A badge appears live on a room with unread messages", + "steps": [ + { + "do": "send", + "actor": "alice", + "text": "unread one" + }, + { + "do": "expect", + "actor": "bob", + "testid": "unread-badge", + "in": { + "testid": "room-item", + "contains": "{room:unread-main}" + }, + "within": 5000 + } + ] + }, + { + "id": "badge-counts", + "points": 1, + "desc": "The badge reflects the number of unread messages (paced above the app's send rate limit)", + "steps": [ + { + "do": "wait", + "actor": "alice", + "ms": 1400 + }, + { + "do": "send", + "actor": "alice", + "text": "unread two" + }, + { + "do": "expect", + "actor": "bob", + "testid": "unread-badge", + "contains": "2", + "in": { + "testid": "room-item", + "contains": "{room:unread-main}" + }, + "within": 5000 + } + ] + }, + { + "id": "badge-clears", + "points": 1, + "desc": "The badge clears when the room is opened", + "steps": [ + { + "do": "enterRoom", + "actor": "bob", + "room": "unread-main" + }, + { + "do": "waitUntilAbsent", + "actor": "bob", + "testid": "unread-badge", + "in": { + "testid": "room-item", + "contains": "{room:unread-main}" + }, + "within": 5000 + } + ] + } + ] + }, + { + "id": 6, + "name": "Room Members and Roles", + "actors": [ + "alice", + "bob" + ], + "setup": [ + { + "do": "signUp", + "actor": "alice", + "name": "Alice" + }, + { + "do": "signUp", + "actor": "bob", + "name": "Bob" + }, + { + "do": "createRoom", + "actor": "alice", + "room": "team" + }, + { + "do": "enterRoom", + "actor": "alice", + "room": "team" + } + ], + "criteria": [ + { + "id": "members-appear-live", + "points": 1, + "desc": "Someone joining a room appears in its member panel without anyone reloading", + "steps": [ + { + "do": "enterRoom", + "actor": "bob", + "room": "team" + }, + { + "do": "expect", + "actor": "alice", + "testid": "member-item", + "contains": "Bob", + "within": 8000 + } + ] + }, + { + "id": "owner-is-shown", + "points": 1, + "desc": "The room's creator is marked as its owner", + "steps": [ + { + "do": "expect", + "actor": "alice", + "testid": "member-owner", + "within": 6000 + } + ] + }, + { + "id": "removed-member-disappears", + "points": 1, + "desc": "Removing a member takes them out of the panel live, for everyone still in the room", + "steps": [ + { + "do": "click", + "actor": "alice", + "testid": "member-remove", + "in": { + "testid": "member-item", + "contains": "Bob" + } + }, + { + "do": "waitUntilAbsent", + "actor": "alice", + "testid": "member-item", + "contains": "Bob", + "within": 8000 + } + ] + } + ] + }, + { + "id": 7, + "name": "Direct Messages", + "actors": [ + "alice", + "bob" + ], + "setup": [ + { + "do": "signUp", + "actor": "alice", + "name": "Alice" + }, + { + "do": "signUp", + "actor": "bob", + "name": "Bob" + }, + { + "do": "wait", + "actor": "alice", + "ms": 2000 + } + ], + "criteria": [ + { + "id": "dm-can-be-started", + "points": 1, + "desc": "A user can open a direct conversation with another user", + "steps": [ + { + "do": "click", + "actor": "alice", + "testid": "dm-start", + "in": { + "testid": "online-user", + "contains": "Bob" + } + }, + { + "do": "expect", + "actor": "alice", + "testid": "dm-item", + "contains": "Bob", + "within": 8000 + } + ] + }, + { + "id": "dm-delivers-live", + "points": 1, + "desc": "A direct message reaches the other participant live, with no reload", + "steps": [ + { + "do": "click", + "actor": "alice", + "testid": "dm-item", + "in": { + "testid": "dm-list", + "contains": "Bob" + } + }, + { + "do": "click", + "actor": "bob", + "testid": "dm-item", + "contains": "Alice" + }, + { + "do": "send", + "actor": "alice", + "text": "a private word" + }, + { + "do": "expect", + "actor": "bob", + "testid": "message-item", + "contains": "a private word", + "within": 10000 + } + ] + }, + { + "id": "dm-persists", + "points": 1, + "desc": "A direct conversation and its messages survive a reload", + "steps": [ + { + "do": "reload", + "actor": "bob", + "settleMs": 3500 + }, + { + "do": "expect", + "actor": "bob", + "testid": "dm-item", + "contains": "Alice", + "within": 10000 + }, + { + "do": "click", + "actor": "bob", + "testid": "dm-item", + "contains": "Alice" + }, + { + "do": "expect", + "actor": "bob", + "testid": "message-item", + "contains": "a private word", + "within": 10000 + } + ] + } + ] + }, + { + "id": 8, + "name": "Message Ownership", + "actors": [ + "alice", + "bob" + ], + "setup": [ + { + "do": "signUp", + "actor": "alice", + "name": "Alice" + }, + { + "do": "signUp", + "actor": "bob", + "name": "Bob" + }, + { + "do": "createRoom", + "actor": "alice", + "room": "owned" + }, + { + "do": "enterRoom", + "actor": "alice", + "room": "owned" + }, + { + "do": "enterRoom", + "actor": "bob", + "room": "owned" + }, + { + "do": "send", + "actor": "alice", + "text": "a message of my own" + } + ], + "criteria": [ + { + "id": "author-can-edit-and-delete", + "points": 1, + "desc": "The author of a message is offered edit and delete on it", + "steps": [ + { + "do": "expect", + "actor": "alice", + "testid": "message-edit", + "in": { + "testid": "message-item", + "contains": "a message of my own" + }, + "within": 6000 + }, + { + "do": "expect", + "actor": "alice", + "testid": "message-delete", + "in": { + "testid": "message-item", + "contains": "a message of my own" + }, + "within": 6000 + } + ] + }, + { + "id": "others-are-not", + "points": 1, + "desc": "Nobody else is offered edit or delete on that message", + "steps": [ + { + "do": "expect", + "actor": "bob", + "testid": "message-item", + "contains": "a message of my own", + "within": 8000 + }, + { + "do": "expect", + "actor": "bob", + "testid": "message-edit", + "in": { + "testid": "message-item", + "contains": "a message of my own" + }, + "absent": true, + "within": 5000 + }, + { + "do": "expect", + "actor": "bob", + "testid": "message-delete", + "in": { + "testid": "message-item", + "contains": "a message of my own" + }, + "absent": true, + "within": 5000 + } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/chat/scenarios/01-contention-wip.json b/tools/stack-bench/tracks/chat/scenarios/01-contention-wip.json new file mode 100644 index 00000000000..9a291fd3871 --- /dev/null +++ b/tools/stack-bench/tracks/chat/scenarios/01-contention-wip.json @@ -0,0 +1,291 @@ +{ + "schemaVersion": 1, + "level": 1, + "note": "Inactive WIP: not selected by track.json. Concurrent client dispatch does not prove overlapping server execution. Promotion requires matching reference and defect controls on every supported stack.", + "features": [ + { + "id": 106, + "name": "Concurrent State Integrity", + "max": 4, + "actors": [ + "alice", + "bob", + "carol", + "dave" + ], + "note": "The pin cap is the real contention target: it must read the current count and decide, which is check-then-insert and genuinely racy. Reactions cannot show a lost update \u2014 one row per user per emoji with onConflictDoNothing cannot lose one whatever the ordering \u2014 so they test agreement and correctness rather than atomicity.", + "setup": [ + { + "do": "signUp", + "actor": "alice", + "name": "Alice" + }, + { + "do": "signUp", + "actor": "bob", + "name": "Bob" + }, + { + "do": "signUp", + "actor": "carol", + "name": "Carol" + }, + { + "do": "signUp", + "actor": "dave", + "name": "Dave" + }, + { + "do": "createRoom", + "actor": "alice", + "room": "busy" + }, + { + "do": "enterRoom", + "actor": "alice", + "room": "busy" + }, + { + "do": "enterRoom", + "actor": "bob", + "room": "busy" + }, + { + "do": "enterRoom", + "actor": "carol", + "room": "busy" + }, + { + "do": "enterRoom", + "actor": "dave", + "room": "busy" + }, + { + "do": "send", + "actor": "alice", + "text": "react to this" + } + ], + "criteria": [ + { + "id": "reaction-count-is-exact", + "points": 1, + "desc": "Four concurrent reaction replays leave exactly four reactions; client dispatch alone does not prove server overlap.", + "steps": [ + { + "do": "clickConcurrently", + "actors": [ + "alice", + "bob", + "carol", + "dave" + ], + "testid": "react-button", + "settleMs": 3500 + }, + { + "do": "clickConcurrently", + "actors": [ + "alice", + "bob", + "carol", + "dave" + ], + "testid": "react-button", + "settleMs": 3500 + }, + { + "do": "replayConcurrently", + "actors": [ + "alice", + "bob", + "carol", + "dave" + ], + "settleMs": 4000 + }, + { + "do": "expectNumber", + "actor": "alice", + "testid": "reaction-count", + "within": 12000, + "equals": 4 + } + ] + }, + { + "id": "reaction-count-agrees-across-clients", + "points": 1, + "desc": "Every client shows the same count. One client being right while another is wrong is what a broadcast that is not atomic with the write produces.", + "steps": [ + { + "do": "expectAgreement", + "actors": [ + "alice", + "bob", + "carol", + "dave" + ], + "testid": "reaction-count", + "within": 10000 + } + ] + }, + { + "id": "removing-a-reaction-decrements-once", + "points": 1, + "desc": "A user removing their reaction takes exactly one off the count, and every client agrees on the new total. One user contributes at most one either way.", + "steps": [ + { + "do": "click", + "actor": "alice", + "testid": "react-button" + }, + { + "do": "wait", + "actor": "alice", + "ms": 3000 + }, + { + "do": "expectNumber", + "actor": "alice", + "testid": "reaction-count", + "within": 8000, + "equals": 3 + }, + { + "do": "expectAgreement", + "actors": [ + "alice", + "bob", + "carol", + "dave" + ], + "testid": "reaction-count", + "within": 10000 + } + ] + }, + { + "id": "pin-cap-holds-under-concurrency", + "points": 1, + "desc": "Four replayed pin requests against a cap of three leave three pinned messages.", + "steps": [ + { + "do": "sendMany", + "actor": "alice", + "prefix": "PIN", + "count": 4, + "delayMs": 1400 + }, + { + "do": "wait", + "actor": "alice", + "ms": 2000 + }, + { + "do": "click", + "actor": "alice", + "testid": "pin-button", + "in": { + "testid": "message-item", + "contains": "PIN-1" + } + }, + { + "do": "click", + "actor": "bob", + "testid": "pin-button", + "in": { + "testid": "message-item", + "contains": "PIN-2" + } + }, + { + "do": "click", + "actor": "carol", + "testid": "pin-button", + "in": { + "testid": "message-item", + "contains": "PIN-3" + } + }, + { + "do": "wait", + "actor": "alice", + "ms": 3000 + }, + { + "do": "click", + "actor": "dave", + "testid": "pin-button", + "in": { + "testid": "message-item", + "contains": "PIN-4" + } + }, + { + "do": "click", + "actor": "alice", + "testid": "pinned-unpin", + "in": { + "testid": "pinned-item", + "contains": "PIN-1" + } + }, + { + "do": "click", + "actor": "bob", + "testid": "pinned-unpin", + "in": { + "testid": "pinned-item", + "contains": "PIN-2" + } + }, + { + "do": "click", + "actor": "carol", + "testid": "pinned-unpin", + "in": { + "testid": "pinned-item", + "contains": "PIN-3" + } + }, + { + "do": "wait", + "actor": "alice", + "ms": 3000 + }, + { + "do": "expect", + "actor": "alice", + "testid": "pinned-item", + "count": 0, + "within": 8000 + }, + { + "do": "replayConcurrently", + "actors": [ + "alice", + "bob", + "carol", + "dave" + ], + "method": "POST", + "settleMs": 5000 + }, + { + "do": "expect", + "actor": "alice", + "testid": "pinned-item", + "count": 3, + "within": 12000 + } + ] + } + ] + } + ], + "blocked_on": "The write path is a WebSocket, so replaying captured HTTP requests cannot reach it. The rebuilt PostgreSQL app pins over socket.io ('pin:add'), authenticated by a session token from the signup response and passed as socket.handshake.auth.token. Driving contention needs a socket-aware mechanism: connect a client with that token and emit the captured event several times without awaiting the acknowledgements. Everything needed is already captured \u2014 Actor.lastWsWrite records the socket.io event name and payload.", + "why_it_matters": "This rebuild is genuinely vulnerable and would be the first real contention finding. Its pin cap reads the count and inserts inside one transaction with NO advisory lock, so at READ COMMITTED concurrent transactions can all see room under the cap and all insert. The previous build defended the same code with pg_advisory_xact_lock and a comment explaining the race. Two builds of the same spec, one safe and one not, is exactly the variance a benchmark exists to measure \u2014 and the harness currently cannot see it." +} diff --git a/tools/stack-bench/tracks/chat/scenarios/01-delivery.json b/tools/stack-bench/tracks/chat/scenarios/01-delivery.json new file mode 100644 index 00000000000..41c3032c9a2 --- /dev/null +++ b/tools/stack-bench/tracks/chat/scenarios/01-delivery.json @@ -0,0 +1,231 @@ +{ + "schemaVersion": 1, + "level": 1, + "note": "Delivery and resilience invariants \u2014 properties of the transport, not of any feature. These are the ones a human grader cannot test at all (you cannot generate concurrency or simulate a network drop by clicking), and where the asymmetry is structural: Express stacks broadcast with io.emit() AFTER awaiting the write, so delivery is not atomic with the commit; a SpacetimeDB subscription update IS the commit. Senders pace themselves above the apps' rate limits (observed: 400ms SpacetimeDB, 350ms Mongo) \u2014 the spec requires spam prevention, so a 0-delay burst is correctly rejected and would test nothing. Concurrency comes from multiple senders overlapping, not from one sender bursting. Actors sign up with credentials, as level 1 requires.", + "features": [ + { + "id": 201, + "name": "Delivery Integrity", + "max": 3, + "actors": [ + "alice", + "bob" + ], + "setup": [ + { + "do": "signUp", + "actor": "alice", + "name": "Alice" + }, + { + "do": "signUp", + "actor": "bob", + "name": "Bob" + }, + { + "do": "createRoom", + "actor": "alice", + "room": "storm" + }, + { + "do": "enterRoom", + "actor": "alice", + "room": "storm" + }, + { + "do": "enterRoom", + "actor": "bob", + "room": "storm" + } + ], + "criteria": [ + { + "id": "no-loss-no-duplication-sender", + "points": 1, + "desc": "Under a concurrent write storm from two clients, the sender sees every message exactly once \u2014 no drops, no double-render from optimistic insert plus broadcast echo", + "steps": [ + { + "do": "sendConcurrently", + "senders": [ + { + "actor": "alice", + "prefix": "AA", + "count": 6 + }, + { + "actor": "bob", + "prefix": "BB", + "count": 6 + } + ], + "delayMs": 1400 + }, + { + "do": "expectAllPresent", + "actor": "alice", + "prefix": "AA", + "count": 6, + "within": 20000 + } + ] + }, + { + "id": "no-loss-no-duplication-receiver", + "points": 1, + "desc": "The receiving client also observes every message from the other sender exactly once", + "steps": [ + { + "do": "expectAllPresent", + "actor": "alice", + "prefix": "BB", + "count": 6, + "within": 20000 + }, + { + "do": "expectAllPresent", + "actor": "bob", + "prefix": "AA", + "count": 6, + "within": 20000 + } + ] + }, + { + "id": "consistent-order-across-clients", + "points": 1, + "desc": "Both clients observe the interleaved messages in the SAME order. Non-atomic broadcast lets concurrent senders interleave differently per client.", + "steps": [ + { + "do": "expectOrderMatches", + "actors": [ + "alice", + "bob" + ], + "prefix": "(?:AA|BB)" + } + ] + } + ] + }, + { + "id": 202, + "name": "Connection Resilience", + "max": 2, + "actors": [ + "alice", + "bob" + ], + "setup": [ + { + "do": "signUp", + "actor": "alice", + "name": "Alice" + }, + { + "do": "signUp", + "actor": "bob", + "name": "Bob" + }, + { + "do": "createRoom", + "actor": "alice", + "room": "resilience" + }, + { + "do": "enterRoom", + "actor": "alice", + "room": "resilience" + }, + { + "do": "enterRoom", + "actor": "bob", + "room": "resilience" + }, + { + "do": "sendMany", + "actor": "bob", + "prefix": "BEFORE", + "count": 2, + "delayMs": 1400 + }, + { + "do": "expectAllPresent", + "actor": "alice", + "prefix": "BEFORE", + "count": 2, + "within": 10000 + } + ], + "criteria": [ + { + "id": "resync-after-disconnect", + "points": 1, + "desc": "A client that loses its connection and regains it must catch up on everything it missed, WITHOUT a reload.", + "steps": [ + { + "do": "setOffline", + "actor": "alice", + "offline": true, + "settleMs": 1500 + }, + { + "do": "sendMany", + "actor": "bob", + "prefix": "OFFLINE", + "count": 3, + "delayMs": 1400 + }, + { + "do": "setOffline", + "actor": "alice", + "offline": false, + "settleMs": 3000 + }, + { + "do": "expectAllPresent", + "actor": "alice", + "prefix": "OFFLINE", + "count": 3, + "within": 15000 + } + ] + }, + { + "id": "no-duplication-after-resync", + "points": 1, + "desc": "Catching up must not double-deliver messages the client already had (resubscribe replaying history on top of existing state)", + "steps": [ + { + "do": "sendMany", + "actor": "bob", + "prefix": "AFTER", + "count": 2, + "delayMs": 1400 + }, + { + "do": "expectAllPresent", + "actor": "alice", + "prefix": "AFTER", + "count": 2, + "within": 10000 + }, + { + "do": "expectAllPresent", + "actor": "alice", + "prefix": "OFFLINE", + "count": 3, + "within": 5000 + }, + { + "do": "expectAllPresent", + "actor": "alice", + "prefix": "BEFORE", + "count": 2, + "within": 10000 + } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/chat/scenarios/01-invariants.json b/tools/stack-bench/tracks/chat/scenarios/01-invariants.json new file mode 100644 index 00000000000..e94cbe15f62 --- /dev/null +++ b/tools/stack-bench/tracks/chat/scenarios/01-invariants.json @@ -0,0 +1,1029 @@ +{ + "schemaVersion": 1, + "level": 1, + "track": "stack-bench", + "note": "Cross-cutting properties, scored on a separate axis from features. A feature-only score cannot see these: an app can implement every listed feature and still let one user take over another's account. Each invariant is its own feature with fresh contexts, so one root defect cannot cascade into several apparent problems.", + "features": [ + { + "id": 101, + "name": "Account Integrity", + "max": 2, + "actors": [ + "victim", + "impostor" + ], + "setup": [ + { + "do": "signUp", + "actor": "victim", + "name": "Victim" + }, + { + "do": "createRoom", + "actor": "victim", + "room": "private-notes" + }, + { + "do": "enterRoom", + "actor": "victim", + "room": "private-notes" + }, + { + "do": "send", + "actor": "victim", + "text": "victim private message" + } + ], + "criteria": [ + { + "id": "username-is-not-a-credential", + "points": 1, + "desc": "Knowing a username must not grant access to that account. Signing up with a taken username, or signing in with the wrong password, must be rejected \u2014 not silently resolved to the existing account.", + "steps": [ + { + "do": "signUp", + "actor": "impostor", + "name": "Victim", + "expectFailure": true + }, + { + "do": "expect", + "actor": "impostor", + "testid": "current-user", + "contains": "Victim", + "absent": true, + "within": 5000 + } + ] + }, + { + "id": "wrong-password-rejected", + "points": 1, + "desc": "Signing in with a wrong password is rejected and grants nothing", + "steps": [ + { + "do": "signIn", + "actor": "impostor", + "name": "Victim", + "password": "not-the-password", + "expectFailure": true + }, + { + "do": "expect", + "actor": "impostor", + "testid": "auth-error", + "within": 6000 + }, + { + "do": "expect", + "actor": "impostor", + "testid": "message-item", + "contains": "victim private message", + "absent": true, + "within": 4000 + } + ] + } + ] + }, + { + "id": 102, + "name": "Session Durability", + "max": 3, + "actors": [ + "alice", + "bob" + ], + "setup": [ + { + "do": "signUp", + "actor": "alice", + "name": "Alice" + }, + { + "do": "signUp", + "actor": "bob", + "name": "Bob" + }, + { + "do": "createRoom", + "actor": "alice", + "room": "mine" + }, + { + "do": "enterRoom", + "actor": "alice", + "room": "mine" + }, + { + "do": "send", + "actor": "alice", + "text": "alice message" + }, + { + "do": "enterRoom", + "actor": "bob", + "room": "mine" + } + ], + "criteria": [ + { + "id": "session-survives-reload", + "points": 1, + "desc": "Reloading keeps the user signed in as the same account", + "steps": [ + { + "do": "reload", + "actor": "alice", + "settleMs": 3000 + }, + { + "do": "expect", + "actor": "alice", + "testid": "current-user", + "contains": "Alice", + "within": 8000 + } + ] + }, + { + "id": "session-survives-reconnect", + "points": 1, + "desc": "A user who loses and regains their connection while sitting on the page is still signed in as the same account. The connection drops and re-establishes on its own; re-identifying the client here would orphan everything it owns.", + "steps": [ + { + "do": "setOffline", + "actor": "alice", + "offline": true, + "settleMs": 3000 + }, + { + "do": "setOffline", + "actor": "alice", + "offline": false, + "settleMs": 6000 + }, + { + "do": "expect", + "actor": "alice", + "testid": "current-user", + "contains": "Alice", + "within": 20000 + } + ] + }, + { + "id": "own-content-still-owned-after-reconnect", + "points": 1, + "desc": "After reconnecting, the room owner can still remove a member, and that member loses access.", + "steps": [ + { + "do": "enterRoom", + "actor": "alice", + "room": "mine" + }, + { + "do": "expect", + "actor": "alice", + "testid": "message-item", + "contains": "alice message", + "within": 15000 + }, + { + "do": "click", + "actor": "alice", + "testid": "member-remove", + "in": { + "testid": "member-item", + "contains": "Bob" + } + }, + { + "do": "waitUntilAbsent", + "actor": "alice", + "testid": "member-item", + "contains": "Bob", + "within": 10000 + }, + { + "do": "waitUntilAbsent", + "actor": "bob", + "testid": "message-input", + "within": 10000 + } + ] + } + ], + "note": "Session durability is tested by dropping the client's connection, not by restarting the backend. Both force the same reconnect, but only one needs a server the harness owns \u2014 and restarting a shared host takes unrelated databases down with it. Backend restarts belong where they test storage rather than session handling (level 4)." + }, + { + "id": 103, + "name": "Data Isolation", + "max": 2, + "actors": [ + "alice", + "bob" + ], + "setup": [ + { + "do": "signUp", + "actor": "alice", + "name": "Alice" + }, + { + "do": "signUp", + "actor": "bob", + "name": "Bob" + }, + { + "do": "createRoom", + "actor": "alice", + "room": "room-a" + }, + { + "do": "createRoom", + "actor": "bob", + "room": "room-b" + }, + { + "do": "enterRoom", + "actor": "alice", + "room": "room-a" + }, + { + "do": "enterRoom", + "actor": "bob", + "room": "room-b" + }, + { + "do": "send", + "actor": "alice", + "text": "message for room a" + } + ], + "criteria": [ + { + "id": "messages-scoped-to-room", + "points": 1, + "desc": "A message sent in one room never appears in another", + "steps": [ + { + "do": "expect", + "actor": "bob", + "testid": "message-item", + "contains": "message for room a", + "absent": true, + "within": 5000 + } + ] + }, + { + "id": "no-duplicate-rooms", + "points": 1, + "desc": "A room renders exactly once in the list \u2014 an optimistic insert plus a broadcast echo duplicates it", + "steps": [ + { + "do": "expect", + "actor": "alice", + "testid": "room-item", + "contains": "{room:room-a}", + "count": 1, + "within": 5000 + } + ] + } + ] + }, + { + "id": 104, + "name": "Per-User State Integrity", + "max": 3, + "actors": [ + "alice", + "bob" + ], + "note": "Unread counts and read receipts are per-user state derived from other people's actions. The historical grading data has more defects here than anywhere else in level 1.", + "setup": [ + { + "do": "signUp", + "actor": "alice", + "name": "Alice" + }, + { + "do": "signUp", + "actor": "bob", + "name": "Bob" + }, + { + "do": "createRoom", + "actor": "alice", + "room": "main" + }, + { + "do": "createRoom", + "actor": "alice", + "room": "side" + }, + { + "do": "enterRoom", + "actor": "alice", + "room": "main" + }, + { + "do": "enterRoom", + "actor": "bob", + "room": "main" + } + ], + "criteria": [ + { + "id": "own-messages-not-unread", + "points": 1, + "desc": "Your own message must never count as unread for you. (historical: a sender's own badge went to 1 on SpacetimeDB, twice)", + "steps": [ + { + "do": "send", + "actor": "alice", + "text": "alice speaking" + }, + { + "do": "wait", + "actor": "alice", + "ms": 1400 + }, + { + "do": "enterRoom", + "actor": "alice", + "room": "side" + }, + { + "do": "expect", + "actor": "alice", + "testid": "unread-badge", + "in": { + "testid": "room-item", + "contains": "{room:main}" + }, + "absent": true, + "within": 5000 + } + ] + }, + { + "id": "unread-does-not-resurrect", + "points": 1, + "desc": "Once a room is open its badge stays clear while new messages arrive in it \u2014 you are looking at them. (historical: badges that never cleared)", + "steps": [ + { + "do": "enterRoom", + "actor": "alice", + "room": "main" + }, + { + "do": "wait", + "actor": "alice", + "ms": 1400 + }, + { + "do": "send", + "actor": "bob", + "text": "bob speaking" + }, + { + "do": "expect", + "actor": "alice", + "testid": "message-item", + "contains": "bob speaking", + "within": 6000 + }, + { + "do": "expect", + "actor": "alice", + "testid": "unread-badge", + "in": { + "testid": "room-item", + "contains": "{room:main}" + }, + "absent": true, + "within": 4000 + } + ] + }, + { + "id": "receipts-keep-updating", + "points": 1, + "desc": "Receipts stay live for later messages, not just the first one. (historical: 'Seen by' that only appeared after leaving and re-entering the room)", + "steps": [ + { + "do": "wait", + "actor": "alice", + "ms": 1400 + }, + { + "do": "send", + "actor": "alice", + "text": "second message from alice" + }, + { + "do": "expect", + "actor": "bob", + "testid": "message-item", + "contains": "second message from alice", + "within": 6000 + }, + { + "do": "expect", + "actor": "alice", + "testid": "read-receipt", + "contains": "Bob", + "within": 10000, + "in": { + "testid": "message-item", + "contains": "second message from alice" + } + } + ] + } + ] + }, + { + "id": 105, + "name": "Presence and Display Stability", + "max": 2, + "actors": [ + "alice", + "bob", + "carol" + ], + "note": "Presence was wrong in six of the historically graded runs, and 'Seen by' lists were seen flickering between orderings. Neither is visible to a single assertion.", + "setup": [ + { + "do": "signUp", + "actor": "alice", + "name": "Alice" + }, + { + "do": "signUp", + "actor": "bob", + "name": "Bob" + }, + { + "do": "signUp", + "actor": "carol", + "name": "Carol" + }, + { + "do": "createRoom", + "actor": "alice", + "room": "lobby" + }, + { + "do": "enterRoom", + "actor": "alice", + "room": "lobby" + }, + { + "do": "enterRoom", + "actor": "bob", + "room": "lobby" + }, + { + "do": "enterRoom", + "actor": "carol", + "room": "lobby" + }, + { + "do": "send", + "actor": "alice", + "text": "who is here" + } + ], + "criteria": [ + { + "id": "receipt-text-does-not-flicker", + "points": 1, + "desc": "The 'Seen by' list must not change while nothing is happening. (historical: names flipping order continuously from an unstable sort)", + "steps": [ + { + "do": "expect", + "actor": "alice", + "testid": "read-receipt", + "contains": "Bob", + "within": 10000 + }, + { + "do": "expect", + "actor": "alice", + "testid": "read-receipt", + "contains": "Carol", + "within": 10000 + }, + { + "do": "expectStable", + "actor": "alice", + "testid": "read-receipt", + "samples": 4, + "intervalMs": 700 + } + ] + }, + { + "id": "presence-drops-signed-out-user", + "points": 1, + "desc": "A user who signs out disappears from the online list. (historical: presence lists stale or stuck, six runs)", + "steps": [ + { + "do": "expect", + "actor": "alice", + "testid": "online-users", + "contains": "Bob", + "within": 8000 + }, + { + "do": "click", + "actor": "bob", + "testid": "signout" + }, + { + "do": "waitUntilAbsent", + "actor": "alice", + "testid": "online-users", + "contains": "Bob", + "within": 12000 + } + ] + } + ] + }, + { + "id": 107, + "name": "Server-Side Truth", + "max": 2, + "actors": [ + "alice", + "bob" + ], + "note": "Whatever the acting client shows must actually exist on the server. Optimistic inserts that were never confirmed, and counts incremented only in one browser, are invisible to any assertion made from that same browser.", + "setup": [ + { + "do": "signUp", + "actor": "alice", + "name": "Alice" + }, + { + "do": "signUp", + "actor": "bob", + "name": "Bob" + }, + { + "do": "createRoom", + "actor": "alice", + "room": "truth" + }, + { + "do": "enterRoom", + "actor": "alice", + "room": "truth" + }, + { + "do": "enterRoom", + "actor": "bob", + "room": "truth" + } + ], + "criteria": [ + { + "id": "rejected-message-does-not-linger", + "points": 1, + "desc": "A message the server refused for exceeding the rate limit must not stay visible to its sender. An optimistic insert without rollback leaves the sender believing they said something nobody received.", + "steps": [ + { + "do": "send", + "actor": "alice", + "text": "first of a burst" + }, + { + "do": "send", + "actor": "alice", + "text": "REJECTED BY RATE LIMIT" + }, + { + "do": "wait", + "actor": "alice", + "ms": 4000 + }, + { + "do": "expect", + "actor": "bob", + "testid": "message-item", + "contains": "REJECTED BY RATE LIMIT", + "absent": true, + "within": 4000 + }, + { + "do": "expect", + "actor": "alice", + "testid": "message-item", + "contains": "REJECTED BY RATE LIMIT", + "absent": true, + "within": 4000 + } + ] + }, + { + "id": "a-fresh-client-sees-the-same-thing", + "points": 1, + "desc": "A brand-new browser sees what the acting client claims. Anything visible only to the client that performed the action never reached the server.", + "steps": [ + { + "do": "wait", + "actor": "alice", + "ms": 1500 + }, + { + "do": "send", + "actor": "alice", + "text": "durable statement" + }, + { + "do": "expect", + "actor": "alice", + "testid": "message-item", + "contains": "durable statement", + "within": 6000 + }, + { + "do": "freshClient", + "actor": "bob" + }, + { + "do": "signUp", + "actor": "bob-fresh", + "name": "Observer" + }, + { + "do": "enterRoom", + "actor": "bob-fresh", + "room": "truth" + }, + { + "do": "expect", + "actor": "bob-fresh", + "testid": "message-item", + "contains": "durable statement", + "within": 10000 + } + ] + } + ] + }, + { + "id": 109, + "name": "Direct Message Privacy", + "max": 3, + "actors": [ + "alice", + "bob", + "snoop" + ], + "note": "A direct conversation is the first data in level 1 that must not reach someone. The wire check is the one that matters: an app that broadcasts widely and filters in the browser looks perfect on screen while the data is already in every client.", + "setup": [ + { + "do": "signUp", + "actor": "alice", + "name": "Alice" + }, + { + "do": "signUp", + "actor": "bob", + "name": "Bob" + }, + { + "do": "signUp", + "actor": "snoop", + "name": "Snoop" + }, + { + "do": "createRoom", + "actor": "alice", + "room": "lobby" + }, + { + "do": "enterRoom", + "actor": "alice", + "room": "lobby" + }, + { + "do": "enterRoom", + "actor": "snoop", + "room": "lobby" + }, + { + "do": "send", + "actor": "alice", + "text": "a public message in the lobby" + }, + { + "do": "wait", + "actor": "alice", + "ms": 2500 + }, + { + "do": "click", + "actor": "alice", + "testid": "dm-start", + "in": { + "testid": "online-user", + "contains": "Bob" + } + }, + { + "do": "wait", + "actor": "alice", + "ms": 2000 + }, + { + "do": "click", + "actor": "alice", + "testid": "dm-item", + "in": { + "testid": "dm-list", + "contains": "Bob" + } + }, + { + "do": "send", + "actor": "alice", + "text": "SECRET between alice and bob" + } + ], + "criteria": [ + { + "id": "outsider-cannot-see-the-conversation", + "points": 1, + "desc": "A third party does not see the conversation in their list", + "steps": [ + { + "do": "expect", + "actor": "snoop", + "testid": "dm-item", + "contains": "Alice", + "absent": true, + "within": 6000 + } + ] + }, + { + "id": "outsider-cannot-read-the-messages", + "points": 1, + "desc": "A third party never sees the messages", + "steps": [ + { + "do": "expect", + "actor": "snoop", + "testid": "message-item", + "contains": "SECRET between alice and bob", + "absent": true, + "within": 6000 + } + ] + }, + { + "id": "outsider-never-receives-it-over-the-wire", + "points": 1, + "desc": "A third party's client is never SENT the private message. Hiding it in the interface is not privacy \u2014 anyone with developer tools can read what their own browser received.", + "steps": [ + { + "do": "expectReceived", + "actor": "snoop", + "contains": "a public message in the lobby", + "within": 8000 + }, + { + "do": "click", + "actor": "bob", + "testid": "dm-item", + "in": { + "testid": "dm-list", + "contains": "Alice" + } + }, + { + "do": "wait", + "actor": "bob", + "ms": 2000 + }, + { + "do": "send", + "actor": "alice", + "text": "SECOND SECRET for bob only" + }, + { + "do": "wait", + "actor": "alice", + "ms": 4000 + }, + { + "do": "expectReceived", + "actor": "bob", + "contains": "SECOND SECRET for bob only", + "within": 8000 + }, + { + "do": "expectNotReceived", + "actor": "snoop", + "contains": "SECOND SECRET for bob only" + } + ] + } + ] + }, + { + "id": 108, + "name": "Authorization", + "max": 4, + "actors": [ + "owner", + "member", + "target", + "outsider" + ], + "note": "HTTP replays test authenticated server refusal when a matching request can be captured. Without an observable replay path, the criterion is inconclusive. DOM-only criteria establish interface behavior, not server authorization.", + "setup": [ + { + "do": "signUp", + "actor": "owner", + "name": "Owner" + }, + { + "do": "signUp", + "actor": "member", + "name": "Member" + }, + { + "do": "signUp", + "actor": "target", + "name": "Target" + }, + { + "do": "signUp", + "actor": "outsider", + "name": "Outsider" + }, + { + "do": "createRoom", + "actor": "owner", + "room": "guarded" + }, + { + "do": "enterRoom", + "actor": "owner", + "room": "guarded" + }, + { + "do": "enterRoom", + "actor": "member", + "room": "guarded" + }, + { + "do": "enterRoom", + "actor": "target", + "room": "guarded" + }, + { + "do": "send", + "actor": "owner", + "text": "owner message" + }, + { + "do": "wait", + "actor": "owner", + "ms": 2500 + } + ], + "criteria": [ + { + "id": "outsider-cannot-read-the-room", + "points": 1, + "desc": "Someone who never joined the room cannot read its history", + "steps": [ + { + "do": "expect", + "actor": "outsider", + "testid": "message-item", + "contains": "owner message", + "absent": true, + "within": 6000 + } + ] + }, + { + "id": "outsider-cannot-post-to-the-room", + "points": 1, + "desc": "Someone who never joined cannot post to the room, even by re-issuing a member's own send request", + "steps": [ + { + "do": "replayAs", + "actor": "outsider", + "from": "owner", + "match": "owner message", + "swap": { + "find": "owner message", + "with": "posted by an outsider" + } + }, + { + "do": "expectReplayRejected", + "actor": "outsider" + }, + { + "do": "expect", + "actor": "owner", + "testid": "message-item", + "contains": "posted by an outsider", + "absent": true, + "within": 6000 + } + ] + }, + { + "id": "removed-member-stops-receiving", + "points": 1, + "desc": "A removed user stops receiving the room's messages immediately, without reloading. Revoking access has to stop the stream, not just stop the interface drawing it.", + "steps": [ + { + "do": "expectReceived", + "actor": "target", + "contains": "owner message", + "within": 8000 + }, + { + "do": "click", + "actor": "owner", + "testid": "member-remove", + "in": { + "testid": "member-item", + "contains": "Target" + } + }, + { + "do": "wait", + "actor": "owner", + "ms": 3000 + }, + { + "do": "send", + "actor": "owner", + "text": "after the removal" + }, + { + "do": "wait", + "actor": "owner", + "ms": 4000 + }, + { + "do": "expectNotReceived", + "actor": "target", + "contains": "after the removal" + } + ] + }, + { + "id": "only-an-owner-can-remove-a-member", + "points": 1, + "desc": "A member who is not an owner cannot remove anyone \u2014 not through the interface, and not by re-issuing the owner's own removal request against a different person", + "steps": [ + { + "do": "expect", + "actor": "member", + "testid": "member-remove", + "absent": true, + "within": 5000 + }, + { + "do": "replayAs", + "actor": "member", + "from": "owner", + "match": "member", + "swap": { + "find": "{user:Target}", + "with": "{user:Owner}" + } + }, + { + "do": "expectReplayRejected", + "actor": "member" + }, + { + "do": "expect", + "actor": "owner", + "testid": "member-item", + "contains": "Owner", + "within": 6000 + } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/chat/scenarios/01-systems.json b/tools/stack-bench/tracks/chat/scenarios/01-systems.json new file mode 100644 index 00000000000..dca4feac96d --- /dev/null +++ b/tools/stack-bench/tracks/chat/scenarios/01-systems.json @@ -0,0 +1,242 @@ +{ + "schemaVersion": 1, + "level": 1, + "note": "Properties of the system, not of any one feature: what happens when the database changes underneath the app, and when two things happen at the same time. A broadcast layer only announces writes that went through it \u2014 a moderation script that deletes rows directly leaves every open client showing messages that no longer exist. Every criterion is withheld at 0 points until it demonstrably fails a real build AND its mutant is CAUGHT, the same promotion rule as contention.", + "features": [ + { + "id": 901, + "name": "A write from outside the app is visible without reload", + "actors": [ + "alice", + "bob", + "auditor" + ], + "setup": [ + { + "do": "signUp", + "actor": "alice", + "name": "SysAlice" + }, + { + "do": "signUp", + "actor": "bob", + "name": "SysBob" + }, + { + "do": "createRoom", + "actor": "alice", + "room": "sysroom" + }, + { + "do": "enterRoom", + "actor": "alice", + "room": "sysroom" + }, + { + "do": "enterRoom", + "actor": "bob", + "room": "sysroom" + }, + { + "do": "send", + "actor": "bob", + "text": "purge-me-alpha" + }, + { + "do": "wait", + "actor": "bob", + "ms": 1400 + }, + { + "do": "send", + "actor": "bob", + "text": "purge-me-beta" + }, + { + "do": "expect", + "actor": "alice", + "testid": "message-item", + "contains": "purge-me-beta", + "within": 8000 + } + ], + "criteria": [ + { + "id": "901a", + "desc": "a back-office script purges SysBob's messages directly in the database; a client with the room open sees them disappear without touching the page", + "points": 0, + "note": "WITHHELD until proven against a mutant (backoffice-writes-via-server) and a real failing build. The script is the app's own scripts/backoffice.mjs, required by the spec to write to the database without the web server's help \u2014 moderation, compliance takedowns and data-retention jobs all look exactly like this.", + "steps": [ + { + "do": "runScript", + "script": "scripts/backoffice.mjs", + "args": [ + "purge-user", + "{user:SysBob}" + ], + "settleMs": 4000 + }, + { + "do": "expectElementCount", + "actor": "alice", + "testid": "message-item", + "contains": "purge-me", + "equals": 0, + "within": 10000 + } + ] + }, + { + "id": "901b", + "desc": "a customer arriving after the purge never sees the messages \u2014 the rows are gone from the database, not merely from one client's memory", + "points": 0, + "note": "WITHHELD \u2014 promoted together with 901a. A fresh actor rather than a reload, so this cannot be confused by session-persistence behaviour, which invariant 105 scores separately.", + "steps": [ + { + "do": "signUp", + "actor": "auditor", + "name": "SysAuditor" + }, + { + "do": "enterRoom", + "actor": "auditor", + "room": "sysroom" + }, + { + "do": "expectElementCount", + "actor": "auditor", + "testid": "message-item", + "contains": "purge-me", + "equals": 0, + "within": 10000 + } + ] + }, + { + "id": "901c", + "desc": "a purge that lands while the app server is down is still reflected to an open client once the server returns \u2014 the deploy window loses nothing", + "points": 0, + "note": "WITHHELD \u2014 same promotion rule, plus this is the anti-cheat phase: a script that talks to the app server fails here outright. NOTIFY-style event relays also fail unless the app refetches on reconnect, because a notification fired while nobody listened is gone forever.", + "steps": [ + { + "do": "send", + "actor": "alice", + "text": "deploy-window-msg" + }, + { + "do": "expect", + "actor": "auditor", + "testid": "message-item", + "contains": "deploy-window-msg", + "within": 8000 + }, + { + "do": "stopAppServer" + }, + { + "do": "runScript", + "script": "scripts/backoffice.mjs", + "args": [ + "purge-user", + "{user:SysAlice}" + ] + }, + { + "do": "startAppServer" + }, + { + "do": "expectElementCount", + "actor": "auditor", + "testid": "message-item", + "contains": "deploy-window-msg", + "equals": 0, + "within": 20000 + } + ] + } + ] + }, + { + "id": 902, + "name": "A list being opened while its contents change", + "actors": [ + "host", + "watcher", + "joiner" + ], + "setup": [ + { + "do": "signUp", + "actor": "host", + "name": "RaceHost" + }, + { + "do": "signUp", + "actor": "watcher", + "name": "RaceWatcher" + }, + { + "do": "signUp", + "actor": "joiner", + "name": "RaceJoiner" + }, + { + "do": "createRoom", + "actor": "host", + "room": "raceroom" + }, + { + "do": "enterRoom", + "actor": "host", + "room": "raceroom" + } + ], + "criteria": [ + { + "id": "902a", + "desc": "one user enters the room \u2014 fetching its member list \u2014 at the same moment another joins: the fetcher ends with the joiner exactly once, not zero (stale fetch) and not twice (fetch merged with the join broadcast)", + "points": 0, + "note": "WITHHELD until proven against a mutant (fetch-then-append-dupe) and a real failing build. The race window is real; INCONCLUSIVE on a missed race, never a false FAIL.", + "steps": [ + { + "do": "race", + "branches": [ + [ + { + "do": "enterRoom", + "actor": "watcher", + "room": "raceroom" + } + ], + [ + { + "do": "enterRoom", + "actor": "joiner", + "room": "raceroom" + } + ] + ], + "settleMs": 4000 + }, + { + "do": "expectElementCount", + "actor": "watcher", + "testid": "member-item", + "contains": "RaceJoiner", + "equals": 1, + "within": 10000 + }, + { + "do": "expectElementCount", + "actor": "watcher", + "testid": "member-item", + "contains": "RaceHost", + "equals": 1, + "within": 10000 + } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/chat/scenarios/02-features.json b/tools/stack-bench/tracks/chat/scenarios/02-features.json new file mode 100644 index 00000000000..1079dab243f --- /dev/null +++ b/tools/stack-bench/tracks/chat/scenarios/02-features.json @@ -0,0 +1,208 @@ +{ + "schemaVersion": 1, + "level": 2, + "note": "Level 2 features: access control and the social layer. Levels are cumulative — these run against the same app as level 1's suites, after the upgrade.", + "features": [ + { + "id": 201, + "name": "Private Rooms", + "actors": ["owner", "outsider"], + "setup": [ + { "do": "signUp", "actor": "owner", "name": "Pria" }, + { "do": "signUp", "actor": "outsider", "name": "Oleg" }, + { "do": "createRoom", "actor": "owner", "room": "vault", "private": true }, + { "do": "enterRoom", "actor": "owner", "room": "vault" }, + { "do": "send", "actor": "owner", "text": "vault-secret-note" } + ], + "criteria": [ + { + "id": "201a", + "desc": "the owner sees the private badge on their room", + "points": 1, + "steps": [ + { "do": "expect", "actor": "owner", "testid": "room-private-badge", "within": 8000 } + ] + }, + { + "id": "201b", + "desc": "a private room does not appear in a non-member's room list at all", + "points": 2, + "steps": [ + { "do": "expect", "actor": "outsider", "testid": "room-item", "contains": "{room:vault}", "absent": true, "within": 6000 } + ] + } + ] + }, + { + "id": 202, + "name": "Invitation and Removal", + "actors": ["owner", "guest"], + "setup": [ + { "do": "signUp", "actor": "owner", "name": "Held" }, + { "do": "signUp", "actor": "guest", "name": "Gina" }, + { "do": "createRoom", "actor": "owner", "room": "committee", "private": true }, + { "do": "enterRoom", "actor": "owner", "room": "committee" }, + { "do": "send", "actor": "owner", "text": "pre-invite-history" } + ], + "criteria": [ + { + "id": "202a", + "desc": "an invited user gains access and can read the room's history", + "points": 2, + "steps": [ + { "do": "fill", "actor": "owner", "testid": "invite-input", "text": "{user:Gina}" }, + { "do": "click", "actor": "owner", "testid": "invite-submit", "settleMs": 1200 }, + { "do": "enterRoom", "actor": "guest", "room": "committee" }, + { "do": "expect", "actor": "guest", "testid": "message-item", "contains": "pre-invite-history", "within": 10000 } + ] + }, + { + "id": "202b", + "desc": "removal takes effect live: the removed member stops receiving without reloading", + "points": 2, + "steps": [ + { "do": "click", "actor": "owner", "testid": "member-remove", "in": { "testid": "member-item", "contains": "Gina" }, "settleMs": 1500 }, + { "do": "send", "actor": "owner", "text": "post-removal-note" }, + { "do": "wait", "actor": "owner", "ms": 2500 }, + { "do": "expect", "actor": "guest", "testid": "message-item", "contains": "post-removal-note", "absent": true, "within": 4000 } + ] + } + ] + }, + { + "id": 203, + "name": "Profiles", + "actors": ["subject", "viewer"], + "setup": [ + { "do": "signUp", "actor": "subject", "name": "Sana" }, + { "do": "signUp", "actor": "viewer", "name": "Vero" }, + { "do": "createRoom", "actor": "subject", "room": "profroom" }, + { "do": "enterRoom", "actor": "subject", "room": "profroom" }, + { "do": "enterRoom", "actor": "viewer", "room": "profroom" }, + { "do": "click", "actor": "subject", "testid": "profile-close", "ifAvailable": true }, + { "do": "click", "actor": "subject", "testid": "member-item", "contains": "Sana" }, + { "do": "fill", "actor": "subject", "testid": "bio-input", "text": "first-bio-text" }, + { "do": "click", "actor": "subject", "testid": "bio-save", "settleMs": 1200 } + ], + "criteria": [ + { + "id": "203a", + "desc": "clicking a username opens that person's profile with their name and bio", + "points": 1, + "steps": [ + { "do": "click", "actor": "viewer", "testid": "member-item", "contains": "Sana", "settleMs": 1200 }, + { "do": "expect", "actor": "viewer", "testid": "profile-name", "contains": "Sana", "within": 8000 }, + { "do": "expect", "actor": "viewer", "testid": "profile-bio", "contains": "first-bio-text", "within": 8000 } + ] + }, + { + "id": "203b", + "desc": "an open profile panel is live: a bio edit appears to a viewer without reopening", + "points": 2, + "steps": [ + { "do": "click", "actor": "subject", "testid": "profile-close", "ifAvailable": true }, + { "do": "click", "actor": "subject", "testid": "member-item", "contains": "Sana" }, + { "do": "fill", "actor": "subject", "testid": "bio-input", "text": "second-bio-text" }, + { "do": "click", "actor": "subject", "testid": "bio-save" }, + { "do": "expect", "actor": "viewer", "testid": "profile-bio", "contains": "second-bio-text", "within": 10000 } + ] + }, + { + "id": "203c", + "desc": "the panel closes, and reopening shows current data", + "points": 1, + "steps": [ + { "do": "click", "actor": "viewer", "testid": "profile-close", "settleMs": 800 }, + { "do": "waitUntilAbsent", "actor": "viewer", "testid": "profile-bio", "within": 4000 }, + { "do": "click", "actor": "viewer", "testid": "member-item", "contains": "Sana", "settleMs": 1200 }, + { "do": "expect", "actor": "viewer", "testid": "profile-bio", "contains": "second-bio-text", "within": 8000 } + ] + } + ] + }, + { + "id": 204, + "name": "Friends", + "actors": ["asker", "asked", "third"], + "setup": [ + { "do": "signUp", "actor": "asker", "name": "Finn" }, + { "do": "signUp", "actor": "asked", "name": "Freya" }, + { "do": "signUp", "actor": "third", "name": "Falk" }, + { "do": "createRoom", "actor": "asker", "room": "friendroom" }, + { "do": "enterRoom", "actor": "asker", "room": "friendroom" }, + { "do": "enterRoom", "actor": "asked", "room": "friendroom" }, + { "do": "enterRoom", "actor": "third", "room": "friendroom" } + ], + "criteria": [ + { + "id": "204a", + "desc": "a request reaches the recipient as pending, and accepting puts each user in the other's friends list — live, no reloads anywhere", + "points": 2, + "steps": [ + { "do": "click", "actor": "asker", "testid": "member-item", "contains": "Freya", "settleMs": 1200 }, + { "do": "click", "actor": "asker", "testid": "friend-request", "settleMs": 1200 }, + { "do": "click", "actor": "asker", "testid": "profile-close", "ifAvailable": true }, + { "do": "expect", "actor": "asked", "testid": "friend-pending", "contains": "Finn", "within": 10000 }, + { "do": "click", "actor": "asked", "testid": "friend-accept", "in": { "testid": "friend-pending", "contains": "Finn" }, "settleMs": 1500 }, + { "do": "expect", "actor": "asker", "testid": "friend-item", "contains": "Freya", "within": 10000 }, + { "do": "expect", "actor": "asked", "testid": "friend-item", "contains": "Finn", "within": 10000 } + ] + }, + { + "id": "204b", + "desc": "a declined request creates no friendship on either side", + "points": 1, + "steps": [ + { "do": "click", "actor": "third", "testid": "member-item", "contains": "Freya", "settleMs": 1200 }, + { "do": "click", "actor": "third", "testid": "friend-request", "settleMs": 1200 }, + { "do": "click", "actor": "third", "testid": "profile-close", "ifAvailable": true }, + { "do": "click", "actor": "asked", "testid": "friend-decline", "in": { "testid": "friend-pending", "contains": "Falk" }, "settleMs": 1500 }, + { "do": "expect", "actor": "third", "testid": "friend-item", "contains": "Freya", "absent": true, "within": 5000 }, + { "do": "expect", "actor": "asked", "testid": "friend-item", "contains": "Falk", "absent": true, "within": 5000 } + ] + }, + { + "id": "204c", + "desc": "unfriending removes the row live from BOTH lists", + "points": 2, + "steps": [ + { "do": "click", "actor": "asker", "testid": "unfriend", "in": { "testid": "friend-item", "contains": "Freya" }, "settleMs": 1500 }, + { "do": "waitUntilAbsent", "actor": "asker", "testid": "friend-item", "contains": "Freya", "within": 8000 }, + { "do": "waitUntilAbsent", "actor": "asked", "testid": "friend-item", "contains": "Finn", "within": 8000 } + ] + } + ] + }, + { + "id": 205, + "name": "Friend Presence", + "actors": ["watcher", "flapper"], + "setup": [ + { "do": "signUp", "actor": "watcher", "name": "Wava" }, + { "do": "signUp", "actor": "flapper", "name": "Flip" }, + { "do": "createRoom", "actor": "watcher", "room": "presroom" }, + { "do": "enterRoom", "actor": "watcher", "room": "presroom" }, + { "do": "enterRoom", "actor": "flapper", "room": "presroom" }, + { "do": "click", "actor": "watcher", "testid": "member-item", "contains": "Flip", "settleMs": 1200 }, + { "do": "click", "actor": "watcher", "testid": "friend-request", "settleMs": 1200 }, + { "do": "click", "actor": "watcher", "testid": "profile-close", "ifAvailable": true }, + { "do": "click", "actor": "flapper", "testid": "friend-accept", "in": { "testid": "friend-pending", "contains": "Wava" }, "settleMs": 1500 } + ], + "criteria": [ + { + "id": "205a", + "desc": "a friend's presence flips to offline live when they leave, and back when they return", + "points": 2, + "steps": [ + { "do": "expect", "actor": "watcher", "testid": "friend-presence", "in": { "testid": "friend-item", "contains": "Flip" }, "contains": "online", "within": 10000 }, + { "do": "closeClient", "actor": "flapper" }, + { "do": "expect", "actor": "watcher", "testid": "friend-presence", "in": { "testid": "friend-item", "contains": "Flip" }, "contains": "offline", "within": 15000 }, + { "do": "openClient", "actor": "flapper" }, + { "do": "ensureSignedIn", "actor": "flapper", "name": "Flip" }, + { "do": "expect", "actor": "watcher", "testid": "friend-presence", "in": { "testid": "friend-item", "contains": "Flip" }, "contains": "online", "within": 15000 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/chat/scenarios/02-invariants.json b/tools/stack-bench/tracks/chat/scenarios/02-invariants.json new file mode 100644 index 00000000000..b81e04e702d --- /dev/null +++ b/tools/stack-bench/tracks/chat/scenarios/02-invariants.json @@ -0,0 +1,114 @@ +{ + "schemaVersion": 1, + "level": 2, + "note": "What level 2 must hold true regardless of UI: privacy is a property of the wire, not of what a page chooses to render; a friendship is one relationship; a profile belongs to its owner. Three-way verdicts follow level 1's replay conventions.", + "features": [ + { + "id": 211, + "name": "Private Room Privacy", + "actors": ["owner", "member", "outsider"], + "setup": [ + { "do": "signUp", "actor": "owner", "name": "Opal" }, + { "do": "signUp", "actor": "member", "name": "Mia" }, + { "do": "signUp", "actor": "outsider", "name": "Xeno" }, + { "do": "createRoom", "actor": "owner", "room": "sealed", "private": true }, + { "do": "enterRoom", "actor": "owner", "room": "sealed" }, + { "do": "fill", "actor": "owner", "testid": "invite-input", "text": "{user:Mia}" }, + { "do": "click", "actor": "owner", "testid": "invite-submit", "settleMs": 1500 }, + { "do": "enterRoom", "actor": "member", "room": "sealed" } + ], + "criteria": [ + { + "id": "211a", + "desc": "a private message reaches a member and is never sent over the wire to a non-member — regardless of what any page renders", + "points": 2, + "steps": [ + { "do": "send", "actor": "owner", "text": "sealed-room-secret" }, + { "do": "wait", "actor": "owner", "ms": 4000 }, + { "do": "expectReceived", "actor": "member", "contains": "sealed-room-secret", "within": 8000 }, + { "do": "expectNotReceived", "actor": "outsider", "contains": "sealed-room-secret" } + ] + }, + { + "id": "211b", + "desc": "a non-member's post into a private room, issued as a raw request, is rejected and appears to nobody", + "points": 2, + "steps": [ + { "do": "replayAs", "actor": "outsider", "from": "owner", "match": "sealed-room-secret", "swap": { "find": "sealed-room-secret", "with": "smuggled-outsider-post" } }, + { "do": "expectReplayRejected", "actor": "outsider" }, + { "do": "expect", "actor": "member", "testid": "message-item", "contains": "smuggled-outsider-post", "absent": true, "within": 5000 } + ] + } + ] + }, + { + "id": 212, + "name": "Friendship Is One Relationship", + "actors": ["a", "b"], + "setup": [ + { "do": "signUp", "actor": "a", "name": "Ada" }, + { "do": "signUp", "actor": "b", "name": "Ben" }, + { "do": "createRoom", "actor": "a", "room": "symroom" }, + { "do": "enterRoom", "actor": "a", "room": "symroom" }, + { "do": "enterRoom", "actor": "b", "room": "symroom" }, + { "do": "click", "actor": "a", "testid": "member-item", "contains": "Ben", "settleMs": 1200 }, + { "do": "click", "actor": "a", "testid": "friend-request", "settleMs": 1200 }, + { "do": "click", "actor": "a", "testid": "profile-close", "ifAvailable": true }, + { "do": "click", "actor": "b", "testid": "friend-accept", "in": { "testid": "friend-pending", "contains": "Ada" }, "settleMs": 1500 } + ], + "criteria": [ + { + "id": "212a", + "desc": "after an accept, each side holds exactly one row for the other — never two, never a half-friendship", + "points": 1, + "steps": [ + { "do": "expectElementCount", "actor": "a", "testid": "friend-item", "contains": "Ben", "equals": 1, "within": 10000 }, + { "do": "expectElementCount", "actor": "b", "testid": "friend-item", "contains": "Ada", "equals": 1, "within": 10000 } + ] + }, + { + "id": "212b", + "desc": "replaying the accept a second time as a raw request creates no second friendship", + "points": 2, + "steps": [ + { "do": "replayAs", "actor": "b", "from": "b", "match": "friend" }, + { "do": "expectReplayCompleted", "actor": "b" }, + { "do": "wait", "actor": "b", "ms": 2500 }, + { "do": "expectElementCount", "actor": "a", "testid": "friend-item", "contains": "Ben", "equals": 1, "within": 8000 }, + { "do": "expectElementCount", "actor": "b", "testid": "friend-item", "contains": "Ada", "equals": 1, "within": 8000 } + ] + } + ] + }, + { + "id": 213, + "name": "A Profile Belongs to Its Owner", + "actors": ["victim", "attacker"], + "setup": [ + { "do": "signUp", "actor": "victim", "name": "Vik" }, + { "do": "signUp", "actor": "attacker", "name": "Axl" }, + { "do": "createRoom", "actor": "victim", "room": "bioroom" }, + { "do": "enterRoom", "actor": "victim", "room": "bioroom" }, + { "do": "enterRoom", "actor": "attacker", "room": "bioroom" }, + { "do": "click", "actor": "victim", "testid": "profile-close", "ifAvailable": true }, + { "do": "click", "actor": "victim", "testid": "member-item", "contains": "Vik" }, + { "do": "fill", "actor": "victim", "testid": "bio-input", "text": "authentic-bio-text" }, + { "do": "click", "actor": "victim", "testid": "bio-save", "settleMs": 1500 } + ], + "criteria": [ + { + "id": "213a", + "desc": "another user's raw request to edit someone's bio is rejected, and the bio is unchanged for everyone", + "points": 2, + "steps": [ + { "do": "replayAs", "actor": "attacker", "from": "victim", "match": "authentic-bio-text", "swap": { "find": "authentic-bio-text", "with": "defaced-bio-text" } }, + { "do": "expectReplayRejected", "actor": "attacker" }, + { "do": "click", "actor": "attacker", "testid": "member-item", "contains": "Vik", "settleMs": 1200 }, + { "do": "expect", "actor": "attacker", "testid": "profile-bio", "contains": "authentic-bio-text", "within": 8000 }, + { "do": "expect", "actor": "attacker", "testid": "profile-bio", "contains": "defaced-bio-text", "absent": true, "within": 4000 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/chat/scenarios/02-systems.json b/tools/stack-bench/tracks/chat/scenarios/02-systems.json new file mode 100644 index 00000000000..4a09146c594 --- /dev/null +++ b/tools/stack-bench/tracks/chat/scenarios/02-systems.json @@ -0,0 +1,89 @@ +{ + "schemaVersion": 1, + "level": 2, + "note": "The subscription-lifecycle races level 2's surfaces make possible: a panel that opens at the moment its contents change, and a relationship that forms while the list showing it is being fetched. Fetch-on-open architectures pass every sequential test and fail exactly here. Every criterion is withheld at 0 points until it demonstrably fails a real build AND its mutant is CAUGHT — same promotion rule as contention.", + "features": [ + { + "id": 921, + "name": "A panel opened at the moment its contents change", + "actors": ["subject", "viewer"], + "setup": [ + { "do": "signUp", "actor": "subject", "name": "Zed" }, + { "do": "signUp", "actor": "viewer", "name": "Yara" }, + { "do": "createRoom", "actor": "subject", "room": "panelrace" }, + { "do": "enterRoom", "actor": "subject", "room": "panelrace" }, + { "do": "enterRoom", "actor": "viewer", "room": "panelrace" }, + { "do": "click", "actor": "subject", "testid": "profile-close", "ifAvailable": true }, + { "do": "click", "actor": "subject", "testid": "member-item", "contains": "Zed" }, + { "do": "fill", "actor": "subject", "testid": "bio-input", "text": "bio-before-race" }, + { "do": "click", "actor": "subject", "testid": "bio-save", "settleMs": 1500 }, + { "do": "click", "actor": "subject", "testid": "profile-close", "ifAvailable": true }, + { "do": "click", "actor": "subject", "testid": "member-item", "contains": "Zed" }, + { "do": "fill", "actor": "subject", "testid": "bio-input", "text": "bio-after-race" } + ], + "criteria": [ + { + "id": "921a", + "desc": "a profile panel opened at the same moment the bio is saved converges on the NEW bio — the panel's subscription outlives its opening fetch", + "points": 0, + "note": "WITHHELD until proven against a mutant (fetch-on-open-no-subscribe) and a real failing build. A fetch-on-open panel that races the save shows bio-before-race forever; a subscribed panel converges.", + "steps": [ + { + "do": "race", + "branches": [ + [ + { "do": "click", "actor": "viewer", "testid": "member-item", "contains": "Zed" } + ], + [ + { "do": "click", "actor": "subject", "testid": "bio-save" } + ] + ], + "settleMs": 4000 + }, + { "do": "expect", "actor": "viewer", "testid": "profile-bio", "contains": "bio-after-race", "within": 10000 } + ] + } + ] + }, + { + "id": 922, + "name": "A friendship formed while the list showing it is fetched", + "actors": ["asker", "asked"], + "setup": [ + { "do": "signUp", "actor": "asker", "name": "Quin" }, + { "do": "signUp", "actor": "asked", "name": "Rune" }, + { "do": "createRoom", "actor": "asker", "room": "listrace" }, + { "do": "enterRoom", "actor": "asker", "room": "listrace" }, + { "do": "enterRoom", "actor": "asked", "room": "listrace" }, + { "do": "click", "actor": "asker", "testid": "member-item", "contains": "Rune", "settleMs": 1200 }, + { "do": "click", "actor": "asker", "testid": "friend-request", "settleMs": 1500 }, + { "do": "click", "actor": "asker", "testid": "profile-close", "ifAvailable": true } + ], + "criteria": [ + { + "id": "922a", + "desc": "the accept lands while the asker's client is mid-refresh: the new friend appears exactly once — not zero (stale fetch), not twice (fetch merged with the accept broadcast)", + "points": 0, + "note": "WITHHELD until proven against a mutant (fetch-then-append-dupe) and a real failing build. The reload is what forces the asker's client into a full refetch for the accept to race.", + "steps": [ + { + "do": "race", + "branches": [ + [ + { "do": "reload", "actor": "asker", "settleMs": 500 } + ], + [ + { "do": "click", "actor": "asked", "testid": "friend-accept", "in": { "testid": "friend-pending", "contains": "Quin" } } + ] + ], + "settleMs": 5000 + }, + { "do": "ensureSignedIn", "actor": "asker", "name": "Quin" }, + { "do": "expectElementCount", "actor": "asker", "testid": "friend-item", "contains": "Rune", "equals": 1, "within": 12000 }, + { "do": "expectElementCount", "actor": "asked", "testid": "friend-item", "contains": "Quin", "equals": 1, "within": 12000 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/chat/track.json b/tools/stack-bench/tracks/chat/track.json new file mode 100644 index 00000000000..949d983864c --- /dev/null +++ b/tools/stack-bench/tracks/chat/track.json @@ -0,0 +1,51 @@ +{ + "schemaVersion": 1, + "title": "Chat", + "slug": "", + "validatedThrough": 0, + "plannedThrough": 5, + "portOffset": 0, + "restartProbe": "/api/rooms", + "reseedOnReset": true, + "suites": { + "1": [ + { + "id": "features", + "inherit": "none", + "spec": "scenarios/01-basic-chat.json" + }, + { + "id": "invariants", + "inherit": "all-higher-levels", + "spec": "scenarios/01-invariants.json" + }, + { + "id": "delivery", + "inherit": "none", + "spec": "scenarios/01-delivery.json" + }, + { + "id": "systems", + "inherit": "all-higher-levels", + "spec": "scenarios/01-systems.json" + } + ], + "2": [ + { + "id": "features", + "inherit": "none", + "spec": "scenarios/02-features.json" + }, + { + "id": "invariants", + "inherit": "all-higher-levels", + "spec": "scenarios/02-invariants.json" + }, + { + "id": "systems", + "inherit": "all-higher-levels", + "spec": "scenarios/02-systems.json" + } + ] + } +} diff --git a/tools/stack-bench/tracks/chat/walk.ts b/tools/stack-bench/tracks/chat/walk.ts new file mode 100644 index 00000000000..5d9c4d5c0d3 --- /dev/null +++ b/tools/stack-bench/tracks/chat/walk.ts @@ -0,0 +1,85 @@ +import type { LintWalkContext } from '../../linter/lint.js'; + +// Keep this recorded contract walk stable. + +export async function walk({ page, args, hooks, byStage, blocked, checkHook, results, uniq, tid, CHECK_TIMEOUT }: LintWalkContext): Promise { + await page.goto(args.url ?? '', { waitUntil: 'domcontentloaded', timeout: 15000 }); + + // Stage: landing (fresh identity -> registration UI must be shown) + let ok = true; + for (const h of byStage('landing')) ok = (await checkHook(page, h, results)) && ok; + + if (ok) { + // Use alphanumeric credentials because username character rules are not part of the contract. + await page.locator(tid('signup-username')).first().fill(`lint${uniq}`); + await page.locator(tid('signup-password')).first().fill(`pwlint${uniq}`); + await page.locator(tid('signup-submit')).first().click(); + } + + // Stage: main (registered -> rooms + online users) + if (ok) { + for (const h of byStage('main')) ok = (await checkHook(page, h, results)) && ok; + } else blocked('main'); + + // Create a room, then check hooks that need one to exist. The created room + // must arrive in the list via the app's live update path (no reload) — this + // is itself a contract requirement, so we wait for OUR room, not any room. + const ourRoom = page.locator(tid('room-item'), { hasText: `lint-room-${uniq}` }).first(); + if (ok) { + const nameInput = page.locator(tid('room-name-input')).first(); + if (!(await nameInput.isVisible().catch(() => false))) { + await page.locator(tid('room-create')).first().click(); + } + await nameInput.fill(`lint-room-${uniq}`); + await page.locator(tid('room-name-submit')).first().click(); + for (const h of byStage('main-after-create')) { + if (h.id === 'room-item') { + try { + await ourRoom.waitFor({ state: 'visible', timeout: CHECK_TIMEOUT }); + results.push({ id: h.id, status: 'PASS' }); + } catch { + results.push({ + id: h.id, status: 'FAIL', + detail: `created room "lint-room-${uniq}" but no ${tid(h.id)} containing it appeared without a reload`, + }); + ok = false; + } + } else ok = (await checkHook(page, h, results)) && ok; + } + } else blocked('main-after-create'); + + // Stage: room (enter our room, send a probe message via Enter). + // Apps may implement click-to-join then click-to-enter; allow a second click. + if (ok) { + await ourRoom.click(); + const msgInput = page.locator(tid('message-input')).first(); + if (!(await msgInput.isVisible().catch(() => false))) { + await page.waitForTimeout(750); + if (!(await msgInput.isVisible().catch(() => false))) await ourRoom.click(); + } + for (const h of byStage('room')) await checkHook(page, h, results); // non-blocking: check all + ok = !results.some(r => r.status === 'FAIL' && hooks.find(h => h.id === r.id)?.stage === 'room'); + } else blocked('room'); + + if (ok && !results.some(r => r.id === 'message-input' && r.status !== 'PASS')) { + const probe = `lint probe ${uniq}`; + await page.locator(tid('message-input')).first().fill(probe); + await page.locator(tid('message-input')).first().press('Enter'); + for (const h of byStage('room-after-send')) { + const loc = page.locator(tid(h.id), { hasText: probe }).first(); + try { + await loc.waitFor({ state: 'visible', timeout: CHECK_TIMEOUT }); + results.push({ id: h.id, status: 'PASS' }); + } catch { + results.push({ + id: h.id, status: 'FAIL', + detail: `sent "${probe}" via Enter but no ${tid(h.id)} containing it appeared — expected: ${h.element}`, + }); + } + } + } else blocked('room-after-send'); + + for (const h of byStage('scenario')) { + results.push({ id: h.id, status: 'SCENARIO', detail: h.note }); + } +} diff --git a/tools/stack-bench/tracks/ecommerce/LEVELS.md b/tools/stack-bench/tracks/ecommerce/LEVELS.md new file mode 100644 index 00000000000..075f47db7a3 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/LEVELS.md @@ -0,0 +1,35 @@ +# Ecommerce track + +The ecommerce track is an online store with a warehouse: shared stock, customer +accounts and orders, staff operations, and scheduled work. It tests shared +numeric state, per-account state, access control, concurrency, and deferred +work. What the checks observe is in [grading](../../docs/grading.md); packs, +recipes, and calibrations are in [composition](composition/README.md). + +## Dependency mode + +The default mode follows the feature graph in `progression/ecommerce.json` +(see **feature** in the [terms](../../docs/system-design.md#terms)). A +feature's depth is how far down the graph it sits, and a campaign targets a +depth: + +- **Depth 1**, 4 features that build on nothing: accounts, catalog items, + staff access, support intake. +- **Depth 2**, 10 features that build on depth 1: catalog discovery, customer + profile, purchasing and orders, cart, warehouse administration, staff roles, + support triage, support history, promotion rules, notification preferences. +- **Depth 3**, 13 features that build on depth 2: checkout, reviews, + fulfilment queue, stock transfers, order cancellation, inventory dashboard, + sales dashboard, recommendations, scheduled restocks, catalog management, + faceted search, managed support cases, stock alerts. + +The graph continues to depth 6 with price history, reservations, delivery, +payments, promotions, bundles, subscriptions, store credit, returns, and +split-tender refunds. Convex and Supabase have reference apps for depths 1 to +3; SpacetimeDB, PostgreSQL, and MongoDB through depth 6. + +## Sequential mode + +Sequential mode runs cumulative levels, L1 to L3, and a level must pass before +the next starts. The recipes are `composition/recipes/sequential-l1.json` to +`sequential-l3.json`. diff --git a/tools/stack-bench/tracks/ecommerce/composition/README.md b/tools/stack-bench/tracks/ecommerce/composition/README.md new file mode 100644 index 00000000000..77d5a2dc6b4 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/README.md @@ -0,0 +1,45 @@ +# Ecommerce composition + +This directory defines the ecommerce work StackBench can request and grade. + +- `packs/`: selectable features and specifications. A pack names the packs it + depends on by stable ID. +- `fixtures/`: exact starting data. +- `recipes/`: which packs, fixture, prompts, execution, and scoring a campaign + gets. +- `sequential.json` and `dependency.json`: mode aliases to recipes. +- `calibrations/`: a recipe bound to its reference apps, seeded bugs, empty-app + controls, repetition policy, and qualification evidence. + +These files are the source of truth. Do not copy versions or qualification +status into documentation. + +Selecting a pack changes the requested work; its dependencies join both the +prompt and the grading scope. Selecting checks narrows measurement; the request +is unchanged. A specification's treatment is explained in +[prompting](../../../docs/prompting.md). + +## Identity + +Recipes are normalized JSON with sorted keys; arrays keep their order where +order matters. `meaningSha256` covers task text, contracts, checks, roles, and +points. `executionSha256` covers fixtures, actions, timing, capabilities, and +budgets. `contentSha256` binds both, and `sourceManifestSha256` binds the raw +source bytes. Version labels are for people; hashes identify content. + +## Commands + +From `tools/stack-bench`: + +```bash +npm run check:composition +npm run check:calibration +npm run pack -- validate --track ecommerce +npm run recipe -- validate --track ecommerce +npm run recipe -- show --track ecommerce +npm run recipe -- diff --track ecommerce +``` + +`recipe show --pack ` or `--check ` inspects one selected scope and its +hash. Qualification status comes from `qualification status`; see the +[qualify-checks skill](../../../skills/qualify-checks/SKILL.md). diff --git a/tools/stack-bench/tracks/ecommerce/composition/calibrations/dependency-l3.json b/tools/stack-bench/tracks/ecommerce/composition/calibrations/dependency-l3.json new file mode 100644 index 00000000000..1b0eabb321a --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/calibrations/dependency-l3.json @@ -0,0 +1,236 @@ +{ + "schemaVersion": 2, + "kind": "calibration-manifest", + "id": "ecommerce.dependency-l3-calibration", + "title": "Ecommerce dependency L3 calibration", + "track": "ecommerce", + "recipe": { + "path": "../recipes/progression-catalog.json", + "id": "ecommerce.progression-catalog", + "meaningSha256": "965ba5bd4b59ed814df7dda9cf563821d3e623cccf0da9d7e7270921aa79d5e5", + "executionSha256": "a5299e64aca9cbe0a7dece622aa02f710fafc0bbfe3362daf3c9edfac801ae22", + "contentSha256": "aa2ad046d068db85f5c3e878713832dd5d51fe1a42c0e8189e5ffe123c4beb9b" + }, + "fixture": { + "id": "ecommerce.operations", + "sourceSha256": "d06444b72dc94fe1ef5e08867d875e1f3bbaa5cd82c35a3558f399c1fcb5ceae" + }, + "references": { + "registryPath": "reference-apps/registry.json", + "entries": [ + { + "backend": "mongodb", + "id": "ecommerce-reference-mongodb", + "sourceSha256": "8a167a23e422534dcf58d664edcd0506ec454c5880b54e170da391c600dfe6c2" + }, + { + "backend": "postgres", + "id": "ecommerce-reference-postgres", + "sourceSha256": "b54d02f7b19b4f2e59349c776780fbec6560575a998f2c8592cf4cda6d24acac" + }, + { + "backend": "spacetime", + "id": "ecommerce-reference-spacetime", + "sourceSha256": "218a8e6298b8becef513b68d4b9a5068b84bde9c14789e0ae45ebc0d6ce703aa" + }, + { + "backend": "convex", + "id": "ecommerce-reference-convex", + "sourceSha256": "8c647a96fbab91b8f51c1d7b6831f926a99d8e681b145b66add7ae52ce24e6c6" + }, + { + "backend": "supabase", + "id": "ecommerce-reference-supabase", + "sourceSha256": "0f6a67cbb1922d02cee1750b2b802431fc6cc05e055748dd6885e9545c12eeef" + } + ] + }, + "mutations": [ + { + "backend": "mongodb", + "path": "grader/mutations/mongodb-ecommerce.json", + "sha256": "c3702c52edcf1ff904dbb5e6f93441d3ba036c2c408a067d0a284cf8e811e9cf", + "referenceId": "ecommerce-reference-mongodb" + }, + { + "backend": "postgres", + "path": "grader/mutations/postgres-ecommerce.json", + "sha256": "a15d56e6cc9c08630a27eadb3a297df182e5a925235a802628eb5b0efdf9da85", + "referenceId": "ecommerce-reference-postgres" + }, + { + "backend": "spacetime", + "path": "grader/mutations/spacetime-ecommerce.json", + "sha256": "9df40afd886890d7ab1d763940b5dbbdb5e20265194922fa0c7b05fa35bde4c3", + "referenceId": "ecommerce-reference-spacetime" + }, + { + "backend": "convex", + "path": "grader/mutations/convex-ecommerce.json", + "sha256": "4a2db5b58dd9b00e322cf565f19fb58b66283838fcdaeec74c6ab2a5b8c68232", + "referenceId": "ecommerce-reference-convex" + }, + { + "backend": "supabase", + "path": "grader/mutations/supabase-ecommerce.json", + "sha256": "8419868903cb7daaaf9a854e75af4ba6bbf05a8ee402e257a1f918122090b3f9", + "referenceId": "ecommerce-reference-supabase" + } + ], + "nullControl": { + "pointBearing": "must-fail-conclusively", + "zeroPoint": "typed-policy", + "repetitions": 1 + }, + "controls": [], + "qualification": { + "exactCombinationRequired": true, + "referenceRepetitions": 1, + "mutationRepetitions": 1, + "checks": [ + "ecommerce.feature.accounts.accounts.1a", + "ecommerce.feature.accounts.accounts.1b", + "ecommerce.spec.access-control.account-password.1c", + "ecommerce.feature.accounts.accounts.1d", + "ecommerce.feature.cart-checkout.cart.4a", + "ecommerce.feature.cart-checkout.cart.4d", + "ecommerce.feature.catalog.catalog-ranking.2b", + "ecommerce.feature.catalog.catalog-search.2d", + "ecommerce.feature.catalog.catalog-values.2a", + "ecommerce.feature.purchasing.purchase-order.3c", + "ecommerce.feature.reviews.reviews.6a", + "ecommerce.feature.warehouse-admin.admin-write.103a", + "ecommerce.feature.warehouse-admin.warehouse-view.7b", + "ecommerce.inventory-operations.operational-views.5c", + "ecommerce.inventory-operations.operational-views.5d", + "ecommerce.inventory-operations.operational-views.5e", + "ecommerce.inventory-operations.operational-views.5f", + "ecommerce.inventory-operations.shipping-accounting.202e", + "ecommerce.inventory-operations.stock-conservation.202b", + "ecommerce.inventory-operations.stock-conservation.202d", + "ecommerce.inventory-operations.stock-conservation.202f", + "ecommerce.inventory-operations.warehouse-transfer.2a", + "ecommerce.l3.deferred-access.scheduled-work-access.317a", + "ecommerce.l3.deferred-integrity.exactly-once.311a", + "ecommerce.l3.scheduled-restocks.scheduled-restocks.302a", + "ecommerce.l3.scheduled-restocks.scheduled-restocks.305a", + "ecommerce.l3.scheduled-restocks.scheduled-restocks.306a", + "ecommerce.l3.server-time.server-time.312a", + "ecommerce.operations-access.fulfilment-queue.1b", + "ecommerce.operations-access.fulfilment-queue.1c", + "ecommerce.operations-access.operator-authorization.201a", + "ecommerce.operations-access.operator-authorization.201c", + "ecommerce.operations-access.order-owner.204a", + "ecommerce.progression.catalog-management.catalog-management.622a", + "ecommerce.progression.catalog-management.catalog-management.622b", + "ecommerce.progression.customer-profile.customer-profile.620c", + "ecommerce.progression.faceted-search.faceted-search.401a", + "ecommerce.progression.faceted-search.faceted-search.402a", + "ecommerce.progression.managed-support.managed-support.613c", + "ecommerce.progression.notification-preferences.notification-preferences.630c", + "ecommerce.progression.promotion-rules.promotion-rule-values.620a", + "ecommerce.progression.review-access-specifications.review-eligibility-direct.618a", + "ecommerce.progression.review-access-specifications.stored-review-script.9180a", + "ecommerce.progression.staff-access.staff-access.601a", + "ecommerce.progression.staff-roles.staff-roles.621c", + "ecommerce.progression.stock-alerts.stock-alert-delivery.631c", + "ecommerce.progression.support-history.support-history.612c", + "ecommerce.progression.support-intake.support-intake.610a", + "ecommerce.progression.support-triage.support-assignment.611a", + "ecommerce.progression.support-triage.support-priority.611b", + "ecommerce.progression.support-triage.support-status.611c", + "ecommerce.returns-pricing.cancellation-and-return.3a", + "ecommerce.returns-pricing.cancellation-and-return.3b", + "ecommerce.returns-pricing.refund-accounting.203a", + "ecommerce.spec.access-control.account-disclosure.110a", + "ecommerce.spec.access-control.cart-boundary.109a", + "ecommerce.spec.access-control.cart-boundary.109b", + "ecommerce.spec.access-control.customer-profile-privacy.620b", + "ecommerce.spec.access-control.fulfilment-area-boundary.1d", + "ecommerce.spec.access-control.managed-support-privacy.613b", + "ecommerce.spec.access-control.notification-preferences-privacy.630b", + "ecommerce.spec.access-control.order-ownership.106a", + "ecommerce.spec.access-control.promotion-management-boundary.620b", + "ecommerce.spec.access-control.purchase-attribution.102a", + "ecommerce.spec.access-control.purchase-session.101a", + "ecommerce.spec.access-control.review-eligibility.108a", + "ecommerce.spec.access-control.review-eligibility.108b", + "ecommerce.spec.access-control.signed-out-purchase.3a", + "ecommerce.spec.access-control.staff-area-boundary.601b", + "ecommerce.spec.access-control.staff-role-boundary.621b", + "ecommerce.spec.access-control.staff-role-revocation.621d", + "ecommerce.spec.access-control.stock-alert-privacy.631b", + "ecommerce.spec.access-control.support-history-logout.612d", + "ecommerce.spec.access-control.support-history-privacy.612b", + "ecommerce.spec.access-control.warehouse-area-boundary.7a", + "ecommerce.spec.access-control.warehouse-write-boundary.103b", + "ecommerce.spec.concurrency-safety.duplicate-checkout.203a", + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b", + "ecommerce.spec.concurrency-safety.last-unit.201a", + "ecommerce.spec.concurrency-safety.last-unit.201b", + "ecommerce.spec.concurrency-safety.last-unit.201c", + "ecommerce.spec.concurrency-safety.restock-race.202a", + "ecommerce.spec.external-data-sync.external-stock-alert.631d", + "ecommerce.spec.external-data-sync.external-stock.901a", + "ecommerce.spec.external-data-sync.external-stock.901d", + "ecommerce.spec.live-state.fulfilment-queue.1a", + "ecommerce.spec.live-state.inventory-dashboard.5a", + "ecommerce.spec.live-state.managed-support.613a", + "ecommerce.spec.live-state.open-list.902a", + "ecommerce.spec.live-state.purchase-stock.3b", + "ecommerce.spec.live-state.ranking.2c", + "ecommerce.spec.live-state.rating.6c", + "ecommerce.spec.live-state.sales-dashboard.5b", + "ecommerce.spec.live-state.shared-cart.4c", + "ecommerce.spec.live-state.warehouse-stock.7c", + "ecommerce.spec.search-ordering.search-ordering.402b", + "ecommerce.spec.state-durability.account-state-recovery.105a", + "ecommerce.spec.state-durability.account-state-recovery.105b", + "ecommerce.spec.state-durability.cart-reload.4b", + "ecommerce.spec.state-durability.checkout-crash-durability.910b", + "ecommerce.spec.state-durability.checkout-crash-integrity.910a", + "ecommerce.spec.state-durability.customer-profile-reload.620a", + "ecommerce.spec.state-durability.notification-preferences-reload.630a", + "ecommerce.spec.state-durability.session-reload.1e", + "ecommerce.spec.state-durability.staff-role-reload.621a", + "ecommerce.spec.state-durability.support-history-reload.612a", + "ecommerce.spec.transactional-integrity.books-balance.107a", + "ecommerce.spec.transactional-integrity.books-balance.107b", + "ecommerce.spec.transactional-integrity.catalog-volume.622c", + "ecommerce.spec.transactional-integrity.server-price.104a", + "ecommerce.spec.transactional-integrity.stock-alert-deduplication.631a", + "ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c", + "ecommerce.spec.transactional-integrity.unique-review.6b" + ], + "runner": { + "schemaVersion": 1, + "mode": "appliance", + "platform": "linux", + "architecture": "x64" + }, + "stacks": [ + "mongodb", + "postgres", + "spacetime", + "convex", + "supabase" + ], + "evidence": [], + "featureCatalog": { + "path": "progression/ecommerce.json", + "contentSha256": "0620dd76657d2ab081d0a991b8bc6ffe178d7b2c6afbf569bb910736d02ca2fe", + "id": "ecommerce.questlines" + } + }, + "equivalenceDecisions": [], + "selection": { + "path": "composition/dependency.json", + "sha256": "e73002df8a7a42e612f2bd1ee5de1f9db2d6c13cb9c819e405a5f2512eca3048", + "alias": "L3", + "coveredAliases": [ + "L1", + "L2", + "L3" + ] + } +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/calibrations/dependency-l6.json b/tools/stack-bench/tracks/ecommerce/composition/calibrations/dependency-l6.json new file mode 100644 index 00000000000..778ff106ea5 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/calibrations/dependency-l6.json @@ -0,0 +1,284 @@ +{ + "schemaVersion": 2, + "kind": "calibration-manifest", + "id": "ecommerce.dependency-l6-calibration", + "title": "Ecommerce dependency L6 calibration", + "track": "ecommerce", + "recipe": { + "path": "../recipes/progression-catalog.json", + "id": "ecommerce.progression-catalog", + "meaningSha256": "965ba5bd4b59ed814df7dda9cf563821d3e623cccf0da9d7e7270921aa79d5e5", + "executionSha256": "a5299e64aca9cbe0a7dece622aa02f710fafc0bbfe3362daf3c9edfac801ae22", + "contentSha256": "aa2ad046d068db85f5c3e878713832dd5d51fe1a42c0e8189e5ffe123c4beb9b" + }, + "fixture": { + "id": "ecommerce.operations", + "sourceSha256": "d06444b72dc94fe1ef5e08867d875e1f3bbaa5cd82c35a3558f399c1fcb5ceae" + }, + "references": { + "registryPath": "reference-apps/registry.json", + "entries": [ + { + "backend": "mongodb", + "id": "ecommerce-reference-mongodb", + "sourceSha256": "8a167a23e422534dcf58d664edcd0506ec454c5880b54e170da391c600dfe6c2" + }, + { + "backend": "postgres", + "id": "ecommerce-reference-postgres", + "sourceSha256": "b54d02f7b19b4f2e59349c776780fbec6560575a998f2c8592cf4cda6d24acac" + }, + { + "backend": "spacetime", + "id": "ecommerce-reference-spacetime", + "sourceSha256": "218a8e6298b8becef513b68d4b9a5068b84bde9c14789e0ae45ebc0d6ce703aa" + } + ] + }, + "mutations": [ + { + "backend": "mongodb", + "path": "grader/mutations/mongodb-ecommerce.json", + "sha256": "c3702c52edcf1ff904dbb5e6f93441d3ba036c2c408a067d0a284cf8e811e9cf", + "referenceId": "ecommerce-reference-mongodb" + }, + { + "backend": "postgres", + "path": "grader/mutations/postgres-ecommerce.json", + "sha256": "a15d56e6cc9c08630a27eadb3a297df182e5a925235a802628eb5b0efdf9da85", + "referenceId": "ecommerce-reference-postgres" + }, + { + "backend": "spacetime", + "path": "grader/mutations/spacetime-ecommerce.json", + "sha256": "9df40afd886890d7ab1d763940b5dbbdb5e20265194922fa0c7b05fa35bde4c3", + "referenceId": "ecommerce-reference-spacetime" + } + ], + "nullControl": { + "pointBearing": "must-fail-conclusively", + "zeroPoint": "typed-policy", + "repetitions": 1 + }, + "controls": [], + "qualification": { + "exactCombinationRequired": true, + "referenceRepetitions": 1, + "mutationRepetitions": 1, + "checks": [ + "ecommerce.feature.accounts.accounts.1a", + "ecommerce.feature.accounts.accounts.1b", + "ecommerce.feature.accounts.accounts.1d", + "ecommerce.feature.bundle-checkout.bundle-checkout.741a", + "ecommerce.feature.bundle-returns.bundle-returns.742a", + "ecommerce.feature.cart-checkout.cart.4a", + "ecommerce.feature.cart-checkout.cart.4d", + "ecommerce.feature.catalog.catalog-ranking.2b", + "ecommerce.feature.catalog.catalog-search.2d", + "ecommerce.feature.catalog.catalog-values.2a", + "ecommerce.feature.product-bundles.product-bundles.740a", + "ecommerce.feature.purchasing.purchase-order.3c", + "ecommerce.feature.reviews.reviews.6a", + "ecommerce.feature.split-tender-refunds.return-refund-interaction.757a", + "ecommerce.feature.split-tender-refunds.return-refund-interaction.757b", + "ecommerce.feature.split-tender-refunds.split-tender-refunds-751.751a", + "ecommerce.feature.store-credit.store-credit-750.750a", + "ecommerce.feature.subscriptions.subscriptions-760.760a", + "ecommerce.feature.warehouse-admin.admin-write.103a", + "ecommerce.feature.warehouse-admin.warehouse-view.7b", + "ecommerce.inventory-operations.operational-views.5c", + "ecommerce.inventory-operations.operational-views.5d", + "ecommerce.inventory-operations.operational-views.5e", + "ecommerce.inventory-operations.operational-views.5f", + "ecommerce.inventory-operations.shipping-accounting.202e", + "ecommerce.inventory-operations.stock-conservation.202b", + "ecommerce.inventory-operations.stock-conservation.202d", + "ecommerce.inventory-operations.stock-conservation.202f", + "ecommerce.inventory-operations.warehouse-transfer.2a", + "ecommerce.l3.cart-expiration.cart-expiration.304a", + "ecommerce.l3.deferred-access.scheduled-work-access.317a", + "ecommerce.l3.deferred-durability.restart-survival.314a", + "ecommerce.l3.deferred-durability.restart-survival.315a", + "ecommerce.l3.deferred-durability.restart-survival.316a", + "ecommerce.l3.deferred-integrity.exactly-once.311a", + "ecommerce.l3.deferred-integrity.exactly-once.312a", + "ecommerce.l3.deferred-integrity.stock-conservation.313a", + "ecommerce.l3.deferred-integrity.stock-conservation.314a", + "ecommerce.l3.order-delivery.order-delivery.303a", + "ecommerce.l3.order-delivery.order-delivery.305a", + "ecommerce.l3.reservations.reservations.301a", + "ecommerce.l3.reservations.reservations.305a", + "ecommerce.l3.reservations.reservations.306a", + "ecommerce.l3.reservations.reservations.307a", + "ecommerce.l3.reservations.reservations.308a", + "ecommerce.l3.scheduled-restocks.scheduled-restocks.302a", + "ecommerce.l3.scheduled-restocks.scheduled-restocks.305a", + "ecommerce.l3.scheduled-restocks.scheduled-restocks.306a", + "ecommerce.l3.server-time.server-time.312a", + "ecommerce.l3.server-time.server-time.313a", + "ecommerce.operations-access.fulfilment-queue.1b", + "ecommerce.operations-access.fulfilment-queue.1c", + "ecommerce.operations-access.operator-authorization.201a", + "ecommerce.operations-access.operator-authorization.201b", + "ecommerce.operations-access.operator-authorization.201c", + "ecommerce.operations-access.order-owner.204a", + "ecommerce.progression.automatic-reorder.automatic-reorder.502a", + "ecommerce.progression.cart-recovery.cart-recovery.503a", + "ecommerce.progression.cart-recovery.cart-recovery.503b", + "ecommerce.progression.catalog-management.catalog-management.622a", + "ecommerce.progression.catalog-management.catalog-management.622b", + "ecommerce.progression.customer-profile.customer-profile.620c", + "ecommerce.progression.delivery-notifications.delivery-notification-delivery.501a", + "ecommerce.progression.faceted-search.faceted-search.401a", + "ecommerce.progression.faceted-search.faceted-search.402a", + "ecommerce.progression.managed-support.managed-support.613c", + "ecommerce.progression.notification-preferences.notification-preferences.630c", + "ecommerce.progression.order-support.order-support-owned.614a", + "ecommerce.progression.payment-records.payment-records.623a", + "ecommerce.progression.personalized-recommendations.personalized-recommendations.403a", + "ecommerce.progression.promotion-checkout.promotion-checkout-active.621a", + "ecommerce.progression.promotion-checkout.promotion-checkout-exhausted.621c", + "ecommerce.progression.promotion-checkout.promotion-checkout-expired.621b", + "ecommerce.progression.promotion-reporting.promotion-report-redemptions.622a", + "ecommerce.progression.promotion-reporting.promotion-report-revenue.622b", + "ecommerce.progression.promotion-rules.promotion-rule-values.620a", + "ecommerce.progression.recommendation-feedback.recommendation-feedback.504a", + "ecommerce.progression.review-access-specifications.review-eligibility-direct.618a", + "ecommerce.progression.review-access-specifications.stored-review-script.9180a", + "ecommerce.progression.staff-access.staff-access.601a", + "ecommerce.progression.staff-activity.staff-activity.624a", + "ecommerce.progression.staff-roles.staff-roles.621c", + "ecommerce.progression.stock-alerts.stock-alert-delivery.631c", + "ecommerce.progression.support-history.support-history.612c", + "ecommerce.progression.support-intake.support-intake.610a", + "ecommerce.progression.support-refunds.support-refunds-resolution.615a", + "ecommerce.progression.support-triage.support-assignment.611a", + "ecommerce.progression.support-triage.support-priority.611b", + "ecommerce.progression.support-triage.support-status.611c", + "ecommerce.returns-pricing.cancellation-and-return.3a", + "ecommerce.returns-pricing.cancellation-and-return.3b", + "ecommerce.returns-pricing.cancellation-and-return.3c", + "ecommerce.returns-pricing.cancellation-and-return.3d", + "ecommerce.returns-pricing.cancellation-and-return.3e", + "ecommerce.returns-pricing.cancellation-and-return.3f", + "ecommerce.returns-pricing.price-history.4a", + "ecommerce.returns-pricing.price-history.4b", + "ecommerce.returns-pricing.price-history.4c", + "ecommerce.returns-pricing.refund-accounting.203a", + "ecommerce.returns-pricing.refund-accounting.203b", + "ecommerce.spec.access-control.account-disclosure.110a", + "ecommerce.spec.access-control.account-password.1c", + "ecommerce.spec.access-control.automatic-reorder-access.502c", + "ecommerce.spec.access-control.cart-boundary.109a", + "ecommerce.spec.access-control.cart-boundary.109b", + "ecommerce.spec.access-control.customer-profile-privacy.620b", + "ecommerce.spec.access-control.delivery-notification-privacy.501b", + "ecommerce.spec.access-control.fulfilment-area-boundary.1d", + "ecommerce.spec.access-control.managed-support-privacy.613b", + "ecommerce.spec.access-control.notification-preferences-privacy.630b", + "ecommerce.spec.access-control.order-ownership.106a", + "ecommerce.spec.access-control.order-support-ownership.614b", + "ecommerce.spec.access-control.promotion-management-boundary.620b", + "ecommerce.spec.access-control.purchase-attribution.102a", + "ecommerce.spec.access-control.purchase-session.101a", + "ecommerce.spec.access-control.recommendation-feedback-privacy.504b", + "ecommerce.spec.access-control.recommendation-profile-isolation.403b", + "ecommerce.spec.access-control.review-eligibility.108a", + "ecommerce.spec.access-control.review-eligibility.108b", + "ecommerce.spec.access-control.signed-out-purchase.3a", + "ecommerce.spec.access-control.staff-activity-privacy.624b", + "ecommerce.spec.access-control.staff-area-boundary.601b", + "ecommerce.spec.access-control.staff-role-boundary.621b", + "ecommerce.spec.access-control.staff-role-revocation.621d", + "ecommerce.spec.access-control.stock-alert-privacy.631b", + "ecommerce.spec.access-control.support-history-logout.612d", + "ecommerce.spec.access-control.support-history-privacy.612b", + "ecommerce.spec.access-control.support-refund-access.615c", + "ecommerce.spec.access-control.warehouse-area-boundary.7a", + "ecommerce.spec.access-control.warehouse-write-boundary.103b", + "ecommerce.spec.bundle-integrity.bundle-742.742b", + "ecommerce.spec.bundle-integrity.bundle-743.743a", + "ecommerce.spec.bundle-integrity.bundle-744.744a", + "ecommerce.spec.bundle-integrity.bundle-745.745a", + "ecommerce.spec.bundle-integrity.bundle-746.746a", + "ecommerce.spec.bundle-integrity.bundle-747.747a", + "ecommerce.spec.bundle-integrity.bundle-748.748a", + "ecommerce.spec.concurrency-safety.duplicate-checkout.203a", + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b", + "ecommerce.spec.concurrency-safety.last-unit.201a", + "ecommerce.spec.concurrency-safety.last-unit.201b", + "ecommerce.spec.concurrency-safety.last-unit.201c", + "ecommerce.spec.concurrency-safety.restock-race.202a", + "ecommerce.spec.concurrency-safety.stock-limit.3d", + "ecommerce.spec.external-data-sync.external-stock-alert.631d", + "ecommerce.spec.external-data-sync.external-stock.901a", + "ecommerce.spec.external-data-sync.external-stock.901d", + "ecommerce.spec.live-state.fulfilment-queue.1a", + "ecommerce.spec.live-state.inventory-dashboard.5a", + "ecommerce.spec.live-state.managed-support.613a", + "ecommerce.spec.live-state.open-list.902a", + "ecommerce.spec.live-state.purchase-stock.3b", + "ecommerce.spec.live-state.ranking.2c", + "ecommerce.spec.live-state.rating.6c", + "ecommerce.spec.live-state.sales-dashboard.5b", + "ecommerce.spec.live-state.shared-cart.4c", + "ecommerce.spec.live-state.warehouse-stock.7c", + "ecommerce.spec.search-ordering.search-ordering.402b", + "ecommerce.spec.split-tender-refunds.production-756.756a", + "ecommerce.spec.state-durability.account-state-recovery.105a", + "ecommerce.spec.state-durability.account-state-recovery.105b", + "ecommerce.spec.state-durability.cart-reload.4b", + "ecommerce.spec.state-durability.checkout-crash-durability.910b", + "ecommerce.spec.state-durability.checkout-crash-integrity.910a", + "ecommerce.spec.state-durability.customer-profile-reload.620a", + "ecommerce.spec.state-durability.notification-preferences-reload.630a", + "ecommerce.spec.state-durability.recommendation-feedback-restart.504c", + "ecommerce.spec.state-durability.session-reload.1e", + "ecommerce.spec.state-durability.staff-role-reload.621a", + "ecommerce.spec.state-durability.support-history-reload.612a", + "ecommerce.spec.store-credit.production-752.752a", + "ecommerce.spec.store-credit.production-753.753a", + "ecommerce.spec.store-credit.production-754.754a", + "ecommerce.spec.store-credit.production-755.755a", + "ecommerce.spec.subscriptions.production-761.761a", + "ecommerce.spec.subscriptions.production-762.762a", + "ecommerce.spec.subscriptions.production-763.763a", + "ecommerce.spec.transactional-integrity.automatic-reorder-deduplication.502b", + "ecommerce.spec.transactional-integrity.books-balance.107a", + "ecommerce.spec.transactional-integrity.books-balance.107b", + "ecommerce.spec.transactional-integrity.catalog-volume.622c", + "ecommerce.spec.transactional-integrity.payment-deduplication.623b", + "ecommerce.spec.transactional-integrity.server-price.104a", + "ecommerce.spec.transactional-integrity.stock-alert-deduplication.631a", + "ecommerce.spec.transactional-integrity.stock-transfer-overdraw.2c", + "ecommerce.spec.transactional-integrity.support-refund-accounting.615b", + "ecommerce.spec.transactional-integrity.unique-review.6b" + ], + "runner": { + "schemaVersion": 1, + "mode": "appliance", + "platform": "linux", + "architecture": "x64" + }, + "stacks": [ + "mongodb", + "postgres", + "spacetime" + ], + "evidence": [], + "featureCatalog": { + "path": "progression/ecommerce.json", + "contentSha256": "61620196cde2a4c03e8a65f57d8a6a578e18f9cb23b13e24b20b77e753505d94", + "id": "ecommerce.questlines" + } + }, + "equivalenceDecisions": [], + "selection": { + "path": "composition/dependency.json", + "sha256": "e73002df8a7a42e612f2bd1ee5de1f9db2d6c13cb9c819e405a5f2512eca3048", + "alias": "L6", + "coveredAliases": [ + "L6" + ] + } +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/calibrations/sequential-l1.json b/tools/stack-bench/tracks/ecommerce/composition/calibrations/sequential-l1.json new file mode 100644 index 00000000000..01c6d5e69fc --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/calibrations/sequential-l1.json @@ -0,0 +1,151 @@ +{ + "schemaVersion": 2, + "kind": "calibration-manifest", + "id": "ecommerce.sequential-l1-calibration", + "title": "Ecommerce sequential L1 calibration", + "track": "ecommerce", + "recipe": { + "path": "../recipes/sequential-l1.json", + "id": "ecommerce.sequential-l1", + "meaningSha256": "97f890c3c95034df4b52746b104b95ef68924366148f5a5d6e435585ea44e9c1", + "executionSha256": "64f9cf7834e06a6c4fda5a406f56ca0433f491f93ef6c00137077db4eff54a9a", + "contentSha256": "ea3ba0bc5423c90bd53012234873a37f38f0e1c91b91b70a5d9a04eab84b3e47" + }, + "fixture": { + "id": "ecommerce.storefront", + "sourceSha256": "5b320fb646efc8c7efc4f8c643130f8e41695ea4c4b2217b18963044e666b80b" + }, + "references": { + "registryPath": "reference-apps/registry.json", + "entries": [ + { + "backend": "mongodb", + "id": "ecommerce-reference-mongodb", + "sourceSha256": "8a167a23e422534dcf58d664edcd0506ec454c5880b54e170da391c600dfe6c2" + }, + { + "backend": "postgres", + "id": "ecommerce-reference-postgres", + "sourceSha256": "b54d02f7b19b4f2e59349c776780fbec6560575a998f2c8592cf4cda6d24acac" + }, + { + "backend": "spacetime", + "id": "ecommerce-reference-spacetime", + "sourceSha256": "218a8e6298b8becef513b68d4b9a5068b84bde9c14789e0ae45ebc0d6ce703aa" + } + ] + }, + "mutations": [ + { + "backend": "mongodb", + "path": "grader/mutations/mongodb-ecommerce.json", + "sha256": "c3702c52edcf1ff904dbb5e6f93441d3ba036c2c408a067d0a284cf8e811e9cf", + "referenceId": "ecommerce-reference-mongodb" + }, + { + "backend": "postgres", + "path": "grader/mutations/postgres-ecommerce.json", + "sha256": "a15d56e6cc9c08630a27eadb3a297df182e5a925235a802628eb5b0efdf9da85", + "referenceId": "ecommerce-reference-postgres" + }, + { + "backend": "spacetime", + "path": "grader/mutations/spacetime-ecommerce.json", + "sha256": "9df40afd886890d7ab1d763940b5dbbdb5e20265194922fa0c7b05fa35bde4c3", + "referenceId": "ecommerce-reference-spacetime" + } + ], + "nullControl": { + "pointBearing": "must-fail-conclusively", + "zeroPoint": "typed-policy", + "repetitions": 1 + }, + "controls": [ + { + "stableKey": "ecommerce.spec.concurrency-safety.restock-race.202-control", + "role": "precondition", + "promotionPolicy": "must-pass-reference", + "mutationTargets": [], + "reason": "This proves that ordinary restock works before the contended restock check; it does not duplicate the existing scored restock behavior." + }, + { + "stableKey": "ecommerce.spec.external-data-sync.external-stock.901b", + "role": "precondition", + "promotionPolicy": "must-pass-reference", + "mutationTargets": [], + "reason": "This reload assertion confirms that the scored live update came from persisted data rather than a page-only patch." + } + ], + "qualification": { + "exactCombinationRequired": true, + "referenceRepetitions": 1, + "mutationRepetitions": 1, + "checks": [ + "ecommerce.feature.accounts.accounts.1a", + "ecommerce.feature.accounts.accounts.1b", + "ecommerce.spec.access-control.account-password.1c", + "ecommerce.spec.state-durability.session-reload.1e", + "ecommerce.feature.accounts.accounts.1d", + "ecommerce.feature.warehouse-admin.admin-write.103a", + "ecommerce.spec.live-state.purchase-stock.3b", + "ecommerce.spec.access-control.cart-boundary.109a", + "ecommerce.spec.access-control.cart-boundary.109b", + "ecommerce.feature.cart-checkout.cart.4a", + "ecommerce.feature.cart-checkout.cart.4d", + "ecommerce.spec.state-durability.cart-reload.4b", + "ecommerce.spec.live-state.shared-cart.4c", + "ecommerce.feature.catalog.catalog-ranking.2b", + "ecommerce.feature.catalog.catalog-search.2d", + "ecommerce.feature.catalog.catalog-values.2a", + "ecommerce.spec.live-state.ranking.2c", + "ecommerce.spec.concurrency-safety.duplicate-checkout.203a", + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b", + "ecommerce.spec.external-data-sync.external-stock.901a", + "ecommerce.spec.external-data-sync.external-stock.901d", + "ecommerce.spec.external-data-sync.external-stock.901b", + "ecommerce.spec.concurrency-safety.last-unit.201a", + "ecommerce.spec.concurrency-safety.last-unit.201b", + "ecommerce.spec.concurrency-safety.last-unit.201c", + "ecommerce.spec.access-control.order-ownership.106a", + "ecommerce.spec.access-control.purchase-attribution.102a", + "ecommerce.spec.access-control.purchase-session.101a", + "ecommerce.spec.concurrency-safety.restock-race.202-control", + "ecommerce.spec.concurrency-safety.restock-race.202a", + "ecommerce.spec.access-control.review-eligibility.108a", + "ecommerce.spec.access-control.review-eligibility.108b", + "ecommerce.spec.live-state.rating.6c", + "ecommerce.spec.transactional-integrity.unique-review.6b", + "ecommerce.feature.reviews.reviews.6a", + "ecommerce.spec.transactional-integrity.server-price.104a", + "ecommerce.spec.access-control.warehouse-area-boundary.7a", + "ecommerce.feature.warehouse-admin.warehouse-view.7b", + "ecommerce.spec.live-state.warehouse-stock.7c", + "ecommerce.spec.state-durability.account-state-recovery.105b", + "ecommerce.spec.state-durability.account-state-recovery.105a", + "ecommerce.spec.transactional-integrity.books-balance.107a", + "ecommerce.spec.transactional-integrity.books-balance.107b", + "ecommerce.spec.live-state.open-list.902a", + "ecommerce.feature.purchasing.purchase-order.3c", + "ecommerce.spec.access-control.signed-out-purchase.3a", + "ecommerce.spec.concurrency-safety.stock-limit.3d" + ], + "runner": { + "schemaVersion": 1, + "mode": "appliance", + "platform": "linux", + "architecture": "x64" + }, + "stacks": [ + "mongodb", + "postgres", + "spacetime" + ], + "evidence": [] + }, + "equivalenceDecisions": [], + "selection": { + "path": "composition/sequential.json", + "sha256": "84e8147089c325eb28068d3a2c86fcb8118768938f575bdfd5b5a14467047a9a", + "alias": "L1" + } +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/calibrations/sequential-l2.json b/tools/stack-bench/tracks/ecommerce/composition/calibrations/sequential-l2.json new file mode 100644 index 00000000000..199e97a17bb --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/calibrations/sequential-l2.json @@ -0,0 +1,176 @@ +{ + "schemaVersion": 2, + "kind": "calibration-manifest", + "id": "ecommerce.sequential-l2-calibration", + "title": "Ecommerce sequential L2 calibration", + "track": "ecommerce", + "recipe": { + "path": "../recipes/sequential-l2.json", + "id": "ecommerce.sequential-l2", + "meaningSha256": "c75e481c5a7f3ab64c2d654a4647e751ff7e36c2190b3fda37bc5a3e1612cee0", + "executionSha256": "c69700de9278e28e2972bc0c1cb67af5d0cc28564d4a6a0cbd320b2f2bf5cd98", + "contentSha256": "750e2415f701db8dbab4753afa5ef3eaa27a2d0603eeba0843c3bcea2e64827a" + }, + "fixture": { + "id": "ecommerce.operations", + "sourceSha256": "d06444b72dc94fe1ef5e08867d875e1f3bbaa5cd82c35a3558f399c1fcb5ceae" + }, + "references": { + "registryPath": "reference-apps/registry.json", + "entries": [ + { + "backend": "mongodb", + "id": "ecommerce-reference-mongodb", + "sourceSha256": "8a167a23e422534dcf58d664edcd0506ec454c5880b54e170da391c600dfe6c2" + }, + { + "backend": "postgres", + "id": "ecommerce-reference-postgres", + "sourceSha256": "b54d02f7b19b4f2e59349c776780fbec6560575a998f2c8592cf4cda6d24acac" + }, + { + "backend": "spacetime", + "id": "ecommerce-reference-spacetime", + "sourceSha256": "218a8e6298b8becef513b68d4b9a5068b84bde9c14789e0ae45ebc0d6ce703aa" + } + ] + }, + "mutations": [ + { + "backend": "mongodb", + "path": "grader/mutations/mongodb-ecommerce.json", + "sha256": "c3702c52edcf1ff904dbb5e6f93441d3ba036c2c408a067d0a284cf8e811e9cf", + "referenceId": "ecommerce-reference-mongodb" + }, + { + "backend": "postgres", + "path": "grader/mutations/postgres-ecommerce.json", + "sha256": "a15d56e6cc9c08630a27eadb3a297df182e5a925235a802628eb5b0efdf9da85", + "referenceId": "ecommerce-reference-postgres" + }, + { + "backend": "spacetime", + "path": "grader/mutations/spacetime-ecommerce.json", + "sha256": "9df40afd886890d7ab1d763940b5dbbdb5e20265194922fa0c7b05fa35bde4c3", + "referenceId": "ecommerce-reference-spacetime" + } + ], + "nullControl": { + "pointBearing": "must-fail-conclusively", + "zeroPoint": "typed-policy", + "repetitions": 1 + }, + "controls": [ + { + "stableKey": "ecommerce.spec.concurrency-safety.restock-race.202-control", + "role": "precondition", + "promotionPolicy": "must-pass-reference", + "mutationTargets": [], + "reason": "This proves ordinary restock before the scored overlap check without counting the same behavior twice." + }, + { + "stableKey": "ecommerce.spec.external-data-sync.external-stock.901b", + "role": "precondition", + "promotionPolicy": "must-pass-reference", + "mutationTargets": [], + "reason": "This proves the external write persisted before the scored live-update behavior is evaluated." + } + ], + "qualification": { + "exactCombinationRequired": true, + "referenceRepetitions": 1, + "mutationRepetitions": 1, + "checks": [ + "ecommerce.feature.accounts.accounts.1a", + "ecommerce.feature.accounts.accounts.1b", + "ecommerce.spec.access-control.account-password.1c", + "ecommerce.spec.state-durability.session-reload.1e", + "ecommerce.feature.accounts.accounts.1d", + "ecommerce.feature.warehouse-admin.admin-write.103a", + "ecommerce.spec.live-state.purchase-stock.3b", + "ecommerce.spec.access-control.cart-boundary.109a", + "ecommerce.spec.access-control.cart-boundary.109b", + "ecommerce.feature.cart-checkout.cart.4a", + "ecommerce.feature.cart-checkout.cart.4d", + "ecommerce.spec.state-durability.cart-reload.4b", + "ecommerce.spec.live-state.shared-cart.4c", + "ecommerce.feature.catalog.catalog-ranking.2b", + "ecommerce.feature.catalog.catalog-search.2d", + "ecommerce.feature.catalog.catalog-values.2a", + "ecommerce.spec.live-state.ranking.2c", + "ecommerce.spec.concurrency-safety.duplicate-checkout.203a", + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b", + "ecommerce.spec.external-data-sync.external-stock.901a", + "ecommerce.spec.external-data-sync.external-stock.901d", + "ecommerce.spec.external-data-sync.external-stock.901b", + "ecommerce.spec.concurrency-safety.last-unit.201a", + "ecommerce.spec.concurrency-safety.last-unit.201b", + "ecommerce.spec.concurrency-safety.last-unit.201c", + "ecommerce.spec.access-control.order-ownership.106a", + "ecommerce.spec.access-control.purchase-attribution.102a", + "ecommerce.spec.access-control.purchase-session.101a", + "ecommerce.spec.concurrency-safety.restock-race.202-control", + "ecommerce.spec.concurrency-safety.restock-race.202a", + "ecommerce.spec.access-control.review-eligibility.108a", + "ecommerce.spec.access-control.review-eligibility.108b", + "ecommerce.spec.live-state.rating.6c", + "ecommerce.spec.transactional-integrity.unique-review.6b", + "ecommerce.feature.reviews.reviews.6a", + "ecommerce.spec.transactional-integrity.server-price.104a", + "ecommerce.spec.access-control.warehouse-area-boundary.7a", + "ecommerce.feature.warehouse-admin.warehouse-view.7b", + "ecommerce.spec.live-state.warehouse-stock.7c", + "ecommerce.operations-access.fulfilment-queue.1a", + "ecommerce.operations-access.fulfilment-queue.1d", + "ecommerce.returns-pricing.cancellation-and-return.3a", + "ecommerce.returns-pricing.price-history.4c", + "ecommerce.inventory-operations.operational-views.5b", + "ecommerce.inventory-operations.operational-views.5c", + "ecommerce.returns-pricing.refund-accounting.203a", + "ecommerce.returns-pricing.refund-accounting.203b", + "ecommerce.returns-pricing.price-history.4b", + "ecommerce.inventory-operations.operational-views.5a", + "ecommerce.returns-pricing.price-history.4a", + "ecommerce.operations-access.fulfilment-queue.1b", + "ecommerce.operations-access.fulfilment-queue.1c", + "ecommerce.operations-access.fulfilment-queue.1e", + "ecommerce.returns-pricing.cancellation-and-return.3b", + "ecommerce.inventory-operations.stock-conservation.202b", + "ecommerce.operations-access.operator-authorization.201c", + "ecommerce.inventory-operations.stock-conservation.202d", + "ecommerce.operations-access.order-owner.204a", + "ecommerce.inventory-operations.warehouse-transfer.2a", + "ecommerce.returns-pricing.cancellation-and-return.3c", + "ecommerce.inventory-operations.operational-views.5d", + "ecommerce.operations-access.operator-authorization.201a", + "ecommerce.operations-access.operator-authorization.201b", + "ecommerce.spec.state-durability.account-state-recovery.105b", + "ecommerce.spec.state-durability.account-state-recovery.105a", + "ecommerce.spec.transactional-integrity.books-balance.107a", + "ecommerce.spec.transactional-integrity.books-balance.107b", + "ecommerce.spec.live-state.open-list.902a", + "ecommerce.feature.purchasing.purchase-order.3c", + "ecommerce.spec.access-control.signed-out-purchase.3a", + "ecommerce.spec.concurrency-safety.stock-limit.3d", + "ecommerce.inventory-operations.warehouse-transfer.2c" + ], + "runner": { + "schemaVersion": 1, + "mode": "appliance", + "platform": "linux", + "architecture": "x64" + }, + "stacks": [ + "mongodb", + "postgres", + "spacetime" + ], + "evidence": [] + }, + "equivalenceDecisions": [], + "selection": { + "path": "composition/sequential.json", + "sha256": "84e8147089c325eb28068d3a2c86fcb8118768938f575bdfd5b5a14467047a9a", + "alias": "L2" + } +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/dependency.json b/tools/stack-bench/tracks/ecommerce/composition/dependency.json new file mode 100644 index 00000000000..41b92c34384 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/dependency.json @@ -0,0 +1,12 @@ +{ + "schemaVersion": 1, + "kind": "recipe-selection", + "entries": [ + { "alias": "L1", "recipe": { "path": "recipes/progression-catalog.json", "id": "ecommerce.progression-catalog" } }, + { "alias": "L2", "recipe": { "path": "recipes/progression-catalog.json", "id": "ecommerce.progression-catalog" } }, + { "alias": "L3", "recipe": { "path": "recipes/progression-catalog.json", "id": "ecommerce.progression-catalog" } }, + { "alias": "L4", "recipe": { "path": "recipes/progression-catalog.json", "id": "ecommerce.progression-catalog" } }, + { "alias": "L5", "recipe": { "path": "recipes/progression-catalog.json", "id": "ecommerce.progression-catalog" } }, + { "alias": "L6", "recipe": { "path": "recipes/progression-catalog.json", "id": "ecommerce.progression-catalog" } } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/fixtures/operations.json b/tools/stack-bench/tracks/ecommerce/composition/fixtures/operations.json new file mode 100644 index 00000000000..d1dddebb1c9 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/fixtures/operations.json @@ -0,0 +1,27 @@ +{ + "schemaVersion": 1, + "kind": "fixture-set", + "id": "ecommerce.operations", + "title": "Ecommerce operations starting data", + "warehouses": ["East", "West"], + "items": [ + { "name": "Air Purifier", "price": "189.00", "category": "Home", "stock": { "East": 60, "West": 40 } }, + { "name": "Bluetooth Speaker", "price": "79.50", "category": "Audio", "stock": { "East": 50, "West": 50 } }, + { "name": "Coffee Grinder", "price": "64.00", "category": "Home", "stock": { "East": 70, "West": 30 } }, + { "name": "Desk Lamp", "price": "42.00", "category": "Home", "stock": { "East": 55, "West": 45 } }, + { "name": "Espresso Machine", "price": "449.00", "category": "Home", "stock": { "East": 80, "West": 20 } }, + { "name": "Gaming Mouse", "price": "59.00", "category": "Computing", "stock": { "East": 50, "West": 50 } }, + { "name": "Headphones", "price": "199.00", "category": "Audio", "stock": { "East": 60, "West": 40 } }, + { "name": "Induction Cooktop", "price": "329.00", "category": "Home", "stock": { "East": 50, "West": 50 } }, + { "name": "Keyboard", "price": "89.00", "category": "Computing", "stock": { "East": 70, "West": 30 } }, + { "name": "Laptop Stand", "price": "29.00", "category": "Computing", "stock": { "East": 90, "West": 10 } }, + { "name": "Mirrorless Camera", "price": "1299.00", "category": "Photo", "stock": { "East": 2, "West": 1 } }, + { "name": "USB Cable", "price": "65.00", "category": "Home", "stock": { "East": 0, "West": 0 } }, + { "name": "Webcam", "price": "69.00", "category": "Computing", "stock": { "East": 60, "West": 40 } } + ], + "accounts": [ + { "username": "admin", "password": "stackbench-admin-2026", "roles": ["admin"] }, + { "username": "staff", "password": "stackbench-staff-2026", "roles": ["staff"] } + ], + "empty": ["carts", "orders", "reviews", "returns"] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/fixtures/storefront.json b/tools/stack-bench/tracks/ecommerce/composition/fixtures/storefront.json new file mode 100644 index 00000000000..41426c67f2a --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/fixtures/storefront.json @@ -0,0 +1,26 @@ +{ + "schemaVersion": 1, + "kind": "fixture-set", + "id": "ecommerce.storefront", + "title": "Ecommerce storefront starting data", + "warehouses": ["East", "West"], + "items": [ + { "name": "Air Purifier", "price": "189.00", "category": "Home", "stock": { "East": 60, "West": 40 } }, + { "name": "Bluetooth Speaker", "price": "79.50", "category": "Audio", "stock": { "East": 50, "West": 50 } }, + { "name": "Coffee Grinder", "price": "64.00", "category": "Home", "stock": { "East": 70, "West": 30 } }, + { "name": "Desk Lamp", "price": "42.00", "category": "Home", "stock": { "East": 55, "West": 45 } }, + { "name": "Espresso Machine", "price": "449.00", "category": "Home", "stock": { "East": 80, "West": 20 } }, + { "name": "Gaming Mouse", "price": "59.00", "category": "Computing", "stock": { "East": 50, "West": 50 } }, + { "name": "Headphones", "price": "199.00", "category": "Audio", "stock": { "East": 60, "West": 40 } }, + { "name": "Induction Cooktop", "price": "329.00", "category": "Home", "stock": { "East": 50, "West": 50 } }, + { "name": "Keyboard", "price": "89.00", "category": "Computing", "stock": { "East": 70, "West": 30 } }, + { "name": "Laptop Stand", "price": "29.00", "category": "Computing", "stock": { "East": 90, "West": 10 } }, + { "name": "Mirrorless Camera", "price": "1299.00", "category": "Photo", "stock": { "East": 2, "West": 1 } }, + { "name": "USB Cable", "price": "65.00", "category": "Home", "stock": { "East": 0, "West": 0 } }, + { "name": "Webcam", "price": "69.00", "category": "Computing", "stock": { "East": 60, "West": 40 } } + ], + "accounts": [ + { "username": "admin", "password": "stackbench-admin-2026", "roles": ["admin"] } + ], + "empty": ["carts", "orders", "reviews"] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/framing/smoke-requirements.md b/tools/stack-bench/tracks/ecommerce/composition/framing/smoke-requirements.md new file mode 100644 index 00000000000..f2a68253a8d --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/framing/smoke-requirements.md @@ -0,0 +1,7 @@ +# Ecommerce capability smoke + +Build a real-time store with the selected capabilities below. Keep shared state +current across open clients without a reload. Show useful loading, empty, and +error states, and keep the interface keyboard accessible. + +## Features diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/feature-accounts.json b/tools/stack-bench/tracks/ecommerce/composition/packs/feature-accounts.json new file mode 100644 index 00000000000..5025e0649b9 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/feature-accounts.json @@ -0,0 +1,76 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.feature.accounts", + "title": "Accounts", + "moduleType": "feature", + "description": "Account creation, sign-in, and sign-out.", + "requiresPacks": [], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 59000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.feature.accounts.requirement", + "path": "prompts/modular/accounts.md", + "order": 100, + "modes": [ + "fresh", + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.feature.accounts.hooks", + "path": "contracts/accounts.md", + "order": 2100, + "modes": [ + "fresh", + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "account-create", + "stableId": "accounts", + "source": "scenarios/01-account-create.json", + "feature": 1, + "criteria": [ + "1a" + ], + "role": "feature" + }, + { + "id": "account-duplicate", + "stableId": "accounts", + "source": "scenarios/01-account-duplicate.json", + "feature": 1, + "criteria": [ + "1b" + ], + "role": "feature" + }, + { + "id": "account-signout", + "stableId": "accounts", + "source": "scenarios/01-account-signout.json", + "feature": 1, + "criteria": [ + "1d" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/feature-bundle-checkout.json b/tools/stack-bench/tracks/ecommerce/composition/packs/feature-bundle-checkout.json new file mode 100644 index 00000000000..342b085c479 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/feature-bundle-checkout.json @@ -0,0 +1,60 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.feature.bundle-checkout", + "moduleType": "feature", + "title": "Bundle checkout", + "description": "Bundle checkout using existing products and orders.", + "requiresPacks": [ + "ecommerce.feature.product-bundles", + "ecommerce.l3.reservations-features" + ], + "conflictsWith": [], + "capabilities": [ + "browser", + "database-observation", + "direct-database-write" + ], + "evidence": [ + "browser-observation", + "database-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 120000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.feature.bundle-checkout", + "path": "prompts/modular/bundle-checkout.md", + "order": 8005, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.interface.bundle-checkout", + "path": "contracts/bundle-checkout.md", + "order": 8105, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "bundle-checkout", + "stableId": "bundle-checkout", + "source": "scenarios/progression-bundle-checkout.json", + "feature": 741, + "criteria": [ + "741a" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/feature-bundle-returns.json b/tools/stack-bench/tracks/ecommerce/composition/packs/feature-bundle-returns.json new file mode 100644 index 00000000000..4a09f3d0242 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/feature-bundle-returns.json @@ -0,0 +1,60 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.feature.bundle-returns", + "moduleType": "feature", + "title": "Bundle returns", + "description": "Bundle returns using existing products and orders.", + "requiresPacks": [ + "ecommerce.feature.bundle-checkout", + "ecommerce.l3.order-returns-features" + ], + "conflictsWith": [], + "capabilities": [ + "browser", + "database-observation", + "direct-database-write" + ], + "evidence": [ + "browser-observation", + "database-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 120000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.feature.bundle-returns", + "path": "prompts/modular/bundle-returns.md", + "order": 8006, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.interface.bundle-returns", + "path": "contracts/bundle-returns.md", + "order": 8106, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "bundle-returns", + "stableId": "bundle-returns", + "source": "scenarios/progression-bundle-returns.json", + "feature": 742, + "criteria": [ + "742a" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/feature-cart.json b/tools/stack-bench/tracks/ecommerce/composition/packs/feature-cart.json new file mode 100644 index 00000000000..55c4dae465d --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/feature-cart.json @@ -0,0 +1,60 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.feature.cart", + "stableId": "ecommerce.feature.cart-checkout", + "title": "Cart", + "moduleType": "feature", + "description": "A signed-in customer cart.", + "requiresPacks": [ + "ecommerce.feature.accounts", + "ecommerce.feature.catalog-items" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 23000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.feature.cart.requirement", + "path": "prompts/modular/cart.md", + "order": 400, + "modes": [ + "fresh", + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.feature.cart.hooks", + "path": "contracts/cart.md", + "order": 2300, + "modes": [ + "fresh", + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "cart", + "stableId": "cart", + "source": "scenarios/progression-cart-checkout.json", + "feature": 4, + "criteria": [ + "4a" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/feature-catalog-discovery.json b/tools/stack-bench/tracks/ecommerce/composition/packs/feature-catalog-discovery.json new file mode 100644 index 00000000000..06c55b24d2f --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/feature-catalog-discovery.json @@ -0,0 +1,69 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.feature.catalog-discovery", + "stableId": "ecommerce.feature.catalog", + "title": "Catalog discovery", + "moduleType": "feature", + "description": "Public catalog ranking and full-catalog search.", + "requiresPacks": [ + "ecommerce.feature.catalog-items" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 5000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.feature.catalog-discovery.requirement", + "path": "prompts/modular/catalog-discovery.md", + "order": 210, + "modes": [ + "fresh", + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.feature.catalog-discovery.hooks", + "path": "contracts/catalog-discovery.md", + "order": 2010, + "modes": [ + "fresh", + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "ranking", + "stableId": "catalog-ranking", + "source": "scenarios/01-catalog-ranking.json", + "feature": 2, + "criteria": [ + "2b" + ], + "role": "feature" + }, + { + "id": "search", + "stableId": "catalog-search", + "source": "scenarios/01-catalog-search.json", + "feature": 2, + "criteria": [ + "2d" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/feature-catalog-items.json b/tools/stack-bench/tracks/ecommerce/composition/packs/feature-catalog-items.json new file mode 100644 index 00000000000..51a17174520 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/feature-catalog-items.json @@ -0,0 +1,57 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.feature.catalog-items", + "stableId": "ecommerce.feature.catalog", + "title": "Catalog items", + "moduleType": "feature", + "description": "Public catalog item values used by shopping features.", + "requiresPacks": [], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 1000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.feature.catalog-items.requirement", + "path": "prompts/modular/catalog-items.md", + "order": 200, + "modes": [ + "fresh", + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.feature.catalog-items.hooks", + "path": "contracts/catalog-items.md", + "order": 2000, + "modes": [ + "fresh", + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "values", + "stableId": "catalog-values", + "source": "scenarios/01-catalog-values.json", + "feature": 2, + "criteria": [ + "2a" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/feature-checkout.json b/tools/stack-bench/tracks/ecommerce/composition/packs/feature-checkout.json new file mode 100644 index 00000000000..e409f13456e --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/feature-checkout.json @@ -0,0 +1,65 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.feature.checkout", + "stableId": "ecommerce.feature.cart-checkout", + "title": "Checkout", + "moduleType": "feature", + "description": "Turn a cart into an order.", + "requiresPacks": [ + "ecommerce.feature.cart" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 69000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.feature.checkout.requirement", + "path": "prompts/modular/checkout.md", + "order": 410, + "modes": [ + "fresh", + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.orders.data", + "path": "contracts/order-data.md", + "order": 2315, + "modes": ["fresh", "upgrade"] + }, + { + "id": "ecommerce.feature.checkout.hooks", + "path": "contracts/checkout.md", + "order": 2310, + "modes": [ + "fresh", + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "checkout", + "stableId": "cart", + "source": "scenarios/progression-cart-checkout.json", + "feature": 4, + "criteria": [ + "4d" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/feature-product-bundles.json b/tools/stack-bench/tracks/ecommerce/composition/packs/feature-product-bundles.json new file mode 100644 index 00000000000..9a81b30be7e --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/feature-product-bundles.json @@ -0,0 +1,59 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.feature.product-bundles", + "moduleType": "feature", + "title": "Product bundles", + "description": "Product bundles using existing products and orders.", + "requiresPacks": [ + "ecommerce.progression.catalog-management" + ], + "conflictsWith": [], + "capabilities": [ + "browser", + "database-observation", + "direct-database-write" + ], + "evidence": [ + "browser-observation", + "database-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 120000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.feature.product-bundles", + "path": "prompts/modular/product-bundles.md", + "order": 8004, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.interface.product-bundles", + "path": "contracts/product-bundles.md", + "order": 8104, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "product-bundles", + "stableId": "product-bundles", + "source": "scenarios/progression-product-bundles.json", + "feature": 740, + "criteria": [ + "740a" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/feature-purchasing.json b/tools/stack-bench/tracks/ecommerce/composition/packs/feature-purchasing.json new file mode 100644 index 00000000000..3415686aea9 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/feature-purchasing.json @@ -0,0 +1,64 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.feature.purchasing", + "title": "Purchasing and orders", + "moduleType": "feature", + "description": "Immediate purchase and customer order history.", + "requiresPacks": [ + "ecommerce.feature.accounts", + "ecommerce.feature.catalog-items" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 10000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.feature.purchasing.requirement", + "path": "prompts/modular/purchasing.md", + "order": 300, + "modes": [ + "fresh", + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.orders.data", + "path": "contracts/order-data.md", + "order": 2315, + "modes": ["fresh", "upgrade"] + }, + { + "id": "ecommerce.feature.purchasing.hooks", + "path": "contracts/purchasing.md", + "order": 2200, + "modes": [ + "fresh", + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "purchase-order", + "source": "scenarios/progression-purchasing.json", + "feature": 3, + "criteria": [ + "3c" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/feature-reviews.json b/tools/stack-bench/tracks/ecommerce/composition/packs/feature-reviews.json new file mode 100644 index 00000000000..fd8b3eab73d --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/feature-reviews.json @@ -0,0 +1,59 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.feature.reviews", + "title": "Reviews", + "moduleType": "feature", + "description": "Item ratings and comments.", + "requiresPacks": [ + "ecommerce.feature.accounts", + "ecommerce.feature.catalog-items", + "ecommerce.feature.purchasing" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 10000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.feature.reviews.requirement", + "path": "prompts/modular/reviews.md", + "order": 500, + "modes": [ + "fresh", + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.feature.reviews.hooks", + "path": "contracts/reviews.md", + "order": 2400, + "modes": [ + "fresh", + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "reviews", + "source": "scenarios/01-review-visibility.json", + "feature": 6, + "criteria": [ + "6a" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/feature-split-tender-refunds.json b/tools/stack-bench/tracks/ecommerce/composition/packs/feature-split-tender-refunds.json new file mode 100644 index 00000000000..634df00ace2 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/feature-split-tender-refunds.json @@ -0,0 +1,83 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.feature.split-tender-refunds", + "moduleType": "feature", + "title": "Split-tender refunds", + "description": "Split-tender refunds", + "requiresPacks": [ + "ecommerce.feature.store-credit", + "ecommerce.progression.support-refunds", + "ecommerce.l3.order-returns-features" + ], + "conflictsWith": [], + "capabilities": [ + "browser", + "database-observation", + "direct-server-call" + ], + "evidence": [ + "browser-observation", + "database-observation", + "fresh-client-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 120000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.feature.split-tender-refunds", + "path": "prompts/modular/split-tender-refunds.md", + "order": 6100, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.feature.split-tender-refunds.interface", + "path": "contracts/split-tender-refunds.md", + "order": 6900, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "feature-751", + "stableId": "split-tender-refunds-751", + "source": "scenarios/progression-split-tender-refunds.json", + "feature": 751, + "criteria": [ + "751a" + ], + "role": "feature", + "requiresFeatures": [ + "ecommerce.feature.split-tender-refunds", + "ecommerce.feature.store-credit", + "ecommerce.progression.support-refunds" + ] + }, + { + "id": "return-refund-interaction", + "source": "scenarios/progression-support-return-interaction.json", + "feature": 757, + "criteria": [ + "757a", + "757b" + ], + "role": "feature", + "requiresFeatures": [ + "ecommerce.l3.order-returns-features", + "ecommerce.progression.support-refunds", + "ecommerce.progression.fulfilment-queue" + ], + "stableId": "return-refund-interaction" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/feature-store-credit.json b/tools/stack-bench/tracks/ecommerce/composition/packs/feature-store-credit.json new file mode 100644 index 00000000000..d661287a9a9 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/feature-store-credit.json @@ -0,0 +1,59 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.feature.store-credit", + "moduleType": "feature", + "title": "Store credit", + "description": "Store credit", + "requiresPacks": [ + "ecommerce.progression.payment-records", + "ecommerce.progression.staff-roles" + ], + "conflictsWith": [], + "capabilities": [ + "browser", + "direct-server-call" + ], + "evidence": [ + "browser-observation", + "fresh-client-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 180000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.feature.store-credit", + "path": "prompts/modular/store-credit.md", + "order": 5100, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.feature.store-credit.interface", + "path": "contracts/store-credit.md", + "order": 5900, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "feature-750", + "stableId": "store-credit-750", + "source": "scenarios/progression-store-credit.json", + "feature": 750, + "criteria": [ + "750a" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/feature-subscriptions.json b/tools/stack-bench/tracks/ecommerce/composition/packs/feature-subscriptions.json new file mode 100644 index 00000000000..fbd6e57a2f3 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/feature-subscriptions.json @@ -0,0 +1,60 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.feature.subscriptions", + "moduleType": "feature", + "title": "Scheduled purchases", + "description": "Scheduled purchases", + "requiresPacks": [ + "ecommerce.progression.payment-records" + ], + "conflictsWith": [], + "capabilities": [ + "browser", + "direct-server-call", + "database-observation" + ], + "evidence": [ + "browser-observation", + "fresh-client-observation", + "database-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 180000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.feature.subscriptions", + "path": "prompts/modular/subscriptions.md", + "order": 5100, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.feature.subscriptions.interface", + "path": "contracts/subscriptions.md", + "order": 5900, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "feature-760", + "stableId": "subscriptions-760", + "source": "scenarios/progression-subscriptions.json", + "feature": 760, + "criteria": [ + "760a" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/feature-warehouse-admin.json b/tools/stack-bench/tracks/ecommerce/composition/packs/feature-warehouse-admin.json new file mode 100644 index 00000000000..936bdc9eefb --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/feature-warehouse-admin.json @@ -0,0 +1,67 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.feature.warehouse-admin", + "title": "Warehouse administration", + "moduleType": "feature", + "description": "Warehouse inventory, restocking, and revenue views.", + "requiresPacks": [ + "ecommerce.feature.catalog-items", + "ecommerce.progression.staff-access" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 29000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.feature.warehouse-admin.requirement", + "path": "prompts/modular/warehouse-administration.md", + "order": 600, + "modes": [ + "fresh", + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.feature.warehouse-admin.hooks", + "path": "contracts/warehouse-administration.md", + "order": 2500, + "modes": [ + "fresh", + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "warehouse-view", + "source": "scenarios/01-warehouse-admin-staff.json", + "feature": 7, + "criteria": [ + "7b" + ], + "role": "feature" + }, + { + "id": "admin-write", + "source": "scenarios/01-admin-write-staff.json", + "feature": 103, + "criteria": [ + "103a" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/inventory-operations-features.json b/tools/stack-bench/tracks/ecommerce/composition/packs/inventory-operations-features.json new file mode 100644 index 00000000000..cb154ffc061 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/inventory-operations-features.json @@ -0,0 +1,129 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.inventory-operations-features", + "stableId": "ecommerce.inventory-operations", + "moduleType": "feature", + "title": "Inventory operations", + "description": "Warehouse transfers and live operational views.", + "requiresPacks": [ + "ecommerce.returns-pricing-features" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 86000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.l2.stock-transfer", + "path": "prompts/02-operations.md", + "order": 30, + "from": "### Moving stock", + "until": "### Cancelling and returning", + "modes": [ + "upgrade" + ] + }, + { + "id": "ecommerce.l2.operational-views", + "path": "prompts/02-operations.md", + "order": 50, + "from": "### Operational views", + "until": "### What must stay true", + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.l2.transfer-hooks", + "path": "contracts/appendix-02.md", + "order": 2030, + "from": "| `transfer-from`", + "until": "| `price-input`", + "modes": [ + "upgrade" + ] + }, + { + "id": "ecommerce.l2.operational-view-hooks", + "path": "contracts/appendix-02.md", + "order": 2050, + "from": "| `low-stock-list`", + "until": "| `order-status`", + "modes": [ + "upgrade" + ] + }, + { + "id": "ecommerce.l2.inventory-action-inputs", + "path": "prompts/l2-action-inputs.md", + "order": 2090, + "from": "## Inventory contention action inputs", + "modes": [ + "upgrade" + ] + }, + { + "id": "ecommerce.l2.transfer-testing-call", + "path": "prompts/fragments/named-actions-inventory-operations.md", + "order": 2095, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "warehouse-transfer-corrected", + "stableId": "warehouse-transfer", + "source": "scenarios/02-strengthened.json", + "feature": 2, + "criteria": [ + "2a" + ], + "role": "feature" + }, + { + "id": "operational-views-low-stock", + "stableId": "operational-views", + "source": "scenarios/02-low-stock.json", + "feature": 5, + "criteria": [ + "5a" + ], + "role": "feature" + }, + { + "id": "operational-views-core", + "stableId": "operational-views", + "source": "scenarios/02-features.json", + "feature": 5, + "criteria": [ + "5b", + "5c" + ], + "role": "feature" + }, + { + "id": "operational-views-kept-purchases", + "stableId": "operational-views", + "source": "scenarios/02-strengthened.json", + "feature": 5, + "criteria": [ + "5d" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/inventory-operations-specifications.json b/tools/stack-bench/tracks/ecommerce/composition/packs/inventory-operations-specifications.json new file mode 100644 index 00000000000..f4e527fad38 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/inventory-operations-specifications.json @@ -0,0 +1,74 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.inventory-operations-specifications", + "stableId": "ecommerce.inventory-operations", + "moduleType": "specification", + "title": "Inventory conservation", + "description": "Stock remains exact across transfers, cancellations, clients, and overlapping writes.", + "requiresPacks": [], + "conflictsWith": [], + "capabilities": ["browser", "concurrency", "direct-server-call"], + "evidence": ["browser-observation", "server-response"], + "budget": { "status": "bounded", "maxRuntimeMs": 52000 }, + "task": { + "requirements": [ + { "id": "ecommerce.l2.inventory-invariants", "path": "prompts/02-operations.md", "order": 60, "from": "### What must stay true", "until": "- Revenue always", "modes": ["upgrade"], "requiresFeatures": ["ecommerce.inventory-operations-features"] } + ], + "contracts": [] + }, + "checks": [ + { + "id": "stock-conservation-cancel", + "stableId": "stock-conservation", + "source": "scenarios/02-self-contained.json", + "feature": 202, + "criteria": [ + "202b" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.inventory-operations-features", + "ecommerce.returns-pricing-features" + ] + }, + { + "id": "stock-conservation-contention", + "stableId": "stock-conservation", + "source": "scenarios/02-server-actions.json", + "feature": 202, + "criteria": [ + "202d" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.inventory-operations-features" + ] + }, + { + "id": "warehouse-transfer-overdraw", + "stableId": "warehouse-transfer", + "source": "scenarios/02-transfer-overdraw.json", + "feature": 2, + "criteria": [ + "2c" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.inventory-operations-features" + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/l2-inventory-dashboard.json b/tools/stack-bench/tracks/ecommerce/composition/packs/l2-inventory-dashboard.json new file mode 100644 index 00000000000..dd5acb99877 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/l2-inventory-dashboard.json @@ -0,0 +1,57 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.l2.inventory-dashboard", + "stableId": "ecommerce.inventory-operations", + "moduleType": "feature", + "title": "Inventory dashboard", + "description": "A low-stock dashboard.", + "requiresPacks": [ + "ecommerce.feature.warehouse-admin" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 24000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.l2.inventory-dashboard", + "path": "prompts/modular/inventory-dashboard.md", + "order": 50, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.l2.inventory-dashboard-hooks", + "path": "contracts/inventory-dashboard.md", + "order": 2050, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "low-stock", + "stableId": "operational-views", + "source": "scenarios/02-low-stock.json", + "feature": 5, + "criteria": [ + "5e" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/l2-order-cancellation-features.json b/tools/stack-bench/tracks/ecommerce/composition/packs/l2-order-cancellation-features.json new file mode 100644 index 00000000000..b0b69706566 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/l2-order-cancellation-features.json @@ -0,0 +1,68 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.l2.order-cancellation-features", + "stableId": "ecommerce.returns-pricing", + "moduleType": "feature", + "title": "Order cancellation", + "description": "Cancel pending orders without losing stock or accounting accuracy.", + "requiresPacks": [ + "ecommerce.feature.purchasing", + "ecommerce.feature.warehouse-admin" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 48000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.l2.order-cancellation", + "path": "prompts/modular/order-cancellation.md", + "order": 2040, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.l2.order-cancellation-hooks", + "path": "contracts/order-cancellation.md", + "order": 2060, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "cancellation-core", + "stableId": "cancellation-and-return", + "source": "scenarios/02-order-cancellation-core.json", + "feature": 3, + "criteria": [ + "3a" + ], + "role": "feature" + }, + { + "id": "cancellation-history", + "stableId": "cancellation-and-return", + "source": "scenarios/02-order-cancellation-history.json", + "feature": 3, + "criteria": [ + "3b" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/l2-price-history-features.json b/tools/stack-bench/tracks/ecommerce/composition/packs/l2-price-history-features.json new file mode 100644 index 00000000000..1b2183a2c5a --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/l2-price-history-features.json @@ -0,0 +1,14 @@ +{ + "schemaVersion": 1, "kind": "test-pack", "id": "ecommerce.l2.price-history-features", "stableId": "ecommerce.returns-pricing", "moduleType": "feature", + "title": "Price history", "description": "Administrators update prices and public prices update live.", + "requiresPacks": ["ecommerce.progression.catalog-management"], + "conflictsWith": [], "capabilities": ["browser"], "evidence": ["browser-observation"], + "budget": { "status": "bounded", "maxRuntimeMs": 94000 }, + "task": { + "requirements": [{ "id": "ecommerce.l2.price-history", "path": "prompts/modular/price-history.md", "order": 2040, "modes": ["upgrade"] }], + "contracts": [{ "id": "ecommerce.l2.price-hooks", "path": "contracts/price-history.md", "order": 2041, "modes": ["upgrade"] }] + }, + "checks": [ + { "id": "price-live-independent", "stableId": "price-history", "source": "scenarios/02-live-price.json", "feature": 4, "criteria": ["4b"], "role": "feature" } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/l2-recommendations.json b/tools/stack-bench/tracks/ecommerce/composition/packs/l2-recommendations.json new file mode 100644 index 00000000000..d3fe13e1877 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/l2-recommendations.json @@ -0,0 +1,58 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.l2.recommendations", + "stableId": "ecommerce.inventory-operations", + "moduleType": "feature", + "title": "Recommendations", + "description": "Customer recommendations based on purchase activity and cart contents.", + "requiresPacks": [ + "ecommerce.feature.cart", + "ecommerce.feature.purchasing" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 16000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.l2.recommendations", + "path": "prompts/modular/recommendations.md", + "order": 70, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.l2.recommendations-hooks", + "path": "contracts/recommendations.md", + "order": 2070, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "recommendations", + "stableId": "operational-views", + "source": "scenarios/02-operational-recommendations.json", + "feature": 5, + "criteria": [ + "5c" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/l2-sales-dashboard.json b/tools/stack-bench/tracks/ecommerce/composition/packs/l2-sales-dashboard.json new file mode 100644 index 00000000000..d0194d6b6ee --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/l2-sales-dashboard.json @@ -0,0 +1,67 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.l2.sales-dashboard", + "stableId": "ecommerce.inventory-operations", + "moduleType": "feature", + "title": "Sales dashboard", + "description": "Category totals and best sellers.", + "requiresPacks": [ + "ecommerce.feature.purchasing" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 36000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.l2.sales-dashboard", + "path": "prompts/modular/sales-dashboard.md", + "order": 60, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.l2.sales-dashboard-hooks", + "path": "contracts/sales-dashboard.md", + "order": 2060, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "category-totals", + "stableId": "operational-views", + "source": "scenarios/02-operational-category-totals.json", + "feature": 5, + "criteria": [ + "5f" + ], + "role": "feature" + }, + { + "id": "best-sellers", + "stableId": "operational-views", + "source": "scenarios/02-operational-best-sellers.json", + "feature": 5, + "criteria": [ + "5d" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/l2-stock-transfers-features.json b/tools/stack-bench/tracks/ecommerce/composition/packs/l2-stock-transfers-features.json new file mode 100644 index 00000000000..335dda1376e --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/l2-stock-transfers-features.json @@ -0,0 +1,57 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.l2.stock-transfers-features", + "stableId": "ecommerce.inventory-operations", + "moduleType": "feature", + "title": "Stock transfers", + "description": "Move stock between warehouses without changing the total.", + "requiresPacks": [ + "ecommerce.feature.warehouse-admin" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 12000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.l2.stock-transfer", + "path": "prompts/modular/stock-transfers.md", + "order": 2030, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.l2.transfer-hooks", + "path": "contracts/stock-transfers.md", + "order": 2031, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "transfer", + "stableId": "warehouse-transfer", + "source": "scenarios/02-strengthened.json", + "feature": 2, + "criteria": [ + "2a" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/l3-cart-expiration-features.json b/tools/stack-bench/tracks/ecommerce/composition/packs/l3-cart-expiration-features.json new file mode 100644 index 00000000000..2fcea81c9fb --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/l3-cart-expiration-features.json @@ -0,0 +1,54 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.l3.cart-expiration-features", + "stableId": "ecommerce.l3.cart-expiration", + "moduleType": "feature", + "title": "Cart expiration", + "description": "Inactive carts expire, release stock, and return empty.", + "requiresPacks": [ + "ecommerce.l3.reservations-features" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 360000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.l3.cart-expiration", + "path": "prompts/modular/cart-expiration.md", + "order": 3030, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.l3.cart-expiration-hooks", + "path": "contracts/cart-expiration.md", + "order": 3930, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "inactive-cart", + "stableId": "cart-expiration", + "source": "scenarios/03-cart-expiration.json", + "feature": 304, + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/l3-deferred-access-specifications.json b/tools/stack-bench/tracks/ecommerce/composition/packs/l3-deferred-access-specifications.json new file mode 100644 index 00000000000..e5784a3e36e --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/l3-deferred-access-specifications.json @@ -0,0 +1,21 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.l3.deferred-access-specifications", + "stableId": "ecommerce.l3.deferred-access", + "moduleType": "specification", + "title": "Scheduled-work access", + "description": "Customers cannot schedule or cancel restocks.", + "requiresPacks": [], + "conflictsWith": [], + "capabilities": ["browser", "direct-server-call", "request-replay"], + "evidence": ["browser-observation", "server-refusal", "server-response"], + "budget": { "status": "bounded", "maxRuntimeMs": 56000 }, + "task": { + "requirements": [{ "id": "ecommerce.l3.deferred-access", "path": "prompts/modular/l3-specifications.md", "order": 3130, "from": "## Deferred-work access", "until": "## Stock conservation", "modes": ["upgrade"], "requiresFeatures": ["ecommerce.l3.scheduled-restocks-features"] }], + "contracts": [] + }, + "checks": [ + { "id": "customer-refused", "stableId": "scheduled-work-access", "source": "scenarios/03-deferred-access.json", "feature": 317, "role": "guarantee", "observations": ["requested", "unmentioned"], "requiresFeatures": ["ecommerce.l3.scheduled-restocks-features"] } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/l3-deferred-durability-specifications.json b/tools/stack-bench/tracks/ecommerce/composition/packs/l3-deferred-durability-specifications.json new file mode 100644 index 00000000000..8206ae12113 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/l3-deferred-durability-specifications.json @@ -0,0 +1,28 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.l3.deferred-durability-specifications", + "stableId": "ecommerce.l3.deferred-durability", + "moduleType": "specification", + "title": "Deferred-work durability", + "description": "Pending work survives a backend restart.", + "requiresPacks": [], + "conflictsWith": [], + "capabilities": ["browser", "backend-lifecycle"], + "evidence": ["browser-observation", "database-observation"], + "budget": { "status": "bounded", "maxRuntimeMs": 632000 }, + "task": { + "requirements": [ + { "id": "ecommerce.l3.durable-reservations", "path": "prompts/modular/l3-specifications.md", "order": 3100, "from": "## Durable reservations", "until": "## Durable restocks", "modes": ["upgrade"], "requiresFeatures": ["ecommerce.l3.reservations-features"] }, + { "id": "ecommerce.l3.durable-restocks", "path": "prompts/modular/l3-specifications.md", "order": 3101, "from": "## Durable restocks", "until": "## Durable order delivery", "modes": ["upgrade"], "requiresFeatures": ["ecommerce.l3.scheduled-restocks-features"] }, + { "id": "ecommerce.l3.durable-order-delivery", "path": "prompts/modular/l3-specifications.md", "order": 3102, "from": "## Durable order delivery", "until": "## Durable cart expiration", "modes": ["upgrade"], "requiresFeatures": ["ecommerce.l3.order-delivery-features"] }, + { "id": "ecommerce.l3.durable-cart-expiration", "path": "prompts/modular/l3-specifications.md", "order": 3103, "from": "## Durable cart expiration", "until": "## Exactly-once restocks", "modes": ["upgrade"], "requiresFeatures": ["ecommerce.l3.cart-expiration-features"] } + ], + "contracts": [] + }, + "checks": [ + { "id": "reservation-restart", "stableId": "restart-survival", "source": "scenarios/03-deferred-durability.json", "feature": 314, "role": "guarantee", "observations": ["requested", "unmentioned"], "requiresFeatures": ["ecommerce.l3.reservations-features"] }, + { "id": "delivery-restart", "stableId": "restart-survival", "source": "scenarios/03-deferred-durability.json", "feature": 315, "role": "guarantee", "observations": ["requested", "unmentioned"], "requiresFeatures": ["ecommerce.l3.order-delivery-features"] }, + { "id": "cart-restart", "stableId": "restart-survival", "source": "scenarios/03-deferred-durability.json", "feature": 316, "role": "guarantee", "observations": ["requested", "unmentioned"], "requiresFeatures": ["ecommerce.l3.cart-expiration-features"] } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/l3-deferred-integrity-specifications.json b/tools/stack-bench/tracks/ecommerce/composition/packs/l3-deferred-integrity-specifications.json new file mode 100644 index 00000000000..e7a4d841f12 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/l3-deferred-integrity-specifications.json @@ -0,0 +1,28 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.l3.deferred-integrity-specifications", + "stableId": "ecommerce.l3.deferred-integrity", + "moduleType": "specification", + "title": "Deferred-work integrity", + "description": "Deferred work applies once and preserves stock.", + "requiresPacks": [], + "conflictsWith": [], + "capabilities": ["browser", "backend-lifecycle"], + "evidence": ["browser-observation", "database-observation"], + "budget": { "status": "bounded", "maxRuntimeMs": 351000 }, + "task": { + "requirements": [ + { "id": "ecommerce.l3.exactly-once-restocks", "path": "prompts/modular/l3-specifications.md", "order": 3110, "from": "## Exactly-once restocks", "until": "## Exactly-once delivery", "modes": ["upgrade"], "requiresFeatures": ["ecommerce.l3.scheduled-restocks-features"] }, + { "id": "ecommerce.l3.exactly-once-delivery", "path": "prompts/modular/l3-specifications.md", "order": 3111, "from": "## Exactly-once delivery", "until": "## Server-timed restocks", "modes": ["upgrade"], "requiresFeatures": ["ecommerce.l3.order-delivery-features"] }, + { "id": "ecommerce.l3.stock-conservation", "path": "prompts/modular/l3-specifications.md", "order": 3140, "from": "## Stock conservation", "modes": ["upgrade"], "requiresFeatures": ["ecommerce.l3.reservations-features"] } + ], + "contracts": [] + }, + "checks": [ + { "id": "restock-once", "stableId": "exactly-once", "source": "scenarios/03-deferred-integrity.json", "feature": 311, "role": "guarantee", "observations": ["requested", "unmentioned"], "requiresFeatures": ["ecommerce.l3.scheduled-restocks-features"] }, + { "id": "delivery-once", "stableId": "exactly-once", "source": "scenarios/03-deferred-integrity.json", "feature": 312, "role": "guarantee", "observations": ["requested", "unmentioned"], "requiresFeatures": ["ecommerce.l3.order-delivery-features"] }, + { "id": "reservation-stock", "stableId": "stock-conservation", "source": "scenarios/03-deferred-integrity.json", "feature": 313, "role": "guarantee", "observations": ["requested", "unmentioned"], "requiresFeatures": ["ecommerce.l3.reservations-features"] }, + { "id": "reserved-checkout", "stableId": "stock-conservation", "source": "scenarios/03-deferred-integrity.json", "feature": 314, "role": "guarantee", "observations": ["requested", "unmentioned"], "requiresFeatures": ["ecommerce.l3.reservations-features"] } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/l3-order-delivery-features.json b/tools/stack-bench/tracks/ecommerce/composition/packs/l3-order-delivery-features.json new file mode 100644 index 00000000000..afa0c041ad2 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/l3-order-delivery-features.json @@ -0,0 +1,62 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.l3.order-delivery-features", + "stableId": "ecommerce.l3.order-delivery", + "moduleType": "feature", + "title": "Order delivery", + "description": "Automatic delivery and final cancellation.", + "requiresPacks": [ + "ecommerce.progression.fulfilment-queue", + "ecommerce.l2.order-cancellation-features" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 190000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.l3.order-delivery", + "path": "prompts/modular/order-delivery.md", + "order": 3020, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.l3.order-delivery-hooks", + "path": "contracts/order-delivery.md", + "order": 3920, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "automatic-delivery", + "stableId": "order-delivery", + "source": "scenarios/03-order-delivery.json", + "feature": 303, + "role": "feature" + }, + { + "id": "cancel-final", + "stableId": "order-delivery", + "source": "scenarios/03-order-delivery.json", + "feature": 305, + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/l3-order-returns-features.json b/tools/stack-bench/tracks/ecommerce/composition/packs/l3-order-returns-features.json new file mode 100644 index 00000000000..6cbd1bfc377 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/l3-order-returns-features.json @@ -0,0 +1,93 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.l3.order-returns-features", + "stableId": "ecommerce.returns-pricing", + "moduleType": "feature", + "title": "Returns", + "description": "Return shipped items without losing stock or accounting accuracy.", + "requiresPacks": [ + "ecommerce.l3.order-delivery-features", + "ecommerce.feature.warehouse-admin" + ], + "conflictsWith": [], + "capabilities": [ + "browser", + "database-observation" + ], + "evidence": [ + "browser-observation", + "database-observation", + "fresh-client-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 120000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.l3.order-returns", + "path": "prompts/modular/order-returns.md", + "order": 4040, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.l3.order-return-hooks", + "path": "contracts/order-returns.md", + "order": 4041, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "return-complete", + "stableId": "cancellation-and-return", + "source": "scenarios/progression-order-return-complete.json", + "feature": 330, + "criteria": [ + "3c" + ], + "role": "feature", + "requiresFeatures": [ + "ecommerce.feature.purchasing", + "ecommerce.progression.fulfilment-queue" + ] + }, + { + "id": "return-before-shipping", + "stableId": "cancellation-and-return", + "source": "scenarios/progression-order-return-boundary.json", + "feature": 331, + "criteria": [ + "3e" + ], + "role": "feature", + "requiresFeatures": [ + "ecommerce.feature.purchasing", + "ecommerce.progression.fulfilment-queue" + ] + }, + { + "id": "return-server-boundary", + "stableId": "cancellation-and-return", + "source": "scenarios/progression-order-return-boundary.json", + "feature": 332, + "criteria": [ + "3f" + ], + "role": "feature", + "requiresFeatures": [ + "ecommerce.feature.purchasing", + "ecommerce.progression.fulfilment-queue" + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/l3-reservations-features.json b/tools/stack-bench/tracks/ecommerce/composition/packs/l3-reservations-features.json new file mode 100644 index 00000000000..cda5ba7f9dd --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/l3-reservations-features.json @@ -0,0 +1,83 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.l3.reservations-features", + "stableId": "ecommerce.l3.reservations", + "moduleType": "feature", + "title": "Reservations", + "description": "Timed cart reservations, renewal, checkout, and expiry.", + "requiresPacks": [ + "ecommerce.feature.checkout", + "ecommerce.feature.warehouse-admin" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 300000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.l3.reservations", + "path": "prompts/modular/reservations.md", + "order": 3000, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.l3.reservation-hooks", + "path": "contracts/reservations.md", + "order": 3900, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "stock-hold", + "stableId": "reservations", + "source": "scenarios/03-reservations.json", + "feature": 301, + "role": "feature" + }, + { + "id": "countdown", + "stableId": "reservations", + "source": "scenarios/03-reservations.json", + "feature": 305, + "role": "feature" + }, + { + "id": "checkout", + "stableId": "reservations", + "source": "scenarios/03-reservations.json", + "feature": 306, + "role": "feature" + }, + { + "id": "expiry", + "stableId": "reservations", + "source": "scenarios/03-reservations.json", + "feature": 307, + "role": "feature" + }, + { + "id": "renewal", + "stableId": "reservations", + "source": "scenarios/03-reservations.json", + "feature": 308, + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/l3-scheduled-restocks-features.json b/tools/stack-bench/tracks/ecommerce/composition/packs/l3-scheduled-restocks-features.json new file mode 100644 index 00000000000..2a2541b8017 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/l3-scheduled-restocks-features.json @@ -0,0 +1,68 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.l3.scheduled-restocks-features", + "stableId": "ecommerce.l3.scheduled-restocks", + "moduleType": "feature", + "title": "Scheduled restocks", + "description": "Pending, completed, and cancelled scheduled restocks.", + "requiresPacks": [ + "ecommerce.feature.warehouse-admin" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 159000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.l3.scheduled-restocks", + "path": "prompts/modular/scheduled-restocks.md", + "order": 3010, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.l3.scheduled-restock-hooks", + "path": "contracts/scheduled-restocks.md", + "order": 3910, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "pending", + "stableId": "scheduled-restocks", + "source": "scenarios/03-scheduled-restocks.json", + "feature": 302, + "role": "feature" + }, + { + "id": "applies", + "stableId": "scheduled-restocks", + "source": "scenarios/03-scheduled-restock-apply.json", + "feature": 305, + "role": "feature" + }, + { + "id": "cancels", + "stableId": "scheduled-restocks", + "source": "scenarios/03-scheduled-restock-cancel.json", + "feature": 306, + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/l3-server-time-specifications.json b/tools/stack-bench/tracks/ecommerce/composition/packs/l3-server-time-specifications.json new file mode 100644 index 00000000000..4574529b065 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/l3-server-time-specifications.json @@ -0,0 +1,25 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.l3.server-time-specifications", + "stableId": "ecommerce.l3.server-time", + "moduleType": "specification", + "title": "Server-controlled time", + "description": "Deferred work does not require a browser and does not run early.", + "requiresPacks": [], + "conflictsWith": [], + "capabilities": ["browser", "backend-lifecycle"], + "evidence": ["browser-observation", "database-observation"], + "budget": { "status": "bounded", "maxRuntimeMs": 287000 }, + "task": { + "requirements": [ + { "id": "ecommerce.l3.server-timed-restocks", "path": "prompts/modular/l3-specifications.md", "order": 3120, "from": "## Server-timed restocks", "until": "## Server-timed reservations", "modes": ["upgrade"], "requiresFeatures": ["ecommerce.l3.scheduled-restocks-features"] }, + { "id": "ecommerce.l3.server-timed-reservations", "path": "prompts/modular/l3-specifications.md", "order": 3121, "from": "## Server-timed reservations", "until": "## Deferred-work access", "modes": ["upgrade"], "requiresFeatures": ["ecommerce.l3.reservations-features"] } + ], + "contracts": [] + }, + "checks": [ + { "id": "not-early", "stableId": "server-time", "source": "scenarios/03-server-time.json", "feature": 312, "role": "guarantee", "observations": ["requested", "unmentioned"], "requiresFeatures": ["ecommerce.l3.scheduled-restocks-features"] }, + { "id": "no-browser", "stableId": "server-time", "source": "scenarios/03-server-time.json", "feature": 313, "role": "guarantee", "observations": ["requested", "unmentioned"], "requiresFeatures": ["ecommerce.l3.reservations-features"] } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/operations-access-features.json b/tools/stack-bench/tracks/ecommerce/composition/packs/operations-access-features.json new file mode 100644 index 00000000000..3bc99fb7830 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/operations-access-features.json @@ -0,0 +1,34 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.operations-access-features", + "stableId": "ecommerce.operations-access", + "moduleType": "feature", + "title": "Fulfilment operations", + "description": "Customer, staff, and administrator fulfilment workflows.", + "requiresPacks": [ + "ecommerce.feature.catalog-discovery", + "ecommerce.feature.purchasing", + "ecommerce.feature.checkout", + "ecommerce.feature.warehouse-admin" + ], + "conflictsWith": [], + "capabilities": ["browser", "direct-database-write", "direct-server-call"], + "evidence": ["browser-observation", "server-refusal"], + "budget": { "status": "bounded", "maxRuntimeMs": 76000 }, + "task": { + "requirements": [ + { "id": "ecommerce.l2.staff-fulfilment", "path": "prompts/02-operations.md", "order": 20, "from": "## New: warehouse staff", "until": "- Staff cannot change prices", "modes": ["upgrade"] }, + { "id": "ecommerce.l2.fulfilment-feature", "path": "prompts/02-operations.md", "order": 22, "from": "## Features", "until": "The\nsame authentication and staff-only authorization", "modes": ["upgrade"] } + ], + "contracts": [ + { "id": "ecommerce.l2.fulfilment-hooks", "path": "contracts/appendix-02.md", "order": 2020, "from": "| `staff-link`", "until": "| `transfer-from`", "modes": ["upgrade"] }, + { "id": "ecommerce.l2.operator-action-inputs", "path": "prompts/l2-action-inputs.md", "order": 2080, "until": "## Price action input", "modes": ["upgrade"] } + ] + }, + "checks": [ + { "id": "fulfilment-core", "stableId": "fulfilment-queue", "source": "scenarios/02-features.json", "feature": 1, "criteria": ["1a", "1d"], "role": "feature" }, + { "id": "fulfilment-warehouse", "stableId": "fulfilment-queue", "source": "scenarios/02-queue-warehouse.json", "feature": 1, "criteria": ["1b"], "role": "feature" }, + { "id": "fulfilment-independent", "stableId": "fulfilment-queue", "source": "scenarios/02-self-contained.json", "feature": 1, "criteria": ["1c", "1e"], "role": "feature" } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/operations-access-specifications.json b/tools/stack-bench/tracks/ecommerce/composition/packs/operations-access-specifications.json new file mode 100644 index 00000000000..604553cf90c --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/operations-access-specifications.json @@ -0,0 +1,27 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.operations-access-specifications", + "stableId": "ecommerce.operations-access", + "moduleType": "specification", + "title": "Operator authorization", + "description": "Server-enforced operator permissions and customer order ownership.", + "requiresPacks": [], + "conflictsWith": [], + "capabilities": ["browser", "direct-server-call", "request-replay"], + "evidence": ["browser-observation", "server-refusal", "server-response"], + "budget": { "status": "bounded", "maxRuntimeMs": 84000 }, + "task": { + "requirements": [ + { "id": "ecommerce.l2.operator-authorization", "path": "prompts/02-operations.md", "order": 21, "from": "- Staff cannot change prices", "until": "## Features", "modes": ["upgrade"], "requiresFeatures": ["ecommerce.operations-access-features"] }, + { "id": "ecommerce.l2.shipping-authorization", "path": "prompts/02-operations.md", "order": 23, "from": "The\nsame authentication and staff-only authorization", "until": "### Moving stock", "modes": ["upgrade"], "requiresFeatures": ["ecommerce.operations-access-features"] } + ], + "contracts": [] + }, + "checks": [ + { "id": "operator-transfer-authorization", "stableId": "operator-authorization", "source": "scenarios/02-strengthened.json", "feature": 201, "criteria": ["201a"], "role": "guarantee", "observations": ["requested", "unmentioned"], "requiresFeatures": ["ecommerce.inventory-operations-features"] }, + { "id": "operator-price-authorization", "stableId": "operator-authorization", "source": "scenarios/02-strengthened.json", "feature": 201, "criteria": ["201b"], "role": "guarantee", "observations": ["requested", "unmentioned"], "requiresFeatures": ["ecommerce.returns-pricing-features"] }, + { "id": "operator-authorization-direct", "stableId": "operator-authorization", "source": "scenarios/02-server-actions.json", "feature": 201, "criteria": ["201c"], "role": "guarantee", "observations": ["requested", "unmentioned"], "requiresFeatures": ["ecommerce.operations-access-features", "ecommerce.feature.purchasing"] }, + { "id": "order-owner-direct", "stableId": "order-owner", "source": "scenarios/02-server-actions.json", "feature": 204, "criteria": ["204a"], "role": "guarantee", "observations": ["requested", "unmentioned"], "requiresFeatures": ["ecommerce.operations-access-features", "ecommerce.feature.purchasing"] } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-automatic-reorder.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-automatic-reorder.json new file mode 100644 index 00000000000..91157844bac --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-automatic-reorder.json @@ -0,0 +1,61 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.progression.automatic-reorder", + "moduleType": "feature", + "title": "Automatic reorder rules", + "description": "Threshold rules schedule one pending restock.", + "requiresPacks": [ + "ecommerce.feature.purchasing", + "ecommerce.l3.scheduled-restocks-features", + "ecommerce.progression.staff-roles" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 90000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.progression.automatic-reorder", + "path": "prompts/modular/progression-automatic-reorder.md", + "order": 5010, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.progression.automatic-reorder-hooks", + "path": "contracts/progression-automatic-reorder.md", + "order": 5910, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "threshold", + "stableId": "automatic-reorder", + "source": "scenarios/progression-automatic-reorder.json", + "feature": 502, + "criteria": [ + "502a" + ], + "role": "feature", + "requiresFeatures": [ + "ecommerce.feature.purchasing" + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-cancellation-accounting-specifications.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-cancellation-accounting-specifications.json new file mode 100644 index 00000000000..77424595a39 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-cancellation-accounting-specifications.json @@ -0,0 +1,21 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.progression.cancellation-accounting-specifications", + "stableId": "ecommerce.returns-pricing", + "moduleType": "specification", + "title": "Cancellation accounting", + "description": "Cancellation removes the exact cancelled amount from revenue.", + "requiresPacks": [], + "conflictsWith": [], + "capabilities": ["browser", "concurrent-actors", "direct-server-call"], + "evidence": ["browser-observation", "concurrent-outcome", "database-observation"], + "budget": { "status": "bounded", "maxRuntimeMs": 46000 }, + "task": { + "requirements": [{ "id": "ecommerce.progression.cancellation-accounting", "path": "prompts/modular/order-accounting.md", "order": 2100, "from": "## Cancellation accounting", "until": "## Price accounting", "modes": ["upgrade"], "requiresFeatures": ["ecommerce.l2.order-cancellation-features"] }], + "contracts": [] + }, + "checks": [ + { "id": "cancellation-accounting", "stableId": "refund-accounting", "source": "scenarios/02-invariants.json", "feature": 203, "criteria": ["203a"], "role": "guarantee", "observations": ["requested", "unmentioned"], "requiresFeatures": ["ecommerce.l2.order-cancellation-features"] } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-cancellation-queue-specifications.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-cancellation-queue-specifications.json new file mode 100644 index 00000000000..b4d3a523b22 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-cancellation-queue-specifications.json @@ -0,0 +1,23 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.progression.cancellation-queue-specifications", + "stableId": "ecommerce.returns-pricing", + "moduleType": "specification", + "title": "Cancellation and fulfilment", + "description": "Cancellation leaves the live queue and remains consistent with shipping.", + "requiresPacks": [], + "conflictsWith": [], + "capabilities": ["browser", "direct-server-call", "database-observation", "concurrent-actors"], + "evidence": ["browser-observation", "database-observation", "concurrent-outcome"], + "budget": { "status": "bounded", "maxRuntimeMs": 90000 }, + "task": { + "requirements": [ + { "id": "ecommerce.progression.cancellation-queue", "path": "prompts/modular/cancellation-queue.md", "order": 2045, "modes": ["upgrade"], "requiresFeatures": ["ecommerce.l2.order-cancellation-features", "ecommerce.progression.fulfilment-queue"] } + ], + "contracts": [] + }, + "checks": [ + { "id": "queue-removal", "stableId": "cancellation-and-return", "source": "scenarios/02-cancellation-queue.json", "feature": 3, "criteria": ["3d"], "role": "guarantee", "observations": ["requested", "unmentioned"], "requiresFeatures": ["ecommerce.l2.order-cancellation-features", "ecommerce.progression.fulfilment-queue", "ecommerce.feature.purchasing", "ecommerce.feature.warehouse-admin"] } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-cart-recovery.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-cart-recovery.json new file mode 100644 index 00000000000..e42ebdddb25 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-cart-recovery.json @@ -0,0 +1,66 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.progression.cart-recovery", + "moduleType": "feature", + "title": "Cart recovery", + "description": "Safe restoration of available items from an expired cart.", + "requiresPacks": [ + "ecommerce.l3.cart-expiration-features" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 350000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.progression.cart-recovery", + "path": "prompts/modular/progression-cart-recovery.md", + "order": 5020, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.progression.cart-recovery-hooks", + "path": "contracts/progression-cart-recovery.md", + "order": 5920, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "available", + "stableId": "cart-recovery", + "source": "scenarios/progression-cart-recovery.json", + "feature": 503, + "criteria": [ + "503a" + ], + "role": "feature" + }, + { + "id": "partial", + "stableId": "cart-recovery", + "source": "scenarios/progression-cart-recovery.json", + "feature": 503, + "criteria": [ + "503b" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-catalog-management.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-catalog-management.json new file mode 100644 index 00000000000..9e45940e322 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-catalog-management.json @@ -0,0 +1,12 @@ +{ + "schemaVersion": 1, "kind": "test-pack", "id": "ecommerce.progression.catalog-management", "moduleType": "feature", + "title": "Catalog management", "description": "Staff-managed products and named variants.", + "requiresPacks": ["ecommerce.feature.catalog-discovery", "ecommerce.progression.staff-roles"], + "conflictsWith": [], "capabilities": ["browser"], "evidence": ["browser-observation"], + "budget": { "status": "bounded", "maxRuntimeMs": 16000 }, + "task": { + "requirements": [{ "id": "ecommerce.progression.catalog-management", "path": "prompts/modular/catalog-management.md", "order": 1300, "modes": ["upgrade"] }], + "contracts": [{ "id": "ecommerce.progression.catalog-management-hooks", "path": "contracts/catalog-management.md", "order": 1970, "modes": ["upgrade"] }] + }, + "checks": [{ "id": "products", "stableId": "catalog-management", "source": "scenarios/progression-catalog-management.json", "feature": 622, "role": "feature" }] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-customer-profile.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-customer-profile.json new file mode 100644 index 00000000000..e34803d4ab6 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-customer-profile.json @@ -0,0 +1,11 @@ +{ + "schemaVersion": 1, "kind": "test-pack", "id": "ecommerce.progression.customer-profile", "moduleType": "feature", + "title": "Customer profile", "description": "Customer details and shipping address.", + "requiresPacks": ["ecommerce.feature.accounts"], "conflictsWith": [], "capabilities": ["browser"], "evidence": ["browser-observation"], + "budget": { "status": "bounded", "maxRuntimeMs": 18000 }, + "task": { + "requirements": [{ "id": "ecommerce.progression.customer-profile", "path": "prompts/modular/customer-profile.md", "order": 1200, "modes": ["upgrade"] }], + "contracts": [{ "id": "ecommerce.progression.customer-profile-hooks", "path": "contracts/customer-profile.md", "order": 1950, "modes": ["upgrade"] }] + }, + "checks": [{ "id": "profile", "stableId": "customer-profile", "source": "scenarios/progression-customer-profile.json", "feature": 620, "criteria": ["620c"], "role": "feature" }] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-delivery-notifications.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-delivery-notifications.json new file mode 100644 index 00000000000..d2e07c0f80e --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-delivery-notifications.json @@ -0,0 +1,61 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.progression.delivery-notifications", + "moduleType": "feature", + "title": "Delivery notifications", + "description": "Exactly one private notification for a delivered order.", + "requiresPacks": [ + "ecommerce.l3.order-delivery-features", + "ecommerce.progression.notification-preferences" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 110000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.progression.delivery-notifications", + "path": "prompts/modular/delivery-notifications.md", + "order": 5000, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.progression.delivery-notification-hooks", + "path": "contracts/delivery-notifications.md", + "order": 5900, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "delivery", + "stableId": "delivery-notification-delivery", + "source": "scenarios/progression-delivery-notifications.json", + "feature": 501, + "criteria": [ + "501a" + ], + "role": "feature", + "requiresFeatures": [ + "ecommerce.feature.purchasing", + "ecommerce.progression.fulfilment-queue" + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-faceted-search.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-faceted-search.json new file mode 100644 index 00000000000..83b853ee0ae --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-faceted-search.json @@ -0,0 +1,66 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.progression.faceted-search", + "moduleType": "feature", + "title": "Faceted search", + "description": "Combined catalogue filters with stable pagination.", + "requiresPacks": [ + "ecommerce.feature.catalog-discovery" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 147000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.progression.faceted-search", + "path": "prompts/modular/faceted-search.md", + "order": 4000, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.progression.faceted-search-hooks", + "path": "contracts/faceted-search.md", + "order": 4900, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "filters", + "stableId": "faceted-search", + "source": "scenarios/progression-faceted-filters.json", + "feature": 401, + "criteria": [ + "401a" + ], + "role": "feature" + }, + { + "id": "pagination", + "stableId": "faceted-search", + "source": "scenarios/progression-faceted-pagination.json", + "feature": 402, + "criteria": [ + "402a" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-fulfilment-queue.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-fulfilment-queue.json new file mode 100644 index 00000000000..d739dc16ae4 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-fulfilment-queue.json @@ -0,0 +1,71 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.progression.fulfilment-queue", + "stableId": "ecommerce.operations-access", + "moduleType": "feature", + "title": "Fulfilment queue", + "description": "Staff and administrator fulfilment workflows.", + "requiresPacks": [ + "ecommerce.feature.purchasing", + "ecommerce.feature.warehouse-admin" + ], + "conflictsWith": [], + "capabilities": [ + "browser", + "direct-server-call" + ], + "evidence": [ + "browser-observation", + "server-refusal" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 105000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.progression.fulfilment-queue", + "path": "prompts/modular/operations-access.md", + "order": 2020, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.progression.fulfilment-queue-hooks", + "path": "contracts/operations-access.md", + "order": 2920, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "fulfilment-warehouse", + "stableId": "fulfilment-queue", + "source": "scenarios/02-queue-warehouse.json", + "feature": 1, + "criteria": [ + "1b" + ], + "role": "feature" + }, + { + "id": "fulfilment-ship", + "stableId": "fulfilment-queue", + "source": "scenarios/02-fulfilment-ship.json", + "feature": 1, + "criteria": [ + "1c" + ], + "role": "feature", + "requiresFeatures": ["ecommerce.feature.purchasing"] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-inventory-conservation-specifications.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-inventory-conservation-specifications.json new file mode 100644 index 00000000000..4f89c9ba694 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-inventory-conservation-specifications.json @@ -0,0 +1,27 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.progression.inventory-conservation-specifications", + "stableId": "ecommerce.inventory-operations", + "moduleType": "specification", + "title": "Inventory conservation", + "description": "Stock remains exact across transfers, cancellations, clients, and overlapping writes.", + "requiresPacks": [], + "conflictsWith": [], + "capabilities": ["browser", "concurrent-actors", "direct-server-call", "database-observation"], + "evidence": ["browser-observation", "server-response", "database-observation"], + "budget": { "status": "bounded", "maxRuntimeMs": 325000 }, + "task": { + "requirements": [ + { "id": "ecommerce.progression.transfer-conservation", "path": "prompts/modular/stock-conservation.md", "order": 2970, "modes": ["upgrade"], "requiresFeatures": ["ecommerce.l2.stock-transfers-features"] }, + { "id": "ecommerce.progression.cancellation-conservation", "path": "prompts/modular/cancellation-conservation.md", "order": 2980, "modes": ["upgrade"], "requiresFeatures": ["ecommerce.l2.order-cancellation-features"] } + ], + "contracts": [] + }, + "checks": [ + { "id": "shipping-accounting", "stableId": "shipping-accounting", "source": "scenarios/progression-shipping-accounting.json", "feature": 202, "criteria": ["202e"], "role": "guarantee", "observations": ["requested", "unmentioned"], "requiresFeatures": ["ecommerce.progression.fulfilment-queue", "ecommerce.feature.purchasing", "ecommerce.feature.warehouse-admin"] }, + { "id": "stock-conservation-cancel", "stableId": "stock-conservation", "source": "scenarios/02-self-contained.json", "feature": 202, "criteria": ["202b"], "role": "guarantee", "observations": ["requested", "unmentioned"], "requiresFeatures": ["ecommerce.l2.order-cancellation-features"] }, + { "id": "stock-conservation-contention", "stableId": "stock-conservation", "source": "scenarios/02-server-actions.json", "feature": 202, "criteria": ["202d"], "role": "guarantee", "observations": ["requested", "unmentioned"], "requiresFeatures": ["ecommerce.l2.stock-transfers-features", "ecommerce.feature.purchasing"] }, + { "id": "stock-conservation-history", "stableId": "stock-conservation", "source": "scenarios/mixed-operation-history.json", "feature": 202, "criteria": ["202f"], "role": "guarantee", "observations": ["requested", "unmentioned"], "requiresFeatures": ["ecommerce.feature.accounts", "ecommerce.feature.catalog-items", "ecommerce.feature.purchasing", "ecommerce.feature.cart", "ecommerce.feature.checkout", "ecommerce.feature.warehouse-admin", "ecommerce.l2.stock-transfers-features", "ecommerce.l2.order-cancellation-features"] } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-managed-support.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-managed-support.json new file mode 100644 index 00000000000..b054876465a --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-managed-support.json @@ -0,0 +1,44 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.progression.managed-support", + "moduleType": "feature", + "title": "Managed support cases", + "description": "Customer and staff support case updates.", + "requiresPacks": [ + "ecommerce.progression.support-history", + "ecommerce.progression.support-triage" + ], + "conflictsWith": [], + "capabilities": ["browser"], + "evidence": ["browser-observation"], + "budget": { "status": "bounded", "maxRuntimeMs": 29000 }, + "task": { + "requirements": [ + { + "id": "ecommerce.progression.managed-support", + "path": "prompts/modular/managed-support.md", + "order": 3100, + "modes": ["upgrade"] + } + ], + "contracts": [ + { + "id": "ecommerce.progression.managed-support-hooks", + "path": "contracts/managed-support.md", + "order": 3900, + "modes": ["upgrade"] + } + ] + }, + "checks": [ + { + "id": "case-updates", + "stableId": "managed-support", + "source": "scenarios/progression-managed-support-shared.json", + "feature": 613, + "criteria": ["613c"], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-notification-preferences.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-notification-preferences.json new file mode 100644 index 00000000000..61b3b08a5b9 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-notification-preferences.json @@ -0,0 +1,56 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.progression.notification-preferences", + "moduleType": "feature", + "title": "Notification preferences", + "description": "Customer notification choices.", + "requiresPacks": [ + "ecommerce.feature.accounts" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 21000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.progression.notification-preferences", + "path": "prompts/modular/notification-preferences.md", + "order": 2300, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.progression.notification-preferences-hooks", + "path": "contracts/notification-preferences.md", + "order": 2990, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "preferences", + "stableId": "notification-preferences", + "source": "scenarios/progression-notification-preferences.json", + "feature": 630, + "criteria": [ + "630c" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-operations-access-specifications.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-operations-access-specifications.json new file mode 100644 index 00000000000..84b258c9fd7 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-operations-access-specifications.json @@ -0,0 +1,29 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.progression.operations-access-specifications", + "stableId": "ecommerce.operations-access", + "moduleType": "specification", + "title": "Server-enforced operations access", + "description": "Direct authorization checks owned by the product work they protect.", + "requiresPacks": [], + "conflictsWith": [], + "capabilities": ["browser", "direct-server-call", "request-replay"], + "evidence": ["server-refusal", "server-response"], + "budget": { "status": "bounded", "maxRuntimeMs": 184000 }, + "task": { + "requirements": [ + { "id": "ecommerce.progression.transfer-authorization", "path": "prompts/modular/transfer-authorization.md", "order": 2930, "modes": ["upgrade"], "requiresFeatures": ["ecommerce.l2.stock-transfers-features"] }, + { "id": "ecommerce.progression.price-authorization", "path": "prompts/modular/price-authorization.md", "order": 2940, "modes": ["upgrade"], "requiresFeatures": ["ecommerce.l2.price-history-features"] }, + { "id": "ecommerce.progression.shipping-authorization", "path": "prompts/modular/shipping-authorization.md", "order": 2950, "modes": ["upgrade"], "requiresFeatures": ["ecommerce.progression.fulfilment-queue"] }, + { "id": "ecommerce.progression.order-ownership", "path": "prompts/modular/order-ownership.md", "order": 2960, "modes": ["upgrade"], "requiresFeatures": ["ecommerce.l2.order-cancellation-features"] } + ], + "contracts": [] + }, + "checks": [ + { "id": "operator-transfer-authorization", "stableId": "operator-authorization", "source": "scenarios/02-strengthened.json", "feature": 201, "criteria": ["201a"], "role": "guarantee", "observations": ["requested", "unmentioned"], "requiresFeatures": ["ecommerce.l2.stock-transfers-features"] }, + { "id": "operator-price-authorization", "stableId": "operator-authorization", "source": "scenarios/02-strengthened.json", "feature": 201, "criteria": ["201b"], "role": "guarantee", "observations": ["requested", "unmentioned"], "requiresFeatures": ["ecommerce.l2.price-history-features"] }, + { "id": "operator-authorization-direct", "stableId": "operator-authorization", "source": "scenarios/02-server-actions.json", "feature": 201, "criteria": ["201c"], "role": "guarantee", "observations": ["requested", "unmentioned"], "requiresFeatures": ["ecommerce.progression.fulfilment-queue", "ecommerce.feature.purchasing"] }, + { "id": "order-owner-direct", "stableId": "order-owner", "source": "scenarios/02-server-actions.json", "feature": 204, "criteria": ["204a"], "role": "guarantee", "observations": ["requested", "unmentioned"], "requiresFeatures": ["ecommerce.l2.order-cancellation-features", "ecommerce.feature.purchasing"] } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-order-support.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-order-support.json new file mode 100644 index 00000000000..502c01bd33e --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-order-support.json @@ -0,0 +1,57 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.progression.order-support", + "moduleType": "feature", + "title": "Order-linked support", + "description": "Owned order links that authorized staff can inspect.", + "requiresPacks": [ + "ecommerce.progression.managed-support", + "ecommerce.feature.purchasing" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 65000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.progression.order-support", + "path": "prompts/modular/order-support.md", + "order": 4100, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.progression.order-support-hooks", + "path": "contracts/order-support.md", + "order": 4900, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "owned-order", + "stableId": "order-support-owned", + "source": "scenarios/progression-order-support-owned.json", + "feature": 614, + "criteria": [ + "614a" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-payment-records.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-payment-records.json new file mode 100644 index 00000000000..cc9c3ef236f --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-payment-records.json @@ -0,0 +1,57 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.progression.payment-records", + "moduleType": "feature", + "title": "Payment records", + "description": "Exact and non-duplicated payment records at checkout.", + "requiresPacks": [ + "ecommerce.feature.checkout", + "ecommerce.feature.purchasing" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 30000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.progression.payment-records", + "path": "prompts/modular/payment-records.md", + "order": 1350, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.progression.payment-record-hooks", + "path": "contracts/payment-records.md", + "order": 1980, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "payment", + "stableId": "payment-records", + "source": "scenarios/progression-core-business.json", + "feature": 623, + "role": "feature", + "criteria": [ + "623a" + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-personalized-recommendations.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-personalized-recommendations.json new file mode 100644 index 00000000000..7eb6aa35711 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-personalized-recommendations.json @@ -0,0 +1,56 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.progression.personalized-recommendations", + "moduleType": "feature", + "title": "Personalized recommendations", + "description": "Deterministic recommendations isolated by customer.", + "requiresPacks": [ + "ecommerce.l2.recommendations" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 70000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.progression.personalized-recommendations", + "path": "prompts/modular/progression-personalized-recommendations.md", + "order": 4010, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.progression.personalized-recommendation-hooks", + "path": "contracts/progression-personalized-recommendations.md", + "order": 4910, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "ordering", + "stableId": "personalized-recommendations", + "source": "scenarios/progression-personalized-recommendations.json", + "feature": 403, + "criteria": [ + "403a" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-price-accounting-specifications.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-price-accounting-specifications.json new file mode 100644 index 00000000000..afbbe337acb --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-price-accounting-specifications.json @@ -0,0 +1,21 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.progression.price-accounting-specifications", + "stableId": "ecommerce.returns-pricing", + "moduleType": "specification", + "title": "Price accounting", + "description": "Later price changes do not rewrite recorded order value.", + "requiresPacks": [], + "conflictsWith": [], + "capabilities": ["browser"], + "evidence": ["browser-observation"], + "budget": { "status": "bounded", "maxRuntimeMs": 24000 }, + "task": { + "requirements": [{ "id": "ecommerce.progression.price-accounting", "path": "prompts/modular/order-accounting.md", "order": 2110, "from": "## Price accounting", "modes": ["upgrade"], "requiresFeatures": ["ecommerce.l2.price-history-features"] }], + "contracts": [] + }, + "checks": [ + { "id": "price-accounting", "stableId": "refund-accounting", "source": "scenarios/02-invariants.json", "feature": 203, "criteria": ["203b"], "role": "guarantee", "observations": ["requested", "unmentioned"], "requiresFeatures": ["ecommerce.l2.price-history-features"] } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-price-history-specifications.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-price-history-specifications.json new file mode 100644 index 00000000000..c3281cceffa --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-price-history-specifications.json @@ -0,0 +1,76 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.progression.price-history-specifications", + "stableId": "ecommerce.returns-pricing", + "moduleType": "specification", + "title": "Price history order behavior", + "description": "Orders and carts keep the correct price through later changes.", + "requiresPacks": [], + "conflictsWith": [], + "capabilities": ["browser","direct-server-call"], + "evidence": ["browser-observation","fresh-client-observation"], + "budget": { "status": "bounded", "maxRuntimeMs": 94000 }, + "task": { + "requirements": [ + { + "id": "ecommerce.progression.price-history-orders", + "path": "prompts/modular/price-history-orders.md", + "order": 2041, + "from": "## Price history: completed orders", + "until": "## Price history: cart and checkout", + "modes": ["upgrade"], + "requiresFeatures": ["ecommerce.feature.purchasing", "ecommerce.l2.price-history-features"] + }, + { + "id": "ecommerce.progression.price-history-cart-checkout", + "path": "prompts/modular/price-history-orders.md", + "order": 2042, + "from": "## Price history: cart and checkout", + "modes": ["upgrade"], + "requiresFeatures": ["ecommerce.feature.checkout", "ecommerce.l2.price-history-features"] + } + ], + "contracts": [ + { + "id": "ecommerce.progression.price-history-order-hooks", + "path": "contracts/price-history-orders.md", + "order": 2042, + "from": "# Price history completed-order interface", + "until": "# Price history cart and checkout interface", + "modes": ["upgrade"], + "requiresFeatures": ["ecommerce.feature.purchasing", "ecommerce.l2.price-history-features"] + }, + { + "id": "ecommerce.progression.price-history-cart-hooks", + "path": "contracts/price-history-orders.md", + "order": 2043, + "from": "# Price history cart and checkout interface", + "modes": ["upgrade"], + "requiresFeatures": ["ecommerce.feature.checkout", "ecommerce.l2.price-history-features"] + } + ] + }, + "checks": [ + { + "id": "price-history-exact", + "stableId": "price-history", + "source": "scenarios/02-paid-price-history.json", + "feature": 4, + "criteria": ["4a"], + "role": "guarantee", + "observations": ["requested", "unmentioned"], + "requiresFeatures": ["ecommerce.feature.purchasing", "ecommerce.l2.price-history-features"] + }, + { + "id": "price-cart-checkout", + "stableId": "price-history", + "source": "scenarios/progression-price-cart-checkout.json", + "feature": 420, + "criteria": ["4c"], + "role": "guarantee", + "observations": ["requested", "unmentioned"], + "requiresFeatures": ["ecommerce.feature.checkout", "ecommerce.l2.price-history-features"] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-promotion-checkout.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-promotion-checkout.json new file mode 100644 index 00000000000..c90eaf53fa8 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-promotion-checkout.json @@ -0,0 +1,77 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.progression.promotion-checkout", + "moduleType": "feature", + "title": "Promotion checkout", + "description": "Server-calculated discounts recorded on orders.", + "requiresPacks": [ + "ecommerce.progression.promotion-rules", + "ecommerce.feature.checkout" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 55000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.progression.promotion-checkout", + "path": "prompts/modular/promotion-checkout.md", + "order": 3200, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.progression.promotion-checkout-hooks", + "path": "contracts/promotion-checkout.md", + "order": 3950, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "active", + "stableId": "promotion-checkout-active", + "source": "scenarios/progression-promotion-checkout.json", + "feature": 621, + "criteria": [ + "621a" + ], + "role": "feature" + }, + { + "id": "expired", + "stableId": "promotion-checkout-expired", + "source": "scenarios/progression-promotion-checkout.json", + "feature": 621, + "criteria": [ + "621b" + ], + "role": "feature" + }, + { + "id": "exhausted", + "stableId": "promotion-checkout-exhausted", + "source": "scenarios/progression-promotion-checkout.json", + "feature": 621, + "criteria": [ + "621c" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-promotion-reporting.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-promotion-reporting.json new file mode 100644 index 00000000000..7fc0f16b8db --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-promotion-reporting.json @@ -0,0 +1,66 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.progression.promotion-reporting", + "moduleType": "feature", + "title": "Promotion reporting", + "description": "Exact promotion redemptions and revenue.", + "requiresPacks": [ + "ecommerce.progression.promotion-checkout" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 40000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.progression.promotion-reporting", + "path": "prompts/modular/promotion-reporting.md", + "order": 4200, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.progression.promotion-reporting-hooks", + "path": "contracts/promotion-reporting.md", + "order": 4950, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "redemptions", + "stableId": "promotion-report-redemptions", + "source": "scenarios/progression-promotion-reporting.json", + "feature": 622, + "criteria": [ + "622a" + ], + "role": "feature" + }, + { + "id": "revenue", + "stableId": "promotion-report-revenue", + "source": "scenarios/progression-promotion-reporting.json", + "feature": 622, + "criteria": [ + "622b" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-promotion-rules.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-promotion-rules.json new file mode 100644 index 00000000000..0d4d7707ccb --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-promotion-rules.json @@ -0,0 +1,57 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.progression.promotion-rules", + "moduleType": "feature", + "title": "Promotion rules", + "description": "Staff-managed bounded promotion codes.", + "requiresPacks": [ + "ecommerce.progression.staff-access", + "ecommerce.feature.catalog-items" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 15000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.progression.promotion-rules", + "path": "prompts/modular/promotion-rules.md", + "order": 2200, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.progression.promotion-rules-hooks", + "path": "contracts/promotion-rules.md", + "order": 2950, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "values", + "stableId": "promotion-rule-values", + "source": "scenarios/progression-promotion-rules.json", + "feature": 620, + "criteria": [ + "620a" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-recommendation-feedback.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-recommendation-feedback.json new file mode 100644 index 00000000000..dec08fff3e0 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-recommendation-feedback.json @@ -0,0 +1,52 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.progression.recommendation-feedback", + "moduleType": "feature", + "title": "Recommendation feedback", + "description": "Durable customer-specific recommendation dismissal.", + "requiresPacks": [ + "ecommerce.progression.personalized-recommendations" + ], + "conflictsWith": [], + "capabilities": ["browser","backend-lifecycle"], + "evidence": ["browser-observation","fresh-client-observation"], + "budget": { + "status": "bounded", + "maxRuntimeMs": 130000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.progression.recommendation-feedback", + "path": "prompts/modular/recommendation-feedback.md", + "order": 5030, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.progression.recommendation-feedback-hooks", + "path": "contracts/recommendation-feedback.md", + "order": 5930, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "dismissal", + "stableId": "recommendation-feedback", + "source": "scenarios/progression-recommendation-feedback.json", + "feature": 504, + "role": "feature", + "criteria": [ + "504a" + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-review-access-specifications.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-review-access-specifications.json new file mode 100644 index 00000000000..4bf1c1bd45e --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-review-access-specifications.json @@ -0,0 +1,49 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.progression.review-access-specifications", + "moduleType": "specification", + "title": "Review safety", + "description": "Server-enforced review eligibility and safe display of stored review content.", + "requiresPacks": [], + "conflictsWith": [], + "capabilities": ["browser", "direct-server-call", "request-replay"], + "evidence": ["browser-observation", "server-refusal", "server-response", "fresh-client-observation"], + "budget": { + "status": "bounded", + "maxRuntimeMs": 68000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.progression.review-access", + "path": "prompts/modular/review-access.md", + "order": 2970, + "modes": ["upgrade"], + "requiresFeatures": ["ecommerce.feature.reviews"] + } + ], + "contracts": [] + }, + "checks": [ + { + "id": "review-eligibility-direct", + "stableId": "review-eligibility-direct", + "source": "scenarios/progression-review-access.json", + "feature": 618, + "criteria": ["618a"], + "role": "guarantee", + "observations": ["requested", "unmentioned"], + "requiresFeatures": ["ecommerce.feature.reviews", "ecommerce.feature.purchasing"] + }, + { + "id": "stored-review-script", + "source": "scenarios/progression-review-script.json", + "feature": 9180, + "criteria": ["9180a"], + "role": "guarantee", + "observations": ["unmentioned"], + "requiresFeatures": ["ecommerce.feature.reviews", "ecommerce.feature.purchasing"] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-staff-access.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-staff-access.json new file mode 100644 index 00000000000..9e6d1f7b086 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-staff-access.json @@ -0,0 +1,11 @@ +{ + "schemaVersion": 1, "kind": "test-pack", "id": "ecommerce.progression.staff-access", "moduleType": "feature", + "title": "Staff access", "description": "Staff and administrator sign-in.", + "requiresPacks": [], "conflictsWith": [], "capabilities": ["browser"], "evidence": ["browser-observation"], + "budget": { "status": "bounded", "maxRuntimeMs": 10000 }, + "task": { + "requirements": [{ "id": "ecommerce.progression.staff-access", "path": "prompts/modular/staff-access.md", "order": 1100, "modes": ["fresh", "upgrade"] }], + "contracts": [{ "id": "ecommerce.progression.staff-access-hooks", "path": "contracts/staff-access.md", "order": 1900, "modes": ["fresh", "upgrade"] }] + }, + "checks": [{ "id": "staff-sign-in", "stableId": "staff-access", "source": "scenarios/progression-staff-access.json", "feature": 601, "criteria": ["601a"], "role": "feature" }] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-staff-activity.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-staff-activity.json new file mode 100644 index 00000000000..7693169aabc --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-staff-activity.json @@ -0,0 +1,12 @@ +{ + "schemaVersion": 1, "kind": "test-pack", "id": "ecommerce.progression.staff-activity", "moduleType": "feature", + "title": "Staff activity history", "description": "Attributable and private history of administrative changes.", + "requiresPacks": ["ecommerce.progression.staff-roles", "ecommerce.progression.catalog-management"], + "conflictsWith": [], "capabilities": ["browser"], "evidence": ["browser-observation"], + "budget": { "status": "bounded", "maxRuntimeMs": 30000 }, + "task": { + "requirements": [{ "id": "ecommerce.progression.staff-activity", "path": "prompts/modular/staff-activity.md", "order": 1400, "modes": ["upgrade"] }], + "contracts": [{ "id": "ecommerce.progression.staff-activity-hooks", "path": "contracts/staff-activity.md", "order": 1990, "modes": ["upgrade"] }] + }, + "checks": [{"id":"activity","stableId":"staff-activity","source":"scenarios/progression-staff-activity.json","feature":624,"role":"feature","criteria":["624a"]}] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-staff-roles.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-staff-roles.json new file mode 100644 index 00000000000..b00fe9f4649 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-staff-roles.json @@ -0,0 +1,56 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.progression.staff-roles", + "moduleType": "feature", + "title": "Staff roles", + "description": "Role assignment for staff accounts.", + "requiresPacks": [ + "ecommerce.progression.staff-access" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 15000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.progression.staff-roles", + "path": "prompts/modular/staff-roles.md", + "order": 1250, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.progression.staff-role-hooks", + "path": "contracts/staff-roles.md", + "order": 1960, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "assignment", + "stableId": "staff-roles", + "source": "scenarios/progression-staff-roles.json", + "feature": 621, + "criteria": [ + "621c" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-stock-alerts.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-stock-alerts.json new file mode 100644 index 00000000000..cb21c00e71f --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-stock-alerts.json @@ -0,0 +1,59 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.progression.stock-alerts", + "moduleType": "feature", + "title": "Stock alerts", + "description": "Customer alerts when stock returns.", + "requiresPacks": [ + "ecommerce.progression.notification-preferences", + "ecommerce.feature.warehouse-admin" + ], + "conflictsWith": [], + "capabilities": [ + "browser", + "direct-database-write" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 227000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.progression.stock-alerts", + "path": "prompts/modular/stock-alerts.md", + "order": 3300, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.progression.stock-alert-hooks", + "path": "contracts/stock-alerts.md", + "order": 3990, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "delivery", + "stableId": "stock-alert-delivery", + "source": "scenarios/progression-stock-alert-delivery.json", + "feature": 631, + "criteria": [ + "631c" + ], + "role": "feature", + "requiresFeatures": ["ecommerce.feature.warehouse-admin"] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-support-history.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-support-history.json new file mode 100644 index 00000000000..c7342e32326 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-support-history.json @@ -0,0 +1,57 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.progression.support-history", + "moduleType": "feature", + "title": "Customer support history", + "description": "Customer support ticket history.", + "requiresPacks": [ + "ecommerce.feature.accounts", + "ecommerce.progression.support-intake" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 21000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.progression.support-history", + "path": "prompts/modular/support-history.md", + "order": 2110, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.progression.support-history-hooks", + "path": "contracts/support-history.md", + "order": 2900, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "history", + "stableId": "support-history", + "source": "scenarios/progression-support-history.json", + "feature": 612, + "criteria": [ + "612c" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-support-intake.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-support-intake.json new file mode 100644 index 00000000000..50a4673875c --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-support-intake.json @@ -0,0 +1,11 @@ +{ + "schemaVersion": 1, "kind": "test-pack", "id": "ecommerce.progression.support-intake", "moduleType": "feature", + "title": "Support intake", "description": "Public support ticket creation and references.", + "requiresPacks": [], "conflictsWith": [], "capabilities": ["browser"], "evidence": ["browser-observation"], + "budget": { "status": "bounded", "maxRuntimeMs": 2000 }, + "task": { + "requirements": [{ "id": "ecommerce.progression.support-intake", "path": "prompts/modular/support-intake.md", "order": 1200, "modes": ["fresh", "upgrade"] }], + "contracts": [{ "id": "ecommerce.progression.support-intake-hooks", "path": "contracts/support-intake.md", "order": 2900, "modes": ["fresh", "upgrade"] }] + }, + "checks": [{ "id": "ticket-create", "stableId": "support-intake", "source": "scenarios/progression-support-intake.json", "feature": 610, "role": "feature" }] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-support-refunds.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-support-refunds.json new file mode 100644 index 00000000000..e943bd918ea --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-support-refunds.json @@ -0,0 +1,57 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.progression.support-refunds", + "moduleType": "feature", + "title": "Support refunds", + "description": "Authorized, exact, and single-application support refunds.", + "requiresPacks": [ + "ecommerce.progression.order-support", + "ecommerce.l2.order-cancellation-features" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 70000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.progression.support-refunds", + "path": "prompts/modular/support-refunds.md", + "order": 5100, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.progression.support-refund-hooks", + "path": "contracts/support-refunds.md", + "order": 5900, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "resolution", + "stableId": "support-refunds-resolution", + "source": "scenarios/progression-support-refunds-resolution.json", + "feature": 615, + "criteria": [ + "615a" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/progression-support-triage.json b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-support-triage.json new file mode 100644 index 00000000000..b0bb71ead80 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/progression-support-triage.json @@ -0,0 +1,77 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.progression.support-triage", + "moduleType": "feature", + "title": "Support triage", + "description": "Staff assignment, priority, and status changes.", + "requiresPacks": [ + "ecommerce.progression.staff-access", + "ecommerce.progression.support-intake" + ], + "conflictsWith": [], + "capabilities": [ + "browser" + ], + "evidence": [ + "browser-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 20000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.progression.support-triage", + "path": "prompts/modular/support-triage.md", + "order": 2100, + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.progression.support-triage-hooks", + "path": "contracts/support-triage.md", + "order": 2900, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "assignment", + "stableId": "support-assignment", + "source": "scenarios/progression-support-triage.json", + "feature": 611, + "criteria": [ + "611a" + ], + "role": "feature" + }, + { + "id": "priority", + "stableId": "support-priority", + "source": "scenarios/progression-support-triage.json", + "feature": 611, + "criteria": [ + "611b" + ], + "role": "feature" + }, + { + "id": "status", + "stableId": "support-status", + "source": "scenarios/progression-support-triage.json", + "feature": 611, + "criteria": [ + "611c" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/returns-pricing-features.json b/tools/stack-bench/tracks/ecommerce/composition/packs/returns-pricing-features.json new file mode 100644 index 00000000000..56b63588adf --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/returns-pricing-features.json @@ -0,0 +1,123 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.returns-pricing-features", + "stableId": "ecommerce.returns-pricing", + "moduleType": "feature", + "title": "Returns and pricing", + "description": "Cancellation, returns, live catalog prices, and immutable receipts.", + "requiresPacks": [ + "ecommerce.operations-access-features" + ], + "conflictsWith": [], + "capabilities": ["browser","direct-server-call"], + "evidence": ["browser-observation","fresh-client-observation"], + "budget": { + "status": "bounded", + "maxRuntimeMs": 94000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.l2.returns-pricing", + "path": "prompts/02-operations.md", + "order": 40, + "from": "### Cancelling and returning", + "until": "### Operational views", + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.l2.price-hooks", + "path": "contracts/appendix-02.md", + "order": 2040, + "from": "| `price-input`", + "until": "| `low-stock-list`", + "modes": [ + "upgrade" + ] + }, + { + "id": "ecommerce.l2.return-hooks", + "path": "contracts/l2-order-hooks.md", + "order": 2060, + "modes": [ + "upgrade" + ] + }, + { + "id": "ecommerce.l2.price-action-input", + "path": "prompts/l2-action-inputs.md", + "order": 2085, + "from": "## Price action input", + "until": "## Inventory contention action inputs", + "modes": [ + "upgrade" + ] + }, + { + "id": "ecommerce.l2.returns-pricing-testing-calls", + "path": "prompts/fragments/named-actions-returns-pricing.md", + "order": 2087, + "modes": [ + "upgrade" + ] + } + ] + }, + "checks": [ + { + "id": "cancellation-core", + "stableId": "cancellation-and-return", + "source": "scenarios/02-features.json", + "feature": 3, + "criteria": [ + "3a" + ], + "role": "feature" + }, + { + "id": "cancellation-history", + "stableId": "cancellation-and-return", + "source": "scenarios/02-self-contained.json", + "feature": 3, + "criteria": [ + "3b" + ], + "role": "feature" + }, + { + "id": "return-complete", + "stableId": "cancellation-and-return", + "source": "scenarios/02-strengthened.json", + "feature": 3, + "criteria": [ + "3c" + ], + "role": "feature" + }, + { + "id": "price-history-exact", + "stableId": "price-history", + "source": "scenarios/02-paid-price-history.json", + "feature": 4, + "criteria": [ + "4a" + ], + "role": "feature" + }, + { + "id": "price-live-independent", + "stableId": "price-history", + "source": "scenarios/02-live-price.json", + "feature": 4, + "criteria": [ + "4b" + ], + "role": "feature" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/returns-pricing-specifications.json b/tools/stack-bench/tracks/ecommerce/composition/packs/returns-pricing-specifications.json new file mode 100644 index 00000000000..c16d21fa1ed --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/returns-pricing-specifications.json @@ -0,0 +1,53 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.returns-pricing-specifications", + "stableId": "ecommerce.returns-pricing", + "moduleType": "specification", + "title": "Refund accounting", + "description": "Revenue remains exact through cancellations, returns, and later price changes.", + "requiresPacks": [], + "conflictsWith": [], + "capabilities": ["browser", "concurrent-actors", "direct-server-call"], + "evidence": ["browser-observation", "concurrent-outcome", "database-observation"], + "budget": { "status": "bounded", "maxRuntimeMs": 66000 }, + "task": { + "requirements": [ + { "id": "ecommerce.l2.refund-invariants", "path": "prompts/02-operations.md", "order": 61, "from": "- Revenue always", "until": "## Starting data", "modes": ["upgrade"], "requiresFeatures": ["ecommerce.returns-pricing-features"] } + ], + "contracts": [] + }, + "checks": [ + { + "id": "refund-accounting", + "stableId": "refund-accounting", + "source": "scenarios/02-invariants.json", + "feature": 203, + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.returns-pricing-features" + ] + }, + { + "id": "price-cart-checkout", + "stableId": "price-history", + "source": "scenarios/progression-price-cart-checkout.json", + "feature": 420, + "criteria": [ + "4c" + ], + "role": "guarantee", + "requiresFeatures": [ + "ecommerce.returns-pricing-features" + ], + "observations": [ + "requested", + "unmentioned" + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/spec-access-control.json b/tools/stack-bench/tracks/ecommerce/composition/packs/spec-access-control.json new file mode 100644 index 00000000000..af18c4b9814 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/spec-access-control.json @@ -0,0 +1,613 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.spec.access-control", + "title": "Access control", + "moduleType": "specification", + "description": "Server-enforced authentication, authorization, ownership, and privacy.", + "requiresPacks": [], + "conflictsWith": [], + "capabilities": [ + "browser", + "direct-server-call", + "request-replay", + "backend-lifecycle" + ], + "evidence": [ + "browser-observation", + "server-response", + "database-observation", + "fresh-client-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 1150000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.spec.access-control.purchasing", + "path": "prompts/modular/l1-specifications.md", + "order": 1000, + "from": "## Access control: purchasing", + "until": "## Access control: warehouse administration", + "modes": [ + "fresh" + ], + "requiresFeatures": [ + "ecommerce.feature.purchasing" + ] + }, + { + "id": "ecommerce.spec.access-control.warehouse-admin", + "path": "prompts/modular/l1-specifications.md", + "order": 1010, + "from": "## Access control: warehouse administration", + "until": "## Access control: reviews", + "modes": [ + "fresh" + ], + "requiresFeatures": [ + "ecommerce.feature.warehouse-admin" + ] + }, + { + "id": "ecommerce.spec.access-control.reviews", + "path": "prompts/modular/l1-specifications.md", + "order": 1020, + "from": "## Access control: reviews", + "until": "## Access control: cart", + "modes": [ + "fresh" + ], + "requiresFeatures": [ + "ecommerce.feature.reviews" + ] + }, + { + "id": "ecommerce.spec.access-control.cart", + "path": "prompts/modular/l1-specifications.md", + "order": 1030, + "from": "## Access control: cart", + "until": "## State durability: accounts", + "modes": [ + "fresh" + ], + "requiresFeatures": [ + "ecommerce.feature.cart" + ] + }, + { + "id": "ecommerce.spec.access-control.automatic-reorder-access", + "path": "prompts/modular/production-specifications.md", + "order": 6001, + "from": "## automatic-reorder-access", + "until": "## recommendation-profile-isolation", + "modes": [ + "upgrade" + ], + "requiresFeatures": [ + "ecommerce.progression.automatic-reorder" + ] + }, + { + "id": "ecommerce.spec.access-control.recommendation-profile-isolation", + "path": "prompts/modular/production-specifications.md", + "order": 6002, + "from": "## recommendation-profile-isolation", + "until": "## staff-activity-privacy", + "modes": [ + "upgrade" + ], + "requiresFeatures": [ + "ecommerce.progression.personalized-recommendations" + ] + }, + { + "id": "ecommerce.spec.access-control.staff-activity-privacy", + "path": "prompts/modular/production-specifications.md", + "order": 6003, + "from": "## staff-activity-privacy", + "until": "## order-support-ownership", + "modes": [ + "upgrade" + ], + "requiresFeatures": [ + "ecommerce.progression.staff-activity" + ] + }, + { + "id": "ecommerce.spec.access-control.order-support-ownership", + "path": "prompts/modular/production-specifications.md", + "order": 6004, + "from": "## order-support-ownership", + "until": "## delivery-notification-privacy", + "modes": [ + "upgrade" + ], + "requiresFeatures": [ + "ecommerce.progression.order-support" + ] + }, + { + "id": "ecommerce.spec.access-control.delivery-notification-privacy", + "path": "prompts/modular/production-specifications.md", + "order": 6005, + "from": "## delivery-notification-privacy", + "until": "## support-refund-access", + "modes": [ + "upgrade" + ], + "requiresFeatures": [ + "ecommerce.progression.delivery-notifications", + "ecommerce.feature.purchasing", + "ecommerce.progression.fulfilment-queue" + ] + }, + { + "id": "ecommerce.spec.access-control.support-refund-access", + "path": "prompts/modular/production-specifications.md", + "order": 6006, + "from": "## support-refund-access", + "until": "## recommendation-feedback-privacy", + "modes": [ + "upgrade" + ], + "requiresFeatures": [ + "ecommerce.progression.support-refunds" + ] + }, + { + "id": "ecommerce.spec.access-control.recommendation-feedback-privacy", + "path": "prompts/modular/production-specifications.md", + "order": 6007, + "from": "## recommendation-feedback-privacy", + "until": "## recommendation-feedback-restart", + "modes": [ + "upgrade" + ], + "requiresFeatures": [ + "ecommerce.progression.recommendation-feedback" + ] + } + ], + "contracts": [] + }, + "checks": [ + { + "id": "account-password", + "source": "scenarios/01-account-password.json", + "feature": 1, + "criteria": ["1c"], + "role": "guarantee", + "observations": ["requested", "unmentioned"], + "requiresFeatures": ["ecommerce.feature.accounts"] + }, + { + "id": "signed-out-purchase", + "source": "scenarios/progression-signed-out-purchase.json", + "feature": 3, + "criteria": [ + "3a" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.purchasing" + ] + }, + { + "id": "purchase-session", + "source": "scenarios/01-purchase-session.json", + "feature": 101, + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.purchasing" + ] + }, + { + "id": "purchase-attribution", + "source": "scenarios/01-purchase-attribution.json", + "feature": 102, + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.purchasing" + ] + }, + { + "id": "order-ownership", + "source": "scenarios/01-order-ownership.json", + "feature": 106, + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.purchasing" + ] + }, + { + "id": "review-eligibility", + "source": "scenarios/01-review-eligibility.json", + "feature": 108, + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.reviews", + "ecommerce.feature.purchasing" + ] + }, + { + "id": "cart-boundary", + "source": "scenarios/01-cart-boundary.json", + "feature": 109, + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.cart" + ] + }, + { + "id": "staff-area-boundary", + "source": "scenarios/progression-staff-access.json", + "feature": 601, + "criteria": [ + "601b" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.progression.staff-access" + ] + }, + { + "id": "warehouse-area-boundary", + "source": "scenarios/01-warehouse-admin-staff.json", + "feature": 7, + "criteria": [ + "7a" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.warehouse-admin" + ] + }, + { + "id": "warehouse-write-boundary", + "source": "scenarios/01-admin-write-staff.json", + "feature": 103, + "criteria": [ + "103b" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.warehouse-admin", + "ecommerce.feature.accounts" + ] + }, + { + "id": "fulfilment-area-boundary", + "source": "scenarios/02-fulfilment-access.json", + "feature": 1, + "criteria": [ + "1d" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.progression.fulfilment-queue" + ] + }, + { + "id": "customer-profile-privacy", + "source": "scenarios/progression-customer-profile.json", + "feature": 620, + "criteria": [ + "620b" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.progression.customer-profile" + ] + }, + { + "id": "notification-preferences-privacy", + "source": "scenarios/progression-notification-preferences.json", + "feature": 630, + "criteria": [ + "630b" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.progression.notification-preferences" + ] + }, + { + "id": "promotion-management-boundary", + "source": "scenarios/progression-promotion-rules.json", + "feature": 620, + "criteria": [ + "620b" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.progression.promotion-rules" + ] + }, + { + "id": "staff-role-revocation", + "source": "scenarios/progression-staff-roles.json", + "feature": 621, + "criteria": [ + "621d" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.progression.staff-roles" + ] + }, + { + "id": "staff-role-boundary", + "source": "scenarios/progression-staff-roles.json", + "feature": 621, + "criteria": [ + "621b" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.progression.staff-roles" + ] + }, + { + "id": "support-history-privacy", + "source": "scenarios/progression-support-history.json", + "feature": 612, + "criteria": [ + "612b" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.progression.support-history" + ] + }, + { + "id": "support-history-logout", + "source": "scenarios/progression-support-history.json", + "feature": 612, + "criteria": [ + "612d" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.progression.support-history" + ] + }, + { + "id": "managed-support-privacy", + "source": "scenarios/progression-managed-support-privacy.json", + "feature": 613, + "criteria": [ + "613b" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.progression.managed-support" + ] + }, + { + "id": "stock-alert-privacy", + "source": "scenarios/progression-stock-alerts.json", + "feature": 631, + "criteria": [ + "631b" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.progression.stock-alerts", + "ecommerce.feature.warehouse-admin" + ] + }, + { + "id": "automatic-reorder-access", + "stableId": "automatic-reorder-access", + "source": "scenarios/progression-automatic-reorder-access.json", + "feature": 502, + "criteria": [ + "502c" + ], + "role": "guarantee", + "requiresFeatures": [ + "ecommerce.progression.automatic-reorder" + ], + "observations": [ + "requested", + "unmentioned" + ] + }, + { + "id": "recommendation-profile-isolation", + "stableId": "recommendation-profile-isolation", + "source": "scenarios/progression-personalized-recommendations.json", + "feature": 403, + "criteria": [ + "403b" + ], + "role": "guarantee", + "requiresFeatures": [ + "ecommerce.progression.personalized-recommendations" + ], + "observations": [ + "requested", + "unmentioned" + ] + }, + { + "id": "staff-activity-privacy", + "stableId": "staff-activity-privacy", + "source": "scenarios/progression-staff-activity.json", + "feature": 624, + "role": "guarantee", + "criteria": [ + "624b" + ], + "requiresFeatures": [ + "ecommerce.progression.staff-activity" + ], + "observations": [ + "requested", + "unmentioned" + ] + }, + { + "id": "order-support-ownership", + "stableId": "order-support-ownership", + "source": "scenarios/progression-order-support-boundary.json", + "feature": 614, + "criteria": [ + "614b" + ], + "role": "guarantee", + "requiresFeatures": [ + "ecommerce.progression.order-support" + ], + "observations": [ + "requested", + "unmentioned" + ] + }, + { + "id": "delivery-notification-privacy", + "stableId": "delivery-notification-privacy", + "source": "scenarios/progression-delivery-notifications.json", + "feature": 501, + "criteria": [ + "501b" + ], + "role": "guarantee", + "requiresFeatures": [ + "ecommerce.progression.delivery-notifications", + "ecommerce.feature.purchasing", + "ecommerce.progression.fulfilment-queue" + ], + "observations": [ + "requested", + "unmentioned" + ] + }, + { + "id": "support-refund-access", + "stableId": "support-refund-access", + "source": "scenarios/progression-support-refunds-access.json", + "feature": 615, + "criteria": [ + "615c" + ], + "role": "guarantee", + "requiresFeatures": [ + "ecommerce.progression.support-refunds" + ], + "observations": [ + "requested", + "unmentioned" + ] + }, + { + "id": "recommendation-feedback-privacy", + "stableId": "recommendation-feedback-privacy", + "source": "scenarios/progression-recommendation-feedback.json", + "feature": 504, + "role": "guarantee", + "criteria": [ + "504b" + ], + "requiresFeatures": [ + "ecommerce.progression.recommendation-feedback" + ], + "observations": [ + "requested", + "unmentioned" + ] + }, + { + "id": "account-disclosure", + "source": "scenarios/01-account-disclosure.json", + "feature": 110, + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.accounts" + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/spec-bundle-integrity.json b/tools/stack-bench/tracks/ecommerce/composition/packs/spec-bundle-integrity.json new file mode 100644 index 00000000000..76c11ef5c16 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/spec-bundle-integrity.json @@ -0,0 +1,187 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.spec.bundle-integrity", + "moduleType": "specification", + "title": "Bundle integrity", + "description": "Shared inventory, authority, and historical return accounting.", + "requiresPacks": [], + "conflictsWith": [], + "capabilities": [ + "browser", + "database-observation", + "direct-database-write", + "direct-server-call", + "backend-lifecycle" + ], + "evidence": [ + "browser-observation", + "database-observation", + "fresh-client-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 480000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.spec.bundle-integrity.product-bundles", + "path": "prompts/modular/later-specifications.md", + "order": 6500, + "from": "## Product bundles", + "until": "## Bundle checkout", + "modes": [ + "fresh", + "upgrade" + ], + "requiresFeatures": [ + "ecommerce.feature.product-bundles" + ] + }, + { + "id": "ecommerce.spec.bundle-integrity.bundle-checkout", + "path": "prompts/modular/later-specifications.md", + "order": 6501, + "from": "## Bundle checkout", + "until": "## Bundle returns", + "modes": [ + "fresh", + "upgrade" + ], + "requiresFeatures": [ + "ecommerce.feature.bundle-checkout" + ] + }, + { + "id": "ecommerce.spec.bundle-integrity.bundle-returns", + "path": "prompts/modular/later-specifications.md", + "order": 6502, + "from": "## Bundle returns", + "until": "## Store credit", + "modes": [ + "fresh", + "upgrade" + ], + "requiresFeatures": [ + "ecommerce.feature.bundle-returns" + ] + } + ], + "contracts": [] + }, + "checks": [ + { + "id": "bundle-743", + "source": "scenarios/progression-product-bundles.json", + "feature": 743, + "criteria": [ + "743a" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.product-bundles" + ] + }, + { + "id": "bundle-744", + "source": "scenarios/progression-bundle-checkout.json", + "feature": 744, + "criteria": [ + "744a" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.bundle-checkout" + ] + }, + { + "id": "bundle-745", + "source": "scenarios/progression-bundle-checkout.json", + "feature": 745, + "criteria": [ + "745a" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.bundle-checkout" + ] + }, + { + "id": "bundle-746", + "source": "scenarios/progression-bundle-checkout.json", + "feature": 746, + "criteria": [ + "746a" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.bundle-checkout" + ] + }, + { + "id": "bundle-747", + "source": "scenarios/progression-bundle-checkout.json", + "feature": 747, + "criteria": [ + "747a" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.bundle-checkout" + ] + }, + { + "id": "bundle-742", + "source": "scenarios/progression-bundle-returns.json", + "feature": 742, + "criteria": [ + "742b" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.bundle-returns" + ] + }, + { + "id": "bundle-748", + "source": "scenarios/progression-bundle-returns.json", + "feature": 748, + "criteria": [ + "748a" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.bundle-returns" + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/spec-concurrency-safety.json b/tools/stack-bench/tracks/ecommerce/composition/packs/spec-concurrency-safety.json new file mode 100644 index 00000000000..3ca28d2b289 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/spec-concurrency-safety.json @@ -0,0 +1,128 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.spec.concurrency-safety", + "title": "Concurrency safety", + "moduleType": "specification", + "description": "Stock race safety and duplicate-submit protection.", + "requiresPacks": [], + "conflictsWith": [], + "capabilities": [ + "browser", + "concurrent-actors", + "direct-server-call" + ], + "evidence": [ + "concurrent-outcome", + "database-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 309000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.spec.concurrency-safety.purchasing", + "path": "prompts/modular/l1-specifications.md", + "order": 1300, + "from": "## Concurrency safety: purchasing", + "until": "## Concurrency safety: restocking", + "modes": [ + "fresh" + ], + "requiresFeatures": [ + "ecommerce.feature.purchasing" + ] + }, + { + "id": "ecommerce.spec.concurrency-safety.restocking", + "path": "prompts/modular/l1-specifications.md", + "order": 1310, + "from": "## Concurrency safety: restocking", + "until": "## Concurrency safety: checkout", + "modes": [ + "fresh" + ], + "requiresFeatures": [ + "ecommerce.feature.purchasing", + "ecommerce.feature.warehouse-admin" + ] + }, + { + "id": "ecommerce.spec.concurrency-safety.checkout", + "path": "prompts/modular/l1-specifications.md", + "order": 1320, + "from": "## Concurrency safety: checkout", + "until": "## Transactional integrity: reviews", + "modes": [ + "fresh" + ], + "requiresFeatures": [ + "ecommerce.feature.checkout" + ] + } + ], + "contracts": [] + }, + "checks": [ + { + "id": "stock-limit", + "source": "scenarios/progression-stock-limit.json", + "feature": 3, + "criteria": [ + "3d" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.purchasing" + ] + }, + { + "id": "last-unit", + "source": "scenarios/01-last-unit.json", + "feature": 201, + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.catalog-items", + "ecommerce.feature.purchasing", + "ecommerce.feature.warehouse-admin" + ] + }, + { + "id": "restock-race", + "source": "scenarios/01-restock-race.json", + "feature": 202, + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.purchasing", + "ecommerce.feature.warehouse-admin" + ] + }, + { + "id": "duplicate-checkout", + "source": "scenarios/01-duplicate-checkout.json", + "feature": 203, + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.checkout" + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/spec-external-data-sync.json b/tools/stack-bench/tracks/ecommerce/composition/packs/spec-external-data-sync.json new file mode 100644 index 00000000000..6d7d2eecba3 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/spec-external-data-sync.json @@ -0,0 +1,111 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.spec.external-data-sync", + "title": "External data synchronization", + "moduleType": "specification", + "description": "Independent probes for live external writes, reload persistence, server restart recovery, and client reconnect catch-up.", + "requiresPacks": [], + "conflictsWith": [], + "capabilities": [ + "browser", + "direct-database-write", + "backend-lifecycle" + ], + "evidence": [ + "database-observation", + "fresh-client-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 247000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.spec.external-data-sync.requirement", + "path": "prompts/modular/l1-external-sync-specifications.md", + "order": 1500, + "from": "## External data synchronization", + "modes": [ + "fresh", + "upgrade" + ], + "requiresFeatures": [ + "ecommerce.feature.warehouse-admin" + ] + } + ], + "contracts": [] + }, + "checks": [ + { + "id": "external-live", + "stableId": "external-stock", + "source": "scenarios/01-external-live-sync.json", + "feature": 901, + "criteria": [ + "901a" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.warehouse-admin" + ] + }, + { + "id": "external-reload", + "stableId": "external-stock", + "source": "scenarios/01-external-reload-sync.json", + "feature": 901, + "criteria": [ + "901b" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.warehouse-admin" + ] + }, + { + "id": "external-reconnect", + "stableId": "external-stock", + "source": "scenarios/01-external-reconnect-sync.json", + "feature": 901, + "criteria": [ + "901d" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.warehouse-admin" + ] + }, + { + "id": "external-stock-alert", + "source": "scenarios/progression-stock-alert-direct-restore.json", + "feature": 631, + "criteria": [ + "631d" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.progression.stock-alerts", + "ecommerce.feature.warehouse-admin" + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/spec-live-state.json b/tools/stack-bench/tracks/ecommerce/composition/packs/spec-live-state.json new file mode 100644 index 00000000000..5f488f6f555 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/spec-live-state.json @@ -0,0 +1,212 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.spec.live-state", + "title": "Live state", + "moduleType": "specification", + "description": "Selected live-view capability: keep open buying views, one account's carts, and staff operational views current across clients. This is not a universal production-readiness requirement; select requested for a product requirement or observed for an unscored diagnostic.", + "requiresPacks": [], + "conflictsWith": [], + "capabilities": [ + "browser", + "concurrent-actors" + ], + "evidence": [ + "browser-observation", + "fresh-client-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 215000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.spec.live-state.catalog-purchasing", + "path": "prompts/modular/l1-specifications.md", + "order": 1200, + "from": "## Live state: catalog and purchasing", + "until": "## Live state: cart", + "modes": [ + "fresh" + ], + "requiresFeatures": [ + "ecommerce.feature.catalog-items", + "ecommerce.feature.purchasing" + ] + }, + { + "id": "ecommerce.spec.live-state.cart", + "path": "prompts/modular/l1-specifications.md", + "order": 1210, + "from": "## Live state: cart", + "until": "## Live state: reviews", + "modes": [ + "fresh" + ], + "requiresFeatures": [ + "ecommerce.feature.cart" + ] + }, + { + "id": "ecommerce.spec.live-state.reviews", + "path": "prompts/modular/l1-specifications.md", + "order": 1220, + "from": "## Live state: reviews", + "until": "## Live state: warehouse administration", + "modes": [ + "fresh" + ], + "requiresFeatures": [ + "ecommerce.feature.reviews" + ] + }, + { + "id": "ecommerce.spec.live-state.warehouse-admin", + "path": "prompts/modular/l1-specifications.md", + "order": 1230, + "from": "## Live state: warehouse administration", + "until": "## Concurrency safety: purchasing", + "modes": [ + "fresh" + ], + "requiresFeatures": [ + "ecommerce.feature.catalog-items", + "ecommerce.feature.warehouse-admin" + ] + } + ], + "contracts": [] + }, + "checks": [ + { + "id": "ranking", + "source": "scenarios/01-core.json", + "feature": 2, + "criteria": [ + "2c" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.catalog-items", + "ecommerce.feature.purchasing" + ] + }, + { + "id": "purchase-stock", + "source": "scenarios/01-buying.json", + "feature": 3, + "criteria": [ + "3b" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.catalog-items", + "ecommerce.feature.purchasing" + ] + }, + { + "id": "shared-cart", + "source": "scenarios/01-cart.json", + "feature": 4, + "criteria": [ + "4c" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.cart" + ] + }, + { + "id": "rating", + "source": "scenarios/01-review-rating-live.json", + "feature": 6, + "criteria": [ + "6c" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.reviews", + "ecommerce.feature.purchasing" + ] + }, + { + "id": "open-list", + "source": "scenarios/progression-open-list-live.json", + "feature": 902, + "criteria": [ + "902a" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.reviews", + "ecommerce.feature.purchasing" + ] + }, + { + "id": "warehouse-stock", + "source": "scenarios/01-warehouse-stock-live-staff.json", + "feature": 7, + "criteria": ["7c"], + "role": "guarantee", + "observations": ["requested", "unmentioned"], + "requiresFeatures": ["ecommerce.feature.catalog-items", "ecommerce.feature.warehouse-admin"] + }, + { + "id": "fulfilment-queue", + "source": "scenarios/02-fulfilment-live.json", + "feature": 1, + "criteria": ["1a"], + "role": "guarantee", + "observations": ["requested", "unmentioned"], + "requiresFeatures": ["ecommerce.progression.fulfilment-queue"] + }, + { + "id": "inventory-dashboard", + "source": "scenarios/02-low-stock.json", + "feature": 5, + "criteria": ["5a"], + "role": "guarantee", + "observations": ["requested", "unmentioned"], + "requiresFeatures": ["ecommerce.l2.inventory-dashboard", "ecommerce.feature.purchasing"] + }, + { + "id": "sales-dashboard", + "source": "scenarios/02-operational-category-totals.json", + "feature": 5, + "criteria": ["5b"], + "role": "guarantee", + "observations": ["requested", "unmentioned"], + "requiresFeatures": ["ecommerce.l2.sales-dashboard", "ecommerce.feature.purchasing"] + }, + { + "id": "managed-support", + "source": "scenarios/progression-managed-support-shared.json", + "feature": 613, + "criteria": ["613a"], + "role": "guarantee", + "observations": ["requested", "unmentioned"], + "requiresFeatures": ["ecommerce.progression.managed-support"] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/spec-search-ordering.json b/tools/stack-bench/tracks/ecommerce/composition/packs/spec-search-ordering.json new file mode 100644 index 00000000000..a7131cea705 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/spec-search-ordering.json @@ -0,0 +1,60 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.spec.search-ordering", + "title": "Search ordering after purchases", + "moduleType": "specification", + "description": "Purchase ranking does not override active search and filter ordering.", + "requiresPacks": [], + "conflictsWith": [], + "capabilities": [ + "browser", + "database-observation" + ], + "evidence": [ + "browser-observation", + "database-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 61000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.spec.search-ordering", + "path": "prompts/modular/search-ordering-specification.md", + "order": 4010, + "modes": [ + "fresh", + "upgrade" + ], + "requiresFeatures": [ + "ecommerce.progression.faceted-search", + "ecommerce.feature.purchasing" + ] + } + ], + "contracts": [] + }, + "checks": [ + { + "id": "after-purchase", + "stableId": "search-ordering", + "source": "scenarios/progression-search-ordering.json", + "feature": 402, + "criteria": [ + "402b" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.progression.faceted-search", + "ecommerce.feature.purchasing" + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/spec-split-tender-refunds.json b/tools/stack-bench/tracks/ecommerce/composition/packs/spec-split-tender-refunds.json new file mode 100644 index 00000000000..e857911ca3c --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/spec-split-tender-refunds.json @@ -0,0 +1,64 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.spec.split-tender-refunds", + "moduleType": "specification", + "title": "Split-tender refunds integrity", + "description": "Bounded production checks for split-tender refunds.", + "requiresPacks": [], + "conflictsWith": [], + "capabilities": [ + "browser", + "database-observation", + "direct-server-call", + "backend-lifecycle" + ], + "evidence": [ + "browser-observation", + "fresh-client-observation", + "database-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 300000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.spec.split-tender-refunds.split-tender-refunds", + "path": "prompts/modular/later-specifications.md", + "order": 6504, + "from": "## Split-tender refunds", + "until": "## Scheduled purchases", + "modes": [ + "fresh", + "upgrade" + ], + "requiresFeatures": [ + "ecommerce.feature.split-tender-refunds" + ] + } + ], + "contracts": [] + }, + "checks": [ + { + "id": "production-756", + "source": "scenarios/progression-split-tender-refunds.json", + "feature": 756, + "criteria": [ + "756a" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.split-tender-refunds", + "ecommerce.feature.store-credit", + "ecommerce.progression.support-refunds" + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/spec-state-durability.json b/tools/stack-bench/tracks/ecommerce/composition/packs/spec-state-durability.json new file mode 100644 index 00000000000..0dfed711682 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/spec-state-durability.json @@ -0,0 +1,264 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.spec.state-durability", + "title": "State durability", + "moduleType": "specification", + "description": "Session continuity and persisted account state across reload, reconnect, and runtime restart.", + "requiresPacks": [], + "conflictsWith": [], + "capabilities": [ + "browser", + "backend-lifecycle", + "database-read", + "process-crash" + ], + "evidence": [ + "browser-observation", + "fresh-client-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 299000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.spec.state-durability.accounts", + "path": "prompts/modular/l1-specifications.md", + "order": 1100, + "from": "## State durability: accounts", + "until": "## State durability: account data", + "modes": [ + "fresh" + ], + "requiresFeatures": [ + "ecommerce.feature.accounts" + ] + }, + { + "id": "ecommerce.spec.state-durability.account-data", + "path": "prompts/modular/l1-specifications.md", + "order": 1110, + "from": "## State durability: account data", + "until": "## Live state: catalog and purchasing", + "modes": [ + "fresh" + ], + "requiresFeatures": [ + "ecommerce.feature.accounts", + "ecommerce.feature.cart" + ] + }, + { + "id": "ecommerce.spec.state-durability.recommendation-feedback-restart", + "path": "prompts/modular/production-specifications.md", + "order": 6008, + "from": "## recommendation-feedback-restart", + "until": "## automatic-reorder-deduplication", + "modes": [ + "upgrade" + ], + "requiresFeatures": [ + "ecommerce.progression.recommendation-feedback" + ] + }, + { + "id": "ecommerce.spec.state-durability.checkout-crash", + "path": "prompts/modular/checkout-recovery-specification.md", + "order": 1120, + "modes": [ + "fresh", + "upgrade" + ], + "requiresFeatures": [ + "ecommerce.feature.checkout" + ] + } + ], + "contracts": [] + }, + "checks": [ + { + "id": "session-reload", + "source": "scenarios/01-account-reload.json", + "feature": 1, + "criteria": [ + "1e" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.accounts" + ] + }, + { + "id": "cart-reload", + "source": "scenarios/01-cart.json", + "feature": 4, + "criteria": [ + "4b" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.cart" + ] + }, + { + "id": "account-state-reload", + "stableId": "account-state-recovery", + "source": "scenarios/progression-account-state-reload.json", + "feature": 105, + "criteria": [ + "105a" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.accounts", + "ecommerce.feature.checkout" + ] + }, + { + "id": "account-state-reconnect", + "stableId": "account-state-recovery", + "source": "scenarios/progression-account-state-reconnect.json", + "feature": 105, + "criteria": [ + "105b" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.accounts", + "ecommerce.feature.checkout" + ] + }, + { + "id": "customer-profile-reload", + "source": "scenarios/progression-customer-profile.json", + "feature": 620, + "criteria": [ + "620a" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.progression.customer-profile" + ] + }, + { + "id": "notification-preferences-reload", + "source": "scenarios/progression-notification-preferences.json", + "feature": 630, + "criteria": [ + "630a" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.progression.notification-preferences" + ] + }, + { + "id": "staff-role-reload", + "source": "scenarios/progression-staff-roles.json", + "feature": 621, + "criteria": [ + "621a" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.progression.staff-roles" + ] + }, + { + "id": "support-history-reload", + "source": "scenarios/progression-support-history.json", + "feature": 612, + "criteria": [ + "612a" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.progression.support-history" + ] + }, + { + "id": "recommendation-feedback-restart", + "stableId": "recommendation-feedback-restart", + "source": "scenarios/progression-recommendation-feedback.json", + "feature": 504, + "role": "guarantee", + "criteria": [ + "504c" + ], + "requiresFeatures": [ + "ecommerce.progression.recommendation-feedback" + ], + "observations": [ + "requested", + "unmentioned" + ] + }, + { + "id": "checkout-crash-integrity", + "source": "scenarios/progression-checkout-crash.json", + "feature": 910, + "criteria": [ + "910a" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.checkout" + ] + }, + { + "id": "checkout-crash-durability", + "source": "scenarios/progression-checkout-crash.json", + "feature": 910, + "criteria": [ + "910b" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.checkout" + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/spec-store-credit.json b/tools/stack-bench/tracks/ecommerce/composition/packs/spec-store-credit.json new file mode 100644 index 00000000000..0f9a8077447 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/spec-store-credit.json @@ -0,0 +1,110 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.spec.store-credit", + "moduleType": "specification", + "title": "Store credit integrity", + "description": "Bounded production checks for store credit.", + "requiresPacks": [], + "conflictsWith": [], + "capabilities": [ + "browser", + "database-observation", + "direct-server-call", + "backend-lifecycle" + ], + "evidence": [ + "browser-observation", + "fresh-client-observation", + "database-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 300000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.spec.store-credit.store-credit", + "path": "prompts/modular/later-specifications.md", + "order": 6503, + "from": "## Store credit", + "until": "## Split-tender refunds", + "modes": [ + "fresh", + "upgrade" + ], + "requiresFeatures": [ + "ecommerce.feature.store-credit" + ] + } + ], + "contracts": [] + }, + "checks": [ + { + "id": "production-752", + "source": "scenarios/progression-store-credit.json", + "feature": 752, + "criteria": [ + "752a" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.store-credit" + ] + }, + { + "id": "production-753", + "source": "scenarios/progression-store-credit.json", + "feature": 753, + "criteria": [ + "753a" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.store-credit" + ] + }, + { + "id": "production-754", + "source": "scenarios/progression-store-credit.json", + "feature": 754, + "criteria": [ + "754a" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.store-credit" + ] + }, + { + "id": "production-755", + "source": "scenarios/progression-store-credit.json", + "feature": 755, + "criteria": [ + "755a" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.store-credit" + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/spec-subscriptions.json b/tools/stack-bench/tracks/ecommerce/composition/packs/spec-subscriptions.json new file mode 100644 index 00000000000..4a8cdcbeb7e --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/spec-subscriptions.json @@ -0,0 +1,93 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.spec.subscriptions", + "moduleType": "specification", + "title": "Scheduled purchases integrity", + "description": "Bounded production checks for scheduled purchases.", + "requiresPacks": [], + "conflictsWith": [], + "capabilities": [ + "browser", + "database-observation", + "direct-server-call", + "backend-lifecycle" + ], + "evidence": [ + "browser-observation", + "fresh-client-observation", + "database-observation" + ], + "budget": { + "status": "bounded", + "maxRuntimeMs": 480000 + }, + "task": { + "requirements": [ + { + "id": "ecommerce.spec.subscriptions.subscriptions", + "path": "prompts/modular/later-specifications.md", + "order": 6505, + "from": "## Scheduled purchases", + "modes": [ + "fresh", + "upgrade" + ], + "requiresFeatures": [ + "ecommerce.feature.subscriptions" + ] + } + ], + "contracts": [] + }, + "checks": [ + { + "id": "production-761", + "source": "scenarios/progression-subscriptions.json", + "feature": 761, + "criteria": [ + "761a" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.subscriptions" + ] + }, + { + "id": "production-762", + "source": "scenarios/progression-subscriptions.json", + "feature": 762, + "criteria": [ + "762a" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.subscriptions" + ] + }, + { + "id": "production-763", + "source": "scenarios/progression-subscriptions.json", + "feature": 763, + "criteria": [ + "763a" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.subscriptions" + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/packs/spec-transactional-integrity.json b/tools/stack-bench/tracks/ecommerce/composition/packs/spec-transactional-integrity.json new file mode 100644 index 00000000000..453000c2f07 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/packs/spec-transactional-integrity.json @@ -0,0 +1,246 @@ +{ + "schemaVersion": 1, + "kind": "test-pack", + "id": "ecommerce.spec.transactional-integrity", + "title": "Transactional integrity", + "moduleType": "specification", + "description": "Server authority, accounting, valid quantities, and unique reviews.", + "requiresPacks": [], + "conflictsWith": [], + "capabilities": ["browser", "database-observation", "direct-server-call"], + "evidence": ["browser-observation", "database-observation", "server-response"], + "budget": { "status": "bounded", "maxRuntimeMs": 2114000 }, + "task": { + "requirements": [ + { + "id": "ecommerce.spec.transactional-integrity.reviews", + "path": "prompts/modular/l1-specifications.md", + "order": 1400, + "from": "## Transactional integrity: reviews", + "until": "## Transactional integrity: purchasing", + "modes": [ + "fresh" + ], + "requiresFeatures": [ + "ecommerce.feature.reviews" + ] + }, + { + "id": "ecommerce.spec.transactional-integrity.purchasing", + "path": "prompts/modular/l1-specifications.md", + "order": 1410, + "from": "## Transactional integrity: purchasing", + "until": "## Transactional integrity: warehouse accounting", + "modes": [ + "fresh" + ], + "requiresFeatures": [ + "ecommerce.feature.purchasing" + ] + }, + { + "id": "ecommerce.spec.transactional-integrity.warehouse-accounting", + "path": "prompts/modular/l1-specifications.md", + "order": 1420, + "from": "## Transactional integrity: warehouse accounting", + "until": "## External data synchronization", + "modes": [ + "fresh" + ], + "requiresFeatures": [ + "ecommerce.feature.purchasing", + "ecommerce.feature.warehouse-admin" + ] + }, + { + "id": "ecommerce.spec.transactional-integrity.automatic-reorder-deduplication", + "path": "prompts/modular/production-specifications.md", + "order": 6009, + "from": "## automatic-reorder-deduplication", + "until": "## payment-deduplication", + "modes": [ + "upgrade" + ], + "requiresFeatures": [ + "ecommerce.progression.automatic-reorder", + "ecommerce.feature.purchasing" + ] + }, + { + "id": "ecommerce.spec.transactional-integrity.payment-deduplication", + "path": "prompts/modular/production-specifications.md", + "order": 6010, + "from": "## payment-deduplication", + "until": "## support-refund-accounting", + "modes": [ + "upgrade" + ], + "requiresFeatures": [ + "ecommerce.progression.payment-records" + ] + }, + { + "id": "ecommerce.spec.transactional-integrity.support-refund-accounting", + "path": "prompts/modular/production-specifications.md", + "order": 6011, + "from": "## support-refund-accounting", + "modes": [ + "upgrade" + ], + "requiresFeatures": [ + "ecommerce.progression.support-refunds" + ] + } + ], + "contracts": [] + }, + "checks": [ + { + "id": "unique-review", + "source": "scenarios/01-review-uniqueness.json", + "feature": 6, + "criteria": [ + "6b" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.reviews", + "ecommerce.feature.purchasing" + ] + }, + { + "id": "server-price", + "source": "scenarios/01-server-price.json", + "feature": 104, + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.purchasing" + ] + }, + { + "id": "books-balance", + "source": "scenarios/progression-books-balance.json", + "feature": 107, + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.feature.purchasing", + "ecommerce.feature.warehouse-admin" + ] + }, + { + "id": "stock-transfer-overdraw", + "source": "scenarios/02-transfer-overdraw.json", + "feature": 2, + "criteria": [ + "2c" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.l2.stock-transfers-features" + ] + }, + { + "id": "stock-alert-deduplication", + "source": "scenarios/progression-stock-alerts.json", + "feature": 631, + "criteria": [ + "631a" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.progression.stock-alerts", + "ecommerce.feature.warehouse-admin" + ] + }, + { + "id": "automatic-reorder-deduplication", + "stableId": "automatic-reorder-deduplication", + "source": "scenarios/progression-automatic-reorder-duplicate.json", + "feature": 502, + "criteria": [ + "502b" + ], + "role": "guarantee", + "requiresFeatures": [ + "ecommerce.progression.automatic-reorder", + "ecommerce.feature.purchasing" + ], + "observations": [ + "requested", + "unmentioned" + ] + }, + { + "id": "payment-deduplication", + "stableId": "payment-deduplication", + "source": "scenarios/progression-core-business.json", + "feature": 623, + "role": "guarantee", + "criteria": [ + "623b" + ], + "requiresFeatures": [ + "ecommerce.progression.payment-records" + ], + "observations": [ + "requested", + "unmentioned" + ] + }, + { + "id": "support-refund-accounting", + "stableId": "support-refund-accounting", + "source": "scenarios/progression-support-refunds-accounting.json", + "feature": 615, + "criteria": [ + "615b" + ], + "role": "guarantee", + "requiresFeatures": [ + "ecommerce.progression.support-refunds" + ], + "observations": [ + "requested", + "unmentioned" + ] + }, + { + "id": "catalog-volume", + "source": "scenarios/progression-catalog-volume.json", + "feature": 622, + "criteria": [ + "622c" + ], + "role": "guarantee", + "observations": [ + "requested", + "unmentioned" + ], + "requiresFeatures": [ + "ecommerce.progression.catalog-management", + "ecommerce.progression.faceted-search", + "ecommerce.feature.purchasing" + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/recipes/progression-catalog.json b/tools/stack-bench/tracks/ecommerce/composition/recipes/progression-catalog.json new file mode 100644 index 00000000000..0e882acfa2f --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/recipes/progression-catalog.json @@ -0,0 +1,550 @@ +{ + "schemaVersion": 1, + "kind": "benchmark-recipe", + "id": "ecommerce.progression-catalog", + "title": "Ecommerce progression catalog", + "track": "ecommerce", + "fixture": { + "path": "../fixtures/operations.json", + "id": "ecommerce.operations" + }, + "task": { + "mode": "action", + "framing": { + "requirements": [ + { + "id": "ecommerce.progression.fresh", + "path": "prompts/modular/progression-framing.md", + "order": 0, + "until": "## Existing application", + "modes": [ + "fresh" + ] + }, + { + "id": "ecommerce.progression.upgrade", + "path": "prompts/modular/progression-framing.md", + "order": 0, + "from": "## Existing application", + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.application-interface", + "path": "contracts/application-interface.md", + "order": 0, + "modes": [ + "fresh", + "upgrade" + ] + } + ] + } + }, + "packs": [ + { + "path": "../packs/feature-accounts.json", + "id": "ecommerce.feature.accounts", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/feature-cart.json", + "id": "ecommerce.feature.cart", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/feature-checkout.json", + "id": "ecommerce.feature.checkout", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/feature-catalog-items.json", + "id": "ecommerce.feature.catalog-items", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/feature-catalog-discovery.json", + "id": "ecommerce.feature.catalog-discovery", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/feature-purchasing.json", + "id": "ecommerce.feature.purchasing", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/feature-reviews.json", + "id": "ecommerce.feature.reviews", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/feature-warehouse-admin.json", + "id": "ecommerce.feature.warehouse-admin", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/progression-inventory-conservation-specifications.json", + "id": "ecommerce.progression.inventory-conservation-specifications", + "includeRoles": [ + "guarantee" + ] + }, + { + "path": "../packs/l2-inventory-dashboard.json", + "id": "ecommerce.l2.inventory-dashboard", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/l2-sales-dashboard.json", + "id": "ecommerce.l2.sales-dashboard", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/l2-recommendations.json", + "id": "ecommerce.l2.recommendations", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/l2-order-cancellation-features.json", + "id": "ecommerce.l2.order-cancellation-features", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/l2-price-history-features.json", + "id": "ecommerce.l2.price-history-features", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/progression-price-history-specifications.json", + "id": "ecommerce.progression.price-history-specifications", + "includeRoles": [ + "guarantee" + ] + }, + { + "path": "../packs/l2-stock-transfers-features.json", + "id": "ecommerce.l2.stock-transfers-features", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/l3-cart-expiration-features.json", + "id": "ecommerce.l3.cart-expiration-features", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/l3-deferred-access-specifications.json", + "id": "ecommerce.l3.deferred-access-specifications", + "includeRoles": [ + "guarantee" + ] + }, + { + "path": "../packs/l3-deferred-durability-specifications.json", + "id": "ecommerce.l3.deferred-durability-specifications", + "includeRoles": [ + "guarantee" + ] + }, + { + "path": "../packs/l3-deferred-integrity-specifications.json", + "id": "ecommerce.l3.deferred-integrity-specifications", + "includeRoles": [ + "guarantee" + ] + }, + { + "path": "../packs/l3-order-delivery-features.json", + "id": "ecommerce.l3.order-delivery-features", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/l3-order-returns-features.json", + "id": "ecommerce.l3.order-returns-features", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/l3-reservations-features.json", + "id": "ecommerce.l3.reservations-features", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/l3-scheduled-restocks-features.json", + "id": "ecommerce.l3.scheduled-restocks-features", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/l3-server-time-specifications.json", + "id": "ecommerce.l3.server-time-specifications", + "includeRoles": [ + "guarantee" + ] + }, + { + "path": "../packs/progression-fulfilment-queue.json", + "id": "ecommerce.progression.fulfilment-queue", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/progression-operations-access-specifications.json", + "id": "ecommerce.progression.operations-access-specifications", + "includeRoles": [ + "guarantee" + ] + }, + { + "path": "../packs/progression-automatic-reorder.json", + "id": "ecommerce.progression.automatic-reorder", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/progression-cancellation-accounting-specifications.json", + "id": "ecommerce.progression.cancellation-accounting-specifications", + "includeRoles": [ + "guarantee" + ] + }, + { + "path": "../packs/progression-cancellation-queue-specifications.json", + "id": "ecommerce.progression.cancellation-queue-specifications", + "includeRoles": [ + "guarantee" + ] + }, + { + "path": "../packs/progression-cart-recovery.json", + "id": "ecommerce.progression.cart-recovery", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/progression-catalog-management.json", + "id": "ecommerce.progression.catalog-management", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/progression-customer-profile.json", + "id": "ecommerce.progression.customer-profile", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/progression-delivery-notifications.json", + "id": "ecommerce.progression.delivery-notifications", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/progression-faceted-search.json", + "id": "ecommerce.progression.faceted-search", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/progression-managed-support.json", + "id": "ecommerce.progression.managed-support", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/progression-notification-preferences.json", + "id": "ecommerce.progression.notification-preferences", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/progression-order-support.json", + "id": "ecommerce.progression.order-support", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/progression-payment-records.json", + "id": "ecommerce.progression.payment-records", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/progression-personalized-recommendations.json", + "id": "ecommerce.progression.personalized-recommendations", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/progression-price-accounting-specifications.json", + "id": "ecommerce.progression.price-accounting-specifications", + "includeRoles": [ + "guarantee" + ] + }, + { + "path": "../packs/progression-promotion-checkout.json", + "id": "ecommerce.progression.promotion-checkout", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/progression-promotion-reporting.json", + "id": "ecommerce.progression.promotion-reporting", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/progression-promotion-rules.json", + "id": "ecommerce.progression.promotion-rules", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/progression-recommendation-feedback.json", + "id": "ecommerce.progression.recommendation-feedback", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/progression-review-access-specifications.json", + "id": "ecommerce.progression.review-access-specifications", + "includeRoles": [ + "guarantee" + ] + }, + { + "path": "../packs/progression-staff-access.json", + "id": "ecommerce.progression.staff-access", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/progression-staff-activity.json", + "id": "ecommerce.progression.staff-activity", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/progression-staff-roles.json", + "id": "ecommerce.progression.staff-roles", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/progression-stock-alerts.json", + "id": "ecommerce.progression.stock-alerts", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/progression-support-history.json", + "id": "ecommerce.progression.support-history", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/progression-support-intake.json", + "id": "ecommerce.progression.support-intake", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/progression-support-refunds.json", + "id": "ecommerce.progression.support-refunds", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/progression-support-triage.json", + "id": "ecommerce.progression.support-triage", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/spec-access-control.json", + "id": "ecommerce.spec.access-control", + "includeRoles": [ + "guarantee" + ] + }, + { + "path": "../packs/spec-concurrency-safety.json", + "id": "ecommerce.spec.concurrency-safety", + "includeRoles": [ + "guarantee" + ] + }, + { + "path": "../packs/spec-external-data-sync.json", + "id": "ecommerce.spec.external-data-sync", + "includeRoles": [ + "guarantee" + ] + }, + { + "path": "../packs/spec-live-state.json", + "id": "ecommerce.spec.live-state", + "includeRoles": [ + "guarantee" + ] + }, + { + "path": "../packs/spec-state-durability.json", + "id": "ecommerce.spec.state-durability", + "includeRoles": [ + "guarantee" + ] + }, + { + "path": "../packs/spec-transactional-integrity.json", + "id": "ecommerce.spec.transactional-integrity", + "includeRoles": [ + "guarantee" + ] + }, + { + "path": "../packs/feature-product-bundles.json", + "id": "ecommerce.feature.product-bundles", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/feature-bundle-checkout.json", + "id": "ecommerce.feature.bundle-checkout", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/feature-bundle-returns.json", + "id": "ecommerce.feature.bundle-returns", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/feature-store-credit.json", + "id": "ecommerce.feature.store-credit", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/feature-split-tender-refunds.json", + "id": "ecommerce.feature.split-tender-refunds", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/feature-subscriptions.json", + "id": "ecommerce.feature.subscriptions", + "includeRoles": [ + "feature" + ] + }, + { + "path": "../packs/spec-bundle-integrity.json", + "id": "ecommerce.spec.bundle-integrity", + "includeRoles": [ + "guarantee" + ] + }, + { + "path": "../packs/spec-store-credit.json", + "id": "ecommerce.spec.store-credit", + "includeRoles": [ + "guarantee" + ] + }, + { + "path": "../packs/spec-split-tender-refunds.json", + "id": "ecommerce.spec.split-tender-refunds", + "includeRoles": [ + "guarantee" + ] + }, + { + "path": "../packs/spec-subscriptions.json", + "id": "ecommerce.spec.subscriptions", + "includeRoles": [ + "guarantee" + ] + }, + { + "path": "../packs/spec-search-ordering.json", + "id": "ecommerce.spec.search-ordering", + "includeRoles": [ + "guarantee" + ] + } + ], + "execution": "all-selected-sources", + "scoring": { + "mode": "source-points" + } +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/recipes/sequential-l1.json b/tools/stack-bench/tracks/ecommerce/composition/recipes/sequential-l1.json new file mode 100644 index 00000000000..b5e35e2e31d --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/recipes/sequential-l1.json @@ -0,0 +1,316 @@ +{ + "schemaVersion": 1, + "kind": "benchmark-recipe", + "id": "ecommerce.sequential-l1", + "title": "Ecommerce sequential L1", + "track": "ecommerce", + "sequence": { + "level": 1 + }, + "fixture": { + "path": "../fixtures/storefront.json", + "id": "ecommerce.storefront" + }, + "task": { + "mode": "fresh", + "framing": { + "requirements": [ + { + "id": "ecommerce.sequential-l1.framing", + "path": "prompts/modular/l1-framing.md", + "order": 0, + "modes": [ + "fresh" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.sequential-l1.interface", + "path": "prompts/modular/l1-hooks.md", + "order": 1900, + "from": "## Public application interface", + "until": "## Catalog controls", + "modes": [ + "fresh" + ] + }, + { + "id": "ecommerce.sequential-l1.purchase-action-input", + "path": "prompts/modular/l1-action-inputs.md", + "order": 2210, + "from": "## Purchase action input", + "until": "## Cart quantity action input", + "modes": [ + "fresh" + ], + "requiresFeatures": [ + "ecommerce.feature.purchasing" + ] + }, + { + "id": "ecommerce.sequential-l1.cart-action-input", + "path": "prompts/modular/l1-action-inputs.md", + "order": 2410, + "from": "## Cart quantity action input", + "until": "## Restock action input", + "modes": [ + "fresh" + ], + "requiresFeatures": [ + "ecommerce.feature.cart-checkout" + ] + }, + { + "id": "ecommerce.sequential-l1.restock-action-input", + "path": "prompts/modular/l1-action-inputs.md", + "order": 2510, + "from": "## Restock action input", + "modes": [ + "fresh" + ], + "requiresFeatures": [ + "ecommerce.feature.warehouse-admin" + ] + }, + { + "id": "ecommerce.sequential-l1.startup-probe", + "path": "prompts/fragments/startup-probe.md", + "order": 2600, + "modes": [ + "fresh" + ], + "requiresFeatures": [ + "ecommerce.feature.catalog-items" + ] + } + ] + } + }, + "packs": [ + { + "path": "../packs/feature-accounts.json", + "id": "ecommerce.feature.accounts", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "account-create", + "account-duplicate", + "account-signout" + ] + }, + { + "path": "../packs/feature-catalog-items.json", + "id": "ecommerce.feature.catalog-items", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "values" + ] + }, + { + "path": "../packs/feature-catalog-discovery.json", + "id": "ecommerce.feature.catalog-discovery", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "ranking", + "search" + ] + }, + { + "path": "../packs/feature-purchasing.json", + "id": "ecommerce.feature.purchasing", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "purchase-order" + ] + }, + { + "path": "../packs/feature-cart.json", + "id": "ecommerce.feature.cart", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "cart" + ] + }, + { + "path": "../packs/feature-checkout.json", + "id": "ecommerce.feature.checkout", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "checkout" + ] + }, + { + "path": "../packs/feature-reviews.json", + "id": "ecommerce.feature.reviews", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "reviews" + ] + }, + { + "path": "../packs/feature-warehouse-admin.json", + "id": "ecommerce.feature.warehouse-admin", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "warehouse-view", + "admin-write" + ] + }, + { + "path": "../packs/spec-access-control.json", + "id": "ecommerce.spec.access-control", + "includeRoles": [ + "guarantee" + ], + "includeCheckGroups": [ + "account-password", + "signed-out-purchase", + "purchase-session", + "purchase-attribution", + "order-ownership", + "review-eligibility", + "cart-boundary", + "warehouse-area-boundary" + ] + }, + { + "path": "../packs/spec-state-durability.json", + "id": "ecommerce.spec.state-durability", + "includeRoles": [ + "guarantee" + ], + "includeCheckGroups": [ + "session-reload", + "cart-reload", + "account-state-reload", + "account-state-reconnect" + ] + }, + { + "path": "../packs/spec-live-state.json", + "id": "ecommerce.spec.live-state", + "includeRoles": [ + "guarantee" + ], + "includeCheckGroups": [ + "ranking", + "purchase-stock", + "shared-cart", + "rating", + "open-list", + "warehouse-stock" + ] + }, + { + "path": "../packs/spec-concurrency-safety.json", + "id": "ecommerce.spec.concurrency-safety", + "includeRoles": [ + "guarantee" + ], + "includeCheckGroups": [ + "stock-limit", + "last-unit", + "restock-race", + "duplicate-checkout" + ] + }, + { + "path": "../packs/spec-transactional-integrity.json", + "id": "ecommerce.spec.transactional-integrity", + "includeRoles": [ + "guarantee" + ], + "includeCheckGroups": [ + "unique-review", + "server-price", + "books-balance" + ] + }, + { + "path": "../packs/spec-external-data-sync.json", + "id": "ecommerce.spec.external-data-sync", + "includeRoles": [ + "guarantee" + ], + "includeCheckGroups": [ + "external-live", + "external-reload", + "external-reconnect" + ] + }, + { + "path": "../packs/progression-staff-access.json", + "id": "ecommerce.progression.staff-access", + "includeRoles": [] + } + ], + "execution": "all-selected-sources", + "scoring": { + "mode": "explicit", + "weights": { + "ecommerce.feature.accounts.accounts.1a": 1, + "ecommerce.feature.accounts.accounts.1b": 1, + "ecommerce.spec.access-control.account-password.1c": 1, + "ecommerce.feature.accounts.accounts.1d": 1, + "ecommerce.spec.state-durability.session-reload.1e": 1, + "ecommerce.feature.catalog.catalog-values.2a": 1, + "ecommerce.feature.catalog.catalog-ranking.2b": 1, + "ecommerce.spec.live-state.ranking.2c": 1, + "ecommerce.feature.catalog.catalog-search.2d": 1, + "ecommerce.spec.access-control.signed-out-purchase.3a": 1, + "ecommerce.spec.live-state.purchase-stock.3b": 2, + "ecommerce.feature.purchasing.purchase-order.3c": 1, + "ecommerce.spec.concurrency-safety.stock-limit.3d": 2, + "ecommerce.feature.cart-checkout.cart.4a": 1, + "ecommerce.spec.state-durability.cart-reload.4b": 1, + "ecommerce.spec.live-state.shared-cart.4c": 2, + "ecommerce.feature.cart-checkout.cart.4d": 2, + "ecommerce.feature.reviews.reviews.6a": 2, + "ecommerce.spec.transactional-integrity.unique-review.6b": 1, + "ecommerce.spec.live-state.rating.6c": 1, + "ecommerce.spec.access-control.warehouse-area-boundary.7a": 1, + "ecommerce.feature.warehouse-admin.warehouse-view.7b": 1, + "ecommerce.spec.live-state.warehouse-stock.7c": 2, + "ecommerce.spec.access-control.purchase-session.101a": 2, + "ecommerce.spec.access-control.purchase-attribution.102a": 2, + "ecommerce.feature.warehouse-admin.admin-write.103a": 2, + "ecommerce.spec.transactional-integrity.server-price.104a": 2, + "ecommerce.spec.state-durability.account-state-recovery.105a": 1, + "ecommerce.spec.state-durability.account-state-recovery.105b": 1, + "ecommerce.spec.access-control.order-ownership.106a": 1, + "ecommerce.spec.transactional-integrity.books-balance.107a": 1, + "ecommerce.spec.transactional-integrity.books-balance.107b": 1, + "ecommerce.spec.access-control.review-eligibility.108a": 2, + "ecommerce.spec.access-control.review-eligibility.108b": 1, + "ecommerce.spec.access-control.cart-boundary.109a": 2, + "ecommerce.spec.access-control.cart-boundary.109b": 1, + "ecommerce.spec.concurrency-safety.last-unit.201a": 1, + "ecommerce.spec.concurrency-safety.last-unit.201b": 1, + "ecommerce.spec.concurrency-safety.last-unit.201c": 1, + "ecommerce.spec.concurrency-safety.restock-race.202-control": 0, + "ecommerce.spec.concurrency-safety.restock-race.202a": 1, + "ecommerce.spec.concurrency-safety.duplicate-checkout.203a": 1, + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b": 1, + "ecommerce.spec.external-data-sync.external-stock.901a": 1, + "ecommerce.spec.external-data-sync.external-stock.901b": 0, + "ecommerce.spec.external-data-sync.external-stock.901d": 1, + "ecommerce.spec.live-state.open-list.902a": 1 + } + } +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/recipes/sequential-l2.json b/tools/stack-bench/tracks/ecommerce/composition/recipes/sequential-l2.json new file mode 100644 index 00000000000..000b1b79ae1 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/recipes/sequential-l2.json @@ -0,0 +1,389 @@ +{ + "schemaVersion": 1, + "kind": "benchmark-recipe", + "id": "ecommerce.sequential-l2", + "title": "Ecommerce sequential L2", + "track": "ecommerce", + "sequence": { + "level": 2 + }, + "fixture": { + "path": "../fixtures/operations.json", + "id": "ecommerce.operations" + }, + "task": { + "mode": "upgrade", + "baseRecipe": { + "path": "sequential-l1.json", + "id": "ecommerce.sequential-l1" + }, + "framing": { + "requirements": [ + { + "id": "ecommerce.l2.introduction", + "path": "prompts/02-operations.md", + "order": 10, + "until": "## New: warehouse staff", + "modes": [ + "upgrade" + ] + }, + { + "id": "ecommerce.l2.starting-data", + "path": "prompts/02-operations.md", + "order": 70, + "from": "## Starting data", + "modes": [ + "upgrade" + ] + } + ], + "contracts": [ + { + "id": "ecommerce.l2.contract-heading", + "path": "contracts/appendix-02.md", + "order": 2000, + "until": "| `app-title`", + "modes": [ + "upgrade" + ] + }, + { + "id": "ecommerce.l2.contract-footer", + "path": "contracts/appendix-02.md", + "order": 2070, + "from": "Before declaring", + "modes": [ + "upgrade" + ] + } + ] + } + }, + "packs": [ + { + "path": "../packs/feature-accounts.json", + "id": "ecommerce.feature.accounts", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "account-create", + "account-duplicate", + "account-signout" + ] + }, + { + "path": "../packs/feature-catalog-items.json", + "id": "ecommerce.feature.catalog-items", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "values" + ] + }, + { + "path": "../packs/feature-catalog-discovery.json", + "id": "ecommerce.feature.catalog-discovery", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "ranking", + "search" + ] + }, + { + "path": "../packs/feature-purchasing.json", + "id": "ecommerce.feature.purchasing", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "purchase-order" + ] + }, + { + "path": "../packs/feature-cart.json", + "id": "ecommerce.feature.cart", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "cart" + ] + }, + { + "path": "../packs/feature-checkout.json", + "id": "ecommerce.feature.checkout", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "checkout" + ] + }, + { + "path": "../packs/feature-reviews.json", + "id": "ecommerce.feature.reviews", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "reviews" + ] + }, + { + "path": "../packs/feature-warehouse-admin.json", + "id": "ecommerce.feature.warehouse-admin", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "warehouse-view", + "admin-write" + ] + }, + { + "path": "../packs/spec-access-control.json", + "id": "ecommerce.spec.access-control", + "includeRoles": [ + "guarantee" + ], + "includeCheckGroups": [ + "account-password", + "signed-out-purchase", + "purchase-session", + "purchase-attribution", + "order-ownership", + "review-eligibility", + "cart-boundary", + "warehouse-area-boundary" + ] + }, + { + "path": "../packs/spec-state-durability.json", + "id": "ecommerce.spec.state-durability", + "includeRoles": [ + "guarantee" + ], + "includeCheckGroups": [ + "session-reload", + "cart-reload", + "account-state-reload", + "account-state-reconnect" + ] + }, + { + "path": "../packs/spec-live-state.json", + "id": "ecommerce.spec.live-state", + "includeRoles": [ + "guarantee" + ], + "includeCheckGroups": [ + "ranking", + "purchase-stock", + "shared-cart", + "rating", + "open-list", + "warehouse-stock" + ] + }, + { + "path": "../packs/spec-concurrency-safety.json", + "id": "ecommerce.spec.concurrency-safety", + "includeRoles": [ + "guarantee" + ], + "includeCheckGroups": [ + "stock-limit", + "last-unit", + "restock-race", + "duplicate-checkout" + ] + }, + { + "path": "../packs/spec-transactional-integrity.json", + "id": "ecommerce.spec.transactional-integrity", + "includeRoles": [ + "guarantee" + ], + "includeCheckGroups": [ + "unique-review", + "server-price", + "books-balance" + ] + }, + { + "path": "../packs/spec-external-data-sync.json", + "id": "ecommerce.spec.external-data-sync", + "includeRoles": [ + "guarantee" + ], + "includeCheckGroups": [ + "external-live", + "external-reload", + "external-reconnect" + ] + }, + { + "path": "../packs/operations-access-features.json", + "id": "ecommerce.operations-access-features", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "fulfilment-core", + "fulfilment-warehouse", + "fulfilment-independent" + ] + }, + { + "path": "../packs/operations-access-specifications.json", + "id": "ecommerce.operations-access-specifications", + "includeRoles": [ + "guarantee" + ], + "includeCheckGroups": [ + "operator-transfer-authorization", + "operator-price-authorization", + "operator-authorization-direct", + "order-owner-direct" + ] + }, + { + "path": "../packs/inventory-operations-features.json", + "id": "ecommerce.inventory-operations-features", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "warehouse-transfer-corrected", + "operational-views-low-stock", + "operational-views-core", + "operational-views-kept-purchases" + ] + }, + { + "path": "../packs/inventory-operations-specifications.json", + "id": "ecommerce.inventory-operations-specifications", + "includeRoles": [ + "guarantee" + ], + "includeCheckGroups": [ + "stock-conservation-cancel", + "stock-conservation-contention", + "warehouse-transfer-overdraw" + ] + }, + { + "path": "../packs/returns-pricing-features.json", + "id": "ecommerce.returns-pricing-features", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "cancellation-core", + "cancellation-history", + "return-complete", + "price-history-exact", + "price-live-independent" + ] + }, + { + "path": "../packs/returns-pricing-specifications.json", + "id": "ecommerce.returns-pricing-specifications", + "includeRoles": [ + "guarantee" + ], + "includeCheckGroups": [ + "refund-accounting", + "price-cart-checkout" + ] + }, + { + "path": "../packs/progression-staff-access.json", + "id": "ecommerce.progression.staff-access", + "includeRoles": [] + } + ], + "execution": "all-selected-sources", + "scoring": { + "mode": "explicit", + "weights": { + "ecommerce.feature.accounts.accounts.1a": 1, + "ecommerce.feature.accounts.accounts.1b": 1, + "ecommerce.spec.access-control.account-password.1c": 1, + "ecommerce.feature.accounts.accounts.1d": 1, + "ecommerce.spec.state-durability.session-reload.1e": 1, + "ecommerce.feature.catalog.catalog-values.2a": 1, + "ecommerce.feature.catalog.catalog-ranking.2b": 1, + "ecommerce.spec.live-state.ranking.2c": 1, + "ecommerce.feature.catalog.catalog-search.2d": 1, + "ecommerce.spec.access-control.signed-out-purchase.3a": 1, + "ecommerce.spec.live-state.purchase-stock.3b": 2, + "ecommerce.feature.purchasing.purchase-order.3c": 1, + "ecommerce.spec.concurrency-safety.stock-limit.3d": 2, + "ecommerce.feature.cart-checkout.cart.4a": 1, + "ecommerce.spec.state-durability.cart-reload.4b": 1, + "ecommerce.spec.live-state.shared-cart.4c": 2, + "ecommerce.feature.cart-checkout.cart.4d": 2, + "ecommerce.feature.reviews.reviews.6a": 2, + "ecommerce.spec.transactional-integrity.unique-review.6b": 1, + "ecommerce.spec.live-state.rating.6c": 1, + "ecommerce.spec.access-control.warehouse-area-boundary.7a": 1, + "ecommerce.feature.warehouse-admin.warehouse-view.7b": 1, + "ecommerce.spec.live-state.warehouse-stock.7c": 2, + "ecommerce.spec.access-control.purchase-session.101a": 2, + "ecommerce.spec.access-control.purchase-attribution.102a": 2, + "ecommerce.feature.warehouse-admin.admin-write.103a": 2, + "ecommerce.spec.transactional-integrity.server-price.104a": 2, + "ecommerce.spec.state-durability.account-state-recovery.105a": 1, + "ecommerce.spec.state-durability.account-state-recovery.105b": 1, + "ecommerce.spec.access-control.order-ownership.106a": 1, + "ecommerce.spec.transactional-integrity.books-balance.107a": 1, + "ecommerce.spec.transactional-integrity.books-balance.107b": 1, + "ecommerce.spec.access-control.review-eligibility.108a": 2, + "ecommerce.spec.access-control.review-eligibility.108b": 1, + "ecommerce.spec.access-control.cart-boundary.109a": 2, + "ecommerce.spec.access-control.cart-boundary.109b": 1, + "ecommerce.spec.concurrency-safety.last-unit.201a": 1, + "ecommerce.spec.concurrency-safety.last-unit.201b": 1, + "ecommerce.spec.concurrency-safety.last-unit.201c": 1, + "ecommerce.spec.concurrency-safety.restock-race.202-control": 0, + "ecommerce.spec.concurrency-safety.restock-race.202a": 1, + "ecommerce.spec.concurrency-safety.duplicate-checkout.203a": 1, + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b": 1, + "ecommerce.spec.external-data-sync.external-stock.901a": 1, + "ecommerce.spec.external-data-sync.external-stock.901b": 0, + "ecommerce.spec.external-data-sync.external-stock.901d": 1, + "ecommerce.spec.live-state.open-list.902a": 1, + "ecommerce.operations-access.fulfilment-queue.1a": 2, + "ecommerce.operations-access.fulfilment-queue.1b": 1, + "ecommerce.operations-access.fulfilment-queue.1c": 2, + "ecommerce.operations-access.fulfilment-queue.1d": 1, + "ecommerce.operations-access.fulfilment-queue.1e": 2, + "ecommerce.inventory-operations.warehouse-transfer.2a": 3, + "ecommerce.returns-pricing.cancellation-and-return.3a": 3, + "ecommerce.returns-pricing.cancellation-and-return.3b": 1, + "ecommerce.returns-pricing.cancellation-and-return.3c": 3, + "ecommerce.returns-pricing.price-history.4a": 3, + "ecommerce.returns-pricing.price-history.4b": 2, + "ecommerce.inventory-operations.operational-views.5a": 3, + "ecommerce.inventory-operations.operational-views.5b": 3, + "ecommerce.inventory-operations.operational-views.5c": 2, + "ecommerce.inventory-operations.operational-views.5d": 1, + "ecommerce.operations-access.operator-authorization.201a": 2, + "ecommerce.operations-access.operator-authorization.201b": 2, + "ecommerce.inventory-operations.stock-conservation.202b": 2, + "ecommerce.returns-pricing.refund-accounting.203a": 3, + "ecommerce.returns-pricing.refund-accounting.203b": 2, + "ecommerce.operations-access.operator-authorization.201c": 2, + "ecommerce.inventory-operations.stock-conservation.202d": 2, + "ecommerce.operations-access.order-owner.204a": 2, + "ecommerce.returns-pricing.price-history.4c": 3, + "ecommerce.inventory-operations.warehouse-transfer.2c": 2 + } + } +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/recipes/sequential-l3.json b/tools/stack-bench/tracks/ecommerce/composition/recipes/sequential-l3.json new file mode 100644 index 00000000000..a524a6c7eed --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/recipes/sequential-l3.json @@ -0,0 +1,494 @@ +{ + "schemaVersion": 1, + "kind": "benchmark-recipe", + "id": "ecommerce.sequential-l3", + "title": "Ecommerce sequential L3", + "track": "ecommerce", + "sequence": { + "level": 3 + }, + "fixture": { + "path": "../fixtures/operations.json", + "id": "ecommerce.operations" + }, + "task": { + "mode": "upgrade", + "baseRecipe": { + "path": "sequential-l2.json", + "id": "ecommerce.sequential-l2" + }, + "framing": { + "requirements": [ + { + "id": "ecommerce.l3.introduction", + "path": "prompts/03-scheduled.md", + "order": 10, + "until": "## Reservations that expire", + "modes": [ + "upgrade" + ] + }, + { + "id": "ecommerce.l3.starting-data", + "path": "prompts/03-scheduled.md", + "order": 3700, + "from": "## Starting data", + "modes": [ + "upgrade" + ] + } + ], + "contracts": [] + } + }, + "packs": [ + { + "path": "../packs/feature-accounts.json", + "id": "ecommerce.feature.accounts", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "account-create", + "account-duplicate", + "account-signout" + ] + }, + { + "path": "../packs/feature-catalog-items.json", + "id": "ecommerce.feature.catalog-items", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "values" + ] + }, + { + "path": "../packs/feature-catalog-discovery.json", + "id": "ecommerce.feature.catalog-discovery", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "ranking", + "search" + ] + }, + { + "path": "../packs/feature-purchasing.json", + "id": "ecommerce.feature.purchasing", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "purchase-order" + ] + }, + { + "path": "../packs/feature-cart.json", + "id": "ecommerce.feature.cart", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "cart" + ] + }, + { + "path": "../packs/feature-checkout.json", + "id": "ecommerce.feature.checkout", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "checkout" + ] + }, + { + "path": "../packs/feature-reviews.json", + "id": "ecommerce.feature.reviews", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "reviews" + ] + }, + { + "path": "../packs/feature-warehouse-admin.json", + "id": "ecommerce.feature.warehouse-admin", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "warehouse-view", + "admin-write" + ] + }, + { + "path": "../packs/spec-access-control.json", + "id": "ecommerce.spec.access-control", + "includeRoles": [ + "guarantee" + ], + "includeCheckGroups": [ + "account-password", + "signed-out-purchase", + "purchase-session", + "purchase-attribution", + "order-ownership", + "review-eligibility", + "cart-boundary", + "warehouse-area-boundary" + ] + }, + { + "path": "../packs/spec-state-durability.json", + "id": "ecommerce.spec.state-durability", + "includeRoles": [ + "guarantee" + ], + "includeCheckGroups": [ + "session-reload", + "cart-reload", + "account-state-reload", + "account-state-reconnect" + ] + }, + { + "path": "../packs/spec-live-state.json", + "id": "ecommerce.spec.live-state", + "includeRoles": [ + "guarantee" + ], + "includeCheckGroups": [ + "ranking", + "purchase-stock", + "shared-cart", + "rating", + "open-list", + "warehouse-stock" + ] + }, + { + "path": "../packs/spec-concurrency-safety.json", + "id": "ecommerce.spec.concurrency-safety", + "includeRoles": [ + "guarantee" + ], + "includeCheckGroups": [ + "stock-limit", + "last-unit", + "restock-race", + "duplicate-checkout" + ] + }, + { + "path": "../packs/spec-transactional-integrity.json", + "id": "ecommerce.spec.transactional-integrity", + "includeRoles": [ + "guarantee" + ], + "includeCheckGroups": [ + "unique-review", + "server-price", + "books-balance" + ] + }, + { + "path": "../packs/spec-external-data-sync.json", + "id": "ecommerce.spec.external-data-sync", + "includeRoles": [ + "guarantee" + ], + "includeCheckGroups": [ + "external-live", + "external-reload", + "external-reconnect" + ] + }, + { + "path": "../packs/operations-access-features.json", + "id": "ecommerce.operations-access-features", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "fulfilment-core", + "fulfilment-warehouse", + "fulfilment-independent" + ] + }, + { + "path": "../packs/operations-access-specifications.json", + "id": "ecommerce.operations-access-specifications", + "includeRoles": [ + "guarantee" + ], + "includeCheckGroups": [ + "operator-transfer-authorization", + "operator-price-authorization", + "operator-authorization-direct", + "order-owner-direct" + ] + }, + { + "path": "../packs/inventory-operations-features.json", + "id": "ecommerce.inventory-operations-features", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "warehouse-transfer-corrected", + "operational-views-low-stock", + "operational-views-core", + "operational-views-kept-purchases" + ] + }, + { + "path": "../packs/inventory-operations-specifications.json", + "id": "ecommerce.inventory-operations-specifications", + "includeRoles": [ + "guarantee" + ], + "includeCheckGroups": [ + "stock-conservation-cancel", + "stock-conservation-contention", + "warehouse-transfer-overdraw" + ] + }, + { + "path": "../packs/returns-pricing-features.json", + "id": "ecommerce.returns-pricing-features", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "cancellation-core", + "cancellation-history", + "return-complete", + "price-history-exact", + "price-live-independent" + ] + }, + { + "path": "../packs/returns-pricing-specifications.json", + "id": "ecommerce.returns-pricing-specifications", + "includeRoles": [ + "guarantee" + ], + "includeCheckGroups": [ + "refund-accounting", + "price-cart-checkout" + ] + }, + { + "path": "../packs/l3-cart-expiration-features.json", + "id": "ecommerce.l3.cart-expiration-features", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "inactive-cart" + ] + }, + { + "path": "../packs/l3-deferred-access-specifications.json", + "id": "ecommerce.l3.deferred-access-specifications", + "includeRoles": [ + "guarantee" + ], + "includeCheckGroups": [ + "customer-refused" + ] + }, + { + "path": "../packs/l3-deferred-durability-specifications.json", + "id": "ecommerce.l3.deferred-durability-specifications", + "includeRoles": [ + "guarantee" + ], + "includeCheckGroups": [ + "reservation-restart", + "delivery-restart", + "cart-restart" + ] + }, + { + "path": "../packs/l3-deferred-integrity-specifications.json", + "id": "ecommerce.l3.deferred-integrity-specifications", + "includeRoles": [ + "guarantee" + ], + "includeCheckGroups": [ + "restock-once", + "delivery-once", + "reservation-stock", + "reserved-checkout" + ] + }, + { + "path": "../packs/l3-order-delivery-features.json", + "id": "ecommerce.l3.order-delivery-features", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "automatic-delivery", + "cancel-final" + ] + }, + { + "path": "../packs/l3-reservations-features.json", + "id": "ecommerce.l3.reservations-features", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "stock-hold", + "countdown", + "checkout", + "expiry", + "renewal" + ] + }, + { + "path": "../packs/l3-scheduled-restocks-features.json", + "id": "ecommerce.l3.scheduled-restocks-features", + "includeRoles": [ + "feature" + ], + "includeCheckGroups": [ + "pending", + "applies", + "cancels" + ] + }, + { + "path": "../packs/l3-server-time-specifications.json", + "id": "ecommerce.l3.server-time-specifications", + "includeRoles": [ + "guarantee" + ], + "includeCheckGroups": [ + "not-early", + "no-browser" + ] + }, + { + "path": "../packs/progression-staff-access.json", + "id": "ecommerce.progression.staff-access", + "includeRoles": [] + }, + { + "path": "../packs/progression-fulfilment-queue.json", + "id": "ecommerce.progression.fulfilment-queue", + "includeRoles": [] + }, + { + "path": "../packs/l2-order-cancellation-features.json", + "id": "ecommerce.l2.order-cancellation-features", + "includeRoles": [] + } + ], + "execution": "all-selected-sources", + "scoring": { + "mode": "explicit", + "weights": { + "ecommerce.feature.accounts.accounts.1a": 1, + "ecommerce.feature.accounts.accounts.1b": 1, + "ecommerce.spec.access-control.account-password.1c": 1, + "ecommerce.feature.accounts.accounts.1d": 1, + "ecommerce.spec.state-durability.session-reload.1e": 1, + "ecommerce.feature.catalog.catalog-values.2a": 1, + "ecommerce.feature.catalog.catalog-ranking.2b": 1, + "ecommerce.spec.live-state.ranking.2c": 1, + "ecommerce.feature.catalog.catalog-search.2d": 1, + "ecommerce.spec.access-control.signed-out-purchase.3a": 1, + "ecommerce.spec.live-state.purchase-stock.3b": 2, + "ecommerce.feature.purchasing.purchase-order.3c": 1, + "ecommerce.spec.concurrency-safety.stock-limit.3d": 2, + "ecommerce.feature.cart-checkout.cart.4a": 1, + "ecommerce.spec.state-durability.cart-reload.4b": 1, + "ecommerce.spec.live-state.shared-cart.4c": 2, + "ecommerce.feature.cart-checkout.cart.4d": 2, + "ecommerce.feature.reviews.reviews.6a": 2, + "ecommerce.spec.transactional-integrity.unique-review.6b": 1, + "ecommerce.spec.live-state.rating.6c": 1, + "ecommerce.spec.access-control.warehouse-area-boundary.7a": 1, + "ecommerce.feature.warehouse-admin.warehouse-view.7b": 1, + "ecommerce.spec.live-state.warehouse-stock.7c": 2, + "ecommerce.spec.access-control.purchase-session.101a": 2, + "ecommerce.spec.access-control.purchase-attribution.102a": 2, + "ecommerce.feature.warehouse-admin.admin-write.103a": 2, + "ecommerce.spec.transactional-integrity.server-price.104a": 2, + "ecommerce.spec.state-durability.account-state-recovery.105a": 1, + "ecommerce.spec.state-durability.account-state-recovery.105b": 1, + "ecommerce.spec.access-control.order-ownership.106a": 1, + "ecommerce.spec.transactional-integrity.books-balance.107a": 1, + "ecommerce.spec.transactional-integrity.books-balance.107b": 1, + "ecommerce.spec.access-control.review-eligibility.108a": 2, + "ecommerce.spec.access-control.review-eligibility.108b": 1, + "ecommerce.spec.access-control.cart-boundary.109a": 2, + "ecommerce.spec.access-control.cart-boundary.109b": 1, + "ecommerce.spec.concurrency-safety.last-unit.201a": 1, + "ecommerce.spec.concurrency-safety.last-unit.201b": 1, + "ecommerce.spec.concurrency-safety.last-unit.201c": 1, + "ecommerce.spec.concurrency-safety.restock-race.202-control": 0, + "ecommerce.spec.concurrency-safety.restock-race.202a": 1, + "ecommerce.spec.concurrency-safety.duplicate-checkout.203a": 1, + "ecommerce.spec.concurrency-safety.duplicate-checkout.203b": 1, + "ecommerce.spec.external-data-sync.external-stock.901a": 1, + "ecommerce.spec.external-data-sync.external-stock.901b": 0, + "ecommerce.spec.external-data-sync.external-stock.901d": 1, + "ecommerce.spec.live-state.open-list.902a": 1, + "ecommerce.operations-access.fulfilment-queue.1a": 2, + "ecommerce.operations-access.fulfilment-queue.1b": 1, + "ecommerce.operations-access.fulfilment-queue.1c": 2, + "ecommerce.operations-access.fulfilment-queue.1d": 1, + "ecommerce.operations-access.fulfilment-queue.1e": 2, + "ecommerce.inventory-operations.warehouse-transfer.2a": 3, + "ecommerce.returns-pricing.cancellation-and-return.3a": 3, + "ecommerce.returns-pricing.cancellation-and-return.3b": 1, + "ecommerce.returns-pricing.cancellation-and-return.3c": 3, + "ecommerce.returns-pricing.price-history.4a": 3, + "ecommerce.returns-pricing.price-history.4b": 2, + "ecommerce.inventory-operations.operational-views.5a": 3, + "ecommerce.inventory-operations.operational-views.5b": 3, + "ecommerce.inventory-operations.operational-views.5c": 2, + "ecommerce.inventory-operations.operational-views.5d": 1, + "ecommerce.operations-access.operator-authorization.201a": 2, + "ecommerce.operations-access.operator-authorization.201b": 2, + "ecommerce.inventory-operations.stock-conservation.202b": 2, + "ecommerce.returns-pricing.refund-accounting.203a": 3, + "ecommerce.returns-pricing.refund-accounting.203b": 2, + "ecommerce.operations-access.operator-authorization.201c": 2, + "ecommerce.inventory-operations.stock-conservation.202d": 2, + "ecommerce.operations-access.order-owner.204a": 2, + "ecommerce.l3.cart-expiration.cart-expiration.304a": 4, + "ecommerce.l3.deferred-access.scheduled-work-access.317a": 3, + "ecommerce.l3.deferred-durability.restart-survival.314a": 4, + "ecommerce.l3.deferred-durability.restart-survival.315a": 4, + "ecommerce.l3.deferred-durability.restart-survival.316a": 4, + "ecommerce.l3.deferred-integrity.exactly-once.311a": 3, + "ecommerce.l3.deferred-integrity.exactly-once.312a": 3, + "ecommerce.l3.deferred-integrity.stock-conservation.313a": 3, + "ecommerce.l3.deferred-integrity.stock-conservation.314a": 3, + "ecommerce.l3.order-delivery.order-delivery.303a": 3, + "ecommerce.l3.order-delivery.order-delivery.305a": 2, + "ecommerce.l3.reservations.reservations.301a": 2, + "ecommerce.l3.reservations.reservations.305a": 1, + "ecommerce.l3.reservations.reservations.306a": 2, + "ecommerce.l3.reservations.reservations.307a": 3, + "ecommerce.l3.reservations.reservations.308a": 2, + "ecommerce.l3.scheduled-restocks.scheduled-restocks.302a": 2, + "ecommerce.l3.scheduled-restocks.scheduled-restocks.305a": 3, + "ecommerce.l3.scheduled-restocks.scheduled-restocks.306a": 2, + "ecommerce.l3.server-time.server-time.312a": 3, + "ecommerce.l3.server-time.server-time.313a": 3, + "ecommerce.returns-pricing.price-history.4c": 3, + "ecommerce.inventory-operations.warehouse-transfer.2c": 2 + } + } +} diff --git a/tools/stack-bench/tracks/ecommerce/composition/sequential.json b/tools/stack-bench/tracks/ecommerce/composition/sequential.json new file mode 100644 index 00000000000..29068bddf76 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/composition/sequential.json @@ -0,0 +1,9 @@ +{ + "schemaVersion": 1, + "kind": "recipe-selection", + "entries": [ + { "alias": "L1", "recipe": { "path": "recipes/sequential-l1.json", "id": "ecommerce.sequential-l1" } }, + { "alias": "L2", "recipe": { "path": "recipes/sequential-l2.json", "id": "ecommerce.sequential-l2" } }, + { "alias": "L3", "recipe": { "path": "recipes/sequential-l3.json", "id": "ecommerce.sequential-l3" } } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/contracts/accounts.md b/tools/stack-bench/tracks/ecommerce/contracts/accounts.md new file mode 100644 index 00000000000..98e90e1c1d8 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/accounts.md @@ -0,0 +1,46 @@ +# Account application interface + +Use these exact `id` attributes on the corresponding visible controls. They do not prescribe UI +structure, data modeling, libraries, or implementation strategy. + +The signed-out page is the page the application's address opens for a signed-out visitor. +From the signed-out page, show the sign-up inputs, `signup-toggle`, or +`signin-toggle`. If sign-up is inside the sign-in dialog, opening `signin-toggle` +must reveal the sign-up inputs or `signup-toggle`. That control must reveal the +sign-up form. No other navigation is required to reach it. +Show the sign-in inputs or `signin-toggle` on the signed-out page. +While signed in, show `signout` directly or reveal it by clicking `current-user`. +No other navigation is required to reach sign-out. + +| Element ID | Observable element | +|---|---| +| `signup-username` | sign-up username input | +| `signup-password` | sign-up password input | +| `signup-submit` | sign-up submit control | +| `signin-toggle` | control that reveals sign-in | +| `signup-toggle` | reveals sign-up; available on the signed-out page or after opening `signin-toggle`; omit when sign-up inputs are already visible | +| `signin-username` | sign-in username input | +| `signin-password` | sign-in password input | +| `signin-submit` | sign-in submit control | +| `current-user` | Active account name, present only while signed in. Do not use this hook for a signed-out message. | +| `signout` | sign-out control | +| `auth-error` | visible account error | + +Accept any username of up to 48 characters made of letters, digits, and hyphens, and any +password of up to 64 characters. + +Expose `window.getSessionToken()` as a synchronous function. For token-based +authentication, return the caller's existing token accepted by the application's +request transport, such as an HTTP header, native connection, or token argument. +For cookie-only authentication, return `null`. Do not return a cookie value unless +the application also accepts it through its token transport. +Return `null` when no such token exists. A database connection token can exist before +account login or after a rejected login. Return that token too; its presence does +not mean the caller has an authenticated application account. +This hook does not prescribe credential storage. Return the caller's real credential; +do not create a separate identity. + +Account creation and login must use the application's real authentication path. +The interface does not prescribe password-taking backend function names or routes. +Keep credentials out of durable application logs. A failed registration or login +must not give the caller an application session or access to protected operations. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/appendix-02.md b/tools/stack-bench/tracks/ecommerce/contracts/appendix-02.md new file mode 100644 index 00000000000..0c1ba52354d --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/appendix-02.md @@ -0,0 +1,101 @@ + + +## Appendix: Application interface (required) + +Use each interface name below as an exact HTML attribute value. Use `id` for a +one-off element. Use `data-role` when the same interface can appear more than +once. These attributes must not change the design, architecture, or selected +stack. + +Rules: +- Values are exactly as listed in kebab-case. +- An element that is hidden until a menu/toggle opens still counts, as long as it is in the DOM after its toggle is clicked. +- Do not add element IDs beyond this list to elements that could be confused with these. +- Each exposed server operation completes every valid request, including requests that overlap, without relying on the browser to retry it. A business refusal, such as insufficient stock, still completes the request. + +| Element ID | Element | +|---|---| +| `app-title` | the app's visible title, `Storefront` | +| `item-list` | the container holding the storefront's item cards | +| `item-card` | one per item shown on the storefront; contains that item's name, price and stock | +| `catalog-link` | control that returns to the storefront catalog | +| `item-name` | the item's name, inside its card; activating it opens `item-detail` | +| `item-price` | the item's price, inside its card | +| `item-stock` | the item's current stock as a number, inside its card | +| `search-input` | the search box, usable signed out | +| `signup-username` | username input on the sign-up form | +| `signup-password` | password input on the sign-up form | +| `signup-submit` | button that submits the sign-up form | +| `signin-toggle` | available on the signed-out page to reveal sign-in; omit when sign-in inputs are already visible | +| `signup-toggle` | reveals sign-up; available on the signed-out page or after opening `signin-toggle`; omit when sign-up inputs are already visible | +| `signin-username` | username input on the sign-in form | +| `signin-password` | password input on the sign-in form | +| `signin-submit` | button that submits the sign-in form | +| `current-user` | the signed-in account's name; absent while signed out | +| `signout` | control that signs the current account out | +| `buy-now` | button on an item card that buys one unit immediately; shown only to signed-in customers | +| `add-to-cart` | button on an item card that adds one unit to the cart; shown only to signed-in customers | +| `cart-toggle` | control that opens the cart | +| `cart-count` | the number of items currently in the cart | +| `orders-toggle` | control that opens the customer's order history | +| `search-results` | the container holding search results; its item cards use the same controls as the storefront's | +| `item-detail` | the panel or page showing one item, opened from its card | +| `review-rating` | the form control choosing a rating from 1 to 5 — an input or a select, so it can be driven by automation | +| `review-input` | the text input for a new review's comment | +| `review-submit` | button that submits the review | +| `review-average` | the item's average rating as a number | +| `review-item` | one per review on the item, containing its comment | +| `cart-panel` | the cart's container, opened by cart-toggle | +| `cart-item` | one per line in the cart, containing that item's name | +| `cart-quantity` | the quantity of a cart line, as a number | +| `cart-total` | the cart's total price | +| `checkout-submit` | button that turns the cart into an order | +| `order-list` | the container holding the customer's orders, opened by orders-toggle | +| `order-item` | one per order, containing the names of the items bought; set `data-entity-id` to that order's server identifier | +| `order-total` | an order's total price | +| `admin-panel` | the admin area's tools; not a loading or error message | +| `admin-item-row` | one per item in the admin item list, containing that item's name; all item and holding rows are available in the open admin panel without pagination | +| `admin-stock` | an item's total stock in the admin item list, as a number | +| `admin-warehouse-item` | one per warehouse in the admin warehouse list, containing its name | +| `admin-location-row` | one per item-in-warehouse holding, containing the item and warehouse it refers to | +| `admin-location-qty` | the quantity held in one warehouse for one item, as a number | +| `restock-input` | the input for how many units to add, inside the item-in-warehouse row it restocks — one per admin-location-row, so the row identifies both the item and the warehouse | +| `restock-submit` | button that applies the restock, inside the same admin-location-row as its restock-input | +| `admin-revenue` | total revenue across all orders, as a number | +| `buy-error` | appears when buying an out-of-stock item, or checking out more than is available | +| `auth-error` | appears on a taken username or a wrong password | +| `review-error` | visible review error | +| `review-toggle` | opens the review form on item details for eligible customers; omit when already shown | +| `out-of-stock` | visible out-of-stock state | +| `empty-cart` | shown before anything is added, and again after checkout empties the cart | +| `cart-remove` | checked with a populated cart by the application | +| `admin-link` | opens administrator tools; the link may remain visible to other accounts | +| `staff-link` | opens staff tools; the link may remain visible to customers | +| `fulfilment-panel` | fulfilment tools, including an empty queue; not a loading or error message | +| `queue-item` | one per order waiting to ship, oldest first, naming the items it contains | +| `queue-warehouse` | the warehouse a queued order's item will ship from, inside its queue-item | +| `queue-depth` | how many orders are waiting, as a number | +| `ship-submit` | button on a queued order that marks it shipped | +| `transfer-from` | the control choosing the warehouse a transfer takes stock from, inside the item's admin row; for a `select`, each warehouse has exactly one option whose value or label is its exact name; the other field may include extra display text or use the warehouse ID | +| `transfer-to` | the control choosing the warehouse a transfer sends stock to, inside the same row; for a `select`, each warehouse has exactly one option whose value or label is its exact name; the other field may include extra display text or use the warehouse ID | +| `transfer-qty` | how many units to transfer | +| `transfer-submit` | button that performs the transfer | +| `price-input` | the input for an item's new price, inside its admin row | +| `price-submit` | button that applies the new price | +| `low-stock-list` | the container listing items at or below ten units | +| `low-stock-link` | control that opens the low-stock list when it is on a separate screen within the admin area; omit it when the list is already shown there | +| `low-stock-item` | one low-stock item | +| `warehouse-total` | the number of units a warehouse holds, inside its admin-warehouse-item | +| `category-row` | one per category in the admin totals, naming the category; present for a category with no sales yet, reading 0 | +| `sales-link` | opens category totals from the admin area when they are on a separate tab or screen; omit when already shown | +| `category-units` | units sold in that category, as a number, inside its category-row | +| `category-revenue` | revenue earned by that category, as a number, inside its category-row | +| `recommended-list` | the container holding a signed-in customer's recommendations | +| `recommended-item` | appears once the customer has bought from a category; excludes items already in their cart | +| `order-status` | an order's state — pending, shipped, cancelled — inside its order-item | +| `cancel-order` | button on a pending order that cancels it | +| `return-item` | only on orders that have shipped | +| `order-error` | visible order or transfer error | + +Before declaring DEPLOY_COMPLETE, verify the interface by running the contract +linter (command provided in your build instructions) and fix any failures. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/application-interface.md b/tools/stack-bench/tracks/ecommerce/contracts/application-interface.md new file mode 100644 index 00000000000..823753ba7c5 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/application-interface.md @@ -0,0 +1,40 @@ +# Application interface + +Use each interface name below as an exact HTML attribute value. Use `id` for +a one-off element. Use `data-role` when the same interface can +appear more than once. These attributes do not prescribe the layout, data +model, libraries, or transport. + +Each exposed server operation completes every valid request, including requests that +overlap, without relying on the browser to retry it. A business refusal, such as insufficient +stock, still completes the request. + + +For expected refusals, use HTTP 400, 401, 403, 409, or 422. Prefer 401 or 403 for +access errors and 400, 409, or 422 for input or business-rule errors. +An existing operation may return 404 to conceal a resource from an unauthorized user. + + + +Report reducer errors by failing the call. + + + +Expose the named application operations below as public native Convex mutations in `convex/api.ts`. +Use the same mutations from the visible controls. Pass arguments as JSON objects with the +names shown below. Preserve native document identifiers as strings. Handlers reject expected +access or input errors with `ConvexError`. Native argument validation may also refuse a request +before its handler runs. Successful calls use the normal Convex return value. +Do not add HTTP routes merely to wrap these mutations. + + + +Expose each named application operation below as a PostgreSQL function in the `public` +schema, called through the data API at `$SUPABASE_URL/rest/v1/rpc/`. Use the same +functions from the visible controls. Name the parameters exactly as the JSON arguments +shown below. Raise expected access errors with SQLSTATE `42501`, and input errors or +business refusals with SQLSTATE `P0001` or `22023`; successful calls return the function's +normal result. Do not add Edge Functions or HTTP routes merely to wrap these functions. + + +Human-readable status text is case-insensitive. Machine identifiers and protocol values keep their specified spelling. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/bundle-checkout.md b/tools/stack-bench/tracks/ecommerce/contracts/bundle-checkout.md new file mode 100644 index 00000000000..6e5e2a60b93 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/bundle-checkout.md @@ -0,0 +1,25 @@ +# Bundle checkout interface + +Use `bundle-add-to-cart` inside `bundle-card`. A bundle cart line uses the existing +`cart-item`, `cart-reservation-timer`, and `cart-item-expired` interfaces, with +`bundle-remove` to remove it. `checkout-submit` buys the cart through the existing checkout +action. The existing `order-item` and `payment-amount` show the bundle name and price paid. + + +Add one bundle with `POST /api/cart/bundles` and `{ bundleId }`. + + + +Add one bundle with `add_bundle_to_cart(bundleId: u64)`. + + + +Add one bundle with `api:add_bundle_to_cart` and `{ bundleId }`. + + + +Add one bundle with the `add_bundle_to_cart` PostgreSQL function and `{ bundleId }`. + + +Use the same application action as the visible control. The `data-bundle-input` attribute +supplies its bundle ID. The cart and order interfaces remain shared with individual products. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/bundle-returns.md b/tools/stack-bench/tracks/ecommerce/contracts/bundle-returns.md new file mode 100644 index 00000000000..4c6e0a7dd83 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/bundle-returns.md @@ -0,0 +1,25 @@ +# Bundle return interface + +Use `return-bundle` inside the existing `order-item`. Mark each returned bundle line +`returned`. The order's `order-status` reads `returned` when all its lines have been +returned; otherwise keep its current fulfilment status. `bundle-refund-amount` shows +the refunded amount in currency units. +Each bundle `order-item` exposes `data-bundle-return-input` as JSON `{ "orderId": "..." }`. + + +Return a whole bundle with `POST /api/bundle-orders/:orderId/return`. + + + +Return a whole bundle with `return_bundle(orderId: u64)`. + + + +Return a whole bundle with `api:return_bundle` and `{ orderId }`. + + + +Return a whole bundle with the `return_bundle` PostgreSQL function and `{ orderId }`. + + +Use the same application action as the visible control. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/cart-expiration.md b/tools/stack-bench/tracks/ecommerce/contracts/cart-expiration.md new file mode 100644 index 00000000000..8bffbeb1c3a --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/cart-expiration.md @@ -0,0 +1,5 @@ +# Cart expiration application interface + +Use `cart-reservation-timer` inside a `cart-item` for the remaining reservation time in seconds. +Use `cart-item-expired` inside an expired cart line and `cart-expired-notice` after cart +expiration. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/cart.md b/tools/stack-bench/tracks/ecommerce/contracts/cart.md new file mode 100644 index 00000000000..4422b2cf99b --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/cart.md @@ -0,0 +1,39 @@ +# Cart application interface + +Use `catalog-link` to return to the catalog. Use `add-to-cart` inside an `item-card` to add one unit. Use `cart-toggle` to open the cart. +While signed in, show `cart-toggle` directly or reveal it by clicking `current-user`. +No other navigation is required to reach it. +If an overlay blocks catalog navigation, expose a visible `overlay-close` control +that dismisses it before `catalog-link` is used. Screens without a blocking +overlay need no such control. Dialogs, panels, and ordinary page layouts are all allowed. +Use `cart-count` for the total units, `cart-item` for each line, `cart-quantity` for its +quantity, and `cart-total` for the numeric total, shown only in the open cart. Use `cart-remove` to remove a line and +`empty-cart` for an empty cart. Keep `cart-count` visible, showing 0, while the cart is empty. + +Put `data-buy-input` on each `item-card`. Its value is a JSON object containing that item's +server identifier, for example `{"itemId":42}`. Put `data-cart-input` on each `cart-item`. +Its value contains the item identifier, for example `{"itemId":42}`. The identifier may be a +JSON number or string. + +Expose the same add and quantity-update operations used by the cart controls. + + +Use `POST /api/cart`. Put `itemId` in the JSON body. +Use `PATCH /api/cart/:itemId`. Put `quantity` in the JSON body. + + + +Use the `add_to_cart` reducer with the item identifier. +Use the `update_cart_quantity` reducer with the item identifier and quantity. + + + +Use `api:add_to_cart` with `{ itemId }` and `api:update_cart_quantity` with `{ itemId, quantity }`. + + + +Use the `add_to_cart` PostgreSQL function with `{ itemId }` and `update_cart_quantity` with `{ itemId, quantity }`. + + +Use `checkout-submit` to check out. Use `orders-toggle` to open order history and `order-item` +for each order created by checkout. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/catalog-discovery.md b/tools/stack-bench/tracks/ecommerce/contracts/catalog-discovery.md new file mode 100644 index 00000000000..9813141284e --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/catalog-discovery.md @@ -0,0 +1,12 @@ +# Catalog discovery application interface + +New catalog data starts with zero purchases. Do not seed sample orders or purchase counts. +When adding this feature to an existing app, preserve purchases made through the app. + +| Element ID | Required element | +| --- | --- | +| `item-list` | Contains exactly the ten ranked storefront items. | +| `item-card` | Shows one storefront or search result. | +| `item-name` | Shows exactly the item name inside its `item-card`. Put icons, badges, and other text outside this element. | +| `search-input` | Searches the full catalog as the visitor types or when they press Enter; no separate control runs the search. | +| `search-results` | Contains matching `item-card` results. | diff --git a/tools/stack-bench/tracks/ecommerce/contracts/catalog-items.md b/tools/stack-bench/tracks/ecommerce/contracts/catalog-items.md new file mode 100644 index 00000000000..c58b7495e3c --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/catalog-items.md @@ -0,0 +1,14 @@ +# Catalog item application interface + +The public catalog may be on the first page or a separate page. If navigation is +needed, show `catalog-link` directly on the signed-out first page to open the catalog. + +| Element ID | Required element | +| --- | --- | +| `catalog-link` | Opens the public catalog; may be omitted when the catalog is already visible. | +| `item-list` | Contains the public catalog items. | +| `item-card` | Shows one catalog item. | +| `item-name` | Shows exactly the item name inside its `item-card`; activating it opens `item-detail`. Put icons, badges, and other text outside this element. | +| `item-price` | Shows the numeric item price inside its `item-card`. | +| `item-stock` | Shows total stock inside its `item-card`. | +| `item-detail` | Contains the selected item's details. | diff --git a/tools/stack-bench/tracks/ecommerce/contracts/catalog-management.md b/tools/stack-bench/tracks/ecommerce/contracts/catalog-management.md new file mode 100644 index 00000000000..0b8c8040fe8 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/catalog-management.md @@ -0,0 +1,10 @@ +# Catalog management application interface + +Use `admin-link` to open the administrator area containing the product controls. +If these controls are on a separate tab or screen within it, expose `catalog-management-link` +there to open them. Omit this control when the product controls are already shown. +Use `catalog-name`, `catalog-category`, `catalog-price`, and `catalog-variants` for the product +values; `catalog-category` accepts a new category name as text, and `catalog-variants` accepts +comma-separated variant names. Use `catalog-save` to add the product. Saving may close the product +controls; `catalog-management-link` then opens them again. Use `item-variant` for each named variant shown +to a visitor. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/checkout.md b/tools/stack-bench/tracks/ecommerce/contracts/checkout.md new file mode 100644 index 00000000000..a4271594fff --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/checkout.md @@ -0,0 +1,27 @@ +# Checkout application interface + +Use `checkout-submit` to check out and `buy-error` for a failed checkout. Use `orders-toggle` +to open order history and `order-item` for each order created by checkout. +Opening a nonempty cart with `cart-toggle` makes `checkout-submit` available without another +navigation step. A signed-in customer can check out that cart without entering additional +required fields. Any extra checkout fields are optional. +While signed in, show `orders-toggle` directly or reveal it by clicking `current-user`. +No other navigation is required to reach it. + +Expose the same checkout used by `checkout-submit`. + + +Use `POST /api/checkout`. + + + +Use the `checkout` reducer. + + + +Use the `api:checkout` mutation with `{}`. + + + +Use the `checkout` PostgreSQL function with `{}`. + diff --git a/tools/stack-bench/tracks/ecommerce/contracts/customer-profile.md b/tools/stack-bench/tracks/ecommerce/contracts/customer-profile.md new file mode 100644 index 00000000000..37c6a1460de --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/customer-profile.md @@ -0,0 +1,16 @@ +# Customer profile application interface + +Use `catalog-link` to return to the catalog. If a profile overlay blocks navigation, +expose `overlay-close` to dismiss it. Saving may leave the profile open. + +While signed in on the catalog, show `profile-link` directly or reveal it by clicking +`current-user`. No other navigation is required to reach it. Use it to open the profile. +Use `profile-name` and `profile-address` for the editable +values. Use `profile-save` to save them. Use `profile-address-summary` to display +the saved address in the profile view. + +Expose `profile-save-state` as the status of the latest save, with `data-submit-state`: +`idle` initially, `pending` immediately when submitted, `succeeded` only after the +server confirms success, or `failed` after rejection or transport failure. Keep this +status visible if the editor closes. Keep its terminal state until the next submission +or page navigation. A new submission must replace the old state immediately. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/delivery-notifications.md b/tools/stack-bench/tracks/ecommerce/contracts/delivery-notifications.md new file mode 100644 index 00000000000..b13b5ed53da --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/delivery-notifications.md @@ -0,0 +1,7 @@ +# Delivery notification application interface + +Use `notifications-toggle` to open notifications. Use `notification-item` for each notification +and `notification-unread-count` for the unread total. + +The `notifications-panel` has `aria-busy="false"` only when the signed-in account's +notifications have loaded. Keep the panel present when the list is empty. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/faceted-search.md b/tools/stack-bench/tracks/ecommerce/contracts/faceted-search.md new file mode 100644 index 00000000000..5b7375f61fc --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/faceted-search.md @@ -0,0 +1,19 @@ +# Faceted search application interface + +| Element ID | Required element | +| --- | --- | +| `category-filter` | Sets the category filter; a text input, or a `select` whose option values are the category names. | +| `minimum-price` | Sets the inclusive minimum price. | +| `maximum-price` | Sets the inclusive maximum price. | +| `in-stock-filter` | Toggles the in-stock-only filter, which starts off. | +| `search-results` | Contains the filtered page; with no search text and no filter selected, the current page of the full catalog. | +| `item-card` | Shows one result inside `search-results`. | +| `search-next-page` | Opens the next page. | +| `search-previous-page` | Opens the previous page. | + +If a `filter-apply` control exists, activating it applies the filters; otherwise results update +as each filter changes. + +Search text or any active filter selects alphabetical ordering. With neither, use +purchase ranking and break ties by item name. Clearing all search text and filters +restores purchase ranking. Both modes can use the same rendered list. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/inventory-dashboard.md b/tools/stack-bench/tracks/ecommerce/contracts/inventory-dashboard.md new file mode 100644 index 00000000000..7402f10840b --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/inventory-dashboard.md @@ -0,0 +1,6 @@ +# Inventory dashboard application interface + +Use `admin-link` to open the administrator area. If `low-stock-list` is on a separate screen +within it, expose `low-stock-link` there to reach it. Use `low-stock-list` for items with 10 units +or fewer and `low-stock-item` for each item. Use +`buy-now` inside an `item-card` to create sales that change available stock. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/l2-order-hooks.md b/tools/stack-bench/tracks/ecommerce/contracts/l2-order-hooks.md new file mode 100644 index 00000000000..c71bc32aa92 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/l2-order-hooks.md @@ -0,0 +1,4 @@ +| `order-status` | an order's state — pending, shipped, or cancelled — inside its `order-item` | +| `cancel-order` | button on a pending order that cancels it | +| `return-item` | only on orders that have shipped | +| `order-error` | visible order or transfer error | diff --git a/tools/stack-bench/tracks/ecommerce/contracts/managed-support.md b/tools/stack-bench/tracks/ecommerce/contracts/managed-support.md new file mode 100644 index 00000000000..05d65b70a02 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/managed-support.md @@ -0,0 +1,30 @@ +# Managed support application interface + +## Managed support controls + +Use `support-ticket` for each case and set `data-entity-id` to that case's server identifier. +Within a case, use `support-status` for the current status, `support-reply` for the reply field, +`support-reply-submit` to send a reply, and `support-reply-item` for each reply. Show these +wherever the case is listed, without another click. + +Expose the same reply operation used by `support-reply-submit`. + + +Use `POST /api/support/:id/replies`, where `:id` is the case identifier from `data-entity-id`. +The JSON body is `{ "body": "" }`. + + + +Use the `reply_support` reducer with arguments in this order: `ticketId: u64`, `body: string`. +Render `data-entity-id` as the decimal case identifier without precision loss. + + + +Use `api:reply_support` with `{ ticketId, body }`. Render `data-entity-id` as +the native string case identifier. + + + +Use the `reply_support` PostgreSQL function with `{ ticketId, body }`. Render `data-entity-id` +as the case identifier accepted by `ticketId`, without precision loss. + diff --git a/tools/stack-bench/tracks/ecommerce/contracts/notification-preferences.md b/tools/stack-bench/tracks/ecommerce/contracts/notification-preferences.md new file mode 100644 index 00000000000..b8dd84ed601 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/notification-preferences.md @@ -0,0 +1,18 @@ +# Notification preference application interface + +Saving may leave settings open. Use `catalog-link` to return to the catalog. If +a settings overlay blocks navigation, expose `overlay-close` to dismiss it. + +While signed in on the catalog, show `notification-settings` directly or reveal it by clicking +`current-user`. No other navigation is required to reach it. Use it to open the settings. +Use `notification-order` and +`notification-stock` for the choices, and `notification-save` to save them. Each choice exposes +its current state in `data-state` as `on` or `off`. +Both choices start `off` for a new account. Activating `notification-order` or +`notification-stock` switches it between `on` and `off`. + +Expose `notification-save-state` as the status of the latest save, with `data-submit-state`: +`idle` initially, `pending` immediately when submitted, `succeeded` only after the +server confirms success, or `failed` after rejection or transport failure. Keep this +status visible if the editor closes. Keep its terminal state until the next submission +or page navigation. A new submission must replace the old state immediately. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/operations-access.md b/tools/stack-bench/tracks/ecommerce/contracts/operations-access.md new file mode 100644 index 00000000000..57912fc7c7a --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/operations-access.md @@ -0,0 +1,41 @@ +# Fulfilment application interface + +| Element ID | Required element | +| --- | --- | +| `staff-link` | Opens the fulfilment area. | +| `fulfilment-panel` | Contains fulfilment tools, including an empty queue; not a loading or error message. | +| `queue-depth` | Shows the number of pending orders. | +| `queue-item` | Shows one pending order and names its items. | +| `queue-warehouse` | Shows the selected warehouse inside its `queue-item`. | +| `ship-submit` | Marks the order in its `queue-item` as shipped. | + +On `fulfilment-panel`, expose `data-submit-state` for the latest shipping submission: +`idle` initially, `pending` immediately when submitted, `succeeded` only after the server +confirms success, or `failed` after rejection or transport failure. Keep the terminal state +on the panel when the shipped row disappears. A new submission must replace the old state. + +`order-status` reads `pending` until the order ships, `shipped` once it has, and `cancelled` +after a cancellation. Later features may add further states after `shipped`. +The visible `order-status` text may have whitespace, a middle dot (`·`), or a bullet (`•`) +before or after the state name. Do not add other words or state names to this element. + +Each customer `order-item` must have `data-ship-input` containing a JSON object with +exactly `orderId`. + +Use the identifier representation required by the selected stack. + + +Use `POST /api/fulfilment/ship` with `{ "orderId": ... }`. + + + +Use the `ship_order` reducer. + + + +Use `api:ship_order` with `{ orderId }`. + + + +Use the `ship_order` PostgreSQL function with `{ orderId }`. + diff --git a/tools/stack-bench/tracks/ecommerce/contracts/order-cancellation.md b/tools/stack-bench/tracks/ecommerce/contracts/order-cancellation.md new file mode 100644 index 00000000000..d7a28db8bf7 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/order-cancellation.md @@ -0,0 +1,27 @@ +# Order cancellation application interface + +Use `order-status` for an order's state inside its `order-item`. Use `cancel-order` on a pending +order. Use `catalog-link` to return to the catalog. +`order-status` reads `pending` until the order ships, `shipped` once it has, and `cancelled` +after a cancellation. Later features may add further states after `shipped`. +The visible `order-status` text may have whitespace, a middle dot (`·`), or a bullet (`•`) +before or after the state name. Do not add other words or state names to this element. + +Each customer `order-item` must have `data-cancel-input` containing a JSON object with +exactly `orderId`. Use the identifier representation required by the selected stack. + + +Use `POST /api/orders/:id/cancel`. + + + +Use the `cancel_order` reducer. + + + +Use `api:cancel_order` with `{ orderId }`. + + + +Use the `cancel_order` PostgreSQL function with `{ orderId }`. + diff --git a/tools/stack-bench/tracks/ecommerce/contracts/order-data.md b/tools/stack-bench/tracks/ecommerce/contracts/order-data.md new file mode 100644 index 00000000000..5c8b611488f --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/order-data.md @@ -0,0 +1,35 @@ +# Order data interface + +Expose the following data through the database's native read tools. These names +describe a read interface; tables, collections, or database-native views over the +application's current records are allowed. Do not maintain separate copies for +this interface. Extra columns are allowed. + +- `item(id, name, price)` identifies catalog items and their current prices. These are the same items shown in the catalog. +- `order_account(id, username)` identifies customer accounts. Do not include passwords or tokens in this interface. +- `order_header(id, account_id, total, refunded, status)` contains every order from direct purchase or checkout, including cancelled orders. `total` is the amount booked for the order. `refunded` is the amount refunded so far, initially zero. +- `order_line(id, order_id, item_id, quantity, unit_price)` contains each order's purchased lines and their booked unit prices. +- When carts are available, `order_cart(account_id, item_id, quantity)` contains their current lines. An empty cart has no lines. +- When carts and warehouse stock are available, `order_reservation(account_id, item_id, warehouse_id, quantity)` contains any stock held for those carts and already deducted from available `stock.quantity`. If the app does not hold stock for carts, this read interface is empty. This does not require adding stock reservations to the app. +- When warehouse stock is available, `order_allocation(order_line_id, warehouse_id, quantity)` contains the original warehouse quantities used for each order line. Keep these quantities available after cancellation. + +Each `id` is a nonempty string or an exact nonnegative integer. Related identifiers +refer to that same `id`; a document may use `_id` when it has no `id`. Item and warehouse +identifiers match the existing `item` and `warehouse` data interfaces and the visible +application actions. Quantities are whole numbers. Money fields use the same currency +units as displayed prices, with at most two decimal places. Status uses the states in +the order interface. + +The names and fields above must remain readable by the supplied database credentials +as features are added. Customer screens and writes must use the same underlying records. +This does not require making customer data available to unauthenticated app users. + + +For Convex, each `id` in this interface is the native document `_id`. Store related +identifiers as references to those same `_id` values. Keep the table and field names +above; no additional `id` column or copy of the application data is needed. + + + +For Supabase, the names above are tables or views in the `public` schema. + diff --git a/tools/stack-bench/tracks/ecommerce/contracts/order-delivery.md b/tools/stack-bench/tracks/ecommerce/contracts/order-delivery.md new file mode 100644 index 00000000000..92eac91a893 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/order-delivery.md @@ -0,0 +1,5 @@ +# Order delivery application interface + +Use `completed-order-item` for each completed order in the staff view. Use +`completed-order-status` inside it for the current state. This extends the order lifecycle: +after `shipped`, `order-status` and `completed-order-status` read `delivered`. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/order-returns.md b/tools/stack-bench/tracks/ecommerce/contracts/order-returns.md new file mode 100644 index 00000000000..e3690b943db --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/order-returns.md @@ -0,0 +1,27 @@ +# Order return application interface + +Use `return-item` inside an `order-item` for each item that can be returned. After a return, +the same `order-item` contains the word `returned`. + +Each ordinary item has an `order-line` containing its name, including while pending. +Set `data-return-input` on that line to JSON with `orderId` and `itemId`, using the +identifiers accepted by the return action. + + +`returnItem` is `POST /api/orders/{orderId}/items/{itemId}/return`. + + + +`returnItem` is `return_order_item(orderId, itemId)`. + + + +`returnItem` is the `api:return_order_item` mutation with `{ orderId, itemId }`. + + + +`returnItem` is the `return_order_item` PostgreSQL function with `{ orderId, itemId }`. + + +The existing `orders-toggle`, `order-item`, `item-stock`, `admin-revenue`, and `catalog-link` +interfaces expose the order, stock, and accounting results. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/order-support.md b/tools/stack-bench/tracks/ecommerce/contracts/order-support.md new file mode 100644 index 00000000000..0ee7e8b5d1b --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/order-support.md @@ -0,0 +1,24 @@ +# Order-linked support application interface + +## Order-linked support controls + +Within a `support-ticket`, use `support-order-option` for each order that the customer can attach, +`support-link-order` to attach the selected order, and `support-order` for the attached order. The +link action must expose its input in `data-action-input` for the named +`linkSupportOrder` application action. + + +`linkSupportOrder` is `POST /api/support/cases/{caseId}/order`. + + + +`linkSupportOrder` is the `link_support_order(caseId, orderId)` reducer. + + + +`linkSupportOrder` is the `api:link_support_order` mutation with `{ caseId, orderId }`. + + + +`linkSupportOrder` is the `link_support_order` PostgreSQL function with `{ caseId, orderId }`. + diff --git a/tools/stack-bench/tracks/ecommerce/contracts/payment-records.md b/tools/stack-bench/tracks/ecommerce/contracts/payment-records.md new file mode 100644 index 00000000000..064d0e8ac8d --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/payment-records.md @@ -0,0 +1,3 @@ +# Payment record application interface + +Use `payment-record`, `payment-status`, and `payment-amount` inside the matching `order-item`. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/price-history-orders.md b/tools/stack-bench/tracks/ecommerce/contracts/price-history-orders.md new file mode 100644 index 00000000000..bce06187990 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/price-history-orders.md @@ -0,0 +1,8 @@ +# Price history completed-order interface + +Use `orders-toggle`, `order-item`, and `order-total` to inspect the order created at checkout. +Use `buy-now` inside an `item-card` to create a paid order before a price change. + +# Price history cart and checkout interface + +Use `add-to-cart` inside an `item-card`, `cart-toggle`, `cart-total`, and `checkout-submit`. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/price-history.md b/tools/stack-bench/tracks/ecommerce/contracts/price-history.md new file mode 100644 index 00000000000..9926f4f4576 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/price-history.md @@ -0,0 +1,25 @@ +# Price history application interface + +Put `price-input` and `price-submit` inside the applicable `admin-item-row`. + +Put a `data-price-input` attribute on each `admin-item-row`. Its value is a JSON object with +`itemId` and numeric `price` from the current price input. Identifiers can be +JSON numbers or strings. + + +Expose `POST /api/admin/price`. + + + +Expose the `admin_change_price` reducer. + + + +Use `api:admin_change_price` with `{ itemId, price }`. + + + +Use the `admin_change_price` PostgreSQL function with `{ itemId, price }`. + + +Use the same action as the visible price control. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/product-bundles.md b/tools/stack-bench/tracks/ecommerce/contracts/product-bundles.md new file mode 100644 index 00000000000..546ccf916aa --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/product-bundles.md @@ -0,0 +1,34 @@ +# Product bundle interface + +The catalog has a `bundles-link`. The bundle panel contains `bundle-card` rows with +`bundle-name`, numeric `bundle-price`, and `bundle-component` rows. Each component row has +`bundle-component-name` and numeric `bundle-component-quantity`. Each component row also +exposes its quantity in `data-quantity`. + +Catalog staff use `bundle-name-input`, `bundle-price-input` (currency units), and +`bundle-components-input` (JSON array of `{ "item": "product name", "quantity": 1 }`), +then `bundle-save`. Saving an existing name edits that bundle. Each `bundle-card` exposes +`data-bundle-input` as JSON `{ "bundleId": "..." }`. +The save button exposes `data-bundle-save-input` with `{ name, price, componentsJson }` +from the current form values. + +Use the same application write for the visible form and this named action: + + +Save a bundle with `POST /api/bundles` and `{ name, price, componentsJson }`. + + + +Save a bundle with `save_bundle(name: string, price: number, componentsJson: string)`. + + + +Save a bundle with `api:save_bundle` and `{ name, price, componentsJson }`. + + + +Save a bundle with the `save_bundle` PostgreSQL function and `{ name, price, componentsJson }`. + + +Each component uses an existing product's exact name. `componentsJson` contains the +component array as a JSON string. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/progression-automatic-reorder.md b/tools/stack-bench/tracks/ecommerce/contracts/progression-automatic-reorder.md new file mode 100644 index 00000000000..d0e64711732 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/progression-automatic-reorder.md @@ -0,0 +1,32 @@ +# Automatic reorder application interface + +Use these application interface names: + +- `reorder-link` opens the automatic reorder rules for warehouse staff. +- `reorder-item`, `reorder-threshold`, and `reorder-quantity` identify the rule inputs. +- `reorder-submit` saves the rule. +- `reorder-rule-item` identifies each saved rule, sets `data-entity-id` to the rule's item + identifier, and contains its item name, threshold, quantity, and current state. + Set `data-threshold` and `data-quantity` to their integer values. Set + `data-action-input` on `reorder-submit` to JSON with `itemId`, `threshold`, and `quantity` + for the current form values. + +Saving a rule is the named `saveReorderRule` application action. + + +`saveReorderRule` is `PUT /api/reorders/{itemId}` with `threshold` and `quantity` in the body. + + + +`saveReorderRule` is the `save_reorder_rule(itemId, threshold, quantity)` reducer. + + + +`saveReorderRule` is the `api:save_reorder_rule` mutation with `{ itemId, threshold, quantity }`. + + + +`saveReorderRule` is the `save_reorder_rule` PostgreSQL function with `{ itemId, threshold, quantity }`. + + +Use `buy-now` inside an `item-card` to create stock changes that evaluate a reorder rule. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/progression-cart-recovery.md b/tools/stack-bench/tracks/ecommerce/contracts/progression-cart-recovery.md new file mode 100644 index 00000000000..4166d6b37b8 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/progression-cart-recovery.md @@ -0,0 +1,9 @@ +# Cart recovery application interface + +Use these application interface names: + +- `expired-cart` identifies the expired cart. +- `restore-cart` restores that cart. +- `cart-restore-warning` lists the names of items that could not be restored. + +The existing `cart-item` control identifies each item restored to the active cart. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/progression-personalized-recommendations.md b/tools/stack-bench/tracks/ecommerce/contracts/progression-personalized-recommendations.md new file mode 100644 index 00000000000..f0eb1c10b71 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/progression-personalized-recommendations.md @@ -0,0 +1,5 @@ +# Personalized recommendation application interface + +Use `recommendations` for the ordered recommendation list. Use `recommended-item` for each +item in that list. Each item contains its item name. Use `recommendation-rank` inside each item +for its one-based position in the list. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/promotion-checkout.md b/tools/stack-bench/tracks/ecommerce/contracts/promotion-checkout.md new file mode 100644 index 00000000000..82da5035970 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/promotion-checkout.md @@ -0,0 +1,4 @@ +# Promotion checkout application interface + +Use `cart-promotion` for the code and `apply-promotion` to apply it. Use `promotion-error` when a +code is refused. Expose the saved discount as `order-discount` inside its `order-item`. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/promotion-reporting.md b/tools/stack-bench/tracks/ecommerce/contracts/promotion-reporting.md new file mode 100644 index 00000000000..6366246a68f --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/promotion-reporting.md @@ -0,0 +1,4 @@ +# Promotion reporting application interface + +Use `promotions-link` to open the staff view. Use `promotion-report` for each promotion and +`promotion-redemptions` and `promotion-revenue` for its totals. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/promotion-rules.md b/tools/stack-bench/tracks/ecommerce/contracts/promotion-rules.md new file mode 100644 index 00000000000..beedeef3c6d --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/promotion-rules.md @@ -0,0 +1,35 @@ +# Promotion rule application interface + +Use `staff-link` to open the staff area. In that area, use `promotions-link` for promotion +management. Use `promotion-code`, `promotion-discount`, +`promotion-start`, `promotion-end`, `promotion-limit`, and `promotion-submit` to create a rule. +List rules as `promotion-item` elements and expose the saved values with the matching field IDs. +The `promotions-link` control may be visible to any account. + +Expose the same rule creation used by `promotion-submit`. + + +Use `POST /api/promotions` with a JSON object containing `code` (string), +`discountPercent` (number), `startMicros` and `endMicros` (integer numbers of microseconds +since the Unix epoch), and `usageLimit` (positive integer). + + + +Use the `create_promotion` reducer with arguments in this order: `code: string`, +`discountPercent: f64`, `startMicros: i64`, `endMicros: i64`, `usageLimit: u32`. +Both time arguments are microseconds since the Unix epoch. + + + +Use `api:create_promotion` with `{ code, discountPercent, startMicros, endMicros, usageLimit }`. +Both time arguments are numeric microseconds since the Unix epoch. + + + +Use the `create_promotion` PostgreSQL function with +`{ code, discountPercent, startMicros, endMicros, usageLimit }`. +Both time arguments are numeric microseconds since the Unix epoch. + + +On a listed rule, `promotion-start` and `promotion-end` show the dates as entered, in ISO +`YYYY-MM-DD` form. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/purchasing.md b/tools/stack-bench/tracks/ecommerce/contracts/purchasing.md new file mode 100644 index 00000000000..aeedbcaf457 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/purchasing.md @@ -0,0 +1,33 @@ +# Purchasing application interface + +Use `catalog-link` to return to the catalog. Use `buy-now` inside an `item-card` to buy one unit. Use `orders-toggle` to open order history. +While signed in, show `orders-toggle` directly or reveal it by clicking `current-user`. +No other navigation is required to reach it. +If an overlay blocks catalog navigation, expose a visible `overlay-close` control +that dismisses it before `catalog-link` is used. Screens without a blocking +overlay need no such control. Dialogs, panels, and ordinary page layouts are all allowed. +Use `order-item` for each order, containing the names of its purchased items. Inside that +`order-item`, use `order-total` for its numeric total and `order-status` for its current state. Show `out-of-stock` inside an `item-card` once that item's stock reaches zero. +Use `buy-error` for a failed purchase. + +Put `data-buy-input` on each `item-card`. Its value is a JSON object containing that item's +server identifier, for example `{"itemId":42}`. The identifier may be a JSON number or string. +Use the same identifier for the visible buy action. + +Expose the same purchase used by `buy-now`. + + +Use `POST /api/items/:id/buy`, where `:id` is the item identifier. + + + +Use the `buy_now` reducer with the item identifier. + + + +Use the `api:buy_now` mutation with `{ itemId }`. + + + +Use the `buy_now` PostgreSQL function with `{ itemId }`. + diff --git a/tools/stack-bench/tracks/ecommerce/contracts/recommendation-feedback.md b/tools/stack-bench/tracks/ecommerce/contracts/recommendation-feedback.md new file mode 100644 index 00000000000..0583f7d9e8a --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/recommendation-feedback.md @@ -0,0 +1,3 @@ +# Recommendation feedback application interface + +Use `dismiss-recommendation` inside each `recommended-item`. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/recommendations.md b/tools/stack-bench/tracks/ecommerce/contracts/recommendations.md new file mode 100644 index 00000000000..a87af1b2514 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/recommendations.md @@ -0,0 +1,6 @@ +# Recommendations application interface + +Use `recommended-list` for recommendations and `recommended-item` for each item in the list. +While a customer is signed in, these hooks hold only that customer's recommendations, +never best sellers. +Use `catalog-link` to open the catalog with this list visible. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/reservations.md b/tools/stack-bench/tracks/ecommerce/contracts/reservations.md new file mode 100644 index 00000000000..61f92d9986b --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/reservations.md @@ -0,0 +1,5 @@ +# Reservation application interface + +Use `cart-reservation-timer` inside a `cart-item` for the remaining reservation time in seconds. +Use `cart-item-expired` inside an expired cart line and `cart-expired-notice` after cart +expiration. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/reviews.md b/tools/stack-bench/tracks/ecommerce/contracts/reviews.md new file mode 100644 index 00000000000..5f19076e038 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/reviews.md @@ -0,0 +1,28 @@ +# Review application interface + +On item details, show the review form for eligible customers or expose +`review-toggle` to open it. Omit this control when the form is already shown. + +Use `review-rating` on an input or select with values 1 through 5, `review-input` for the comment, and +`review-submit` to submit it. Put the item's server identifier in +`data-review-item-id` on `review-submit`. Use `review-average` for the numeric average, +`review-item` for each visible review, and `review-error` for a failed submission. + +Expose the same review operation used by `review-submit`. + + +Use `POST /api/items/:id/reviews`, where `:id` is the item identifier. Send `rating` and +`comment` in the request body. + + + +Use the `submit_review` reducer with the item identifier, rating, and comment. + + + +Use `api:submit_review` with `{ itemId, rating, comment }`. + + + +Use the `submit_review` PostgreSQL function with `{ itemId, rating, comment }`. + diff --git a/tools/stack-bench/tracks/ecommerce/contracts/sales-dashboard.md b/tools/stack-bench/tracks/ecommerce/contracts/sales-dashboard.md new file mode 100644 index 00000000000..3214d659713 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/sales-dashboard.md @@ -0,0 +1,10 @@ +# Sales dashboard application interface + +Open the administrator area with `admin-link`. If category totals are on a +separate tab or screen, expose `sales-link` there to open them. Omit this control +when the totals are already shown. + +Use `category-row` for each product category, including a category with no sales yet, whose +units and revenue read 0. Use `category-units` and `category-revenue` inside each row. Use `recommended-list` for signed-out +best sellers, shown on the catalog page, and `recommended-item` for each item. Use `buy-now` inside an `item-card` to create +sales activity for the dashboard. Each best seller contains its one-based `recommendation-rank`. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/scheduled-restocks.md b/tools/stack-bench/tracks/ecommerce/contracts/scheduled-restocks.md new file mode 100644 index 00000000000..324375d5ca1 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/scheduled-restocks.md @@ -0,0 +1,40 @@ +# Scheduled restock application interface + +Use `admin-link` to open the administrator area. If these controls are on a separate screen within +it, expose `restocks-link` there to reach them. Use `schedule-restock-item`, `schedule-restock-warehouse`, `schedule-restock-qty`, and +`schedule-restock-delay` for the inputs. For a `select`, each item and warehouse must have exactly +one option whose value or label is its exact name. `schedule-restock-delay` takes whole seconds. +Use `schedule-restock-submit` to schedule the restock. Scheduling may close these controls; +`restocks-link` then opens them again. +Set its `data-action-input` to a JSON object with exactly `item`, `warehouse`, `quantity`, and +`delaySeconds`. `item` and `warehouse` are their names as strings; `quantity` and +`delaySeconds` are JSON integers. Use `pending-restock-item` for each pending row and set its +`data-entity-id` to the restock's server identifier. +Each row contains the item name and sets `data-quantity` to its integer quantity. Use +`pending-restock-remaining` for its remaining seconds, counting down while the page is open, +`pending-restock-cancel` to cancel it, and `stock-ledger-entry` for a completed stock movement. +Show the ledger entries on the same screen as the pending restocks. + + +Expose `POST /api/admin/scheduled-restocks` and `DELETE /api/admin/scheduled-restocks/:id`. +The POST body has the same fields as `data-action-input`. +Write the restock's `data-entity-id` as a decimal number. + + + +Expose `schedule_restock` with arguments in this order: `item: string`, `warehouse: string`, +`quantity: u32`, `delaySeconds: u32`; and `cancel_scheduled_restock` with `restockId: u64`. +Write the restock's `data-entity-id` as a decimal number. + + + +Use `api:schedule_restock` with `{ item, warehouse, quantity, delaySeconds }` and +`api:cancel_scheduled_restock` with `{ restockId }`. The item and warehouse arguments are names. +Use the scheduled-restock document's native `_id` string for `data-entity-id` and `restockId`. + + + +Use the `schedule_restock` PostgreSQL function with `{ item, warehouse, quantity, delaySeconds }` +and `cancel_scheduled_restock` with `{ restockId }`. The item and warehouse arguments are names. +Use the identifier accepted by `restockId` for the restock's `data-entity-id`. + diff --git a/tools/stack-bench/tracks/ecommerce/contracts/split-tender-refunds.md b/tools/stack-bench/tracks/ecommerce/contracts/split-tender-refunds.md new file mode 100644 index 00000000000..69237349e10 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/split-tender-refunds.md @@ -0,0 +1,3 @@ +# Split refund interface + +Reuse `support-refund` and its existing named action. Each `refund-entry` shows `refund-credit-amount` and `refund-external-amount` in major currency units. Show the restored credit in the existing credit balance and history. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/staff-access.md b/tools/stack-bench/tracks/ecommerce/contracts/staff-access.md new file mode 100644 index 00000000000..96bdb23bad4 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/staff-access.md @@ -0,0 +1,14 @@ +## Staff access controls + +Use the existing account sign-in controls. Use `staff-link` for the staff area and `admin-link` +for the administrator area. Use `staff-area` for the staff tools, not for a loading or error message. + +For an account allowed to use an area, show its link directly or reveal it by clicking +`current-user`. No other navigation is required to reach these links. The links may be +visible to any account. + +Use these provided accounts: + +- staff: `staff` / `stackbench-staff-2026` +- administrator: `admin` / `stackbench-admin-2026` +- customer: `customer` / `stackbench-customer-2026` diff --git a/tools/stack-bench/tracks/ecommerce/contracts/staff-activity.md b/tools/stack-bench/tracks/ecommerce/contracts/staff-activity.md new file mode 100644 index 00000000000..8e65af6c6ba --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/staff-activity.md @@ -0,0 +1,4 @@ +# Staff activity application interface + +Use `activity-link` to open staff activity history and `activity-entry` for each change. Inside +each entry, use `activity-actor`, `activity-action`, `activity-subject`, and `activity-time`. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/staff-roles.md b/tools/stack-bench/tracks/ecommerce/contracts/staff-roles.md new file mode 100644 index 00000000000..9d3801925d5 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/staff-roles.md @@ -0,0 +1,46 @@ +# Staff role application interface + +Put role management in the administrator area opened by `admin-link`. +If the role controls are on a separate tab or screen within it, expose `staff-roles-link` +there to open them. Omit this control when the role controls are already shown. +Use `staff-role-row` for each staff account and set `data-account-id` to that account's server +identifier. Put `staff-role-select` and `staff-role-save` inside the row. +When the page loads, `staff-role-select` shows the account's saved role. Keep each account +listed after its role changes, including an account whose role becomes `admin`. +On that row, expose `data-submit-state` for the latest role assignment: `idle` +initially, `pending` immediately when submitted, `succeeded` only after the server +confirms success, or `failed` after rejection or transport failure. Keep the terminal +state until the next submission. A new submission must replace the old state. +Also set the row's HTML `id` to `staff-role-account-` followed by +`encodeURIComponent(username)`, using the exact account username without changing its case. +For example, username `staff` has row ID `staff-role-account-staff`. This identifies the +account independently of the role options or other text in the row. + +The staff sign-in and staff-area controls come from the staff access feature. + +Also provide a second staff account named `staff2`, with the initial role `staff`. +Include it in role management like the other staff accounts. + +Expose the same role assignment used by `staff-role-save`. + + +Use `PUT /api/staff/:id/role`, where `:id` is the account identifier from `data-account-id`. +The JSON body is `{ "role": "" }`. + + + +Use the `assign_staff_role` reducer with arguments in this order: `accountId: u64`, +`role: string`. Render `data-account-id` as the decimal account identifier without precision loss. + + + +Use `api:assign_staff_role` with `{ accountId, role }`. Render `data-account-id` as +the native string account identifier. + + + +Use the `assign_staff_role` PostgreSQL function with `{ accountId, role }`. Render +`data-account-id` as the account identifier accepted by `accountId`, without precision loss. + + +`staff-role-select` offers the roles `staff`, `inventory`, and `admin` as its option values. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/stock-alerts.md b/tools/stack-bench/tracks/ecommerce/contracts/stock-alerts.md new file mode 100644 index 00000000000..1c4a67f34b4 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/stock-alerts.md @@ -0,0 +1,21 @@ +# Stock alert application interface + +Use `stock-alert` inside an unavailable `item-card` to request an alert. Use +`notifications-toggle` to open notifications and `notification-item` for each alert. +While signed in on the catalog, show `notifications-toggle` directly, reveal it by clicking +`current-user`, or make it available in the settings opened by `notification-settings`. +No other navigation is required. An already open `notifications-panel` needs no toggle. +On that `item-card`, expose `data-submit-state` for the latest stock-alert request: +`idle` initially, `pending` immediately when submitted, `succeeded` only after the server +confirms success, or `failed` after rejection or transport failure. Keep the terminal state +if the request button disappears. A new submission must replace the old state. +Use `notifications-panel` for the opened notification view, including while its contents +load. Set its `aria-busy` attribute to `false` only when the signed-in account's contents +have loaded successfully, including an empty list; keep it `true` while loading or after +a failed read. Its toggle may also close it. +Within a delivered stock alert, expose `stock-alert-delivery` containing the item's +displayed name. A pending request must not expose `stock-alert-delivery`. + +Use `catalog-link` to return to the catalog. If an overlay blocks navigation, +expose a visible `overlay-close` control that dismisses it before navigation. +Screens without a blocking overlay do not need this control. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/stock-transfers.md b/tools/stack-bench/tracks/ecommerce/contracts/stock-transfers.md new file mode 100644 index 00000000000..8279792c1ea --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/stock-transfers.md @@ -0,0 +1,31 @@ +# Stock transfer application interface + +Put `transfer-from`, `transfer-to`, `transfer-qty`, and `transfer-submit` inside the applicable +`admin-item-row`. Use `warehouse-total` inside each `admin-warehouse-item` for its numeric stock +total. Show `order-error` when a transfer is refused. + +For a `select` used by `transfer-from` or `transfer-to`, each warehouse must have exactly one +option whose value or label is the exact warehouse name. The other field may include extra +display text or use the warehouse ID. + +Put `data-transfer-input` on each `admin-item-row`. Its value is a JSON object with exactly +`itemId`, `fromWarehouseId`, and `toWarehouseId` for the currently selected source and destination. +Identifiers can be JSON numbers or strings. + + +Expose `POST /api/admin/transfer`. The JSON body has `itemId`, `fromWarehouseId`, +`toWarehouseId`, and `quantity`. + + + +Expose `admin_transfer_stock` with arguments in this order: `itemId: u64`, +`fromWarehouseId: u64`, `toWarehouseId: u64`, `quantity`. + + + +Use `api:admin_transfer_stock` with `{ itemId, fromWarehouseId, toWarehouseId, quantity }`. + + + +Use the `admin_transfer_stock` PostgreSQL function with `{ itemId, fromWarehouseId, toWarehouseId, quantity }`. + diff --git a/tools/stack-bench/tracks/ecommerce/contracts/store-credit.md b/tools/stack-bench/tracks/ecommerce/contracts/store-credit.md new file mode 100644 index 00000000000..5bee4b5baac --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/store-credit.md @@ -0,0 +1,25 @@ +# Store credit interface + +Open customer credit with `credit-link`. The `credit-panel` exposes the signed-in account's `data-account-id` and `aria-busy="false"` when loaded. Show `credit-balance` in major currency units and one `credit-entry` per movement. + +Staff use `credit-customer`, `credit-amount-input` (major units), `credit-reference-input`, and `credit-grant`. The grant control exposes `data-action-input` as JSON with `accountId`, `amountMinor`, and `reference`. + +Use `credit-checkout` in the cart. Each `order-item` shows `payment-credit-amount` and `payment-external-amount` in major units. Their sum is `payment-amount`. + + +`grantCredit` is `POST /api/staff/credit` with `accountId`, `amountMinor`, and `reference`. +`checkoutCredit` is `POST /api/checkout/credit` with no body fields. + + + +`grantCredit` is `grant_credit(accountId, amountMinor, reference)`. +`checkoutCredit` is `checkout_credit()`. + + + +Use `api:grant_credit` with `{ accountId, amountMinor, reference }` and `api:checkout_credit` with `{}`. + + + +Use the `grant_credit` PostgreSQL function with `{ accountId, amountMinor, reference }` and `checkout_credit` with `{}`. + diff --git a/tools/stack-bench/tracks/ecommerce/contracts/subscriptions.md b/tools/stack-bench/tracks/ecommerce/contracts/subscriptions.md new file mode 100644 index 00000000000..14dea13ad01 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/subscriptions.md @@ -0,0 +1,27 @@ +# Subscription interface + +Open with `subscriptions-link`. Use `subscription-item-input` (item name), `subscription-quantity-input`, `subscription-interval-input` (whole seconds, minimum 30), `subscription-deliveries-input` (1–12), and `subscription-create`. + +The loaded `subscriptions-panel` has `aria-busy="false"`. Each `subscription-row` includes the item name and `subscription-status`: `active`, `paused`, `cancelled`, or `complete`. Use `subscription-pause`, `subscription-resume`, and `subscription-cancel`; each exposes `data-action-input` with `subscriptionId`. Each processed slot has one `subscription-delivery` inside the row, with `subscription-delivery-status` (`paid` or `skipped`). `subscription-total` shows the sum of its recorded payments in major currency units. Ordinary orders and payment records include their item names. + + +`pauseSubscription` is `POST /api/subscriptions/{subscriptionId}/pause`. +`resumeSubscription` is `POST /api/subscriptions/{subscriptionId}/resume`. +`cancelSubscription` is `POST /api/subscriptions/{subscriptionId}/cancel`. + + + +`pauseSubscription` is `pause_subscription(subscriptionId)`. +`resumeSubscription` is `resume_subscription(subscriptionId)`. +`cancelSubscription` is `cancel_subscription(subscriptionId)`. + + + +Use `api:pause_subscription`, `api:resume_subscription`, and `api:cancel_subscription`, +each with `{ subscriptionId }`. + + + +Use the `pause_subscription`, `resume_subscription`, and `cancel_subscription` database +functions, each with `{ subscriptionId }`. + diff --git a/tools/stack-bench/tracks/ecommerce/contracts/support-history.md b/tools/stack-bench/tracks/ecommerce/contracts/support-history.md new file mode 100644 index 00000000000..82a3566350f --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/support-history.md @@ -0,0 +1,5 @@ +## Customer support history controls + +`support-link` opens support. Use `support-ticket` for each ticket in the history, showing +its subject. While signed in, opening support makes both the intake form and the account's +history available without another tab or menu. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/support-intake.md b/tools/stack-bench/tracks/ecommerce/contracts/support-intake.md new file mode 100644 index 00000000000..31f09195a37 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/support-intake.md @@ -0,0 +1,15 @@ +## Support intake controls + +Use `support-link` to open support. Show it directly while signed out. While signed in, +show it directly or reveal it by clicking `current-user`. No other navigation is required +to reach it. Opening support makes the intake form available without another tab or menu. +If a support dialog, panel, or confirmation overlay blocks leaving support, account +navigation, or the intake form, expose a visible `overlay-close` control that dismisses +the blocking overlay. Screens without a blocking overlay need no such control. +Dialogs, panels, and ordinary page layouts are all allowed. +Use `support-email`, `support-subject`, and +`support-message` for the ticket fields. Use `support-submit` to submit the ticket and +`support-reference` to show its reference. + +An email address, a subject, and a message are sufficient to submit a ticket. Any +additional fields must be optional. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/support-refunds.md b/tools/stack-bench/tracks/ecommerce/contracts/support-refunds.md new file mode 100644 index 00000000000..70b11c7ea2b --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/support-refunds.md @@ -0,0 +1,27 @@ +# Support refund application interface + +## Support refund controls + +Within a `support-ticket`, use `support-refund` for the refund action and +`support-refund-total` for the recorded refund amount. The refund action must expose its input in +`data-action-input` for the named `supportRefund` application action. The `support-ticket` also +exposes the same JSON `{ "caseId": "..." }` in `data-refund-input` so the case remains addressable +after its refund button is disabled or removed. Within an `order-item`, use +`order-refund-total` for the refunded amount and `refund-entry` for each refund record. Each +`refund-entry` includes the order item name. + + +`supportRefund` is `POST /api/support/cases/{caseId}/refund`. + + + +`supportRefund` is the `support_refund(caseId)` reducer. + + + +`supportRefund` is the `api:support_refund` mutation with `{ caseId }`. + + + +`supportRefund` is the `support_refund` PostgreSQL function with `{ caseId }`. + diff --git a/tools/stack-bench/tracks/ecommerce/contracts/support-triage.md b/tools/stack-bench/tracks/ecommerce/contracts/support-triage.md new file mode 100644 index 00000000000..0bf6a6c7f2d --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/support-triage.md @@ -0,0 +1,18 @@ +## Support triage controls + +Open the staff area with `staff-link`. If its ticket queue is on a separate tab or +screen, expose `support-queue-link` there to open it. Omit this control when the +editable ticket controls are already shown. + +Use `support-ticket` for each ticket in the staff view, showing its subject. Within a ticket, use +`support-assignee`, `support-priority`, and `support-status-input` for the editable fields. +Use `support-update` to apply the changes. Use `support-status` to show the current status. +On that `support-ticket`, expose `data-submit-state` for the latest update: +`idle` initially, `pending` immediately when submitted, `succeeded` only after the server +confirms success, or `failed` after rejection or transport failure. Keep the terminal state +until the next submission. A new submission must replace the old state. +`support-assignee` takes the assignee's username; if it is a select, its option values are the +usernames. `support-priority` offers `low`, `normal`, and `high` as its option values. + +`support-status-input` offers the statuses `open`, `in progress`, and `resolved` as its option +values; `support-status` shows the one chosen. diff --git a/tools/stack-bench/tracks/ecommerce/contracts/warehouse-administration.md b/tools/stack-bench/tracks/ecommerce/contracts/warehouse-administration.md new file mode 100644 index 00000000000..549d91f1fac --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/contracts/warehouse-administration.md @@ -0,0 +1,48 @@ +# Warehouse administration application interface + +Use `admin-link` to open `admin-panel`. Use `admin-item-row` for each item and `admin-stock` for +its numeric total stock. Use `admin-warehouse-item` for each warehouse. Use `admin-location-row` +for every item in every warehouse, including zero quantities; the row shows the item name and +the warehouse name, with +`admin-location-qty` for its quantity. Use +`restock-input` and `restock-submit` inside that row. Use `admin-revenue` for numeric total +revenue. + +Keep all item, warehouse, and holding rows available in the open admin panel, without pagination. + +Put a `data-restock-input` attribute on each `admin-location-row`. Its value is a JSON object +with exactly `itemId`, `warehouseId`, and a valid one-unit `quantity`. Identifiers can be JSON numbers or +strings. + +Use the same restock action as the visible control. + +## Stock data interface + +Expose singular native tables or views `item(id, name, price)`, `warehouse(id, name)`, and +`stock(item_id, warehouse_id, quantity)` for direct database access. +`stock.item_id` and `stock.warehouse_id` reference `item.id` and `warehouse.id`; in a document +store they hold the referenced document's `id` value, or its `_id` when it has no `id`. +Each item and warehouse must have a unique ID and name, with one stock holding per item and +warehouse pair. Keep this interface readable with the database's own tools. Native updates to +`stock.quantity` must change the same stock that the application uses. Physical foreign-key +constraints are not required. + + +Expose `POST /api/admin/restock`. The JSON body has the same fields as `data-restock-input`. + + + +Expose `admin_restock` with arguments in this order: `itemId: u64`, `warehouseId: u64`, +`quantity`. + + + +Use `api:admin_restock` with `{ itemId, warehouseId, quantity }`. +The singular native `item` and `warehouse` tables use Convex `_id` identifiers. +The `stock.item_id` and `stock.warehouse_id` fields refer to those same identifiers. + + + +Use the `admin_restock` PostgreSQL function with `{ itemId, warehouseId, quantity }`. +The `item`, `warehouse`, and `stock` tables are in the `public` schema. + diff --git a/tools/stack-bench/tracks/ecommerce/progression/ecommerce.json b/tools/stack-bench/tracks/ecommerce/progression/ecommerce.json new file mode 100644 index 00000000000..ed401fca0e9 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/progression/ecommerce.json @@ -0,0 +1,1105 @@ +{ + "schemaVersion": 2, + "kind": "progression-definition", + "id": "ecommerce.questlines", + "title": "Ecommerce progression", + "nodes": [ + { + "id": "accounts", + "title": "Accounts", + "questline": "identity-profiles", + "featureRefs": [ + "ecommerce.feature.accounts" + ], + "gradingGroups": [ + "ecommerce.feature.accounts#account-create", + "ecommerce.feature.accounts#account-duplicate", + "ecommerce.spec.access-control#account-password", + "ecommerce.feature.accounts#account-signout", + "ecommerce.spec.access-control#account-disclosure", + "ecommerce.spec.state-durability#session-reload" + ], + "dependencies": [] + }, + { + "id": "catalog", + "title": "Catalog items", + "questline": "catalog-discovery", + "featureRefs": [ + "ecommerce.feature.catalog-items" + ], + "gradingGroups": [ + "ecommerce.feature.catalog-items#values" + ], + "dependencies": [] + }, + { + "id": "catalog-discovery", + "title": "Catalog discovery", + "questline": "catalog-discovery", + "featureRefs": [ + "ecommerce.feature.catalog-discovery" + ], + "gradingGroups": [ + "ecommerce.feature.catalog-discovery#ranking", + "ecommerce.feature.catalog-discovery#search" + ], + "dependencies": [ + { + "id": "catalog", + "reason": "Discovery ranks and searches catalog items." + } + ] + }, + { + "id": "customer-profile", + "title": "Customer profile", + "questline": "identity-profiles", + "featureRefs": [ + "ecommerce.progression.customer-profile" + ], + "gradingGroups": [ + "ecommerce.progression.customer-profile#profile", + "ecommerce.spec.access-control#customer-profile-privacy", + "ecommerce.spec.state-durability#customer-profile-reload" + ], + "dependencies": [ + { + "id": "accounts", + "reason": "A profile belongs to a customer account." + } + ] + }, + { + "id": "purchasing", + "title": "Purchasing and orders", + "questline": "orders-fulfillment", + "featureRefs": [ + "ecommerce.feature.purchasing" + ], + "gradingGroups": [ + "ecommerce.feature.purchasing#purchase-order", + "ecommerce.spec.access-control#order-ownership", + "ecommerce.spec.access-control#purchase-attribution", + "ecommerce.spec.access-control#purchase-session", + "ecommerce.spec.access-control#signed-out-purchase", + "ecommerce.spec.live-state#purchase-stock", + "ecommerce.spec.live-state#ranking", + "ecommerce.spec.transactional-integrity#server-price" + ], + "dependencies": [ + { + "id": "accounts", + "reason": "A purchase needs a buyer." + }, + { + "id": "catalog", + "reason": "A purchase needs an item." + } + ] + }, + { + "id": "cart", + "title": "Cart", + "questline": "shopping-checkout", + "featureRefs": [ + "ecommerce.feature.cart" + ], + "gradingGroups": [ + "ecommerce.feature.cart#cart", + "ecommerce.spec.access-control#cart-boundary", + "ecommerce.spec.live-state#shared-cart", + "ecommerce.spec.state-durability#cart-reload" + ], + "dependencies": [ + { + "id": "accounts", + "reason": "An account cart belongs to a buyer." + }, + { + "id": "catalog", + "reason": "A cart contains catalog items." + } + ] + }, + { + "id": "checkout", + "title": "Checkout", + "questline": "shopping-checkout", + "featureRefs": [ + "ecommerce.feature.checkout" + ], + "gradingGroups": [ + "ecommerce.feature.checkout#checkout", + "ecommerce.spec.concurrency-safety#duplicate-checkout", + "ecommerce.spec.state-durability#account-state-reload", + "ecommerce.spec.state-durability#account-state-reconnect", + "ecommerce.spec.state-durability#checkout-crash-integrity", + "ecommerce.spec.state-durability#checkout-crash-durability" + ], + "dependencies": [ + { + "id": "cart", + "reason": "Checkout turns a cart into an order." + } + ] + }, + { + "id": "warehouse-admin", + "title": "Warehouse administration", + "questline": "inventory", + "featureRefs": [ + "ecommerce.feature.warehouse-admin" + ], + "gradingGroups": [ + "ecommerce.feature.warehouse-admin#warehouse-view", + "ecommerce.feature.warehouse-admin#admin-write", + "ecommerce.spec.access-control#warehouse-area-boundary", + "ecommerce.spec.access-control#warehouse-write-boundary", + "ecommerce.spec.external-data-sync#external-live", + "ecommerce.spec.external-data-sync#external-reconnect", + "ecommerce.spec.live-state#warehouse-stock" + ], + "dependencies": [ + { + "id": "catalog", + "reason": "Warehouse stock belongs to catalog items." + }, + { + "id": "staff-access", + "reason": "Warehouse changes require staff access." + } + ] + }, + { + "id": "reviews", + "title": "Reviews", + "questline": "reviews", + "featureRefs": [ + "ecommerce.feature.reviews" + ], + "gradingGroups": [ + "ecommerce.feature.reviews#reviews", + "ecommerce.progression.review-access-specifications#review-eligibility-direct", + "ecommerce.spec.access-control#review-eligibility", + "ecommerce.spec.live-state#open-list", + "ecommerce.spec.live-state#rating", + "ecommerce.spec.transactional-integrity#unique-review", + "ecommerce.progression.review-access-specifications#stored-review-script" + ], + "dependencies": [ + { + "id": "purchasing", + "reason": "A verified review needs a purchase." + } + ] + }, + { + "id": "fulfilment-queue", + "title": "Fulfilment queue", + "questline": "orders-fulfillment", + "featureRefs": [ + "ecommerce.progression.fulfilment-queue" + ], + "gradingGroups": [ + "ecommerce.progression.fulfilment-queue#fulfilment-ship", + "ecommerce.progression.inventory-conservation-specifications#shipping-accounting", + "ecommerce.progression.fulfilment-queue#fulfilment-warehouse", + "ecommerce.progression.operations-access-specifications#operator-authorization-direct", + "ecommerce.spec.access-control#fulfilment-area-boundary", + "ecommerce.spec.concurrency-safety#last-unit", + "ecommerce.spec.concurrency-safety#restock-race", + "ecommerce.spec.live-state#fulfilment-queue" + ], + "dependencies": [ + { + "id": "purchasing", + "reason": "Fulfilment needs orders." + }, + { + "id": "warehouse-admin", + "reason": "Fulfilment needs warehouse stock." + } + ] + }, + { + "id": "stock-transfers", + "title": "Stock transfers", + "questline": "inventory", + "featureRefs": [ + "ecommerce.l2.stock-transfers-features" + ], + "gradingGroups": [ + "ecommerce.l2.stock-transfers-features#transfer", + "ecommerce.progression.inventory-conservation-specifications#stock-conservation-contention", + "ecommerce.progression.operations-access-specifications#operator-transfer-authorization", + "ecommerce.spec.transactional-integrity#stock-transfer-overdraw" + ], + "dependencies": [ + { + "id": "warehouse-admin", + "reason": "Transfers operate on warehouse stock." + } + ] + }, + { + "id": "order-cancellation", + "title": "Order cancellation", + "questline": "orders-fulfillment", + "featureRefs": [ + "ecommerce.l2.order-cancellation-features" + ], + "gradingGroups": [ + "ecommerce.l2.order-cancellation-features#cancellation-core", + "ecommerce.l2.order-cancellation-features#cancellation-history", + "ecommerce.progression.inventory-conservation-specifications#stock-conservation-cancel", + "ecommerce.progression.inventory-conservation-specifications#stock-conservation-history", + "ecommerce.progression.cancellation-accounting-specifications#cancellation-accounting", + "ecommerce.progression.operations-access-specifications#order-owner-direct" + ], + "dependencies": [ + { + "id": "purchasing", + "reason": "Cancellation needs an order." + }, + { + "id": "warehouse-admin", + "reason": "Cancelled stock returns to its original warehouse." + } + ] + }, + { + "id": "price-history", + "title": "Price history", + "questline": "catalog-discovery", + "featureRefs": [ + "ecommerce.l2.price-history-features" + ], + "gradingGroups": [ + "ecommerce.progression.operations-access-specifications#operator-price-authorization", + "ecommerce.progression.price-history-specifications#price-cart-checkout", + "ecommerce.progression.price-history-specifications#price-history-exact", + "ecommerce.l2.price-history-features#price-live-independent", + "ecommerce.progression.price-accounting-specifications#price-accounting" + ], + "dependencies": [ + { + "id": "catalog-management", + "reason": "Price changes are a catalog management operation." + } + ] + }, + { + "id": "inventory-dashboard", + "title": "Inventory dashboard", + "questline": "business-operations", + "featureRefs": [ + "ecommerce.l2.inventory-dashboard" + ], + "gradingGroups": [ + "ecommerce.l2.inventory-dashboard#low-stock", + "ecommerce.spec.live-state#inventory-dashboard" + ], + "dependencies": [ + { + "id": "warehouse-admin", + "reason": "The dashboard shows warehouse stock." + } + ] + }, + { + "id": "sales-dashboard", + "title": "Sales dashboard", + "questline": "business-operations", + "featureRefs": [ + "ecommerce.l2.sales-dashboard" + ], + "gradingGroups": [ + "ecommerce.l2.sales-dashboard#category-totals", + "ecommerce.l2.sales-dashboard#best-sellers", + "ecommerce.spec.live-state#sales-dashboard", + "ecommerce.spec.transactional-integrity#books-balance" + ], + "dependencies": [ + { + "id": "purchasing", + "reason": "The dashboard summarizes sales." + } + ] + }, + { + "id": "recommendations", + "title": "Recommendations", + "questline": "recommendations", + "featureRefs": [ + "ecommerce.l2.recommendations" + ], + "gradingGroups": [ + "ecommerce.l2.recommendations#recommendations" + ], + "dependencies": [ + { + "id": "cart", + "reason": "Recommendations exclude items already in the cart." + }, + { + "id": "purchasing", + "reason": "Recommendations use customer purchase activity." + } + ] + }, + { + "id": "reservations", + "title": "Reservations", + "questline": "shopping-checkout", + "featureRefs": [ + "ecommerce.l3.reservations-features" + ], + "gradingGroups": [ + "ecommerce.l3.deferred-durability-specifications#reservation-restart", + "ecommerce.l3.deferred-integrity-specifications#reservation-stock", + "ecommerce.l3.deferred-integrity-specifications#reserved-checkout", + "ecommerce.l3.reservations-features#checkout", + "ecommerce.l3.reservations-features#countdown", + "ecommerce.l3.reservations-features#expiry", + "ecommerce.l3.reservations-features#renewal", + "ecommerce.l3.reservations-features#stock-hold", + "ecommerce.l3.server-time-specifications#no-browser" + ], + "dependencies": [ + { + "id": "checkout", + "reason": "Reservations confirm checkout availability." + }, + { + "id": "warehouse-admin", + "reason": "Reservations hold warehouse stock." + } + ] + }, + { + "id": "order-delivery", + "title": "Order delivery", + "questline": "orders-fulfillment", + "featureRefs": [ + "ecommerce.l3.order-delivery-features" + ], + "gradingGroups": [ + "ecommerce.progression.cancellation-queue-specifications#queue-removal", + "ecommerce.l3.deferred-durability-specifications#delivery-restart", + "ecommerce.l3.deferred-integrity-specifications#delivery-once", + "ecommerce.l3.order-delivery-features#automatic-delivery", + "ecommerce.l3.order-delivery-features#cancel-final" + ], + "dependencies": [ + { + "id": "fulfilment-queue", + "reason": "Delivery follows fulfilment." + }, + { + "id": "order-cancellation", + "reason": "Delivery must respect final cancellation state." + } + ] + }, + { + "id": "order-returns", + "title": "Returns", + "questline": "orders-fulfillment", + "featureRefs": [ + "ecommerce.l3.order-returns-features" + ], + "gradingGroups": [ + "ecommerce.l3.order-returns-features#return-complete", + "ecommerce.l3.order-returns-features#return-before-shipping", + "ecommerce.l3.order-returns-features#return-server-boundary" + ], + "dependencies": [ + { + "id": "order-delivery", + "reason": "Only a shipped or delivered item can be returned." + } + ] + }, + { + "id": "scheduled-restocks", + "title": "Scheduled restocks", + "questline": "inventory", + "featureRefs": [ + "ecommerce.l3.scheduled-restocks-features" + ], + "gradingGroups": [ + "ecommerce.l3.deferred-access-specifications#customer-refused", + "ecommerce.l3.deferred-integrity-specifications#restock-once", + "ecommerce.l3.scheduled-restocks-features#applies", + "ecommerce.l3.scheduled-restocks-features#cancels", + "ecommerce.l3.scheduled-restocks-features#pending", + "ecommerce.l3.server-time-specifications#not-early" + ], + "dependencies": [ + { + "id": "warehouse-admin", + "reason": "Scheduled restocking changes warehouse stock." + } + ] + }, + { + "id": "cart-expiration", + "title": "Cart expiration", + "questline": "shopping-checkout", + "featureRefs": [ + "ecommerce.l3.cart-expiration-features" + ], + "gradingGroups": [ + "ecommerce.l3.cart-expiration-features#inactive-cart", + "ecommerce.l3.deferred-durability-specifications#cart-restart" + ], + "dependencies": [ + { + "id": "reservations", + "reason": "Expiration releases existing reservations." + } + ] + }, + { + "id": "staff-access", + "title": "Staff access", + "questline": "staff-access", + "featureRefs": [ + "ecommerce.progression.staff-access" + ], + "gradingGroups": [ + "ecommerce.progression.staff-access#staff-sign-in", + "ecommerce.spec.access-control#staff-area-boundary" + ], + "dependencies": [] + }, + { + "id": "staff-roles", + "title": "Staff roles", + "questline": "staff-access", + "featureRefs": [ + "ecommerce.progression.staff-roles" + ], + "gradingGroups": [ + "ecommerce.progression.staff-roles#assignment", + "ecommerce.spec.access-control#staff-role-boundary", + "ecommerce.spec.access-control#staff-role-revocation", + "ecommerce.spec.state-durability#staff-role-reload" + ], + "dependencies": [ + { + "id": "staff-access", + "reason": "Roles apply to staff identities." + } + ] + }, + { + "id": "catalog-management", + "title": "Catalog management", + "questline": "business-operations", + "featureRefs": [ + "ecommerce.progression.catalog-management" + ], + "gradingGroups": [ + "ecommerce.progression.catalog-management#products", + "ecommerce.spec.transactional-integrity#catalog-volume" + ], + "dependencies": [ + { + "id": "catalog-discovery", + "reason": "Staff use catalog discovery to find managed products." + }, + { + "id": "staff-roles", + "reason": "Catalog changes require an authorized staff role." + } + ] + }, + { + "id": "support-intake", + "title": "Support intake", + "questline": "customer-support", + "featureRefs": [ + "ecommerce.progression.support-intake" + ], + "gradingGroups": [ + "ecommerce.progression.support-intake#ticket-create" + ], + "dependencies": [] + }, + { + "id": "support-triage", + "title": "Support triage", + "questline": "customer-support", + "featureRefs": [ + "ecommerce.progression.support-triage" + ], + "gradingGroups": [ + "ecommerce.progression.support-triage#assignment", + "ecommerce.progression.support-triage#priority", + "ecommerce.progression.support-triage#status" + ], + "dependencies": [ + { + "id": "staff-access", + "reason": "Triage requires a staff member." + }, + { + "id": "support-intake", + "reason": "Triage operates on submitted tickets." + } + ] + }, + { + "id": "support-history", + "title": "Customer support history", + "questline": "customer-support", + "featureRefs": [ + "ecommerce.progression.support-history" + ], + "gradingGroups": [ + "ecommerce.progression.support-history#history", + "ecommerce.spec.access-control#support-history-privacy", + "ecommerce.spec.access-control#support-history-logout", + "ecommerce.spec.state-durability#support-history-reload" + ], + "dependencies": [ + { + "id": "accounts", + "reason": "Owned ticket history requires an account." + }, + { + "id": "support-intake", + "reason": "History contains submitted tickets." + } + ] + }, + { + "id": "promotion-rules", + "title": "Promotion rules", + "questline": "promotions", + "featureRefs": [ + "ecommerce.progression.promotion-rules" + ], + "gradingGroups": [ + "ecommerce.progression.promotion-rules#values", + "ecommerce.spec.access-control#promotion-management-boundary" + ], + "dependencies": [ + { + "id": "catalog", + "reason": "Promotions apply to catalog sales." + }, + { + "id": "staff-access", + "reason": "Promotion rules require staff access." + } + ] + }, + { + "id": "notification-preferences", + "title": "Notification preferences", + "questline": "notifications", + "featureRefs": [ + "ecommerce.progression.notification-preferences" + ], + "gradingGroups": [ + "ecommerce.progression.notification-preferences#preferences", + "ecommerce.spec.access-control#notification-preferences-privacy", + "ecommerce.spec.state-durability#notification-preferences-reload" + ], + "dependencies": [ + { + "id": "accounts", + "reason": "Preferences belong to an account." + } + ] + }, + { + "id": "faceted-search", + "title": "Faceted search", + "questline": "catalog-discovery", + "featureRefs": [ + "ecommerce.progression.faceted-search" + ], + "gradingGroups": [ + "ecommerce.progression.faceted-search#filters", + "ecommerce.progression.faceted-search#pagination", + "ecommerce.spec.search-ordering#after-purchase" + ], + "dependencies": [ + { + "id": "catalog-discovery", + "reason": "Facets refine catalog discovery." + } + ] + }, + { + "id": "payment-records", + "title": "Payment records", + "questline": "orders-fulfillment", + "featureRefs": [ + "ecommerce.progression.payment-records" + ], + "gradingGroups": [ + "ecommerce.progression.payment-records#payment", + "ecommerce.spec.concurrency-safety#stock-limit", + "ecommerce.spec.transactional-integrity#payment-deduplication" + ], + "dependencies": [ + { + "id": "checkout", + "reason": "Checkout creates a payment record." + }, + { + "id": "purchasing", + "reason": "The payment belongs to an order." + } + ] + }, + { + "id": "managed-support", + "title": "Managed support cases", + "questline": "customer-support", + "featureRefs": [ + "ecommerce.progression.managed-support" + ], + "gradingGroups": [ + "ecommerce.progression.managed-support#case-updates", + "ecommerce.spec.access-control#managed-support-privacy", + "ecommerce.spec.live-state#managed-support" + ], + "dependencies": [ + { + "id": "support-history", + "reason": "Customers need an owned case history." + }, + { + "id": "support-triage", + "reason": "Staff need an assigned ticket queue." + } + ] + }, + { + "id": "promotion-checkout", + "title": "Promotion checkout", + "questline": "promotions", + "featureRefs": [ + "ecommerce.progression.promotion-checkout" + ], + "gradingGroups": [ + "ecommerce.progression.promotion-checkout#active", + "ecommerce.progression.promotion-checkout#expired", + "ecommerce.progression.promotion-checkout#exhausted" + ], + "dependencies": [ + { + "id": "checkout", + "reason": "A promotion is applied at checkout." + }, + { + "id": "promotion-rules", + "reason": "Checkout needs an active promotion rule." + } + ] + }, + { + "id": "stock-alerts", + "title": "Stock alerts", + "questline": "notifications", + "featureRefs": [ + "ecommerce.progression.stock-alerts" + ], + "gradingGroups": [ + "ecommerce.progression.stock-alerts#delivery", + "ecommerce.spec.access-control#stock-alert-privacy", + "ecommerce.spec.transactional-integrity#stock-alert-deduplication", + "ecommerce.spec.external-data-sync#external-stock-alert" + ], + "dependencies": [ + { + "id": "notification-preferences", + "reason": "The customer chooses whether to receive alerts." + }, + { + "id": "warehouse-admin", + "reason": "Alerts respond to warehouse stock changes." + } + ] + }, + { + "id": "personalized-recommendations", + "title": "Personalized recommendations", + "questline": "recommendations", + "featureRefs": [ + "ecommerce.progression.personalized-recommendations" + ], + "gradingGroups": [ + "ecommerce.progression.personalized-recommendations#ordering", + "ecommerce.spec.access-control#recommendation-profile-isolation" + ], + "dependencies": [ + { + "id": "recommendations", + "reason": "Personalization extends customer recommendations." + } + ] + }, + { + "id": "staff-activity", + "title": "Staff activity history", + "questline": "business-operations", + "featureRefs": [ + "ecommerce.progression.staff-activity" + ], + "gradingGroups": [ + "ecommerce.progression.staff-activity#activity", + "ecommerce.spec.access-control#staff-activity-privacy" + ], + "dependencies": [ + { + "id": "catalog-management", + "reason": "Catalog changes provide attributable staff activity." + } + ] + }, + { + "id": "order-support", + "title": "Order-linked support", + "questline": "customer-support", + "featureRefs": [ + "ecommerce.progression.order-support" + ], + "gradingGroups": [ + "ecommerce.progression.order-support#owned-order", + "ecommerce.spec.access-control#order-support-ownership" + ], + "dependencies": [ + { + "id": "managed-support", + "reason": "The order is attached to an existing managed case." + }, + { + "id": "purchasing", + "reason": "The linked order must exist." + } + ] + }, + { + "id": "promotion-reporting", + "title": "Promotion reporting", + "questline": "promotions", + "featureRefs": [ + "ecommerce.progression.promotion-reporting" + ], + "gradingGroups": [ + "ecommerce.progression.promotion-reporting#redemptions", + "ecommerce.progression.promotion-reporting#revenue" + ], + "dependencies": [ + { + "id": "promotion-checkout", + "reason": "Reports count promotion redemptions." + } + ] + }, + { + "id": "automatic-reorder", + "title": "Automatic reorder rules", + "questline": "inventory", + "featureRefs": [ + "ecommerce.progression.automatic-reorder" + ], + "gradingGroups": [ + "ecommerce.progression.automatic-reorder#threshold", + "ecommerce.spec.access-control#automatic-reorder-access", + "ecommerce.spec.transactional-integrity#automatic-reorder-deduplication" + ], + "dependencies": [ + { + "id": "purchasing", + "reason": "Purchases reduce stock and can cross reorder thresholds." + }, + { + "id": "scheduled-restocks", + "reason": "Rules create scheduled restocks." + }, + { + "id": "staff-roles", + "reason": "Only warehouse staff can manage reorder rules." + } + ] + }, + { + "id": "cart-recovery", + "title": "Cart recovery", + "questline": "shopping-checkout", + "featureRefs": [ + "ecommerce.progression.cart-recovery" + ], + "gradingGroups": [ + "ecommerce.progression.cart-recovery#available", + "ecommerce.progression.cart-recovery#partial" + ], + "dependencies": [ + { + "id": "cart-expiration", + "reason": "Recovery restores an expired cart." + } + ] + }, + { + "id": "delivery-notifications", + "title": "Delivery notifications", + "questline": "notifications", + "featureRefs": [ + "ecommerce.progression.delivery-notifications" + ], + "gradingGroups": [ + "ecommerce.progression.delivery-notifications#delivery", + "ecommerce.spec.access-control#delivery-notification-privacy" + ], + "dependencies": [ + { + "id": "notification-preferences", + "reason": "The customer chooses whether to receive delivery notices." + }, + { + "id": "order-delivery", + "reason": "A notice follows a delivered order." + } + ] + }, + { + "id": "recommendation-feedback", + "title": "Recommendation feedback", + "questline": "recommendations", + "featureRefs": [ + "ecommerce.progression.recommendation-feedback" + ], + "gradingGroups": [ + "ecommerce.progression.recommendation-feedback#dismissal", + "ecommerce.spec.access-control#recommendation-feedback-privacy", + "ecommerce.spec.state-durability#recommendation-feedback-restart" + ], + "dependencies": [ + { + "id": "personalized-recommendations", + "reason": "Feedback applies to personal recommendations." + } + ] + }, + { + "id": "support-refunds", + "title": "Support refunds", + "questline": "customer-support", + "featureRefs": [ + "ecommerce.progression.support-refunds" + ], + "gradingGroups": [ + "ecommerce.progression.support-refunds#resolution", + "ecommerce.spec.transactional-integrity#support-refund-accounting", + "ecommerce.spec.access-control#support-refund-access" + ], + "dependencies": [ + { + "id": "order-cancellation", + "reason": "A support refund uses the established order refund path." + }, + { + "id": "order-support", + "reason": "The refund resolves an order-linked case." + } + ] + }, + { + "id": "product-bundles", + "title": "Catalog bundles", + "questline": "catalog-discovery", + "featureRefs": [ + "ecommerce.feature.product-bundles" + ], + "gradingGroups": [ + "ecommerce.feature.product-bundles#product-bundles", + "ecommerce.spec.bundle-integrity#bundle-743" + ], + "dependencies": [ + { + "id": "catalog-management", + "reason": "Uses catalog management." + } + ] + }, + { + "id": "bundle-checkout", + "title": "Bundle checkout", + "questline": "shopping-checkout", + "featureRefs": [ + "ecommerce.feature.bundle-checkout" + ], + "gradingGroups": [ + "ecommerce.feature.bundle-checkout#bundle-checkout", + "ecommerce.spec.bundle-integrity#bundle-744", + "ecommerce.spec.bundle-integrity#bundle-745", + "ecommerce.spec.bundle-integrity#bundle-746", + "ecommerce.spec.bundle-integrity#bundle-747" + ], + "dependencies": [ + { + "id": "product-bundles", + "reason": "Uses catalog bundles." + }, + { + "id": "reservations", + "reason": "Uses reservations." + } + ] + }, + { + "id": "bundle-returns", + "title": "Bundle returns", + "questline": "orders-fulfillment", + "featureRefs": [ + "ecommerce.feature.bundle-returns" + ], + "gradingGroups": [ + "ecommerce.feature.bundle-returns#bundle-returns", + "ecommerce.spec.bundle-integrity#bundle-742", + "ecommerce.spec.bundle-integrity#bundle-748" + ], + "dependencies": [ + { + "id": "bundle-checkout", + "reason": "Uses bundle checkout." + }, + { + "id": "order-returns", + "reason": "Uses returns." + } + ] + }, + { + "id": "store-credit", + "title": "Store credit", + "questline": "business-operations", + "featureRefs": [ + "ecommerce.feature.store-credit" + ], + "gradingGroups": [ + "ecommerce.feature.store-credit#feature-750", + "ecommerce.spec.store-credit#production-752", + "ecommerce.spec.store-credit#production-753", + "ecommerce.spec.store-credit#production-754", + "ecommerce.spec.store-credit#production-755" + ], + "dependencies": [ + { + "id": "payment-records", + "reason": "Uses payment records." + }, + { + "id": "staff-roles", + "reason": "Uses staff roles." + } + ] + }, + { + "id": "split-tender-refunds", + "title": "Split-tender refunds", + "questline": "customer-support", + "featureRefs": [ + "ecommerce.feature.split-tender-refunds" + ], + "gradingGroups": [ + "ecommerce.feature.split-tender-refunds#feature-751", + "ecommerce.spec.split-tender-refunds#production-756", + "ecommerce.feature.split-tender-refunds#return-refund-interaction" + ], + "dependencies": [ + { + "id": "store-credit", + "reason": "Uses store credit." + }, + { + "id": "support-refunds", + "reason": "Uses support refunds." + }, + { + "id": "order-returns", + "reason": "Checks that returns and refunds share one financial total while stock is restored once." + } + ] + }, + { + "id": "subscriptions", + "title": "Scheduled purchases", + "questline": "orders-fulfillment", + "featureRefs": [ + "ecommerce.feature.subscriptions" + ], + "gradingGroups": [ + "ecommerce.feature.subscriptions#feature-760", + "ecommerce.spec.subscriptions#production-761", + "ecommerce.spec.subscriptions#production-762", + "ecommerce.spec.subscriptions#production-763" + ], + "dependencies": [ + { + "id": "payment-records", + "reason": "Uses payment records." + } + ] + } + ], + "questlines": [ + { + "id": "identity-profiles", + "title": "Identity and profiles" + }, + { + "id": "catalog-discovery", + "title": "Catalog and discovery" + }, + { + "id": "shopping-checkout", + "title": "Shopping and checkout" + }, + { + "id": "orders-fulfillment", + "title": "Orders and fulfilment" + }, + { + "id": "inventory", + "title": "Inventory" + }, + { + "id": "reviews", + "title": "Reviews" + }, + { + "id": "recommendations", + "title": "Recommendations" + }, + { + "id": "notifications", + "title": "Notifications" + }, + { + "id": "staff-access", + "title": "Staff access" + }, + { + "id": "business-operations", + "title": "Business operations" + }, + { + "id": "promotions", + "title": "Promotions" + }, + { + "id": "customer-support", + "title": "Customer support" + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/prompts/02-operations.md b/tools/stack-bench/tracks/ecommerce/prompts/02-operations.md new file mode 100644 index 00000000000..ccde909b62c --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/02-operations.md @@ -0,0 +1,72 @@ +# Store operations + +Add warehouse operations, order fulfilment, cancellation, returns, and price management. + +## New: warehouse staff + +- A **staff** account signs in like anyone else but sees the fulfilment area + instead of the admin area +- Staff accounts are seeded (below); sign-up still creates ordinary customers +- Staff cannot change prices or create items; admins can do everything staff can + +## Features + +### Fulfilment + +Show pending orders in a fulfilment queue, oldest first. Each entry shows its items and +shipping warehouse. Staff can mark an order shipped. Show the status in order history. + +Name the shipping write `ship`: server-based stacks expose `POST /api/fulfilment/ship` +with `{ "orderId": ... }`, and SpacetimeDB exposes reducer `shipOrder`. + +The +same authentication and staff-only authorization apply through this path. + +### Moving stock + +An admin can transfer units of an item from one warehouse to another. + +### Cancelling and returning + +Customers can cancel an order before it ships. Refund the purchase and return its stock +to the supplying warehouse. After shipping, customers can return an item for its purchase +price and the item is restocked. Show cancelled and returned states in order history. + +### Prices + +An admin can change an item's price. Show prices in the catalog. + +### Operational views + +- Low stock: items with 10 units or fewer, most urgent first. +- Warehouse utilisation: total units in each warehouse. +- Category totals: units sold and revenue per category. +- Fulfilment queue depth: number of pending orders. +- Recommended for you: items from categories the customer bought from, most-purchased + first, excluding items already in their cart. Signed-out visitors see best sellers. + +### What must stay true + +- An item's stock is always the sum of its warehouse rows, no matter which of the + actions above last touched it +- Revenue always equals the sum of orders that are still standing — not cancelled, + minus what was returned +- The best-seller ranking always reflects kept purchases +- Every one of these numbers is the same for every person looking at it + +## Starting data + +Add to the level 1 catalogue, without disturbing it: + +**Categories** — every existing item gets one: + +| Category | Items | +|---|---| +| `Home` | Air Purifier, Desk Lamp, Induction Cooktop, Espresso Machine, Coffee Grinder, USB Cable | +| `Audio` | Bluetooth Speaker, Headphones | +| `Computing` | Gaming Mouse, Keyboard, Laptop Stand, Webcam | +| `Photo` | Mirrorless Camera | + +**Staff account:** username `staff`, password `stackbench-staff-2026`. + +No orders, cancellations, returns or price changes exist at the start. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/03-scheduled.md b/tools/stack-bench/tracks/ecommerce/prompts/03-scheduled.md new file mode 100644 index 00000000000..494caba035a --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/03-scheduled.md @@ -0,0 +1,60 @@ +# Scheduled work + +Add stock reservations, scheduled restocks, order delivery, and cart expiration. + +## Reservations that expire + +- Adding an item to the cart takes a **reservation** on one unit for **90 + seconds**: storefront stock drops immediately, for everyone +- If the customer checks out within the window, the reservation becomes a sale +- If they do not, the reservation **expires on its own** and the unit returns to + storefront stock — visibly, without anyone reloading or clicking +- A cart line whose reservation expired is marked expired, not silently removed +- The cart line shows the reservation's remaining time, live, the way the + pending restocks list does +- Extending is allowed: touching the quantity renews the 90 seconds + +## Scheduled restocks + +- An admin can schedule a restock: item, warehouse, quantity, and a delay in + seconds +- The **pending restocks** list shows each one with its remaining time, live +- When the delay elapses the restock applies itself — stock rises for every + viewer, the entry leaves the pending list, and it appears in the stock ledger +- An admin can cancel a pending restock before it fires; cancelled restocks + never apply + +## Orders that advance themselves + +- Shipping remains the immediate staff action introduced at level 2 +- A shipped order moves to `delivered` **60 seconds** after shipping +- Delivery is visible live in the customer's order history and the staff view +- A cancelled order never advances + +## Abandoned carts + +- A cart untouched for **5 minutes** is abandoned: its reservations are released + and the customer is told the cart expired +- An abandoned cart is empty, not hidden — a customer who returns sees an empty + cart, and the stock is back on the shelf + +## What must stay true + +- **Deferred work survives a restart.** Everything above must still happen if + the backend restarts between the scheduling and the firing. Work scheduled + before a restart fires after it, once, at the right time. +- **Exactly once.** A backend restart must never double-apply a scheduled + restock or advance an order twice. If it fired, it fired once. +- **Nothing fires early.** A restart must not cause pending work to run + immediately just because it is being re-read. +- **The clock is the database's, not the browser's.** Countdowns shown in the UI + may be rendered client-side, but what actually expires is decided by the + server: a client with a wrong clock, or no client at all, changes nothing. +- **Stock conservation still holds** across reservations, expiries, scheduled + restocks and cancellations: units are never created or destroyed by the + passage of time. + +## Starting data + +Everything from level 1 and level 2, unchanged. No pending restocks, no +reservations and no abandoned carts exist at the start. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/fragments/named-actions-inventory-operations.md b/tools/stack-bench/tracks/ecommerce/prompts/fragments/named-actions-inventory-operations.md new file mode 100644 index 00000000000..1db483b2520 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/fragments/named-actions-inventory-operations.md @@ -0,0 +1,11 @@ +### Named action for stock transfers + +The transfer write already appears in the requested features. Give it the exact +named action shown here so the same request can be made without a browser. Use +your stack's ordinary write path; do not add another transport for this action. + +| Action | MongoDB/PostgreSQL app | SpacetimeDB app | +|---|---|---| +| transfer stock between warehouses | `POST /api/admin/transfer` | reducer `adminTransferStock` | + +Use the same action as the visible transfer control. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/fragments/named-actions-returns-pricing.md b/tools/stack-bench/tracks/ecommerce/prompts/fragments/named-actions-returns-pricing.md new file mode 100644 index 00000000000..27a7e4f9ea1 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/fragments/named-actions-returns-pricing.md @@ -0,0 +1,12 @@ +### Named actions for pricing and cancellation + +Every write below already appears in the requested features. Give it the exact +named action shown here so the same request can be made without a browser. Use +your stack's ordinary write path; do not add another transport for this action. + +| Action | MongoDB/PostgreSQL app | SpacetimeDB app | +|---|---|---| +| change an item's price | `POST /api/admin/price` | reducer `adminChangePrice` | +| cancel an order | `POST /api/orders/:id/cancel` | reducer `cancelOrder` | + +Use the same actions as the visible controls. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/fragments/startup-probe.md b/tools/stack-bench/tracks/ecommerce/prompts/fragments/startup-probe.md new file mode 100644 index 00000000000..2cc3f5f662a --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/fragments/startup-probe.md @@ -0,0 +1,6 @@ +## Catalogue readiness + +On server-based stacks (MongoDB/PostgreSQL), expose `GET /api/items`. Return +HTTP 200 and a JSON object with an `items` array. This interface does not define +how the app stores or restores its data. SpacetimeDB apps use the module and do +not need this endpoint. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/l2-action-inputs.md b/tools/stack-bench/tracks/ecommerce/prompts/l2-action-inputs.md new file mode 100644 index 00000000000..5a3a1f2e452 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/l2-action-inputs.md @@ -0,0 +1,22 @@ +## Operator action inputs + +Expose machine-readable inputs for the public application interface. These are inert HTML attributes; they do not prescribe the application framework, database API, or internal implementation. + +- Each `order-item` exposes `data-ship-input` containing a JSON object with exactly `orderId`. +- Each `order-item` exposes `data-cancel-input` containing a JSON object with exactly `orderId`. +Use the identifier representation expected by the stack's declared server operation. Do not put credentials or account identifiers in these attributes. + +## Price action input + +- Each `admin-item-row` exposes `data-price-input` containing a JSON object with exactly `itemId` and numeric `price` from the current price input. + +Use the identifier representation expected by the stack's declared server operation. Do not put credentials or account identifiers in this attribute. + +## Inventory contention action inputs + +Expose machine-readable inputs for the public application interface. These are inert HTML attributes; they do not prescribe the application framework, database API, or internal implementation. + +- Each catalog `item-card` exposes `data-buy-input` containing a JSON object with exactly `itemId`. +- Each `admin-item-row` exposes `data-transfer-input` containing a JSON object with exactly `itemId`, `fromWarehouseId`, and `toWarehouseId` for the currently selected source and destination. + +Use the identifier representation expected by the stack's declared server operation. Do not put credentials or account identifiers in these attributes. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/accounts.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/accounts.md new file mode 100644 index 00000000000..0836bedb6d8 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/accounts.md @@ -0,0 +1,5 @@ +## Accounts + +Visitors can create an account with a username and password. Returning users can sign in with +those credentials, see which account is active, and sign out. Show a useful error for a taken +username or an incorrect password. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/bundle-checkout.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/bundle-checkout.md new file mode 100644 index 00000000000..2355d633cfb --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/bundle-checkout.md @@ -0,0 +1,5 @@ +## Bundle checkout + +Customers can add a whole bundle to their cart and buy it through checkout. Its components +use the existing stock reservations and reservation lifetime. Removing a bundle releases +its reservation. Show the bundle as one order line at its bundle price. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/bundle-returns.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/bundle-returns.md new file mode 100644 index 00000000000..625fcbfc80a --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/bundle-returns.md @@ -0,0 +1,5 @@ +## Bundle returns + +Customers can return a shipped bundle as a whole. Restore the purchased component quantities +to their original warehouses and refund the price paid. Show the return and refund on the +order. Partial bundle returns are not supported. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/cancellation-conservation.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/cancellation-conservation.md new file mode 100644 index 00000000000..e76c7f20b04 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/cancellation-conservation.md @@ -0,0 +1,4 @@ +## Cancellation conservation + +Cancelling an order restores its stock to the warehouse that supplied it. The restored total is +the same for current and newly opened clients. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/cancellation-queue.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/cancellation-queue.md new file mode 100644 index 00000000000..d98b69a9e52 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/cancellation-queue.md @@ -0,0 +1,4 @@ +## Fulfilment queue integration + +When fulfilment is available, cancelling a pending order also removes it from the fulfilment +queue. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/cart-expiration.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/cart-expiration.md new file mode 100644 index 00000000000..c9f5fddfe22 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/cart-expiration.md @@ -0,0 +1,4 @@ +## Cart expiration + +A cart with no activity for five minutes expires and releases its reservations. The customer +sees an empty cart and an expiration notice. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/cart.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/cart.md new file mode 100644 index 00000000000..019841246fa --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/cart.md @@ -0,0 +1,4 @@ +## Cart + +A signed-in customer has one cart. They can add an item, change its quantity, remove it, and +see the total. Adding the same item again increases the existing line quantity. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/catalog-discovery.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/catalog-discovery.md new file mode 100644 index 00000000000..984185ca62a --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/catalog-discovery.md @@ -0,0 +1,4 @@ +## Catalog discovery + +Show the ten most-purchased items. Break ties by item name. Search matches any part of +an item name, without regard to case, across the full catalog. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/catalog-items.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/catalog-items.md new file mode 100644 index 00000000000..70e5776538a --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/catalog-items.md @@ -0,0 +1,3 @@ +## Catalog items + +Show public catalog items. Each item shows its name, price, and total stock. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/catalog-management.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/catalog-management.md new file mode 100644 index 00000000000..a0dcfce7534 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/catalog-management.md @@ -0,0 +1,4 @@ +## Catalog management + +Authorized staff can add a product with named variants. The new product and its variants appear +in the public catalog. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/checkout-recovery-specification.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/checkout-recovery-specification.md new file mode 100644 index 00000000000..91790502ca2 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/checkout-recovery-specification.md @@ -0,0 +1,4 @@ +After the application or database process restarts, an interrupted checkout leaves +either its unchanged cart or one complete order with the cart cleared. A checkout +reported as complete and the account's earlier orders remain recorded correctly. +The application can accept new checkouts after recovery. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/checkout.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/checkout.md new file mode 100644 index 00000000000..3aeddbf75b4 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/checkout.md @@ -0,0 +1,4 @@ +## Checkout + +Checkout creates one order, reduces stock for every line, and empties the cart. Show an +explanation if checkout fails. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/customer-profile.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/customer-profile.md new file mode 100644 index 00000000000..022eff8b1cc --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/customer-profile.md @@ -0,0 +1,3 @@ +## Customer profile + +A signed-in customer can save and view their name and shipping address. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/delivery-notifications.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/delivery-notifications.md new file mode 100644 index 00000000000..8524ae5d726 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/delivery-notifications.md @@ -0,0 +1,3 @@ +## Delivery notifications + +Notify customers when their orders are delivered, using their notification preferences. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/faceted-search.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/faceted-search.md new file mode 100644 index 00000000000..aa0265b6c41 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/faceted-search.md @@ -0,0 +1,7 @@ +# Faceted search + +Let visitors filter the catalog by category, minimum price, maximum price, and availability. +Combine the selected filters. While search text or any filter is active, order matching +items by name, with ten results per page. With no search text or active filter, use +the storefront's purchase ranking, breaking ties by item name. Clearing all search +text and filters returns to that ranking. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/inventory-dashboard.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/inventory-dashboard.md new file mode 100644 index 00000000000..03b8ce57636 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/inventory-dashboard.md @@ -0,0 +1,3 @@ +## Inventory dashboard + +Give administrators a low-stock view. It lists items with 10 units or fewer, most urgent first. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/l1-action-inputs.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/l1-action-inputs.md new file mode 100644 index 00000000000..6469d2e0b01 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/l1-action-inputs.md @@ -0,0 +1,21 @@ +## Purchase action input + +Put a `data-buy-input` attribute on each `item-card`. Its value is a JSON +object containing that card's server item identifier, for example +`{"itemId":42}`. The identifier may be a JSON number or string. Use this value +with the same ordinary buy operation as the visible button. + +## Cart quantity action input + +Put a `data-cart-input` attribute on each `cart-item`. Its value is a JSON +object containing that line's item identifier, for example `{"itemId":42}`. +The identifier may be a JSON number or string. + +## Restock action input + +Put a `data-restock-input` attribute on each `admin-location-row`. Its value is +a JSON object containing the row's server item and warehouse identifiers plus +a valid one-unit restock, for example +`{"itemId":42,"warehouseId":7,"quantity":1}`. Identifiers may be JSON numbers +or strings. Use this value with the same ordinary restock operation as the +visible control. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/l1-external-sync-specifications.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/l1-external-sync-specifications.md new file mode 100644 index 00000000000..7dbc7a0a287 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/l1-external-sync-specifications.md @@ -0,0 +1,83 @@ +## Access control: purchasing + +Treat identity as server-enforced authority, not UI decoration. Unauthenticated +callers cannot purchase, one account cannot place an order for another account, +and order history is visible only to its owner. Knowing a username never grants +access to that account. + +## Access control: warehouse administration + +Customer accounts cannot perform warehouse-administration writes. Enforce this +on the server even if the corresponding controls are hidden in the UI. + +## Access control: reviews + +Only a customer who purchased an item may review it. Enforce this on the server +instead of relying on whether the review form is visible. + +## Access control: cart + +One account cannot read or change another account's cart. Refuse a cart request +whose quantity is negative without changing state. + +## State durability: accounts + +A signed-in session survives a page reload as the same account. + +## State durability: account data + +The same account keeps its cart and orders across reload and connection loss. +After reconnect it has current state without another sign-in. Restarting the +application must not duplicate starting data or reset state users changed. + +## Live state: catalog and purchasing + +Stock and best-seller ranking update every affected open storefront without a +reload, including signed-out storefronts. + +## Live state: cart + +The same account open in two clients sees one current cart; a change in either +client reaches the other without a reload. + +## Live state: reviews + +Reviews and average ratings update affected open item views without a reload. +A view opened while a review is submitted converges to the current review list. + +## Live state: warehouse administration + +Restocking updates warehouse quantities, total item stock, and open storefronts +without a reload. + +## Concurrency safety: purchasing + +Stock never becomes negative and only one customer can receive the last unit. + +## Concurrency safety: restocking + +Concurrent restocks and purchases preserve every accepted stock change. + +## Concurrency safety: checkout + +Repeating or racing checkout for the same cart creates only one order. + +## Transactional integrity: reviews + +A customer has at most one review per item; another submission updates it. + +## Transactional integrity: purchasing + +The server controls prices and order attribution rather than trusting client +values. Historical order prices do not change. + +## Transactional integrity: warehouse accounting + +Every stock reduction caused by a sale has its matching order, revenue equals +the orders' recorded totals, and fresh clients agree with those results. + +## External data synchronization + +Other systems may write stock directly without calling the application. Open +pages and newly loaded pages must converge to a direct stock correction, +including a correction made while the application server is down. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/l1-framing.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/l1-framing.md new file mode 100644 index 00000000000..98b4cacf649 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/l1-framing.md @@ -0,0 +1,24 @@ +# Ecommerce storefront + +Build a polished ecommerce web application using the selected stack and only +the feature sections that follow. Use `Storefront` as the visible page title. +Use a clear responsive layout, useful loading, empty, and error states, and +ordinary keyboard support. + +Start with exactly two warehouses, `East` and `West`, and these items: + +| Item | Price | East | West | +|---|---:|---:|---:| +| Air Purifier | 189.00 | 60 | 40 | +| Bluetooth Speaker | 79.50 | 50 | 50 | +| Coffee Grinder | 64.00 | 70 | 30 | +| Desk Lamp | 42.00 | 55 | 45 | +| Espresso Machine | 449.00 | 80 | 20 | +| Gaming Mouse | 59.00 | 50 | 50 | +| Headphones | 199.00 | 60 | 40 | +| Induction Cooktop | 329.00 | 50 | 50 | +| Keyboard | 89.00 | 70 | 30 | +| Laptop Stand | 29.00 | 90 | 10 | +| Mirrorless Camera | 1299.00 | 2 | 1 | +| USB Cable | 65.00 | 0 | 0 | +| Webcam | 69.00 | 60 | 40 | diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/l1-hooks.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/l1-hooks.md new file mode 100644 index 00000000000..162ed40af5f --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/l1-hooks.md @@ -0,0 +1,95 @@ +## Public application interface + +Use each interface name below as an exact HTML attribute value. Use `id` for a +one-off element. Use `data-role` when the same interface can appear more than +once. These attributes do not prescribe the layout, data model, libraries, or +transport. + +Each exposed server operation completes every valid request, including requests +that overlap, without relying on the browser to retry it. A business refusal, such as +insufficient stock, still completes the request. + +## Catalog controls + +| Element ID | Observable element | +|---|---| +| `app-title` | visible application title, `Storefront` | +| `item-list` | storefront item-card container | +| `catalog-link` | control that returns to the storefront catalog | +| `item-card` | one storefront or search-result item | +| `item-name` | item name within a card; activating it opens `item-detail` | +| `item-price` | numeric item price within a card | +| `item-stock` | numeric total stock within a card | +| `search-input` | catalog search input | +| `search-results` | search-result item-card container | +| `item-detail` | opened item detail view | +| `out-of-stock` | visible out-of-stock state | + +## Account controls + +| Element ID | Observable element | +|---|---| +| `signup-username` | sign-up username input | +| `signup-password` | sign-up password input | +| `signup-submit` | sign-up submit control | +| `signin-toggle` | available on the signed-out page to reveal sign-in; omit when sign-in inputs are already visible | +| `signup-toggle` | reveals sign-up; available on the signed-out page or after opening `signin-toggle`; omit when sign-up inputs are already visible | +| `signin-username` | sign-in username input | +| `signin-password` | sign-in password input | +| `signin-submit` | sign-in submit control | +| `current-user` | active account name | +| `signout` | sign-out control | +| `auth-error` | visible account error | + +## Purchasing controls + +| Element ID | Observable element | +|---|---| +| `buy-now` | item-card control that buys one unit | +| `orders-toggle` | control that opens order history | +| `order-list` | order-history container | +| `order-item` | one order containing its item names | +| `order-total` | numeric order total | +| `buy-error` | visible purchase or checkout error | + +## Cart controls + +| Element ID | Observable element | +|---|---| +| `add-to-cart` | item-card control that adds one unit | +| `cart-toggle` | control that opens the cart | +| `cart-count` | numeric count of units in the cart | +| `cart-panel` | cart container | +| `cart-item` | one cart line containing its item name | +| `cart-quantity` | numeric quantity for a cart line | +| `cart-total` | numeric cart total | +| `cart-remove` | control that removes a cart line | +| `checkout-submit` | checkout control | +| `empty-cart` | visible empty-cart state | + +## Review controls + +| Element ID | Observable element | +|---|---| +| `review-rating` | rating input or select with values one through five | +| `review-toggle` | opens the review form on item details for eligible customers; omit when already shown | +| `review-input` | review comment input | +| `review-submit` | review submit control | +| `review-average` | numeric average rating | +| `review-item` | one visible review containing its comment | +| `review-error` | visible review error | + +## Warehouse administration controls + +| Element ID | Observable element | +|---|---| +| `admin-link` | control that opens warehouse administration | +| `admin-panel` | warehouse-administration container | +| `admin-item-row` | one item row containing its name; all item and holding rows are available in the open admin panel without pagination | +| `admin-stock` | numeric total stock within an item row | +| `admin-warehouse-item` | one warehouse entry containing its name | +| `admin-location-row` | one item-and-warehouse holding row | +| `admin-location-qty` | numeric quantity within a holding row | +| `restock-input` | quantity input within a holding row | +| `restock-submit` | restock control within the same holding row | +| `admin-revenue` | numeric total revenue | diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/l1-specifications.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/l1-specifications.md new file mode 100644 index 00000000000..bf884b0cf86 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/l1-specifications.md @@ -0,0 +1,87 @@ +## Access control: purchasing + +Treat identity as server-enforced authority, not UI decoration. Unauthenticated +callers cannot purchase, one account cannot place an order for another account, +and order history is visible only to its owner. Knowing a username never grants +access to that account. + +## Access control: warehouse administration + +Customer accounts cannot perform warehouse-administration writes. Enforce this +on the server even if the corresponding controls are hidden in the UI. + +## Access control: reviews + +Only a customer who purchased an item may review it. Enforce this on the server +instead of relying on whether the review form is visible. + +## Access control: cart + +One account cannot read or change another account's cart. Refuse a cart request +whose quantity is negative without changing state. + +## State durability: accounts + +A signed-in session survives a page reload as the same account. + +## State durability: account data + +The same account keeps its cart and orders across reload and connection loss. +After reconnect it has current state without another sign-in. Restarting the +application must not duplicate starting data or reset state users changed. + +## Live state: catalog and purchasing + +Stock and best-seller ranking update every affected open storefront without a +reload, including signed-out storefronts. + +## Live state: cart + +The same account open in two clients sees one current cart; a change in either +client reaches the other without a reload. + +## Live state: reviews + +Reviews and average ratings update affected open item views without a reload. +A view opened while a review is submitted converges to the current review list. + +## Live state: warehouse administration + +Restocking updates warehouse quantities, total item stock, and open storefronts +without a reload. + +## Concurrency safety: purchasing + +Stock never becomes negative and only one customer can receive the last unit. + +## Concurrency safety: restocking + +Concurrent restocks and purchases preserve every accepted stock change. + +## Concurrency safety: checkout + +Repeating or racing checkout for the same cart creates only one order. + +## Transactional integrity: reviews + +A customer has at most one review per item. A later submission must not create +a duplicate; it may update the existing review or be refused. + +## Transactional integrity: purchasing + +The server controls prices and order attribution rather than trusting client +values. Historical order prices do not change. + +## Transactional integrity: warehouse accounting + +Every stock reduction caused by a sale has its matching order, revenue equals +the orders' recorded totals, and fresh clients agree with those results. + +## External data synchronization + +Other systems may write stock directly without calling the application. Use +singular tables `item(id, name, price)`, `warehouse(id, name)`, and +`stock(item_id, warehouse_id, quantity)` as the source of truth for this +interoperability surface. Open pages and newly loaded pages must converge to a +direct stock correction, including a correction made while the application +server is down. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/l3-specifications.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/l3-specifications.md new file mode 100644 index 00000000000..ca2bab9966f --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/l3-specifications.md @@ -0,0 +1,41 @@ +# Level 3 production behavior + +## Durable reservations + +Pending reservations survive a backend restart. + +## Durable restocks + +Pending restocks survive a backend restart. + +## Durable order delivery + +Pending order delivery survives a backend restart. + +## Durable cart expiration + +Pending cart expiration survives a backend restart. + +## Exactly-once restocks + +Restarting the backend cannot apply a restock more than once. + +## Exactly-once delivery + +Restarting the backend cannot apply an order transition more than once. + +## Server-timed restocks + +A pending restock does not run early after a restart. + +## Server-timed reservations + +A reservation expires without an open browser. + +## Deferred-work access + +Only an admin can schedule or cancel a restock. + +## Stock conservation + +Reservation expiry returns exactly the stock that the reservation took. Checkout does not take reserved stock twice. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/later-specifications.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/later-specifications.md new file mode 100644 index 00000000000..96ab8148ea9 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/later-specifications.md @@ -0,0 +1,25 @@ +# Expected production behavior + +## Product bundles + +Only authorized staff can change bundles. + +## Bundle checkout + +Reserve all components or none. Competing purchases share the same stock. Release component reservations when the cart expires or the bundle is removed. + +## Bundle returns + +Return the original component allocations and price paid. Repeating a return must not change stock or refunds again. Customers cannot return another account's order. + +## Store credit + +Only authorized staff can grant credit. Repeating a reference must not issue credit twice. Concurrent checkout must not duplicate the order or credit use. Accepted credit survives a backend restart. + +## Split-tender refunds + +Concurrent or repeated refunds must restore each original payment portion only once. The resulting balance and refund records survive a backend restart. + +## Scheduled purchases + +Pending deliveries and pauses survive a backend restart. Process elapsed pending slots after recovery. Completed delivery slots must not run again. Customers cannot change another account's subscription. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/managed-support.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/managed-support.md new file mode 100644 index 00000000000..a977b4bfbaa --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/managed-support.md @@ -0,0 +1,3 @@ +## Managed support cases + +Customers and staff can exchange replies and update the status of a support case. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/notification-preferences.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/notification-preferences.md new file mode 100644 index 00000000000..6dbefe203a9 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/notification-preferences.md @@ -0,0 +1,3 @@ +## Notification preferences + +Signed-in customers can turn order and stock notifications on or off. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/operations-access.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/operations-access.md new file mode 100644 index 00000000000..e00aa4457cd --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/operations-access.md @@ -0,0 +1,5 @@ +# Fulfilment operations + +Add a fulfilment area for staff and administrators. Show pending orders, their items, and the +warehouse that will ship them. Staff and administrators can mark an order as shipped. Show the +new status in the fulfilment area and the customer's order history. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/order-accounting.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/order-accounting.md new file mode 100644 index 00000000000..c91a8b726e1 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/order-accounting.md @@ -0,0 +1,7 @@ +## Cancellation accounting + +Revenue equals the sum of orders that remain paid. Cancellation removes the cancelled amount. + +## Price accounting + +Later catalog price changes do not change the amount recorded on an existing order. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/order-cancellation.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/order-cancellation.md new file mode 100644 index 00000000000..bad4a33d7e8 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/order-cancellation.md @@ -0,0 +1,4 @@ +## Order cancellation + +A customer can cancel an order before it ships. Refund the purchase, return its stock to +the supplying warehouse, and show its cancelled status in order history. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/order-delivery.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/order-delivery.md new file mode 100644 index 00000000000..a92ebd43f38 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/order-delivery.md @@ -0,0 +1,4 @@ +## Order delivery + +A shipped order becomes delivered 60 seconds after shipping. Show its status in the +customer's order history and the staff view. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/order-ownership.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/order-ownership.md new file mode 100644 index 00000000000..efccb83a174 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/order-ownership.md @@ -0,0 +1,3 @@ +## Order ownership + +The server allows a customer to act only on that customer's own orders. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/order-returns.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/order-returns.md new file mode 100644 index 00000000000..edea068225b --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/order-returns.md @@ -0,0 +1,4 @@ +## Order returns + +A customer can return an item after the order ships for a refund of its purchase price. +Restock the item and mark it returned in order history. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/order-support.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/order-support.md new file mode 100644 index 00000000000..28c1a80d25c --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/order-support.md @@ -0,0 +1,4 @@ +## Order-linked support + +A customer can attach an order to a support case. Staff can inspect the linked order +from the case. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/payment-records.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/payment-records.md new file mode 100644 index 00000000000..9f31664b219 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/payment-records.md @@ -0,0 +1,3 @@ +## Payment records + +Show the payment status and amount paid on each order. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/price-authorization.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/price-authorization.md new file mode 100644 index 00000000000..b66fde8cb92 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/price-authorization.md @@ -0,0 +1,3 @@ +## Price authorization + +The server allows only administrators to change catalog prices. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/price-history-orders.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/price-history-orders.md new file mode 100644 index 00000000000..1d5f5dd6c46 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/price-history-orders.md @@ -0,0 +1,8 @@ +## Price history: completed orders + +Completed orders keep the price paid at checkout. A later price change does not alter a receipt +or revenue already recorded. + +## Price history: cart and checkout + +Every open cart shows the new price without a reload. Checkout uses the current price. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/price-history.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/price-history.md new file mode 100644 index 00000000000..4e0c7e0517f --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/price-history.md @@ -0,0 +1,3 @@ +## Price history + +An administrator can change an item's price. Show the price in the public catalog. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/product-bundles.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/product-bundles.md new file mode 100644 index 00000000000..a547edd0619 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/product-bundles.md @@ -0,0 +1,5 @@ +## Product bundles + +Catalog staff can create and edit a named bundle of existing products, with a quantity +for each component and one bundle price. Show bundles and their components in the catalog. +Bundles contain products only, not other bundles. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/production-specifications.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/production-specifications.md new file mode 100644 index 00000000000..6a19d23bdb7 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/production-specifications.md @@ -0,0 +1,43 @@ +## automatic-reorder-access + +Only warehouse staff can manage automatic reorder rules. + +## recommendation-profile-isolation + +Customer recommendation profiles are isolated. + +## staff-activity-privacy + +Customers cannot open staff activity history. + +## order-support-ownership + +A customer cannot attach or inspect another customer's order. + +## delivery-notification-privacy + +Delivery notifications are private to the order owner. + +## support-refund-access + +Only authorized staff can refund an order from a support case. + +## recommendation-feedback-privacy + +Recommendation feedback belongs to one customer. + +## recommendation-feedback-restart + +Dismissed recommendations stay dismissed after reconnecting or restarting the server. + +## automatic-reorder-deduplication + +A pending automatic restock is not scheduled twice. + +## payment-deduplication + +A checkout does not create duplicate payments. + +## support-refund-accounting + +The recorded refund equals the amount paid and is applied only once. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/progression-automatic-reorder.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/progression-automatic-reorder.md new file mode 100644 index 00000000000..e31fccd6dd9 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/progression-automatic-reorder.md @@ -0,0 +1,6 @@ +# Automatic reorder rules + +Warehouse staff can manage automatic reorder rules. A rule names an item, a stock +threshold, and a restock quantity. Schedule a restock when stock falls from above the +threshold to the threshold or below. +The automatic restock is due 60 seconds after it is scheduled. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/progression-cart-recovery.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/progression-cart-recovery.md new file mode 100644 index 00000000000..77d03bcbb7d --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/progression-cart-recovery.md @@ -0,0 +1,4 @@ +# Cart recovery + +Let a signed-in customer restore an expired cart. Reserve each item again only when its full +quantity is available. Restore the available items and list each item that could not be restored. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/progression-framing.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/progression-framing.md new file mode 100644 index 00000000000..ad1050c1693 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/progression-framing.md @@ -0,0 +1,10 @@ +## New application + +Build an ecommerce application from the product work below. Use `Storefront` +as the visible page title. Use the application interface names where they are +provided. Start with no orders or purchase history. + +## Existing application + +Add the current product work to the existing ecommerce application. Preserve its data; +do not add sample orders or purchase history while adding features. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/progression-personalized-recommendations.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/progression-personalized-recommendations.md new file mode 100644 index 00000000000..236b1c7e84f --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/progression-personalized-recommendations.md @@ -0,0 +1,4 @@ +# Personalized recommendations + +Recommend items from categories in the signed-in customer's purchase history. Exclude +items they already purchased. Order by global units sold, highest first, then by item name. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/promotion-checkout.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/promotion-checkout.md new file mode 100644 index 00000000000..e3498d90b0c --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/promotion-checkout.md @@ -0,0 +1,4 @@ +## Promotion checkout + +Customers can apply an active promotion code to a cart. The final order records the applied +discount. Expired and fully redeemed promotions are refused. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/promotion-reporting.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/promotion-reporting.md new file mode 100644 index 00000000000..c22c5610e16 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/promotion-reporting.md @@ -0,0 +1,3 @@ +## Promotion reporting + +Staff can see each promotion's redemption count and revenue after discounts. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/promotion-rules.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/promotion-rules.md new file mode 100644 index 00000000000..b76d1333648 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/promotion-rules.md @@ -0,0 +1,4 @@ +## Promotion rules + +Staff can create promotion codes with a percentage discount, a start date, an end date, and a +redemption limit. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/purchasing.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/purchasing.md new file mode 100644 index 00000000000..4c5d624f09a --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/purchasing.md @@ -0,0 +1,5 @@ +## Purchasing and orders + +A signed-in customer can buy one unit of an available item. The purchase reduces stock and +creates an order for that customer at the price paid. Their order history shows the newest +orders first, including items, quantities, prices, and totals. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/recommendation-feedback.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/recommendation-feedback.md new file mode 100644 index 00000000000..25a62b98e6a --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/recommendation-feedback.md @@ -0,0 +1,3 @@ +## Recommendation feedback + +A customer can dismiss a recommendation from their list. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/recommendations.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/recommendations.md new file mode 100644 index 00000000000..287d0e378e3 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/recommendations.md @@ -0,0 +1,4 @@ +## Recommendations + +Show customers a recommended-for-you list on the catalog page. It recommends items from categories the customer has +bought from and excludes items already in the cart. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/reservations.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/reservations.md new file mode 100644 index 00000000000..adfe4098d6a --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/reservations.md @@ -0,0 +1,5 @@ +## Reservations + +Adding an item to a cart reserves its stock for 90 seconds. The cart shows the remaining time. +Checkout consumes a live reservation. An expired reservation releases its stock and remains +visible as expired. Adding the item again renews the reservation. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/review-access.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/review-access.md new file mode 100644 index 00000000000..b0103c3de65 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/review-access.md @@ -0,0 +1,3 @@ +## Review access + +Only a customer who bought an item can review it. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/reviews.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/reviews.md new file mode 100644 index 00000000000..1e026fcfc8e --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/reviews.md @@ -0,0 +1,4 @@ +## Reviews + +Customers can rate items they purchased from one to five and add a comment. Show reviews +and the average rating on the item detail. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/sales-dashboard.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/sales-dashboard.md new file mode 100644 index 00000000000..7d16cac20a3 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/sales-dashboard.md @@ -0,0 +1,4 @@ +## Sales dashboard + +Give customers and administrators sales views. Category totals show units sold and revenue for +each category. Signed-out visitors see best sellers on the storefront. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/scheduled-restocks.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/scheduled-restocks.md new file mode 100644 index 00000000000..c00a2afb659 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/scheduled-restocks.md @@ -0,0 +1,4 @@ +## Scheduled restocks + +An admin can schedule and cancel a restock. Show pending restocks and their remaining +time. Show completed stock movements in a stock ledger. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/search-ordering-specification.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/search-ordering-specification.md new file mode 100644 index 00000000000..5e1ee495977 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/search-ordering-specification.md @@ -0,0 +1,4 @@ +# Search ordering after purchases + +Purchases must not change the alphabetical order of active search or filter results. +Clearing all search text and filters restores the current purchase ranking. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/shipping-authorization.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/shipping-authorization.md new file mode 100644 index 00000000000..cde076db312 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/shipping-authorization.md @@ -0,0 +1,3 @@ +## Shipping authorization + +The server allows only staff and administrators to mark orders as shipped. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/split-tender-refunds.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/split-tender-refunds.md new file mode 100644 index 00000000000..4fa1d02906f --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/split-tender-refunds.md @@ -0,0 +1,3 @@ +## Refunds with store credit + +Extend support refunds and item returns to orders paid with credit and another payment. Return each part to its original source. A full refund restores the original credit portion to the customer balance and refunds the original external portion. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/staff-access.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/staff-access.md new file mode 100644 index 00000000000..eb4f5715325 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/staff-access.md @@ -0,0 +1,3 @@ +## Staff access + +Staff and administrators can sign in and use staff areas. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/staff-activity.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/staff-activity.md new file mode 100644 index 00000000000..fd062bd4d2a --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/staff-activity.md @@ -0,0 +1,4 @@ +## Staff activity history + +Staff can inspect a history of administrative changes. Each entry shows the staff member, +action, subject, and time. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/staff-roles.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/staff-roles.md new file mode 100644 index 00000000000..70a06b8ee4d --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/staff-roles.md @@ -0,0 +1,6 @@ +## Staff roles + +An administrator can assign a role to an existing staff account. The `admin` role grants +administrator access. The `staff` and `inventory` roles grant staff access without +administrator access. +A role change takes effect immediately, including for accounts that are already signed in. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/stock-alerts.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/stock-alerts.md new file mode 100644 index 00000000000..7a3f35808d1 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/stock-alerts.md @@ -0,0 +1,4 @@ +## Stock alerts + +A signed-in customer can request an alert for an unavailable item. Show an alert when stock +returns. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/stock-conservation.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/stock-conservation.md new file mode 100644 index 00000000000..b68aaca3e26 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/stock-conservation.md @@ -0,0 +1,4 @@ +## Stock conservation + +Stock moves between warehouses without changing the total. A refused transfer changes nothing. +When a purchase and a transfer overlap, the final total reflects the sold units exactly once. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/stock-transfers.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/stock-transfers.md new file mode 100644 index 00000000000..fe9b06bef48 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/stock-transfers.md @@ -0,0 +1,3 @@ +## Stock transfers + +An administrator can move units of an item from one warehouse to another. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/store-credit.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/store-credit.md new file mode 100644 index 00000000000..0cbb6393551 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/store-credit.md @@ -0,0 +1,3 @@ +## Store credit + +Staff can issue customer credit with a reference. Customers can see their balance and history and choose credit at cart checkout. Use credit first, up to the order total; record any remainder as the existing payment. Amounts use the shop currency and whole minor units. A reference identifies one grant. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/subscriptions.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/subscriptions.md new file mode 100644 index 00000000000..3b482c83911 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/subscriptions.md @@ -0,0 +1,3 @@ +## Scheduled purchases + +Customers can subscribe to an individual catalog item (not a bundle) and quantity for a chosen number of deliveries at a chosen interval. Use the item price at subscription creation. The first delivery is due after one interval. Each delivery creates an ordinary order and payment. Skip an unavailable delivery without charging; it still uses one delivery slot. Customers can pause, resume, or cancel future deliveries; a pause moves future due times by the pause duration. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/support-history.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/support-history.md new file mode 100644 index 00000000000..a578a2fda30 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/support-history.md @@ -0,0 +1,3 @@ +## Customer support history + +Signed-in customers can see their support ticket history. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/support-intake.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/support-intake.md new file mode 100644 index 00000000000..245e7052d9c --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/support-intake.md @@ -0,0 +1,4 @@ +## Support intake + +Anyone can open a support ticket with contact details, a subject, and a message. Return a +reference that the visitor can use to identify the ticket. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/support-refunds.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/support-refunds.md new file mode 100644 index 00000000000..553307f56f9 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/support-refunds.md @@ -0,0 +1,7 @@ +## Support refunds + +Staff can refund an entire order from its support case. A successful refund resolves +the case. Show the refund amount and case status. + +A refund does not prevent a later physical return of shipped goods. Refund only +the amount not already refunded, and restock goods when they are returned. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/support-triage.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/support-triage.md new file mode 100644 index 00000000000..32767f66133 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/support-triage.md @@ -0,0 +1,3 @@ +## Support triage + +Staff can view new support tickets, assign a ticket, set its priority, and change its status. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/transfer-authorization.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/transfer-authorization.md new file mode 100644 index 00000000000..897342d96c0 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/transfer-authorization.md @@ -0,0 +1,3 @@ +## Transfer authorization + +The server allows only staff and administrators to transfer warehouse stock. diff --git a/tools/stack-bench/tracks/ecommerce/prompts/modular/warehouse-administration.md b/tools/stack-bench/tracks/ecommerce/prompts/modular/warehouse-administration.md new file mode 100644 index 00000000000..e89ba5c9440 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/prompts/modular/warehouse-administration.md @@ -0,0 +1,5 @@ +## Warehouse administration + +Provide an administration area that lists every item, both warehouses, and the quantity held in +each warehouse. An administrator can add units to a selected item and warehouse. The area also +shows total revenue across orders. diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-account-create.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-account-create.json new file mode 100644 index 00000000000..c283418861a --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-account-create.json @@ -0,0 +1,28 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Account creation", + "note": "Account creation is exercised inside its own scored check so a broken signup cannot invalidate unrelated account checks during setup.", + "features": [ + { + "id": 1, + "name": "Accounts", + "actors": ["shopper"], + "setup": [], + "criteria": [ + { + "id": "1a", + "category": "feature", + "desc": "a visitor can create an account and sign in as it", + "points": 1, + "steps": [ + {"do":"signUp","actor":"shopper","name":"ann","awaitSignedIn":false}, + {"do":"expect","actor":"shopper","testid":"current-user","contains":"ann","within":10000} + ], + "statedBy": "a visitor can create an account with a username and password" + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-account-disclosure.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-account-disclosure.json new file mode 100644 index 00000000000..9b9ac6c847c --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-account-disclosure.json @@ -0,0 +1,93 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Account password disclosure", + "note": "Draft: create and sign in with a real password, then inspect data delivered to another account and a fresh signed-out page. This bounded sentinel checks plaintext disclosure in captured HTTP data/HTML/JavaScript/CSS (including error bodies), WebSocket and EventSource messages. It does not prove password hashing, absence of encoded secrets, logs, unfetched resources, or all endpoints. Missing/incomplete transport evidence is not a privacy pass.", + "features": [ + { + "id": 110, + "name": "Account password disclosure", + "actors": [ + "owner", + "returning", + "other", + "guest" + ], + "setup": [ + { + "do": "signUp", + "actor": "owner", + "name": "private-password", + "password": "sb-private-pw-76dc492ea153" + }, + { + "do": "signUp", + "actor": "other", + "name": "different-password" + } + ], + "criteria": [ + { + "id": "110a", + "category": "production", + "desc": "an account password is not delivered to another account or a signed-out visitor", + "points": 1, + "steps": [ + { + "do": "signIn", + "actor": "returning", + "name": "private-password", + "password": "sb-private-pw-76dc492ea153", + "awaitSignedIn": false + }, + { + "do": "expect", + "actor": "returning", + "testid": "current-user", + "contains": "private-password", + "within": 10000 + }, + { + "do": "reload", + "actor": "other", + "application": true, + "settleMs": 1000 + }, + { + "do": "expect", + "actor": "other", + "testid": "current-user", + "contains": "different-password" + }, + { + "do": "reload", + "actor": "guest", + "application": true, + "settleMs": 1000 + }, + { + "do": "expect", + "actor": "guest", + "testid": "current-user", + "absent": true + }, + { + "do": "expectNotReceived", + "actor": "other", + "contains": "sb-private-pw-76dc492ea153", + "within": 1000 + }, + { + "do": "expectNotReceived", + "actor": "guest", + "contains": "sb-private-pw-76dc492ea153", + "within": 1000 + } + ], + "statedBy": "account credentials authorize their owner and are not public application data" + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-account-duplicate.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-account-duplicate.json new file mode 100644 index 00000000000..6d34394d2bb --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-account-duplicate.json @@ -0,0 +1,29 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Duplicate account refusal", + "features": [ + { + "id": 1, + "name": "Accounts", + "actors": ["owner","impostor"], + "setup": [{"do":"signUp","actor":"owner","name":"ann"}], + "criteria": [ + { + "id": "1b", + "category": "production", + "desc": "a taken username is refused and does not sign the visitor in as the existing account", + "points": 1, + "steps": [ + {"do":"signUp","actor":"impostor","name":"ann","password":"different-pw","expectFailure":true}, + {"do":"expect","actor":"impostor","testid":"auth-error","within":6000}, + {"do":"reload","actor":"impostor","application":true,"settleMs":1000}, + {"do":"expect","actor":"impostor","testid":"current-user","absent":true} + ], + "statedBy": "signing up with a taken username fails with a visible error and must never sign the visitor in as the existing account" + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-account-password.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-account-password.json new file mode 100644 index 00000000000..d863fc63dde --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-account-password.json @@ -0,0 +1,93 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Password refusal", + "features": [ + { + "id": 1, + "name": "Accounts", + "actors": [ + "ordinary", + "ordinary-returning", + "ordinary-impostor", + "owner", + "returning", + "impostor" + ], + "setup": [], + "criteria": [ + { + "id": "1c", + "category": "production", + "desc": "the complete accepted password is required to sign in", + "points": 1, + "steps": [ + { + "do": "signUp", + "actor": "ordinary", + "name": "password-ordinary", + "password": "ordinary-password-A" + }, + { + "do": "expect", + "actor": "ordinary", + "testid": "current-user", + "contains": "password-ordinary", + "within": 6000 + }, + { + "do": "signIn", + "actor": "ordinary-returning", + "name": "password-ordinary", + "password": "ordinary-password-A", + "awaitSignedIn": false + }, + { + "do": "expect", + "actor": "ordinary-returning", + "testid": "current-user", + "contains": "password-ordinary", + "within": 10000 + }, + { + "do": "signIn", + "actor": "ordinary-impostor", + "name": "password-ordinary", + "password": "ordinary-password-B", + "expectFailure": true + }, + { + "do": "expect", + "actor": "ordinary-impostor", + "testid": "auth-error", + "within": 6000 + }, + { + "do": "reload", + "actor": "ordinary-impostor", + "application": true, + "settleMs": 1000 + }, + { + "do": "expect", + "actor": "ordinary-impostor", + "testid": "current-user", + "absent": true + }, + { + "do": "probePasswordBoundary", + "actor": "owner", + "returningActor": "returning", + "impostorActor": "impostor", + "name": "password-boundary", + "password": "界界界界界界界界界界界界界界界界界界界界界界界界-A", + "wrongPassword": "界界界界界界界界界界界界界界界界界界界界界界界界-B" + } + ], + "statedBy": "signing in with a wrong password fails with a visible error" + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-account-reload.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-account-reload.json new file mode 100644 index 00000000000..27fb17db3d3 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-account-reload.json @@ -0,0 +1,27 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Session reload", + "features": [ + { + "id": 1, + "name": "Accounts", + "actors": ["shopper"], + "setup": [{"do":"signUp","actor":"shopper","name":"ann"}], + "criteria": [ + { + "id": "1e", + "category": "production", + "desc": "the session survives a reload", + "points": 1, + "steps": [ + {"do":"reload","actor":"shopper","settleMs":4000}, + {"do":"expect","actor":"shopper","testid":"current-user","contains":"ann","within":6000} + ], + "statedBy": "a signed-in session persists across a page reload" + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-account-signout.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-account-signout.json new file mode 100644 index 00000000000..74f38869f25 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-account-signout.json @@ -0,0 +1,76 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Sign out and return", + "features": [ + { + "id": 1, + "name": "Accounts", + "actors": [ + "shopper" + ], + "setup": [ + { + "do": "signUp", + "actor": "shopper", + "name": "ann" + } + ], + "criteria": [ + { + "id": "1d", + "category": "feature", + "desc": "signing out and back in returns the same account", + "points": 1, + "steps": [ + { "do": "click", "actor": "shopper", "testid": "current-user", "unlessVisible": "signout" }, + { + "do": "click", + "actor": "shopper", + "testid": "signout" + }, + { + "do": "waitUntilAbsent", + "actor": "shopper", + "testid": "current-user", + "within": 6000 + }, + { + "do": "expect", + "actor": "shopper", + "testid": "current-user", + "absent": true + }, + { + "do": "reload", + "actor": "shopper", + "application": true, + "settleMs": 1000 + }, + { + "do": "expect", + "actor": "shopper", + "testid": "current-user", + "absent": true + }, + { + "do": "signIn", + "actor": "shopper", + "name": "ann", + "awaitSignedIn": false + }, + { + "do": "expect", + "actor": "shopper", + "testid": "current-user", + "contains": "ann", + "within": 10000 + } + ], + "statedBy": "a signed-in user can sign out, returning to the signed-out state" + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-admin-write-staff.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-admin-write-staff.json new file mode 100644 index 00000000000..56b9b9c0e36 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-admin-write-staff.json @@ -0,0 +1,75 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Only an administrator can restock through staff access", + "note": "Draft: staff cannot gain restock authority by submitting role=admin. The same real write receives an extra body field on HTTP/Convex or an extra positional reducer argument. Native schema rejection is valid; it does not prove an application role branch ran. Stored stock must remain unchanged, including after a refusal response. Normal administrator writes must work before and after the probe. This bounded claim does not cover arbitrary authority fields or query injection. Signup submits role=admin separately to each observed registration write, using a fresh account and browser session for each write. Rejection or an ordinary account is valid; it must not grant stock-write authority. A normal signup must still work. Missing, changed, or incomplete registration-write capture is inconclusive. This does not cover query injection.", + "features": [ + { + "id": 103, + "name": "Only an administrator can restock", + "actors": ["admin", "staff", "claimant"], + "setup": [ + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + { "do": "signIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true }, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + { "do": "click", "actor": "admin", "testid": "admin-link", "unlessVisible": "admin-panel" }, + { "do": "click", "actor": "staff", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "103a", + "category": "feature", + "desc": "an administrator can restock a warehouse", + "points": 1, + "statedBy": "An administrator can add units to a selected item and warehouse.", + "steps": [ + { "do": "recordNumber", "actor": "staff", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Air Purifier" }, "as": "purifier-before-control" }, + { "do": "callAction", "actor": "admin", "action": "restock", "namedAction": { "id": "restock", "path": "/api/admin/restock", "reducer": "admin_restock", "args": [0, 0, 1], "params": [{ "name": "itemId", "in": "body", "wireType": "u64" }, { "name": "warehouseId", "in": "body", "wireType": "u64" }, { "name": "quantity", "in": "body" }] }, "input": { "testid": "admin-location-row", "contains": "Air Purifier", "attribute": "data-restock-input" }, "settleMs": 2000 }, + { "do": "expectActionOutcome", "actor": "admin", "outcome": "accepted" }, + { "do": "reload", "actor": "staff", "settleMs": 2000 }, + { "do": "ensureSignedIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true, "readyTestid": "current-user" }, + { "do": "click", "actor": "staff", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "expectNumber", "actor": "staff", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Air Purifier" }, "relativeTo": "purifier-before-control", "plus": 1 } + ] + }, + { + "id": "103b", + "category": "production", + "desc": "staff or client-supplied signup claims cannot grant administrator stock-write authority", + "points": 2, + "statedBy": "Staff and signup claims cannot gain administrator restock authority.", + "note": "The ordinary signup can establish a session automatically or through a separate sign-in. The next assertion requires the created account. A patched signup refusal must not be followed by a sign-in.", + "steps": [ + { "do": "callAction", "actor": "admin", "action": "restock", "namedAction": { "id": "restock", "path": "/api/admin/restock", "reducer": "admin_restock", "args": [0, 0, 1], "params": [{ "name": "itemId", "in": "body", "wireType": "u64" }, { "name": "warehouseId", "in": "body", "wireType": "u64" }, { "name": "quantity", "in": "body" }] }, "input": { "testid": "admin-location-row", "contains": "Air Purifier", "attribute": "data-restock-input" }, "settleMs": 2000 }, + { "do": "expectActionOutcome", "actor": "admin", "outcome": "accepted" }, + { "do": "reload", "actor": "staff", "settleMs": 2000 }, + { "do": "ensureSignedIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true, "readyTestid": "current-user" }, + { "do": "click", "actor": "staff", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "recordNumber", "actor": "staff", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Air Purifier" }, "as": "purifier-before-refusal" }, + { "do": "callAction", "actor": "staff", "from": "admin", "action": "restock", "namedAction": { "id": "restock", "path": "/api/admin/restock", "reducer": "admin_restock", "args": [0, 0, 1], "params": [{ "name": "itemId", "in": "body", "wireType": "u64" }, { "name": "warehouseId", "in": "body", "wireType": "u64" }, { "name": "quantity", "in": "body" }] }, "input": { "testid": "admin-location-row", "contains": "Air Purifier", "attribute": "data-restock-input" }, "settleMs": 2000 }, + { "do": "expectActionOutcome", "actor": "staff", "outcome": "refused", "routeProvenBy": "admin" }, + { "do": "reload", "actor": "staff", "settleMs": 2000 }, + { "do": "ensureSignedIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true, "readyTestid": "current-user" }, + { "do": "click", "actor": "staff", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "expectNumber", "actor": "staff", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Air Purifier" }, "relativeTo": "purifier-before-refusal", "plus": 0 }, + { "do": "dbRecordStock", "item": "Air Purifier", "as": "before-admin-claim" }, + { "do": "callAction", "actor": "staff", "from": "admin", "action": "restock", "namedAction": { "id": "restock", "path": "/api/admin/restock", "reducer": "admin_restock", "args": [0, 0, 1, "admin"], "params": [{ "name": "itemId", "in": "body", "wireType": "u64" }, { "name": "warehouseId", "in": "body", "wireType": "u64" }, { "name": "quantity", "in": "body" }, { "name": "role", "in": "body" }] }, "input": { "testid": "admin-location-row", "contains": "Air Purifier", "attribute": "data-restock-input", "fields": ["itemId", "warehouseId", "quantity"] }, "settleMs": 1000 }, + { "do": "expectActionOutcome", "actor": "staff", "outcome": "completed", "routeProvenBy": "admin" }, + { "do": "dbExpectStock", "item": "Air Purifier", "relativeTo": "before-admin-claim", "plus": 0 }, + { "do": "expectActionOutcome", "actor": "staff", "outcome": "application-refused", "routeProvenBy": "admin" }, + { "do": "probeSignupClaims", "actor": "claimant", "name": "claimant", "fields": { "role": "admin" }, "branches": [[ + { "do": "callAction", "actor": "claimant", "action": "restock", "namedAction": { "id": "restock", "path": "/api/admin/restock", "reducer": "admin_restock", "args": [0, 0, 1], "params": [{ "name": "itemId", "in": "body", "wireType": "u64" }, { "name": "warehouseId", "in": "body", "wireType": "u64" }, { "name": "quantity", "in": "body" }] }, "input": { "testid": "admin-location-row", "contains": "Air Purifier", "attribute": "data-restock-input" }, "settleMs": 1000, "from": "admin", "authentication": "optional" }, + { "do": "expectActionOutcome", "actor": "claimant", "outcome": "completed", "routeProvenBy": "admin" }, + { "do": "dbExpectStock", "item": "Air Purifier", "relativeTo": "before-admin-claim", "plus": 0 }, + { "do": "expectActionOutcome", "actor": "claimant", "outcome": "application-refused", "routeProvenBy": "admin" } + ]] }, + { "do": "callAction", "actor": "admin", "action": "restock", "namedAction": { "id": "restock", "path": "/api/admin/restock", "reducer": "admin_restock", "args": [0, 0, 1], "params": [{ "name": "itemId", "in": "body", "wireType": "u64" }, { "name": "warehouseId", "in": "body", "wireType": "u64" }, { "name": "quantity", "in": "body" }] }, "input": { "testid": "admin-location-row", "contains": "Air Purifier", "attribute": "data-restock-input" }, "settleMs": 1000 }, + { "do": "expectActionOutcome", "actor": "admin", "outcome": "accepted" }, + { "do": "dbExpectStock", "item": "Air Purifier", "relativeTo": "before-admin-claim", "plus": 1 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-buying.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-buying.json new file mode 100644 index 00000000000..f75f5a2d748 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-buying.json @@ -0,0 +1,82 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Purchasing and orders", + "note": "Each purchase check owns its state. Checks inspect durable results after reload instead of requiring panels to close through a particular keyboard interaction.", + "features": [ + { + "id": 3, + "name": "Buying", + "actors": ["buyer", "watcher", "visitor"], + "setup": [ + { "do": "signUp", "actor": "buyer", "name": "eli" }, + { "do": "signUp", "actor": "watcher", "name": "fay" }, + { "do": "click", "actor": "watcher", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "click", "actor": "buyer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "click", "actor": "visitor", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "3a", + "desc": "a signed-out visitor cannot use purchasing controls", + "points": 1, + "steps": [ + { "do": "expectUnavailable", "actor": "visitor", "testid": "buy-now" }, + { "do": "expectUnavailable", "actor": "visitor", "testid": "add-to-cart" } + ], + "statedBy": "buy and add-to-cart actions are unavailable while signed out" + }, + { + "id": "3b", + "category": "production", + "desc": "buying reduces the stock every other client sees, without a reload", + "points": 2, + "steps": [ + { "do": "expectNumber", "actor": "watcher", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Espresso Machine" }, "equals": 100 }, + { "do": "expectNumber", "actor": "visitor", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Espresso Machine" }, "equals": 100 }, + { "do": "click", "actor": "buyer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Espresso Machine" } }, + { "do": "expectNumber", "actor": "watcher", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Espresso Machine" }, "equals": 99, "within": 10000 }, + { "do": "expectNumber", "actor": "visitor", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Espresso Machine" }, "equals": 99, "within": 10000 } + ], + "statedBy": "buying an item reduces its stock by one for everyone" + }, + { + "id": "3c", + "desc": "the purchase is recorded in the buyer's order history at the price paid", + "points": 1, + "steps": [ + { "do": "fill", "actor": "buyer", "testid": "search-input", "text": "Coffee Grinder", "enter": true, "settleMs": 1500 }, + { "do": "click", "actor": "buyer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Coffee Grinder" } }, + {"do":"click","actor":"buyer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { "do": "click", "actor": "buyer", "testid": "orders-toggle" }, + { "do": "expect", "actor": "buyer", "testid": "order-item", "contains": "Coffee Grinder" }, + { "do": "expectNumber", "actor": "buyer", "testid": "order-total", "in": { "testid": "order-item", "contains": "Coffee Grinder" }, "equals": 64 } + ], + "statedBy": "it creates an order for the buyer, visible in their order history" + }, + { + "id": "3d", + "desc": "the last unit sells and the next purchase is refused without changing stock", + "points": 2, + "steps": [ + { "do": "fill", "actor": "buyer", "testid": "search-input", "text": "Mirrorless Camera", "enter": true, "settleMs": 1500 }, + { "do": "click", "actor": "buyer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Mirrorless Camera" } }, + { "do": "wait", "actor": "buyer", "ms": 800 }, + { "do": "click", "actor": "buyer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Mirrorless Camera" } }, + { "do": "wait", "actor": "buyer", "ms": 800 }, + { "do": "click", "actor": "buyer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Mirrorless Camera" } }, + { "do": "expectNumber", "actor": "buyer", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Mirrorless Camera" }, "equals": 0 }, + { "do": "expect", "actor": "buyer", "testid": "out-of-stock", "in": { "testid": "item-card", "contains": "Mirrorless Camera" } }, + { "do": "callAction", "actor": "buyer", "action": "buy", "namedAction": { "id": "buy", "path": "/api/items/:id/buy", "reducer": "buy_now", "args": [0], "params": [{ "name": "itemId", "in": "path", "placeholder": ":id", "wireType": "u64" }] }, "input": { "testid": "item-card", "contains": "Mirrorless Camera", "attribute": "data-buy-input" }, "settleMs": 2000 }, + { "do": "expectActionOutcome", "actor": "buyer", "outcome": "validation-refused" }, + { "do": "reload", "actor": "buyer", "settleMs": 3000 }, + { "do": "ensureSignedIn", "actor": "buyer", "name": "eli", "readyTestid": "current-user" }, + { "do": "expectNumber", "actor": "buyer", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Mirrorless Camera" }, "equals": 0 } + ], + "statedBy": "an item with zero stock cannot be bought" + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-cart-boundary.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-cart-boundary.json new file mode 100644 index 00000000000..a3750d7415d --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-cart-boundary.json @@ -0,0 +1,84 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "A cart is nobody else's business", + "writeUrlPattern": "\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses", + "note": "Every criterion receives all state it relies on from its feature setup. Security assertions include a working positive path where an empty or broken view could otherwise look secure.", + "features": [ + { + "id": 109, + "name": "A cart is nobody else's business", + "actors": ["owner", "stranger"], + "setup": [ + { "do": "signUp", "actor": "owner", "name": "vic" }, + { "do": "signUp", "actor": "stranger", "name": "wes" }, + { "do": "click", "actor": "owner", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "click", "actor": "owner", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Coffee Grinder" } }, + { "do": "wait", "actor": "owner", "ms": 1500 }, + {"do": "click", "actor": "owner", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["cart-toggle", "cart-total"]}, + { "do": "click", "actor": "owner", "testid": "cart-toggle", "unlessVisible": "cart-total" }, + { "do": "click", "actor": "stranger", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "109a", + "category": "production", + "desc": "the same cart action run by another customer changes only that customer's cart", + "points": 2, + "steps": [ + { "do": "callAction", "actor": "stranger", "action": "cart-add", "namedAction": { "id": "cart-add", "path": "/api/cart", "reducer": "add_to_cart", "args": [0], "params": [{ "name": "itemId", "in": "body", "wireType": "u64" }] }, "input": { "testid": "item-card", "contains": "Coffee Grinder", "attribute": "data-buy-input" }, "settleMs": 2000 }, + { "do": "expectActionOutcome", "actor": "stranger", "outcome": "accepted" }, + { "do": "reload", "actor": "owner", "settleMs": 1500 }, + { "do": "ensureSignedIn", "actor": "owner", "name": "vic", "readyTestid": "current-user" }, + { "do": "click", "actor": "owner", "testid": "overlay-close", "ifAvailable": true }, + { "do": "click", "actor": "owner", "testid": "catalog-link", "ifAvailable": true }, + {"do": "click", "actor": "owner", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["cart-toggle", "cart-total"]}, + { "do": "click", "actor": "owner", "testid": "cart-toggle", "unlessVisible": "cart-total" }, + { "do": "reload", "actor": "stranger", "settleMs": 1500 }, + { "do": "ensureSignedIn", "actor": "stranger", "name": "wes", "readyTestid": "current-user" }, + { "do": "click", "actor": "stranger", "testid": "overlay-close", "ifAvailable": true }, + { "do": "click", "actor": "stranger", "testid": "catalog-link", "ifAvailable": true }, + { "do": "expect", "actor": "owner", "testid": "cart-item", "contains": "Coffee Grinder", "count": 1 }, + { "do": "expectNumber", "actor": "owner", "testid": "cart-quantity", "in": { "testid": "cart-item", "contains": "Coffee Grinder" }, "equals": 1 }, + {"do": "click", "actor": "stranger", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["cart-toggle", "cart-total"]}, + { "do": "click", "actor": "stranger", "testid": "cart-toggle", "unlessVisible": "cart-total" }, + { "do": "expect", "actor": "stranger", "testid": "cart-item", "contains": "Coffee Grinder", "count": 1 }, + { "do": "expectNumber", "actor": "stranger", "testid": "cart-quantity", "in": { "testid": "cart-item", "contains": "Coffee Grinder" }, "equals": 1 } + ], + "statedBy": "one customer cannot read or change another customer's cart" + }, + { + "id": "109b", + "category": "production", + "desc": "a negative quantity is refused and leaves the cart unchanged", + "points": 1, + "steps": [ + { "do": "callAction", "actor": "owner", "action": "cart-set-quantity", "namedAction": { "id": "cart-set-quantity", "path": "/api/cart/:itemId", "method": "PATCH", "reducer": "update_cart_quantity", "args": [0, 2], "params": [{ "name": "itemId", "in": "path", "placeholder": ":itemId", "wireType": "u64" }, { "name": "quantity", "in": "body" }] }, "input": { "testid": "cart-item", "contains": "Coffee Grinder", "attribute": "data-cart-input", "fields": ["itemId"] }, "settleMs": 2000 }, + { "do": "expectActionOutcome", "actor": "owner", "outcome": "accepted" }, + { "do": "reload", "actor": "owner", "settleMs": 1500 }, + { "do": "ensureSignedIn", "actor": "owner", "name": "vic", "readyTestid": "current-user" }, + { "do": "click", "actor": "owner", "testid": "overlay-close", "ifAvailable": true }, + { "do": "click", "actor": "owner", "testid": "catalog-link", "ifAvailable": true }, + {"do": "click", "actor": "owner", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["cart-toggle", "cart-total"]}, + { "do": "click", "actor": "owner", "testid": "cart-toggle", "unlessVisible": "cart-total" }, + { "do": "expectNumber", "actor": "owner", "testid": "cart-quantity", "in": { "testid": "cart-item", "contains": "Coffee Grinder" }, "equals": 2 }, + { "do": "recordNumber", "actor": "owner", "testid": "cart-total", "as": "cart-total-before-invalid" }, + { "do": "recordNumber", "actor": "owner", "testid": "cart-quantity", "in": { "testid": "cart-item", "contains": "Coffee Grinder" }, "as": "cart-quantity-before-invalid" }, + { "do": "callAction", "actor": "owner", "action": "cart-set-quantity", "namedAction": { "id": "cart-set-quantity", "path": "/api/cart/:itemId", "method": "PATCH", "reducer": "update_cart_quantity", "args": [0, -3], "params": [{ "name": "itemId", "in": "path", "placeholder": ":itemId", "wireType": "u64" }, { "name": "quantity", "in": "body" }] }, "input": { "testid": "cart-item", "contains": "Coffee Grinder", "attribute": "data-cart-input", "fields": ["itemId"] }, "settleMs": 2000 }, + { "do": "expectActionOutcome", "actor": "owner", "outcome": "validation-refused" }, + { "do": "reload", "actor": "owner", "settleMs": 1500 }, + { "do": "ensureSignedIn", "actor": "owner", "name": "vic", "readyTestid": "current-user" }, + { "do": "click", "actor": "owner", "testid": "overlay-close", "ifAvailable": true }, + { "do": "click", "actor": "owner", "testid": "catalog-link", "ifAvailable": true }, + {"do": "click", "actor": "owner", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["cart-toggle", "cart-total"]}, + { "do": "click", "actor": "owner", "testid": "cart-toggle", "unlessVisible": "cart-total" }, + { "do": "expectNumber", "actor": "owner", "testid": "cart-total", "relativeTo": "cart-total-before-invalid", "plus": 0 }, + { "do": "expectNumber", "actor": "owner", "testid": "cart-quantity", "in": { "testid": "cart-item", "contains": "Coffee Grinder" }, "relativeTo": "cart-quantity-before-invalid", "plus": 0 } + ], + "statedBy": "a request carrying a negative quantity is refused and changes nothing" + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-cart.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-cart.json new file mode 100644 index 00000000000..b38c2e0786a --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-cart.json @@ -0,0 +1,92 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Cart and checkout", + "note": "Each check owns its account and cart state. Checkout is verified after reload through persistent cart count, stock, and order state, without requiring a particular panel lifecycle or Escape-key handler.", + "features": [ + { + "id": 4, + "name": "Cart belongs to the account", + "actors": ["quantity", "reload", "live1", "live2", "checkout"], + "setup": [ + { "do": "signUp", "actor": "quantity", "name": "nora" }, + { "do": "signUp", "actor": "reload", "name": "omar" }, + { "do": "signUp", "actor": "live1", "name": "pia" }, + { "do": "signUp", "actor": "checkout", "name": "quinn" }, + { "do": "click", "actor": "reload", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "click", "actor": "live1", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "4a", + "desc": "adding the same item twice raises its quantity instead of adding a second line", + "points": 1, + "steps": [ + { "do": "click", "actor": "quantity", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Headphones" } }, + { "do": "wait", "actor": "quantity", "ms": 800 }, + { "do": "click", "actor": "quantity", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Headphones" } }, + { "do": "click", "actor": "quantity", "testid": "cart-toggle" }, + { "do": "expect", "actor": "quantity", "testid": "cart-item", "contains": "Headphones", "count": 1 }, + { "do": "expectNumber", "actor": "quantity", "testid": "cart-quantity", "in": { "testid": "cart-item", "contains": "Headphones" }, "equals": 2 } + ], + "statedBy": "adding an item that is already in the cart raises its quantity" + }, + { + "id": "4b", + "category": "production", + "desc": "the cart survives a reload", + "points": 1, + "steps": [ + { "do": "click", "actor": "reload", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Laptop Stand" } }, + {"do": "click", "actor": "reload", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["cart-toggle", "cart-total"]}, + { "do": "click", "actor": "reload", "testid": "cart-toggle", "unlessVisible": "cart-total" }, + { "do": "expect", "actor": "reload", "testid": "cart-item", "contains": "Laptop Stand", "within": 10000 }, + { "do": "reload", "actor": "reload", "settleMs": 3000 }, + { "do": "ensureSignedIn", "actor": "reload", "name": "omar", "readyTestid": "current-user" }, + { "do": "click", "actor": "reload", "testid": "overlay-close", "ifAvailable": true, "within": 1000 }, + { "do": "click", "actor": "reload", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "reload", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["cart-toggle", "cart-total"]}, + { "do": "click", "actor": "reload", "testid": "cart-toggle", "unlessVisible": "cart-total" }, + { "do": "expect", "actor": "reload", "testid": "cart-item", "contains": "Laptop Stand" } + ], + "statedBy": "the cart survives a reload" + }, + { + "id": "4c", + "category": "production", + "desc": "the same account signed in elsewhere sees one cart, live", + "points": 2, + "steps": [ + { "do": "signIn", "actor": "live2", "name": "pia" }, + { "do": "click", "actor": "live2", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "live2", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["cart-toggle", "cart-total"]}, + { "do": "click", "actor": "live2", "testid": "cart-toggle", "unlessVisible": "cart-total" }, + { "do": "click", "actor": "live1", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Induction Cooktop" } }, + { "do": "expect", "actor": "live2", "testid": "cart-item", "contains": "Induction Cooktop", "within": 10000 } + ], + "statedBy": "the same account signed in twice sees one cart, and a change made in one place appears in the other without a reload" + }, + { + "id": "4d", + "desc": "checkout creates one order, reduces stock and empties the cart", + "points": 2, + "steps": [ + { "do": "expectNumber", "actor": "checkout", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Headphones" }, "equals": 100 }, + { "do": "click", "actor": "checkout", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Headphones" } }, + { "do": "expectNumber", "actor": "checkout", "testid": "cart-count", "equals": 1 }, + { "do": "click", "actor": "checkout", "testid": "cart-toggle" }, + { "do": "click", "actor": "checkout", "testid": "checkout-submit" }, + { "do": "wait", "actor": "checkout", "ms": 1500 }, + { "do": "expectNumber", "actor": "checkout", "testid": "cart-count", "equals": 0 }, + { "do": "expectNumber", "actor": "checkout", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Headphones" }, "equals": 99 }, + {"do":"click","actor":"checkout","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { "do": "click", "actor": "checkout", "testid": "orders-toggle" }, + { "do": "expect", "actor": "checkout", "testid": "order-item", "contains": "Headphones", "count": 1 } + ], + "statedBy": "checkout turns the cart into one order, reduces stock for every line, and empties the cart" + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-catalog-ranking.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-catalog-ranking.json new file mode 100644 index 00000000000..a7c77e909ee --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-catalog-ranking.json @@ -0,0 +1,46 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Ecommerce L1 catalog ranking", + "note": "This scenario owns only the initial storefront ranking check.", + "features": [ + { + "id": 2, + "name": "Public catalog ranking", + "actors": ["visitor"], + "setup": [ + { "do": "click", "actor": "visitor", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "2b", + "category": "feature", + "desc": "the storefront shows the exact alphabetical top ten before any purchase", + "points": 1, + "statedBy": "Show the ten most-purchased items and break ties by item name", + "steps": [ + { + "do": "expectSequence", + "actor": "visitor", + "testid": "item-name", + "in": { "testid": "item-list" }, + "equals": [ + "Air Purifier", + "Bluetooth Speaker", + "Coffee Grinder", + "Desk Lamp", + "Espresso Machine", + "Gaming Mouse", + "Headphones", + "Induction Cooktop", + "Keyboard", + "Laptop Stand" + ] + } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-catalog-search.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-catalog-search.json new file mode 100644 index 00000000000..0067853cccc --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-catalog-search.json @@ -0,0 +1,30 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Ecommerce L1 catalog search", + "note": "This scenario owns only the full-catalog search check.", + "features": [ + { + "id": 2, + "name": "Public catalog search", + "actors": ["visitor"], + "setup": [ + { "do": "click", "actor": "visitor", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "search-input" } + ], + "criteria": [ + { + "id": "2d", + "category": "feature", + "desc": "case-insensitive partial search finds an item outside the storefront top ten", + "points": 1, + "statedBy": "Search matches any part of an item name without regard to case across the full catalog", + "steps": [ + { "do": "fill", "actor": "visitor", "testid": "search-input", "text": "mirrorLESS", "enter": true, "settleMs": 1500 }, + { "do": "expect", "actor": "visitor", "testid": "item-card", "contains": "Mirrorless Camera", "in": { "testid": "search-results" } } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-catalog-values.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-catalog-values.json new file mode 100644 index 00000000000..31f88259714 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-catalog-values.json @@ -0,0 +1,31 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Ecommerce L1 catalog values", + "note": "This scenario owns only the public catalog value check.", + "features": [ + { + "id": 2, + "name": "Public catalog values", + "actors": ["visitor"], + "setup": [ + { "do": "click", "actor": "visitor", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "2a", + "category": "feature", + "desc": "a signed-out visitor sees the seeded item name, price, and total stock", + "points": 1, + "statedBy": "Each item shows its name, price, and total stock", + "steps": [ + { "do": "expect", "actor": "visitor", "testid": "item-card", "contains": "Air Purifier" }, + { "do": "expectNumber", "actor": "visitor", "testid": "item-price", "in": { "testid": "item-card", "contains": "Air Purifier" }, "equals": 189 }, + { "do": "expectNumber", "actor": "visitor", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Air Purifier" }, "equals": 100 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-contention.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-contention.json new file mode 100644 index 00000000000..2168e0a231f --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-contention.json @@ -0,0 +1,596 @@ +{ + "schemaVersion": 1, + "level": 1, + "track": "ecommerce", + "name": "Storefront and warehouse — contention (WITHHELD)", + "writeUrlPattern": "\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses", + "note": "Candidate checks score zero until qualification proves the reference passes and the declared mutation fails only its own criterion. Setup failure, inconclusive evidence, a different failed criterion, or collateral regressions do not qualify.", + "features": [ + { + "id": 201, + "name": "The last unit is sold once", + "actors": [ + "a", + "b", + "c", + "d", + "e", + "f" + ], + "setup": [ + { + "do": "signUp", + "actor": "a", + "name": "c1" + }, + { + "do": "signUp", + "actor": "b", + "name": "c2" + }, + { + "do": "signUp", + "actor": "c", + "name": "c3" + }, + { + "do": "signUp", + "actor": "d", + "name": "c4" + }, + { + "do": "signUp", + "actor": "e", + "name": "c5" + }, + { + "do": "signUp", + "actor": "f", + "name": "c6" + }, + { + "do": "fill", + "actor": "a", + "testid": "search-input", + "text": "Mirrorless Camera", + "enter": true, + "settleMs": 1200 + }, + { + "do": "fill", + "actor": "b", + "testid": "search-input", + "text": "Mirrorless Camera", + "enter": true, + "settleMs": 1200 + }, + { + "do": "fill", + "actor": "c", + "testid": "search-input", + "text": "Mirrorless Camera", + "enter": true, + "settleMs": 1200 + }, + { + "do": "fill", + "actor": "d", + "testid": "search-input", + "text": "Mirrorless Camera", + "enter": true, + "settleMs": 1200 + }, + { + "do": "fill", + "actor": "e", + "testid": "search-input", + "text": "Mirrorless Camera", + "enter": true, + "settleMs": 1200 + }, + { + "do": "fill", + "actor": "f", + "testid": "search-input", + "text": "Mirrorless Camera", + "enter": true, + "settleMs": 1200 + } + ], + "criteria": [ + { + "id": "201a", + "desc": "six customers buy the last three units at once: stock lands on exactly zero, never below", + "points": 1, + "steps": [ + { + "do": "clickConcurrently", + "testid": "buy-now", + "actors": [ + "a", + "b", + "c", + "d", + "e", + "f" + ], + "in": { + "testid": "item-card", + "contains": "Mirrorless Camera" + }, + "settleMs": 6000 + }, + { + "do": "expectNumber", + "actor": "a", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Mirrorless Camera" + }, + "equals": 0 + }, + { + "do": "expectAgreement", + "actors": [ + "a", + "b", + "c", + "d", + "e", + "f" + ], + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Mirrorless Camera" + }, + "numeric": true + } + ], + "provenBy": "Candidate backend-specific mutants are declared for this criterion and must pass the live Docker mutation gate before this oracle is called production-validated. PostgreSQL and MongoDB remove an explicit concurrency control; SpacetimeDB omits inventory reservation because reducer transactions make the equivalent interleaving defect inexpressible.", + "statedBy": "stock may never go negative, and two customers must never both get the last unit" + }, + { + "id": "201b", + "desc": "exactly three of the six customers end up with an order, and none is charged twice", + "points": 0, + "steps": [ + { + "do": "clickConcurrently", + "testid": "orders-toggle", + "actors": [ + "a", + "b", + "c", + "d", + "e", + "f" + ], + "settleMs": 2500 + }, + { + "do": "expectActorsWith", + "actors": [ + "a", + "b", + "c", + "d", + "e", + "f" + ], + "testid": "order-item", + "contains": "Mirrorless Camera", + "equals": 3, + "maxEach": 1 + } + ], + "withheld": "rewritten to count the population instead of one participant: the old phrasing asked actor a for at most one order, which passes when nobody got one. Stays at 0 points until it kills a mutant on a real build.", + "statedBy": "two customers must never both get the last unit" + }, + { + "id": "201c", + "desc": "revenue counts three sales, not six", + "points": 0, + "withheld": "needs an admin actor; wire once 201a has demonstrably failed somewhere", + "steps": [], + "statedBy": "stock must never leave a warehouse without a corresponding order" + } + ] + }, + { + "id": 202, + "name": "A restock during a rush is not lost", + "actors": [ + "admin", + "a", + "b", + "c" + ], + "setup": [ + { + "do": "signIn", + "actor": "admin", + "name": "admin", + "password": "stackbench-admin-2026", + "exact": true + }, + { + "do": "signUp", + "actor": "a", + "name": "r1" + }, + { + "do": "signUp", + "actor": "b", + "name": "r2" + }, + { + "do": "signUp", + "actor": "c", + "name": "r3" + } + ], + "criteria": [ + { + "id": "202-control", + "points": 0, + "withheld": "diagnostic: establishes that a restock is applied and reaches the storefront", + "desc": "an uncontended restock of five is stored by the server and shows on the storefront", + "steps": [ + { + "do": "click", + "actor": "admin", + "testid": "admin-link" + }, + { + "do": "recordNumber", + "actor": "admin", + "testid": "admin-location-qty", + "in": { + "testid": "admin-location-row", + "contains": "Bluetooth Speaker" + }, + "as": "warehouse-before" + }, + { + "do": "recordNumber", + "actor": "a", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Bluetooth Speaker" + }, + "as": "storefront-before" + }, + { + "do": "fill", + "actor": "admin", + "testid": "restock-input", + "in": { + "testid": "admin-location-row", + "contains": "Bluetooth Speaker" + }, + "text": "5" + }, + { + "do": "click", + "actor": "admin", + "testid": "restock-submit", + "in": { + "testid": "admin-location-row", + "contains": "Bluetooth Speaker" + } + }, + { + "do": "wait", + "actor": "admin", + "ms": 4000 + }, + { + "do": "expectNumber", + "actor": "admin", + "testid": "admin-location-qty", + "in": { + "testid": "admin-location-row", + "contains": "Bluetooth Speaker" + }, + "relativeTo": "warehouse-before", + "plus": 5, + "within": 8000 + }, + { + "do": "pressKey", + "actor": "a", + "key": "Escape", + "settleMs": 800 + }, + { + "do": "expectNumber", + "actor": "a", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Bluetooth Speaker" + }, + "relativeTo": "storefront-before", + "plus": 5, + "within": 8000 + } + ] + }, + { + "id": "202a", + "points": 0, + "withheld": "A candidate cross-feature mutation now omits purchase inventory reservation and directly makes this read start + 5 instead of start + 2. Keep at zero until that exact target passes the repeated live Docker mutation gate.", + "desc": "three purchases and a restock of five during the rush leave stock at start + 2", + "steps": [ + { + "do": "recordNumber", + "actor": "a", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Bluetooth Speaker" + }, + "as": "rush-before" + }, + { + "do": "clickConcurrently", + "actors": [ + "a", + "b", + "c" + ], + "testid": "buy-now", + "in": { + "testid": "item-card", + "contains": "Bluetooth Speaker" + }, + "settleMs": 500 + }, + { + "do": "fill", + "actor": "admin", + "testid": "restock-input", + "in": { + "testid": "admin-location-row", + "contains": "Bluetooth Speaker" + }, + "text": "5" + }, + { + "do": "click", + "actor": "admin", + "testid": "restock-submit", + "in": { + "testid": "admin-location-row", + "contains": "Bluetooth Speaker" + } + }, + { + "do": "wait", + "actor": "admin", + "ms": 6000 + }, + { + "do": "pressKey", + "actor": "a", + "key": "Escape", + "settleMs": 800 + }, + { + "do": "expectNumber", + "actor": "a", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Bluetooth Speaker" + }, + "relativeTo": "rush-before", + "plus": 2, + "within": 10000 + } + ], + "statedBy": "a restock in one warehouse raises the storefront number live" + } + ], + "note": "The rush assertion only means something if a restock works at all, so the control runs first and has to move the number. Without it a restock that never reached the server and a restock lost to a race look identical: both leave stock at start - purchases. The control also separates the two, by reading what the server stored (admin-location-qty) before what the buyer sees (item-stock). Totals are asserted as CHANGES rather than absolute values, because earlier features buy from the same catalogue." + }, + { + "id": 203, + "name": "One cart, two tabs, one checkout", + "actors": [ + "tab1", + "tab2", + "filler" + ], + "setup": [ + { + "do": "signUp", + "actor": "tab1", + "name": "twin" + }, + { + "do": "signIn", + "actor": "tab2", + "name": "twin" + }, + { + "do": "signIn", + "actor": "filler", + "name": "twin" + }, + { + "do": "fill", + "actor": "tab1", + "testid": "search-input", + "text": "Gaming Mouse", + "enter": true, + "settleMs": 1500 + }, + { + "do": "fill", + "actor": "tab2", + "testid": "search-input", + "text": "Gaming Mouse", + "enter": true, + "settleMs": 1500 + } + ], + "criteria": [ + { + "id": "203a", + "desc": "the same item added from two tabs at once becomes one line of two", + "points": 0, + "withheld": "runs clean (2/2) but no mutant has been attempted against it yet.", + "steps": [ + { + "do": "clickConcurrently", + "testid": "add-to-cart", + "actors": [ + "tab1", + "tab2" + ], + "in": { + "testid": "item-card", + "contains": "Gaming Mouse" + }, + "settleMs": 4000 + }, + { + "do": "click", + "actor": "tab1", + "testid": "cart-toggle" + }, + { + "do": "expect", + "actor": "tab1", + "testid": "cart-item", + "contains": "Gaming Mouse", + "count": 1 + }, + { + "do": "expectNumber", + "actor": "tab1", + "testid": "cart-quantity", + "in": { + "testid": "cart-item", + "contains": "Gaming Mouse" + }, + "equals": 2 + } + ], + "statedBy": "raises its quantity rather than adding a second line" + }, + { + "id": "203b", + "desc": "checking the same cart out twice at once produces one order", + "points": 1, + "steps": [ + { + "do": "click", + "actor": "tab1", + "testid": "checkout-submit" + }, + { + "do": "wait", + "actor": "tab1", + "ms": 2500 + }, + { + "do": "fill", + "actor": "tab2", + "testid": "search-input", + "text": "Webcam", + "enter": true, + "settleMs": 1500 + }, + { + "do": "click", + "actor": "tab2", + "testid": "add-to-cart", + "in": { + "testid": "item-card", + "contains": "Webcam" + } + }, + { + "do": "wait", + "actor": "tab2", + "ms": 1200 + }, + { + "do": "click", + "actor": "tab2", + "testid": "cart-toggle" + }, + { + "do": "click", + "actor": "tab2", + "testid": "checkout-submit" + }, + { + "do": "wait", + "actor": "tab2", + "ms": 2500 + }, + { + "do": "fill", + "actor": "filler", + "testid": "search-input", + "text": "Keyboard", + "enter": true, + "settleMs": 1500 + }, + { + "do": "click", + "actor": "filler", + "testid": "add-to-cart", + "in": { + "testid": "item-card", + "contains": "Keyboard" + } + }, + { + "do": "wait", + "actor": "filler", + "ms": 2000 + }, + { + "do": "callConcurrently", + "actors": [ + "tab1", + "tab2" + ], + "action": "checkout", + "settleMs": 5000 + }, + { + "do": "expectCallOutcomes", + "accepted": 1 + }, + {"do":"click","actor":"filler","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { + "do": "click", + "actor": "filler", + "testid": "orders-toggle" + }, + { + "do": "expect", + "actor": "filler", + "testid": "order-item", + "contains": "Keyboard", + "count": 1 + } + ], + "note": "replayConcurrently replays each actor's LAST POST, so a racing tab must not touch the cart after its checkout is captured — that would overwrite the captured request with an add-to-cart. A third session on the same account fills the cart instead, leaving tab1 and tab2 holding nothing but their checkouts. A checkout carries no JSON body, so it never lands in lastWrites — the replay must select it by URL with the match option, or it silently races the last add-to-cart instead.", + "provenBy": "checkout-not-idempotent (postgres-ecom-l1.json): stripping FOR UPDATE from the cart read at checkout made two replayed checkout requests each create an order — the criterion failed on its own assertion (expected exactly 1 order containing Keyboard, found 2). The unmutated app passes, refusing the second checkout with a 400.", + "withheld": "uses the spec's named `checkout` action so it can be issued on either stack. It previously replayed a captured HTTP write, which SpacetimeDB never produces — that client writes over WebSocket — so the criterion reported INCONCLUSIVE there and the two backends were scored out of different totals. Points retained: it has already failed a mutant on the stacks that could run it.", + "statedBy": "checking out twice must not produce two orders" + } + ], + "note": "Both tabs reach the item by search rather than trusting the storefront: earlier features in this suite buy things, which re-ranks the top ten. Each tab opens its OWN cart — panel state belongs to a browser context, not to the feature." + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-core.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-core.json new file mode 100644 index 00000000000..af0175cb52a --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-core.json @@ -0,0 +1,65 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Catalog", + "note": "Catalog checks assert the exact seeded values and visible ranking order rather than treating mere presence as ordering evidence.", + "features": [ + { + "id": 2, + "name": "Storefront is public and live", + "actors": ["buyer", "visitor", "inspector"], + "setup": [ + { "do": "signUp", "actor": "buyer", "name": "dov" }, + { "do": "click", "actor": "buyer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "click", "actor": "visitor", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "2a", + "desc": "a signed-out visitor sees the catalogue with the seeded prices and stock", + "points": 1, + "steps": [ + { "do": "fill", "actor": "inspector", "testid": "search-input", "text": "Air Purifier", "enter": true, "settleMs": 1500 }, + { "do": "expect", "actor": "inspector", "testid": "item-card", "contains": "Air Purifier", "in": { "testid": "search-results" } }, + { "do": "expectNumber", "actor": "inspector", "testid": "item-price", "in": { "testid": "item-card", "contains": "Air Purifier" }, "equals": 189 }, + { "do": "expectNumber", "actor": "inspector", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Air Purifier" }, "equals": 100 } + ], + "statedBy": "each card shows its name, current price, and total stock" + }, + { + "id": "2b", + "desc": "the storefront shows the exact alphabetical top ten while nothing has been bought", + "points": 1, + "steps": [ + { "do": "expectSequence", "actor": "visitor", "testid": "item-name", "in": { "testid": "item-list" }, "equals": ["Air Purifier", "Bluetooth Speaker", "Coffee Grinder", "Desk Lamp", "Espresso Machine", "Gaming Mouse", "Headphones", "Induction Cooktop", "Keyboard", "Laptop Stand"] } + ], + "statedBy": "the storefront shows the ten most-purchased items, with alphabetical order breaking ties" + }, + { + "id": "2c", + "category": "production", + "desc": "a purchase moves the bought item to the front of the ranking, live", + "points": 1, + "steps": [ + { "do": "expectSequence", "actor": "visitor", "testid": "item-name", "in": { "testid": "item-list" }, "equals": ["Air Purifier", "Bluetooth Speaker", "Coffee Grinder", "Desk Lamp", "Espresso Machine", "Gaming Mouse", "Headphones", "Induction Cooktop", "Keyboard", "Laptop Stand"] }, + { "do": "click", "actor": "buyer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Coffee Grinder" } }, + { "do": "expectSequence", "actor": "visitor", "testid": "item-name", "in": { "testid": "item-list" }, "within": 10000, "equals": ["Coffee Grinder", "Air Purifier", "Bluetooth Speaker", "Desk Lamp", "Espresso Machine", "Gaming Mouse", "Headphones", "Induction Cooktop", "Keyboard", "Laptop Stand"] }, + { "do": "expectAgreement", "actors": ["buyer", "visitor"], "testid": "item-stock", "in": { "testid": "item-card", "contains": "Coffee Grinder" }, "numeric": true } + ], + "statedBy": "a purchase immediately changes the ranking for every open client" + }, + { + "id": "2d", + "desc": "search finds an item that is not on the storefront", + "points": 1, + "steps": [ + { "do": "fill", "actor": "visitor", "testid": "search-input", "text": "Mirrorless", "enter": true, "settleMs": 1500 }, + { "do": "expect", "actor": "visitor", "testid": "item-card", "contains": "Mirrorless Camera", "in": { "testid": "search-results" } } + ], + "statedBy": "search matches any part of an item name across the full catalogue" + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-duplicate-checkout.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-duplicate-checkout.json new file mode 100644 index 00000000000..f59a899db14 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-duplicate-checkout.json @@ -0,0 +1,132 @@ +{ + "schemaVersion": 1, + "level": 1, + "track": "ecommerce", + "name": "Storefront — duplicate checkout", + "writeUrlPattern": "\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses", + "note": "This focused suite owns the shared cart preparation needed by both duplicate-cart and duplicate-checkout checks, so either check can run independently.", + "features": [ + { + "id": 203, + "name": "One cart, two tabs, one checkout", + "actors": ["tab1", "tab2", "filler", "reply"], + "setup": [ + { "do": "signUp", "actor": "tab1", "name": "twin" }, + { "do": "signIn", "actor": "tab2", "name": "twin" }, + { "do": "signIn", "actor": "filler", "name": "twin" }, + { "do": "click", "actor": "tab1", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "click", "actor": "tab2", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "clickConcurrently", "testid": "add-to-cart", "actors": ["tab1", "tab2"], "in": { "testid": "item-card", "contains": "Gaming Mouse" }, "settleMs": 4000 }, + { "do": "reload", "actor": "tab1", "settleMs": 2500 }, + { "do": "ensureSignedIn", "actor": "tab1", "name": "twin", "readyTestid": "current-user" }, + { "do": "click", "actor": "tab1", "testid": "overlay-close", "ifAvailable": true, "within": 1000 }, + { "do": "click", "actor": "tab1", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "tab1", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["cart-toggle", "cart-total"]}, + { "do": "click", "actor": "tab1", "testid": "cart-toggle", "unlessVisible": "cart-total" }, + { "do": "click", "actor": "filler", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "203a", + "category": "production", + "desc": "the same item added from two tabs at once becomes one line of two", + "points": 1, + "steps": [ + { "do": "expect", "actor": "tab1", "testid": "cart-item", "contains": "Gaming Mouse", "count": 1 }, + { "do": "expectNumber", "actor": "tab1", "testid": "cart-quantity", "in": { "testid": "cart-item", "contains": "Gaming Mouse" }, "equals": 2 } + ], + "statedBy": "raises its quantity rather than adding a second line" + }, + { + "id": "203b", + "category": "production", + "desc": "the same consumed cart produces one order after a lost reply or concurrent submissions", + "points": 1, + "steps": [ + {"do":"signUp","actor":"reply","name":"lost-reply"}, + {"do":"prepareResponseLoss","actor":"reply"}, + {"do":"ensureSignedIn","actor":"reply","name":"lost-reply","readyTestid":"current-user"}, + {"do":"click","actor":"reply","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"reply","testid":"catalog-link","ifAvailable":true,"within":1000}, + {"do":"dbRecordCheckout","account":"{user:lost-reply}","item":"Desk Lamp","as":"control-before","storage":{"kind":"order-data","cart":true,"warehouses":false}}, + {"do":"click","awaitWrites":true,"actor":"reply","testid":"add-to-cart","in":{"testid":"item-card","contains":"Desk Lamp"}}, + {"do": "click", "actor": "reply", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["cart-toggle", "checkout-submit"]}, + {"do":"click","actor":"reply","testid":"cart-toggle","unlessVisible":"checkout-submit"}, + {"do":"expectNumber","actor":"reply","testid":"cart-count","equals":1}, + {"do":"dbRecordCheckout","account":"{user:lost-reply}","item":"Desk Lamp","as":"control-cart","storage":{"kind":"order-data","cart":true,"warehouses":false}}, + {"do":"click","actor":"reply","testid":"checkout-submit"}, + {"do":"dbExpectCheckout","before":"control-before","prepared":"control-cart","quantity":1,"within":10000}, + {"do":"reload","actor":"reply","settleMs":0}, + {"do":"ensureSignedIn","actor":"reply","name":"lost-reply","readyTestid":"current-user"}, + {"do":"click","actor":"reply","testid":"overlay-close","ifAvailable":true,"unlessVisible":"cart-total","within":1000}, + {"do":"click","actor":"reply","testid":"current-user","ifAvailable":true,"unlessVisible":["cart-toggle","cart-total"]}, + {"do":"click","actor":"reply","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"reply","testid":"cart-count","equals":0}, + {"do":"click","actor":"reply","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"reply","testid":"catalog-link","ifAvailable":true,"within":1000}, + {"do":"recordNumber","actor":"reply","testid":"item-stock","in":{"testid":"item-card","contains":"Desk Lamp"},"as":"reply-stock-before"}, + {"do":"dbRecordCheckout","account":"{user:lost-reply}","item":"Desk Lamp","as":"before","storage":{"kind":"order-data","cart":true,"warehouses":false}}, + {"do":"click","awaitWrites":true,"actor":"reply","testid":"add-to-cart","in":{"testid":"item-card","contains":"Desk Lamp"}}, + {"do": "click", "actor": "reply", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["cart-toggle", "checkout-submit"]}, + {"do":"click","actor":"reply","testid":"cart-toggle","unlessVisible":"checkout-submit"}, + {"do":"expectNumber","actor":"reply","testid":"cart-count","equals":1}, + {"do":"dbRecordCheckout","account":"{user:lost-reply}","item":"Desk Lamp","as":"prepared","storage":{"kind":"order-data","cart":true,"warehouses":false}}, + {"do":"loseCheckoutResponse","actor":"reply","before":"before","prepared":"prepared","quantity":1}, + {"do":"reload","actor":"reply","settleMs":0}, + {"do":"ensureSignedIn","actor":"reply","name":"lost-reply","readyTestid":"current-user"}, + {"do":"click","actor":"reply","testid":"overlay-close","ifAvailable":true,"unlessVisible":"cart-total","within":1000}, + {"do":"click","actor":"reply","testid":"current-user","ifAvailable":true,"unlessVisible":["cart-toggle","cart-total"]}, + {"do":"click","actor":"reply","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"reply","testid":"cart-count","equals":0}, + {"do":"callAction","actor":"reply","action":"checkout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]}}, + {"do":"expectActionOutcome","actor":"reply","outcome":"completed"}, + {"do":"dbExpectCheckout","before":"before","prepared":"prepared","quantity":1}, + {"do":"reload","actor":"reply","settleMs":0}, + {"do":"ensureSignedIn","actor":"reply","name":"lost-reply","readyTestid":"current-user"}, + {"do":"click","actor":"reply","testid":"overlay-close","ifAvailable":true,"within":1000}, + { "do": "click", "actor": "reply", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do":"expectNumber","actor":"reply","testid":"item-stock","in":{"testid":"item-card","contains":"Desk Lamp"},"relativeTo":"reply-stock-before","plus":-1}, + {"do":"dbRecordCheckout","account":"{user:lost-reply}","item":"Desk Lamp","as":"recovered-before","storage":{"kind":"order-data","cart":true,"warehouses":false}}, + {"do":"click","awaitWrites":true,"actor":"reply","testid":"add-to-cart","in":{"testid":"item-card","contains":"Desk Lamp"}}, + {"do": "click", "actor": "reply", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["cart-toggle", "checkout-submit"]}, + {"do":"click","actor":"reply","testid":"cart-toggle","unlessVisible":"checkout-submit"}, + {"do":"expectNumber","actor":"reply","testid":"cart-count","equals":1}, + {"do":"dbRecordCheckout","account":"{user:lost-reply}","item":"Desk Lamp","as":"recovered-cart","storage":{"kind":"order-data","cart":true,"warehouses":false}}, + {"do":"click","actor":"reply","testid":"checkout-submit"}, + {"do":"dbExpectCheckout","before":"recovered-before","prepared":"recovered-cart","quantity":1,"within":10000}, + {"do":"reload","actor":"reply","settleMs":0}, + {"do":"ensureSignedIn","actor":"reply","name":"lost-reply","readyTestid":"current-user"}, + {"do":"click","actor":"reply","testid":"overlay-close","ifAvailable":true,"unlessVisible":"cart-total","within":1000}, + {"do":"click","actor":"reply","testid":"current-user","ifAvailable":true,"unlessVisible":["cart-toggle","cart-total"]}, + {"do":"click","actor":"reply","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"reply","testid":"cart-count","equals":0}, + { "do": "click", "awaitWrites": true, "actor": "tab1", "testid": "checkout-submit" }, + { "do": "wait", "actor": "tab1", "ms": 2500 }, + { "do": "recordNumber", "actor": "filler", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Keyboard" }, "as": "keyboard-before-checkout" }, + { "do": "dbRecordCheckout", "account": "{user:twin}", "item": "Keyboard", "as": "checkout-before", "storage": { "kind": "order-data", "cart": true, "warehouses": false } }, + { "do": "click", "awaitWrites": true, "actor": "filler", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Keyboard" } }, + { "do": "wait", "actor": "filler", "ms": 2000 }, + { "do": "dbRecordCheckout", "account": "{user:twin}", "item": "Keyboard", "as": "checkout-prepared", "storage": { "kind": "order-data", "cart": true, "warehouses": false } }, + { "do": "callConcurrently", "actors": ["tab1", "tab2"], "action": "checkout", "settleMs": 5000 }, + { "do": "expectCallOutcomes" }, + { "do": "dbExpectCheckout", "before": "checkout-before", "prepared": "checkout-prepared", "quantity": 1 }, + {"do":"click","actor":"filler","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do": "click", "actor": "filler", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + { "do": "click", "actor": "filler", "testid": "orders-toggle", "unlessVisible": "order-item" }, + { "do": "expect", "actor": "filler", "testid": "order-item", "contains": "Keyboard", "count": 1 }, + { "do": "reload", "actor": "filler", "settleMs": 2000 }, + { "do": "ensureSignedIn", "actor": "filler", "name": "twin", "readyTestid": "current-user" }, + { "do": "click", "actor": "filler", "testid": "overlay-close", "ifAvailable": true }, + { "do": "click", "actor": "filler", "testid": "catalog-link", "ifAvailable": true }, + { "do": "expectNumber", "actor": "filler", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Keyboard" }, "relativeTo": "keyboard-before-checkout", "plus": -1 } + ], + "note": "callConcurrently resolves the track's named checkout action through the selected backend adapter and issues it at the same time with tab1 and tab2's own session credentials. The filler prepares the shared Keyboard cart. Native order evidence requires one complete order for that account, the booked price and quantity, a cleared cart, and unchanged prior orders. Checkout can precede warehouse administration, so this check reads no warehouse data and retains its visible stock reduction assertion. The second call may succeed idempotently or deliberately refuse; server errors and no progress fail. The reply actor first proves normal checkout, loses a real browser reply, reloads and explicitly retries the unchanged consumed cart. Independent order/cart reads and visible stock reconciliation require one effect and a later usable checkout. Failure to establish commit and response suppression is inconclusive. This does not claim automatic SDK retry or identity for a cart filled again.", + "provenBy": "The L1 2.3 candidate set binds an exact 203b defect on every reference stack: MongoDB bypasses the atomic cart claim-and-clear, while PostgreSQL and SpacetimeDB leave checked-out cart lines behind. These defects let both named checkout calls reuse the same cart, which the call-outcome and final order assertions are designed to catch. Exact Docker mutation qualification remains the promotion gate.", + "withheld": "The named checkout action has an HTTP route for MongoDB and PostgreSQL and a reducer mapping for SpacetimeDB. The stack adapter issues the corresponding credentialed request for each actor, giving all three stacks the same one-cart, two-call expectation.", + "statedBy": "checking out twice must not produce two orders" + } + ], + "note": "Each tab opens its own cart. Cart panel state belongs to the browser context." + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-external-live-sync.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-external-live-sync.json new file mode 100644 index 00000000000..b995592aacb --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-external-live-sync.json @@ -0,0 +1,53 @@ +{ + "schemaVersion": 1, + "level": 1, + "track": "ecommerce", + "name": "Storefront and warehouse — external write live synchronization", + "writeUrlPattern": "\\/api\\/|\\/items|\\/stock|\\/warehouses", + "note": "A focused, independently runnable probe for an already-open storefront. The direct database writer is outside the application server, and this scenario measures only live propagation.", + "features": [ + { + "id": 901, + "name": "An open storefront follows a direct database write", + "actors": ["viewer"], + "setup": [ + { "do": "click", "actor": "viewer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { + "do": "expectNumber", + "actor": "viewer", + "testid": "item-stock", + "in": { "testid": "item-card", "contains": "Desk Lamp" }, + "equals": 100, + "within": 10000 + } + ], + "criteria": [ + { + "id": "901a", + "category": "production", + "desc": "a direct database write sets Desk Lamp's East stock to 5, and the already-open storefront updates from 100 to 50 without a reload or page action", + "points": 1, + "note": "This proposed score remains draft until the focused pristine and mutation controls are qualified.", + "steps": [ + { + "do": "dbSetStock", + "item": "Desk Lamp", + "warehouse": "East", + "quantity": 5, + "settleMs": 4000 + }, + { + "do": "expectNumber", + "actor": "viewer", + "testid": "item-stock", + "in": { "testid": "item-card", "contains": "Desk Lamp" }, + "equals": 50, + "within": 10000 + } + ], + "statedBy": "The storefront reflects current database values after changes made outside the application server." + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-external-reconnect-sync.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-external-reconnect-sync.json new file mode 100644 index 00000000000..ce621c9c160 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-external-reconnect-sync.json @@ -0,0 +1,55 @@ +{ + "schemaVersion": 1, + "level": 1, + "track": "ecommerce", + "name": "Storefront and warehouse — client reconnect synchronization", + "writeUrlPattern": "\\/api\\/|\\/items|\\/stock|\\/warehouses", + "note": "A focused reconnect probe with a known seed. West remains 45, so changing East from 55 to 7 must produce 52. It does not inherit the West=10 state from the server-restart criterion.", + "features": [ + { + "id": 901, + "name": "A reconnecting storefront catches up to an external write", + "actors": ["viewer"], + "setup": [ + { "do": "click", "actor": "viewer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { + "do": "expectNumber", + "actor": "viewer", + "testid": "item-stock", + "in": { "testid": "item-card", "contains": "Desk Lamp" }, + "equals": 100, + "within": 10000 + } + ], + "criteria": [ + { + "id": "901d", + "category": "production", + "desc": "while the storefront is offline, a direct database write sets Desk Lamp's East stock to 7; after reconnecting, the same page catches up from 100 to the authoritative total of 52", + "points": 1, + "note": "This proposed score remains draft until the focused pristine and mutation controls are qualified.", + "steps": [ + { "do": "setOffline", "actor": "viewer", "offline": true, "settleMs": 1000 }, + { + "do": "dbSetStock", + "item": "Desk Lamp", + "warehouse": "East", + "quantity": 7, + "settleMs": 4000 + }, + { "do": "setOffline", "actor": "viewer", "offline": false, "settleMs": 1000 }, + { + "do": "expectNumber", + "actor": "viewer", + "testid": "item-stock", + "in": { "testid": "item-card", "contains": "Desk Lamp" }, + "equals": 52, + "within": 20000 + } + ], + "statedBy": "The storefront reflects current database values after changes made outside the application server." + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-external-reload-sync.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-external-reload-sync.json new file mode 100644 index 00000000000..89eb012da16 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-external-reload-sync.json @@ -0,0 +1,52 @@ +{ + "schemaVersion": 1, + "level": 1, + "track": "ecommerce", + "name": "Storefront and warehouse — external write reload persistence", + "writeUrlPattern": "\\/api\\/|\\/items|\\/stock|\\/warehouses", + "note": "A self-contained supporting probe. It performs its own direct database write before reloading, so it can be selected or rerun without depending on the live-propagation check.", + "features": [ + { + "id": 901, + "name": "A direct database write survives reload", + "actors": ["viewer"], + "setup": [ + { + "do": "expectNumber", + "actor": "viewer", + "testid": "item-stock", + "in": { "testid": "item-card", "contains": "Desk Lamp" }, + "equals": 100, + "within": 10000 + } + ], + "criteria": [ + { + "id": "901b", + "desc": "after a direct database write sets Desk Lamp's East stock to 5, a reload reads the persisted total of 50", + "points": 0, + "note": "Supporting evidence for 901a rather than a second score. It rules out a page-only patch but has no distinct calibrated failure mode.", + "steps": [ + { + "do": "dbSetStock", + "item": "Desk Lamp", + "warehouse": "East", + "quantity": 5, + "settleMs": 1000 + }, + { "do": "reload", "actor": "viewer", "settleMs": 3000 }, + { + "do": "expectNumber", + "actor": "viewer", + "testid": "item-stock", + "in": { "testid": "item-card", "contains": "Desk Lamp" }, + "equals": 50, + "within": 10000 + } + ], + "statedBy": "The storefront reflects current database values after changes made outside the application server." + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-features.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-features.json new file mode 100644 index 00000000000..3dd2d7b117d --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-features.json @@ -0,0 +1,451 @@ +{ + "schemaVersion": 1, + "level": 1, + "track": "ecommerce", + "name": "Storefront and warehouse — features", + "writeUrlPattern": "\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses", + "note": "Each feature is given its own seeded items where it mutates stock, because features share one database and a purchase in one would otherwise move another's numbers. Items reserved per feature: F2 Coffee Grinder, F3 Espresso Machine + Mirrorless Camera, F4 Headphones + Laptop Stand, F6 Induction Cooktop, F7 Gaming Mouse. Panels are closed after use and items are reached via search, because the storefront shows only the top ten and earlier features change that ranking.", + "features": [ + { + "id": 1, + "name": "Accounts", + "actors": ["shopper", "impostor", "password-owner", "password-returning", "password-impostor", "password-ordinary", "password-ordinary-returning", "password-ordinary-impostor"], + "setup": [ + { "do": "signUp", "actor": "shopper", "name": "ann" } + ], + "note": "The refusals are attempted by a second, signed-out actor: a client that is already signed in has no sign-up form to submit, so re-using one actor would test nothing.", + "criteria": [ + { + "id": "1a", + "desc": "a visitor can create an account and is signed in as it", + "points": 1, + "steps": [ + { "do": "expect", "actor": "shopper", "testid": "current-user", "contains": "ann", "within": 6000 } + ], + "statedBy": "a visitor can create an account with a username and password" + }, + { + "id": "1b", + "desc": "a taken username is refused and does not sign the visitor in as the existing account", + "points": 1, + "steps": [ + { "do": "signUp", "actor": "impostor", "name": "ann", "password": "different-pw", "expectFailure": true }, + { "do": "expect", "actor": "impostor", "testid": "auth-error", "within": 6000 }, + { "do": "reload", "actor": "impostor", "application": true, "settleMs": 1000 }, + { "do": "expect", "actor": "impostor", "testid": "current-user", "absent": true } + ], + "statedBy": "signing up with a taken username fails with a visible error and\nmust never sign the visitor in as the existing account" + }, + { + "id": "1c", + "desc": "the complete accepted password is required to sign in", + "points": 1, + "steps": [ + { + "do": "signUp", + "actor": "password-ordinary", + "name": "password-ordinary", + "password": "ordinary-password-A" + }, + { + "do": "expect", + "actor": "password-ordinary", + "testid": "current-user", + "contains": "password-ordinary", + "within": 6000 + }, + { + "do": "signIn", + "actor": "password-ordinary-returning", + "name": "password-ordinary", + "password": "ordinary-password-A", + "awaitSignedIn": false + }, + { + "do": "expect", + "actor": "password-ordinary-returning", + "testid": "current-user", + "contains": "password-ordinary", + "within": 10000 + }, + { + "do": "signIn", + "actor": "password-ordinary-impostor", + "name": "password-ordinary", + "password": "ordinary-password-B", + "expectFailure": true + }, + { + "do": "expect", + "actor": "password-ordinary-impostor", + "testid": "auth-error", + "within": 6000 + }, + { + "do": "reload", + "actor": "password-ordinary-impostor", + "application": true, + "settleMs": 1000 + }, + { + "do": "expect", + "actor": "password-ordinary-impostor", + "testid": "current-user", + "absent": true + }, + { + "do": "probePasswordBoundary", + "actor": "password-owner", + "returningActor": "password-returning", + "impostorActor": "password-impostor", + "name": "password-boundary", + "password": "界界界界界界界界界界界界界界界界界界界界界界界界-A", + "wrongPassword": "界界界界界界界界界界界界界界界界界界界界界界界界-B" + } + ], + "statedBy": "signing in with a wrong password fails with a visible error" + }, + { + "id": "1d", + "desc": "signing out and back in returns the same account", + "points": 1, + "steps": [ + { "do": "click", "actor": "shopper", "testid": "current-user", "unlessVisible": "signout" }, + { "do": "click", "actor": "shopper", "testid": "signout" }, + { "do": "waitUntilAbsent", "actor": "shopper", "testid": "current-user", "within": 6000 }, + { "do": "signIn", "actor": "shopper", "name": "ann" }, + { "do": "expect", "actor": "shopper", "testid": "current-user", "contains": "ann", "within": 6000 } + ], + "statedBy": "a signed-in user can sign out, returning to the signed-out state" + }, + { + "id": "1e", + "desc": "the session survives a reload", + "points": 1, + "steps": [ + { "do": "reload", "actor": "shopper", "settleMs": 4000 }, + { "do": "expect", "actor": "shopper", "testid": "current-user", "contains": "ann", "within": 6000 } + ], + "statedBy": "a signed-in session persists across a page reload" + } + ] + }, + { + "id": 2, + "name": "Storefront is public and live", + "actors": ["buyer", "visitor"], + "setup": [ + { "do": "signUp", "actor": "buyer", "name": "dov" } + ], + "criteria": [ + { + "id": "2a", + "desc": "a signed-out visitor sees the catalogue with prices and stock", + "points": 1, + "steps": [ + { "do": "expect", "actor": "visitor", "testid": "item-card", "contains": "Air Purifier" }, + { "do": "expect", "actor": "visitor", "testid": "item-price", "in": { "testid": "item-card", "contains": "Air Purifier" } }, + { "do": "expectNumber", "actor": "visitor", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Air Purifier" }, "atLeast": 1 } + ], + "statedBy": "every item shows its name, price and current stock" + }, + { + "id": "2b", + "desc": "the storefront shows ten items, alphabetically while nothing has been bought", + "points": 1, + "steps": [ + { "do": "expect", "actor": "visitor", "testid": "item-card", "count": 10 }, + { "do": "expect", "actor": "visitor", "testid": "item-card", "contains": "Air Purifier" }, + { "do": "expect", "actor": "visitor", "testid": "item-card", "contains": "Laptop Stand" }, + { "do": "expect", "actor": "visitor", "testid": "item-card", "contains": "Mirrorless Camera", "absent": true } + ], + "statedBy": "the storefront shows the 10 most-purchased items" + }, + { + "id": "2c", + "desc": "a purchase moves the bought item to the front of the ranking, live", + "points": 1, + "steps": [ + { "do": "click", "actor": "buyer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Coffee Grinder" } }, + { "do": "wait", "actor": "visitor", "ms": 2500 }, + { "do": "expect", "actor": "visitor", "testid": "item-name", "in": { "testid": "item-list" }, "contains": "Coffee Grinder" }, + { "do": "expectAgreement", "actors": ["buyer", "visitor"], "testid": "item-stock", "in": { "testid": "item-card", "contains": "Coffee Grinder" }, "numeric": true } + ], + "statedBy": "when anyone buys something, every\nvisitor's storefront updates without a reload" + }, + { + "id": "2d", + "desc": "search finds an item that is not on the storefront", + "points": 1, + "steps": [ + { "do": "fill", "actor": "visitor", "testid": "search-input", "text": "Mirrorless", "enter": true, "settleMs": 1500 }, + { "do": "expect", "actor": "visitor", "testid": "item-card", "contains": "Mirrorless Camera", "in": { "testid": "search-results" } } + ], + "statedBy": "search covers the whole catalogue, not just the items currently on the storefront" + } + ] + }, + { + "id": 3, + "name": "Buying", + "actors": ["buyer", "watcher", "visitor"], + "setup": [ + { "do": "signUp", "actor": "buyer", "name": "eli" }, + { "do": "signUp", "actor": "watcher", "name": "fay" } + ], + "criteria": [ + { + "id": "3a", + "desc": "a signed-out visitor is offered no way to buy", + "points": 1, + "steps": [ + { "do": "expect", "actor": "visitor", "testid": "buy-now", "absent": true }, + { "do": "expect", "actor": "visitor", "testid": "add-to-cart", "absent": true } + ], + "statedBy": "a signed-out visitor sees no buy or\nadd-to-cart action" + }, + { + "id": "3b", + "desc": "buying reduces the stock every other client sees, without a reload", + "points": 2, + "steps": [ + { "do": "expectNumber", "actor": "watcher", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Espresso Machine" }, "equals": 100 }, + { "do": "click", "actor": "buyer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Espresso Machine" } }, + { "do": "expectNumber", "actor": "watcher", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Espresso Machine" }, "equals": 99 }, + { "do": "expectNumber", "actor": "visitor", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Espresso Machine" }, "equals": 99 } + ], + "statedBy": "buying an item reduces its stock by one for everyone" + }, + { + "id": "3c", + "desc": "the purchase is recorded in the buyer's order history at the price paid", + "points": 1, + "steps": [ + { "do": "click", "actor": "buyer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Espresso Machine" } }, + {"do":"click","actor":"buyer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { "do": "click", "actor": "buyer", "testid": "orders-toggle" }, + { "do": "expect", "actor": "buyer", "testid": "order-item", "contains": "Espresso Machine" }, + { "do": "expectNumber", "actor": "buyer", "testid": "order-total", "in": { "testid": "order-item", "contains": "Espresso Machine" }, "atLeast": 449 }, + { "do": "pressKey", "actor": "buyer", "key": "Escape", "settleMs": 800 } + ], + "statedBy": "it creates an order for the buyer,\nvisible in their order history" + }, + { + "id": "3d", + "desc": "an item sells out visibly, and a checkout exceeding stock is refused and changes nothing", + "points": 2, + "steps": [ + { "do": "fill", "actor": "buyer", "testid": "search-input", "text": "Mirrorless Camera", "enter": true, "settleMs": 1500 }, + { "do": "click", "actor": "buyer", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Mirrorless Camera" } }, + { "do": "wait", "actor": "buyer", "ms": 800 }, + { "do": "click", "actor": "buyer", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Mirrorless Camera" } }, + { "do": "wait", "actor": "buyer", "ms": 800 }, + { "do": "click", "actor": "buyer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Mirrorless Camera" } }, + { "do": "wait", "actor": "buyer", "ms": 800 }, + { "do": "click", "actor": "buyer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Mirrorless Camera" } }, + { "do": "wait", "actor": "buyer", "ms": 800 }, + { "do": "click", "actor": "buyer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Mirrorless Camera" } }, + { "do": "expectNumber", "actor": "buyer", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Mirrorless Camera" }, "equals": 0 }, + { "do": "expect", "actor": "buyer", "testid": "out-of-stock", "in": { "testid": "item-card", "contains": "Mirrorless Camera" } }, + { "do": "click", "actor": "buyer", "testid": "cart-toggle" }, + { "do": "click", "actor": "buyer", "testid": "checkout-submit" }, + { "do": "expect", "actor": "buyer", "testid": "buy-error", "within": 6000 }, + { "do": "expect", "actor": "buyer", "testid": "cart-item", "contains": "Mirrorless Camera" }, + { "do": "pressKey", "actor": "buyer", "key": "Escape", "settleMs": 800 }, + { "do": "expectNumber", "actor": "buyer", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Mirrorless Camera" }, "equals": 0 } + ], + "statedBy": "an item with zero stock cannot be bought" + } + ] + }, + { + "id": 4, + "name": "Cart belongs to the account", + "actors": ["tab1", "tab2"], + "setup": [ + { "do": "signUp", "actor": "tab1", "name": "gus" } + ], + "criteria": [ + { + "id": "4a", + "desc": "adding the same item twice raises its quantity instead of adding a second line", + "points": 1, + "steps": [ + { "do": "click", "actor": "tab1", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Headphones" } }, + { "do": "wait", "actor": "tab1", "ms": 800 }, + { "do": "click", "actor": "tab1", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Headphones" } }, + { "do": "click", "actor": "tab1", "testid": "cart-toggle" }, + { "do": "expect", "actor": "tab1", "testid": "cart-item", "contains": "Headphones", "count": 1 }, + { "do": "expectNumber", "actor": "tab1", "testid": "cart-quantity", "in": { "testid": "cart-item", "contains": "Headphones" }, "equals": 2 }, + { "do": "pressKey", "actor": "tab1", "key": "Escape", "settleMs": 800 } + ], + "statedBy": "adding an item that is already in the cart raises its quantity" + }, + { + "id": "4b", + "desc": "the cart survives a reload", + "points": 1, + "steps": [ + { "do": "fill", "actor": "tab1", "testid": "search-input", "text": "Laptop Stand", "enter": true, "settleMs": 1500 }, + { "do": "click", "actor": "tab1", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Laptop Stand" } }, + { "do": "click", "actor": "tab1", "testid": "cart-toggle" }, + { "do": "expect", "actor": "tab1", "testid": "cart-item", "contains": "Laptop Stand", "within": 10000 }, + { "do": "reload", "actor": "tab1", "settleMs": 4000 }, + { "do": "ensureSignedIn", "actor": "tab1", "name": "gus", "readyTestid": "current-user" }, + { "do": "click", "actor": "tab1", "testid": "cart-toggle" }, + { "do": "expect", "actor": "tab1", "testid": "cart-item", "contains": "Laptop Stand" }, + { "do": "pressKey", "actor": "tab1", "key": "Escape", "settleMs": 800 } + ], + "statedBy": "the cart survives a reload" + }, + { + "id": "4c", + "desc": "the same account signed in elsewhere sees one cart, live", + "points": 2, + "steps": [ + { "do": "signIn", "actor": "tab2", "name": "gus" }, + { "do": "click", "actor": "tab2", "testid": "cart-toggle" }, + { "do": "expect", "actor": "tab2", "testid": "cart-item", "contains": "Laptop Stand" }, + { "do": "fill", "actor": "tab1", "testid": "search-input", "text": "Induction Cooktop", "enter": true, "settleMs": 1500 }, + { "do": "click", "actor": "tab1", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Induction Cooktop" } }, + { "do": "expect", "actor": "tab2", "testid": "cart-item", "contains": "Induction Cooktop", "within": 10000 } + ], + "statedBy": "the same account signed\nin twice sees one cart, and a change made in one place appears in the other without a\nreload" + }, + { + "id": "4d", + "desc": "checkout creates one order, reduces stock and empties the cart", + "points": 2, + "steps": [ + { "do": "click", "actor": "tab1", "testid": "cart-toggle" }, + { "do": "click", "actor": "tab1", "testid": "checkout-submit" }, + { "do": "wait", "actor": "tab1", "ms": 2000 }, + { "do": "expect", "actor": "tab1", "testid": "empty-cart" }, + { "do": "pressKey", "actor": "tab1", "key": "Escape", "settleMs": 800 }, + { "do": "fill", "actor": "tab1", "testid": "search-input", "text": "", "enter": true, "settleMs": 1200 }, + { "do": "expectNumber", "actor": "tab1", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Headphones" }, "equals": 98 }, + {"do":"click","actor":"tab1","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { "do": "click", "actor": "tab1", "testid": "orders-toggle" }, + { "do": "expect", "actor": "tab1", "testid": "order-item", "contains": "Headphones" } + ], + "statedBy": "checkout turns the cart into one order, reduces stock for every line, and\nempties the cart" + } + ] + }, + { + "id": 6, + "name": "Reviews", + "actors": ["author", "other", "visitor"], + "setup": [ + { "do": "signUp", "actor": "author", "name": "hal" }, + { "do": "signUp", "actor": "other", "name": "ivy" }, + { "do": "click", "actor": "author", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Induction Cooktop" } }, + { "do": "wait", "actor": "author", "ms": 800 }, + { "do": "click", "actor": "other", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Induction Cooktop" } }, + { "do": "wait", "actor": "other", "ms": 800 } + ], + "criteria": [ + { + "id": "6a", + "desc": "a customer can review an item and everyone sees it, signed out included", + "points": 2, + "steps": [ + { "do": "fill", "actor": "author", "testid": "search-input", "text": "Induction Cooktop", "enter": true, "settleMs": 1500 }, + { "do": "openItem", "actor": "author", "item": "Induction Cooktop" }, + {"do":"click","actor":"author","testid":"review-toggle","unlessVisible":"review-rating","ifAvailable":true}, + { "do": "fill", "actor": "author", "testid": "review-rating", "text": "4" }, + { "do": "fill", "actor": "author", "testid": "review-input", "text": "solid mold" }, + { "do": "click", "actor": "author", "testid": "review-submit" }, + { "do": "expect", "actor": "author", "testid": "review-item", "contains": "solid mold" }, + { "do": "openItem", "actor": "visitor", "item": "Induction Cooktop" }, + { "do": "expect", "actor": "visitor", "testid": "review-item", "contains": "solid mold", "within": 10000 } + ], + "statedBy": "reviews are visible to everyone, including signed-out visitors" + }, + { + "id": "6b", + "desc": "one review per customer per item — a second does not add a duplicate", + "points": 1, + "steps": [ + {"do":"click","actor":"author","testid":"review-toggle","unlessVisible":"review-rating","ifAvailable":true}, + { "do": "fill", "actor": "author", "testid": "review-rating", "text": "2" }, + { "do": "fill", "actor": "author", "testid": "review-input", "text": "changed my mind" }, + { "do": "click", "actor": "author", "testid": "review-submit" }, + { "do": "wait", "actor": "author", "ms": 2000 }, + { "do": "expect", "actor": "author", "testid": "review-item", "contains": "solid mold", "absent": true } + ], + "statedBy": "one review per customer per item" + }, + { + "id": "6c", + "desc": "the average rating reflects both reviewers, live", + "points": 1, + "steps": [ + { "do": "fill", "actor": "other", "testid": "search-input", "text": "Induction Cooktop", "enter": true, "settleMs": 1500 }, + { "do": "openItem", "actor": "other", "item": "Induction Cooktop" }, + {"do":"click","actor":"other","testid":"review-toggle","unlessVisible":"review-rating","ifAvailable":true}, + { "do": "fill", "actor": "other", "testid": "review-rating", "text": "4" }, + { "do": "fill", "actor": "other", "testid": "review-input", "text": "does the job" }, + { "do": "click", "actor": "other", "testid": "review-submit" }, + { "do": "expectNumber", "actor": "other", "testid": "review-average", "equals": 3, "within": 10000 }, + { "do": "expectAgreement", "actors": ["author", "other"], "testid": "review-average", "numeric": true } + ], + "statedBy": "each item shows its average rating, which updates live as reviews arrive" + } + ], + "note": "Both reviewers buy the item in setup: reviews are gated on a purchase, so an unpurchased reviewer would be refused by a compliant app." + }, + { + "id": 7, + "name": "Admin and warehouses", + "actors": ["admin", "authorized", "customer", "visitor"], + "setup": [ + { "do": "signUp", "actor": "customer", "name": "jon" }, + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true } + ], + "criteria": [ + { + "id": "7a", + "desc": "the admin area is for admins only", + "points": 1, + "steps": [ + { "do": "signIn", "actor": "authorized", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + { "do": "click", "actor": "authorized", "testid": "admin-link" }, + { "do": "expect", "actor": "authorized", "testid": "admin-item-row" }, + { "do": "click", "actor": "customer", "testid": "admin-link", "ifAvailable": true, "settleMs": 1500 }, + { "do": "expect", "actor": "customer", "testid": "admin-item-row", "absent": true } + ], + "statedBy": "admin accounts see an admin area; customers do not" + }, + { + "id": "7b", + "desc": "admin lists every item, every warehouse, and what each warehouse holds", + "points": 1, + "steps": [ + { "do": "click", "actor": "admin", "testid": "admin-link" }, + { "do": "expect", "actor": "admin", "testid": "admin-item-row", "count": 13 }, + { "do": "expect", "actor": "admin", "testid": "admin-warehouse-item", "contains": "East" }, + { "do": "expect", "actor": "admin", "testid": "admin-warehouse-item", "contains": "West" }, + { "do": "expectNumber", "actor": "admin", "testid": "admin-stock", "in": { "testid": "admin-item-row", "contains": "Gaming Mouse" }, "equals": 100 } + ], + "statedBy": "admin lists every item with its stock, every warehouse, and the stock of\neach item in each warehouse" + }, + { + "id": "7c", + "desc": "the storefront stock is the sum across warehouses, and a restock raises it live", + "points": 2, + "steps": [ + { "do": "pressKey", "actor": "visitor", "key": "Escape", "settleMs": 800 }, + { "do": "fill", "actor": "visitor", "testid": "search-input", "text": "Gaming Mouse", "enter": true, "settleMs": 1500 }, + { "do": "expectNumber", "actor": "visitor", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Gaming Mouse" }, "equals": 100 }, + { "do": "fill", "actor": "admin", "testid": "restock-input", "text": "25", "in": { "testid": "admin-location-row", "contains": "Gaming Mouse" } }, + { "do": "click", "actor": "admin", "testid": "restock-submit", "in": { "testid": "admin-location-row", "contains": "Gaming Mouse" } }, + { "do": "expectNumber", "actor": "visitor", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Gaming Mouse" }, "equals": 125, "within": 10000 }, + { "do": "expectNumber", "actor": "admin", "testid": "admin-stock", "in": { "testid": "admin-item-row", "contains": "Gaming Mouse" }, "equals": 125 } + ], + "statedBy": "an item's stock on the storefront is the sum of that item's units across all\nwarehouses" + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-invariants.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-invariants.json new file mode 100644 index 00000000000..1dea0993e15 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-invariants.json @@ -0,0 +1,298 @@ +{ + "schemaVersion": 1, + "level": 1, + "track": "ecommerce", + "name": "Storefront and warehouse — invariants", + "writeUrlPattern": "\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses", + "note": "Cross-cutting properties, scored separately from features because an app can implement every feature and still let anyone spend anyone else's money. Each is stated as an outcome a customer would recognise, not as a protocol detail — a backend must not lose points for being harder to inspect, nor gain them for it. 108 and 109 cover what a customer may assert about things that are not theirs: a rating is a claim about a purchase, and a cart belongs to exactly one account.", + "features": [ + { + "id": 101, + "name": "Purchase requires an account", + "actors": ["buyer", "guest"], + "setup": [ + { "do": "signUp", "actor": "buyer", "name": "kim" } + ], + "criteria": [ + { + "id": "101a", + "desc": "a purchase request replayed without a session creates no order and moves no stock", + "points": 2, + "steps": [ + { "do": "expectNumber", "actor": "buyer", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Bluetooth Speaker" }, "equals": 100 }, + { "do": "click", "actor": "buyer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Bluetooth Speaker" } }, + { "do": "wait", "actor": "buyer", "ms": 1500 }, + { "do": "replayAs", "actor": "guest", "from": "buyer", "match": "buy", "settleMs": 2500 }, + { "do": "expectReplayRejected", "actor": "guest" }, + { "do": "expectNumber", "actor": "buyer", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Bluetooth Speaker" }, "equals": 99 } + ], + "statedBy": "the server must refuse a purchase request that arrives without a valid session" + } + ] + }, + { + "id": 102, + "name": "Purchases are attributed to whoever made them", + "actors": ["victim", "attacker"], + "setup": [ + { "do": "signUp", "actor": "victim", "name": "lee" }, + { "do": "signUp", "actor": "attacker", "name": "mel" } + ], + "criteria": [ + { + "id": "102a", + "desc": "an order cannot be placed in someone else's name by editing the request", + "points": 2, + "steps": [ + { "do": "click", "actor": "victim", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Coffee Grinder" } }, + { "do": "click", "actor": "attacker", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Desk Lamp" } }, + { "do": "wait", "actor": "attacker", "ms": 1500 }, + { "do": "forgeWrite", "actor": "attacker", "fromActor": "victim", "settleMs": 2500 }, + { "do": "expectForgeryRejected", "actor": "attacker" }, + {"do":"click","actor":"victim","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { "do": "click", "actor": "victim", "testid": "orders-toggle" }, + { "do": "expect", "actor": "victim", "testid": "order-item", "contains": "Coffee Grinder", "count": 1 }, + { "do": "expect", "actor": "victim", "testid": "order-item", "contains": "Desk Lamp", "absent": true } + ], + "statedBy": "knowing someone's username must never grant access to it" + } + ] + }, + { + "id": 103, + "name": "Stock is administered by admins only", + "actors": ["admin", "customer"], + "setup": [ + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + { "do": "signUp", "actor": "customer", "name": "ned" } + ], + "criteria": [ + { + "id": "103a", + "desc": "a restock replayed by a customer is refused and changes no stock", + "points": 2, + "steps": [ + { "do": "click", "actor": "admin", "testid": "admin-link" }, + { "do": "fill", "actor": "admin", "testid": "restock-input", "text": "10", "in": { "testid": "admin-location-row", "contains": "Air Purifier" } }, + { "do": "click", "actor": "admin", "testid": "restock-submit", "in": { "testid": "admin-location-row", "contains": "Air Purifier" } }, + { "do": "wait", "actor": "admin", "ms": 2000 }, + { "do": "pressKey", "actor": "customer", "key": "Escape", "settleMs": 800 }, + { "do": "recordNumber", "actor": "customer", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Air Purifier" }, "as": "purifier-after-admin-restock" }, + { "do": "replayAs", "actor": "customer", "from": "admin", "match": "restock", "settleMs": 2500 }, + { "do": "expectReplayRejected", "actor": "customer" }, + { "do": "expectNumber", "actor": "customer", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Air Purifier" }, "relativeTo": "purifier-after-admin-restock", "plus": 0 } + ], + "statedBy": "the server must refuse admin actions from a non-admin account" + } + ] + }, + { + "id": 104, + "name": "The price is the store's to set", + "actors": ["buyer"], + "setup": [ + { "do": "signUp", "actor": "buyer", "name": "oli" } + ], + "criteria": [ + { + "id": "104a", + "desc": "a purchase request with the price edited does not sell the item for less", + "points": 2, + "steps": [ + { "do": "click", "actor": "buyer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Espresso Machine" } }, + { "do": "wait", "actor": "buyer", "ms": 1500 }, + { "do": "replayAs", "actor": "buyer", "from": "buyer", "match": "buy", "swap": { "find": "449", "with": "1" }, "settleMs": 2500 }, + { "do": "expectReplayRejected", "actor": "buyer" }, + {"do":"click","actor":"buyer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { "do": "click", "actor": "buyer", "testid": "orders-toggle" }, + { "do": "expectNumber", "actor": "buyer", "testid": "order-total", "in": { "testid": "order-item", "contains": "Espresso Machine" }, "atLeast": 449 } + ], + "statedBy": "the price at the time of purchase" + } + ] + }, + { + "id": 105, + "name": "An account keeps what belongs to it", + "actors": ["shopper"], + "setup": [ + { "do": "signUp", "actor": "shopper", "name": "pat" } + ], + "criteria": [ + { + "id": "105a", + "desc": "cart and order history survive a reload as the same account", + "points": 1, + "steps": [ + { "do": "click", "actor": "shopper", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Keyboard" } }, + { "do": "click", "actor": "shopper", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Headphones" } }, + { "do": "reload", "actor": "shopper", "settleMs": 4000 }, + { "do": "expect", "actor": "shopper", "testid": "current-user", "contains": "pat" }, + { "do": "click", "actor": "shopper", "testid": "cart-toggle" }, + { "do": "expect", "actor": "shopper", "testid": "cart-item", "contains": "Keyboard" }, + {"do":"click","actor":"shopper","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { "do": "click", "actor": "shopper", "testid": "orders-toggle" }, + { "do": "expect", "actor": "shopper", "testid": "order-item", "contains": "Headphones" } + ], + "statedBy": "persists across a page reload" + }, + { + "id": "105b", + "desc": "the same account survives the connection dropping and coming back", + "points": 1, + "steps": [ + { "do": "setOffline", "actor": "shopper", "settleMs": 3000 }, + { "do": "setOffline", "actor": "shopper", "offline": false, "settleMs": 6000 }, + { "do": "expect", "actor": "shopper", "testid": "current-user", "contains": "pat" }, + { "do": "click", "actor": "shopper", "testid": "cart-toggle" }, + { "do": "expect", "actor": "shopper", "testid": "cart-item", "contains": "Keyboard" } + ], + "statedBy": "across the connection dropping and re-establishing" + } + ] + }, + { + "id": 106, + "name": "One customer's orders are not another's", + "actors": ["one", "two"], + "setup": [ + { "do": "signUp", "actor": "one", "name": "quin" }, + { "do": "signUp", "actor": "two", "name": "ros" } + ], + "criteria": [ + { + "id": "106a", + "desc": "a customer's order history contains only their own orders", + "points": 1, + "steps": [ + { "do": "click", "actor": "one", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Coffee Grinder" } }, + { "do": "wait", "actor": "one", "ms": 2000 }, + {"do":"click","actor":"two","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { "do": "click", "actor": "two", "testid": "orders-toggle" }, + { "do": "expect", "actor": "two", "testid": "order-item", "contains": "Coffee Grinder", "absent": true } + ], + "statedBy": "a customer sees only their own orders" + } + ] + }, + { + "id": 107, + "name": "The books balance", + "actors": ["buyer", "admin"], + "setup": [ + { "do": "signUp", "actor": "buyer", "name": "sam" }, + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true } + ], + "criteria": [ + { + "id": "107a", + "desc": "revenue rises by exactly what was bought", + "points": 1, + "steps": [ + { "do": "click", "actor": "admin", "testid": "admin-link" }, + { "do": "recordNumber", "actor": "admin", "testid": "admin-revenue", "as": "revenue-before" }, + { "do": "recordNumber", "actor": "admin", "testid": "admin-stock", "in": { "testid": "admin-item-row", "contains": "Laptop Stand" }, "as": "brick-before" }, + { "do": "pressKey", "actor": "buyer", "key": "Escape", "settleMs": 800 }, + { "do": "click", "actor": "buyer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Laptop Stand" } }, + { "do": "wait", "actor": "buyer", "ms": 1000 }, + { "do": "click", "actor": "buyer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Laptop Stand" } }, + { "do": "wait", "actor": "buyer", "ms": 2000 }, + { "do": "expectNumber", "actor": "admin", "testid": "admin-revenue", "relativeTo": "revenue-before", "plus": 58, "within": 10000 } + ], + "statedBy": "the sum of all order totals" + }, + { + "id": "107b", + "desc": "what the store sold is what left the warehouses, and a fresh client agrees", + "points": 1, + "steps": [ + { "do": "expectNumber", "actor": "admin", "testid": "admin-stock", "in": { "testid": "admin-item-row", "contains": "Laptop Stand" }, "relativeTo": "brick-before", "plus": -2 }, + { "do": "freshClient", "actor": "buyer" }, + { "do": "expectNumber", "actor": "buyer-fresh", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Laptop Stand" }, "relativeTo": "brick-before", "plus": -2 } + ], + "statedBy": "must always equal what the orders themselves add up to" + } + ] + }, + { + "id": 108, + "name": "A review is a claim about a purchase", + "actors": ["owner", "stranger"], + "setup": [ + { "do": "signUp", "actor": "owner", "name": "tam" }, + { "do": "signUp", "actor": "stranger", "name": "uma" } + ], + "criteria": [ + { + "id": "108a", + "desc": "someone who never bought the item cannot review it", + "points": 2, + "steps": [ + { "do": "fill", "actor": "stranger", "testid": "search-input", "text": "Webcam", "enter": true, "settleMs": 1500 }, + { "do": "openItem", "actor": "stranger", "item": "Webcam" }, + {"do":"click","actor":"stranger","testid":"review-toggle","unlessVisible":"review-rating","ifAvailable":true}, + { "do": "fill", "actor": "stranger", "testid": "review-rating", "text": "5" }, + { "do": "fill", "actor": "stranger", "testid": "review-input", "text": "never bought this" }, + { "do": "click", "actor": "stranger", "testid": "review-submit" }, + { "do": "expect", "actor": "stranger", "testid": "review-error", "within": 6000 }, + { "do": "expect", "actor": "stranger", "testid": "review-item", "contains": "never bought this", "absent": true } + ], + "statedBy": "refuses a review from someone who has never ordered the item" + }, + { + "id": "108b", + "desc": "buying the item earns the right to review it", + "points": 1, + "steps": [ + { "do": "fill", "actor": "owner", "testid": "search-input", "text": "Webcam", "enter": true, "settleMs": 1500 }, + { "do": "click", "actor": "owner", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Webcam" } }, + { "do": "wait", "actor": "owner", "ms": 2000 }, + { "do": "openItem", "actor": "owner", "item": "Webcam" }, + {"do":"click","actor":"owner","testid":"review-toggle","unlessVisible":"review-rating","ifAvailable":true}, + { "do": "fill", "actor": "owner", "testid": "review-rating", "text": "4" }, + { "do": "fill", "actor": "owner", "testid": "review-input", "text": "bought and used it" }, + { "do": "click", "actor": "owner", "testid": "review-submit" }, + { "do": "expect", "actor": "owner", "testid": "review-item", "contains": "bought and used it", "within": 8000 } + ], + "statedBy": "a review of an item they have bought" + } + ] + }, + { + "id": 109, + "name": "A cart is nobody else’s business", + "actors": ["owner", "stranger"], + "setup": [ + { "do": "signUp", "actor": "owner", "name": "vic" }, + { "do": "signUp", "actor": "stranger", "name": "wes" } + ], + "criteria": [ + { + "id": "109a", + "desc": "one customer’s cart request replayed by another leaves the owner’s cart untouched", + "points": 2, + "steps": [ + { "do": "click", "actor": "owner", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Coffee Grinder" } }, + { "do": "wait", "actor": "owner", "ms": 1500 }, + { "do": "replayAs", "actor": "stranger", "from": "owner", "match": "cart", "settleMs": 2500 }, + { "do": "click", "actor": "owner", "testid": "cart-toggle" }, + { "do": "expectNumber", "actor": "owner", "testid": "cart-quantity", "in": { "testid": "cart-item", "contains": "Coffee Grinder" }, "equals": 1 } + ], + "note": "A cart add request that contains only an item and quantity is legitimately reusable by another authenticated customer: the server applies it to the caller's own cart. The ownership assertion below is the security boundary; requiring every replay to return non-2xx incorrectly rejects identity-derived APIs.", + "statedBy": "nobody else can read or change it" + }, + { + "id": "109b", + "desc": "a quantity below one is refused rather than credited", + "points": 1, + "steps": [ + { "do": "replayAs", "actor": "owner", "from": "owner", "match": "cart", "swap": { "find": "\"quantity\":1", "with": "\"quantity\":-3" }, "settleMs": 2500 }, + { "do": "expectNumber", "actor": "owner", "testid": "cart-total", "atLeast": 0 }, + { "do": "expectNumber", "actor": "owner", "testid": "cart-quantity", "in": { "testid": "cart-item", "contains": "Coffee Grinder" }, "atLeast": 1 } + ], + "statedBy": "a request carrying a negative quantity is refused and changes nothing" + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-last-unit.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-last-unit.json new file mode 100644 index 00000000000..221440ee538 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-last-unit.json @@ -0,0 +1,170 @@ +{ + "schemaVersion": 1, + "level": 1, + "track": "ecommerce", + "name": "Storefront and warehouse — last-unit accounting", + "writeUrlPattern": "\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses", + "note": "Draft: per-warehouse stored-stock assertions require matching reference and defect-control qualification.", + "features": [ + { + "id": 201, + "name": "The last unit is sold once", + "actors": ["admin", "a", "b", "c", "d", "e", "f"], + "setup": [ + { + "do": "signIn", + "actor": "admin", + "name": "admin", + "password": "stackbench-admin-2026", + "exact": true + }, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + { "do": "click", "actor": "admin", "testid": "admin-link", "unlessVisible": "admin-panel" }, + { + "do": "recordNumber", + "actor": "admin", + "testid": "admin-revenue", + "as": "revenue-before-last-unit" + }, + { "do": "dbSetStock", "item": "Air Purifier", "warehouse": "East", "quantity": 2, "settleMs": 250 }, + { "do": "dbSetStock", "item": "Air Purifier", "warehouse": "West", "quantity": 1, "settleMs": 250 }, + { "do": "signUp", "actor": "a", "name": "c1" }, + { "do": "signUp", "actor": "b", "name": "c2" }, + { "do": "signUp", "actor": "c", "name": "c3" }, + { "do": "signUp", "actor": "d", "name": "c4" }, + { "do": "signUp", "actor": "e", "name": "c5" }, + { "do": "signUp", "actor": "f", "name": "c6" }, + { "do": "dbRecordCheckout", "account": "{user:c1}", "item": "Air Purifier", "as": "buy-a", "storage": { "kind": "order-data", "cart": false, "warehouses": true } }, + { "do": "dbRecordCheckout", "account": "{user:c2}", "item": "Air Purifier", "as": "buy-b", "storage": { "kind": "order-data", "cart": false, "warehouses": true } }, + { "do": "dbRecordCheckout", "account": "{user:c3}", "item": "Air Purifier", "as": "buy-c", "storage": { "kind": "order-data", "cart": false, "warehouses": true } }, + { "do": "dbRecordCheckout", "account": "{user:c4}", "item": "Air Purifier", "as": "buy-d", "storage": { "kind": "order-data", "cart": false, "warehouses": true } }, + { "do": "dbRecordCheckout", "account": "{user:c5}", "item": "Air Purifier", "as": "buy-e", "storage": { "kind": "order-data", "cart": false, "warehouses": true } }, + { "do": "dbRecordCheckout", "account": "{user:c6}", "item": "Air Purifier", "as": "buy-f", "storage": { "kind": "order-data", "cart": false, "warehouses": true } }, + { "do": "click", "actor": "a", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "click", "actor": "b", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "click", "actor": "c", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "click", "actor": "d", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "click", "actor": "e", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "click", "actor": "f", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { + "do": "callConcurrently", + "action": "buy", + "actors": ["a", "b", "c", "d", "e", "f"], + "namedAction": { "id": "buy", "path": "/api/items/:id/buy", "reducer": "buy_now", "args": [0], "params": [{ "name": "itemId", "in": "path", "placeholder": ":id", "wireType": "u64" }] }, + "input": { "testid": "item-card", "contains": "Air Purifier", "attribute": "data-buy-input" }, + "settleMs": 6000 + }, + { "do": "expectCallOutcomes" } + ], + "criteria": [ + { + "id": "201a", + "category": "production", + "desc": "after six customers try to buy the last three units, each warehouse stores zero stock and all observed clients show zero stock", + "note": "Clients are read after a reload: this check measures stock integrity, and live propagation is measured by the live-state checks.", + "points": 1, + "steps": [ + { "do": "dbExpectStock", "item": "Air Purifier", "warehouse": "East", "equals": 0 }, + { "do": "dbExpectStock", "item": "Air Purifier", "warehouse": "West", "equals": 0 }, + { "do": "reload", "actor": "a", "settleMs": 500 }, + { "do": "ensureSignedIn", "actor": "a", "name": "c1", "readyTestid": "current-user" }, + { "do": "click", "actor": "a", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "reload", "actor": "b", "settleMs": 500 }, + { "do": "ensureSignedIn", "actor": "b", "name": "c2", "readyTestid": "current-user" }, + { "do": "click", "actor": "b", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "reload", "actor": "c", "settleMs": 500 }, + { "do": "ensureSignedIn", "actor": "c", "name": "c3", "readyTestid": "current-user" }, + { "do": "click", "actor": "c", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "reload", "actor": "d", "settleMs": 500 }, + { "do": "ensureSignedIn", "actor": "d", "name": "c4", "readyTestid": "current-user" }, + { "do": "click", "actor": "d", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "reload", "actor": "e", "settleMs": 500 }, + { "do": "ensureSignedIn", "actor": "e", "name": "c5", "readyTestid": "current-user" }, + { "do": "click", "actor": "e", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "reload", "actor": "f", "settleMs": 500 }, + { "do": "ensureSignedIn", "actor": "f", "name": "c6", "readyTestid": "current-user" }, + { "do": "click", "actor": "f", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { + "do": "expectNumber", + "actor": "a", + "testid": "item-stock", + "in": { "testid": "item-card", "contains": "Air Purifier" }, + "equals": 0 + }, + { + "do": "expectAgreement", + "actors": ["a", "b", "c", "d", "e", "f"], + "testid": "item-stock", + "in": { "testid": "item-card", "contains": "Air Purifier" }, + "numeric": true + } + ], + "provenBy": "Prior qualification covers the client observations only. The added stored-stock observations need matching three-stack reference and defect-control evidence before promotion.", + "statedBy": "stock may never go negative, and two customers must never both get the last unit" + }, + { + "id": "201c", + "category": "production", + "desc": "revenue increases by exactly three sales, not six", + "points": 1, + "steps": [ + {"do":"reload","actor":"admin","settleMs":1000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","readyTestid":"current-user","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-revenue"]}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":"admin-revenue"}, + {"do":"expectNumber","actor":"admin","testid":"admin-revenue","relativeTo":"revenue-before-last-unit","plus":567,"within":10000} + ], + "statedBy": "stock must never leave a warehouse without a corresponding order" + }, + { + "id": "201b", + "category": "production", + "desc": "the last three units create complete orders for the successful buyers, and all four affordable purchases succeed when stock is sufficient", + "points": 1, + "steps": [ + { "do": "dbExpectPurchases", "before": { "a": "buy-a", "b": "buy-b", "c": "buy-c", "d": "buy-d", "e": "buy-e", "f": "buy-f" }, "purchases": 3 }, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do": "click", "actor": "a", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + { "do": "click", "actor": "a", "testid": "orders-toggle", "unlessVisible": "order-item" }, + {"do":"click","actor":"b","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do": "click", "actor": "b", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + { "do": "click", "actor": "b", "testid": "orders-toggle", "unlessVisible": "order-item" }, + {"do":"click","actor":"c","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do": "click", "actor": "c", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + { "do": "click", "actor": "c", "testid": "orders-toggle", "unlessVisible": "order-item" }, + {"do":"click","actor":"d","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do": "click", "actor": "d", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + { "do": "click", "actor": "d", "testid": "orders-toggle", "unlessVisible": "order-item" }, + {"do":"click","actor":"e","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do": "click", "actor": "e", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + { "do": "click", "actor": "e", "testid": "orders-toggle", "unlessVisible": "order-item" }, + {"do":"click","actor":"f","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do": "click", "actor": "f", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + { "do": "click", "actor": "f", "testid": "orders-toggle", "unlessVisible": "order-item" }, + { + "do": "expectActorsWith", + "actors": ["a", "b", "c", "d", "e", "f"], + "testid": "order-item", + "contains": "Air Purifier", + "equals": 3, + "maxEach": 1 + }, + { "do": "dbSetStock", "item": "Air Purifier", "warehouse": "East", "quantity": 2, "settleMs": 250 }, + { "do": "dbSetStock", "item": "Air Purifier", "warehouse": "West", "quantity": 2, "settleMs": 250 }, + { "do": "reload", "actor": "a", "settleMs": 1000 }, + { "do": "ensureSignedIn", "actor": "a", "name": "c1", "readyTestid": "current-user" }, + { "do": "click", "actor": "a", "testid": "catalog-link", "ifAvailable": true }, + { "do": "dbRecordCheckout", "account": "{user:c1}", "item": "Air Purifier", "as": "ample-a", "storage": { "kind": "order-data", "cart": false, "warehouses": true } }, + { "do": "dbRecordCheckout", "account": "{user:c2}", "item": "Air Purifier", "as": "ample-b", "storage": { "kind": "order-data", "cart": false, "warehouses": true } }, + { "do": "callConcurrently", "actors": ["a", "b"], "from": "a", "action": "buy", "namedAction": { "id": "buy", "path": "/api/items/:id/buy", "reducer": "buy_now", "args": [0], "params": [{ "name": "itemId", "in": "path", "placeholder": ":id", "wireType": "u64" }] }, "requests": 4, "input": { "testid": "item-card", "contains": "Air Purifier", "attribute": "data-buy-input" }, "settleMs": 3000 }, + { "do": "expectCallOutcomes" }, + { "do": "dbExpectPurchases", "before": { "a": "ample-a", "b": "ample-b" }, "purchases": 4 } + ], + "provenBy": "Draft: native order reconciliation and the sufficient-stock progress case require matching reference and defect controls. This criterion runs after the revenue assertion so its extra purchases cannot alter that assertion's baseline.", + "statedBy": "two customers must never both get the last unit" + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-open-list-live.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-open-list-live.json new file mode 100644 index 00000000000..a7d801ddcf0 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-open-list-live.json @@ -0,0 +1,76 @@ +{ + "schemaVersion": 1, + "level": 1, + "track": "ecommerce", + "name": "Storefront — an open review list stays current", + "writeUrlPattern": "\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses", + "note": "This focused candidate checks one mechanism-neutral product behavior: the reader opens the review list first, the reviewer then commits a review, and the reader's already-open list must show it exactly once. The writer's assertion proves the review committed before the reader is graded. A stronger simultaneous snapshot-and-update stress test remains separate future coverage because it cannot be forced across arbitrary app architectures without prescribing their data transport.", + "features": [ + { + "id": 902, + "name": "An open list stays current", + "actors": ["reader", "reviewer"], + "setup": [ + { "do": "signUp", "actor": "reviewer", "name": "raceR" }, + { "do": "signUp", "actor": "reader", "name": "raceL" }, + { + "do": "recordNumber", + "actor": "reviewer", + "testid": "item-stock", + "in": { "testid": "item-card", "contains": "Keyboard" }, + "as": "keyboard-before-review-purchase" + }, + { + "do": "click", + "actor": "reviewer", + "testid": "buy-now", + "in": { "testid": "item-card", "contains": "Keyboard" } + }, + { + "do": "expectNumber", + "actor": "reviewer", + "testid": "item-stock", + "in": { "testid": "item-card", "contains": "Keyboard" }, + "relativeTo": "keyboard-before-review-purchase", + "plus": -1, + "within": 10000 + }, + {"do": "openItem", "actor": "reviewer", "item": "Keyboard"}, + { "do": "expect", "actor": "reviewer", "testid": "item-detail", "within": 10000 }, + {"do":"click","actor":"reviewer","testid":"review-toggle","unlessVisible":"review-rating","ifAvailable":true}, + { "do": "fill", "actor": "reviewer", "testid": "review-rating", "text": "5" } + ], + "criteria": [ + { + "id": "902a", + "desc": "one customer has the Keyboard's reviews open before another posts one; the already-open view shows that review exactly once", + "points": 1, + "note": "Candidate until the exact reference and mutation matrix is run. This deterministic check covers live list contents and exact-once rendering without assuming HTTP, WebSockets, subscriptions, or any project layout.", + "steps": [ + {"do": "openItem", "actor": "reader", "item": "Keyboard"}, + { "do": "expect", "actor": "reader", "testid": "item-detail", "within": 10000 }, + {"do":"click","actor":"reviewer","testid":"review-toggle","unlessVisible":"review-rating","ifAvailable":true}, + { "do": "fill", "actor": "reviewer", "testid": "review-input", "text": "live-review-kbd" }, + { "do": "click", "actor": "reviewer", "testid": "review-submit" }, + { + "do": "expectElementCount", + "actor": "reviewer", + "testid": "review-item", + "contains": "live-review-kbd", + "equals": 1, + "within": 10000 + }, + { + "do": "expectElementCount", + "actor": "reader", + "testid": "review-item", + "contains": "live-review-kbd", + "equals": 1, + "within": 10000 + } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-order-ownership.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-order-ownership.json new file mode 100644 index 00000000000..0d539399443 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-order-ownership.json @@ -0,0 +1,61 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "One customer's orders are not another's", + "writeUrlPattern": "\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses", + "note": "Draft: two accounts first prove separate order histories. The same browser signs out and signs in as the other account without a grader reload. Its old orders must disappear, the new account history must work, and an independently observed purchase must belong to the new account. This does not claim token revocation, all private data surfaces, or transport-level leak detection.", + "features": [ + { + "id": 106, + "name": "One customer's orders are not another's", + "actors": ["one","two"], + "setup": [{"do":"signUp","actor":"one","name":"quin"},{"do":"signUp","actor":"two","name":"ros"}, + { "do": "click", "actor": "one", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "click", "actor": "two", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }], + "criteria": [ + { + "id": "106a", + "category": "production", + "desc": "order history and purchases remain account-scoped when a browser switches accounts", + "points": 1, + "steps": [ + { + "do": "click", + "actor": "one", + "testid": "buy-now", + "in": {"testid":"item-card","contains":"Coffee Grinder"} + }, + {"do":"click","actor":"two","testid":"buy-now","in":{"testid":"item-card","contains":"Desk Lamp"}}, + {"do":"wait","actor":"two","ms":2000}, + {"do":"click","actor":"two","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do": "click", "actor": "two", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + {"do":"click","actor":"two","testid":"orders-toggle", "unlessVisible": "order-item" }, + {"do":"expect","actor":"two","testid":"order-item","contains":"Desk Lamp","count":1}, + {"do":"expect","actor":"two","testid":"order-item","contains":"Coffee Grinder","absent":true}, + {"do":"click","actor":"two","testid":"overlay-close","ifAvailable":true}, + {"do":"click","actor":"two","testid":"current-user","unlessVisible":"signout"}, + {"do":"click","actor":"two","testid":"signout"}, + {"do":"waitUntilAbsent","actor":"two","testid":"current-user","within":6000}, + {"do":"signIn","actor":"two","name":"quin"}, + {"do":"expect","actor":"two","testid":"current-user","contains":"quin"}, + {"do":"click","actor":"two","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { "do": "click", "actor": "two", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": ["item-list", "order-item"] }, + {"do": "click", "actor": "two", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + {"do":"click","actor":"two","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"two","testid":"order-item","contains":"Coffee Grinder","count":1}, + {"do":"expect","actor":"two","testid":"order-item","contains":"Desk Lamp","absent":true}, + {"do":"click","actor":"two","testid":"overlay-close","ifAvailable":true}, + {"do":"click","actor":"two","testid":"catalog-link","ifAvailable":true}, + {"do":"dbRecordCheckout","account":"{user:quin}","item":"Bluetooth Speaker","as":"after-account-switch","storage":{"kind":"order-data","cart":false,"warehouses":false}}, + {"do":"dbRecordStock","item":"Bluetooth Speaker","as":"after-account-switch-stock"}, + {"do":"callAction","actor":"two","action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Bluetooth Speaker","attribute":"data-buy-input"},"settleMs":0}, + {"do":"expectActionOutcome","actor":"two","outcome":"accepted"}, + {"do":"dbExpectPurchase","actor":"two","before":"after-account-switch","stockBefore":"after-account-switch-stock"} + ], + "statedBy": "a customer sees only their own orders" + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-purchase-attribution.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-purchase-attribution.json new file mode 100644 index 00000000000..de2ade90f6f --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-purchase-attribution.json @@ -0,0 +1,53 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Purchases are attributed to whoever made them", + "note": "Refusal checks first prove the same valid route and input with an authorized actor. A missing or malformed endpoint therefore cannot earn authorization credit merely by returning an error.", + "features": [ + { + "id": 102, + "name": "Purchases are attributed to whoever made them", + "actors": ["victim","attacker"], + "setup": [{"do":"signUp","actor":"victim","name":"lee"},{"do":"signUp","actor":"attacker","name":"mel"}, + { "do": "click", "actor": "victim", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do":"click","actor":"victim","testid":"buy-now","in":{"testid":"item-card","contains":"Coffee Grinder"}}, + { "do": "click", "actor": "attacker", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }], + "criteria": [ + { + "id": "102a", + "category": "production", + "desc": "a direct purchase is attributed to the authenticated caller, not another account", + "points": 2, + "steps": [ + { + "do": "callAction", + "actor": "attacker", + "action": "buy", + "namedAction": { + "id": "buy", + "path": "/api/items/:id/buy", + "reducer": "buy_now", + "args": [0], + "params": [{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}] + }, + "input": {"testid":"item-card","contains":"Desk Lamp","attribute":"data-buy-input"}, + "settleMs": 2000 + }, + {"do":"expectActionOutcome","actor":"attacker","outcome":"accepted"}, + {"do":"click","actor":"victim","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do": "click", "actor": "victim", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + {"do":"click","actor":"victim","testid":"orders-toggle", "unlessVisible": "order-item" }, + {"do":"expect","actor":"victim","testid":"order-item","contains":"Coffee Grinder","count":1}, + {"do":"expect","actor":"victim","testid":"order-item","contains":"Desk Lamp","absent":true}, + {"do":"click","actor":"attacker","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do": "click", "actor": "attacker", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + {"do":"click","actor":"attacker","testid":"orders-toggle", "unlessVisible": "order-item" }, + {"do":"expect","actor":"attacker","testid":"order-item","contains":"Desk Lamp","count":1} + ], + "statedBy": "the authenticated caller, not a client-supplied identity, owns the order" + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-purchase-session.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-purchase-session.json new file mode 100644 index 00000000000..87f9873f8f3 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-purchase-session.json @@ -0,0 +1,784 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Purchase requires an account", + "note": "Draft: the purchase-session check includes bounded login-input and credential-tampering probes. The login probe replaces the password in one matched JSON form request with query-like text. Valid purchases must work before and after it. Stored order and stock reads must show no effect from a refused login or forgery. The read scope requires neither carts nor warehouses. Missing, ambiguous, repeated, redirected, non-JSON or incomplete login requests remain inconclusive. This proves one login string cannot grant purchase authority; it does not prove general injection safety. Unsupported or ambiguous credentials remain inconclusive, never a pass or an app defect. This does not test expiry or invalidation of previously copied tokens. After signout, the current browser credential must not authorize a purchase; sign-in must restore access. A bearer-only control must also succeed before cookies are omitted from the tampering probe. Cookie-only sessions still require one unambiguous cookie. Same-site and cross-site simple POSTs use the real browser cookie policy, without copied tokens. Complete opaque responses do not prove refusal: native state must remain unchanged, and valid purchases must still work. Local-network permission is granted only to the controlled origin because the appliance addresses are private.", + "features": [ + { + "id": 101, + "name": "Purchase requires an account", + "actors": [ + "buyer", + "guest", + "duplicate", + "query-input" + ], + "setup": [ + { + "do": "signUp", + "actor": "buyer", + "name": "kim" + }, + { + "do": "dbRecordStock", + "item": "Bluetooth Speaker", + "as": "speaker-before-control" + }, + { "do": "click", "actor": "buyer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { + "do": "callAction", + "actor": "buyer", + "action": "buy", + "namedAction": { + "id": "buy", + "path": "/api/items/:id/buy", + "reducer": "buy_now", + "args": [ + 0 + ], + "params": [ + { + "name": "itemId", + "in": "path", + "placeholder": ":id", + "wireType": "u64" + } + ] + }, + "input": { + "testid": "item-card", + "contains": "Bluetooth Speaker", + "attribute": "data-buy-input" + }, + "settleMs": 2000 + }, + { + "do": "expectActionOutcome", + "actor": "buyer", + "outcome": "accepted" + }, + { + "do": "dbExpectStock", + "item": "Bluetooth Speaker", + "relativeTo": "speaker-before-control", + "plus": -1 + } + ], + "criteria": [ + { + "id": "101a", + "category": "production", + "desc": "a valid direct purchase works for the buyer but is refused without a session, after ordinary or query-like rejected login or duplicate registration, and with a tampered session; an untrusted browser origin cannot purchase with ambient credentials; signing out removes this browser's purchase authority and signing in restores it", + "points": 2, + "steps": [ + { + "do": "dbRecordCheckout", + "account": "{user:kim}", + "item": "Bluetooth Speaker", + "as": "before-rejected-access", + "storage": { + "kind": "order-data", + "cart": false, + "warehouses": false + } + }, + { + "do": "callAction", + "actor": "guest", + "from": "buyer", + "action": "buy", + "namedAction": { + "id": "buy", + "path": "/api/items/:id/buy", + "reducer": "buy_now", + "args": [ + 0 + ], + "params": [ + { + "name": "itemId", + "in": "path", + "placeholder": ":id", + "wireType": "u64" + } + ] + }, + "input": { + "testid": "item-card", + "contains": "Bluetooth Speaker", + "attribute": "data-buy-input" + }, + "authentication": "none", + "settleMs": 2000 + }, + { + "do": "expectActionOutcome", + "actor": "guest", + "outcome": "application-refused", + "routeProvenBy": "buyer" + }, + { + "do": "dbExpectNoPurchase", + "before": "before-rejected-access" + }, + { + "do": "dbExpectStock", + "item": "Bluetooth Speaker", + "relativeTo": "speaker-before-control", + "plus": -1 + }, + { "do": "signIn", "actor": "query-input", "name": "kim", "deferAuthFailureToExpect": true }, + { "do": "expect", "actor": "query-input", "testid": "current-user", "contains": "{user:kim}", "within": 6000 }, + { "do": "click", "actor": "query-input", "testid": "current-user", "unlessVisible": "signout" }, + { "do": "click", "actor": "query-input", "testid": "signout" }, + { "do": "waitUntilAbsent", "actor": "query-input", "testid": "current-user", "within": 6000 }, + { + "do": "signIn", + "actor": "query-input", + "name": "kim", + "password": "not-kims-password", + "expectFailure": true + }, + { + "do": "expect", + "actor": "query-input", + "testid": "auth-error", + "within": 6000 + }, + { + "do": "reload", + "actor": "query-input", + "application": true, + "settleMs": 1000 + }, + { + "do": "expect", + "actor": "query-input", + "testid": "current-user", + "absent": true, + "within": 1000 + }, + { + "do": "callAction", + "actor": "query-input", + "from": "buyer", + "action": "buy", + "namedAction": { + "id": "buy", + "path": "/api/items/:id/buy", + "reducer": "buy_now", + "args": [ + 0 + ], + "params": [ + { + "name": "itemId", + "in": "path", + "placeholder": ":id", + "wireType": "u64" + } + ] + }, + "input": { + "testid": "item-card", + "contains": "Bluetooth Speaker", + "attribute": "data-buy-input" + }, + "authentication": "optional", + "settleMs": 2000 + }, + { + "do": "expectActionOutcome", + "actor": "query-input", + "outcome": "application-refused", + "routeProvenBy": "buyer" + }, + { + "do": "dbExpectNoPurchase", + "before": "before-rejected-access" + }, + { + "do": "dbExpectStock", + "item": "Bluetooth Speaker", + "relativeTo": "speaker-before-control", + "plus": -1 + }, + { + "do": "signIn", + "actor": "query-input", + "name": "kim", + "password": "not-kims-password", + "expectFailure": true, + "requestPatch": { + "password": "' OR '1'='1" + } + }, + { + "do": "callAction", + "actor": "query-input", + "from": "buyer", + "action": "buy", + "namedAction": { + "id": "buy", + "path": "/api/items/:id/buy", + "reducer": "buy_now", + "args": [ + 0 + ], + "params": [ + { + "name": "itemId", + "in": "path", + "placeholder": ":id", + "wireType": "u64" + } + ] + }, + "input": { + "testid": "item-card", + "contains": "Bluetooth Speaker", + "attribute": "data-buy-input" + }, + "authentication": "optional", + "settleMs": 2000 + }, + { + "do": "expectActionOutcome", + "actor": "query-input", + "outcome": "completed", + "routeProvenBy": "buyer" + }, + { + "do": "dbExpectNoPurchase", + "before": "before-rejected-access" + }, + { + "do": "dbExpectStock", + "item": "Bluetooth Speaker", + "relativeTo": "speaker-before-control", + "plus": -1 + }, + { + "do": "expectActionOutcome", + "actor": "query-input", + "outcome": "application-refused", + "routeProvenBy": "buyer" + }, + { + "do": "signUp", + "actor": "duplicate", + "name": "kim", + "password": "not-kims-password", + "expectFailure": true + }, + { + "do": "expect", + "actor": "duplicate", + "testid": "auth-error", + "within": 6000 + }, + { + "do": "reload", + "actor": "duplicate", + "application": true, + "settleMs": 1000 + }, + { + "do": "expect", + "actor": "duplicate", + "testid": "current-user", + "absent": true, + "within": 1000 + }, + { + "do": "callAction", + "actor": "duplicate", + "from": "buyer", + "action": "buy", + "namedAction": { + "id": "buy", + "path": "/api/items/:id/buy", + "reducer": "buy_now", + "args": [ + 0 + ], + "params": [ + { + "name": "itemId", + "in": "path", + "placeholder": ":id", + "wireType": "u64" + } + ] + }, + "input": { + "testid": "item-card", + "contains": "Bluetooth Speaker", + "attribute": "data-buy-input" + }, + "authentication": "optional", + "settleMs": 2000 + }, + { + "do": "expectActionOutcome", + "actor": "duplicate", + "outcome": "application-refused", + "routeProvenBy": "buyer" + }, + { + "do": "dbExpectNoPurchase", + "before": "before-rejected-access" + }, + { + "do": "dbExpectStock", + "item": "Bluetooth Speaker", + "relativeTo": "speaker-before-control", + "plus": -1 + }, + { + "do": "callAction", + "actor": "buyer", + "action": "buy", + "namedAction": { + "id": "buy", + "path": "/api/items/:id/buy", + "reducer": "buy_now", + "args": [ + 0 + ], + "params": [ + { + "name": "itemId", + "in": "path", + "placeholder": ":id", + "wireType": "u64" + } + ] + }, + "input": { + "testid": "item-card", + "contains": "Bluetooth Speaker", + "attribute": "data-buy-input" + }, + "settleMs": 2000, + "authentication": "session-control" + }, + { + "do": "expectActionOutcome", + "actor": "buyer", + "outcome": "accepted" + }, + { + "do": "dbExpectStock", + "item": "Bluetooth Speaker", + "relativeTo": "speaker-before-control", + "plus": -2 + }, + { + "do": "dbRecordCheckout", + "account": "{user:kim}", + "item": "Bluetooth Speaker", + "as": "before-forgery", + "storage": { + "kind": "order-data", + "cart": false, + "warehouses": false + } + }, + { + "do": "callAction", + "actor": "buyer", + "action": "buy", + "namedAction": { + "id": "buy", + "path": "/api/items/:id/buy", + "reducer": "buy_now", + "args": [ + 0 + ], + "params": [ + { + "name": "itemId", + "in": "path", + "placeholder": ":id", + "wireType": "u64" + } + ] + }, + "input": { + "testid": "item-card", + "contains": "Bluetooth Speaker", + "attribute": "data-buy-input" + }, + "settleMs": 1000, + "authentication": "tampered-session" + }, + { + "do": "expectActionOutcome", + "actor": "buyer", + "outcome": "completed" + }, + { + "do": "dbExpectNoPurchase", + "before": "before-forgery" + }, + { + "do": "dbExpectStock", + "item": "Bluetooth Speaker", + "relativeTo": "speaker-before-control", + "plus": -2 + }, + { + "do": "expectActionOutcome", + "actor": "buyer", + "outcome": "application-refused" + }, + { + "do": "callAction", + "actor": "buyer", + "action": "buy", + "namedAction": { + "id": "buy", + "path": "/api/items/:id/buy", + "reducer": "buy_now", + "args": [ + 0 + ], + "params": [ + { + "name": "itemId", + "in": "path", + "placeholder": ":id", + "wireType": "u64" + } + ] + }, + "input": { + "testid": "item-card", + "contains": "Bluetooth Speaker", + "attribute": "data-buy-input" + }, + "settleMs": 0 + }, + { + "do": "expectActionOutcome", + "actor": "buyer", + "outcome": "accepted" + }, + { + "do": "dbExpectStock", + "item": "Bluetooth Speaker", + "relativeTo": "speaker-before-control", + "plus": -3 + }, + { + "do": "dbRecordCheckout", + "account": "{user:kim}", + "item": "Bluetooth Speaker", + "as": "before-same-site", + "storage": { + "kind": "order-data", + "cart": false, + "warehouses": false + } + }, + { + "do": "callAction", + "actor": "buyer", + "action": "buy", + "namedAction": { + "id": "buy", + "path": "/api/items/:id/buy", + "reducer": "buy_now", + "args": [ + 0 + ], + "params": [ + { + "name": "itemId", + "in": "path", + "placeholder": ":id", + "wireType": "u64" + } + ] + }, + "input": { + "testid": "item-card", + "contains": "Bluetooth Speaker", + "attribute": "data-buy-input" + }, + "settleMs": 1000, + "browserOrigin": "same-site" + }, + { + "do": "dbExpectNoPurchase", + "before": "before-same-site" + }, + { + "do": "dbExpectStock", + "item": "Bluetooth Speaker", + "relativeTo": "speaker-before-control", + "plus": -3 + }, + { + "do": "callAction", + "actor": "buyer", + "action": "buy", + "namedAction": { + "id": "buy", + "path": "/api/items/:id/buy", + "reducer": "buy_now", + "args": [ + 0 + ], + "params": [ + { + "name": "itemId", + "in": "path", + "placeholder": ":id", + "wireType": "u64" + } + ] + }, + "input": { + "testid": "item-card", + "contains": "Bluetooth Speaker", + "attribute": "data-buy-input" + }, + "settleMs": 0 + }, + { + "do": "expectActionOutcome", + "actor": "buyer", + "outcome": "accepted" + }, + { + "do": "dbExpectStock", + "item": "Bluetooth Speaker", + "relativeTo": "speaker-before-control", + "plus": -4 + }, + { + "do": "dbRecordCheckout", + "account": "{user:kim}", + "item": "Bluetooth Speaker", + "as": "before-cross-site", + "storage": { + "kind": "order-data", + "cart": false, + "warehouses": false + } + }, + { + "do": "callAction", + "actor": "buyer", + "action": "buy", + "namedAction": { + "id": "buy", + "path": "/api/items/:id/buy", + "reducer": "buy_now", + "args": [ + 0 + ], + "params": [ + { + "name": "itemId", + "in": "path", + "placeholder": ":id", + "wireType": "u64" + } + ] + }, + "input": { + "testid": "item-card", + "contains": "Bluetooth Speaker", + "attribute": "data-buy-input" + }, + "settleMs": 1000, + "browserOrigin": "cross-site" + }, + { + "do": "dbExpectNoPurchase", + "before": "before-cross-site" + }, + { + "do": "dbExpectStock", + "item": "Bluetooth Speaker", + "relativeTo": "speaker-before-control", + "plus": -4 + }, + { + "do": "callAction", + "actor": "buyer", + "action": "buy", + "namedAction": { + "id": "buy", + "path": "/api/items/:id/buy", + "reducer": "buy_now", + "args": [ + 0 + ], + "params": [ + { + "name": "itemId", + "in": "path", + "placeholder": ":id", + "wireType": "u64" + } + ] + }, + "input": { + "testid": "item-card", + "contains": "Bluetooth Speaker", + "attribute": "data-buy-input" + }, + "settleMs": 0 + }, + { + "do": "expectActionOutcome", + "actor": "buyer", + "outcome": "accepted" + }, + { + "do": "dbExpectStock", + "item": "Bluetooth Speaker", + "relativeTo": "speaker-before-control", + "plus": -5 + }, + { + "do": "dbRecordCheckout", + "account": "{user:kim}", + "item": "Bluetooth Speaker", + "as": "before-signout", + "storage": { + "kind": "order-data", + "cart": false, + "warehouses": false + } + }, + { + "do": "click", + "actor": "buyer", + "testid": "current-user", + "unlessVisible": "signout" + }, + { + "do": "click", + "actor": "buyer", + "testid": "signout" + }, + { + "do": "waitUntilAbsent", + "actor": "buyer", + "testid": "current-user", + "within": 6000 + }, + { "do": "click", "actor": "buyer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { + "do": "callAction", + "actor": "buyer", + "action": "buy", + "namedAction": { + "id": "buy", + "path": "/api/items/:id/buy", + "reducer": "buy_now", + "args": [ + 0 + ], + "params": [ + { + "name": "itemId", + "in": "path", + "placeholder": ":id", + "wireType": "u64" + } + ] + }, + "input": { + "testid": "item-card", + "contains": "Bluetooth Speaker", + "attribute": "data-buy-input" + }, + "settleMs": 2000, + "authentication": "optional" + }, + { + "do": "expectActionOutcome", + "actor": "buyer", + "outcome": "completed" + }, + { + "do": "dbExpectNoPurchase", + "before": "before-signout" + }, + { + "do": "dbExpectStock", + "item": "Bluetooth Speaker", + "relativeTo": "speaker-before-control", + "plus": -5 + }, + { + "do": "expectActionOutcome", + "actor": "buyer", + "outcome": "application-refused" + }, + { + "do": "signIn", + "actor": "buyer", + "name": "kim", + "awaitSignedIn": false + }, + { + "do": "expect", + "actor": "buyer", + "testid": "current-user", + "contains": "kim", + "within": 10000 + }, + { "do": "click", "actor": "buyer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { + "do": "callAction", + "actor": "buyer", + "action": "buy", + "namedAction": { + "id": "buy", + "path": "/api/items/:id/buy", + "reducer": "buy_now", + "args": [ + 0 + ], + "params": [ + { + "name": "itemId", + "in": "path", + "placeholder": ":id", + "wireType": "u64" + } + ] + }, + "input": { + "testid": "item-card", + "contains": "Bluetooth Speaker", + "attribute": "data-buy-input" + }, + "settleMs": 2000 + }, + { + "do": "expectActionOutcome", + "actor": "buyer", + "outcome": "accepted" + }, + { + "do": "dbExpectStock", + "item": "Bluetooth Speaker", + "relativeTo": "speaker-before-control", + "plus": -6 + } + ], + "statedBy": "the server refuses a purchase without an authenticated customer" + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-restock-race.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-restock-race.json new file mode 100644 index 00000000000..c91449a9e12 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-restock-race.json @@ -0,0 +1,555 @@ +{ + "schemaVersion": 1, + "level": 1, + "track": "ecommerce", + "name": "Storefront and warehouse — restock during purchases", + "writeUrlPattern": "\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses", + "note": "Draft: retain the UI race, then reconcile a bounded named-action race against native orders and warehouse allocations. Both phases require matching reference and defect-control qualification.", + "features": [ + { + "id": 202, + "name": "A restock during a rush is not lost", + "actors": [ + "admin", + "a", + "b", + "c" + ], + "setup": [ + { + "do": "signIn", + "actor": "admin", + "name": "admin", + "password": "stackbench-admin-2026", + "exact": true + }, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + { + "do": "click", + "actor": "admin", + "testid": "admin-link", + "unlessVisible": "admin-panel" + }, + { + "do": "signUp", + "actor": "a", + "name": "r1" + }, + { + "do": "signUp", + "actor": "b", + "name": "r2" + }, + { + "do": "signUp", + "actor": "c", + "name": "r3" + }, + { + "do": "dbRecordStock", + "item": "Bluetooth Speaker", + "as": "stored-before-serial-purchase" + }, + { "do": "click", "actor": "a", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { + "do": "click", + "actor": "a", + "testid": "buy-now", + "in": { + "testid": "item-card", + "contains": "Bluetooth Speaker" + } + }, + { + "do": "click", + "actor": "a", + "testid": "overlay-close", + "ifAvailable": true, + "unlessVisible": "order-item", + "within": 1000 + }, + {"do": "click", "actor": "a", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + { + "do": "click", + "actor": "a", + "testid": "orders-toggle", + "unlessVisible": "order-item" + }, + { + "do": "expect", + "actor": "a", + "testid": "order-item", + "contains": "Bluetooth Speaker", + "count": 1, + "within": 10000 + }, + { + "do": "dbExpectStock", + "item": "Bluetooth Speaker", + "relativeTo": "stored-before-serial-purchase", + "plus": -1 + }, + { + "do": "reload", + "actor": "a", + "settleMs": 1000 + }, + { + "do": "ensureSignedIn", + "actor": "a", + "name": "r1", + "readyTestid": "current-user" + }, + { + "do": "reload", + "actor": "admin", + "settleMs": 1000 + }, + { + "do": "ensureSignedIn", + "actor": "admin", + "name": "admin", + "password": "stackbench-admin-2026", + "exact": true, + "readyTestid": "current-user" + }, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + { + "do": "click", + "actor": "admin", + "testid": "admin-link", + "unlessVisible": "admin-panel" + }, + { + "do": "expectNumber", + "actor": "admin", + "testid": "admin-stock", + "in": { + "testid": "admin-item-row", + "contains": "Bluetooth Speaker" + }, + "relativeTo": "stored-before-serial-purchase", + "plus": -1, + "within": 10000 + }, + { + "do": "dbRecordStock", + "item": "Bluetooth Speaker", + "as": "stored-before-control" + }, + { + "do": "recordNumber", + "actor": "admin", + "testid": "admin-location-qty", + "in": { + "testid": "admin-location-row", + "contains": "Bluetooth Speaker" + }, + "as": "warehouse-before" + }, + { + "do": "click", + "actor": "a", + "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" + }, + { + "do": "recordNumber", + "actor": "a", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Bluetooth Speaker" + }, + "as": "storefront-before" + }, + { + "do": "fill", + "actor": "admin", + "testid": "restock-input", + "in": { + "testid": "admin-location-row", + "contains": "Bluetooth Speaker" + }, + "text": "5" + }, + { + "do": "click", + "actor": "admin", + "testid": "restock-submit", + "in": { + "testid": "admin-location-row", + "contains": "Bluetooth Speaker" + } + }, + { + "do": "dbExpectStock", + "item": "Bluetooth Speaker", + "relativeTo": "stored-before-control", + "plus": 5, + "within": 8000 + }, + { + "do": "reload", + "actor": "admin", + "settleMs": 1000 + }, + { + "do": "ensureSignedIn", + "actor": "admin", + "name": "admin", + "password": "stackbench-admin-2026", + "exact": true, + "readyTestid": "current-user" + }, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + { + "do": "click", + "actor": "admin", + "testid": "admin-link", + "unlessVisible": "admin-panel" + }, + { + "do": "expectNumber", + "actor": "admin", + "testid": "admin-location-qty", + "in": { + "testid": "admin-location-row", + "contains": "Bluetooth Speaker" + }, + "relativeTo": "warehouse-before", + "plus": 5, + "within": 8000 + }, + { + "do": "reload", + "actor": "a", + "settleMs": 1000 + }, + { + "do": "click", + "actor": "a", + "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" + }, + { + "do": "expectNumber", + "actor": "a", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Bluetooth Speaker" + }, + "relativeTo": "storefront-before", + "plus": 5, + "within": 8000 + }, + { + "do": "dbExpectStock", + "item": "Bluetooth Speaker", + "relativeTo": "stored-before-control", + "plus": 5 + }, + { "do": "click", "actor": "b", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "click", "actor": "c", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "202-control", + "category": "feature", + "points": 0, + "withheld": "The ordinary restock is a setup prerequisite retained when only the race is selected. This zero-point control verifies that setup; it does not add product credit.", + "desc": "an uncontended restock of five is stored by the server and shows on the storefront", + "steps": [ + { + "do": "dbExpectStock", + "item": "Bluetooth Speaker", + "relativeTo": "stored-before-control", + "plus": 5 + }, + { + "do": "expectNumber", + "actor": "a", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Bluetooth Speaker" + }, + "relativeTo": "storefront-before", + "plus": 5, + "within": 8000 + } + ] + }, + { + "id": "202a", + "category": "production", + "points": 1, + "desc": "restocking during purchases preserves stock, complete buyer orders and their warehouse allocations", + "steps": [ + { + "do": "dbRecordStock", + "item": "Bluetooth Speaker", + "as": "stored-before-rush" + }, + { + "do": "fill", + "actor": "admin", + "testid": "restock-input", + "in": { + "testid": "admin-location-row", + "contains": "Bluetooth Speaker" + }, + "text": "5" + }, + { + "do": "recordNumber", + "actor": "a", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Bluetooth Speaker" + }, + "as": "rush-before" + }, + { + "do": "race", + "branches": [ + [ + { + "do": "clickConcurrently", + "actors": [ + "a", + "b", + "c" + ], + "testid": "buy-now", + "in": { + "testid": "item-card", + "contains": "Bluetooth Speaker" + }, + "settleMs": 500 + } + ], + [ + { + "do": "click", + "actor": "admin", + "testid": "restock-submit", + "in": { + "testid": "admin-location-row", + "contains": "Bluetooth Speaker" + } + } + ] + ], + "settleMs": 0 + }, + { + "do": "dbExpectStock", + "item": "Bluetooth Speaker", + "relativeTo": "stored-before-rush", + "within": 10000, + "plus": 2 + }, + { "do": "dbExpectStock", "item": "Bluetooth Speaker", "warehouse": "East", "atLeast": 0 }, + { "do": "dbExpectStock", "item": "Bluetooth Speaker", "warehouse": "West", "atLeast": 0 }, + { + "do": "reload", + "actor": "a", + "settleMs": 1000 + }, + { + "do": "click", + "actor": "a", + "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" + }, + { + "do": "expectNumber", + "actor": "a", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Bluetooth Speaker" + }, + "relativeTo": "rush-before", + "plus": 2, + "within": 10000 + }, + { + "do": "ensureSignedIn", + "actor": "a", + "name": "r1", + "readyTestid": "current-user" + }, + { + "do": "click", + "actor": "a", + "testid": "overlay-close", + "ifAvailable": true, + "unlessVisible": "order-item", + "within": 1000 + }, + { "do": "click", "actor": "a", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": ["item-list", "order-item"] }, + {"do": "click", "actor": "a", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + { + "do": "click", + "actor": "a", + "testid": "orders-toggle", + "unlessVisible": "order-item" + }, + { + "do": "expect", + "actor": "a", + "testid": "order-item", + "contains": "Bluetooth Speaker", + "count": 2, + "within": 10000 + }, + { + "do": "reload", + "actor": "b", + "settleMs": 1000 + }, + { + "do": "ensureSignedIn", + "actor": "b", + "name": "r2", + "readyTestid": "current-user" + }, + { + "do": "click", + "actor": "b", + "testid": "overlay-close", + "ifAvailable": true, + "unlessVisible": "order-item", + "within": 1000 + }, + { "do": "click", "actor": "b", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": ["item-list", "order-item"] }, + {"do": "click", "actor": "b", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + { + "do": "click", + "actor": "b", + "testid": "orders-toggle", + "unlessVisible": "order-item" + }, + { + "do": "expect", + "actor": "b", + "testid": "order-item", + "contains": "Bluetooth Speaker", + "count": 1, + "within": 10000 + }, + { + "do": "reload", + "actor": "c", + "settleMs": 1000 + }, + { + "do": "ensureSignedIn", + "actor": "c", + "name": "r3", + "readyTestid": "current-user" + }, + { + "do": "click", + "actor": "c", + "testid": "overlay-close", + "ifAvailable": true, + "unlessVisible": "order-item", + "within": 1000 + }, + { "do": "click", "actor": "c", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": ["item-list", "order-item"] }, + {"do": "click", "actor": "c", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + { + "do": "click", + "actor": "c", + "testid": "orders-toggle", + "unlessVisible": "order-item" + }, + { + "do": "expect", + "actor": "c", + "testid": "order-item", + "contains": "Bluetooth Speaker", + "count": 1, + "within": 10000 + }, + { + "do": "reload", + "actor": "admin", + "settleMs": 1000 + }, + { + "do": "ensureSignedIn", + "actor": "admin", + "name": "admin", + "password": "stackbench-admin-2026", + "exact": true, + "readyTestid": "current-user" + }, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + { + "do": "click", + "actor": "admin", + "testid": "admin-link", + "ifAvailable": true, + "unlessVisible": "admin-panel" + }, + { + "do": "expectNumber", + "actor": "admin", + "testid": "admin-stock", + "in": { + "testid": "admin-item-row", + "contains": "Bluetooth Speaker" + }, + "relativeTo": "stored-before-rush", + "plus": 2, + "within": 10000 + }, + { + "do": "reload", + "actor": "a", + "settleMs": 1000 + }, + { + "do": "click", + "actor": "a", + "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" + }, + { + "do": "expectNumber", + "actor": "a", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Bluetooth Speaker" + }, + "relativeTo": "stored-before-rush", + "plus": 2, + "within": 10000 + }, + { "do": "ensureSignedIn", "actor": "a", "name": "r1", "readyTestid": "current-user" }, + { "do": "dbRecordCheckout", "account": "{user:r1}", "item": "Bluetooth Speaker", "as": "mixed-a", "storage": { "kind": "order-data", "cart": false, "warehouses": true } }, + { "do": "dbRecordCheckout", "account": "{user:r2}", "item": "Bluetooth Speaker", "as": "mixed-b", "storage": { "kind": "order-data", "cart": false, "warehouses": true } }, + { "do": "dbRecordCheckout", "account": "{user:r3}", "item": "Bluetooth Speaker", "as": "mixed-c", "storage": { "kind": "order-data", "cart": false, "warehouses": true } }, + { "do": "click", "actor": "a", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { + "do": "callConcurrently", "action": "buy", "actors": ["a", "b", "c"], "from": "a", "requests": 3, "settleMs": 3000, + "namedAction": { "id": "buy", "path": "/api/items/:id/buy", "reducer": "buy_now", "args": [0], "params": [{ "name": "itemId", "in": "path", "placeholder": ":id", "wireType": "u64" }] }, + "input": { "testid": "item-card", "contains": "Bluetooth Speaker", "attribute": "data-buy-input" }, + "alongside": [{ + "action": "restock", "actors": ["admin"], "requests": 1, + "namedAction": { "id": "restock", "path": "/api/admin/restock", "reducer": "admin_restock", "args": [0, 0, 1], "params": [{ "name": "itemId", "in": "body", "wireType": "u64" }, { "name": "warehouseId", "in": "body", "wireType": "u64" }, { "name": "quantity", "in": "body" }] }, + "input": { "testid": "admin-location-row", "contains": "Bluetooth Speaker", "attribute": "data-restock-input" } + }] + }, + { "do": "expectCallOutcomes", "accepted": 4 }, + { "do": "dbExpectPurchases", "before": { "a": "mixed-a", "b": "mixed-b", "c": "mixed-c" }, "purchases": 3 } + ], + "statedBy": "a restock in one warehouse raises the storefront number live" + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-review-eligibility.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-review-eligibility.json new file mode 100644 index 00000000000..d34d1110c5e --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-review-eligibility.json @@ -0,0 +1,72 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "A review is a claim about a purchase", + "writeUrlPattern": "\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses", + "note": "Every criterion receives all state it relies on from its feature setup. Security assertions include a working positive path where an empty or broken view could otherwise look secure.", + "features": [ + { + "id": 108, + "name": "A review is a claim about a purchase", + "actors": ["owner", "stranger"], + "setup": [ + { "do": "signUp", "actor": "owner", "name": "tam" }, + { "do": "signUp", "actor": "stranger", "name": "uma" }, + { "do": "click", "actor": "owner", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "click", "actor": "stranger", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "108a", + "category": "production", + "desc": "someone who never bought the item cannot review it", + "points": 2, + "steps": [ + { "do": "callAction", "actor": "owner", "action": "buy", "namedAction": { "id": "buy", "path": "/api/items/:id/buy", "reducer": "buy_now", "args": [0], "params": [{ "name": "itemId", "in": "path", "placeholder": ":id", "wireType": "u64" }] }, "input": { "testid": "item-card", "contains": "Air Purifier", "attribute": "data-buy-input" } }, + + { "do": "expectActionOutcome", "actor": "owner", "outcome": "accepted" }, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do": "click", "actor": "owner", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + { "do": "click", "actor": "owner", "testid": "orders-toggle", "unlessVisible": "order-item" }, + { "do": "expect", "actor": "owner", "testid": "order-item", "contains": "Air Purifier" }, + { "do": "click", "actor": "owner", "testid": "overlay-close", "ifAvailable": true }, + { "do": "click", "actor": "owner", "testid": "catalog-link", "ifAvailable": true }, + { "do": "callAction", "actor": "owner", "from": "owner", "action": "submitReview", "namedAction": { "id": "submitReview", "path": "/api/items/:id/reviews", "method": "POST", "reducer": "submit_review", "args": [0, 4, "eligible review control"], "params": [{ "name": "itemId", "in": "path", "placeholder": ":id", "wireType": "u64" }, { "name": "rating", "in": "body" }, { "name": "comment", "in": "body" }] }, "input": { "testid": "item-card", "contains": "Air Purifier", "attribute": "data-buy-input", "fields": ["itemId"] }, "settleMs": 1500 }, + { "do": "expectActionOutcome", "actor": "owner", "outcome": "accepted" }, + { "do": "openItem", "actor": "stranger", "item": "Air Purifier" }, + { "do": "expect", "actor": "stranger", "testid": "review-item", "contains": "eligible review control" }, + { "do": "callAction", "actor": "stranger", "from": "owner", "action": "submitReview", "namedAction": { "id": "submitReview", "path": "/api/items/:id/reviews", "method": "POST", "reducer": "submit_review", "args": [0, 4, "never bought this"], "params": [{ "name": "itemId", "in": "path", "placeholder": ":id", "wireType": "u64" }, { "name": "rating", "in": "body" }, { "name": "comment", "in": "body" }] }, "input": { "testid": "item-card", "contains": "Air Purifier", "attribute": "data-buy-input", "fields": ["itemId"] }, "settleMs": 1500 }, + { "do": "expectActionOutcome", "actor": "stranger", "outcome": "application-refused", "routeProvenBy": "owner" }, + { "do": "reload", "actor": "stranger", "settleMs": 1500 }, + { "do": "ensureSignedIn", "actor": "stranger", "name": "uma", "readyTestid": "current-user" }, + { "do": "click", "actor": "stranger", "testid": "overlay-close", "ifAvailable": true }, + { "do": "click", "actor": "stranger", "testid": "catalog-link", "ifAvailable": true }, + { "do": "openItem", "actor": "stranger", "item": "Air Purifier" }, + { "do": "expect", "actor": "stranger", "testid": "review-item", "contains": "eligible review control" }, + { "do": "expect", "actor": "stranger", "testid": "review-item", "contains": "never bought this", "absent": true } + ], + "statedBy": "the server refuses a review from someone who has never ordered the item" + }, + { + "id": "108b", + "category": "production", + "desc": "buying the item earns the right to review it", + "points": 1, + "steps": [ + { "do": "callAction", "actor": "owner", "action": "buy", "namedAction": { "id": "buy", "path": "/api/items/:id/buy", "reducer": "buy_now", "args": [0], "params": [{ "name": "itemId", "in": "path", "placeholder": ":id", "wireType": "u64" }] }, "input": { "testid": "item-card", "contains": "Keyboard", "attribute": "data-buy-input" } }, + + { "do": "expectActionOutcome", "actor": "owner", "outcome": "accepted" }, + { "do": "openItem", "actor": "owner", "item": "Keyboard" }, + {"do":"click","actor":"owner","testid":"review-toggle","unlessVisible":"review-rating","ifAvailable":true}, + { "do": "fill", "actor": "owner", "testid": "review-rating", "text": "4" }, + { "do": "fill", "actor": "owner", "testid": "review-input", "text": "bought and used it" }, + { "do": "click", "actor": "owner", "testid": "review-submit" }, + { "do": "expect", "actor": "owner", "testid": "review-item", "contains": "bought and used it", "within": 8000 } + ], + "statedBy": "a customer can review an item they bought" + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-review-rating-live.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-review-rating-live.json new file mode 100644 index 00000000000..f2eb7b9f708 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-review-rating-live.json @@ -0,0 +1,62 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Live review rating", + "note": "This scenario uses two different customers and does not depend on the outcome of the separate repeat-submission check.", + "features": [ + { + "id": 6, + "name": "Reviews", + "actors": ["author", "other"], + "setup": [ + { "do": "signUp", "actor": "author", "name": "leon" }, + { "do": "signUp", "actor": "other", "name": "maya" }, + { "do": "click", "actor": "author", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "callAction", "actor": "author", "action": "buy", "namedAction": { "id": "buy", "path": "/api/items/:id/buy", "reducer": "buy_now", "args": [0], "params": [{ "name": "itemId", "in": "path", "placeholder": ":id", "wireType": "u64" }] }, "input": { "testid": "item-card", "contains": "Gaming Mouse", "attribute": "data-buy-input" } }, + + { "do": "expectActionOutcome", "actor": "author", "outcome": "accepted" }, + {"do":"click","actor":"author","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do": "click", "actor": "author", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + { "do": "click", "actor": "author", "testid": "orders-toggle", "unlessVisible": "order-item" }, + { "do": "expect", "actor": "author", "testid": "order-item", "contains": "Gaming Mouse", "within": 10000 }, + { "do": "click", "actor": "author", "testid": "overlay-close", "ifAvailable": true }, + { "do": "click", "actor": "author", "testid": "catalog-link", "ifAvailable": true }, + { "do": "openItem", "actor": "author", "item": "Gaming Mouse" }, + {"do":"click","actor":"author","testid":"review-toggle","unlessVisible":"review-rating","ifAvailable":true}, + { "do": "fill", "actor": "author", "testid": "review-rating", "text": "2" }, + { "do": "fill", "actor": "author", "testid": "review-input", "text": "works for travel" }, + { "do": "click", "actor": "author", "testid": "review-submit" }, + { "do": "expect", "actor": "author", "testid": "review-item", "contains": "works for travel" }, + { "do": "click", "actor": "other", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "callAction", "actor": "other", "action": "buy", "namedAction": { "id": "buy", "path": "/api/items/:id/buy", "reducer": "buy_now", "args": [0], "params": [{ "name": "itemId", "in": "path", "placeholder": ":id", "wireType": "u64" }] }, "input": { "testid": "item-card", "contains": "Gaming Mouse", "attribute": "data-buy-input" } }, + + { "do": "expectActionOutcome", "actor": "other", "outcome": "accepted" }, + {"do":"click","actor":"other","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do": "click", "actor": "other", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + { "do": "click", "actor": "other", "testid": "orders-toggle", "unlessVisible": "order-item" }, + { "do": "expect", "actor": "other", "testid": "order-item", "contains": "Gaming Mouse", "within": 10000 }, + { "do": "click", "actor": "other", "testid": "overlay-close", "ifAvailable": true }, + { "do": "click", "actor": "other", "testid": "catalog-link", "ifAvailable": true } + ], + "criteria": [ + { + "id": "6c", + "category": "production", + "desc": "the average rating reflects both reviewers and updates live", + "points": 1, + "steps": [ + { "do": "openItem", "actor": "other", "item": "Gaming Mouse" }, + {"do":"click","actor":"other","testid":"review-toggle","unlessVisible":"review-rating","ifAvailable":true}, + { "do": "fill", "actor": "other", "testid": "review-rating", "text": "4" }, + { "do": "fill", "actor": "other", "testid": "review-input", "text": "does the job" }, + { "do": "click", "actor": "other", "testid": "review-submit" }, + { "do": "expectNumber", "actor": "other", "testid": "review-average", "equals": 3, "within": 10000 }, + { "do": "expectAgreement", "actors": ["author", "other"], "testid": "review-average", "numeric": true, "within": 10000 } + ], + "statedBy": "each item shows its average rating, which updates live as reviews arrive" + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-review-uniqueness.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-review-uniqueness.json new file mode 100644 index 00000000000..6cb3c8f8e2b --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-review-uniqueness.json @@ -0,0 +1,54 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Review uniqueness", + "note": "The check measures the product invariant only: a later submission may replace the review or be refused, but it must not create a second review.", + "features": [ + { + "id": 6, + "name": "Reviews", + "actors": ["author"], + "setup": [ + { "do": "signUp", "actor": "author", "name": "kira" }, + { "do": "click", "actor": "author", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "callAction", "actor": "author", "action": "buy", "namedAction": { "id": "buy", "path": "/api/items/:id/buy", "reducer": "buy_now", "args": [0], "params": [{ "name": "itemId", "in": "path", "placeholder": ":id", "wireType": "u64" }] }, "input": { "testid": "item-card", "contains": "Air Purifier", "attribute": "data-buy-input" } }, + + { "do": "expectActionOutcome", "actor": "author", "outcome": "accepted" }, + {"do":"click","actor":"author","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do": "click", "actor": "author", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + { "do": "click", "actor": "author", "testid": "orders-toggle", "unlessVisible": "order-item" }, + { "do": "expect", "actor": "author", "testid": "order-item", "contains": "Air Purifier", "within": 10000 }, + { "do": "click", "actor": "author", "testid": "overlay-close", "ifAvailable": true }, + { "do": "click", "actor": "author", "testid": "catalog-link", "ifAvailable": true }, + { "do": "openItem", "actor": "author", "item": "Air Purifier" }, + {"do":"click","actor":"author","testid":"review-toggle","unlessVisible":"review-rating","ifAvailable":true}, + { "do": "fill", "actor": "author", "testid": "review-rating", "text": "4" }, + { "do": "fill", "actor": "author", "testid": "review-input", "text": "quiet and effective" }, + { "do": "click", "actor": "author", "testid": "review-submit" }, + { "do": "expect", "actor": "author", "testid": "review-item", "contains": "quiet and effective" } + ], + "criteria": [ + { + "id": "6b", + "category": "production", + "desc": "a later review submission does not create a duplicate for the same customer and item", + "points": 1, + "steps": [ + { "do": "click", "actor": "author", "testid": "overlay-close", "ifAvailable": true }, + { "do": "click", "actor": "author", "testid": "catalog-link", "ifAvailable": true }, + { "do": "callAction", "actor": "author", "from": "author", "action": "submitReview", "namedAction": { "id": "submitReview", "path": "/api/items/:id/reviews", "method": "POST", "reducer": "submit_review", "args": [0, 4, "changed my mind"], "params": [{ "name": "itemId", "in": "path", "placeholder": ":id", "wireType": "u64" }, { "name": "rating", "in": "body" }, { "name": "comment", "in": "body" }] }, "input": { "testid": "item-card", "contains": "Air Purifier", "attribute": "data-buy-input", "fields": ["itemId"] }, "settleMs": 1500 }, + { "do": "expectActionOutcome", "actor": "author", "outcome": "completed" }, + { "do": "reload", "actor": "author", "settleMs": 2500 }, + { "do": "ensureSignedIn", "actor": "author", "name": "kira", "readyTestid": "current-user" }, + { "do": "click", "actor": "author", "testid": "overlay-close", "ifAvailable": true }, + { "do": "click", "actor": "author", "testid": "catalog-link", "ifAvailable": true }, + { "do": "openItem", "actor": "author", "item": "Air Purifier" }, + { "do": "expect", "actor": "author", "testid": "review-item", "count": 1, "within": 10000 } + ], + "statedBy": "one customer has at most one review per item" + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-review-visibility.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-review-visibility.json new file mode 100644 index 00000000000..ae482ac0585 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-review-visibility.json @@ -0,0 +1,45 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Review visibility", + "note": "This scenario owns the complete purchase, submission, and signed-out observation needed by 6a, so it is independent of the uniqueness and average-rating checks.", + "features": [ + { + "id": 6, + "name": "Reviews", + "actors": ["author", "visitor"], + "setup": [ + { "do": "signUp", "actor": "author", "name": "hal" }, + { "do": "click", "actor": "author", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "click", "actor": "author", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Induction Cooktop" } }, + {"do":"click","actor":"author","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do": "click", "actor": "author", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + { "do": "click", "actor": "author", "testid": "orders-toggle", "unlessVisible": "order-item" }, + { "do": "expect", "actor": "author", "testid": "order-item", "contains": "Induction Cooktop", "within": 10000 }, + { "do": "click", "actor": "author", "testid": "overlay-close", "ifAvailable": true, "within": 1000 }, + { "do": "click", "actor": "author", "testid": "catalog-link", "ifAvailable": true }, + { "do": "click", "actor": "visitor", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "6a", + "category": "feature", + "desc": "a customer can review an item and everyone sees it, signed out included", + "points": 2, + "steps": [ + { "do": "openItem", "actor": "author", "item": "Induction Cooktop" }, + {"do":"click","actor":"author","testid":"review-toggle","unlessVisible":"review-rating","ifAvailable":true}, + { "do": "fill", "actor": "author", "testid": "review-rating", "text": "4" }, + { "do": "fill", "actor": "author", "testid": "review-input", "text": "solid mold" }, + { "do": "click", "actor": "author", "testid": "review-submit" }, + { "do": "expect", "actor": "author", "testid": "review-item", "contains": "solid mold" }, + { "do": "openItem", "actor": "visitor", "item": "Induction Cooktop" }, + { "do": "expect", "actor": "visitor", "testid": "review-item", "contains": "solid mold", "within": 10000 } + ], + "statedBy": "reviews are visible to everyone, including signed-out visitors" + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-server-price.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-server-price.json new file mode 100644 index 00000000000..d1e136d6cbe --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-server-price.json @@ -0,0 +1,277 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "The price is the store's to set", + "note": "Draft: normal purchases of two differently priced items plus a client-supplied price probe. The app may reject the extra argument without effects or ignore it and book the stored price. Stored orders, line prices and total stock are reconciled against the completed response. Normal purchases must work before and after. This tests one price field, not all checkout or mass-assignment variants.", + "features": [ + { + "id": 104, + "name": "The price is the store's to set", + "actors": [ + "buyer" + ], + "setup": [ + { + "do": "signUp", + "actor": "buyer", + "name": "oli" + }, + { "do": "click", "actor": "buyer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "104a", + "category": "production", + "desc": "direct purchases use the stored price; a submitted price cannot change booked totals or leave effects after refusal", + "points": 2, + "steps": [ + { + "do": "callAction", + "actor": "buyer", + "action": "buy", + "namedAction": { + "id": "buy", + "path": "/api/items/:id/buy", + "reducer": "buy_now", + "args": [ + 0 + ], + "params": [ + { + "name": "itemId", + "in": "path", + "placeholder": ":id", + "wireType": "u64" + } + ] + }, + "input": { + "testid": "item-card", + "contains": "Espresso Machine", + "attribute": "data-buy-input" + }, + "settleMs": 2000 + }, + { + "do": "expectActionOutcome", + "actor": "buyer", + "outcome": "accepted" + }, + { + "do": "callAction", + "actor": "buyer", + "action": "buy", + "namedAction": { + "id": "buy", + "path": "/api/items/:id/buy", + "reducer": "buy_now", + "args": [ + 0 + ], + "params": [ + { + "name": "itemId", + "in": "path", + "placeholder": ":id", + "wireType": "u64" + } + ] + }, + "input": { + "testid": "item-card", + "contains": "Coffee Grinder", + "attribute": "data-buy-input" + }, + "settleMs": 2000 + }, + { + "do": "expectActionOutcome", + "actor": "buyer", + "outcome": "accepted" + }, + { + "do": "freshClient", + "actor": "buyer" + }, + { + "do": "signIn", + "actor": "buyer-fresh", + "name": "oli" + }, + { + "do": "click", + "actor": "buyer-fresh", + "testid": "overlay-close", + "ifAvailable": true, + "unlessVisible": "order-item", + "within": 1000 + }, + { "do": "click", "actor": "buyer-fresh", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": ["item-list", "order-item"] }, + {"do": "click", "actor": "buyer-fresh", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + { + "do": "click", + "actor": "buyer-fresh", + "testid": "orders-toggle", + "unlessVisible": "order-item" + }, + { + "do": "expect", + "actor": "buyer-fresh", + "testid": "order-item", + "count": 2 + }, + { + "do": "expect", + "actor": "buyer-fresh", + "testid": "order-item", + "contains": "Espresso Machine", + "count": 1 + }, + { + "do": "expectNumber", + "actor": "buyer-fresh", + "testid": "order-total", + "in": { + "testid": "order-item", + "contains": "Espresso Machine" + }, + "equals": 449 + }, + { + "do": "expect", + "actor": "buyer-fresh", + "testid": "order-item", + "contains": "Coffee Grinder", + "count": 1 + }, + { + "do": "expectNumber", + "actor": "buyer-fresh", + "testid": "order-total", + "in": { + "testid": "order-item", + "contains": "Coffee Grinder" + }, + "equals": 64 + }, + { + "do": "dbRecordCheckout", + "account": "{user:oli}", + "item": "Espresso Machine", + "as": "before-price-claim", + "storage": { + "kind": "order-data", + "cart": false, + "warehouses": false + } + }, + { + "do": "dbRecordStock", + "item": "Espresso Machine", + "as": "before-price-claim-stock" + }, + { + "do": "callAction", + "actor": "buyer", + "action": "buy", + "namedAction": { + "id": "buy", + "path": "/api/items/:id/buy", + "reducer": "buy_now", + "args": [ + 0, + 1 + ], + "params": [ + { + "name": "itemId", + "in": "path", + "placeholder": ":id", + "wireType": "u64" + }, + { + "name": "price", + "in": "body" + } + ] + }, + "input": { + "testid": "item-card", + "contains": "Espresso Machine", + "attribute": "data-buy-input" + , "fields": ["itemId"] }, + "settleMs": 0 + }, + { + "do": "expectActionOutcome", + "actor": "buyer", + "outcome": "completed" + }, + { + "do": "dbExpectPurchase", + "actor": "buyer", + "before": "before-price-claim", + "stockBefore": "before-price-claim-stock" + }, + { + "do": "dbRecordCheckout", + "account": "{user:oli}", + "item": "Espresso Machine", + "as": "after-price-claim", + "storage": { + "kind": "order-data", + "cart": false, + "warehouses": false + } + }, + { + "do": "dbRecordStock", + "item": "Espresso Machine", + "as": "after-price-claim-stock" + }, + { + "do": "callAction", + "actor": "buyer", + "action": "buy", + "namedAction": { + "id": "buy", + "path": "/api/items/:id/buy", + "reducer": "buy_now", + "args": [ + 0 + ], + "params": [ + { + "name": "itemId", + "in": "path", + "placeholder": ":id", + "wireType": "u64" + } + ] + }, + "input": { + "testid": "item-card", + "contains": "Espresso Machine", + "attribute": "data-buy-input" + }, + "settleMs": 0 + }, + { + "do": "expectActionOutcome", + "actor": "buyer", + "outcome": "accepted" + }, + { + "do": "dbExpectPurchase", + "actor": "buyer", + "before": "after-price-claim", + "stockBefore": "after-price-claim-stock" + } + ], + "statedBy": "the server uses the current stored price when it creates an order" + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-systems.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-systems.json new file mode 100644 index 00000000000..f7b20b21337 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-systems.json @@ -0,0 +1,202 @@ +{ + "schemaVersion": 1, + "level": 1, + "track": "ecommerce", + "name": "Storefront and warehouse — systems", + "writeUrlPattern": "\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses", + "note": "Properties of the system, not of any one feature: what happens when the database changes underneath the app, and when two things happen at the same time. These are the failures that appear in production and never in a demo — a broadcast layer only announces writes that went through it, and a fetch-then-merge list drops or duplicates whatever moved while it was fetching. Every criterion is withheld at 0 points until it demonstrably fails a real build AND its mutant is CAUGHT, the same promotion rule as contention.", + "features": [ + { + "id": 901, + "name": "A write from outside the app is visible without reload", + "actors": [ + "viewer" + ], + "setup": [ + { + "do": "expectNumber", + "actor": "viewer", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Desk Lamp" + }, + "equals": 100 + } + ], + "criteria": [ + { + "id": "901a", + "desc": "a back-office script sets Desk Lamp's East stock to 5, directly in the database; an anonymous viewer with the storefront already open sees 50 without touching the page", + "points": 0, + "note": "WITHHELD until proven against a mutant (backoffice-writes-via-server) and a real failing build. The script is the app's own scripts/backoffice.mjs, required by the spec to write to the database without the web server's help — the way a cron job or another team's service would.", + "steps": [ + { + "do": "dbSetStock", + "item": "Desk Lamp", + "warehouse": "East", + "quantity": 5, + "settleMs": 4000 + }, + { + "do": "expectNumber", + "actor": "viewer", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Desk Lamp" + }, + "equals": 50, + "within": 10000 + } + ], + "statedBy": "The storefront reflects current database values after changes made outside the application server." + }, + { + "id": "901b", + "desc": "the out-of-band change survives a reload — the page did not merely patch its own memory", + "points": 0, + "note": "WITHHELD — promoted together with 901a.", + "steps": [ + { + "do": "reload", + "actor": "viewer", + "settleMs": 3000 + }, + { + "do": "expectNumber", + "actor": "viewer", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Desk Lamp" + }, + "equals": 50, + "within": 10000 + } + ], + "statedBy": "the app must remain correct - and its open pages current" + }, + { + "id": "901d", + "desc": "the client, not the server, misses the window: a correction lands while this browser is disconnected, and the page is current again once it reconnects", + "points": 1, + "steps": [ + { + "do": "setOffline", + "actor": "viewer", + "offline": true + }, + { + "do": "dbSetStock", + "item": "Desk Lamp", + "warehouse": "East", + "quantity": 7, + "settleMs": 4000 + }, + { + "do": "setOffline", + "actor": "viewer", + "offline": false + }, + { + "do": "expectNumber", + "actor": "viewer", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Desk Lamp" + }, + "equals": 17, + "within": 20000 + } + ], + "withheld": "WITHHELD until it has failed one real build and passed another, the same gate 901c cleared. Distinct from 901c: there the SERVER was down, so nothing was listening at all. Here the server stays up and sees the write; it is this CLIENT that was disconnected. An app that resumes its stream on reconnect without asking for current state stays wrong, and a dropped connection is far more common than a deploy — a closed laptop, a tunnel, a flaky phone." + } + ] + }, + { + "id": 902, + "name": "A list being opened while its contents change", + "actors": [ + "reader", + "reviewer" + ], + "setup": [ + { + "do": "signUp", + "actor": "reviewer", + "name": "raceR" + }, + { + "do": "signUp", + "actor": "reader", + "name": "raceL" + }, + { + "do": "click", + "actor": "reviewer", + "testid": "buy-now", + "in": { + "testid": "item-card", + "contains": "Keyboard" + } + }, + { + "do": "wait", + "actor": "reviewer", + "ms": 800 + }, + {"do": "openItem", "actor": "reviewer", "item": "Keyboard", "settleMs": 1200}, + {"do":"click","actor":"reviewer","testid":"review-toggle","unlessVisible":"review-rating","ifAvailable":true}, + { + "do": "fill", + "actor": "reviewer", + "testid": "review-rating", + "text": "5" + } + ], + "criteria": [ + { + "id": "902a", + "desc": "one customer opens the Keyboard's reviews at the same moment another posts one: the opener ends with that review exactly once — not zero (stale fetch), not twice (fetch merged with its own live echo)", + "points": 0, + "note": "WITHHELD until proven against a mutant (fetch-then-append-dupe) and a real failing build. The race window is real and the criterion may need repetition to be reliable; INCONCLUSIVE on a missed race, never a false FAIL.", + "steps": [ + { + "do": "race", + "branches": [ + [ + {"do": "openItem", "actor": "reader", "item": "Keyboard"} + ], + [ + {"do":"click","actor":"reviewer","testid":"review-toggle","unlessVisible":"review-rating","ifAvailable":true}, + { + "do": "fill", + "actor": "reviewer", + "testid": "review-input", + "text": "race-review-kbd" + }, + { + "do": "click", + "actor": "reviewer", + "testid": "review-submit" + } + ] + ], + "settleMs": 4000 + }, + { + "do": "expectElementCount", + "actor": "reader", + "testid": "review-item", + "contains": "race-review-kbd", + "equals": 1, + "within": 10000 + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-warehouse-admin-staff.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-warehouse-admin-staff.json new file mode 100644 index 00000000000..9961c3856df --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-warehouse-admin-staff.json @@ -0,0 +1,58 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Warehouse administration through staff access", + "features": [ + { + "id": 7, + "name": "Admin and warehouses", + "actors": ["admin", "staff"], + "setup": [ + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + { "do": "signIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true }, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + { "do": "click", "actor": "admin", "testid": "admin-link", "unlessVisible": "admin-panel" } + ], + "criteria": [ + { + "id": "7a", + "category": "production", + "desc": "the administrator area stays unavailable to other staff", + "points": 1, + "statedBy": "Administrators can open the administration area; other staff cannot.", + "steps": [ + {"do": "expect", "actor": "admin", "testid": "admin-item-row"}, + {"do": "click", "actor": "staff", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link"]}, + {"do": "click", "actor": "staff", "testid": "admin-link", "ifAvailable": true, "settleMs": 1500}, + {"do": "expect", "actor": "staff", "testid": "admin-item-row", "absent": true} + ] + }, + { + "id": "7b", + "category": "feature", + "desc": "admin lists every item, every warehouse, and what each warehouse holds", + "points": 1, + "statedBy": "admin lists every item with its stock, every warehouse, and the stock of each item in each warehouse", + "steps": [ + { "do": "expectWarehouseInventory", "actor": "admin", "items": [ + {"name": "Air Purifier", "stock": {"East": 60, "West": 40}}, + {"name": "Bluetooth Speaker", "stock": {"East": 50, "West": 50}}, + {"name": "Coffee Grinder", "stock": {"East": 70, "West": 30}}, + {"name": "Desk Lamp", "stock": {"East": 55, "West": 45}}, + {"name": "Espresso Machine", "stock": {"East": 80, "West": 20}}, + {"name": "Gaming Mouse", "stock": {"East": 50, "West": 50}}, + {"name": "Headphones", "stock": {"East": 60, "West": 40}}, + {"name": "Induction Cooktop", "stock": {"East": 50, "West": 50}}, + {"name": "Keyboard", "stock": {"East": 70, "West": 30}}, + {"name": "Laptop Stand", "stock": {"East": 90, "West": 10}}, + {"name": "Mirrorless Camera", "stock": {"East": 2, "West": 1}}, + {"name": "USB Cable", "stock": {"East": 0, "West": 0}}, + {"name": "Webcam", "stock": {"East": 60, "West": 40}} + ] } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/01-warehouse-stock-live-staff.json b/tools/stack-bench/tracks/ecommerce/scenarios/01-warehouse-stock-live-staff.json new file mode 100644 index 00000000000..d6aa93f3063 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/01-warehouse-stock-live-staff.json @@ -0,0 +1,35 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Warehouse restock reaches the storefront through staff access", + "features": [ + { + "id": 7, + "name": "Warehouse stock stays live", + "actors": ["admin", "visitor"], + "setup": [ + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + { "do": "click", "actor": "admin", "testid": "admin-link", "unlessVisible": "admin-panel" }, + { "do": "click", "actor": "visitor", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "expectNumber", "actor": "visitor", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Gaming Mouse" }, "equals": 100 } + ], + "criteria": [ + { + "id": "7c", + "category": "production", + "desc": "the storefront stock is the sum across warehouses, and a restock raises it live", + "points": 2, + "statedBy": "an item's stock on the storefront is the sum of that item's units across all warehouses", + "steps": [ + { "do": "fill", "actor": "admin", "testid": "restock-input", "text": "25", "in": { "testid": "admin-location-row", "contains": "Gaming Mouse" } }, + { "do": "click", "actor": "admin", "testid": "restock-submit", "in": { "testid": "admin-location-row", "contains": "Gaming Mouse" } }, + { "do": "expectNumber", "actor": "visitor", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Gaming Mouse" }, "equals": 125, "within": 10000 }, + { "do": "expectNumber", "actor": "admin", "testid": "admin-stock", "in": { "testid": "admin-item-row", "contains": "Gaming Mouse" }, "equals": 125 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/02-cancellation-queue.json b/tools/stack-bench/tracks/ecommerce/scenarios/02-cancellation-queue.json new file mode 100644 index 00000000000..4006a8120cf --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/02-cancellation-queue.json @@ -0,0 +1,85 @@ +{ + "schemaVersion": 1, "track": "ecommerce", "level": 2, + "name": "Cancellation and shipping remain consistent", + "note": "Draft: existing queue-removal owner also checks cancellation before/after shipping and three independent request pairs. Preserve the live queue observer. One point; no request-count bonus. Native stored-state checks use the declared order/warehouse interface. Client dispatch overlap does not establish all server schedules.", + "features": [{ + "id": 3, "name": "Cancellation and fulfilment", "actors": ["customer","staff","ship-first","race-1","race-2","race-3"], + "setup": [ + {"do":"signUp","actor":"customer","name":"cancel-queue"}, + {"do":"signIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true}, + {"do":"click","actor":"staff","testid":"staff-link"} + ], + "criteria": [{ + "id": "3d", "category": "production", "points": 1, + "desc": "cancellation removes an order from the live queue and cannot combine its refund or stock restoration with shipping", + "statedBy": "A customer can cancel an order before it ships; the order leaves the fulfilment queue.", + "steps": [ + {"do":"dbRecordCheckout","account":"{user:cancel-queue}","item":"Coffee Grinder","as":"before","storage":{"kind":"order-data","cart":false,"warehouses":true}}, + {"do":"callConcurrently","actors":["customer"],"action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Coffee Grinder","attribute":"data-buy-input"},"requests":1,"settleMs":0}, + {"do":"expectCallOutcomes","accepted":1}, + {"do":"expect","actor":"staff","testid":"queue-item","contains":"Coffee Grinder","within":10000}, + {"do":"dbExpectPurchases","before":{"customer":"before"},"purchases":1}, + {"do":"dbRecordCheckout","account":"{user:cancel-queue}","item":"Coffee Grinder","as":"pending","storage":{"kind":"order-data","cart":false,"warehouses":true}}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"customer","testid":"orders-toggle"}, + {"do":"click","actor":"customer","testid":"cancel-order","in":{"testid":"order-item","contains":"Coffee Grinder"}}, + {"do":"waitUntilAbsent","actor":"staff","testid":"queue-item","contains":"Coffee Grinder","within":10000}, + {"do":"dbExpectCancellation","before":"pending"}, + {"do":"callAction","actor":"staff","from":"customer","action":"ship","namedAction":{"id":"ship","path":"/api/fulfilment/ship","reducer":"ship_order","args":[0],"params":[{"name":"orderId","in":"body","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Coffee Grinder","attribute":"data-ship-input"},"settleMs":0}, + {"do":"expectActionOutcome","actor":"staff","outcome":"completed"}, + {"do":"dbExpectCancellation","before":"pending"}, + {"do":"signUp","actor":"ship-first","name":"ship-first"}, + {"do":"dbRecordCheckout","account":"{user:ship-first}","item":"Coffee Grinder","as":"before","storage":{"kind":"order-data","cart":false,"warehouses":true}}, + {"do":"callConcurrently","actors":["ship-first"],"action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Coffee Grinder","attribute":"data-buy-input"},"requests":1,"settleMs":0}, + {"do":"expectCallOutcomes","accepted":1}, + {"do":"dbExpectPurchases","before":{"ship-first":"before"},"purchases":1}, + {"do":"dbRecordCheckout","account":"{user:ship-first}","item":"Coffee Grinder","as":"pending","storage":{"kind":"order-data","cart":false,"warehouses":true}}, + {"do":"click","actor":"ship-first","testid":"overlay-close","ifAvailable":true,"within":1000,"unlessVisible":"order-item"}, + {"do":"click","actor":"ship-first","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"ship-first","testid":"order-status","in":{"testid":"order-item","contains":"Coffee Grinder"},"statusText":"pending","ignoreCase":true}, + {"do":"callAction","actor":"staff","from":"ship-first","action":"ship","namedAction":{"id":"ship","path":"/api/fulfilment/ship","reducer":"ship_order","args":[0],"params":[{"name":"orderId","in":"body","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Coffee Grinder","attribute":"data-ship-input"},"settleMs":0}, + {"do":"expectActionOutcome","actor":"staff","outcome":"accepted"}, + {"do":"dbExpectCancellation","before":"pending","shipping":"wins"}, + {"do":"callAction","actor":"ship-first","from":"ship-first","action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Coffee Grinder","attribute":"data-cancel-input"},"settleMs":0}, + {"do":"expectActionOutcome","actor":"ship-first","outcome":"completed"}, + {"do":"dbExpectCancellation","before":"pending","shipping":"wins"}, + {"do":"signUp","actor":"race-1","name":"race-1"}, + {"do":"dbRecordCheckout","account":"{user:race-1}","item":"Coffee Grinder","as":"before","storage":{"kind":"order-data","cart":false,"warehouses":true}}, + {"do":"callConcurrently","actors":["race-1"],"action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Coffee Grinder","attribute":"data-buy-input"},"requests":1,"settleMs":0}, + {"do":"expectCallOutcomes","accepted":1}, + {"do":"dbExpectPurchases","before":{"race-1":"before"},"purchases":1}, + {"do":"dbRecordCheckout","account":"{user:race-1}","item":"Coffee Grinder","as":"pending","storage":{"kind":"order-data","cart":false,"warehouses":true}}, + {"do":"click","actor":"race-1","testid":"overlay-close","ifAvailable":true,"within":1000,"unlessVisible":"order-item"}, + {"do":"click","actor":"race-1","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"race-1","testid":"order-status","in":{"testid":"order-item","contains":"Coffee Grinder"},"statusText":"pending","ignoreCase":true}, + {"do":"callConcurrently","actors":["race-1"],"from":"race-1","action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Coffee Grinder","attribute":"data-cancel-input"},"requests":1,"settleMs":0,"alongside":[{"actors":["staff"],"from":"race-1","action":"ship","namedAction":{"id":"ship","path":"/api/fulfilment/ship","reducer":"ship_order","args":[0],"params":[{"name":"orderId","in":"body","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Coffee Grinder","attribute":"data-ship-input"},"requests":1}]}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCancellation","before":"pending","shipping":"competes"}, + {"do":"signUp","actor":"race-2","name":"race-2"}, + {"do":"dbRecordCheckout","account":"{user:race-2}","item":"Coffee Grinder","as":"before","storage":{"kind":"order-data","cart":false,"warehouses":true}}, + {"do":"callConcurrently","actors":["race-2"],"action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Coffee Grinder","attribute":"data-buy-input"},"requests":1,"settleMs":0}, + {"do":"expectCallOutcomes","accepted":1}, + {"do":"dbExpectPurchases","before":{"race-2":"before"},"purchases":1}, + {"do":"dbRecordCheckout","account":"{user:race-2}","item":"Coffee Grinder","as":"pending","storage":{"kind":"order-data","cart":false,"warehouses":true}}, + {"do":"click","actor":"race-2","testid":"overlay-close","ifAvailable":true,"within":1000,"unlessVisible":"order-item"}, + {"do":"click","actor":"race-2","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"race-2","testid":"order-status","in":{"testid":"order-item","contains":"Coffee Grinder"},"statusText":"pending","ignoreCase":true}, + {"do":"callConcurrently","actors":["race-2"],"from":"race-2","action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Coffee Grinder","attribute":"data-cancel-input"},"requests":1,"settleMs":0,"alongside":[{"actors":["staff"],"from":"race-2","action":"ship","namedAction":{"id":"ship","path":"/api/fulfilment/ship","reducer":"ship_order","args":[0],"params":[{"name":"orderId","in":"body","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Coffee Grinder","attribute":"data-ship-input"},"requests":1}]}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCancellation","before":"pending","shipping":"competes"}, + {"do":"signUp","actor":"race-3","name":"race-3"}, + {"do":"dbRecordCheckout","account":"{user:race-3}","item":"Coffee Grinder","as":"before","storage":{"kind":"order-data","cart":false,"warehouses":true}}, + {"do":"callConcurrently","actors":["race-3"],"action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Coffee Grinder","attribute":"data-buy-input"},"requests":1,"settleMs":0}, + {"do":"expectCallOutcomes","accepted":1}, + {"do":"dbExpectPurchases","before":{"race-3":"before"},"purchases":1}, + {"do":"dbRecordCheckout","account":"{user:race-3}","item":"Coffee Grinder","as":"pending","storage":{"kind":"order-data","cart":false,"warehouses":true}}, + {"do":"click","actor":"race-3","testid":"overlay-close","ifAvailable":true,"within":1000,"unlessVisible":"order-item"}, + {"do":"click","actor":"race-3","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"race-3","testid":"order-status","in":{"testid":"order-item","contains":"Coffee Grinder"},"statusText":"pending","ignoreCase":true}, + {"do":"callConcurrently","actors":["race-3"],"from":"race-3","action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Coffee Grinder","attribute":"data-cancel-input"},"requests":1,"settleMs":0,"alongside":[{"actors":["staff"],"from":"race-3","action":"ship","namedAction":{"id":"ship","path":"/api/fulfilment/ship","reducer":"ship_order","args":[0],"params":[{"name":"orderId","in":"body","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Coffee Grinder","attribute":"data-ship-input"},"requests":1}]}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCancellation","before":"pending","shipping":"competes"} + ] + }] + }] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/02-features.json b/tools/stack-bench/tracks/ecommerce/scenarios/02-features.json new file mode 100644 index 00000000000..d72773b6b39 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/02-features.json @@ -0,0 +1,1156 @@ +{ + "schemaVersion": 1, + "level": 2, + "track": "ecommerce", + "name": "Running the business — features", + "writeUrlPattern": "\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses|\\/transfer|\\/cancel|\\/return|\\/ship|\\/price|\\/fulfil", + "note": "The level's thesis is that eight actions each touch several of ten live views. These features are therefore written as ONE action watched by SEVERAL viewers at once, not as one action checked once — a backend that updates the view the actor is looking at and forgets the others fails on the watcher, not on the actor. Items are reached by search because earlier features re-rank the storefront. Panels are closed with Escape, which the spec requires, rather than by clicking an opener again.", + "features": [ + { + "id": 1, + "name": "Fulfilment queue", + "actors": [ + "customer", + "staff", + "admin" + ], + "setup": [ + { + "do": "signUp", + "actor": "customer", + "name": "fq" + }, + { + "do": "signIn", + "actor": "staff", + "name": "staff", + "password": "stackbench-staff-2026", + "exact": true + }, + { + "do": "signIn", + "actor": "admin", + "name": "admin", + "password": "stackbench-admin-2026", + "exact": true + } + ], + "criteria": [ + { + "id": "1a", + "desc": "an order placed by a customer appears in the staff queue without a reload", + "points": 2, + "steps": [ + { + "do": "click", + "actor": "staff", + "testid": "staff-link" + }, + { + "do": "recordNumber", + "actor": "staff", + "testid": "queue-depth", + "as": "depth-before" + }, + { + "do": "fill", + "actor": "customer", + "testid": "search-input", + "text": "Desk Lamp", + "enter": true, + "settleMs": 1500 + }, + { + "do": "click", + "actor": "customer", + "testid": "buy-now", + "in": { + "testid": "item-card", + "contains": "Desk Lamp" + } + }, + { + "do": "expect", + "actor": "staff", + "testid": "queue-item", + "contains": "Desk Lamp", + "within": 10000 + }, + { + "do": "expectNumber", + "actor": "staff", + "testid": "queue-depth", + "relativeTo": "depth-before", + "plus": 1 + } + ], + "statedBy": "appears in the fulfilment queue in the order it was placed" + }, + { + "id": "1b", + "desc": "the queue names the warehouse the order will ship from", + "points": 1, + "steps": [ + { + "do": "expect", + "actor": "staff", + "testid": "queue-warehouse", + "in": { + "testid": "queue-item", + "contains": "Desk Lamp" + } + } + ], + "statedBy": "which warehouse each will ship from" + }, + { + "id": "1c", + "desc": "shipping removes the order from the queue for staff and marks it shipped for the customer, live", + "points": 2, + "steps": [ + { + "do": "click", + "actor": "staff", + "testid": "ship-submit", + "in": { + "testid": "queue-item", + "contains": "Desk Lamp" + } + }, + { + "do": "expect", + "actor": "staff", + "testid": "queue-item", + "contains": "Desk Lamp", + "absent": true, + "within": 10000 + }, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { + "do": "click", + "actor": "customer", + "testid": "orders-toggle" + }, + { + "do": "expect", + "actor": "customer", + "testid": "order-status", + "in": { + "testid": "order-item", + "contains": "Desk Lamp" + }, + "value": "shipped", "ignoreCase": true, + "within": 10000 + } + ], + "statedBy": "it leaves the queue for everyone watching the queue, and the customer's order history shows it as shipped, live" + }, + { + "id": "1d", + "desc": "the fulfilment area is for staff and admins, not customers", + "points": 1, + "steps": [ + { + "do": "click", + "actor": "admin", + "testid": "staff-link" + }, + { + "do": "expect", + "actor": "admin", + "testid": "fulfilment-panel" + }, + { + "do": "click", + "actor": "customer", + "testid": "staff-link", + "ifAvailable": true, + "settleMs": 1500 + }, + { + "do": "expect", + "actor": "customer", + "testid": "fulfilment-panel", + "absent": true + } + ], + "statedBy": "a staff account signs in like anyone else but sees the fulfilment area instead of the admin area" + }, + { + "id": "1e", + "desc": "the server refuses a ship request from a customer, not just the button", + "points": 2, + "statedBy": "staff cannot change prices or create items; admins can do everything staff can", + "steps": [ + { + "do": "fill", + "actor": "customer", + "testid": "search-input", + "text": "Webcam", + "enter": true, + "settleMs": 1500 + }, + { + "do": "click", + "actor": "customer", + "testid": "buy-now", + "in": { + "testid": "item-card", + "contains": "Webcam" + } + }, + { + "do": "wait", + "actor": "customer", + "ms": 2500 + }, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { + "do": "click", + "actor": "customer", + "testid": "orders-toggle" + }, + { + "do": "replayAs", + "actor": "customer", + "from": "staff", + "match": "ship", + "namedAction": { + "id": "ship", + "path": "/api/fulfilment/ship", + "reducer": "ship_order", + "args": [0] + }, + "namedTarget": { + "testid": "order-item", + "contains": "Webcam", + "attribute": "data-entity-id", + "valueType": "number" + }, + "swap": { + "find": "Desk Lamp", + "with": "Webcam" + }, + "settleMs": 2500 + }, + { + "do": "expectReplayRejected", + "actor": "customer" + }, + { + "do": "expect", + "actor": "customer", + "testid": "order-status", + "in": { + "testid": "order-item", + "contains": "Webcam" + }, + "value": "pending", "ignoreCase": true + } + ] + } + ] + }, + { + "id": 2, + "name": "Moving stock between warehouses", + "actors": [ + "admin", + "staff", + "visitor" + ], + "setup": [ + { + "do": "signIn", + "actor": "admin", + "name": "admin", + "password": "stackbench-admin-2026", + "exact": true + }, + { + "do": "signIn", + "actor": "staff", + "name": "staff", + "password": "stackbench-staff-2026", + "exact": true + }, + { + "do": "click", + "actor": "admin", + "testid": "admin-link" + } + ], + "criteria": [ + { + "id": "2a", + "desc": "a transfer moves stock without creating or destroying any", + "points": 3, + "steps": [ + { + "do": "recordNumber", + "actor": "visitor", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Headphones" + }, + "as": "total-before" + }, + { + "do": "recordNumber", + "actor": "admin", + "testid": "admin-location-qty", + "in": { + "testid": "admin-location-row", + "contains": "Headphones" + }, + "as": "east-before" + }, + { + "do": "recordNumber", + "actor": "admin", + "testid": "warehouse-total", + "in": { + "testid": "admin-warehouse-item", + "contains": "West" + }, + "as": "west-before" + }, + { + "do": "fill", + "actor": "admin", + "testid": "transfer-from", + "text": "East", + "in": { + "testid": "admin-item-row", + "contains": "Headphones" + } + }, + { + "do": "fill", + "actor": "admin", + "testid": "transfer-to", + "text": "West", + "in": { + "testid": "admin-item-row", + "contains": "Headphones" + } + }, + { + "do": "fill", + "actor": "admin", + "testid": "transfer-qty", + "text": "10", + "in": { + "testid": "admin-item-row", + "contains": "Headphones" + } + }, + { + "do": "click", + "actor": "admin", + "testid": "transfer-submit", + "in": { + "testid": "admin-item-row", + "contains": "Headphones" + } + }, + { + "do": "wait", + "actor": "admin", + "ms": 2500 + }, + { + "do": "expectNumber", + "actor": "admin", + "testid": "admin-location-qty", + "in": { + "testid": "admin-location-row", + "contains": "Headphones" + }, + "relativeTo": "east-before", + "plus": -10 + }, + { + "do": "expectNumber", + "actor": "visitor", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Headphones" + }, + "relativeTo": "total-before", + "plus": 0 + } + ], + "statedBy": "a transfer moves stock, it does not create or destroy it" + }, + { + "id": "2b", + "desc": "the warehouse totals move live as stock is transferred", + "points": 2, + "steps": [ + { + "do": "expectNumber", + "actor": "admin", + "testid": "warehouse-total", + "in": { + "testid": "admin-warehouse-item", + "contains": "West" + }, + "relativeTo": "west-before", + "plus": 10, + "within": 10000 + } + ], + "statedBy": "the per-warehouse numbers staff and admins see both move at once" + }, + { + "id": "2c", + "desc": "a transfer that would overdraw a warehouse is refused and changes nothing", + "points": 2, + "steps": [ + { + "do": "recordNumber", + "actor": "admin", + "testid": "admin-stock", + "in": { + "testid": "admin-item-row", + "contains": "Headphones" + }, + "as": "total-guard" + }, + { + "do": "fill", + "actor": "admin", + "testid": "transfer-from", + "text": "East", + "in": { + "testid": "admin-item-row", + "contains": "Headphones" + } + }, + { + "do": "fill", + "actor": "admin", + "testid": "transfer-to", + "text": "West", + "in": { + "testid": "admin-item-row", + "contains": "Headphones" + } + }, + { + "do": "fill", + "actor": "admin", + "testid": "transfer-qty", + "text": "99999", + "in": { + "testid": "admin-item-row", + "contains": "Headphones" + } + }, + { + "do": "click", + "actor": "admin", + "testid": "transfer-submit", + "in": { + "testid": "admin-item-row", + "contains": "Headphones" + } + }, + { + "do": "expect", + "actor": "admin", + "testid": "order-error", + "within": 6000 + }, + { + "do": "expectNumber", + "actor": "admin", + "testid": "admin-stock", + "in": { + "testid": "admin-item-row", + "contains": "Headphones" + }, + "relativeTo": "total-guard", + "plus": 0 + } + ], + "statedBy": "a transfer that would leave a warehouse short is refused and changes nothing" + } + ] + }, + { + "id": 3, + "name": "Cancelling and returning", + "actors": [ + "customer", + "staff", + "admin" + ], + "setup": [ + { + "do": "signUp", + "actor": "customer", + "name": "cr" + }, + { + "do": "signIn", + "actor": "staff", + "name": "staff", + "password": "stackbench-staff-2026", + "exact": true + }, + { + "do": "signIn", + "actor": "admin", + "name": "admin", + "password": "stackbench-admin-2026", + "exact": true + }, + { + "do": "click", + "actor": "admin", + "testid": "admin-link" + }, + { + "do": "click", + "actor": "staff", + "testid": "staff-link" + } + ], + "criteria": [ + { + "id": "3a", + "desc": "cancelling a pending order returns its stock, empties it from the queue and lowers revenue", + "points": 3, + "steps": [ + {"do":"dbRecordStock","item":"Coffee Grinder","as":"stored-before-cancel-3a"}, + {"do":"recordNumber","actor":"admin","testid":"admin-revenue","as":"rev-before"}, + {"do":"fill","actor":"customer","testid":"search-input","text":"Coffee Grinder","enter":true,"settleMs":1500}, + {"do":"recordNumber","actor":"customer","testid":"item-stock","in":{"testid":"item-card","contains":"Coffee Grinder"},"as":"stock-before"}, + {"do":"click","actor":"customer","testid":"buy-now","in":{"testid":"item-card","contains":"Coffee Grinder"}}, + {"do":"wait","actor":"customer","ms":2500}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"customer","testid":"orders-toggle"}, + {"do":"expect","actor":"customer","testid":"order-item","contains":"Coffee Grinder","count":1,"within":10000}, + {"do":"dbExpectStock","item":"Coffee Grinder","relativeTo":"stored-before-cancel-3a","plus":-1}, + {"do":"expect","actor":"staff","testid":"queue-item","contains":"Coffee Grinder","count":1,"within":10000}, + {"do":"expectNumber","actor":"admin","testid":"admin-revenue","relativeTo":"rev-before","plus":64,"within":10000}, + {"do":"click","actor":"customer","testid":"cancel-order","in":{"testid":"order-item","contains":"Coffee Grinder"}}, + {"do":"wait","actor":"customer","ms":2500}, + {"do":"expect","actor":"staff","testid":"queue-item","contains":"Coffee Grinder","absent":true,"within":10000}, + {"do":"expectNumber","actor":"admin","testid":"admin-revenue","relativeTo":"rev-before","plus":0,"within":10000}, + {"do":"pressKey","actor":"customer","key":"Escape"}, + {"do":"expectNumber","actor":"customer","testid":"item-stock","in":{"testid":"item-card","contains":"Coffee Grinder"},"relativeTo":"stock-before","plus":0}, + {"do":"dbExpectStock","item":"Coffee Grinder","relativeTo":"stored-before-cancel-3a","plus":0} + ], + "statedBy": "the stock goes back to the warehouse it came from, the order leaves the fulfilment queue, and revenue falls" + }, + { + "id": "3b", + "desc": "a cancelled order is shown as cancelled in the customer’s history", + "points": 1, + "steps": [ + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { + "do": "click", + "actor": "customer", + "testid": "orders-toggle", + "settleMs": 1200 + }, + { + "do": "expect", + "actor": "customer", + "testid": "order-status", + "in": { + "testid": "order-item", + "contains": "Coffee Grinder" + }, + "value": "cancelled", "ignoreCase": true, + "within": 8000 + } + ], + "statedBy": "a customer can cancel an order that has not shipped" + }, + { + "id": "3c", + "desc": "returning an item from a shipped order brings the stock back and lowers revenue", + "points": 3, + "steps": [ + { + "do": "fill", + "actor": "customer", + "testid": "search-input", + "text": "Keyboard", + "enter": true, + "settleMs": 1500 + }, + { + "do": "recordNumber", + "actor": "customer", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Keyboard" + }, + "as": "kb-before" + }, + { + "do": "recordNumber", + "actor": "admin", + "testid": "admin-revenue", + "as": "rev2-before" + }, + { + "do": "click", + "actor": "customer", + "testid": "buy-now", + "in": { + "testid": "item-card", + "contains": "Keyboard" + } + }, + { + "do": "wait", + "actor": "customer", + "ms": 2500 + }, + { + "do": "click", + "actor": "staff", + "testid": "ship-submit", + "in": { + "testid": "queue-item", + "contains": "Keyboard" + } + }, + { + "do": "wait", + "actor": "staff", + "ms": 2500 + }, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { + "do": "click", + "actor": "customer", + "testid": "orders-toggle" + }, + { + "do": "click", + "actor": "customer", + "testid": "return-item", + "in": { + "testid": "order-item", + "contains": "Keyboard" + } + }, + { + "do": "wait", + "actor": "customer", + "ms": 2500 + }, + { + "do": "expectNumber", + "actor": "admin", + "testid": "admin-revenue", + "relativeTo": "rev2-before", + "plus": 0, + "within": 10000 + }, + { + "do": "pressKey", + "actor": "customer", + "key": "Escape" + }, + { + "do": "expectNumber", + "actor": "customer", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Keyboard" + }, + "relativeTo": "kb-before", + "plus": 0 + } + ], + "statedBy": "the stock comes back, revenue falls by what was paid for it, and the order shows the item as returned" + } + ] + }, + { + "id": 4, + "name": "Prices change, history does not", + "actors": [ + "admin", + "customer", + "visitor" + ], + "setup": [ + { + "do": "signIn", + "actor": "admin", + "name": "admin", + "password": "stackbench-admin-2026", + "exact": true + }, + { + "do": "signUp", + "actor": "customer", + "name": "pr" + }, + { + "do": "click", + "actor": "admin", + "testid": "admin-link" + } + ], + "criteria": [ + { + "id": "4a", + "desc": "a customer's receipt keeps the price that was paid after the price changes", + "points": 3, + "steps": [ + { + "do": "fill", + "actor": "customer", + "testid": "search-input", + "text": "Air Purifier", + "enter": true, + "settleMs": 1500 + }, + { + "do": "click", + "actor": "customer", + "testid": "buy-now", + "in": { + "testid": "item-card", + "contains": "Air Purifier" + } + }, + { + "do": "wait", + "actor": "customer", + "ms": 2500 + }, + { + "do": "fill", + "actor": "admin", + "testid": "price-input", + "text": "1.00", + "in": { + "testid": "admin-item-row", + "contains": "Air Purifier" + } + }, + { + "do": "click", + "actor": "admin", + "testid": "price-submit", + "in": { + "testid": "admin-item-row", + "contains": "Air Purifier" + } + }, + { + "do": "wait", + "actor": "admin", + "ms": 2500 + }, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { + "do": "click", + "actor": "customer", + "testid": "orders-toggle" + }, + { + "do": "expectNumber", + "actor": "customer", + "testid": "order-total", + "in": { + "testid": "order-item", + "contains": "Air Purifier" + }, + "atLeast": 189 + } + ], + "statedBy": "past orders keep the price that was paid" + }, + { + "id": "4b", + "desc": "the new price reaches a signed-out visitor without a reload", + "points": 2, + "steps": [ + { + "do": "fill", + "actor": "visitor", + "testid": "search-input", + "text": "Air Purifier", + "enter": true, + "settleMs": 1500 + }, + { + "do": "expectNumber", + "actor": "visitor", + "testid": "item-price", + "in": { + "testid": "item-card", + "contains": "Air Purifier" + }, + "equals": 1, + "within": 10000 + } + ], + "statedBy": "the storefront shows the new price immediately, to everyone" + }, + { + "id": "4c", + "desc": "a price change updates an open cart and checkout charges the new price", + "points": 3, + "steps": [ + { + "do": "pressKey", + "actor": "customer", + "key": "Escape" + }, + { + "do": "fill", + "actor": "customer", + "testid": "search-input", + "text": "Desk Lamp", + "enter": true, + "settleMs": 1500 + }, + { + "do": "click", + "actor": "customer", + "testid": "add-to-cart", + "in": { + "testid": "item-card", + "contains": "Desk Lamp" + } + }, + { + "do": "click", + "actor": "customer", + "testid": "cart-toggle" + }, + { + "do": "expectNumber", + "actor": "customer", + "testid": "cart-total", + "equals": 42 + }, + { + "do": "fill", + "actor": "admin", + "testid": "price-input", + "text": "52.00", + "in": { + "testid": "admin-item-row", + "contains": "Desk Lamp" + } + }, + { + "do": "click", + "actor": "admin", + "testid": "price-submit", + "in": { + "testid": "admin-item-row", + "contains": "Desk Lamp" + } + }, + { + "do": "expectNumber", + "actor": "customer", + "testid": "cart-total", + "equals": 52, + "within": 10000 + }, + { + "do": "click", + "actor": "customer", + "testid": "checkout-submit" + }, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { + "do": "click", + "actor": "customer", + "testid": "orders-toggle", + "settleMs": 1500 + }, + { + "do": "expectNumber", + "actor": "customer", + "testid": "order-total", + "in": { + "testid": "order-item", + "contains": "Desk Lamp" + }, + "equals": 52 + } + ], + "statedBy": "an item in someone's cart at the moment the price changes is charged the new price at checkout, and the cart total they are looking at updates to match" + } + ] + }, + { + "id": 5, + "name": "The operational views", + "actors": [ + "admin", + "customer", + "visitor" + ], + "setup": [ + { + "do": "signIn", + "actor": "admin", + "name": "admin", + "password": "stackbench-admin-2026", + "exact": true + }, + { + "do": "signUp", + "actor": "customer", + "name": "ops" + }, + { + "do": "click", + "actor": "admin", + "testid": "admin-link" + }, + { + "do": "click", + "actor": "admin", + "testid": "low-stock-link", + "ifAvailable": true, + "unlessVisible": "low-stock-item", + "within": 10000 + } + ], + "criteria": [ + { + "id": "5a", + "desc": "an item falling to ten units or fewer joins the low-stock list, live", + "points": 3, + "steps": [ + { + "do": "expect", + "actor": "admin", + "testid": "low-stock-item", + "contains": "Mirrorless Camera", + "within": 8000 + }, + { + "do": "fill", + "actor": "admin", + "testid": "restock-input", + "text": "8", + "in": { + "testid": "admin-location-row", + "contains": "Mirrorless Camera" + } + }, + { + "do": "click", + "actor": "admin", + "testid": "restock-submit", + "in": { + "testid": "admin-location-row", + "contains": "Mirrorless Camera" + } + }, + { + "do": "expect", + "actor": "admin", + "testid": "low-stock-item", + "contains": "Mirrorless Camera", + "absent": true, + "within": 10000 + }, + { + "do": "fill", + "actor": "customer", + "testid": "search-input", + "text": "Mirrorless Camera", + "enter": true, + "settleMs": 1500 + }, + { + "do": "click", + "actor": "customer", + "testid": "buy-now", + "in": { + "testid": "item-card", + "contains": "Mirrorless Camera" + } + }, + { + "do": "expect", + "actor": "admin", + "testid": "low-stock-item", + "contains": "Mirrorless Camera", + "within": 10000 + } + ], + "statedBy": "items enter and leave this list as stock moves, sells, is restocked, cancelled or returned" + }, + { + "id": "5b", + "desc": "a purchase moves that category's units and revenue, live", + "points": 3, + "steps": [ + {"do":"click","actor":"admin","testid":"sales-link","unlessVisible":"category-row","ifAvailable":true}, + { + "do": "recordNumber", + "actor": "admin", + "testid": "category-units", + "in": { + "testid": "category-row", + "contains": "Audio" + }, + "as": "audio-units" + }, + { + "do": "recordNumber", + "actor": "admin", + "testid": "category-revenue", + "in": { + "testid": "category-row", + "contains": "Audio" + }, + "as": "audio-rev" + }, + { + "do": "fill", + "actor": "customer", + "testid": "search-input", + "text": "Bluetooth Speaker", + "enter": true, + "settleMs": 1500 + }, + { + "do": "click", + "actor": "customer", + "testid": "buy-now", + "in": { + "testid": "item-card", + "contains": "Bluetooth Speaker" + } + }, + { + "do": "expectNumber", + "actor": "admin", + "testid": "category-units", + "in": { + "testid": "category-row", + "contains": "Audio" + }, + "relativeTo": "audio-units", + "plus": 1, + "within": 10000 + }, + { + "do": "expectNumber", + "actor": "admin", + "testid": "category-revenue", + "in": { + "testid": "category-row", + "contains": "Audio" + }, + "relativeTo": "audio-rev", + "plus": 79.5, + "within": 10000 + } + ], + "statedBy": "for each\ncategory, how many units have been sold and the revenue they earned, live" + }, + { + "id": "5c", + "desc": "buying from a category makes that category's other items recommended", + "points": 2, + "steps": [ + { + "do": "fill", + "actor": "customer", + "testid": "search-input", + "text": "Bluetooth Speaker", + "enter": true, + "settleMs": 1500 + }, + { + "do": "click", + "actor": "customer", + "testid": "buy-now", + "in": { + "testid": "item-card", + "contains": "Bluetooth Speaker" + } + }, + { + "do": "pressKey", + "actor": "customer", + "key": "Escape" + }, + { + "do": "expect", + "actor": "customer", + "testid": "recommended-item", + "contains": "Headphones", + "within": 10000 + }, + { + "do": "fill", + "actor": "customer", + "testid": "search-input", + "text": "Headphones", + "enter": true, + "settleMs": 1500 + }, + { + "do": "click", + "actor": "customer", + "testid": "add-to-cart", + "in": { + "testid": "item-card", + "contains": "Headphones" + } + }, + { + "do": "pressKey", + "actor": "customer", + "key": "Escape" + }, + { + "do": "expect", + "actor": "customer", + "testid": "recommended-item", + "contains": "Headphones", + "absent": true, + "within": 10000 + } + ], + "statedBy": "the items from categories\nthey have bought from, most-purchased first, excluding items already in\ntheir cart" + }, + { + "id": "5d", + "desc": "a signed-out visitor sees best sellers in the recommendations slot", + "points": 1, + "steps": [ + { + "do": "expect", + "actor": "visitor", + "testid": "recommended-item", + "contains": "Bluetooth Speaker", + "within": 10000 + } + ], + "statedBy": "for a signed-out visitor this is simply the best sellers" + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/02-fulfilment-access.json b/tools/stack-bench/tracks/ecommerce/scenarios/02-fulfilment-access.json new file mode 100644 index 00000000000..337864ca5c8 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/02-fulfilment-access.json @@ -0,0 +1,39 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 2, + "name": "Ecommerce L2 fulfilment access", + "note": "This scenario owns only the visible fulfilment access boundary.", + "features": [ + { + "id": 1, + "name": "Fulfilment area access", + "actors": ["customer", "staff", "admin"], + "setup": [ + { "do": "signUp", "actor": "customer", "name": "fq-customer" }, + { "do": "signIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true }, + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true } + ], + "criteria": [ + { + "id": "1d", + "category": "production", + "desc": "staff and administrators can open fulfilment while customers cannot", + "points": 1, + "statedBy": "Customers cannot open the fulfilment area", + "steps": [ + {"do": "click", "actor": "staff", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["staff-link"]}, + { "do": "click", "actor": "staff", "testid": "staff-link" }, + { "do": "expect", "actor": "staff", "testid": "fulfilment-panel" }, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["staff-link"]}, + { "do": "click", "actor": "admin", "testid": "staff-link" }, + { "do": "expect", "actor": "admin", "testid": "fulfilment-panel" }, + {"do": "click", "actor": "customer", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["staff-link"]}, + { "do": "click", "actor": "customer", "testid": "staff-link", "ifAvailable": true, "settleMs": 1500 }, + { "do": "expect", "actor": "customer", "testid": "fulfilment-panel", "absent": true } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/02-fulfilment-live.json b/tools/stack-bench/tracks/ecommerce/scenarios/02-fulfilment-live.json new file mode 100644 index 00000000000..b17c3dd32d6 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/02-fulfilment-live.json @@ -0,0 +1,36 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 2, + "name": "Ecommerce L2 live fulfilment queue", + "note": "This scenario owns only the live queue check.", + "features": [ + { + "id": 1, + "name": "Live fulfilment queue", + "actors": ["customer", "staff"], + "setup": [ + { "do": "signUp", "actor": "customer", "name": "fq-live" }, + { "do": "signIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true }, + {"do": "click", "actor": "staff", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["staff-link", "fulfilment-panel"]}, + { "do": "click", "actor": "staff", "testid": "staff-link", "unlessVisible": "fulfilment-panel" }, + { "do": "click", "actor": "customer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "1a", + "category": "production", + "desc": "an order placed by a customer appears in the staff queue without a reload", + "points": 2, + "statedBy": "New orders appear in the queue without a reload", + "steps": [ + { "do": "recordNumber", "actor": "staff", "testid": "queue-depth", "as": "depth-before" }, + { "do": "click", "actor": "customer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Desk Lamp" } }, + { "do": "expect", "actor": "staff", "testid": "queue-item", "contains": "Desk Lamp", "within": 10000 }, + { "do": "expectNumber", "actor": "staff", "testid": "queue-depth", "relativeTo": "depth-before", "plus": 1 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/02-fulfilment-ship.json b/tools/stack-bench/tracks/ecommerce/scenarios/02-fulfilment-ship.json new file mode 100644 index 00000000000..995c9068bf7 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/02-fulfilment-ship.json @@ -0,0 +1,57 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 2, + "name": "Ecommerce L2 fulfilment shipping", + "note": "This scenario owns only the shipping result check.", + "features": [ + { + "id": 1, + "name": "Ship a pending order", + "actors": ["customer", "staff"], + "setup": [ + { "do": "signUp", "actor": "customer", "name": "fq-ship" }, + { "do": "signIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true }, + {"do": "click", "actor": "staff", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["staff-link", "fulfilment-panel"]}, + { "do": "click", "actor": "staff", "testid": "staff-link", "unlessVisible": "fulfilment-panel" }, + { "do": "click", "actor": "customer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "1c", + "category": "feature", + "desc": "shipping removes the order from the queue and marks the customer's order shipped", + "points": 2, + "statedBy": "Staff can mark an order as shipped; show the new status in the fulfilment area and the customer's order history.", + "steps": [ + { "do": "callAction", "actor": "customer", "action": "buy", "namedAction": { "id": "buy", "path": "/api/items/:id/buy", "reducer": "buy_now", "args": [0], "params": [{ "name": "itemId", "in": "path", "placeholder": ":id", "wireType": "u64" }] }, "input": { "testid": "item-card", "contains": "Keyboard", "attribute": "data-buy-input" } }, + { "do": "expectActionOutcome", "actor": "customer", "outcome": "accepted" }, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do": "click", "actor": "customer", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + { "do": "click", "actor": "customer", "testid": "orders-toggle", "unlessVisible": "order-item" }, + { "do": "expect", "actor": "customer", "testid": "order-status", "in": { "testid": "order-item", "contains": "Keyboard" }, "statusText": "pending", "ignoreCase": true, "within": 10000 }, + { "do": "reload", "actor": "staff", "settleMs": 2000 }, + { "do": "ensureSignedIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true, "readyTestid": "current-user" }, + {"do": "click", "actor": "staff", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["staff-link", "fulfilment-panel"]}, + { "do": "click", "actor": "staff", "testid": "staff-link", "ifAvailable": true, "unlessVisible": "fulfilment-panel"}, + { "do": "expect", "actor": "staff", "testid": "queue-item", "contains": "Keyboard", "within": 10000 }, + { "do": "click", "actor": "staff", "testid": "ship-submit", "in": { "testid": "queue-item", "contains": "Keyboard" } }, + { "do": "expect", "actor": "staff", "testid": "fulfilment-panel", "attribute": "data-submit-state", "value": "succeeded", "within": 10000 }, + { "do": "reload", "actor": "staff", "settleMs": 2000 }, + { "do": "ensureSignedIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true, "readyTestid": "current-user" }, + {"do": "click", "actor": "staff", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["staff-link", "fulfilment-panel"]}, + { "do": "click", "actor": "staff", "testid": "staff-link", "ifAvailable": true, "unlessVisible": "fulfilment-panel"}, + { "do": "expect", "actor": "staff", "testid": "queue-item", "contains": "Keyboard", "absent": true, "within": 10000 }, + { "do": "reload", "actor": "customer", "settleMs": 2000 }, + { "do": "ensureSignedIn", "actor": "customer", "name": "fq-ship", "readyTestid": "current-user" }, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { "do": "click", "actor": "customer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": ["item-list", "order-item"] }, + {"do": "click", "actor": "customer", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + { "do": "click", "actor": "customer", "testid": "orders-toggle", "unlessVisible": "order-item" }, + { "do": "expect", "actor": "customer", "testid": "order-status", "in": { "testid": "order-item", "contains": "Keyboard" }, "statusText": "shipped", "ignoreCase": true, "within": 10000 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/02-invariants.json b/tools/stack-bench/tracks/ecommerce/scenarios/02-invariants.json new file mode 100644 index 00000000000..652db23ad7e --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/02-invariants.json @@ -0,0 +1,715 @@ +{ + "schemaVersion": 1, + "level": 2, + "track": "ecommerce", + "name": "Running the business — invariants", + "writeUrlPattern": "\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses|\\/transfer|\\/cancel|\\/return|\\/ship|\\/price|\\/fulfil", + "note": "Draft: revised observations require matching reference and defect-control qualification.", + "features": [ + { + "id": 201, + "name": "Operating the store requires being an operator", + "actors": [ + "admin", + "customer" + ], + "setup": [ + { + "do": "signIn", + "actor": "admin", + "name": "admin", + "password": "stackbench-admin-2026", + "exact": true + }, + { + "do": "signUp", + "actor": "customer", + "name": "notstaff" + }, + { + "do": "click", + "actor": "admin", + "testid": "admin-link" + } + ], + "criteria": [ + { + "id": "201a", + "desc": "a customer replaying an admin's transfer moves no stock", + "points": 2, + "steps": [ + { + "do": "fill", + "actor": "admin", + "testid": "transfer-from", + "text": "East", + "in": { + "testid": "admin-item-row", + "contains": "Gaming Mouse" + } + }, + { + "do": "fill", + "actor": "admin", + "testid": "transfer-to", + "text": "West", + "in": { + "testid": "admin-item-row", + "contains": "Gaming Mouse" + } + }, + { + "do": "fill", + "actor": "admin", + "testid": "transfer-qty", + "text": "5", + "in": { + "testid": "admin-item-row", + "contains": "Gaming Mouse" + } + }, + { + "do": "click", + "actor": "admin", + "testid": "transfer-submit", + "in": { + "testid": "admin-item-row", + "contains": "Gaming Mouse" + } + }, + { + "do": "wait", + "actor": "admin", + "ms": 2500 + }, + { + "do": "recordNumber", + "actor": "customer", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Gaming Mouse" + }, + "as": "after-admin-move" + }, + { + "do": "replayAs", + "actor": "customer", + "from": "admin", + "match": "transfer", + "settleMs": 2500 + }, + { + "do": "expectReplayRejected", + "actor": "customer" + }, + { + "do": "expectNumber", + "actor": "customer", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Gaming Mouse" + }, + "relativeTo": "after-admin-move", + "plus": 0 + } + ], + "statedBy": "an admin can transfer a number of units of an item from one warehouse to another" + }, + { + "id": "201b", + "desc": "a customer cannot set a price by replaying an admin's change", + "points": 2, + "steps": [ + { + "do": "fill", + "actor": "admin", + "testid": "price-input", + "text": "77.00", + "in": { + "testid": "admin-item-row", + "contains": "Gaming Mouse" + } + }, + { + "do": "click", + "actor": "admin", + "testid": "price-submit", + "in": { + "testid": "admin-item-row", + "contains": "Gaming Mouse" + } + }, + { + "do": "wait", + "actor": "admin", + "ms": 2500 + }, + { + "do": "replayAs", + "actor": "customer", + "from": "admin", + "match": "price", + "swap": { + "find": "77", + "with": "1" + }, + "settleMs": 2500 + }, + { + "do": "expectReplayRejected", + "actor": "customer" + }, + { + "do": "expectNumber", + "actor": "customer", + "testid": "item-price", + "in": { + "testid": "item-card", + "contains": "Gaming Mouse" + }, + "atLeast": 77 + } + ], + "statedBy": "an admin can change an item's price" + }, + { + "id": "201c", + "desc": "a customer cannot ship their own order", + "points": 0, + "steps": [ + { + "do": "pressKey", + "actor": "customer", + "key": "Escape" + }, + { + "do": "click", + "actor": "customer", + "testid": "buy-now", + "in": { + "testid": "item-card", + "contains": "Laptop Stand" + } + }, + { + "do": "wait", + "actor": "customer", + "ms": 2500 + }, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { + "do": "click", + "actor": "customer", + "testid": "orders-toggle", + "unlessVisible": "order-item" + }, + { + "do": "expect", + "actor": "customer", + "testid": "ship-submit", + "absent": true + }, + { + "do": "expect", + "actor": "customer", + "testid": "order-status", + "in": { + "testid": "order-item", + "contains": "Laptop Stand" + }, + "value": "pending", "ignoreCase": true + } + ], + "statedBy": "staff cannot change prices or create items; admins can do everything staff can", + "note": "DOM diagnostic: checks the ship control is absent for a customer. The SERVER-side refusal of a customer's ship request is now criterion 1e, which replays a real ship as the customer and requires the server to reject it. Kept at 0 points so hiding the button is never mistaken for enforcement." + } + ] + }, + { + "id": 202, + "name": "Stock is conserved however it moves", + "actors": [ + "admin", + "customer" + ], + "setup": [ + { + "do": "signIn", + "actor": "admin", + "name": "admin", + "password": "stackbench-admin-2026", + "exact": true + }, + { + "do": "signUp", + "actor": "customer", + "name": "conserve" + }, + { + "do": "click", + "actor": "admin", + "testid": "admin-link" + } + ], + "criteria": [ + { + "id": "202a", + "desc": "a transfer changes where stock is, never how much there is", + "points": 2, + "steps": [ + { + "do": "recordNumber", + "actor": "admin", + "testid": "admin-stock", + "in": { + "testid": "admin-item-row", + "contains": "Espresso Machine" + }, + "as": "espresso-total" + }, + { + "do": "fill", + "actor": "admin", + "testid": "transfer-from", + "text": "East", + "in": { + "testid": "admin-item-row", + "contains": "Espresso Machine" + } + }, + { + "do": "fill", + "actor": "admin", + "testid": "transfer-to", + "text": "West", + "in": { + "testid": "admin-item-row", + "contains": "Espresso Machine" + } + }, + { + "do": "fill", + "actor": "admin", + "testid": "transfer-qty", + "text": "17", + "in": { + "testid": "admin-item-row", + "contains": "Espresso Machine" + } + }, + { + "do": "click", + "actor": "admin", + "testid": "transfer-submit", + "in": { + "testid": "admin-item-row", + "contains": "Espresso Machine" + } + }, + { + "do": "wait", + "actor": "admin", + "ms": 3000 + }, + { + "do": "expectNumber", + "actor": "admin", + "testid": "admin-stock", + "in": { + "testid": "admin-item-row", + "contains": "Espresso Machine" + }, + "relativeTo": "espresso-total", + "plus": 0 + } + ], + "statedBy": "a transfer moves stock, it does not create or destroy it" + }, + { + "id": "202b", + "desc": "a sale and its cancellation leave the shelf exactly as they found it", + "points": 2, + "steps": [ + { + "do": "recordNumber", + "actor": "customer", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Induction Cooktop" + }, + "as": "cooktop-before" + }, + { + "do": "click", + "actor": "customer", + "testid": "buy-now", + "in": { + "testid": "item-card", + "contains": "Induction Cooktop" + } + }, + { + "do": "wait", + "actor": "customer", + "ms": 2000 + }, + { + "do": "expectNumber", + "actor": "customer", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Induction Cooktop" + }, + "relativeTo": "cooktop-before", + "plus": -1 + }, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { + "do": "click", + "actor": "customer", + "testid": "orders-toggle", + "unlessVisible": "order-item" + }, + { + "do": "click", + "actor": "customer", + "testid": "cancel-order", + "in": { + "testid": "order-item", + "contains": "Induction Cooktop" + } + }, + { + "do": "wait", + "actor": "customer", + "ms": 2500 + }, + { + "do": "pressKey", + "actor": "customer", + "key": "Escape" + }, + { + "do": "expectNumber", + "actor": "customer", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Induction Cooktop" + }, + "relativeTo": "cooktop-before", + "plus": 0 + } + ], + "statedBy": "the stock goes back to the warehouse it came from" + }, + { + "id": "202c", + "desc": "a fresh client sees the same totals, so nothing was only corrected in one browser", + "points": 1, + "steps": [ + { + "do": "freshClient", + "actor": "customer" + }, + { + "do": "expectNumber", + "actor": "customer-fresh", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Induction Cooktop" + }, + "relativeTo": "cooktop-before", + "plus": 0 + } + ], + "statedBy": "every one of these numbers is the same for every person looking at it" + }, + { + "id": "202d", + "desc": "a transfer racing a purchase loses no unit between the rows: total = before − sold, exactly, however the two interleave", + "points": 0, + "note": "WITHHELD until proven against a mutant (transfer-read-modify-write) and a real failing build. A non-transactional transfer reads both rows, computes, writes both — a purchase landing between the read and the write is erased. Sequential tests cannot see this; 202a passes either way.", + "steps": [ + { + "do": "recordNumber", + "actor": "admin", + "testid": "admin-stock", + "in": { + "testid": "admin-item-row", + "contains": "Headphones" + }, + "as": "hp-before" + }, + { + "do": "race", + "branches": [ + [ + { + "do": "fill", + "actor": "admin", + "testid": "transfer-from", + "text": "East", + "in": { + "testid": "admin-item-row", + "contains": "Headphones" + } + }, + { + "do": "fill", + "actor": "admin", + "testid": "transfer-to", + "text": "West", + "in": { + "testid": "admin-item-row", + "contains": "Headphones" + } + }, + { + "do": "fill", + "actor": "admin", + "testid": "transfer-qty", + "text": "25", + "in": { + "testid": "admin-item-row", + "contains": "Headphones" + } + }, + { + "do": "click", + "actor": "admin", + "testid": "transfer-submit", + "in": { + "testid": "admin-item-row", + "contains": "Headphones" + } + } + ], + [ + { + "do": "click", + "actor": "customer", + "testid": "buy-now", + "in": { + "testid": "item-card", + "contains": "Headphones" + } + } + ] + ], + "settleMs": 5000 + }, + { + "do": "expectNumber", + "actor": "admin", + "testid": "admin-stock", + "in": { + "testid": "admin-item-row", + "contains": "Headphones" + }, + "relativeTo": "hp-before", + "plus": -1, + "within": 12000 + } + ], + "statedBy": "a transfer moves stock, it does not create or destroy it" + } + ] + }, + { + "id": 203, + "name": "The books still balance once money can flow backwards", + "actors": [ + "admin", + "customer", + "customer2" + ], + "setup": [ + { + "do": "signIn", + "actor": "admin", + "name": "admin", + "password": "stackbench-admin-2026", + "exact": true + }, + { + "do": "signUp", + "actor": "customer", + "name": "books" + }, + { "do": "signIn", "actor": "customer2", "name": "books" }, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + { + "do": "click", + "actor": "admin", + "testid": "admin-link", + "unlessVisible": "admin-panel" + }, + { "do": "click", "actor": "customer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "203a", + "category": "production", + "desc": "concurrent cancellation restores original stock and the booked amount once, while revenue returns to its prior value", + "points": 3, + "steps": [ + {"do":"recordNumber","actor":"admin","testid":"admin-revenue","as":"rev-start"}, + {"do":"click","actor":"customer","testid":"buy-now","in":{"testid":"item-card","contains":"Desk Lamp"}}, + {"do":"wait","actor":"customer","ms":2500}, + {"do":"reload","actor":"admin","settleMs":1000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","readyTestid":"current-user","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-revenue"]}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":"admin-revenue"}, + {"do":"expectNumber","actor":"admin","testid":"admin-revenue","relativeTo":"rev-start","plus":42,"within":10000}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do": "click", "actor": "customer", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + {"do":"click","actor":"customer","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"customer","testid":"order-status","in":{"testid":"order-item","contains":"Desk Lamp"},"statusText":"pending","ignoreCase":true,"within":10000}, + {"do":"dbRecordCheckout","account":"{user:books}","item":"Desk Lamp","as":"cancel-before","storage":{"kind":"order-data","cart":false,"warehouses":true}}, + {"do":"callConcurrently","actors":["customer","customer2"],"from":"customer","action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"requests":4,"input":{"testid":"order-item","contains":"Desk Lamp","attribute":"data-cancel-input"},"settleMs":3000}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCancellation","before":"cancel-before"}, + {"do":"reload","actor":"admin","settleMs":1000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","readyTestid":"current-user","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-revenue"]}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":"admin-revenue"}, + {"do":"expectNumber","actor":"admin","testid":"admin-revenue","relativeTo":"rev-start","plus":0,"within":10000} + ], + "statedBy": "revenue always equals the sum of orders that are still standing", + "note": "Draft: native order, refund and original warehouse allocation evidence supplements the retained revenue assertions. Four calls from two sessions of the owner must cancel once; a repeated call may deliberately refuse or succeed without extra effects. Qualification pending." + }, + { + "id": "203b", + "category": "production", + "desc": "a price change does not rewrite revenue already earned", + "points": 2, + "steps": [ + {"do":"recordNumber","actor":"admin","testid":"admin-revenue","as":"history-revenue-before-sale"}, + {"do":"pressKey","actor":"customer","key":"Escape"}, + {"do":"click","actor":"customer","testid":"buy-now","in":{"testid":"item-card","contains":"Bluetooth Speaker"}}, + {"do":"wait","actor":"customer","ms":2500}, + {"do":"reload","actor":"admin","settleMs":1000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","readyTestid":"current-user","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":"admin-revenue"}, + {"do":"expectNumber","actor":"admin","testid":"admin-revenue","relativeTo":"history-revenue-before-sale","plus":79.5,"within":10000}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"customer","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"customer","testid":"order-item","contains":"Bluetooth Speaker","count":1,"within":10000}, + {"do":"recordNumber","actor":"admin","testid":"admin-revenue","as":"rev-after-sale"}, + {"do":"fill","actor":"admin","testid":"price-input","text":"5.00","in":{"testid":"admin-item-row","contains":"Bluetooth Speaker"}}, + {"do":"click","actor":"admin","testid":"price-submit","in":{"testid":"admin-item-row","contains":"Bluetooth Speaker"}}, + {"do":"wait","actor":"admin","ms":3000}, + {"do":"reload","actor":"admin","settleMs":2000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true,"readyTestid":"current-user"}, + {"do":"click","actor":"admin","testid":"admin-link","ifAvailable":true,"within":1000}, + {"do":"expectNumber","actor":"admin","testid":"price-input","in":{"testid":"admin-item-row","contains":"Bluetooth Speaker"},"equals":5,"within":10000}, + {"do":"reload","actor":"admin","settleMs":1000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","readyTestid":"current-user","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":"admin-revenue"}, + {"do":"expectNumber","actor":"admin","testid":"admin-revenue","relativeTo":"rev-after-sale","plus":0} + ], + "statedBy": "changing a price never alters the history, the revenue already recorded" + } + ] + }, + { + "id": 204, + "name": "An order belongs to the person who placed it", + "actors": [ + "one", + "two" + ], + "setup": [ + { + "do": "signUp", + "actor": "one", + "name": "owner" + }, + { + "do": "signUp", + "actor": "two", + "name": "other" + } + ], + "criteria": [ + { + "id": "204a", + "desc": "one customer cannot cancel another's order", + "points": 0, + "note": "WITHHELD: the current replay happens after the owner's cancellation, so a state-only rejection can masquerade as an ownership check. Restore points only after the harness can replay a captured cancellation against a still-pending order and a cross-account authorization mutant is caught.", + "steps": [ + { + "do": "click", + "actor": "one", + "testid": "buy-now", + "in": { + "testid": "item-card", + "contains": "Laptop Stand" + } + }, + { + "do": "wait", + "actor": "one", + "ms": 2500 + }, + {"do":"click","actor":"one","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { + "do": "click", + "actor": "one", + "testid": "orders-toggle" + }, + { + "do": "click", + "actor": "one", + "testid": "cancel-order", + "in": { + "testid": "order-item", + "contains": "Laptop Stand" + } + }, + { + "do": "wait", + "actor": "one", + "ms": 2000 + }, + { + "do": "replayAs", + "actor": "two", + "from": "one", + "match": "cancel", + "settleMs": 2500 + }, + { + "do": "expectReplayRejected", + "actor": "two" + }, + {"do":"click","actor":"two","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { + "do": "click", + "actor": "two", + "testid": "orders-toggle" + }, + { + "do": "expect", + "actor": "two", + "testid": "order-item", + "contains": "Laptop Stand", + "absent": true + } + ], + "statedBy": "a customer can cancel an order that has not shipped" + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/02-live-price.json b/tools/stack-bench/tracks/ecommerce/scenarios/02-live-price.json new file mode 100644 index 00000000000..6c223fa14e8 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/02-live-price.json @@ -0,0 +1,34 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 2, + "name": "Ecommerce L2 live catalog price", + "note": "The signed-out live-price check owns its catalog state and does not share a mutation grade with unrelated workflows.", + "features": [ + { + "id": 4, + "name": "Live catalog price", + "actors": ["admin", "visitor"], + "setup": [ + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + { "do": "click", "actor": "admin", "testid": "admin-link" }, + { "do": "fill", "actor": "visitor", "testid": "search-input", "text": "Air Purifier", "enter": true, "settleMs": 1500 } + ], + "criteria": [ + { + "id": "4b", + "category": "production", + "desc": "the new price reaches a signed-out visitor without a reload", + "points": 2, + "statedBy": "the storefront shows the new price immediately, to everyone", + "steps": [ + { "do": "expectNumber", "actor": "visitor", "testid": "item-price", "in": { "testid": "item-card", "contains": "Air Purifier" }, "atLeast": 2 }, + { "do": "fill", "actor": "admin", "testid": "price-input", "text": "1.00", "in": { "testid": "admin-item-row", "contains": "Air Purifier" } }, + { "do": "click", "actor": "admin", "testid": "price-submit", "in": { "testid": "admin-item-row", "contains": "Air Purifier" } }, + { "do": "expectNumber", "actor": "visitor", "testid": "item-price", "in": { "testid": "item-card", "contains": "Air Purifier" }, "equals": 1, "within": 10000 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/02-low-stock.json b/tools/stack-bench/tracks/ecommerce/scenarios/02-low-stock.json new file mode 100644 index 00000000000..5b2becc62c3 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/02-low-stock.json @@ -0,0 +1,58 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 2, + "name": "Ecommerce L2 low-stock boundary", + "note": "Draft: repaired probe behavior needs matching qualification evidence. This scenario owns only the low-stock boundary check so a failure cannot prevent unrelated operational-view checks from running.", + "features": [ + { + "id": 5, + "name": "The low-stock view", + "actors": ["admin", "customer", "restocker"], + "setup": [ + { "do": "dbSetStock", "item": "Air Purifier", "warehouse": "East", "quantity": 2, "settleMs": 250 }, + { "do": "dbSetStock", "item": "Air Purifier", "warehouse": "West", "quantity": 1, "settleMs": 250 }, + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + { "do": "signUp", "actor": "customer", "name": "low-stock-focused" }, + { "do": "reload", "actor": "admin", "settleMs": 1000 }, + { "do": "ensureSignedIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true, "readyTestid": "current-user" }, + { "do": "reload", "actor": "customer", "settleMs": 1000 }, + { "do": "ensureSignedIn", "actor": "customer", "name": "low-stock-focused", "readyTestid": "current-user" }, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel", "low-stock-list"]}, + { "do": "click", "actor": "admin", "testid": "admin-link", "unlessVisible": ["admin-panel", "low-stock-list"]}, + { "do": "click", "actor": "admin", "testid": "low-stock-link", "ifAvailable": true, "unlessVisible": "low-stock-list", "within": 10000 }, + { "do": "click", "actor": "customer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "5e", + "category": "feature", + "desc": "the dashboard lists a current low-stock item", + "points": 1, + "statedBy": "It lists items with 10 units or fewer, most urgent first.", + "steps": [ + { "do": "expect", "actor": "admin", "testid": "low-stock-item", "contains": "Air Purifier", "within": 8000 } + ] + }, + { + "id": "5a", + "category": "production", + "desc": "an item falling to ten units or fewer joins the low-stock list, live", + "points": 3, + "statedBy": "items enter and leave this list as stock moves, sells, is restocked, cancelled or returned", + "steps": [ + { "do": "expect", "actor": "admin", "testid": "low-stock-item", "contains": "Air Purifier", "within": 8000 }, + { "do": "signIn", "actor": "restocker", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + {"do": "click", "actor": "restocker", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + { "do": "click", "actor": "restocker", "testid": "admin-link", "unlessVisible": "admin-panel" }, + { "do": "fill", "actor": "restocker", "testid": "restock-input", "text": "8", "in": { "testid": "admin-location-row", "contains": "Air Purifier" } }, + { "do": "click", "actor": "restocker", "testid": "restock-submit", "in": { "testid": "admin-location-row", "contains": "Air Purifier" } }, + { "do": "waitUntilAbsent", "actor": "admin", "testid": "low-stock-item", "contains": "Air Purifier", "within": 10000 }, + { "do": "click", "actor": "customer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Air Purifier" } }, + { "do": "expect", "actor": "admin", "testid": "low-stock-item", "contains": "Air Purifier", "within": 10000 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/02-operational-best-sellers.json b/tools/stack-bench/tracks/ecommerce/scenarios/02-operational-best-sellers.json new file mode 100644 index 00000000000..4b49e16a123 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/02-operational-best-sellers.json @@ -0,0 +1,37 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 2, + "name": "Ecommerce L2 signed-out best sellers", + "note": "This scenario owns only the signed-out recommendation check.", + "features": [ + { + "id": 5, + "name": "Signed-out best sellers", + "actors": ["buyer", "visitor"], + "setup": [ + { "do": "signUp", "actor": "buyer", "name": "best-seller-buyer" }, + { "do": "click", "actor": "buyer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "click", "actor": "buyer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Gaming Mouse" } }, + {"do":"click","actor":"buyer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do": "click", "actor": "buyer", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + { "do": "click", "actor": "buyer", "testid": "orders-toggle", "unlessVisible": "order-item" }, + { "do": "expect", "actor": "buyer", "testid": "order-item", "contains": "Gaming Mouse", "within": 10000 } + ], + "criteria": [ + { + "id": "5d", + "category": "feature", + "desc": "a signed-out visitor sees a best seller in the recommendations list", + "points": 1, + "statedBy": "Signed-out visitors see best sellers", + "steps": [ + { "do": "reload", "actor": "visitor", "settleMs": 2500 }, + { "do": "click", "actor": "visitor", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "expectNumber", "actor": "visitor", "testid": "recommendation-rank", "in": { "testid": "recommended-item", "contains": "Gaming Mouse" }, "equals": 1, "within": 10000 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/02-operational-category-totals.json b/tools/stack-bench/tracks/ecommerce/scenarios/02-operational-category-totals.json new file mode 100644 index 00000000000..4d1562f467f --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/02-operational-category-totals.json @@ -0,0 +1,58 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 2, + "name": "Ecommerce L2 category totals", + "note": "This scenario owns only the category sales totals check.", + "features": [ + { + "id": 5, + "name": "Category sales totals", + "actors": ["admin", "customer"], + "setup": [ + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + { "do": "signUp", "actor": "customer", "name": "category-totals" }, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + { "do": "click", "actor": "admin", "testid": "admin-link", "unlessVisible": "admin-panel" }, + { "do": "click", "actor": "customer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "5f", + "category": "feature", + "desc": "the dashboard shows category units and revenue", + "points": 1, + "statedBy": "Category totals show units sold and revenue for each category.", + "steps": [ + {"do":"click","actor":"admin","testid":"sales-link","unlessVisible":"category-row","ifAvailable":true}, + { "do": "recordNumber", "actor": "admin", "testid": "category-units", "in": { "testid": "category-row", "contains": "Audio" }, "as": "audio-core-units" }, + { "do": "recordNumber", "actor": "admin", "testid": "category-revenue", "in": { "testid": "category-row", "contains": "Audio" }, "as": "audio-core-revenue" }, + { "do": "click", "actor": "customer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Bluetooth Speaker" } }, + { "do": "reload", "actor": "admin", "settleMs": 2000 }, + { "do": "ensureSignedIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true, "readyTestid": "current-user" }, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel", "category-row"]}, + { "do": "click", "actor": "admin", "testid": "admin-link", "unlessVisible": ["admin-panel", "category-row"]}, + {"do":"click","actor":"admin","testid":"sales-link","unlessVisible":"category-row","ifAvailable":true}, + { "do": "expectNumber", "actor": "admin", "testid": "category-units", "in": { "testid": "category-row", "contains": "Audio" }, "relativeTo": "audio-core-units", "plus": 1 }, + { "do": "expectNumber", "actor": "admin", "testid": "category-revenue", "in": { "testid": "category-row", "contains": "Audio" }, "relativeTo": "audio-core-revenue", "plus": 79.5 } + ] + }, + { + "id": "5b", + "category": "production", + "desc": "a purchase updates that category's units and revenue live", + "points": 3, + "statedBy": "Category totals show units sold and revenue for each category", + "steps": [ + {"do":"click","actor":"admin","testid":"sales-link","unlessVisible":"category-row","ifAvailable":true}, + { "do": "recordNumber", "actor": "admin", "testid": "category-units", "in": { "testid": "category-row", "contains": "Audio" }, "as": "audio-units" }, + { "do": "recordNumber", "actor": "admin", "testid": "category-revenue", "in": { "testid": "category-row", "contains": "Audio" }, "as": "audio-revenue" }, + { "do": "click", "actor": "customer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Bluetooth Speaker" } }, + { "do": "expectNumber", "actor": "admin", "testid": "category-units", "in": { "testid": "category-row", "contains": "Audio" }, "relativeTo": "audio-units", "plus": 1, "within": 10000 }, + { "do": "expectNumber", "actor": "admin", "testid": "category-revenue", "in": { "testid": "category-row", "contains": "Audio" }, "relativeTo": "audio-revenue", "plus": 79.5, "within": 10000 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/02-operational-recommendations.json b/tools/stack-bench/tracks/ecommerce/scenarios/02-operational-recommendations.json new file mode 100644 index 00000000000..ffa12c82af8 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/02-operational-recommendations.json @@ -0,0 +1,35 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 2, + "name": "Ecommerce L2 recommendations", + "note": "Draft: updated observations require matching reference and defect-control qualification.", + "features": [ + { + "id": 5, + "name": "Customer recommendations", + "actors": ["customer"], + "setup": [ + { "do": "signUp", "actor": "customer", "name": "recommendations" }, + { "do": "click", "actor": "customer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "5c", + "category": "feature", + "desc": "a purchase recommends another item from that category and excludes an item in the cart", + "points": 2, + "statedBy": "Recommend items from categories the customer bought from and exclude items already in the cart", + "steps": [ + { "do": "click", "actor": "customer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Bluetooth Speaker" } }, + { "do": "click", "actor": "customer", "testid": "overlay-close", "ifAvailable": true }, + { "do": "click", "actor": "customer", "testid": "catalog-link", "ifAvailable": true }, + { "do": "expect", "actor": "customer", "testid": "recommended-item", "in": { "testid": "recommended-list" }, "contains": "Headphones", "within": 10000 }, + { "do": "click", "actor": "customer", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Headphones" } }, + { "do": "waitUntilAbsent", "actor": "customer", "testid": "recommended-item", "in": { "testid": "recommended-list" }, "contains": "Headphones", "within": 10000 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/02-order-cancellation-core.json b/tools/stack-bench/tracks/ecommerce/scenarios/02-order-cancellation-core.json new file mode 100644 index 00000000000..a694246ffdb --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/02-order-cancellation-core.json @@ -0,0 +1,53 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 2, + "name": "Order cancellation", + "features": [ + { + "id": 3, + "name": "Cancel a pending order", + "actors": ["customer", "admin"], + "setup": [ + { "do": "signUp", "actor": "customer", "name": "cancel-core" }, + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + { "do": "click", "actor": "admin", "testid": "admin-link", "unlessVisible": "admin-panel" }, + { "do": "click", "actor": "customer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "3a", + "category": "production", + "desc": "cancelling a pending order restores its stock and revenue", + "points": 2, + "statedBy": "the stock goes back to the warehouse it came from and revenue falls", + "steps": [ + { "do": "recordNumber", "actor": "admin", "testid": "admin-revenue", "as": "revenue-before" }, + { "do": "recordNumber", "actor": "customer", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Coffee Grinder" }, "as": "stock-before" }, + { "do": "click", "actor": "customer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Coffee Grinder" } }, + { "do": "expectNumber", "actor": "customer", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Coffee Grinder" }, "relativeTo": "stock-before", "plus": -1, "within": 10000 }, + { "do": "reload", "actor": "admin", "settleMs": 2000 }, + { "do": "ensureSignedIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true, "readyTestid": "current-user" }, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + { "do": "click", "actor": "admin", "testid": "admin-link", "ifAvailable": true, "unlessVisible": "admin-panel"}, + { "do": "expectNumber", "actor": "admin", "testid": "admin-revenue", "relativeTo": "revenue-before", "plus": 64, "within": 10000 }, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do": "click", "actor": "customer", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + { "do": "click", "actor": "customer", "testid": "orders-toggle", "settleMs": 2000, "unlessVisible": "order-item" }, + { "do": "expect", "actor": "customer", "testid": "order-status", "in": { "testid": "order-item", "contains": "Coffee Grinder" }, "statusText": "pending", "ignoreCase": true }, + { "do": "click", "awaitWrites": true, "actor": "customer", "testid": "cancel-order", "in": { "testid": "order-item", "contains": "Coffee Grinder" } }, + { "do": "reload", "actor": "admin", "settleMs": 2000 }, + { "do": "ensureSignedIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true, "readyTestid": "current-user" }, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + { "do": "click", "actor": "admin", "testid": "admin-link", "ifAvailable": true, "unlessVisible": "admin-panel"}, + { "do": "expectNumber", "actor": "admin", "testid": "admin-revenue", "relativeTo": "revenue-before", "plus": 0, "within": 10000 }, + { "do": "click", "actor": "customer", "testid": "overlay-close", "ifAvailable": true }, + { "do": "click", "actor": "customer", "testid": "catalog-link", "ifAvailable": true }, + { "do": "expectNumber", "actor": "customer", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Coffee Grinder" }, "relativeTo": "stock-before", "plus": 0, "within": 10000 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/02-order-cancellation-history.json b/tools/stack-bench/tracks/ecommerce/scenarios/02-order-cancellation-history.json new file mode 100644 index 00000000000..ea51f424c34 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/02-order-cancellation-history.json @@ -0,0 +1,34 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 2, + "name": "Order cancellation history", + "features": [ + { + "id": 3, + "name": "Cancellation history", + "actors": ["customer"], + "setup": [ + { "do": "signUp", "actor": "customer", "name": "cancel-focused" }, + { "do": "click", "actor": "customer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "3b", + "category": "feature", + "desc": "a cancelled order is shown as cancelled in the customer's history", + "points": 1, + "statedBy": "a customer can cancel an order that has not shipped", + "steps": [ + { "do": "click", "actor": "customer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Coffee Grinder" }, "settleMs": 2000 }, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do": "click", "actor": "customer", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + { "do": "click", "actor": "customer", "testid": "orders-toggle", "unlessVisible": "order-item" }, + { "do": "click", "actor": "customer", "testid": "cancel-order", "in": { "testid": "order-item", "contains": "Coffee Grinder" } }, + { "do": "expect", "actor": "customer", "testid": "order-status", "in": { "testid": "order-item", "contains": "Coffee Grinder" }, "statusText": "cancelled", "ignoreCase": true, "within": 10000 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/02-paid-price-history.json b/tools/stack-bench/tracks/ecommerce/scenarios/02-paid-price-history.json new file mode 100644 index 00000000000..a515af9b7a8 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/02-paid-price-history.json @@ -0,0 +1,42 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 2, + "name": "Ecommerce L2 paid price history", + "note": "The receipt check reloads the customer before reading history, proving persisted order data rather than a stale in-memory response.", + "features": [ + { + "id": 4, + "name": "Prices change, history does not", + "actors": ["admin", "customer", "visitor"], + "setup": [ + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + { "do": "signUp", "actor": "customer", "name": "history-persisted" }, + { "do": "click", "actor": "admin", "testid": "admin-link" } + ], + "criteria": [ + { + "id": "4a", + "category": "production", + "desc": "a price change updates the live catalog but leaves the customer's exact paid price unchanged", + "points": 3, + "statedBy": "past orders keep the price that was paid", + "steps": [ + { "do": "fill", "actor": "customer", "testid": "search-input", "text": "Air Purifier", "enter": true, "settleMs": 1500 }, + { "do": "recordNumber", "actor": "customer", "testid": "item-price", "in": { "testid": "item-card", "contains": "Air Purifier" }, "as": "air-purifier-paid" }, + { "do": "click", "actor": "customer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Air Purifier" }, "settleMs": 2000 }, + { "do": "fill", "actor": "visitor", "testid": "search-input", "text": "Air Purifier", "enter": true, "settleMs": 1500 }, + { "do": "fill", "actor": "admin", "testid": "price-input", "text": "1.00", "in": { "testid": "admin-item-row", "contains": "Air Purifier" } }, + { "do": "click", "actor": "admin", "testid": "price-submit", "in": { "testid": "admin-item-row", "contains": "Air Purifier" } }, + { "do": "expectNumber", "actor": "visitor", "testid": "item-price", "in": { "testid": "item-card", "contains": "Air Purifier" }, "equals": 1, "within": 10000 }, + { "do": "reload", "actor": "customer", "settleMs": 2000 }, + { "do": "ensureSignedIn", "actor": "customer", "name": "history-persisted", "readyTestid": "current-user" }, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { "do": "click", "actor": "customer", "testid": "orders-toggle" }, + { "do": "expectNumber", "actor": "customer", "testid": "order-total", "in": { "testid": "order-item", "contains": "Air Purifier" }, "relativeTo": "air-purifier-paid", "plus": 0 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/02-queue-warehouse.json b/tools/stack-bench/tracks/ecommerce/scenarios/02-queue-warehouse.json new file mode 100644 index 00000000000..320e0d1158e --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/02-queue-warehouse.json @@ -0,0 +1,40 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 2, + "name": "Ecommerce L2 fulfilment warehouse", + "note": "The warehouse-label check owns its order and queue state so another fulfilment check cannot affect its evidence.", + "features": [ + { + "id": 1, + "name": "Fulfilment queue", + "actors": ["customer", "staff"], + "setup": [ + { "do": "signUp", "actor": "customer", "name": "fq-warehouse" }, + { "do": "signIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true }, + {"do": "click", "actor": "staff", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["staff-link", "fulfilment-panel"]}, + { "do": "click", "actor": "staff", "testid": "staff-link", "unlessVisible": "fulfilment-panel" }, + { "do": "click", "actor": "customer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "1b", + "category": "feature", + "desc": "the queue names the warehouse the order will ship from", + "points": 1, + "statedBy": "which warehouse each will ship from", + "steps": [ + { "do": "dbSetStock", "item": "Desk Lamp", "warehouse": "West", "quantity": 0, "settleMs": 2000 }, + { "do": "click", "actor": "customer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Desk Lamp" }, "settleMs": 1500 }, + { "do": "reload", "actor": "staff", "settleMs": 2000 }, + { "do": "ensureSignedIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true, "readyTestid": "current-user" }, + {"do": "click", "actor": "staff", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["staff-link", "fulfilment-panel"]}, + { "do": "click", "actor": "staff", "testid": "staff-link", "ifAvailable": true, "unlessVisible": "fulfilment-panel"}, + { "do": "expect", "actor": "staff", "testid": "queue-item", "contains": "Desk Lamp", "within": 10000 }, + { "do": "expect", "actor": "staff", "testid": "queue-warehouse", "in": { "testid": "queue-item", "contains": "Desk Lamp" }, "contains": "East" } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/02-self-contained.json b/tools/stack-bench/tracks/ecommerce/scenarios/02-self-contained.json new file mode 100644 index 00000000000..e9d343d9769 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/02-self-contained.json @@ -0,0 +1,222 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 2, + "name": "Ecommerce L2 independently selectable checks", + "note": "Each criterion establishes every state and recorded value that it later observes, so it remains valid when selected by itself.", + "features": [ + { + "id": 1, + "name": "Fulfilment queue", + "actors": ["customer", "staff"], + "setup": [ + { "do": "signUp", "actor": "customer", "name": "fq-focused" }, + { "do": "signIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true }, + { "do": "click", "actor": "staff", "testid": "staff-link" } + ], + "criteria": [ + { + "id": "1b", + "desc": "the queue names the warehouse the order will ship from", + "points": 1, + "statedBy": "which warehouse each will ship from", + "steps": [ + { "do": "dbSetStock", "item": "Desk Lamp", "warehouse": "West", "quantity": 0, "settleMs": 2000 }, + { "do": "click", "actor": "customer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Desk Lamp" }, "settleMs": 1500 }, + { "do": "reload", "actor": "staff", "settleMs": 2000 }, + { "do": "ensureSignedIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true, "readyTestid": "current-user" }, + { "do": "click", "actor": "staff", "testid": "staff-link", "ifAvailable": true }, + { "do": "expect", "actor": "staff", "testid": "queue-item", "contains": "Desk Lamp", "within": 10000 }, + { "do": "expect", "actor": "staff", "testid": "queue-warehouse", "in": { "testid": "queue-item", "contains": "Desk Lamp" }, "contains": "East" } + ] + }, + { + "id": "1c", + "desc": "shipping removes the order from the queue for staff and marks it shipped for the customer, live", + "points": 2, + "statedBy": "it leaves the queue for everyone watching the queue, and the customer's order history shows it as shipped, live", + "steps": [ + { "do": "click", "actor": "customer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Keyboard" } }, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { "do": "click", "actor": "customer", "testid": "orders-toggle", "unlessVisible": "order-item" }, + { "do": "expect", "actor": "customer", "testid": "order-status", "in": { "testid": "order-item", "contains": "Keyboard" }, "value": "pending", "ignoreCase": true, "within": 10000 }, + { "do": "expect", "actor": "staff", "testid": "queue-item", "contains": "Keyboard", "within": 10000 }, + { "do": "click", "actor": "staff", "testid": "ship-submit", "in": { "testid": "queue-item", "contains": "Keyboard" } }, + { "do": "expect", "actor": "staff", "testid": "queue-item", "contains": "Keyboard", "absent": true, "within": 10000 }, + { "do": "expect", "actor": "customer", "testid": "order-status", "in": { "testid": "order-item", "contains": "Keyboard" }, "value": "shipped", "ignoreCase": true, "within": 10000 } + ] + }, + { + "id": "1e", + "desc": "the server refuses a customer's direct attempt to ship their own order", + "points": 2, + "statedBy": "Staff mark an order shipped", + "steps": [ + {"do":"click","actor":"customer","testid":"buy-now","in":{"testid":"item-card","contains":"Keyboard"},"settleMs":2000}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"customer","testid":"orders-toggle", "unlessVisible": "order-item" }, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true}, + {"do":"click","actor":"customer","testid":"catalog-link","ifAvailable":true}, + {"do":"click","actor":"customer","testid":"buy-now","in":{"testid":"item-card","contains":"Coffee Grinder"},"settleMs":2000}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"customer","testid":"orders-toggle", "unlessVisible": "order-item" }, + {"do":"expect","actor":"customer","testid":"order-item","contains":"Coffee Grinder","count":1,"within":10000}, + {"do":"callAction","actor":"staff","action":"ship","namedAction":{"id":"ship","path":"/api/fulfilment/ship","reducer":"ship_order","args":[0],"params":[{"name":"orderId","in":"body","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Coffee Grinder","attribute":"data-ship-input"},"settleMs":2000,"from":"customer"}, + {"do":"expectActionOutcome","actor":"staff","outcome":"accepted"}, + {"do":"reload","actor":"customer","settleMs":2000}, + {"do":"ensureSignedIn","actor":"customer","name":"fq-focused","readyTestid":"current-user"}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"customer","testid":"orders-toggle", "unlessVisible": "order-item" }, + {"do":"expect","actor":"customer","testid":"order-status","in":{"testid":"order-item","contains":"Coffee Grinder"},"value":"shipped", "ignoreCase": true,"within":10000}, + {"do":"callAction","actor":"customer","action":"ship","namedAction":{"id":"ship","path":"/api/fulfilment/ship","reducer":"ship_order","args":[0],"params":[{"name":"orderId","in":"body","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Keyboard","attribute":"data-ship-input"},"settleMs":2000}, + {"do":"expectActionOutcome","actor":"customer","outcome":"refused","routeProvenBy":"staff"}, + {"do":"reload","actor":"customer","settleMs":2000}, + {"do":"ensureSignedIn","actor":"customer","name":"fq-focused","readyTestid":"current-user"}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"customer","testid":"orders-toggle", "unlessVisible": "order-item" }, + {"do":"expect","actor":"customer","testid":"order-status","in":{"testid":"order-item","contains":"Keyboard"},"value":"pending", "ignoreCase": true} + ] + } + ] + }, + { + "id": 2, + "name": "Warehouse totals", + "actors": ["admin"], + "setup": [ + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + { "do": "click", "actor": "admin", "testid": "admin-link" } + ], + "criteria": [ + { + "id": "2b", + "desc": "both warehouse totals move live and in opposite directions as stock is transferred", + "points": 2, + "statedBy": "the per-warehouse numbers staff and admins see both move at once", + "steps": [ + {"do":"expect","actor":"admin","testid":"admin-warehouse-item","contains":"East","count":1}, + { "do": "recordNumber", "actor": "admin", "testid": "warehouse-total", "in": { "testid": "admin-warehouse-item", "contains": "East" }, "as": "warehouse-east-before" }, + {"do":"expect","actor":"admin","testid":"admin-warehouse-item","contains":"West","count":1}, + { "do": "recordNumber", "actor": "admin", "testid": "warehouse-total", "in": { "testid": "admin-warehouse-item", "contains": "West" }, "as": "warehouse-west-before" }, + { "do": "fill", "actor": "admin", "testid": "transfer-from", "text": "East", "in": { "testid": "admin-item-row", "contains": "Headphones" } }, + { "do": "fill", "actor": "admin", "testid": "transfer-to", "text": "West", "in": { "testid": "admin-item-row", "contains": "Headphones" } }, + { "do": "fill", "actor": "admin", "testid": "transfer-qty", "text": "10", "in": { "testid": "admin-item-row", "contains": "Headphones" } }, + { "do": "click", "actor": "admin", "testid": "transfer-submit", "in": { "testid": "admin-item-row", "contains": "Headphones" } }, + {"do":"expect","actor":"admin","testid":"admin-warehouse-item","contains":"East","count":1}, + { "do": "expectNumber", "actor": "admin", "testid": "warehouse-total", "in": { "testid": "admin-warehouse-item", "contains": "East" }, "relativeTo": "warehouse-east-before", "plus": -10, "within": 10000 }, + {"do":"expect","actor":"admin","testid":"admin-warehouse-item","contains":"West","count":1}, + { "do": "expectNumber", "actor": "admin", "testid": "warehouse-total", "in": { "testid": "admin-warehouse-item", "contains": "West" }, "relativeTo": "warehouse-west-before", "plus": 10, "within": 10000 } + ] + } + ] + }, + { + "id": 3, + "name": "Cancellation history", + "actors": ["customer"], + "setup": [ + { "do": "signUp", "actor": "customer", "name": "cancel-focused" } + ], + "criteria": [ + { + "id": "3b", + "desc": "a cancelled order is shown as cancelled in the customer's history", + "points": 1, + "statedBy": "a customer can cancel an order that has not shipped", + "steps": [ + { "do": "click", "actor": "customer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Coffee Grinder" }, "settleMs": 2000 }, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { "do": "click", "actor": "customer", "testid": "orders-toggle", "unlessVisible": "order-item" }, + { "do": "click", "actor": "customer", "testid": "cancel-order", "in": { "testid": "order-item", "contains": "Coffee Grinder" } }, + { "do": "expect", "actor": "customer", "testid": "order-status", "in": { "testid": "order-item", "contains": "Coffee Grinder" }, "value": "cancelled", "ignoreCase": true, "within": 10000 } + ] + } + ] + }, + { + "id": 4, + "name": "Live catalog price", + "actors": ["admin", "visitor"], + "setup": [ + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + { "do": "click", "actor": "admin", "testid": "admin-link" } + ], + "criteria": [ + { + "id": "4b", + "desc": "the new price reaches a signed-out visitor without a reload", + "points": 2, + "statedBy": "the storefront shows the new price immediately, to everyone", + "steps": [ + { "do": "expectNumber", "actor": "visitor", "testid": "item-price", "in": { "testid": "item-card", "contains": "Air Purifier" }, "atLeast": 2 }, + { "do": "fill", "actor": "admin", "testid": "price-input", "text": "1.00", "in": { "testid": "admin-item-row", "contains": "Air Purifier" } }, + { "do": "click", "actor": "admin", "testid": "price-submit", "in": { "testid": "admin-item-row", "contains": "Air Purifier" } }, + { "do": "expectNumber", "actor": "visitor", "testid": "item-price", "in": { "testid": "item-card", "contains": "Air Purifier" }, "equals": 1, "within": 10000 } + ] + } + ] + }, + { + "id": 202, + "name": "Stock recovery is durable across clients", + "actors": ["customer"], + "setup": [ + { "do": "signUp", "actor": "customer", "name": "fresh-stock" }, + { "do": "click", "actor": "customer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "202b", + "category": "production", + "desc": "a sale and its cancellation leave the shelf exactly as they found it", + "points": 2, + "statedBy": "the stock goes back to the warehouse it came from", + "steps": [ + {"do":"dbRecordStock","item":"Induction Cooktop","warehouse":"East","as":"east-before-cancel-202b"}, + {"do":"dbRecordStock","item":"Induction Cooktop","warehouse":"West","as":"west-before-cancel-202b"}, + {"do":"dbRecordStock","item":"Induction Cooktop","as":"stored-before-cancel-202b"}, + {"do":"recordNumber","actor":"customer","testid":"item-stock","in":{"testid":"item-card","contains":"Induction Cooktop"},"as":"cancel-stock-before"}, + {"do":"click","actor":"customer","testid":"buy-now","in":{"testid":"item-card","contains":"Induction Cooktop"},"settleMs":2000}, + {"do":"expectNumber","actor":"customer","testid":"item-stock","in":{"testid":"item-card","contains":"Induction Cooktop"},"relativeTo":"cancel-stock-before","plus":-1}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do": "click", "actor": "customer", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + {"do":"click","actor":"customer","testid":"orders-toggle", "unlessVisible": "order-item" }, + {"do":"expect","actor":"customer","testid":"order-item","contains":"Induction Cooktop","count":1,"within":10000}, + {"do":"dbExpectStock","item":"Induction Cooktop","relativeTo":"stored-before-cancel-202b","plus":-1}, + {"do":"click","actor":"customer","testid":"cancel-order","in":{"testid":"order-item","contains":"Induction Cooktop"},"settleMs":2000}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true}, + {"do":"click","actor":"customer","testid":"catalog-link","ifAvailable":true}, + {"do":"expectNumber","actor":"customer","testid":"item-stock","in":{"testid":"item-card","contains":"Induction Cooktop"},"relativeTo":"cancel-stock-before","plus":0}, + {"do":"dbExpectStock","item":"Induction Cooktop","relativeTo":"stored-before-cancel-202b","plus":0}, + {"do":"dbExpectStock","item":"Induction Cooktop","warehouse":"East","relativeTo":"east-before-cancel-202b","plus":0}, + {"do":"dbExpectStock","item":"Induction Cooktop","warehouse":"West","relativeTo":"west-before-cancel-202b","plus":0} + ] + }, + { + "id": "202c", + "category": "production", + "desc": "a fresh client sees the restored total after a sale is cancelled", + "points": 1, + "statedBy": "every one of these numbers is the same for every person looking at it", + "steps": [ + {"do":"dbRecordStock","item":"Headphones","warehouse":"East","as":"east-before-cancel-202c"}, + {"do":"dbRecordStock","item":"Headphones","warehouse":"West","as":"west-before-cancel-202c"}, + {"do":"dbRecordStock","item":"Headphones","as":"stored-before-cancel-202c"}, + {"do":"recordNumber","actor":"customer","testid":"item-stock","in":{"testid":"item-card","contains":"Headphones"},"as":"fresh-stock-before"}, + {"do":"click","actor":"customer","testid":"buy-now","in":{"testid":"item-card","contains":"Headphones"},"settleMs":2000}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"customer","testid":"orders-toggle", "unlessVisible": "order-item" }, + {"do":"expect","actor":"customer","testid":"order-item","contains":"Headphones","count":1,"within":10000}, + {"do":"dbExpectStock","item":"Headphones","relativeTo":"stored-before-cancel-202c","plus":-1}, + {"do":"click","actor":"customer","testid":"cancel-order","in":{"testid":"order-item","contains":"Headphones"},"settleMs":2000}, + {"do":"freshClient","actor":"customer"}, + {"do":"expectNumber","actor":"customer-fresh","testid":"item-stock","in":{"testid":"item-card","contains":"Headphones"},"relativeTo":"fresh-stock-before","plus":0}, + {"do":"dbExpectStock","item":"Headphones","relativeTo":"stored-before-cancel-202c","plus":0}, + {"do":"dbExpectStock","item":"Headphones","warehouse":"East","relativeTo":"east-before-cancel-202c","plus":0}, + {"do":"dbExpectStock","item":"Headphones","warehouse":"West","relativeTo":"west-before-cancel-202c","plus":0} + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/02-server-actions.json b/tools/stack-bench/tracks/ecommerce/scenarios/02-server-actions.json new file mode 100644 index 00000000000..f3effed9a8f --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/02-server-actions.json @@ -0,0 +1,222 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 2, + "name": "Ecommerce L2 direct server checks", + "note": "Draft: revised observations and uncertain click delivery require matching reference and defect-control qualification.", + "features": [ + { + "id": 201, + "name": "Shipping requires an operator", + "actors": [ + "customer", + "staff" + ], + "setup": [ + {"do":"signIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true}, + {"do":"signUp","actor":"customer","name":"direct-notstaff"}, + { "do": "click", "actor": "customer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do":"click","actor":"customer","testid":"buy-now","in":{"testid":"item-card","contains":"Laptop Stand"},"settleMs":2000}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do": "click", "actor": "customer", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + {"do":"click","actor":"customer","testid":"orders-toggle", "unlessVisible": "order-item" } + ], + "criteria": [ + { + "id": "201c", + "category": "production", + "desc": "customers cannot ship a pending order, including by submitting a staff role; legitimate staff can still ship it", + "points": 2, + "statedBy": "Staff mark an order shipped", + "steps": [ + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"customer","testid":"catalog-link","ifAvailable":true,"within":1000}, + {"do":"click","actor":"customer","testid":"buy-now","in":{"testid":"item-card","contains":"Coffee Grinder"},"settleMs":2000}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do": "click", "actor": "customer", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + {"do":"click","actor":"customer","testid":"orders-toggle", "unlessVisible": "order-item" }, + {"do":"expect","actor":"customer","testid":"order-item","contains":"Coffee Grinder","count":1,"within":10000}, + {"do":"callAction","actor":"staff","action":"ship","namedAction":{"id":"ship","path":"/api/fulfilment/ship","reducer":"ship_order","args":[0],"params":[{"name":"orderId","in":"body","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Coffee Grinder","attribute":"data-ship-input"},"settleMs":2000,"from":"customer"}, + {"do":"expectActionOutcome","actor":"staff","outcome":"accepted"}, + {"do":"reload","actor":"customer","settleMs":2000}, + {"do":"ensureSignedIn","actor":"customer","name":"direct-notstaff","readyTestid":"current-user"}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { "do": "click", "actor": "customer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": ["item-list", "order-item"] }, + {"do": "click", "actor": "customer", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + {"do":"click","actor":"customer","testid":"orders-toggle", "unlessVisible": "order-item" }, + {"do":"expect","actor":"customer","testid":"order-status","in":{"testid":"order-item","contains":"Coffee Grinder"},"statusText":"shipped", "ignoreCase": true,"within":10000}, + {"do":"callAction","actor":"customer","action":"ship","namedAction":{"id":"ship","path":"/api/fulfilment/ship","reducer":"ship_order","args":[0],"params":[{"name":"orderId","in":"body","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Laptop Stand","attribute":"data-ship-input"},"settleMs":2000}, + {"do":"expectActionOutcome","actor":"customer","outcome":"refused","routeProvenBy":"staff"}, + {"do":"reload","actor":"customer","settleMs":2000}, + {"do":"ensureSignedIn","actor":"customer","name":"direct-notstaff","readyTestid":"current-user"}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { "do": "click", "actor": "customer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": ["item-list", "order-item"] }, + {"do": "click", "actor": "customer", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + {"do":"click","actor":"customer","testid":"orders-toggle", "unlessVisible": "order-item" }, + {"do":"expect","actor":"customer","testid":"order-status","in":{"testid":"order-item","contains":"Laptop Stand"},"statusText":"pending", "ignoreCase": true}, + {"do":"callAction","actor":"customer","action":"ship","namedAction":{"id":"ship","path":"/api/fulfilment/ship","reducer":"ship_order","args":[0,"staff"],"params":[{"name":"orderId","in":"body","wireType":"u64"},{"name":"role","in":"body"}]},"input":{"testid":"order-item","contains":"Laptop Stand","attribute":"data-ship-input","fields":["orderId"]},"settleMs":0}, + {"do":"expectActionOutcome","actor":"customer","outcome":"completed","routeProvenBy":"staff"}, + {"do":"reload","actor":"customer","settleMs":1000}, + {"do":"ensureSignedIn","actor":"customer","name":"direct-notstaff","readyTestid":"current-user"}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { "do": "click", "actor": "customer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": ["item-list", "order-item"] }, + {"do": "click", "actor": "customer", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + {"do":"click","actor":"customer","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"customer","testid":"order-status","in":{"testid":"order-item","contains":"Laptop Stand"},"statusText":"pending","ignoreCase":true}, + {"do":"expectActionOutcome","actor":"customer","outcome":"application-refused","routeProvenBy":"staff"}, + {"do":"callAction","actor":"staff","from":"customer","action":"ship","namedAction":{"id":"ship","path":"/api/fulfilment/ship","reducer":"ship_order","args":[0],"params":[{"name":"orderId","in":"body","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Laptop Stand","attribute":"data-ship-input"},"settleMs":0}, + {"do":"expectActionOutcome","actor":"staff","outcome":"accepted"}, + {"do":"reload","actor":"customer","settleMs":1000}, + {"do":"ensureSignedIn","actor":"customer","name":"direct-notstaff","readyTestid":"current-user"}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { "do": "click", "actor": "customer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": ["item-list", "order-item"] }, + {"do": "click", "actor": "customer", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + {"do":"click","actor":"customer","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"customer","testid":"order-status","in":{"testid":"order-item","contains":"Laptop Stand"},"statusText":"shipped","ignoreCase":true,"within":10000} + ] + } + ] + }, + { + "id": 202, + "name": "Stock is conserved while operations overlap", + "actors": ["admin", "customer", "reverseAdmin"], + "setup": [ + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + { "do": "signUp", "actor": "customer", "name": "direct-conserve" }, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + { "do": "click", "actor": "admin", "testid": "admin-link", "unlessVisible": "admin-panel" } + ], + "criteria": [ + { + "id": "202d", + "category": "production", + "desc": "overlapping purchases and opposing transfers preserve exact warehouse stock and complete valid operations", + "points": 2, + "statedBy": "a transfer moves stock, it does not create or destroy it", + "steps": [ + {"do":"dbExpectStock","item":"Headphones","warehouse":"East","equals":60}, + {"do":"dbExpectStock","item":"Headphones","warehouse":"West","equals":40}, + {"do":"dbRecordStock","item":"Headphones","as":"direct-race-stock-before"}, + {"do":"reload","actor":"admin","settleMs":2000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true,"readyTestid":"current-user"}, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + {"do":"click","actor":"admin","testid":"admin-link","ifAvailable":true,"within":1000, "unlessVisible": "admin-panel"}, + {"do":"reload","actor":"customer","settleMs":2000}, + {"do":"ensureSignedIn","actor":"customer","name":"direct-conserve","readyTestid":"current-user"}, + {"do":"fill","actor":"admin","testid":"transfer-from","text":"East","in":{"testid":"admin-item-row","contains":"Headphones"}}, + {"do":"fill","actor":"admin","testid":"transfer-to","text":"West","in":{"testid":"admin-item-row","contains":"Headphones"}}, + {"do":"fill","actor":"admin","testid":"transfer-qty","text":"25","in":{"testid":"admin-item-row","contains":"Headphones"}}, + { "do": "click", "actor": "customer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do":"race","settleMs":5000,"branches":[[{"do":"callAction","actor":"admin","action":"transfer","namedAction":{"id":"transfer","path":"/api/admin/transfer","reducer":"admin_transfer_stock","args":[0,0,0,25],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"fromWarehouseId","in":"body","wireType":"u64"},{"name":"toWarehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-item-row","contains":"Headphones","attribute":"data-transfer-input","fields":["itemId","fromWarehouseId","toWarehouseId"]}}],[{"do":"callAction","actor":"customer","action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Headphones","attribute":"data-buy-input"}}]]}, + {"do":"expectActionOutcome","actor":"admin","outcome":"accepted"}, + {"do":"expectActionOutcome","actor":"customer","outcome":"accepted"}, + {"do":"reload","actor":"admin","settleMs":2000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true,"readyTestid":"current-user"}, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + {"do":"click","actor":"admin","testid":"admin-link","ifAvailable":true,"within":1000, "unlessVisible": "admin-panel"}, + {"do":"expectNumber","actor":"admin","testid":"admin-stock","in":{"testid":"admin-item-row","contains":"Headphones"},"relativeTo":"direct-race-stock-before","plus":-1,"within":12000}, + {"do":"reload","actor":"customer","settleMs":2000}, + {"do":"ensureSignedIn","actor":"customer","name":"direct-conserve","readyTestid":"current-user"}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"customer","testid":"catalog-link","ifAvailable":true,"within":1000}, + {"do":"expectNumber","actor":"customer","testid":"item-stock","in":{"testid":"item-card","contains":"Headphones"},"relativeTo":"direct-race-stock-before","plus":-1,"within":12000}, + {"do":"dbExpectStock","item":"Headphones","warehouse":"East","atLeast":34,"atMost":35}, + {"do":"dbExpectStock","item":"Headphones","warehouse":"West","atLeast":64,"atMost":65}, + {"do":"dbExpectStock","item":"Headphones","relativeTo":"direct-race-stock-before","plus":-1}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do": "click", "actor": "customer", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + {"do":"click","actor":"customer","testid":"orders-toggle", "unlessVisible": "order-item" }, + {"do":"expect","actor":"customer","testid":"order-item","contains":"Headphones","count":1,"within":10000}, + {"do":"signIn","actor":"reverseAdmin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do": "click", "actor": "reverseAdmin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + {"do":"click","actor":"reverseAdmin","testid":"admin-link", "unlessVisible": "admin-panel"}, + {"do":"fill","actor":"admin","testid":"transfer-from","text":"East","in":{"testid":"admin-item-row","contains":"Headphones"}}, + {"do":"fill","actor":"admin","testid":"transfer-to","text":"West","in":{"testid":"admin-item-row","contains":"Headphones"}}, + {"do":"fill","actor":"reverseAdmin","testid":"transfer-from","text":"West","in":{"testid":"admin-item-row","contains":"Headphones"}}, + {"do":"fill","actor":"reverseAdmin","testid":"transfer-to","text":"East","in":{"testid":"admin-item-row","contains":"Headphones"}}, + {"do":"dbRecordStock","item":"Headphones","warehouse":"East","as":"opposing-East"}, + {"do":"dbRecordStock","item":"Headphones","warehouse":"West","as":"opposing-West"}, + {"do":"callConcurrently","actors":["admin"],"action":"transfer","namedAction":{"id":"transfer","path":"/api/admin/transfer","reducer":"admin_transfer_stock","args":[0,0,0,7],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"fromWarehouseId","in":"body","wireType":"u64"},{"name":"toWarehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-item-row","contains":"Headphones","attribute":"data-transfer-input","fields":["itemId","fromWarehouseId","toWarehouseId"]},"requests":1,"settleMs":0,"alongside":[{"actors":["reverseAdmin"],"action":"transfer","namedAction":{"id":"transfer","path":"/api/admin/transfer","reducer":"admin_transfer_stock","args":[0,0,0,4],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"fromWarehouseId","in":"body","wireType":"u64"},{"name":"toWarehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-item-row","contains":"Headphones","attribute":"data-transfer-input","fields":["itemId","fromWarehouseId","toWarehouseId"]},"requests":1}]}, + {"do":"expectCallOutcomes","accepted":2}, + {"do":"dbExpectStock","item":"Headphones","warehouse":"East","relativeTo":"opposing-East","plus":-3}, + {"do":"dbExpectStock","item":"Headphones","warehouse":"West","relativeTo":"opposing-West","plus":3}, + {"do":"dbRecordStock","item":"Headphones","warehouse":"East","as":"opposing-East"}, + {"do":"dbRecordStock","item":"Headphones","warehouse":"West","as":"opposing-West"}, + {"do":"callConcurrently","actors":["admin"],"action":"transfer","namedAction":{"id":"transfer","path":"/api/admin/transfer","reducer":"admin_transfer_stock","args":[0,0,0,4],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"fromWarehouseId","in":"body","wireType":"u64"},{"name":"toWarehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-item-row","contains":"Headphones","attribute":"data-transfer-input","fields":["itemId","fromWarehouseId","toWarehouseId"]},"requests":1,"settleMs":0,"alongside":[{"actors":["reverseAdmin"],"action":"transfer","namedAction":{"id":"transfer","path":"/api/admin/transfer","reducer":"admin_transfer_stock","args":[0,0,0,9],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"fromWarehouseId","in":"body","wireType":"u64"},{"name":"toWarehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-item-row","contains":"Headphones","attribute":"data-transfer-input","fields":["itemId","fromWarehouseId","toWarehouseId"]},"requests":1}]}, + {"do":"expectCallOutcomes","accepted":2}, + {"do":"dbExpectStock","item":"Headphones","warehouse":"East","relativeTo":"opposing-East","plus":5}, + {"do":"dbExpectStock","item":"Headphones","warehouse":"West","relativeTo":"opposing-West","plus":-5}, + {"do":"dbRecordStock","item":"Headphones","warehouse":"East","as":"opposing-East"}, + {"do":"dbRecordStock","item":"Headphones","warehouse":"West","as":"opposing-West"}, + {"do":"callConcurrently","actors":["admin"],"action":"transfer","namedAction":{"id":"transfer","path":"/api/admin/transfer","reducer":"admin_transfer_stock","args":[0,0,0,8],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"fromWarehouseId","in":"body","wireType":"u64"},{"name":"toWarehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-item-row","contains":"Headphones","attribute":"data-transfer-input","fields":["itemId","fromWarehouseId","toWarehouseId"]},"requests":1,"settleMs":0,"alongside":[{"actors":["reverseAdmin"],"action":"transfer","namedAction":{"id":"transfer","path":"/api/admin/transfer","reducer":"admin_transfer_stock","args":[0,0,0,3],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"fromWarehouseId","in":"body","wireType":"u64"},{"name":"toWarehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-item-row","contains":"Headphones","attribute":"data-transfer-input","fields":["itemId","fromWarehouseId","toWarehouseId"]},"requests":1}]}, + {"do":"expectCallOutcomes","accepted":2}, + {"do":"dbExpectStock","item":"Headphones","warehouse":"East","relativeTo":"opposing-East","plus":-5}, + {"do":"dbExpectStock","item":"Headphones","warehouse":"West","relativeTo":"opposing-West","plus":5} + ] + } + ] + }, + { + "id": 204, + "name": "An order belongs to the person who placed it", + "actors": ["owner", "other"], + "setup": [ + { "do": "signUp", "actor": "owner", "name": "direct-owner", "exact": true }, + { "do": "signUp", "actor": "other", "name": "direct-other" }, + { "do": "click", "actor": "owner", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "callAction", "actor": "owner", "action": "buy", "namedAction": { "id": "buy", "path": "/api/items/:id/buy", "reducer": "buy_now", "args": [0], "params": [{ "name": "itemId", "in": "path", "placeholder": ":id", "wireType": "u64" }] }, "input": { "testid": "item-card", "contains": "Desk Lamp", "attribute": "data-buy-input" } }, + + { "do": "expectActionOutcome", "actor": "owner", "outcome": "accepted" }, + { "do": "callAction", "actor": "owner", "action": "buy", "namedAction": { "id": "buy", "path": "/api/items/:id/buy", "reducer": "buy_now", "args": [0], "params": [{ "name": "itemId", "in": "path", "placeholder": ":id", "wireType": "u64" }] }, "input": { "testid": "item-card", "contains": "Keyboard", "attribute": "data-buy-input" } }, + + { "do": "expectActionOutcome", "actor": "owner", "outcome": "accepted" }, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do": "click", "actor": "owner", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + { "do": "click", "actor": "owner", "testid": "orders-toggle", "unlessVisible": "order-item" } + ], + "criteria": [ + { + "id": "204a", + "category": "production", + "desc": "customers cannot cancel another account's order, including by submitting its owner's username; the owner can still cancel it", + "points": 2, + "statedBy": "A customer can cancel an order that has not shipped", + "steps": [ + {"do":"callAction","actor":"owner","action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Desk Lamp","attribute":"data-cancel-input"},"settleMs":2000}, + {"do":"expectActionOutcome","actor":"owner","outcome":"accepted"}, + {"do":"callAction","actor":"other","from":"owner","action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Keyboard","attribute":"data-cancel-input"},"settleMs":2000}, + {"do":"expectActionOutcome","actor":"other","outcome":"completed","routeProvenBy":"owner"}, + {"do":"reload","actor":"owner","settleMs":1000}, + {"do":"ensureSignedIn","actor":"owner","name":"direct-owner","exact":true,"readyTestid":"current-user"}, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"within":1000}, + { "do": "click", "actor": "owner", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "owner", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + {"do":"click","actor":"owner","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"owner","testid":"order-status","in":{"testid":"order-item","contains":"Keyboard"},"statusText":"pending","ignoreCase":true}, + {"do":"expectActionOutcome","actor":"other","outcome":"application-refused","routeProvenBy":"owner"}, + {"do":"callAction","actor":"other","from":"owner","action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0,"direct-owner"],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"},{"name":"username","in":"body"}]},"input":{"testid":"order-item","contains":"Keyboard","attribute":"data-cancel-input","fields":["orderId"]},"settleMs":0}, + {"do":"expectActionOutcome","actor":"other","outcome":"completed","routeProvenBy":"owner"}, + {"do":"reload","actor":"owner","settleMs":1000}, + {"do":"ensureSignedIn","actor":"owner","name":"direct-owner","exact":true,"readyTestid":"current-user"}, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"within":1000}, + { "do": "click", "actor": "owner", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "owner", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + {"do":"click","actor":"owner","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"owner","testid":"order-status","in":{"testid":"order-item","contains":"Keyboard"},"statusText":"pending","ignoreCase":true}, + {"do":"expectActionOutcome","actor":"other","outcome":"application-refused","routeProvenBy":"owner"}, + {"do":"callAction","actor":"owner","action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Keyboard","attribute":"data-cancel-input"},"settleMs":0}, + {"do":"expectActionOutcome","actor":"owner","outcome":"accepted"}, + {"do":"reload","actor":"owner","settleMs":1000}, + {"do":"ensureSignedIn","actor":"owner","name":"direct-owner","exact":true,"readyTestid":"current-user"}, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"within":1000}, + { "do": "click", "actor": "owner", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "owner", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + {"do":"click","actor":"owner","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"owner","testid":"order-status","in":{"testid":"order-item","contains":"Keyboard"},"statusText":"cancelled","ignoreCase":true} + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/02-strengthened.json b/tools/stack-bench/tracks/ecommerce/scenarios/02-strengthened.json new file mode 100644 index 00000000000..80f136d9058 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/02-strengthened.json @@ -0,0 +1,299 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 2, + "name": "Ecommerce L2 strengthened observations", + "note": "Draft: revised observations require matching reference and defect-control qualification.", + "features": [ + { + "id": 2, + "name": "Moving stock between warehouses", + "actors": [ + "admin", + "visitor" + ], + "setup": [ + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + {"do":"click","actor":"admin","testid":"admin-link", "unlessVisible": "admin-panel"}, + { "do": "click", "actor": "visitor", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "2a", + "category": "production", + "desc": "a transfer decreases the source, increases the destination, and preserves the item's exact total", + "points": 3, + "statedBy": "a transfer moves stock, it does not create or destroy it", + "steps": [ + {"do":"dbRecordStock","item":"Headphones","warehouse":"East","as":"product-East"}, + {"do":"dbRecordStock","item":"Headphones","warehouse":"West","as":"product-West"}, + {"do":"recordNumber","actor":"visitor","testid":"item-stock","in":{"testid":"item-card","contains":"Headphones"},"as":"transfer-item-before"}, + {"do":"recordNumber","actor":"admin","testid":"warehouse-total","in":{"testid":"admin-warehouse-item","contains":"East"},"as":"transfer-east-before"}, + {"do":"recordNumber","actor":"admin","testid":"warehouse-total","in":{"testid":"admin-warehouse-item","contains":"West"},"as":"transfer-west-before"}, + {"do":"fill","actor":"admin","testid":"transfer-from","text":"East","in":{"testid":"admin-item-row","contains":"Headphones"}}, + {"do":"fill","actor":"admin","testid":"transfer-to","text":"West","in":{"testid":"admin-item-row","contains":"Headphones"}}, + {"do":"fill","actor":"admin","testid":"transfer-qty","text":"10","in":{"testid":"admin-item-row","contains":"Headphones"}}, + {"do":"click","actor":"admin","testid":"transfer-submit","in":{"testid":"admin-item-row","contains":"Headphones"}}, + {"do":"dbExpectStock","item":"Headphones","warehouse":"East","relativeTo":"product-East","plus":-10,"within":10000}, + {"do":"dbExpectStock","item":"Headphones","warehouse":"West","relativeTo":"product-West","plus":10,"within":10000}, + {"do":"expectNumber","actor":"admin","testid":"warehouse-total","in":{"testid":"admin-warehouse-item","contains":"East"},"relativeTo":"transfer-east-before","plus":-10,"within":10000}, + {"do":"expectNumber","actor":"admin","testid":"warehouse-total","in":{"testid":"admin-warehouse-item","contains":"West"},"relativeTo":"transfer-west-before","plus":10,"within":10000}, + {"do":"expectNumber","actor":"visitor","testid":"item-stock","in":{"testid":"item-card","contains":"Headphones"},"relativeTo":"transfer-item-before","plus":0,"within":10000} + ] + } + ] + }, + { + "id": 3, + "name": "Returning a shipped item", + "actors": [ + "customer", + "staff", + "admin" + ], + "setup": [ + {"do":"signUp","actor":"customer","name":"return-focused"}, + {"do":"signIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"staff","testid":"staff-link"}, + {"do":"click","actor":"admin","testid":"admin-link"} + ], + "criteria": [ + { + "id": "3c", + "desc": "returning a shipped item restores stock and revenue and visibly marks the item returned", + "points": 3, + "statedBy": "the stock comes back, revenue falls by what was paid for it, and the order shows the item as returned", + "steps": [ + {"do":"dbRecordStock","item":"Keyboard","as":"stored-before-return"}, + {"do":"recordNumber","actor":"customer","testid":"item-stock","in":{"testid":"item-card","contains":"Keyboard"},"as":"return-stock-before"}, + {"do":"recordNumber","actor":"admin","testid":"admin-revenue","as":"return-revenue-before"}, + {"do":"click","actor":"customer","testid":"buy-now","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"expect","actor":"staff","testid":"queue-item","contains":"Keyboard","within":10000}, + {"do":"click","actor":"staff","testid":"ship-submit","in":{"testid":"queue-item","contains":"Keyboard"}}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"customer","testid":"orders-toggle","settleMs":1500}, + {"do":"expect","actor":"customer","testid":"order-status","in":{"testid":"order-item","contains":"Keyboard"},"value":"shipped","ignoreCase":true,"within":10000}, + {"do":"expectElementCount","actor":"customer","testid":"order-item","contains":"Keyboard","equals":1,"within":10000}, + {"do":"dbExpectStock","item":"Keyboard","relativeTo":"stored-before-return","plus":-1}, + {"do":"expectNumber","actor":"admin","testid":"admin-revenue","relativeTo":"return-revenue-before","plus":89,"within":10000}, + {"do":"click","actor":"customer","testid":"return-item","in":{"testid":"order-item","contains":"Keyboard"}}, + {"do":"expect","actor":"customer","testid":"order-item","contains":"eturned","within":10000}, + {"do":"expectNumber","actor":"admin","testid":"admin-revenue","relativeTo":"return-revenue-before","plus":0,"within":10000}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"customer","testid":"catalog-link","ifAvailable":true,"within":1000}, + {"do":"expectNumber","actor":"customer","testid":"item-stock","in":{"testid":"item-card","contains":"Keyboard"},"relativeTo":"return-stock-before","plus":0}, + {"do":"dbExpectStock","item":"Keyboard","relativeTo":"stored-before-return","plus":0} + ] + } + ] + }, + { + "id": 4, + "name": "Prices change, history does not", + "actors": [ + "admin", + "customer", + "visitor" + ], + "setup": [ + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"signUp","actor":"customer","name":"history-focused"}, + {"do":"click","actor":"admin","testid":"admin-link"} + ], + "criteria": [ + { + "id": "4a", + "desc": "a price change updates the live catalog but leaves the customer's exact paid price unchanged", + "points": 3, + "statedBy": "past orders keep the price that was paid", + "steps": [ + {"do":"recordNumber","actor":"customer","testid":"item-price","in":{"testid":"item-card","contains":"Air Purifier"},"as":"air-purifier-paid"}, + {"do":"click","actor":"customer","testid":"buy-now","in":{"testid":"item-card","contains":"Air Purifier"},"settleMs":2000}, + {"do":"fill","actor":"admin","testid":"price-input","text":"1.00","in":{"testid":"admin-item-row","contains":"Air Purifier"}}, + {"do":"click","actor":"admin","testid":"price-submit","in":{"testid":"admin-item-row","contains":"Air Purifier"}}, + {"do":"expectNumber","actor":"visitor","testid":"item-price","in":{"testid":"item-card","contains":"Air Purifier"},"equals":1,"within":10000}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"customer","testid":"orders-toggle"}, + {"do":"expectNumber","actor":"customer","testid":"order-total","in":{"testid":"order-item","contains":"Air Purifier"},"relativeTo":"air-purifier-paid","plus":0} + ] + } + ] + }, + { + "id": 5, + "name": "Best sellers count only kept purchases", + "actors": [ + "customer", + "staff", + "visitor" + ], + "setup": [ + {"do":"signUp","actor":"customer","name":"best-seller-focused"}, + {"do":"signIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true}, + {"do":"click","actor":"staff","testid":"staff-link"} + ], + "criteria": [ + { + "id": "5d", + "desc": "signed-out best sellers include kept purchases and exclude purchases that are cancelled or returned", + "points": 1, + "statedBy": "A cancelled or returned order's items stop counting as purchases", + "steps": [ + {"do":"click","actor":"customer","testid":"buy-now","in":{"testid":"item-card","contains":"Desk Lamp"}}, + {"do":"expect","actor":"visitor","testid":"recommended-item","contains":"Desk Lamp","within":10000}, + {"do":"click","actor":"customer","testid":"buy-now","in":{"testid":"item-card","contains":"Bluetooth Speaker"}}, + {"do":"expect","actor":"visitor","testid":"recommended-item","contains":"Bluetooth Speaker","within":10000}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"customer","testid":"orders-toggle"}, + {"do":"click","actor":"customer","testid":"cancel-order","in":{"testid":"order-item","contains":"Bluetooth Speaker"}}, + {"do":"expect","actor":"visitor","testid":"recommended-item","contains":"Bluetooth Speaker","absent":true,"within":10000}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"customer","testid":"catalog-link","ifAvailable":true,"within":1000}, + {"do":"click","actor":"customer","testid":"buy-now","in":{"testid":"item-card","contains":"Coffee Grinder"}}, + {"do":"expect","actor":"visitor","testid":"recommended-item","contains":"Coffee Grinder","within":10000}, + {"do":"expect","actor":"staff","testid":"queue-item","contains":"Coffee Grinder","within":10000}, + {"do":"click","actor":"staff","testid":"ship-submit","in":{"testid":"queue-item","contains":"Coffee Grinder"}}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"customer","testid":"orders-toggle","settleMs":1500}, + {"do":"click","actor":"customer","testid":"return-item","in":{"testid":"order-item","contains":"Coffee Grinder"}}, + {"do":"expect","actor":"visitor","testid":"recommended-item","contains":"Coffee Grinder","absent":true,"within":10000}, + {"do":"expect","actor":"visitor","testid":"recommended-item","contains":"Desk Lamp","within":10000} + ] + } + ] + }, + { + "id": 201, + "name": "Operating the store requires authorization", + "actors": [ + "admin", + "customer" + ], + "setup": [ + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"signUp","actor":"customer","name":"not-operator"}, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + {"do":"click","actor":"admin","testid":"admin-link", "unlessVisible": "admin-panel"} + ], + "criteria": [ + { + "id": "201a", + "category": "production", + "desc": "the server refuses a customer's direct transfer and neither warehouse nor the item total changes", + "points": 2, + "statedBy": "An admin can transfer a number of units of an item from one warehouse to another", + "steps": [ + {"do":"dbRecordStock","item":"Headphones","warehouse":"East","as":"authorized-transfer-east"}, + {"do":"dbRecordStock","item":"Headphones","warehouse":"West","as":"authorized-transfer-west"}, + {"do":"fill","actor":"admin","testid":"transfer-from","text":"East","in":{"testid":"admin-item-row","contains":"Headphones"}}, + {"do":"fill","actor":"admin","testid":"transfer-to","text":"West","in":{"testid":"admin-item-row","contains":"Headphones"}}, + {"do":"fill","actor":"admin","testid":"transfer-qty","text":"25","in":{"testid":"admin-item-row","contains":"Headphones"}}, + {"do":"callAction","actor":"admin","action":"transfer","namedAction":{"id":"transfer","path":"/api/admin/transfer","reducer":"admin_transfer_stock","args":[0,0,0,25],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"fromWarehouseId","in":"body","wireType":"u64"},{"name":"toWarehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-item-row","contains":"Headphones","attribute":"data-transfer-input","fields":["itemId","fromWarehouseId","toWarehouseId"]},"settleMs":2000}, + {"do":"expectActionOutcome","actor":"admin","outcome":"accepted"}, + {"do":"dbExpectStock","item":"Headphones","warehouse":"East","relativeTo":"authorized-transfer-east","plus":-25}, + {"do":"dbExpectStock","item":"Headphones","warehouse":"West","relativeTo":"authorized-transfer-west","plus":25}, + {"do":"reload","actor":"admin","settleMs":2000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true,"readyTestid":"current-user"}, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + {"do":"click","actor":"admin","testid":"admin-link","ifAvailable":true,"within":1000, "unlessVisible": "admin-panel"}, + {"do":"reload","actor":"customer","settleMs":2000}, + {"do":"ensureSignedIn","actor":"customer","name":"not-operator","readyTestid":"current-user"}, + { "do": "click", "actor": "customer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do":"recordNumber","actor":"customer","testid":"item-stock","in":{"testid":"item-card","contains":"Headphones"},"as":"unauthorized-item-before"}, + {"do":"recordNumber","actor":"admin","testid":"warehouse-total","in":{"testid":"admin-warehouse-item","contains":"East"},"as":"unauthorized-east-before"}, + {"do":"recordNumber","actor":"admin","testid":"warehouse-total","in":{"testid":"admin-warehouse-item","contains":"West"},"as":"unauthorized-west-before"}, + {"do":"fill","actor":"admin","testid":"transfer-from","text":"East","in":{"testid":"admin-item-row","contains":"Headphones"}}, + {"do":"fill","actor":"admin","testid":"transfer-to","text":"West","in":{"testid":"admin-item-row","contains":"Headphones"}}, + {"do":"dbRecordStock","item":"Headphones","warehouse":"East","as":"refused-East"}, + {"do":"dbRecordStock","item":"Headphones","warehouse":"West","as":"refused-West"}, + {"do":"callAction","actor":"customer","from":"admin","action":"transfer","namedAction":{"id":"transfer","path":"/api/admin/transfer","reducer":"admin_transfer_stock","args":[0,0,0,25],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"fromWarehouseId","in":"body","wireType":"u64"},{"name":"toWarehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-item-row","contains":"Headphones","attribute":"data-transfer-input","fields":["itemId","fromWarehouseId","toWarehouseId"]},"settleMs":2000}, + {"do":"expectActionOutcome","actor":"customer","outcome":"refused","routeProvenBy":"admin"}, + {"do":"reload","actor":"admin","settleMs":2000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true,"readyTestid":"current-user"}, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + {"do":"click","actor":"admin","testid":"admin-link","ifAvailable":true,"within":1000, "unlessVisible": "admin-panel"}, + {"do":"reload","actor":"customer","settleMs":2000}, + {"do":"ensureSignedIn","actor":"customer","name":"not-operator","readyTestid":"current-user"}, + {"do":"dbExpectStock","item":"Headphones","warehouse":"East","relativeTo":"refused-East","plus":0}, + {"do":"dbExpectStock","item":"Headphones","warehouse":"West","relativeTo":"refused-West","plus":0}, + {"do":"expectNumber","actor":"admin","testid":"warehouse-total","in":{"testid":"admin-warehouse-item","contains":"East"},"relativeTo":"unauthorized-east-before","plus":0}, + {"do":"expectNumber","actor":"admin","testid":"warehouse-total","in":{"testid":"admin-warehouse-item","contains":"West"},"relativeTo":"unauthorized-west-before","plus":0}, + { "do": "click", "actor": "customer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do":"expectNumber","actor":"customer","testid":"item-stock","in":{"testid":"item-card","contains":"Headphones"},"relativeTo":"unauthorized-item-before","plus":0} + ] + }, + { + "id": "201b", + "category": "production", + "desc": "the server refuses a customer's direct price change and the last accepted price remains exact", + "points": 2, + "statedBy": "an admin can change an item's price", + "steps": [ + {"do":"fill","actor":"admin","testid":"price-input","text":"77.00","in":{"testid":"admin-item-row","contains":"Gaming Mouse"}}, + {"do":"callAction","actor":"admin","action":"price","namedAction":{"id":"price","path":"/api/admin/price","reducer":"admin_change_price","args":[0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"price","in":"body"}]},"input":{"testid":"admin-item-row","contains":"Gaming Mouse","attribute":"data-price-input"},"settleMs":2000}, + {"do":"expectActionOutcome","actor":"admin","outcome":"accepted"}, + {"do":"reload","actor":"customer","settleMs":2000}, + {"do":"ensureSignedIn","actor":"customer","name":"not-operator","readyTestid":"current-user"}, + {"do":"expectNumber","actor":"customer","testid":"item-price","in":{"testid":"item-card","contains":"Gaming Mouse"},"equals":77,"within":10000}, + {"do":"fill","actor":"admin","testid":"price-input","text":"1.00","in":{"testid":"admin-item-row","contains":"Gaming Mouse"}}, + {"do":"callAction","actor":"customer","from":"admin","action":"price","namedAction":{"id":"price","path":"/api/admin/price","reducer":"admin_change_price","args":[0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"price","in":"body"}]},"input":{"testid":"admin-item-row","contains":"Gaming Mouse","attribute":"data-price-input"},"settleMs":2000}, + {"do":"expectActionOutcome","actor":"customer","outcome":"refused","routeProvenBy":"admin"}, + {"do":"reload","actor":"admin","settleMs":2000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true,"readyTestid":"current-user"}, + {"do":"click","actor":"admin","testid":"admin-link","ifAvailable":true,"within":1000}, + {"do":"reload","actor":"customer","settleMs":2000}, + {"do":"ensureSignedIn","actor":"customer","name":"not-operator","readyTestid":"current-user"}, + {"do":"expectNumber","actor":"customer","testid":"item-price","in":{"testid":"item-card","contains":"Gaming Mouse"},"equals":77,"within":10000} + ] + } + ] + }, + { + "id": 202, + "name": "Stock is conserved however it moves", + "actors": [ + "admin", + "customer" + ], + "setup": [ + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"signUp","actor":"customer","name":"directional-stock"}, + {"do":"click","actor":"admin","testid":"admin-link"} + ], + "criteria": [ + { + "id": "202a", + "category": "production", + "desc": "a transfer decreases East, increases West, and leaves the item's exact total unchanged", + "points": 2, + "statedBy": "a transfer moves stock, it does not create or destroy it", + "steps": [ + {"do":"dbRecordStock","item":"Espresso Machine","warehouse":"East","as":"product-East"}, + {"do":"dbRecordStock","item":"Espresso Machine","warehouse":"West","as":"product-West"}, + {"do":"recordNumber","actor":"customer","testid":"item-stock","in":{"testid":"item-card","contains":"Espresso Machine"},"as":"conservation-item-before"}, + {"do":"expect","actor":"admin","testid":"admin-warehouse-item","contains":"East","count":1}, + {"do":"recordNumber","actor":"admin","testid":"warehouse-total","in":{"testid":"admin-warehouse-item","contains":"East"},"as":"conservation-east-before"}, + {"do":"expect","actor":"admin","testid":"admin-warehouse-item","contains":"West","count":1}, + {"do":"recordNumber","actor":"admin","testid":"warehouse-total","in":{"testid":"admin-warehouse-item","contains":"West"},"as":"conservation-west-before"}, + {"do":"fill","actor":"admin","testid":"transfer-from","text":"East","in":{"testid":"admin-item-row","contains":"Espresso Machine"}}, + {"do":"fill","actor":"admin","testid":"transfer-to","text":"West","in":{"testid":"admin-item-row","contains":"Espresso Machine"}}, + {"do":"fill","actor":"admin","testid":"transfer-qty","text":"17","in":{"testid":"admin-item-row","contains":"Espresso Machine"}}, + {"do":"click","actor":"admin","testid":"transfer-submit","in":{"testid":"admin-item-row","contains":"Espresso Machine"}}, + {"do":"expect","actor":"admin","testid":"admin-warehouse-item","contains":"East","count":1}, + {"do":"dbExpectStock","item":"Espresso Machine","warehouse":"East","relativeTo":"product-East","plus":-17,"within":10000}, + {"do":"dbExpectStock","item":"Espresso Machine","warehouse":"West","relativeTo":"product-West","plus":17,"within":10000}, + {"do":"reload","actor":"admin","settleMs":1000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","readyTestid":"current-user","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":"admin-revenue"}, + {"do":"expectNumber","actor":"admin","testid":"warehouse-total","in":{"testid":"admin-warehouse-item","contains":"East"},"relativeTo":"conservation-east-before","plus":-17,"within":10000}, + {"do":"expect","actor":"admin","testid":"admin-warehouse-item","contains":"West","count":1}, + {"do":"expectNumber","actor":"admin","testid":"warehouse-total","in":{"testid":"admin-warehouse-item","contains":"West"},"relativeTo":"conservation-west-before","plus":17,"within":10000}, + {"do":"expectNumber","actor":"customer","testid":"item-stock","in":{"testid":"item-card","contains":"Espresso Machine"},"relativeTo":"conservation-item-before","plus":0,"within":10000} + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/02-transfer-overdraw.json b/tools/stack-bench/tracks/ecommerce/scenarios/02-transfer-overdraw.json new file mode 100644 index 00000000000..95a690950b0 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/02-transfer-overdraw.json @@ -0,0 +1,72 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 2, + "name": "Warehouse transfer overdraft refusal", + "note": "Draft: revised observations require matching reference and defect-control qualification.", + "features": [ + { + "id": 2, + "name": "Moving stock between warehouses", + "actors": [ + "admin", + "visitor" + ], + "setup": [ + { + "do": "signIn", + "actor": "admin", + "name": "admin", + "password": "stackbench-admin-2026", + "exact": true + }, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + { + "do": "click", + "actor": "admin", + "testid": "admin-link", + "unlessVisible": "admin-panel" + }, + { "do": "click", "actor": "visitor", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "2c", + "category": "production", + "desc": "a transfer that would overdraw a warehouse is refused and changes neither warehouse nor the item total", + "points": 2, + "statedBy": "a transfer that would leave a warehouse short is refused and changes nothing", + "steps": [ + {"do":"dbRecordStock","item":"Headphones","warehouse":"East","as":"valid-East"}, + {"do":"dbRecordStock","item":"Headphones","warehouse":"West","as":"valid-West"}, + {"do":"recordNumber","actor":"admin","testid":"warehouse-total","in":{"testid":"admin-warehouse-item","contains":"East"},"as":"valid-east-before"}, + {"do":"recordNumber","actor":"admin","testid":"warehouse-total","in":{"testid":"admin-warehouse-item","contains":"West"},"as":"valid-west-before"}, + {"do":"fill","actor":"admin","testid":"transfer-from","text":"East","in":{"testid":"admin-item-row","contains":"Headphones"}}, + {"do":"fill","actor":"admin","testid":"transfer-to","text":"West","in":{"testid":"admin-item-row","contains":"Headphones"}}, + {"do":"fill","actor":"admin","testid":"transfer-qty","text":"1","in":{"testid":"admin-item-row","contains":"Headphones"}}, + {"do":"click","actor":"admin","testid":"transfer-submit","in":{"testid":"admin-item-row","contains":"Headphones"}}, + {"do":"expectNumber","actor":"admin","testid":"warehouse-total","in":{"testid":"admin-warehouse-item","contains":"East"},"relativeTo":"valid-east-before","plus":-1,"within":10000}, + {"do":"expectNumber","actor":"admin","testid":"warehouse-total","in":{"testid":"admin-warehouse-item","contains":"West"},"relativeTo":"valid-west-before","plus":1,"within":10000}, + {"do":"dbExpectStock","item":"Headphones","warehouse":"East","relativeTo":"valid-East","plus":-1,"within":10000}, + {"do":"dbExpectStock","item":"Headphones","warehouse":"West","relativeTo":"valid-West","plus":1,"within":10000}, + {"do":"dbRecordStock","item":"Headphones","warehouse":"East","as":"product-East"}, + {"do":"dbRecordStock","item":"Headphones","warehouse":"West","as":"product-West"}, + {"do":"recordNumber","actor":"visitor","testid":"item-stock","in":{"testid":"item-card","contains":"Headphones"},"as":"overdraw-item-before"}, + {"do":"recordNumber","actor":"admin","testid":"warehouse-total","in":{"testid":"admin-warehouse-item","contains":"East"},"as":"overdraw-east-before"}, + {"do":"recordNumber","actor":"admin","testid":"warehouse-total","in":{"testid":"admin-warehouse-item","contains":"West"},"as":"overdraw-west-before"}, + {"do":"fill","actor":"admin","testid":"transfer-from","text":"East","in":{"testid":"admin-item-row","contains":"Headphones"}}, + {"do":"fill","actor":"admin","testid":"transfer-to","text":"West","in":{"testid":"admin-item-row","contains":"Headphones"}}, + {"do":"fill","actor":"admin","testid":"transfer-qty","text":"99999","in":{"testid":"admin-item-row","contains":"Headphones"}}, + {"do":"click","actor":"admin","testid":"transfer-submit","in":{"testid":"admin-item-row","contains":"Headphones"}}, + {"do":"expect","actor":"admin","testid":"order-error","within":6000}, + {"do":"dbExpectStock","item":"Headphones","warehouse":"East","relativeTo":"product-East","plus":0}, + {"do":"dbExpectStock","item":"Headphones","warehouse":"West","relativeTo":"product-West","plus":0}, + {"do":"expectNumber","actor":"admin","testid":"warehouse-total","in":{"testid":"admin-warehouse-item","contains":"East"},"relativeTo":"overdraw-east-before","plus":0}, + {"do":"expectNumber","actor":"admin","testid":"warehouse-total","in":{"testid":"admin-warehouse-item","contains":"West"},"relativeTo":"overdraw-west-before","plus":0}, + {"do":"expectNumber","actor":"visitor","testid":"item-stock","in":{"testid":"item-card","contains":"Headphones"},"relativeTo":"overdraw-item-before","plus":0} + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/02-transfer-totals.json b/tools/stack-bench/tracks/ecommerce/scenarios/02-transfer-totals.json new file mode 100644 index 00000000000..dbb4d03166b --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/02-transfer-totals.json @@ -0,0 +1,45 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 2, + "name": "Ecommerce L2 warehouse transfer totals", + "note": "Draft: revised observations require matching reference and defect-control qualification.", + "features": [ + { + "id": 2, + "name": "Warehouse totals", + "actors": ["admin"], + "setup": [ + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + { "do": "click", "actor": "admin", "testid": "admin-link" } + ], + "criteria": [ + { + "id": "2b", + "category": "production", + "desc": "both warehouse totals move live and in opposite directions as stock is transferred", + "points": 2, + "statedBy": "the per-warehouse numbers staff and admins see both move at once", + "steps": [ + {"do":"dbRecordStock","item":"Headphones","warehouse":"East","as":"product-East"}, + {"do":"dbRecordStock","item":"Headphones","warehouse":"West","as":"product-West"}, + {"do":"expect","actor":"admin","testid":"admin-warehouse-item","contains":"East","count":1}, + {"do":"recordNumber","actor":"admin","testid":"warehouse-total","in":{"testid":"admin-warehouse-item","contains":"East"},"as":"warehouse-east-before"}, + {"do":"expect","actor":"admin","testid":"admin-warehouse-item","contains":"West","count":1}, + {"do":"recordNumber","actor":"admin","testid":"warehouse-total","in":{"testid":"admin-warehouse-item","contains":"West"},"as":"warehouse-west-before"}, + {"do":"fill","actor":"admin","testid":"transfer-from","text":"East","in":{"testid":"admin-item-row","contains":"Headphones"}}, + {"do":"fill","actor":"admin","testid":"transfer-to","text":"West","in":{"testid":"admin-item-row","contains":"Headphones"}}, + {"do":"fill","actor":"admin","testid":"transfer-qty","text":"10","in":{"testid":"admin-item-row","contains":"Headphones"}}, + {"do":"click","actor":"admin","testid":"transfer-submit","in":{"testid":"admin-item-row","contains":"Headphones"}}, + {"do":"expect","actor":"admin","testid":"admin-warehouse-item","contains":"East","count":1}, + {"do":"expectNumber","actor":"admin","testid":"warehouse-total","in":{"testid":"admin-warehouse-item","contains":"East"},"relativeTo":"warehouse-east-before","plus":-10,"within":10000}, + {"do":"expect","actor":"admin","testid":"admin-warehouse-item","contains":"West","count":1}, + {"do":"expectNumber","actor":"admin","testid":"warehouse-total","in":{"testid":"admin-warehouse-item","contains":"West"},"relativeTo":"warehouse-west-before","plus":10,"within":10000}, + {"do":"dbExpectStock","item":"Headphones","warehouse":"East","relativeTo":"product-East","plus":-10,"within":10000}, + {"do":"dbExpectStock","item":"Headphones","warehouse":"West","relativeTo":"product-West","plus":10,"within":10000} + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/03-cart-expiration.json b/tools/stack-bench/tracks/ecommerce/scenarios/03-cart-expiration.json new file mode 100644 index 00000000000..7d34ef4759e --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/03-cart-expiration.json @@ -0,0 +1,36 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 3, + "name": "Cart expiration", + "note": "Draft: revised observations require matching reference and defect-control qualification.", + "features": [{ + "id": 304, + "name": "An inactive cart expires", + "actors": ["customer", "watcher"], + "setup": [ + {"do":"signUp","actor":"customer","name":"cart-expiry"}, + {"do":"recordNumber","actor":"watcher","testid":"item-stock","in":{"testid":"item-card","contains":"Air Purifier"},"as":"before"}, + {"do":"click","actor":"customer","testid":"add-to-cart","in":{"testid":"item-card","contains":"Air Purifier"}}, + {"do":"reload","actor":"watcher","settleMs":1000}, + {"do":"expectNumber","actor":"watcher","testid":"item-stock","in":{"testid":"item-card","contains":"Air Purifier"},"relativeTo":"before","plus":-1,"within":10000}, + {"do":"closeClient","actor":"customer"} + ], + "criteria": [{ + "id": "304a", + "category": "production", + "desc": "an inactive cart expires without a browser, releases stock, and returns empty", + "points": 4, + "statedBy": "A cart with no activity for five minutes expires and releases its reservations.", + "steps": [ + {"do":"wait","actor":"watcher","ms":310000}, + {"do":"reload","actor":"watcher","settleMs":1000}, + {"do":"expectNumber","actor":"watcher","testid":"item-stock","in":{"testid":"item-card","contains":"Air Purifier"},"relativeTo":"before","plus":0,"within":30000}, + {"do":"openClient","actor":"customer","settleMs":3000}, + {"do":"ensureSignedIn","actor":"customer","name":"cart-expiry","readyTestid":"current-user"}, + {"do":"expect","actor":"customer","testid":"cart-expired-notice","within":10000}, + {"do":"expectNumber","actor":"customer","testid":"cart-count","equals":0} + ] + }] + }] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/03-deferred-access.json b/tools/stack-bench/tracks/ecommerce/scenarios/03-deferred-access.json new file mode 100644 index 00000000000..f9bcc92ca2b --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/03-deferred-access.json @@ -0,0 +1,76 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 3, + "name": "Scheduled-work access", + "note": "The admin proves each route with a distinct restock; the customer then attempts a new schedule and a cancellation.", + "features": [ + { + "id": 317, + "name": "Customers cannot manage scheduled restocks", + "actors": ["admin", "customer"], + "setup": [ + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel", "schedule-restock-item"]}, + { "do": "click", "actor": "admin", "testid": "admin-link", "unlessVisible": ["admin-panel", "schedule-restock-item"] }, + { "do": "click", "actor": "admin", "testid": "restocks-link", "ifAvailable": true, "unlessVisible": "schedule-restock-item" }, + { "do": "signUp", "actor": "customer", "name": "restock-outsider" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-item", "text": "Webcam" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-warehouse", "text": "West" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-qty", "text": "3" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-delay", "text": "180" }, + { "do": "click", "actor": "admin", "testid": "schedule-restock-submit" }, + { "do": "expect", "actor": "admin", "testid": "pending-restock-item", "count": 1 } + ], + "criteria": [ + { + "id": "317a", + "category": "production", + "desc": "the server refuses customer scheduling and cancellation of restocks", + "points": 3, + "statedBy": "Only an admin can schedule or cancel a restock.", + "steps": [ + { "do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel", "schedule-restock-item"] }, + { "do": "click", "actor": "admin", "testid": "admin-link", "unlessVisible": ["admin-panel", "schedule-restock-item"] }, + { "do": "click", "actor": "admin", "testid": "restocks-link", "ifAvailable": true, "unlessVisible": "schedule-restock-item" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-item", "text": "Desk Lamp" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-warehouse", "text": "East" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-qty", "text": "3" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-delay", "text": "180" }, + { "do": "callAction", "actor": "admin", "action": "scheduleRestock", "authentication": "actor", "input": { "testid": "schedule-restock-submit", "attribute": "data-action-input" }, "namedAction": { "id": "scheduleRestock", "path": "/api/admin/scheduled-restocks", "method": "POST", "reducer": "schedule_restock", "args": ["", "", 0, 0], "params": [{ "name": "item", "in": "body" }, { "name": "warehouse", "in": "body" }, { "name": "quantity", "in": "body" }, { "name": "delaySeconds", "in": "body" }] }, "settleMs": 2000 }, + { "do": "expectActionOutcome", "actor": "admin", "outcome": "accepted" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-item", "text": "Keyboard" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-warehouse", "text": "East" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-qty", "text": "3" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-delay", "text": "180" }, + { "do": "callAction", "actor": "customer", "from": "admin", "action": "scheduleRestock", "authentication": "actor", "input": { "testid": "schedule-restock-submit", "attribute": "data-action-input" }, "namedAction": { "id": "scheduleRestock", "path": "/api/admin/scheduled-restocks", "method": "POST", "reducer": "schedule_restock", "args": ["", "", 0, 0], "params": [{ "name": "item", "in": "body" }, { "name": "warehouse", "in": "body" }, { "name": "quantity", "in": "body" }, { "name": "delaySeconds", "in": "body" }] }, "settleMs": 2000 }, + { "do": "expectActionOutcome", "actor": "customer", "outcome": "refused", "routeProvenBy": "admin" }, + { "do": "reload", "actor": "admin", "settleMs": 1000 }, + { "do": "ensureSignedIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true, "readyTestid": "current-user" }, + { "do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel", "pending-restock-item"] }, + { "do": "click", "actor": "admin", "testid": "admin-link", "unlessVisible": ["admin-panel", "pending-restock-item"] }, + { "do": "click", "actor": "admin", "testid": "restocks-link", "ifAvailable": true, "unlessVisible": "pending-restock-item" }, + { "do": "expect", "actor": "admin", "testid": "pending-restock-item", "count": 2, "within": 10000 }, + { "do": "replayAs", "actor": "admin", "from": "admin", "match": "__317a-authorized-cancel__", "namedAction": { "id": "cancelScheduledRestock", "path": "/api/admin/scheduled-restocks/{restockId}", "method": "DELETE", "reducer": "cancel_scheduled_restock", "args": [0], "params": [{ "name": "restockId", "in": "path", "placeholder": "{restockId}", "wireType": "u64" }] }, "namedTarget": { "testid": "pending-restock-item", "attribute": "data-entity-id", "valueType": "string" }, "settleMs": 2000 }, + { "do": "expectReplayCompleted", "actor": "admin", "requireAccepted": true }, + { "do": "reload", "actor": "admin", "settleMs": 1000 }, + { "do": "ensureSignedIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true, "readyTestid": "current-user" }, + { "do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel", "pending-restock-item"] }, + { "do": "click", "actor": "admin", "testid": "admin-link", "unlessVisible": ["admin-panel", "pending-restock-item"] }, + { "do": "click", "actor": "admin", "testid": "restocks-link", "ifAvailable": true, "unlessVisible": "pending-restock-item" }, + { "do": "expect", "actor": "admin", "testid": "pending-restock-item", "count": 1, "within": 10000 }, + { "do": "replayAs", "actor": "customer", "from": "admin", "match": "__317a-customer-cancel__", "namedAction": { "id": "cancelScheduledRestock", "path": "/api/admin/scheduled-restocks/{restockId}", "method": "DELETE", "reducer": "cancel_scheduled_restock", "args": [0], "params": [{ "name": "restockId", "in": "path", "placeholder": "{restockId}", "wireType": "u64" }] }, "namedTarget": { "testid": "pending-restock-item", "attribute": "data-entity-id", "valueType": "string" }, "settleMs": 2000 }, + { "do": "expectReplayRejected", "actor": "customer" }, + { "do": "reload", "actor": "admin", "settleMs": 1000 }, + { "do": "ensureSignedIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true, "readyTestid": "current-user" }, + { "do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel", "pending-restock-item"] }, + { "do": "click", "actor": "admin", "testid": "admin-link", "unlessVisible": ["admin-panel", "pending-restock-item"] }, + { "do": "click", "actor": "admin", "testid": "restocks-link", "ifAvailable": true, "unlessVisible": "pending-restock-item" }, + { "do": "expect", "actor": "admin", "testid": "pending-restock-item", "count": 1, "within": 10000 }, + { "do": "click", "actor": "admin", "testid": "pending-restock-cancel", "in": { "testid": "pending-restock-item" } } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/03-deferred-durability.json b/tools/stack-bench/tracks/ecommerce/scenarios/03-deferred-durability.json new file mode 100644 index 00000000000..90e67300290 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/03-deferred-durability.json @@ -0,0 +1,190 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 3, + "name": "Deferred-work durability", + "note": "Draft: revised observations require matching reference and defect-control qualification.", + "features": [ + { + "id": 311, + "name": "A scheduled restock survives restart", + "actors": ["admin"], + "setup": [ + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + { "do": "click", "actor": "admin", "testid": "admin-link" }, + { "do": "click", "actor": "admin", "testid": "restocks-link", "ifAvailable": true }, + { "do": "dbRecordStock", "item": "Air Purifier", "as": "ordinaryBefore" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-item", "text": "Air Purifier" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-warehouse", "text": "East" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-qty", "text": "5" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-delay", "text": "45" }, + { "do": "click", "actor": "admin", "testid": "schedule-restock-submit" }, + { "do": "expect", "actor": "admin", "testid": "pending-restock-item", "count": 1, "within": 10000 }, + { "do": "dbExpectStock", "item": "Air Purifier", "relativeTo": "ordinaryBefore", "plus": 5, "within": 70000 }, + { "do": "reload", "actor": "admin", "settleMs": 1000 }, + { "do": "ensureSignedIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true, "readyTestid": "current-user" }, + { "do": "click", "actor": "admin", "testid": "admin-link" }, + { "do": "click", "actor": "admin", "testid": "restocks-link", "ifAvailable": true }, + { "do": "expect", "actor": "admin", "testid": "schedule-restock-submit" }, + { "do": "expect", "actor": "admin", "testid": "pending-restock-item", "absent": true }, + { "do": "dbRecordStock", "item": "Air Purifier", "as": "before" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-item", "text": "Air Purifier" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-warehouse", "text": "East" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-qty", "text": "5" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-delay", "text": "45" }, + { "do": "click", "actor": "admin", "testid": "schedule-restock-submit" }, + { "do": "expect", "actor": "admin", "testid": "pending-restock-item", "count": 1, "within": 10000 }, + { "do": "dbExpectStock", "item": "Air Purifier", "relativeTo": "before", "plus": 0 }, + { "do": "restartBackend", "settleMs": 15000 } + ], + "criteria": [ + { + "id": "311a", + "category": "production", + "desc": "a restock scheduled before restart still applies", + "points": 4, + "statedBy": "Pending restocks survive a backend restart.", + "steps": [ + { "do": "dbExpectStock", "item": "Air Purifier", "relativeTo": "before", "plus": 5, "within": 70000 }, + { "do": "reload", "actor": "admin", "settleMs": 2000 }, + { "do": "ensureSignedIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true, "readyTestid": "current-user" }, + { "do": "click", "actor": "admin", "testid": "admin-link", "ifAvailable": true }, + { "do": "click", "actor": "admin", "testid": "restocks-link", "ifAvailable": true }, + { "do": "expect", "actor": "admin", "testid": "schedule-restock-submit" }, + { "do": "expect", "actor": "admin", "testid": "pending-restock-item", "absent": true } + ] + } + ] + }, + { + "id": 314, + "name": "A reservation survives restart", + "actors": ["customer", "watcher"], + "setup": [ + {"do":"signUp","actor":"customer","name":"durable-reservation"}, + {"do":"recordNumber","actor":"watcher","testid":"item-stock","in":{"testid":"item-card","contains":"Desk Lamp"},"as":"before"}, + {"do":"recordTime","as":"pending-314"}, + {"do":"click","actor":"customer","testid":"add-to-cart","in":{"testid":"item-card","contains":"Desk Lamp"}}, + {"do":"reload","actor":"watcher","settleMs":1000}, + {"do":"expectNumber","actor":"watcher","testid":"item-stock","in":{"testid":"item-card","contains":"Desk Lamp"},"relativeTo":"before","plus":-1,"within":10000}, + {"do":"recordTime","as":"pending-314-accepted"}, + {"do":"wait","actor":"customer","ms":30000}, + {"do":"restartBackend","settleMs":15000}, + {"do":"reload","actor":"watcher","settleMs":3000} + ], + "criteria": [ + { + "id": "314a", + "category": "production", + "desc": "a reservation pending before restart still expires and returns stock", + "points": 4, + "statedBy": "Pending reservations survive a backend restart.", + "steps": [ + {"do":"expectElapsed","since":"pending-314","atMost":70000}, + {"do":"reload","actor":"watcher","settleMs":1000}, + {"do":"expectElapsed","since":"pending-314","atMost":70000}, + {"do":"expectNumber","actor":"watcher","testid":"item-stock","in":{"testid":"item-card","contains":"Desk Lamp"},"relativeTo":"before","plus":-1,"within":1000}, + {"do":"expectElapsed","since":"pending-314","atMost":70000}, + {"do":"wait","actor":"customer","ms":110000,"since":"pending-314-accepted"}, + {"do":"reload","actor":"watcher","settleMs":2000}, + {"do":"expectNumber","actor":"watcher","testid":"item-stock","in":{"testid":"item-card","contains":"Desk Lamp"},"relativeTo":"before","plus":0,"within":10000}, + {"do":"reload","actor":"customer","settleMs":1000}, + {"do":"ensureSignedIn","actor":"customer","name":"durable-reservation","readyTestid":"current-user"}, + {"do":"click","actor":"customer","testid":"cart-toggle","unlessVisible":"cart-item"}, + {"do":"expect","actor":"customer","testid":"cart-item-expired","in":{"testid":"cart-item","contains":"Desk Lamp"},"within":10000} + ] + } + ] + }, + { + "id": 315, + "name": "An order transition survives restart", + "actors": ["customer", "staff"], + "setup": [ + {"do":"signUp","actor":"customer","name":"durable-delivery"}, + {"do":"signIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true}, + {"do":"click","actor":"staff","testid":"staff-link"}, + {"do":"click","actor":"customer","testid":"buy-now","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"reload","actor":"staff","settleMs":1000}, + {"do":"ensureSignedIn","actor":"staff","name":"staff","readyTestid":"current-user","password":"stackbench-staff-2026","exact":true}, + {"do":"click","actor":"staff","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"staff","testid":"staff-link","unlessVisible":"queue-item"}, + {"do":"expect","actor":"staff","testid":"queue-item","contains":"Keyboard","within":10000}, + {"do":"click","actor":"staff","testid":"ship-submit","in":{"testid":"queue-item","contains":"Keyboard"}}, + {"do":"reload","actor":"customer","settleMs":1000}, + {"do":"ensureSignedIn","actor":"customer","name":"durable-delivery","readyTestid":"current-user"}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"customer","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"customer","testid":"order-status","in":{"testid":"order-item","contains":"Keyboard"},"statusText":"shipped","ignoreCase":true,"within":10000}, + {"do":"recordTime","as":"delivery-start-accepted"}, + {"do":"wait","actor":"customer","ms":20000}, + {"do":"restartBackend","settleMs":15000}, + {"do":"reload","actor":"customer","settleMs":3000}, + {"do":"ensureSignedIn","actor":"customer","name":"durable-delivery","readyTestid":"current-user"}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"customer","testid":"orders-toggle"} + ], + "criteria": [ + { + "id": "315a", + "category": "production", + "desc": "an order shipped before restart still becomes delivered", + "points": 4, + "statedBy": "Pending order delivery survives a backend restart.", + "steps": [ + {"do":"wait","actor":"customer","ms":75000,"since":"delivery-start-accepted"}, + {"do":"reload","actor":"customer","settleMs":1000}, + {"do":"ensureSignedIn","actor":"customer","name":"durable-delivery","readyTestid":"current-user"}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"customer","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"customer","testid":"order-status","in":{"testid":"order-item","contains":"Keyboard"},"statusText":"delivered","ignoreCase":true,"within":10000} + ] + } + ] + }, + { + "id": 316, + "name": "Cart expiration survives restart", + "actors": ["customer", "watcher"], + "setup": [ + {"do":"signUp","actor":"customer","name":"durable-cart"}, + {"do":"recordNumber","actor":"watcher","testid":"item-stock","in":{"testid":"item-card","contains":"Bluetooth Speaker"},"as":"before"}, + {"do":"recordTime","as":"pending-316"}, + {"do":"click","actor":"customer","testid":"add-to-cart","in":{"testid":"item-card","contains":"Bluetooth Speaker"}}, + {"do":"reload","actor":"watcher","settleMs":1000}, + {"do":"expectNumber","actor":"watcher","testid":"item-stock","in":{"testid":"item-card","contains":"Bluetooth Speaker"},"relativeTo":"before","plus":-1,"within":10000}, + {"do":"recordTime","as":"pending-316-accepted"}, + {"do":"wait","actor":"customer","ms":120000}, + {"do":"restartBackend","settleMs":15000}, + {"do":"reload","actor":"watcher","settleMs":3000} + ], + "criteria": [ + { + "id": "316a", + "category": "production", + "desc": "a cart survives restart and expires near its original five-minute deadline", + "points": 4, + "statedBy": "Pending cart expiration survives a backend restart.", + "steps": [ + {"do":"expectElapsed","since":"pending-316","atMost":250000}, + {"do":"reload","actor":"customer","settleMs":1000}, + {"do":"ensureSignedIn","actor":"customer","name":"durable-cart","readyTestid":"current-user"}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"customer","testid":"cart-toggle","unlessVisible":"cart-item"}, + {"do":"expectElapsed","since":"pending-316","atMost":250000}, + {"do":"expectNumber","actor":"customer","testid":"cart-count","equals":1,"within":1000}, + {"do":"expectElapsed","since":"pending-316","atMost":250000}, + {"do":"wait","actor":"customer","ms":310000,"since":"pending-316-accepted"}, + {"do":"reload","actor":"customer","settleMs":3000}, + {"do":"ensureSignedIn","actor":"customer","name":"durable-cart","readyTestid":"current-user"}, + {"do":"click","actor":"customer","testid":"cart-toggle"}, + {"do":"expectNumber","actor":"customer","testid":"cart-count","equals":0,"within":10000}, + {"do":"expect","actor":"customer","testid":"cart-expired-notice","within":10000}, + {"do":"reload","actor":"watcher","settleMs":1000}, + {"do":"expectNumber","actor":"watcher","testid":"item-stock","in":{"testid":"item-card","contains":"Bluetooth Speaker"},"relativeTo":"before","plus":0,"within":10000} + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/03-deferred-integrity.json b/tools/stack-bench/tracks/ecommerce/scenarios/03-deferred-integrity.json new file mode 100644 index 00000000000..5c192c7db0f --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/03-deferred-integrity.json @@ -0,0 +1,154 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 3, + "name": "Deferred-work integrity", + "note": "Draft: revised observations require matching reference and defect-control qualification.", + "features": [ + { + "id": 311, + "name": "A restock applies once", + "actors": ["admin", "watcher"], + "setup": [ + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel", "schedule-restock-item"]}, + { "do": "click", "actor": "admin", "testid": "admin-link", "unlessVisible": ["admin-panel", "schedule-restock-item"] }, + { "do": "click", "actor": "admin", "testid": "restocks-link", "ifAvailable": true, "unlessVisible": "schedule-restock-item" }, + { "do": "dbRecordStock", "item": "Air Purifier", "as": "before" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-item", "text": "Air Purifier" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-warehouse", "text": "East" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-qty", "text": "5" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-delay", "text": "20" }, + { "do": "click", "actor": "admin", "testid": "schedule-restock-submit" }, + { "do": "expect", "actor": "admin", "testid": "pending-restock-item", "count": 1, "within": 10000 }, + { "do": "dbExpectStock", "item": "Air Purifier", "relativeTo": "before", "plus": 0 }, + { "do": "restartBackend", "settleMs": 15000 }, + { "do": "reload", "actor": "watcher", "settleMs": 3000 } + ], + "criteria": [ + { + "id": "311a", + "category": "production", + "desc": "restart cannot replay a completed restock", + "points": 3, + "statedBy": "Restarting the backend cannot apply a restock more than once.", + "steps": [ + { "do": "reload", "actor": "watcher", "settleMs": 2000 }, + { "do": "dbExpectStock", "item": "Air Purifier", "relativeTo": "before", "plus": 5, "within": 45000 }, + { "do": "restartBackend", "settleMs": 15000 }, + { "do": "reload", "actor": "watcher", "settleMs": 3000 }, + { "do": "wait", "actor": "watcher", "ms": 15000 }, + { "do": "dbExpectStock", "item": "Air Purifier", "relativeTo": "before", "plus": 5 } + ] + } + ] + }, + { + "id": 312, + "name": "Delivery applies once", + "actors": ["customer", "staff"], + "setup": [ + {"do":"signUp","actor":"customer","name":"once-delivery"}, + {"do":"signIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true}, + {"do":"click","actor":"staff","testid":"staff-link"}, + {"do":"click","actor":"customer","testid":"buy-now","in":{"testid":"item-card","contains":"Desk Lamp"}}, + {"do":"reload","actor":"staff","settleMs":1000}, + {"do":"ensureSignedIn","actor":"staff","name":"staff","readyTestid":"current-user","password":"stackbench-staff-2026","exact":true}, + {"do":"click","actor":"staff","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"staff","testid":"staff-link","unlessVisible":"queue-item"}, + {"do":"expect","actor":"staff","testid":"queue-item","contains":"Desk Lamp","within":10000}, + {"do":"click","actor":"staff","testid":"ship-submit","in":{"testid":"queue-item","contains":"Desk Lamp"}}, + {"do":"reload","actor":"customer","settleMs":1000}, + {"do":"ensureSignedIn","actor":"customer","name":"once-delivery","readyTestid":"current-user"}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"customer","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"customer","testid":"order-status","in":{"testid":"order-item","contains":"Desk Lamp"},"statusText":"shipped","ignoreCase":true,"within":10000}, + {"do":"wait","actor":"customer","ms":70000}, + {"do":"reload","actor":"customer","settleMs":1000}, + {"do":"ensureSignedIn","actor":"customer","name":"once-delivery","readyTestid":"current-user"}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"customer","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"customer","testid":"order-status","in":{"testid":"order-item","contains":"Desk Lamp"},"statusText":"delivered","ignoreCase":true,"within":15000}, + {"do":"restartBackend","settleMs":15000}, + {"do":"reload","actor":"staff","settleMs":3000}, + {"do":"ensureSignedIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true,"readyTestid":"current-user"}, + {"do":"click","actor":"staff","testid":"staff-link","ifAvailable":true,"within":1000} + ], + "criteria": [ + { + "id": "312a", + "category": "production", + "desc": "restart leaves one delivered order record", + "points": 3, + "statedBy": "A delivered order is not duplicated by a backend restart.", + "steps": [ + { "do": "reload", "actor": "staff", "settleMs": 2000 }, + { "do": "ensureSignedIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true, "readyTestid": "current-user" }, + { "do": "click", "actor": "staff", "testid": "staff-link", "ifAvailable": true }, + { "do": "expect", "actor": "staff", "testid": "completed-order-item", "contains": "Desk Lamp", "count": 1 }, + { "do": "expect", "actor": "staff", "testid": "completed-order-status", "in": { "testid": "completed-order-item", "contains": "Desk Lamp" }, "value": "delivered", "ignoreCase": true } + ] + } + ] + }, + { + "id": 313, + "name": "Reservation expiry conserves stock", + "actors": ["customer", "watcher"], + "setup": [ + {"do":"signUp","actor":"customer","name":"conserve-expiry"}, + {"do":"recordNumber","actor":"watcher","testid":"item-stock","in":{"testid":"item-card","contains":"Keyboard"},"as":"before"}, + {"do":"click","actor":"customer","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"reload","actor":"watcher","settleMs":1000}, + {"do":"expectNumber","actor":"watcher","testid":"item-stock","in":{"testid":"item-card","contains":"Keyboard"},"relativeTo":"before","plus":-1,"within":10000} + ], + "criteria": [ + { + "id": "313a", + "category": "production", + "desc": "expiry returns exactly the unit reserved", + "points": 3, + "statedBy": "Reservation expiry returns exactly the stock that the reservation took.", + "steps": [ + {"do":"wait","actor":"watcher","ms":100000}, + {"do":"reload","actor":"watcher","settleMs":1000}, + {"do":"expectNumber","actor":"watcher","testid":"item-stock","in":{"testid":"item-card","contains":"Keyboard"},"relativeTo":"before","plus":0,"within":30000} + ] + } + ] + }, + { + "id": 314, + "name": "Checkout conserves reserved stock", + "actors": ["customer", "watcher"], + "setup": [ + {"do":"signUp","actor":"customer","name":"conserve-checkout"}, + {"do":"recordNumber","actor":"watcher","testid":"item-stock","in":{"testid":"item-card","contains":"Headphones"},"as":"before"}, + {"do":"click","actor":"customer","testid":"add-to-cart","in":{"testid":"item-card","contains":"Headphones"}}, + {"do":"click","actor":"customer","testid":"cart-toggle"}, + {"do":"reload","actor":"watcher","settleMs":1000}, + {"do":"expectNumber","actor":"watcher","testid":"item-stock","in":{"testid":"item-card","contains":"Headphones"},"relativeTo":"before","plus":-1,"within":10000} + ], + "criteria": [ + { + "id": "314a", + "category": "production", + "desc": "checkout does not decrement stock after the reservation already did", + "points": 3, + "statedBy": "Checkout does not take reserved stock twice.", + "steps": [ + {"do":"click","actor":"customer","testid":"checkout-submit"}, + {"do":"expectNumber","actor":"customer","testid":"cart-count","equals":0,"within":10000}, + {"do":"reload","actor":"customer","settleMs":1000}, + {"do":"ensureSignedIn","actor":"customer","name":"conserve-checkout","readyTestid":"current-user"}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"customer","testid":"orders-toggle"}, + {"do":"expectElementCount","actor":"customer","testid":"order-item","contains":"Headphones","equals":1,"within":10000}, + {"do":"reload","actor":"watcher","settleMs":1000}, + {"do":"expectNumber","actor":"watcher","testid":"item-stock","in":{"testid":"item-card","contains":"Headphones"},"relativeTo":"before","plus":-1,"within":10000} + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/03-features.json b/tools/stack-bench/tracks/ecommerce/scenarios/03-features.json new file mode 100644 index 00000000000..751273becb6 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/03-features.json @@ -0,0 +1,340 @@ +{ + "schemaVersion": 1, + "level": 3, + "track": "ecommerce", + "name": "Work that happens later — features", + "writeUrlPattern": "\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses|\\/schedule", + "note": "Deferred work observed from the outside: a reservation that releases itself, a restock that fires on a timer, an order that advances with nobody watching. Every wait here is real time — these criteria are slow by nature and the timings come from the spec (90s reservation, 60s transitions), not from guesses about implementation.", + "features": [ + { + "id": 301, + "name": "A reservation holds stock and then releases it", + "actors": [ + "shopper", + "watcher" + ], + "setup": [ + { + "do": "signUp", + "actor": "shopper", + "name": "resa" + }, + { + "do": "recordNumber", + "actor": "watcher", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Desk Lamp" + }, + "as": "lamp-before" + } + ], + "criteria": [ + { + "id": "301a", + "desc": "adding to the cart takes a unit off the shelf immediately, for a signed-out viewer who never touched the page", + "points": 2, + "steps": [ + { + "do": "click", + "actor": "shopper", + "testid": "add-to-cart", + "in": { + "testid": "item-card", + "contains": "Desk Lamp" + }, + "settleMs": 1500 + }, + { + "do": "expectNumber", + "actor": "watcher", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Desk Lamp" + }, + "relativeTo": "lamp-before", + "plus": -1, + "within": 10000 + } + ], + "statedBy": "storefront stock drops immediately, for everyone" + }, + { + "id": "301b", + "desc": "the cart line shows the reservation counting down", + "points": 1, + "steps": [ + { + "do": "click", + "actor": "shopper", + "testid": "cart-toggle", + "settleMs": 1200 + }, + { + "do": "expect", + "actor": "shopper", + "testid": "cart-reservation-timer", + "in": { + "testid": "cart-item", + "contains": "Desk Lamp" + }, + "within": 8000 + } + ], + "statedBy": "the cart line shows the reservation's remaining time, live" + }, + { + "id": "301c", + "desc": "left alone, the reservation expires on its own: the unit returns to the shelf for the watcher without a reload, and the line is marked expired rather than vanishing", + "points": 3, + "note": "waits out the spec's 90-second window plus slack; slow by nature.", + "steps": [ + { + "do": "wait", + "actor": "shopper", + "ms": 100000 + }, + { + "do": "expectNumber", + "actor": "watcher", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Desk Lamp" + }, + "relativeTo": "lamp-before", + "plus": 0, + "within": 30000 + }, + { + "do": "expect", + "actor": "shopper", + "testid": "cart-item-expired", + "in": { + "testid": "cart-item", + "contains": "Desk Lamp" + }, + "within": 15000 + } + ], + "statedBy": "the reservation expires on its own and the unit returns to\nstorefront stock - visibly, without anyone reloading" + } + ] + }, + { + "id": 302, + "name": "A restock scheduled for later applies itself", + "actors": [ + "admin", + "watcher" + ], + "setup": [ + { + "do": "signIn", + "actor": "admin", + "name": "admin", + "exact": true, + "password": "stackbench-admin-2026" + }, + { + "do": "click", + "actor": "admin", + "testid": "admin-link", + "settleMs": 1500 + }, + { + "do": "click", + "actor": "admin", + "testid": "restocks-link", + "ifAvailable": true + }, + { + "do": "recordNumber", + "actor": "watcher", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Webcam" + }, + "as": "webcam-before" + } + ], + "criteria": [ + { + "id": "302a", + "desc": "a scheduled restock appears as pending with time remaining, and has not yet changed stock", + "points": 2, + "steps": [ + { + "do": "fill", + "actor": "admin", + "testid": "schedule-restock-item", + "text": "Webcam" + }, + { + "do": "fill", + "actor": "admin", + "testid": "schedule-restock-warehouse", + "text": "East" + }, + { + "do": "fill", + "actor": "admin", + "testid": "schedule-restock-qty", + "text": "7" + }, + { + "do": "fill", + "actor": "admin", + "testid": "schedule-restock-delay", + "text": "45" + }, + { + "do": "click", + "actor": "admin", + "testid": "schedule-restock-submit", + "settleMs": 2000 + }, + { + "do": "expect", + "actor": "admin", + "testid": "pending-restock-item", + "count": 1, + "within": 8000 + }, + { + "do": "expect", + "actor": "admin", + "testid": "pending-restock-remaining", + "in": { + "testid": "pending-restock-item" + }, + "within": 8000 + }, + { + "do": "expectNumber", + "actor": "watcher", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Webcam" + }, + "relativeTo": "webcam-before", + "plus": 0 + } + ], + "statedBy": "the pending restocks list shows each one with its remaining time, live" + }, + { + "id": "302b", + "desc": "when the delay elapses it applies itself: stock rises for a viewer who did nothing, and it leaves the pending list", + "points": 3, + "steps": [ + { + "do": "wait", + "actor": "admin", + "ms": 55000 + }, + { + "do": "expectNumber", + "actor": "watcher", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Webcam" + }, + "relativeTo": "webcam-before", + "plus": 7, + "within": 30000 + }, + { + "do": "expect", + "actor": "admin", + "testid": "pending-restock-item", + "absent": true, + "within": 15000 + } + ], + "statedBy": "the entry leaves the pending list, and it appears in the stock ledger" + }, + { + "id": "302c", + "desc": "a cancelled restock never applies", + "points": 2, + "steps": [ + { + "do": "fill", + "actor": "admin", + "testid": "schedule-restock-item", + "text": "Keyboard" + }, + { + "do": "fill", + "actor": "admin", + "testid": "schedule-restock-warehouse", + "text": "West" + }, + { + "do": "fill", + "actor": "admin", + "testid": "schedule-restock-qty", + "text": "9" + }, + { + "do": "fill", + "actor": "admin", + "testid": "schedule-restock-delay", + "text": "40" + }, + { + "do": "click", + "actor": "admin", + "testid": "schedule-restock-submit", + "settleMs": 2000 + }, + { + "do": "recordNumber", + "actor": "watcher", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Keyboard" + }, + "as": "kbd-before" + }, + { "do": "expect", "actor": "admin", "testid": "pending-restock-item", "count": 1 }, + { + "do": "click", + "actor": "admin", + "testid": "pending-restock-cancel", + "in": { + "testid": "pending-restock-item" + }, + "settleMs": 2000 + }, + { + "do": "wait", + "actor": "admin", + "ms": 50000 + }, + { + "do": "expectNumber", + "actor": "watcher", + "testid": "item-stock", + "in": { + "testid": "item-card", + "contains": "Keyboard" + }, + "relativeTo": "kbd-before", + "plus": 0, + "within": 10000 + } + ], + "statedBy": "an admin can cancel a pending restock before it fires; cancelled restocks\nnever apply" + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/03-invariants.json b/tools/stack-bench/tracks/ecommerce/scenarios/03-invariants.json new file mode 100644 index 00000000000..431a5b0c0d1 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/03-invariants.json @@ -0,0 +1,104 @@ +{ + "schemaVersion": 1, + "level": 3, + "track": "ecommerce", + "name": "Work that happens later — invariants", + "writeUrlPattern": "\\/api\\/|\\/items|\\/cart|\\/orders|\\/checkout|\\/buy|\\/reviews|\\/stock|\\/restock|\\/warehouses|\\/schedule", + "note": "What must hold about deferred work regardless of how it is implemented: it survives a restart, it fires once, it does not fire early, and time never creates or destroys stock. This is the level where in-memory timers stop being an implementation detail and become a defect.", + "features": [ + { + "id": 311, + "name": "Scheduled work survives a restart", + "actors": ["admin", "watcher"], + "setup": [ + { "do": "signIn", "actor": "admin", "name": "admin", "exact": true, "password": "stackbench-admin-2026" }, + { "do": "click", "actor": "admin", "testid": "admin-link", "settleMs": 1500 }, + { "do": "click", "actor": "admin", "testid": "restocks-link", "ifAvailable": true }, + { "do": "recordNumber", "actor": "watcher", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Air Purifier" }, "as": "air-before" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-item", "text": "Air Purifier" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-warehouse", "text": "East" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-qty", "text": "5" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-delay", "text": "70" }, + { "do": "click", "actor": "admin", "testid": "schedule-restock-submit", "settleMs": 2000 } + ], + "criteria": [ + { + "id": "311a", + "desc": "a restock scheduled before a restart still fires after it, once, at its own time — an in-memory timer does not survive and must not be how this was built", + "points": 4, + "steps": [ + { "do": "restartBackend", "settleMs": 15000 }, + { "do": "reload", "actor": "watcher", "settleMs": 5000 }, + { "do": "expectNumber", "actor": "watcher", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Air Purifier" }, "relativeTo": "air-before", "plus": 5, "within": 90000 } + ], + "statedBy": "work scheduled\nbefore a restart fires after it, once, at the right time" + }, + { + "id": "311b", + "desc": "it fired exactly once — the restart did not replay it", + "points": 3, + "steps": [ + { "do": "wait", "actor": "watcher", "ms": 20000 }, + { "do": "expectNumber", "actor": "watcher", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Air Purifier" }, "relativeTo": "air-before", "plus": 5 } + ], + "statedBy": "must never double-apply\na scheduled restock or advance an order twice" + } + ] + }, + { + "id": 312, + "name": "A restart does not make pending work fire early", + "actors": ["admin", "watcher"], + "setup": [ + { "do": "signIn", "actor": "admin", "name": "admin", "exact": true, "password": "stackbench-admin-2026" }, + { "do": "click", "actor": "admin", "testid": "admin-link", "settleMs": 1500 }, + { "do": "click", "actor": "admin", "testid": "restocks-link", "ifAvailable": true }, + { "do": "recordNumber", "actor": "watcher", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Espresso Machine" }, "as": "esp-before" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-item", "text": "Espresso Machine" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-warehouse", "text": "West" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-qty", "text": "4" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-delay", "text": "180" }, + { "do": "click", "actor": "admin", "testid": "schedule-restock-submit", "settleMs": 2000 } + ], + "criteria": [ + { + "id": "312a", + "desc": "a restock due in three minutes has not applied thirty seconds after a restart — re-reading pending work must not run it", + "points": 3, + "steps": [ + { "do": "restartBackend", "settleMs": 15000 }, + { "do": "reload", "actor": "watcher", "settleMs": 5000 }, + { "do": "wait", "actor": "watcher", "ms": 25000 }, + { "do": "expectNumber", "actor": "watcher", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Espresso Machine" }, "relativeTo": "esp-before", "plus": 0 } + ], + "statedBy": "a restart must not cause pending work to run\nimmediately" + } + ] + }, + { + "id": 313, + "name": "Time does not create or destroy stock", + "actors": ["shopper", "admin"], + "setup": [ + { "do": "signUp", "actor": "shopper", "name": "consv" }, + { "do": "signIn", "actor": "admin", "name": "admin", "exact": true, "password": "stackbench-admin-2026" }, + { "do": "click", "actor": "admin", "testid": "admin-link", "settleMs": 1500 }, + { "do": "recordNumber", "actor": "admin", "testid": "admin-stock", "in": { "testid": "admin-item-row", "contains": "Bluetooth Speaker" }, "as": "spk-before" } + ], + "criteria": [ + { + "id": "313a", + "desc": "a reservation taken and left to expire returns exactly what it took, counted in the admin's own totals", + "points": 3, + "steps": [ + { "do": "click", "actor": "shopper", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Bluetooth Speaker" }, "settleMs": 1500 }, + { "do": "expectNumber", "actor": "admin", "testid": "admin-stock", "in": { "testid": "admin-item-row", "contains": "Bluetooth Speaker" }, "relativeTo": "spk-before", "plus": -1, "within": 10000 }, + { "do": "wait", "actor": "shopper", "ms": 100000 }, + { "do": "expectNumber", "actor": "admin", "testid": "admin-stock", "in": { "testid": "admin-item-row", "contains": "Bluetooth Speaker" }, "relativeTo": "spk-before", "plus": 0, "within": 30000 } + ], + "statedBy": "units are never created or destroyed by the\npassage of time" + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/03-order-delivery.json b/tools/stack-bench/tracks/ecommerce/scenarios/03-order-delivery.json new file mode 100644 index 00000000000..e25cceefd4e --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/03-order-delivery.json @@ -0,0 +1,243 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 3, + "name": "Order delivery", + "note": "Draft: revised observations require matching reference and defect-control qualification.", + "features": [ + { + "id": 303, + "name": "A shipped order becomes delivered", + "actors": ["customer", "staff"], + "setup": [ + { + "do": "signUp", + "actor": "customer", + "name": "delivery-live" + }, + { + "do": "signIn", + "actor": "staff", + "name": "staff", + "password": "stackbench-staff-2026", + "exact": true + }, + { + "do": "click", + "actor": "staff", + "testid": "staff-link" + }, + { + "do": "click", + "actor": "customer", + "testid": "buy-now", + "in": { + "testid": "item-card", + "contains": "Gaming Mouse" + } + }, + { + "do": "reload", + "actor": "staff", + "settleMs": 1000 + }, + { + "do": "ensureSignedIn", + "actor": "staff", + "name": "staff", + "readyTestid": "current-user", + "exact": true, + "password": "stackbench-staff-2026" + }, + { + "do": "click", + "actor": "staff", + "testid": "overlay-close", + "ifAvailable": true, + "within": 1000 + }, + { + "do": "click", + "actor": "staff", + "testid": "staff-link", + "unlessVisible": "queue-item" + }, + { + "do": "expect", + "actor": "staff", + "testid": "queue-item", + "contains": "Gaming Mouse", + "within": 10000 + }, + { + "do": "click", + "actor": "staff", + "testid": "ship-submit", + "in": { + "testid": "queue-item", + "contains": "Gaming Mouse" + } + }, + { + "do": "click", + "actor": "customer", + "testid": "overlay-close", + "ifAvailable": true, + "unlessVisible": "order-item", + "within": 1000 + }, + { + "do": "reload", + "actor": "customer", + "settleMs": 1000 + }, + { + "do": "ensureSignedIn", + "actor": "customer", + "name": "delivery-live", + "readyTestid": "current-user" + }, + { + "do": "click", + "actor": "customer", + "testid": "overlay-close", + "ifAvailable": true, + "within": 1000 + }, + { + "do": "click", + "actor": "customer", + "testid": "orders-toggle" + }, + { + "do": "expect", + "actor": "customer", + "testid": "order-status", + "in": { + "testid": "order-item", + "contains": "Gaming Mouse" + }, + "statusText": "shipped", + "ignoreCase": true, + "within": 10000 + } + ], + "criteria": [{ + "id": "303a", + "category": "production", + "desc": "a shipped order becomes delivered in customer and staff views", + "points": 3, + "statedBy": "A shipped order becomes delivered after 60 seconds.", + "steps": [ + { + "do": "wait", + "actor": "customer", + "ms": 70000 + }, + { + "do": "reload", + "actor": "customer", + "settleMs": 1000 + }, + { + "do": "ensureSignedIn", + "actor": "customer", + "name": "delivery-live", + "readyTestid": "current-user" + }, + { + "do": "click", + "actor": "customer", + "testid": "overlay-close", + "ifAvailable": true, + "within": 1000 + }, + { + "do": "click", + "actor": "customer", + "testid": "orders-toggle", + "unlessVisible": "order-item" + }, + { + "do": "expect", + "actor": "customer", + "testid": "order-status", + "in": { + "testid": "order-item", + "contains": "Gaming Mouse" + }, + "statusText": "delivered", + "ignoreCase": true, + "within": 15000 + }, + { + "do": "reload", + "actor": "staff", + "settleMs": 1000 + }, + { + "do": "ensureSignedIn", + "actor": "staff", + "name": "staff", + "readyTestid": "current-user", + "exact": true, + "password": "stackbench-staff-2026" + }, + { + "do": "click", + "actor": "staff", + "testid": "overlay-close", + "ifAvailable": true, + "within": 1000 + }, + { + "do": "click", + "actor": "staff", + "testid": "staff-link", + "unlessVisible": "completed-order-item" + }, + { + "do": "expect", + "actor": "staff", + "testid": "completed-order-status", + "in": { + "testid": "completed-order-item", + "contains": "Gaming Mouse" + }, + "value": "delivered", + "ignoreCase": true, + "within": 15000 + } + ] + }] + }, + { + "id": 305, + "name": "Cancellation is final", + "actors": ["customer"], + "setup": [ + { "do": "signUp", "actor": "customer", "name": "delivery-cancel" }, + { "do": "click", "actor": "customer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Keyboard" } }, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { "do": "click", "actor": "customer", "testid": "orders-toggle" }, + { "do": "click", "actor": "customer", "testid": "cancel-order", "in": { "testid": "order-item", "contains": "Keyboard" } }, + { "do": "expect", "actor": "customer", "testid": "order-status", "in": { "testid": "order-item", "contains": "Keyboard" }, "statusText": "cancelled", "ignoreCase": true, "within": 10000 } + ], + "criteria": [{ + "id": "305a", + "category": "production", + "desc": "a cancelled order remains cancelled after the delivery interval", + "points": 2, + "statedBy": "A cancelled order never advances.", + "steps": [ + {"do":"wait","actor":"customer","ms":70000}, + {"do":"reload","actor":"customer","settleMs":1000}, + {"do":"ensureSignedIn","actor":"customer","name":"delivery-cancel","readyTestid":"current-user"}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"customer","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"customer","testid":"order-status","in":{"testid":"order-item","contains":"Keyboard"},"statusText":"cancelled","ignoreCase":true} + ] + }] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/03-reservations.json b/tools/stack-bench/tracks/ecommerce/scenarios/03-reservations.json new file mode 100644 index 00000000000..77aa0a1c30f --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/03-reservations.json @@ -0,0 +1,121 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 3, + "name": "Reservations", + "note": "Each check creates its own account and reservation.", + "features": [ + { + "id": 301, + "name": "A cart reserves stock", + "actors": ["shopper", "watcher"], + "setup": [ + { "do": "signUp", "actor": "shopper", "name": "reserve-stock" }, + { "do": "recordNumber", "actor": "watcher", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Desk Lamp" }, "as": "before" } + ], + "criteria": [{ + "id": "301a", + "category": "production", + "desc": "adding an item reserves one unit for every open viewer", + "points": 2, + "statedBy": "Adding an item to a cart reserves its stock immediately for 90 seconds.", + "steps": [ + { "do": "click", "actor": "shopper", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Desk Lamp" } }, + { "do": "expectNumber", "actor": "watcher", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Desk Lamp" }, "relativeTo": "before", "plus": -1, "within": 10000 } + ] + }] + }, + { + "id": 305, + "name": "Reservation time is visible", + "actors": ["shopper"], + "setup": [ + { "do": "signUp", "actor": "shopper", "name": "reserve-clock" }, + { "do": "click", "actor": "shopper", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Gaming Mouse" } }, + { "do": "click", "actor": "shopper", "testid": "cart-toggle" } + ], + "criteria": [{ + "id": "305a", + "category": "interface", + "desc": "the reservation timer decreases", + "points": 1, + "statedBy": "The cart shows the remaining reservation time.", + "steps": [ + { "do": "expectNumber", "actor": "shopper", "testid": "cart-reservation-timer", "in": { "testid": "cart-item", "contains": "Gaming Mouse" }, "atLeast": 1, "atMost": 90 }, + { "do": "recordNumber", "actor": "shopper", "testid": "cart-reservation-timer", "in": { "testid": "cart-item", "contains": "Gaming Mouse" }, "as": "initial-countdown" }, + { "do": "wait", "actor": "shopper", "ms": 1000 }, + { "do": "expectNumber", "actor": "shopper", "testid": "cart-reservation-timer", "in": { "testid": "cart-item", "contains": "Gaming Mouse" }, "relativeTo": "initial-countdown", "plus": -1, "comparison": "atMost", "within": 10000 } + ] + }] + }, + { + "id": 306, + "name": "Checkout consumes a reservation", + "actors": ["shopper"], + "setup": [ + { "do": "signUp", "actor": "shopper", "name": "reserve-checkout" }, + { "do": "click", "actor": "shopper", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Keyboard" } }, + { "do": "click", "actor": "shopper", "testid": "cart-toggle" } + ], + "criteria": [{ + "id": "306a", + "category": "feature", + "desc": "checkout converts the reservation into an order and empties the cart", + "points": 2, + "statedBy": "Checkout converts a live reservation into a sale.", + "steps": [ + { "do": "click", "actor": "shopper", "testid": "checkout-submit" }, + { "do": "expectNumber", "actor": "shopper", "testid": "cart-count", "equals": 0, "within": 10000 }, + {"do":"click","actor":"shopper","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { "do": "click", "actor": "shopper", "testid": "orders-toggle" }, + { "do": "expect", "actor": "shopper", "testid": "order-item", "contains": "Keyboard", "count": 1 } + ] + }] + }, + { + "id": 307, + "name": "A reservation expires", + "actors": ["shopper"], + "setup": [ + { "do": "signUp", "actor": "shopper", "name": "reserve-expiry" }, + { "do": "click", "actor": "shopper", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Bluetooth Speaker" } }, + { "do": "click", "actor": "shopper", "testid": "cart-toggle" } + ], + "criteria": [{ + "id": "307a", + "category": "feature", + "desc": "an expired reservation marks its cart line", + "points": 3, + "statedBy": "An expired reservation remains visible as expired.", + "steps": [ + { "do": "wait", "actor": "shopper", "ms": 100000 }, + { "do": "expect", "actor": "shopper", "testid": "cart-item-expired", "in": { "testid": "cart-item", "contains": "Bluetooth Speaker" }, "within": 15000 } + ] + }] + }, + { + "id": 308, + "name": "Changing quantity renews a reservation", + "actors": ["shopper"], + "setup": [ + { "do": "signUp", "actor": "shopper", "name": "reserve-renew" }, + { "do": "click", "actor": "shopper", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Headphones" } }, + { "do": "wait", "actor": "shopper", "ms": 60000 }, + { "do": "click", "actor": "shopper", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Headphones" } }, + { "do": "click", "actor": "shopper", "testid": "cart-toggle" } + ], + "criteria": [{ + "id": "308a", + "category": "feature", + "desc": "raising quantity starts a new reservation window", + "points": 2, + "statedBy": "Adding the item again renews the reservation.", + "steps": [ + { "do": "wait", "actor": "shopper", "ms": 40000 }, + { "do": "expectNumber", "actor": "shopper", "testid": "cart-reservation-timer", "in": { "testid": "cart-item", "contains": "Headphones" }, "atLeast": 35 }, + { "do": "expect", "actor": "shopper", "testid": "cart-item-expired", "in": { "testid": "cart-item", "contains": "Headphones" }, "absent": true } + ] + }] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/03-scheduled-restock-apply.json b/tools/stack-bench/tracks/ecommerce/scenarios/03-scheduled-restock-apply.json new file mode 100644 index 00000000000..c9196419413 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/03-scheduled-restock-apply.json @@ -0,0 +1,50 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 3, + "name": "Scheduled restocks", + "note": "This check owns a separate scenario so pending work from another check cannot affect it.", + "features": [ + { + "id": 305, + "name": "A due restock applies", + "actors": ["admin", "watcher"], + "setup": [ + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel", "schedule-restock-item"]}, + { "do": "click", "actor": "admin", "testid": "admin-link", "unlessVisible": ["admin-panel", "schedule-restock-item"] }, + { "do": "click", "actor": "admin", "testid": "restocks-link", "ifAvailable": true, "unlessVisible": "schedule-restock-item" }, + { "do": "click", "actor": "watcher", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "recordNumber", "actor": "watcher", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Keyboard" }, "as": "before" } + ], + "criteria": [ + { + "id": "305a", + "category": "feature", + "desc": "a due restock updates stock and moves to the ledger", + "points": 3, + "statedBy": "A due restock updates stock, leaves the pending list, and enters the stock ledger.", + "steps": [ + { "do": "recordNumber", "actor": "admin", "testid": "stock-ledger-entry", "count": true, "as": "ledger-before" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-item", "text": "Keyboard" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-warehouse", "text": "West" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-qty", "text": "7" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-delay", "text": "15" }, + { "do": "click", "actor": "admin", "testid": "schedule-restock-submit" }, + { "do": "wait", "actor": "admin", "ms": 25000 }, + { "do": "reload", "actor": "watcher", "settleMs": 2000 }, + { "do": "reload", "actor": "admin", "settleMs": 2000 }, + { "do": "ensureSignedIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true, "readyTestid": "current-user" }, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel", "schedule-restock-submit"]}, + { "do": "click", "actor": "admin", "testid": "admin-link", "ifAvailable": true, "unlessVisible": ["admin-panel", "schedule-restock-submit"]}, + { "do": "click", "actor": "admin", "testid": "restocks-link", "ifAvailable": true, "unlessVisible": "schedule-restock-submit"}, + { "do": "click", "actor": "watcher", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "expectNumber", "actor": "watcher", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Keyboard" }, "relativeTo": "before", "plus": 7, "within": 15000 }, + { "do": "expect", "actor": "admin", "testid": "pending-restock-item", "absent": true }, + { "do": "expectElementCount", "actor": "admin", "testid": "stock-ledger-entry", "relativeTo": "ledger-before", "plus": 1 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/03-scheduled-restock-cancel.json b/tools/stack-bench/tracks/ecommerce/scenarios/03-scheduled-restock-cancel.json new file mode 100644 index 00000000000..ab6450c06b2 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/03-scheduled-restock-cancel.json @@ -0,0 +1,45 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 3, + "name": "Scheduled restocks", + "note": "This check owns a separate scenario so pending work from another check cannot affect it.", + "features": [ + { + "id": 306, + "name": "A scheduled restock can be cancelled", + "actors": ["admin", "watcher"], + "setup": [ + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel", "schedule-restock-item"]}, + { "do": "click", "actor": "admin", "testid": "admin-link", "unlessVisible": ["admin-panel", "schedule-restock-item"] }, + { "do": "click", "actor": "admin", "testid": "restocks-link", "ifAvailable": true, "unlessVisible": "schedule-restock-item" }, + { "do": "click", "actor": "watcher", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "recordNumber", "actor": "watcher", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Desk Lamp" }, "as": "before" } + ], + "criteria": [ + { + "id": "306a", + "category": "production", + "desc": "a cancelled restock never applies", + "points": 2, + "statedBy": "An admin can schedule and cancel a restock.", + "steps": [ + { "do": "fill", "actor": "admin", "testid": "schedule-restock-item", "text": "Desk Lamp" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-warehouse", "text": "East" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-qty", "text": "9" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-delay", "text": "15" }, + { "do": "click", "actor": "admin", "testid": "schedule-restock-submit" }, + { "do": "expect", "actor": "admin", "testid": "pending-restock-item", "count": 1 }, + { "do": "click", "actor": "admin", "testid": "pending-restock-cancel", "in": { "testid": "pending-restock-item" } }, + { "do": "expectElementCount", "actor": "admin", "testid": "pending-restock-item", "equals": 0, "within": 6000 }, + { "do": "wait", "actor": "admin", "ms": 25000 }, + { "do": "reload", "actor": "watcher", "settleMs": 2000 }, + { "do": "click", "actor": "watcher", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "expectNumber", "actor": "watcher", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Desk Lamp" }, "relativeTo": "before", "plus": 0 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/03-scheduled-restocks.json b/tools/stack-bench/tracks/ecommerce/scenarios/03-scheduled-restocks.json new file mode 100644 index 00000000000..5307aaca3c1 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/03-scheduled-restocks.json @@ -0,0 +1,42 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 3, + "name": "Scheduled restocks", + "note": "This check owns a separate scenario so pending work from another check cannot affect it.", + "features": [ + { + "id": 302, + "name": "A restock is pending before it is due", + "actors": ["admin"], + "setup": [ + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel", "schedule-restock-item"]}, + { "do": "click", "actor": "admin", "testid": "admin-link", "unlessVisible": ["admin-panel", "schedule-restock-item"] }, + { "do": "click", "actor": "admin", "testid": "restocks-link", "ifAvailable": true, "unlessVisible": "schedule-restock-item" } + ], + "criteria": [ + { + "id": "302a", + "category": "feature", + "desc": "a scheduled restock is pending and its remaining time decreases", + "points": 2, + "statedBy": "A pending restock shows its remaining time.", + "steps": [ + { "do": "fill", "actor": "admin", "testid": "schedule-restock-item", "text": "Webcam" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-warehouse", "text": "East" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-qty", "text": "7" }, + { "do": "fill", "actor": "admin", "testid": "schedule-restock-delay", "text": "90" }, + { "do": "click", "actor": "admin", "testid": "schedule-restock-submit" }, + { "do": "expect", "actor": "admin", "testid": "pending-restock-item", "count": 1, "within": 10000 }, + { "do": "expectNumber", "actor": "admin", "testid": "pending-restock-remaining", "in": { "testid": "pending-restock-item" }, "atLeast": 1, "atMost": 90 }, + { "do": "recordNumber", "actor": "admin", "testid": "pending-restock-remaining", "in": { "testid": "pending-restock-item" }, "as": "initial-countdown" }, + { "do": "wait", "actor": "admin", "ms": 1000 }, + { "do": "expectNumber", "actor": "admin", "testid": "pending-restock-remaining", "in": { "testid": "pending-restock-item" }, "relativeTo": "initial-countdown", "plus": -1, "comparison": "atMost", "within": 10000 }, + { "do": "click", "actor": "admin", "testid": "pending-restock-cancel", "in": { "testid": "pending-restock-item" } } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/03-server-time.json b/tools/stack-bench/tracks/ecommerce/scenarios/03-server-time.json new file mode 100644 index 00000000000..47444dcd854 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/03-server-time.json @@ -0,0 +1,91 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 3, + "name": "Server-controlled time", + "note": "Draft: revised observations require matching reference and defect-control qualification.", + "features": [ + { + "id": 312, + "name": "Restart does not run work early", + "actors": ["admin", "watcher"], + "setup": [ + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel", "schedule-restock-item"]}, + {"do":"click","actor":"admin","testid":"admin-link", "unlessVisible": ["admin-panel", "schedule-restock-item"]}, + {"do":"click","actor":"admin","testid":"restocks-link","ifAvailable":true,"within":1000, "unlessVisible": "schedule-restock-item"}, + {"do":"dbRecordStock","item":"Espresso Machine","as":"before"}, + {"do":"fill","actor":"admin","testid":"schedule-restock-item","text":"Espresso Machine"}, + {"do":"fill","actor":"admin","testid":"schedule-restock-warehouse","text":"West"}, + {"do":"fill","actor":"admin","testid":"schedule-restock-qty","text":"4"}, + {"do":"fill","actor":"admin","testid":"schedule-restock-delay","text":"120"}, + {"do":"recordTime","as":"restock-start"}, + {"do":"click","actor":"admin","testid":"schedule-restock-submit"}, + {"do":"expect","actor":"admin","testid":"pending-restock-item","count":1,"within":10000}, + {"do":"recordTime","as":"restock-start-accepted"}, + {"do":"dbExpectStock","item":"Espresso Machine","relativeTo":"before","plus":0}, + {"do":"restartBackend","settleMs":15000}, + {"do":"reload","actor":"watcher","settleMs":3000}, + {"do":"reload","actor":"admin","settleMs":3000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true,"readyTestid":"current-user"}, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel", "schedule-restock-submit"]}, + {"do":"click","actor":"admin","testid":"admin-link","ifAvailable":true,"within":1000, "unlessVisible": ["admin-panel", "schedule-restock-submit"]}, + {"do":"click","actor":"admin","testid":"restocks-link","ifAvailable":true,"within":1000, "unlessVisible": "schedule-restock-submit"}, + {"do":"expect","actor":"admin","testid":"schedule-restock-submit"} + ], + "criteria": [ + { + "id": "312a", + "category": "production", + "desc": "restart preserves the due time and the work later completes", + "points": 3, + "statedBy": "A pending restock does not run early after a restart.", + "steps": [ + {"do":"wait","actor":"watcher","ms":30000}, + {"do":"reload","actor":"watcher","settleMs":2000}, + {"do":"expectElapsed","since":"restock-start","atMost":100000}, + {"do":"dbExpectStock","item":"Espresso Machine","relativeTo":"before","plus":0}, + {"do":"expect","actor":"admin","testid":"pending-restock-item","count":1}, + {"do":"expectElapsed","since":"restock-start","atMost":100000}, + {"do":"wait","actor":"admin","ms":130000,"since":"restock-start-accepted"}, + {"do":"dbExpectStock","item":"Espresso Machine","relativeTo":"before","plus":4,"within":10000}, + {"do":"reload","actor":"admin","settleMs":2000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true,"readyTestid":"current-user"}, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel", "schedule-restock-submit"]}, + {"do":"click","actor":"admin","testid":"admin-link","ifAvailable":true,"within":1000, "unlessVisible": ["admin-panel", "schedule-restock-submit"]}, + {"do":"click","actor":"admin","testid":"restocks-link","ifAvailable":true,"within":1000, "unlessVisible": "schedule-restock-submit"}, + {"do":"expect","actor":"admin","testid":"schedule-restock-submit"}, + {"do":"expect","actor":"admin","testid":"pending-restock-item","absent":true} + ] + } + ] + }, + { + "id": 313, + "name": "A browser is not the clock", + "actors": ["customer", "watcher"], + "setup": [ + {"do":"signUp","actor":"customer","name":"server-clock"}, + {"do":"recordNumber","actor":"watcher","testid":"item-stock","in":{"testid":"item-card","contains":"Air Purifier"},"as":"before"}, + {"do":"click","actor":"customer","testid":"add-to-cart","in":{"testid":"item-card","contains":"Air Purifier"}}, + {"do":"reload","actor":"watcher","settleMs":1000}, + {"do":"expectNumber","actor":"watcher","testid":"item-stock","in":{"testid":"item-card","contains":"Air Purifier"},"relativeTo":"before","plus":-1,"within":10000}, + {"do":"closeClient","actor":"customer"} + ], + "criteria": [ + { + "id": "313a", + "category": "production", + "desc": "a reservation expires while its browser is closed", + "points": 3, + "statedBy": "A reservation expires without an open browser.", + "steps": [ + {"do":"wait","actor":"watcher","ms":100000}, + {"do":"reload","actor":"watcher","settleMs":1000}, + {"do":"expectNumber","actor":"watcher","testid":"item-stock","in":{"testid":"item-card","contains":"Air Purifier"},"relativeTo":"before","plus":0,"within":30000} + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/diagnostic-cancellation-contention.json b/tools/stack-bench/tracks/ecommerce/scenarios/diagnostic-cancellation-contention.json new file mode 100644 index 00000000000..2c5f3732235 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/diagnostic-cancellation-contention.json @@ -0,0 +1,429 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 2, + "name": "Cancellation contention diagnostic", + "note": "Draft reference diagnostic; not selected by scored campaigns. Widths 1/4/16/64, three fresh-account cohorts each, over two sessions of one account. Client dispatch overlap is not proof of server execution overlap. Each paid non-credit order is verified before cancellation. Repeated calls may refuse or succeed without extra effects. Reconcile warehouse allocations, cancelled status, preserved records and fresh revenue. Payment rows remain history in these references; this does not qualify payment-provider refunds or account-credit reversal. Reference-only readers require exact schema mapping. Preserve unknown request outcomes; they cannot pass. Qualification pending.", + "features": [ + {"id":9100,"name":"1 concurrent cancellation requests, cohort 1","actors":["a","b","admin"], + "setup":[ + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"recordNumber","actor":"admin","testid":"admin-revenue","as":"revenue-before"}, + {"do":"signUp","actor":"a","name":"cancel-1-1"}, + {"do":"signIn","actor":"b","name":"cancel-1-1"}, + {"do":"dbRecordCheckout","account":"{user:cancel-1-1}","item":"Keyboard","as":"checkout-before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"a","testid":"cart-quantity","in":{"testid":"cart-item","contains":"Keyboard"},"equals":1,"within":10000}, + {"do":"dbRecordCheckout","account":"{user:cancel-1-1}","item":"Keyboard","as":"checkout-prepared"}, + {"do":"callAction","actor":"a","action":"checkout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"settleMs":1000}, + {"do":"dbExpectCheckout","before":"checkout-before","prepared":"checkout-prepared","quantity":1}, + {"do":"dbRecordCheckout","account":"{user:cancel-1-1}","item":"Keyboard","as":"cancel-before"}, + {"do":"reload","actor":"a","settleMs":1000}, + {"do":"ensureSignedIn","actor":"a","name":"cancel-1-1","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"a","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"a","testid":"order-status","in":{"testid":"order-item","contains":"Keyboard"},"value":"pending","ignoreCase":true,"within":10000} + ], + "criteria":[{"id":"9100a","category":"production","points":0, + "desc":"concurrent cancellation restores original warehouse stock once, preserves order and payment history, and removes revenue once", + "steps":[ + {"do":"callConcurrently","actors":["a","b"],"from":"a","action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Keyboard","attribute":"data-cancel-input"},"requests":1,"requestTimeoutMs":30000,"settleMs":3000}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCancellation","before":"cancel-before"}, + {"do":"reload","actor":"admin","settleMs":1000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true,"readyTestid":"current-user"}, + {"do":"click","actor":"admin","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":"admin-revenue"}, + {"do":"expectNumber","actor":"admin","testid":"admin-revenue","relativeTo":"revenue-before","plus":0,"within":10000} + ] + }] + }, + {"id":9101,"name":"1 concurrent cancellation requests, cohort 2","actors":["a","b","admin"], + "setup":[ + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"recordNumber","actor":"admin","testid":"admin-revenue","as":"revenue-before"}, + {"do":"signUp","actor":"a","name":"cancel-1-2"}, + {"do":"signIn","actor":"b","name":"cancel-1-2"}, + {"do":"dbRecordCheckout","account":"{user:cancel-1-2}","item":"Keyboard","as":"checkout-before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"a","testid":"cart-quantity","in":{"testid":"cart-item","contains":"Keyboard"},"equals":1,"within":10000}, + {"do":"dbRecordCheckout","account":"{user:cancel-1-2}","item":"Keyboard","as":"checkout-prepared"}, + {"do":"callAction","actor":"a","action":"checkout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"settleMs":1000}, + {"do":"dbExpectCheckout","before":"checkout-before","prepared":"checkout-prepared","quantity":1}, + {"do":"dbRecordCheckout","account":"{user:cancel-1-2}","item":"Keyboard","as":"cancel-before"}, + {"do":"reload","actor":"a","settleMs":1000}, + {"do":"ensureSignedIn","actor":"a","name":"cancel-1-2","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"a","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"a","testid":"order-status","in":{"testid":"order-item","contains":"Keyboard"},"value":"pending","ignoreCase":true,"within":10000} + ], + "criteria":[{"id":"9101a","category":"production","points":0, + "desc":"concurrent cancellation restores original warehouse stock once, preserves order and payment history, and removes revenue once", + "steps":[ + {"do":"callConcurrently","actors":["a","b"],"from":"a","action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Keyboard","attribute":"data-cancel-input"},"requests":1,"requestTimeoutMs":30000,"settleMs":3000}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCancellation","before":"cancel-before"}, + {"do":"reload","actor":"admin","settleMs":1000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true,"readyTestid":"current-user"}, + {"do":"click","actor":"admin","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":"admin-revenue"}, + {"do":"expectNumber","actor":"admin","testid":"admin-revenue","relativeTo":"revenue-before","plus":0,"within":10000} + ] + }] + }, + {"id":9102,"name":"1 concurrent cancellation requests, cohort 3","actors":["a","b","admin"], + "setup":[ + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"recordNumber","actor":"admin","testid":"admin-revenue","as":"revenue-before"}, + {"do":"signUp","actor":"a","name":"cancel-1-3"}, + {"do":"signIn","actor":"b","name":"cancel-1-3"}, + {"do":"dbRecordCheckout","account":"{user:cancel-1-3}","item":"Keyboard","as":"checkout-before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"a","testid":"cart-quantity","in":{"testid":"cart-item","contains":"Keyboard"},"equals":1,"within":10000}, + {"do":"dbRecordCheckout","account":"{user:cancel-1-3}","item":"Keyboard","as":"checkout-prepared"}, + {"do":"callAction","actor":"a","action":"checkout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"settleMs":1000}, + {"do":"dbExpectCheckout","before":"checkout-before","prepared":"checkout-prepared","quantity":1}, + {"do":"dbRecordCheckout","account":"{user:cancel-1-3}","item":"Keyboard","as":"cancel-before"}, + {"do":"reload","actor":"a","settleMs":1000}, + {"do":"ensureSignedIn","actor":"a","name":"cancel-1-3","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"a","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"a","testid":"order-status","in":{"testid":"order-item","contains":"Keyboard"},"value":"pending","ignoreCase":true,"within":10000} + ], + "criteria":[{"id":"9102a","category":"production","points":0, + "desc":"concurrent cancellation restores original warehouse stock once, preserves order and payment history, and removes revenue once", + "steps":[ + {"do":"callConcurrently","actors":["a","b"],"from":"a","action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Keyboard","attribute":"data-cancel-input"},"requests":1,"requestTimeoutMs":30000,"settleMs":3000}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCancellation","before":"cancel-before"}, + {"do":"reload","actor":"admin","settleMs":1000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true,"readyTestid":"current-user"}, + {"do":"click","actor":"admin","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":"admin-revenue"}, + {"do":"expectNumber","actor":"admin","testid":"admin-revenue","relativeTo":"revenue-before","plus":0,"within":10000} + ] + }] + }, + {"id":9103,"name":"4 concurrent cancellation requests, cohort 1","actors":["a","b","admin"], + "setup":[ + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"recordNumber","actor":"admin","testid":"admin-revenue","as":"revenue-before"}, + {"do":"signUp","actor":"a","name":"cancel-4-1"}, + {"do":"signIn","actor":"b","name":"cancel-4-1"}, + {"do":"dbRecordCheckout","account":"{user:cancel-4-1}","item":"Keyboard","as":"checkout-before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"a","testid":"cart-quantity","in":{"testid":"cart-item","contains":"Keyboard"},"equals":1,"within":10000}, + {"do":"dbRecordCheckout","account":"{user:cancel-4-1}","item":"Keyboard","as":"checkout-prepared"}, + {"do":"callAction","actor":"a","action":"checkout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"settleMs":1000}, + {"do":"dbExpectCheckout","before":"checkout-before","prepared":"checkout-prepared","quantity":1}, + {"do":"dbRecordCheckout","account":"{user:cancel-4-1}","item":"Keyboard","as":"cancel-before"}, + {"do":"reload","actor":"a","settleMs":1000}, + {"do":"ensureSignedIn","actor":"a","name":"cancel-4-1","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"a","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"a","testid":"order-status","in":{"testid":"order-item","contains":"Keyboard"},"value":"pending","ignoreCase":true,"within":10000} + ], + "criteria":[{"id":"9103a","category":"production","points":0, + "desc":"concurrent cancellation restores original warehouse stock once, preserves order and payment history, and removes revenue once", + "steps":[ + {"do":"callConcurrently","actors":["a","b"],"from":"a","action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Keyboard","attribute":"data-cancel-input"},"requests":4,"requestTimeoutMs":30000,"settleMs":3000}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCancellation","before":"cancel-before"}, + {"do":"reload","actor":"admin","settleMs":1000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true,"readyTestid":"current-user"}, + {"do":"click","actor":"admin","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":"admin-revenue"}, + {"do":"expectNumber","actor":"admin","testid":"admin-revenue","relativeTo":"revenue-before","plus":0,"within":10000} + ] + }] + }, + {"id":9104,"name":"4 concurrent cancellation requests, cohort 2","actors":["a","b","admin"], + "setup":[ + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"recordNumber","actor":"admin","testid":"admin-revenue","as":"revenue-before"}, + {"do":"signUp","actor":"a","name":"cancel-4-2"}, + {"do":"signIn","actor":"b","name":"cancel-4-2"}, + {"do":"dbRecordCheckout","account":"{user:cancel-4-2}","item":"Keyboard","as":"checkout-before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"a","testid":"cart-quantity","in":{"testid":"cart-item","contains":"Keyboard"},"equals":1,"within":10000}, + {"do":"dbRecordCheckout","account":"{user:cancel-4-2}","item":"Keyboard","as":"checkout-prepared"}, + {"do":"callAction","actor":"a","action":"checkout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"settleMs":1000}, + {"do":"dbExpectCheckout","before":"checkout-before","prepared":"checkout-prepared","quantity":1}, + {"do":"dbRecordCheckout","account":"{user:cancel-4-2}","item":"Keyboard","as":"cancel-before"}, + {"do":"reload","actor":"a","settleMs":1000}, + {"do":"ensureSignedIn","actor":"a","name":"cancel-4-2","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"a","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"a","testid":"order-status","in":{"testid":"order-item","contains":"Keyboard"},"value":"pending","ignoreCase":true,"within":10000} + ], + "criteria":[{"id":"9104a","category":"production","points":0, + "desc":"concurrent cancellation restores original warehouse stock once, preserves order and payment history, and removes revenue once", + "steps":[ + {"do":"callConcurrently","actors":["a","b"],"from":"a","action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Keyboard","attribute":"data-cancel-input"},"requests":4,"requestTimeoutMs":30000,"settleMs":3000}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCancellation","before":"cancel-before"}, + {"do":"reload","actor":"admin","settleMs":1000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true,"readyTestid":"current-user"}, + {"do":"click","actor":"admin","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":"admin-revenue"}, + {"do":"expectNumber","actor":"admin","testid":"admin-revenue","relativeTo":"revenue-before","plus":0,"within":10000} + ] + }] + }, + {"id":9105,"name":"4 concurrent cancellation requests, cohort 3","actors":["a","b","admin"], + "setup":[ + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"recordNumber","actor":"admin","testid":"admin-revenue","as":"revenue-before"}, + {"do":"signUp","actor":"a","name":"cancel-4-3"}, + {"do":"signIn","actor":"b","name":"cancel-4-3"}, + {"do":"dbRecordCheckout","account":"{user:cancel-4-3}","item":"Keyboard","as":"checkout-before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"a","testid":"cart-quantity","in":{"testid":"cart-item","contains":"Keyboard"},"equals":1,"within":10000}, + {"do":"dbRecordCheckout","account":"{user:cancel-4-3}","item":"Keyboard","as":"checkout-prepared"}, + {"do":"callAction","actor":"a","action":"checkout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"settleMs":1000}, + {"do":"dbExpectCheckout","before":"checkout-before","prepared":"checkout-prepared","quantity":1}, + {"do":"dbRecordCheckout","account":"{user:cancel-4-3}","item":"Keyboard","as":"cancel-before"}, + {"do":"reload","actor":"a","settleMs":1000}, + {"do":"ensureSignedIn","actor":"a","name":"cancel-4-3","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"a","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"a","testid":"order-status","in":{"testid":"order-item","contains":"Keyboard"},"value":"pending","ignoreCase":true,"within":10000} + ], + "criteria":[{"id":"9105a","category":"production","points":0, + "desc":"concurrent cancellation restores original warehouse stock once, preserves order and payment history, and removes revenue once", + "steps":[ + {"do":"callConcurrently","actors":["a","b"],"from":"a","action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Keyboard","attribute":"data-cancel-input"},"requests":4,"requestTimeoutMs":30000,"settleMs":3000}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCancellation","before":"cancel-before"}, + {"do":"reload","actor":"admin","settleMs":1000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true,"readyTestid":"current-user"}, + {"do":"click","actor":"admin","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":"admin-revenue"}, + {"do":"expectNumber","actor":"admin","testid":"admin-revenue","relativeTo":"revenue-before","plus":0,"within":10000} + ] + }] + }, + {"id":9106,"name":"16 concurrent cancellation requests, cohort 1","actors":["a","b","admin"], + "setup":[ + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"recordNumber","actor":"admin","testid":"admin-revenue","as":"revenue-before"}, + {"do":"signUp","actor":"a","name":"cancel-16-1"}, + {"do":"signIn","actor":"b","name":"cancel-16-1"}, + {"do":"dbRecordCheckout","account":"{user:cancel-16-1}","item":"Keyboard","as":"checkout-before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"a","testid":"cart-quantity","in":{"testid":"cart-item","contains":"Keyboard"},"equals":1,"within":10000}, + {"do":"dbRecordCheckout","account":"{user:cancel-16-1}","item":"Keyboard","as":"checkout-prepared"}, + {"do":"callAction","actor":"a","action":"checkout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"settleMs":1000}, + {"do":"dbExpectCheckout","before":"checkout-before","prepared":"checkout-prepared","quantity":1}, + {"do":"dbRecordCheckout","account":"{user:cancel-16-1}","item":"Keyboard","as":"cancel-before"}, + {"do":"reload","actor":"a","settleMs":1000}, + {"do":"ensureSignedIn","actor":"a","name":"cancel-16-1","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"a","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"a","testid":"order-status","in":{"testid":"order-item","contains":"Keyboard"},"value":"pending","ignoreCase":true,"within":10000} + ], + "criteria":[{"id":"9106a","category":"production","points":0, + "desc":"concurrent cancellation restores original warehouse stock once, preserves order and payment history, and removes revenue once", + "steps":[ + {"do":"callConcurrently","actors":["a","b"],"from":"a","action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Keyboard","attribute":"data-cancel-input"},"requests":16,"requestTimeoutMs":30000,"settleMs":3000}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCancellation","before":"cancel-before"}, + {"do":"reload","actor":"admin","settleMs":1000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true,"readyTestid":"current-user"}, + {"do":"click","actor":"admin","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":"admin-revenue"}, + {"do":"expectNumber","actor":"admin","testid":"admin-revenue","relativeTo":"revenue-before","plus":0,"within":10000} + ] + }] + }, + {"id":9107,"name":"16 concurrent cancellation requests, cohort 2","actors":["a","b","admin"], + "setup":[ + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"recordNumber","actor":"admin","testid":"admin-revenue","as":"revenue-before"}, + {"do":"signUp","actor":"a","name":"cancel-16-2"}, + {"do":"signIn","actor":"b","name":"cancel-16-2"}, + {"do":"dbRecordCheckout","account":"{user:cancel-16-2}","item":"Keyboard","as":"checkout-before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"a","testid":"cart-quantity","in":{"testid":"cart-item","contains":"Keyboard"},"equals":1,"within":10000}, + {"do":"dbRecordCheckout","account":"{user:cancel-16-2}","item":"Keyboard","as":"checkout-prepared"}, + {"do":"callAction","actor":"a","action":"checkout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"settleMs":1000}, + {"do":"dbExpectCheckout","before":"checkout-before","prepared":"checkout-prepared","quantity":1}, + {"do":"dbRecordCheckout","account":"{user:cancel-16-2}","item":"Keyboard","as":"cancel-before"}, + {"do":"reload","actor":"a","settleMs":1000}, + {"do":"ensureSignedIn","actor":"a","name":"cancel-16-2","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"a","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"a","testid":"order-status","in":{"testid":"order-item","contains":"Keyboard"},"value":"pending","ignoreCase":true,"within":10000} + ], + "criteria":[{"id":"9107a","category":"production","points":0, + "desc":"concurrent cancellation restores original warehouse stock once, preserves order and payment history, and removes revenue once", + "steps":[ + {"do":"callConcurrently","actors":["a","b"],"from":"a","action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Keyboard","attribute":"data-cancel-input"},"requests":16,"requestTimeoutMs":30000,"settleMs":3000}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCancellation","before":"cancel-before"}, + {"do":"reload","actor":"admin","settleMs":1000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true,"readyTestid":"current-user"}, + {"do":"click","actor":"admin","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":"admin-revenue"}, + {"do":"expectNumber","actor":"admin","testid":"admin-revenue","relativeTo":"revenue-before","plus":0,"within":10000} + ] + }] + }, + {"id":9108,"name":"16 concurrent cancellation requests, cohort 3","actors":["a","b","admin"], + "setup":[ + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"recordNumber","actor":"admin","testid":"admin-revenue","as":"revenue-before"}, + {"do":"signUp","actor":"a","name":"cancel-16-3"}, + {"do":"signIn","actor":"b","name":"cancel-16-3"}, + {"do":"dbRecordCheckout","account":"{user:cancel-16-3}","item":"Keyboard","as":"checkout-before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"a","testid":"cart-quantity","in":{"testid":"cart-item","contains":"Keyboard"},"equals":1,"within":10000}, + {"do":"dbRecordCheckout","account":"{user:cancel-16-3}","item":"Keyboard","as":"checkout-prepared"}, + {"do":"callAction","actor":"a","action":"checkout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"settleMs":1000}, + {"do":"dbExpectCheckout","before":"checkout-before","prepared":"checkout-prepared","quantity":1}, + {"do":"dbRecordCheckout","account":"{user:cancel-16-3}","item":"Keyboard","as":"cancel-before"}, + {"do":"reload","actor":"a","settleMs":1000}, + {"do":"ensureSignedIn","actor":"a","name":"cancel-16-3","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"a","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"a","testid":"order-status","in":{"testid":"order-item","contains":"Keyboard"},"value":"pending","ignoreCase":true,"within":10000} + ], + "criteria":[{"id":"9108a","category":"production","points":0, + "desc":"concurrent cancellation restores original warehouse stock once, preserves order and payment history, and removes revenue once", + "steps":[ + {"do":"callConcurrently","actors":["a","b"],"from":"a","action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Keyboard","attribute":"data-cancel-input"},"requests":16,"requestTimeoutMs":30000,"settleMs":3000}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCancellation","before":"cancel-before"}, + {"do":"reload","actor":"admin","settleMs":1000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true,"readyTestid":"current-user"}, + {"do":"click","actor":"admin","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":"admin-revenue"}, + {"do":"expectNumber","actor":"admin","testid":"admin-revenue","relativeTo":"revenue-before","plus":0,"within":10000} + ] + }] + }, + {"id":9109,"name":"64 concurrent cancellation requests, cohort 1","actors":["a","b","admin"], + "setup":[ + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"recordNumber","actor":"admin","testid":"admin-revenue","as":"revenue-before"}, + {"do":"signUp","actor":"a","name":"cancel-64-1"}, + {"do":"signIn","actor":"b","name":"cancel-64-1"}, + {"do":"dbRecordCheckout","account":"{user:cancel-64-1}","item":"Keyboard","as":"checkout-before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"a","testid":"cart-quantity","in":{"testid":"cart-item","contains":"Keyboard"},"equals":1,"within":10000}, + {"do":"dbRecordCheckout","account":"{user:cancel-64-1}","item":"Keyboard","as":"checkout-prepared"}, + {"do":"callAction","actor":"a","action":"checkout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"settleMs":1000}, + {"do":"dbExpectCheckout","before":"checkout-before","prepared":"checkout-prepared","quantity":1}, + {"do":"dbRecordCheckout","account":"{user:cancel-64-1}","item":"Keyboard","as":"cancel-before"}, + {"do":"reload","actor":"a","settleMs":1000}, + {"do":"ensureSignedIn","actor":"a","name":"cancel-64-1","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"a","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"a","testid":"order-status","in":{"testid":"order-item","contains":"Keyboard"},"value":"pending","ignoreCase":true,"within":10000} + ], + "criteria":[{"id":"9109a","category":"production","points":0, + "desc":"concurrent cancellation restores original warehouse stock once, preserves order and payment history, and removes revenue once", + "steps":[ + {"do":"callConcurrently","actors":["a","b"],"from":"a","action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Keyboard","attribute":"data-cancel-input"},"requests":64,"requestTimeoutMs":30000,"settleMs":3000}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCancellation","before":"cancel-before"}, + {"do":"reload","actor":"admin","settleMs":1000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true,"readyTestid":"current-user"}, + {"do":"click","actor":"admin","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":"admin-revenue"}, + {"do":"expectNumber","actor":"admin","testid":"admin-revenue","relativeTo":"revenue-before","plus":0,"within":10000} + ] + }] + }, + {"id":9110,"name":"64 concurrent cancellation requests, cohort 2","actors":["a","b","admin"], + "setup":[ + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"recordNumber","actor":"admin","testid":"admin-revenue","as":"revenue-before"}, + {"do":"signUp","actor":"a","name":"cancel-64-2"}, + {"do":"signIn","actor":"b","name":"cancel-64-2"}, + {"do":"dbRecordCheckout","account":"{user:cancel-64-2}","item":"Keyboard","as":"checkout-before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"a","testid":"cart-quantity","in":{"testid":"cart-item","contains":"Keyboard"},"equals":1,"within":10000}, + {"do":"dbRecordCheckout","account":"{user:cancel-64-2}","item":"Keyboard","as":"checkout-prepared"}, + {"do":"callAction","actor":"a","action":"checkout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"settleMs":1000}, + {"do":"dbExpectCheckout","before":"checkout-before","prepared":"checkout-prepared","quantity":1}, + {"do":"dbRecordCheckout","account":"{user:cancel-64-2}","item":"Keyboard","as":"cancel-before"}, + {"do":"reload","actor":"a","settleMs":1000}, + {"do":"ensureSignedIn","actor":"a","name":"cancel-64-2","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"a","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"a","testid":"order-status","in":{"testid":"order-item","contains":"Keyboard"},"value":"pending","ignoreCase":true,"within":10000} + ], + "criteria":[{"id":"9110a","category":"production","points":0, + "desc":"concurrent cancellation restores original warehouse stock once, preserves order and payment history, and removes revenue once", + "steps":[ + {"do":"callConcurrently","actors":["a","b"],"from":"a","action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Keyboard","attribute":"data-cancel-input"},"requests":64,"requestTimeoutMs":30000,"settleMs":3000}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCancellation","before":"cancel-before"}, + {"do":"reload","actor":"admin","settleMs":1000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true,"readyTestid":"current-user"}, + {"do":"click","actor":"admin","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":"admin-revenue"}, + {"do":"expectNumber","actor":"admin","testid":"admin-revenue","relativeTo":"revenue-before","plus":0,"within":10000} + ] + }] + }, + {"id":9111,"name":"64 concurrent cancellation requests, cohort 3","actors":["a","b","admin"], + "setup":[ + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"recordNumber","actor":"admin","testid":"admin-revenue","as":"revenue-before"}, + {"do":"signUp","actor":"a","name":"cancel-64-3"}, + {"do":"signIn","actor":"b","name":"cancel-64-3"}, + {"do":"dbRecordCheckout","account":"{user:cancel-64-3}","item":"Keyboard","as":"checkout-before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"a","testid":"cart-quantity","in":{"testid":"cart-item","contains":"Keyboard"},"equals":1,"within":10000}, + {"do":"dbRecordCheckout","account":"{user:cancel-64-3}","item":"Keyboard","as":"checkout-prepared"}, + {"do":"callAction","actor":"a","action":"checkout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"settleMs":1000}, + {"do":"dbExpectCheckout","before":"checkout-before","prepared":"checkout-prepared","quantity":1}, + {"do":"dbRecordCheckout","account":"{user:cancel-64-3}","item":"Keyboard","as":"cancel-before"}, + {"do":"reload","actor":"a","settleMs":1000}, + {"do":"ensureSignedIn","actor":"a","name":"cancel-64-3","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"a","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"a","testid":"order-status","in":{"testid":"order-item","contains":"Keyboard"},"value":"pending","ignoreCase":true,"within":10000} + ], + "criteria":[{"id":"9111a","category":"production","points":0, + "desc":"concurrent cancellation restores original warehouse stock once, preserves order and payment history, and removes revenue once", + "steps":[ + {"do":"callConcurrently","actors":["a","b"],"from":"a","action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Keyboard","attribute":"data-cancel-input"},"requests":64,"requestTimeoutMs":30000,"settleMs":3000}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCancellation","before":"cancel-before"}, + {"do":"reload","actor":"admin","settleMs":1000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true,"readyTestid":"current-user"}, + {"do":"click","actor":"admin","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":"admin-revenue"}, + {"do":"expectNumber","actor":"admin","testid":"admin-revenue","relativeTo":"revenue-before","plus":0,"within":10000} + ] + }] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/diagnostic-checkout-application-crash.json b/tools/stack-bench/tracks/ecommerce/scenarios/diagnostic-checkout-application-crash.json new file mode 100644 index 00000000000..7bb4a9b8a81 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/diagnostic-checkout-application-crash.json @@ -0,0 +1,291 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 2, + "name": "Checkout application crash diagnostic", + "note": "Draft reference diagnostic, zero points, not selected by scored campaigns. Run ten fresh trials per width/offset on disposable leased apps. STDB uses only the database boundary, with confirmed native WebSocket acknowledgements. PG/Mongo test application and database separately. Failed responses may follow a commit. Unknown checkout effects must be absent or complete; confirmed success and earlier confirmed state must survive. Missed fault windows, expired reservations and incomplete injection stay unmeasured. The baseline retains a synchronous HTTP or confirmed native WebSocket completion receipt before stored-state reconciliation. A separate post-recovery checkout tests app progress. State is compared after process recovery and reader connection; valid reconstruction from current durable data is allowed. This tests recovered application consistency, not untouched physical rows or raw database transaction atomicity. No power-loss or storage-device guarantee is claimed.", + "features": [ + { + "id": 9700, + "name": "application crash, 1 checkout calls, 0ms offset", + "actors": ["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"crash"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"baseline-before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"baseline-prepared"}, + {"do":"confirmCheckout","actor":"a","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]}}, + {"do":"dbExpectCheckout","before":"baseline-before","prepared":"baseline-prepared","quantity":1}, + {"do":"reload","actor":"a","settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"prepared"} + ], + "criteria": [ + { + "id": "9700a", + "category": "production", + "points": 0, + "desc": "after a verified process crash and reconnection, recovered state retains acknowledged checkout effects and contains no partial checkout", + "steps": [ + {"do":"crashCheckout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"actor":"a","before":"before","prepared":"prepared","quantity":1,"requests":1,"offsetMs":0,"target":"application"} + ] + }, + { + "id": "9700b", + "category": "production", + "points": 0, + "desc": "a fresh customer can complete checkout after recovery", + "steps": [ + {"do":"signUp","actor":"b","name":"recovered"}, + {"do":"dbRecordCheckout","account":"{user:recovered}","item":"Keyboard","as":"recovered-before"}, + {"do":"click", "awaitWrites": true,"actor":"b","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:recovered}","item":"Keyboard","as":"recovered-prepared"}, + {"do":"click","actor":"b","testid":"cart-toggle"}, + {"do":"click","actor":"b","testid":"checkout-submit"}, + {"do":"expectNumber","actor":"b","testid":"cart-count","equals":0,"within":10000}, + {"do":"click","actor":"b","testid":"overlay-close","ifAvailable":true}, + {"do":"dbExpectCheckout","before":"recovered-before","prepared":"recovered-prepared","quantity":1} + ] + } + ] + }, + { + "id": 9701, + "name": "application crash, 1 checkout calls, 5ms offset", + "actors": ["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"crash"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"baseline-before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"baseline-prepared"}, + {"do":"confirmCheckout","actor":"a","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]}}, + {"do":"dbExpectCheckout","before":"baseline-before","prepared":"baseline-prepared","quantity":1}, + {"do":"reload","actor":"a","settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"prepared"} + ], + "criteria": [ + { + "id": "9701a", + "category": "production", + "points": 0, + "desc": "after a verified process crash and reconnection, recovered state retains acknowledged checkout effects and contains no partial checkout", + "steps": [ + {"do":"crashCheckout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"actor":"a","before":"before","prepared":"prepared","quantity":1,"requests":1,"offsetMs":5,"target":"application"} + ] + }, + { + "id": "9701b", + "category": "production", + "points": 0, + "desc": "a fresh customer can complete checkout after recovery", + "steps": [ + {"do":"signUp","actor":"b","name":"recovered"}, + {"do":"dbRecordCheckout","account":"{user:recovered}","item":"Keyboard","as":"recovered-before"}, + {"do":"click", "awaitWrites": true,"actor":"b","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:recovered}","item":"Keyboard","as":"recovered-prepared"}, + {"do":"click","actor":"b","testid":"cart-toggle"}, + {"do":"click","actor":"b","testid":"checkout-submit"}, + {"do":"expectNumber","actor":"b","testid":"cart-count","equals":0,"within":10000}, + {"do":"click","actor":"b","testid":"overlay-close","ifAvailable":true}, + {"do":"dbExpectCheckout","before":"recovered-before","prepared":"recovered-prepared","quantity":1} + ] + } + ] + }, + { + "id": 9702, + "name": "application crash, 1 checkout calls, 20ms offset", + "actors": ["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"crash"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"baseline-before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"baseline-prepared"}, + {"do":"confirmCheckout","actor":"a","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]}}, + {"do":"dbExpectCheckout","before":"baseline-before","prepared":"baseline-prepared","quantity":1}, + {"do":"reload","actor":"a","settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"prepared"} + ], + "criteria": [ + { + "id": "9702a", + "category": "production", + "points": 0, + "desc": "after a verified process crash and reconnection, recovered state retains acknowledged checkout effects and contains no partial checkout", + "steps": [ + {"do":"crashCheckout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"actor":"a","before":"before","prepared":"prepared","quantity":1,"requests":1,"offsetMs":20,"target":"application"} + ] + }, + { + "id": "9702b", + "category": "production", + "points": 0, + "desc": "a fresh customer can complete checkout after recovery", + "steps": [ + {"do":"signUp","actor":"b","name":"recovered"}, + {"do":"dbRecordCheckout","account":"{user:recovered}","item":"Keyboard","as":"recovered-before"}, + {"do":"click", "awaitWrites": true,"actor":"b","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:recovered}","item":"Keyboard","as":"recovered-prepared"}, + {"do":"click","actor":"b","testid":"cart-toggle"}, + {"do":"click","actor":"b","testid":"checkout-submit"}, + {"do":"expectNumber","actor":"b","testid":"cart-count","equals":0,"within":10000}, + {"do":"click","actor":"b","testid":"overlay-close","ifAvailable":true}, + {"do":"dbExpectCheckout","before":"recovered-before","prepared":"recovered-prepared","quantity":1} + ] + } + ] + }, + { + "id": 9703, + "name": "application crash, 16 checkout calls, 0ms offset", + "actors": ["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"crash"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"baseline-before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"baseline-prepared"}, + {"do":"confirmCheckout","actor":"a","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]}}, + {"do":"dbExpectCheckout","before":"baseline-before","prepared":"baseline-prepared","quantity":1}, + {"do":"reload","actor":"a","settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"prepared"} + ], + "criteria": [ + { + "id": "9703a", + "category": "production", + "points": 0, + "desc": "after a verified process crash and reconnection, recovered state retains acknowledged checkout effects and contains no partial checkout", + "steps": [ + {"do":"crashCheckout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"actor":"a","before":"before","prepared":"prepared","quantity":1,"requests":16,"offsetMs":0,"target":"application"} + ] + }, + { + "id": "9703b", + "category": "production", + "points": 0, + "desc": "a fresh customer can complete checkout after recovery", + "steps": [ + {"do":"signUp","actor":"b","name":"recovered"}, + {"do":"dbRecordCheckout","account":"{user:recovered}","item":"Keyboard","as":"recovered-before"}, + {"do":"click", "awaitWrites": true,"actor":"b","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:recovered}","item":"Keyboard","as":"recovered-prepared"}, + {"do":"click","actor":"b","testid":"cart-toggle"}, + {"do":"click","actor":"b","testid":"checkout-submit"}, + {"do":"expectNumber","actor":"b","testid":"cart-count","equals":0,"within":10000}, + {"do":"click","actor":"b","testid":"overlay-close","ifAvailable":true}, + {"do":"dbExpectCheckout","before":"recovered-before","prepared":"recovered-prepared","quantity":1} + ] + } + ] + }, + { + "id": 9704, + "name": "application crash, 16 checkout calls, 5ms offset", + "actors": ["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"crash"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"baseline-before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"baseline-prepared"}, + {"do":"confirmCheckout","actor":"a","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]}}, + {"do":"dbExpectCheckout","before":"baseline-before","prepared":"baseline-prepared","quantity":1}, + {"do":"reload","actor":"a","settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"prepared"} + ], + "criteria": [ + { + "id": "9704a", + "category": "production", + "points": 0, + "desc": "after a verified process crash and reconnection, recovered state retains acknowledged checkout effects and contains no partial checkout", + "steps": [ + {"do":"crashCheckout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"actor":"a","before":"before","prepared":"prepared","quantity":1,"requests":16,"offsetMs":5,"target":"application"} + ] + }, + { + "id": "9704b", + "category": "production", + "points": 0, + "desc": "a fresh customer can complete checkout after recovery", + "steps": [ + {"do":"signUp","actor":"b","name":"recovered"}, + {"do":"dbRecordCheckout","account":"{user:recovered}","item":"Keyboard","as":"recovered-before"}, + {"do":"click", "awaitWrites": true,"actor":"b","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:recovered}","item":"Keyboard","as":"recovered-prepared"}, + {"do":"click","actor":"b","testid":"cart-toggle"}, + {"do":"click","actor":"b","testid":"checkout-submit"}, + {"do":"expectNumber","actor":"b","testid":"cart-count","equals":0,"within":10000}, + {"do":"click","actor":"b","testid":"overlay-close","ifAvailable":true}, + {"do":"dbExpectCheckout","before":"recovered-before","prepared":"recovered-prepared","quantity":1} + ] + } + ] + }, + { + "id": 9705, + "name": "application crash, 16 checkout calls, 20ms offset", + "actors": ["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"crash"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"baseline-before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"baseline-prepared"}, + {"do":"confirmCheckout","actor":"a","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]}}, + {"do":"dbExpectCheckout","before":"baseline-before","prepared":"baseline-prepared","quantity":1}, + {"do":"reload","actor":"a","settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"prepared"} + ], + "criteria": [ + { + "id": "9705a", + "category": "production", + "points": 0, + "desc": "after a verified process crash and reconnection, recovered state retains acknowledged checkout effects and contains no partial checkout", + "steps": [ + {"do":"crashCheckout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"actor":"a","before":"before","prepared":"prepared","quantity":1,"requests":16,"offsetMs":20,"target":"application"} + ] + }, + { + "id": "9705b", + "category": "production", + "points": 0, + "desc": "a fresh customer can complete checkout after recovery", + "steps": [ + {"do":"signUp","actor":"b","name":"recovered"}, + {"do":"dbRecordCheckout","account":"{user:recovered}","item":"Keyboard","as":"recovered-before"}, + {"do":"click", "awaitWrites": true,"actor":"b","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:recovered}","item":"Keyboard","as":"recovered-prepared"}, + {"do":"click","actor":"b","testid":"cart-toggle"}, + {"do":"click","actor":"b","testid":"checkout-submit"}, + {"do":"expectNumber","actor":"b","testid":"cart-count","equals":0,"within":10000}, + {"do":"click","actor":"b","testid":"overlay-close","ifAvailable":true}, + {"do":"dbExpectCheckout","before":"recovered-before","prepared":"recovered-prepared","quantity":1} + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/diagnostic-checkout-contention.json b/tools/stack-bench/tracks/ecommerce/scenarios/diagnostic-checkout-contention.json new file mode 100644 index 00000000000..9e54d5e727d --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/diagnostic-checkout-contention.json @@ -0,0 +1,357 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 2, + "name": "Checkout contention diagnostic", + "note": "Draft diagnostic only; not selected by scored campaigns. Widths 1/4/16/64 are parallel requests cycled over two sessions of one fresh account, not distinct clients. Three fresh-account cohorts per width share one otherwise idle app. Reference-only stored-state readers require matching source schemas. Each cohort records initial and prepared state, then verifies one order, exact lines and warehouse allocations, one payment, cleared reservations/cart and one stored stock decrement; rejecting all requests cannot pass. Timing is client dispatch through response, not server execution overlap or committed-operation latency. Use a reset disposable app copy and the same runtime/resources for each stack. Qualification is pending.", + "features": [ + {"id": 9000, "name": "1 parallel checkout requests, cohort 1", "actors": ["a", "b"], + "setup": [ + {"do":"signUp","actor":"a","name":"cohort-1-1"}, + {"do":"signIn","actor":"b","name":"cohort-1-1"}, + {"do":"dbRecordCheckout","account":"{user:cohort-1-1}","item":"Keyboard","as":"checkout-before"}, + {"do":"dbRecordStock","item":"Keyboard","as":"stock-before"}, + {"do":"click","actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"a","testid":"cart-quantity","in":{"testid":"cart-item","contains":"Keyboard"},"equals":1,"within":10000} + ], + "criteria": [{"id": "9000a", "category": "production", "points": 0, + "desc": "one prepared cart produces one order and one stored stock decrement", + "steps": [ + {"do":"dbRecordCheckout","account":"{user:cohort-1-1}","item":"Keyboard","as":"checkout-prepared"}, + {"do":"callConcurrently","actors":["a","b"],"action":"checkout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"requests":1,"requestTimeoutMs":30000,"settleMs":3000}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCheckout","before":"checkout-before","prepared":"checkout-prepared","quantity":1}, + {"do":"dbExpectStock","item":"Keyboard","relativeTo":"stock-before","plus":-1}, + {"do":"reload","actor":"a","settleMs":1000}, + {"do":"ensureSignedIn","actor":"a","name":"cohort-1-1","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"a","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"a","testid":"order-item","contains":"Keyboard","count":1,"within":10000}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expect","actor":"a","testid":"cart-item","absent":true,"within":10000} + ] + }] + }, + {"id": 9001, "name": "1 parallel checkout requests, cohort 2", "actors": ["a", "b"], + "setup": [ + {"do":"signUp","actor":"a","name":"cohort-1-2"}, + {"do":"signIn","actor":"b","name":"cohort-1-2"}, + {"do":"dbRecordCheckout","account":"{user:cohort-1-2}","item":"Keyboard","as":"checkout-before"}, + {"do":"dbRecordStock","item":"Keyboard","as":"stock-before"}, + {"do":"click","actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"a","testid":"cart-quantity","in":{"testid":"cart-item","contains":"Keyboard"},"equals":1,"within":10000} + ], + "criteria": [{"id": "9001a", "category": "production", "points": 0, + "desc": "one prepared cart produces one order and one stored stock decrement", + "steps": [ + {"do":"dbRecordCheckout","account":"{user:cohort-1-2}","item":"Keyboard","as":"checkout-prepared"}, + {"do":"callConcurrently","actors":["a","b"],"action":"checkout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"requests":1,"requestTimeoutMs":30000,"settleMs":3000}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCheckout","before":"checkout-before","prepared":"checkout-prepared","quantity":1}, + {"do":"dbExpectStock","item":"Keyboard","relativeTo":"stock-before","plus":-1}, + {"do":"reload","actor":"a","settleMs":1000}, + {"do":"ensureSignedIn","actor":"a","name":"cohort-1-2","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"a","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"a","testid":"order-item","contains":"Keyboard","count":1,"within":10000}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expect","actor":"a","testid":"cart-item","absent":true,"within":10000} + ] + }] + }, + {"id": 9002, "name": "1 parallel checkout requests, cohort 3", "actors": ["a", "b"], + "setup": [ + {"do":"signUp","actor":"a","name":"cohort-1-3"}, + {"do":"signIn","actor":"b","name":"cohort-1-3"}, + {"do":"dbRecordCheckout","account":"{user:cohort-1-3}","item":"Keyboard","as":"checkout-before"}, + {"do":"dbRecordStock","item":"Keyboard","as":"stock-before"}, + {"do":"click","actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"a","testid":"cart-quantity","in":{"testid":"cart-item","contains":"Keyboard"},"equals":1,"within":10000} + ], + "criteria": [{"id": "9002a", "category": "production", "points": 0, + "desc": "one prepared cart produces one order and one stored stock decrement", + "steps": [ + {"do":"dbRecordCheckout","account":"{user:cohort-1-3}","item":"Keyboard","as":"checkout-prepared"}, + {"do":"callConcurrently","actors":["a","b"],"action":"checkout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"requests":1,"requestTimeoutMs":30000,"settleMs":3000}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCheckout","before":"checkout-before","prepared":"checkout-prepared","quantity":1}, + {"do":"dbExpectStock","item":"Keyboard","relativeTo":"stock-before","plus":-1}, + {"do":"reload","actor":"a","settleMs":1000}, + {"do":"ensureSignedIn","actor":"a","name":"cohort-1-3","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"a","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"a","testid":"order-item","contains":"Keyboard","count":1,"within":10000}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expect","actor":"a","testid":"cart-item","absent":true,"within":10000} + ] + }] + }, + {"id": 9003, "name": "4 parallel checkout requests, cohort 1", "actors": ["a", "b"], + "setup": [ + {"do":"signUp","actor":"a","name":"cohort-4-1"}, + {"do":"signIn","actor":"b","name":"cohort-4-1"}, + {"do":"dbRecordCheckout","account":"{user:cohort-4-1}","item":"Keyboard","as":"checkout-before"}, + {"do":"dbRecordStock","item":"Keyboard","as":"stock-before"}, + {"do":"click","actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"a","testid":"cart-quantity","in":{"testid":"cart-item","contains":"Keyboard"},"equals":1,"within":10000} + ], + "criteria": [{"id": "9003a", "category": "production", "points": 0, + "desc": "one prepared cart produces one order and one stored stock decrement", + "steps": [ + {"do":"dbRecordCheckout","account":"{user:cohort-4-1}","item":"Keyboard","as":"checkout-prepared"}, + {"do":"callConcurrently","actors":["a","b"],"action":"checkout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"requests":4,"requestTimeoutMs":30000,"settleMs":3000}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCheckout","before":"checkout-before","prepared":"checkout-prepared","quantity":1}, + {"do":"dbExpectStock","item":"Keyboard","relativeTo":"stock-before","plus":-1}, + {"do":"reload","actor":"a","settleMs":1000}, + {"do":"ensureSignedIn","actor":"a","name":"cohort-4-1","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"a","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"a","testid":"order-item","contains":"Keyboard","count":1,"within":10000}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expect","actor":"a","testid":"cart-item","absent":true,"within":10000} + ] + }] + }, + {"id": 9004, "name": "4 parallel checkout requests, cohort 2", "actors": ["a", "b"], + "setup": [ + {"do":"signUp","actor":"a","name":"cohort-4-2"}, + {"do":"signIn","actor":"b","name":"cohort-4-2"}, + {"do":"dbRecordCheckout","account":"{user:cohort-4-2}","item":"Keyboard","as":"checkout-before"}, + {"do":"dbRecordStock","item":"Keyboard","as":"stock-before"}, + {"do":"click","actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"a","testid":"cart-quantity","in":{"testid":"cart-item","contains":"Keyboard"},"equals":1,"within":10000} + ], + "criteria": [{"id": "9004a", "category": "production", "points": 0, + "desc": "one prepared cart produces one order and one stored stock decrement", + "steps": [ + {"do":"dbRecordCheckout","account":"{user:cohort-4-2}","item":"Keyboard","as":"checkout-prepared"}, + {"do":"callConcurrently","actors":["a","b"],"action":"checkout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"requests":4,"requestTimeoutMs":30000,"settleMs":3000}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCheckout","before":"checkout-before","prepared":"checkout-prepared","quantity":1}, + {"do":"dbExpectStock","item":"Keyboard","relativeTo":"stock-before","plus":-1}, + {"do":"reload","actor":"a","settleMs":1000}, + {"do":"ensureSignedIn","actor":"a","name":"cohort-4-2","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"a","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"a","testid":"order-item","contains":"Keyboard","count":1,"within":10000}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expect","actor":"a","testid":"cart-item","absent":true,"within":10000} + ] + }] + }, + {"id": 9005, "name": "4 parallel checkout requests, cohort 3", "actors": ["a", "b"], + "setup": [ + {"do":"signUp","actor":"a","name":"cohort-4-3"}, + {"do":"signIn","actor":"b","name":"cohort-4-3"}, + {"do":"dbRecordCheckout","account":"{user:cohort-4-3}","item":"Keyboard","as":"checkout-before"}, + {"do":"dbRecordStock","item":"Keyboard","as":"stock-before"}, + {"do":"click","actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"a","testid":"cart-quantity","in":{"testid":"cart-item","contains":"Keyboard"},"equals":1,"within":10000} + ], + "criteria": [{"id": "9005a", "category": "production", "points": 0, + "desc": "one prepared cart produces one order and one stored stock decrement", + "steps": [ + {"do":"dbRecordCheckout","account":"{user:cohort-4-3}","item":"Keyboard","as":"checkout-prepared"}, + {"do":"callConcurrently","actors":["a","b"],"action":"checkout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"requests":4,"requestTimeoutMs":30000,"settleMs":3000}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCheckout","before":"checkout-before","prepared":"checkout-prepared","quantity":1}, + {"do":"dbExpectStock","item":"Keyboard","relativeTo":"stock-before","plus":-1}, + {"do":"reload","actor":"a","settleMs":1000}, + {"do":"ensureSignedIn","actor":"a","name":"cohort-4-3","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"a","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"a","testid":"order-item","contains":"Keyboard","count":1,"within":10000}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expect","actor":"a","testid":"cart-item","absent":true,"within":10000} + ] + }] + }, + {"id": 9006, "name": "16 parallel checkout requests, cohort 1", "actors": ["a", "b"], + "setup": [ + {"do":"signUp","actor":"a","name":"cohort-16-1"}, + {"do":"signIn","actor":"b","name":"cohort-16-1"}, + {"do":"dbRecordCheckout","account":"{user:cohort-16-1}","item":"Keyboard","as":"checkout-before"}, + {"do":"dbRecordStock","item":"Keyboard","as":"stock-before"}, + {"do":"click","actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"a","testid":"cart-quantity","in":{"testid":"cart-item","contains":"Keyboard"},"equals":1,"within":10000} + ], + "criteria": [{"id": "9006a", "category": "production", "points": 0, + "desc": "one prepared cart produces one order and one stored stock decrement", + "steps": [ + {"do":"dbRecordCheckout","account":"{user:cohort-16-1}","item":"Keyboard","as":"checkout-prepared"}, + {"do":"callConcurrently","actors":["a","b"],"action":"checkout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"requests":16,"requestTimeoutMs":30000,"settleMs":3000}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCheckout","before":"checkout-before","prepared":"checkout-prepared","quantity":1}, + {"do":"dbExpectStock","item":"Keyboard","relativeTo":"stock-before","plus":-1}, + {"do":"reload","actor":"a","settleMs":1000}, + {"do":"ensureSignedIn","actor":"a","name":"cohort-16-1","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"a","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"a","testid":"order-item","contains":"Keyboard","count":1,"within":10000}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expect","actor":"a","testid":"cart-item","absent":true,"within":10000} + ] + }] + }, + {"id": 9007, "name": "16 parallel checkout requests, cohort 2", "actors": ["a", "b"], + "setup": [ + {"do":"signUp","actor":"a","name":"cohort-16-2"}, + {"do":"signIn","actor":"b","name":"cohort-16-2"}, + {"do":"dbRecordCheckout","account":"{user:cohort-16-2}","item":"Keyboard","as":"checkout-before"}, + {"do":"dbRecordStock","item":"Keyboard","as":"stock-before"}, + {"do":"click","actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"a","testid":"cart-quantity","in":{"testid":"cart-item","contains":"Keyboard"},"equals":1,"within":10000} + ], + "criteria": [{"id": "9007a", "category": "production", "points": 0, + "desc": "one prepared cart produces one order and one stored stock decrement", + "steps": [ + {"do":"dbRecordCheckout","account":"{user:cohort-16-2}","item":"Keyboard","as":"checkout-prepared"}, + {"do":"callConcurrently","actors":["a","b"],"action":"checkout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"requests":16,"requestTimeoutMs":30000,"settleMs":3000}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCheckout","before":"checkout-before","prepared":"checkout-prepared","quantity":1}, + {"do":"dbExpectStock","item":"Keyboard","relativeTo":"stock-before","plus":-1}, + {"do":"reload","actor":"a","settleMs":1000}, + {"do":"ensureSignedIn","actor":"a","name":"cohort-16-2","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"a","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"a","testid":"order-item","contains":"Keyboard","count":1,"within":10000}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expect","actor":"a","testid":"cart-item","absent":true,"within":10000} + ] + }] + }, + {"id": 9008, "name": "16 parallel checkout requests, cohort 3", "actors": ["a", "b"], + "setup": [ + {"do":"signUp","actor":"a","name":"cohort-16-3"}, + {"do":"signIn","actor":"b","name":"cohort-16-3"}, + {"do":"dbRecordCheckout","account":"{user:cohort-16-3}","item":"Keyboard","as":"checkout-before"}, + {"do":"dbRecordStock","item":"Keyboard","as":"stock-before"}, + {"do":"click","actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"a","testid":"cart-quantity","in":{"testid":"cart-item","contains":"Keyboard"},"equals":1,"within":10000} + ], + "criteria": [{"id": "9008a", "category": "production", "points": 0, + "desc": "one prepared cart produces one order and one stored stock decrement", + "steps": [ + {"do":"dbRecordCheckout","account":"{user:cohort-16-3}","item":"Keyboard","as":"checkout-prepared"}, + {"do":"callConcurrently","actors":["a","b"],"action":"checkout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"requests":16,"requestTimeoutMs":30000,"settleMs":3000}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCheckout","before":"checkout-before","prepared":"checkout-prepared","quantity":1}, + {"do":"dbExpectStock","item":"Keyboard","relativeTo":"stock-before","plus":-1}, + {"do":"reload","actor":"a","settleMs":1000}, + {"do":"ensureSignedIn","actor":"a","name":"cohort-16-3","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"a","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"a","testid":"order-item","contains":"Keyboard","count":1,"within":10000}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expect","actor":"a","testid":"cart-item","absent":true,"within":10000} + ] + }] + }, + {"id": 9009, "name": "64 parallel checkout requests, cohort 1", "actors": ["a", "b"], + "setup": [ + {"do":"signUp","actor":"a","name":"cohort-64-1"}, + {"do":"signIn","actor":"b","name":"cohort-64-1"}, + {"do":"dbRecordCheckout","account":"{user:cohort-64-1}","item":"Keyboard","as":"checkout-before"}, + {"do":"dbRecordStock","item":"Keyboard","as":"stock-before"}, + {"do":"click","actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"a","testid":"cart-quantity","in":{"testid":"cart-item","contains":"Keyboard"},"equals":1,"within":10000} + ], + "criteria": [{"id": "9009a", "category": "production", "points": 0, + "desc": "one prepared cart produces one order and one stored stock decrement", + "steps": [ + {"do":"dbRecordCheckout","account":"{user:cohort-64-1}","item":"Keyboard","as":"checkout-prepared"}, + {"do":"callConcurrently","actors":["a","b"],"action":"checkout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"requests":64,"requestTimeoutMs":30000,"settleMs":3000}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCheckout","before":"checkout-before","prepared":"checkout-prepared","quantity":1}, + {"do":"dbExpectStock","item":"Keyboard","relativeTo":"stock-before","plus":-1}, + {"do":"reload","actor":"a","settleMs":1000}, + {"do":"ensureSignedIn","actor":"a","name":"cohort-64-1","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"a","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"a","testid":"order-item","contains":"Keyboard","count":1,"within":10000}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expect","actor":"a","testid":"cart-item","absent":true,"within":10000} + ] + }] + }, + {"id": 9010, "name": "64 parallel checkout requests, cohort 2", "actors": ["a", "b"], + "setup": [ + {"do":"signUp","actor":"a","name":"cohort-64-2"}, + {"do":"signIn","actor":"b","name":"cohort-64-2"}, + {"do":"dbRecordCheckout","account":"{user:cohort-64-2}","item":"Keyboard","as":"checkout-before"}, + {"do":"dbRecordStock","item":"Keyboard","as":"stock-before"}, + {"do":"click","actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"a","testid":"cart-quantity","in":{"testid":"cart-item","contains":"Keyboard"},"equals":1,"within":10000} + ], + "criteria": [{"id": "9010a", "category": "production", "points": 0, + "desc": "one prepared cart produces one order and one stored stock decrement", + "steps": [ + {"do":"dbRecordCheckout","account":"{user:cohort-64-2}","item":"Keyboard","as":"checkout-prepared"}, + {"do":"callConcurrently","actors":["a","b"],"action":"checkout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"requests":64,"requestTimeoutMs":30000,"settleMs":3000}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCheckout","before":"checkout-before","prepared":"checkout-prepared","quantity":1}, + {"do":"dbExpectStock","item":"Keyboard","relativeTo":"stock-before","plus":-1}, + {"do":"reload","actor":"a","settleMs":1000}, + {"do":"ensureSignedIn","actor":"a","name":"cohort-64-2","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"a","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"a","testid":"order-item","contains":"Keyboard","count":1,"within":10000}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expect","actor":"a","testid":"cart-item","absent":true,"within":10000} + ] + }] + }, + {"id": 9011, "name": "64 parallel checkout requests, cohort 3", "actors": ["a", "b"], + "setup": [ + {"do":"signUp","actor":"a","name":"cohort-64-3"}, + {"do":"signIn","actor":"b","name":"cohort-64-3"}, + {"do":"dbRecordCheckout","account":"{user:cohort-64-3}","item":"Keyboard","as":"checkout-before"}, + {"do":"dbRecordStock","item":"Keyboard","as":"stock-before"}, + {"do":"click","actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"a","testid":"cart-quantity","in":{"testid":"cart-item","contains":"Keyboard"},"equals":1,"within":10000} + ], + "criteria": [{"id": "9011a", "category": "production", "points": 0, + "desc": "one prepared cart produces one order and one stored stock decrement", + "steps": [ + {"do":"dbRecordCheckout","account":"{user:cohort-64-3}","item":"Keyboard","as":"checkout-prepared"}, + {"do":"callConcurrently","actors":["a","b"],"action":"checkout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"requests":64,"requestTimeoutMs":30000,"settleMs":3000}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCheckout","before":"checkout-before","prepared":"checkout-prepared","quantity":1}, + {"do":"dbExpectStock","item":"Keyboard","relativeTo":"stock-before","plus":-1}, + {"do":"reload","actor":"a","settleMs":1000}, + {"do":"ensureSignedIn","actor":"a","name":"cohort-64-3","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"a","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"a","testid":"order-item","contains":"Keyboard","count":1,"within":10000}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expect","actor":"a","testid":"cart-item","absent":true,"within":10000} + ] + }] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/diagnostic-checkout-database-crash.json b/tools/stack-bench/tracks/ecommerce/scenarios/diagnostic-checkout-database-crash.json new file mode 100644 index 00000000000..31cc421df5d --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/diagnostic-checkout-database-crash.json @@ -0,0 +1,291 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 2, + "name": "Checkout database crash diagnostic", + "note": "Draft reference diagnostic, zero points, not selected by scored campaigns. Run ten fresh trials per width/offset on disposable leased apps. STDB uses only the database boundary, with confirmed native WebSocket acknowledgements. PG/Mongo test application and database separately. Failed responses may follow a commit. Unknown checkout effects must be absent or complete; confirmed success and earlier confirmed state must survive. Missed fault windows, expired reservations and incomplete injection stay unmeasured. The baseline retains a synchronous HTTP or confirmed native WebSocket completion receipt before stored-state reconciliation. A separate post-recovery checkout tests app progress. State is compared after process recovery and reader connection; valid reconstruction from current durable data is allowed. This tests recovered application consistency, not untouched physical rows or raw database transaction atomicity. No power-loss or storage-device guarantee is claimed.", + "features": [ + { + "id": 9800, + "name": "database crash, 1 checkout calls, 0ms offset", + "actors": ["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"crash"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"baseline-before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"baseline-prepared"}, + {"do":"confirmCheckout","actor":"a","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]}}, + {"do":"dbExpectCheckout","before":"baseline-before","prepared":"baseline-prepared","quantity":1}, + {"do":"reload","actor":"a","settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"prepared"} + ], + "criteria": [ + { + "id": "9800a", + "category": "production", + "points": 0, + "desc": "after a verified process crash and reconnection, recovered state retains acknowledged checkout effects and contains no partial checkout", + "steps": [ + {"do":"crashCheckout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"actor":"a","before":"before","prepared":"prepared","quantity":1,"requests":1,"offsetMs":0,"target":"database"} + ] + }, + { + "id": "9800b", + "category": "production", + "points": 0, + "desc": "a fresh customer can complete checkout after recovery", + "steps": [ + {"do":"signUp","actor":"b","name":"recovered"}, + {"do":"dbRecordCheckout","account":"{user:recovered}","item":"Keyboard","as":"recovered-before"}, + {"do":"click", "awaitWrites": true,"actor":"b","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:recovered}","item":"Keyboard","as":"recovered-prepared"}, + {"do":"click","actor":"b","testid":"cart-toggle"}, + {"do":"click","actor":"b","testid":"checkout-submit"}, + {"do":"expectNumber","actor":"b","testid":"cart-count","equals":0,"within":10000}, + {"do":"click","actor":"b","testid":"overlay-close","ifAvailable":true}, + {"do":"dbExpectCheckout","before":"recovered-before","prepared":"recovered-prepared","quantity":1} + ] + } + ] + }, + { + "id": 9801, + "name": "database crash, 1 checkout calls, 5ms offset", + "actors": ["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"crash"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"baseline-before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"baseline-prepared"}, + {"do":"confirmCheckout","actor":"a","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]}}, + {"do":"dbExpectCheckout","before":"baseline-before","prepared":"baseline-prepared","quantity":1}, + {"do":"reload","actor":"a","settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"prepared"} + ], + "criteria": [ + { + "id": "9801a", + "category": "production", + "points": 0, + "desc": "after a verified process crash and reconnection, recovered state retains acknowledged checkout effects and contains no partial checkout", + "steps": [ + {"do":"crashCheckout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"actor":"a","before":"before","prepared":"prepared","quantity":1,"requests":1,"offsetMs":5,"target":"database"} + ] + }, + { + "id": "9801b", + "category": "production", + "points": 0, + "desc": "a fresh customer can complete checkout after recovery", + "steps": [ + {"do":"signUp","actor":"b","name":"recovered"}, + {"do":"dbRecordCheckout","account":"{user:recovered}","item":"Keyboard","as":"recovered-before"}, + {"do":"click", "awaitWrites": true,"actor":"b","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:recovered}","item":"Keyboard","as":"recovered-prepared"}, + {"do":"click","actor":"b","testid":"cart-toggle"}, + {"do":"click","actor":"b","testid":"checkout-submit"}, + {"do":"expectNumber","actor":"b","testid":"cart-count","equals":0,"within":10000}, + {"do":"click","actor":"b","testid":"overlay-close","ifAvailable":true}, + {"do":"dbExpectCheckout","before":"recovered-before","prepared":"recovered-prepared","quantity":1} + ] + } + ] + }, + { + "id": 9802, + "name": "database crash, 1 checkout calls, 20ms offset", + "actors": ["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"crash"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"baseline-before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"baseline-prepared"}, + {"do":"confirmCheckout","actor":"a","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]}}, + {"do":"dbExpectCheckout","before":"baseline-before","prepared":"baseline-prepared","quantity":1}, + {"do":"reload","actor":"a","settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"prepared"} + ], + "criteria": [ + { + "id": "9802a", + "category": "production", + "points": 0, + "desc": "after a verified process crash and reconnection, recovered state retains acknowledged checkout effects and contains no partial checkout", + "steps": [ + {"do":"crashCheckout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"actor":"a","before":"before","prepared":"prepared","quantity":1,"requests":1,"offsetMs":20,"target":"database"} + ] + }, + { + "id": "9802b", + "category": "production", + "points": 0, + "desc": "a fresh customer can complete checkout after recovery", + "steps": [ + {"do":"signUp","actor":"b","name":"recovered"}, + {"do":"dbRecordCheckout","account":"{user:recovered}","item":"Keyboard","as":"recovered-before"}, + {"do":"click", "awaitWrites": true,"actor":"b","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:recovered}","item":"Keyboard","as":"recovered-prepared"}, + {"do":"click","actor":"b","testid":"cart-toggle"}, + {"do":"click","actor":"b","testid":"checkout-submit"}, + {"do":"expectNumber","actor":"b","testid":"cart-count","equals":0,"within":10000}, + {"do":"click","actor":"b","testid":"overlay-close","ifAvailable":true}, + {"do":"dbExpectCheckout","before":"recovered-before","prepared":"recovered-prepared","quantity":1} + ] + } + ] + }, + { + "id": 9803, + "name": "database crash, 16 checkout calls, 0ms offset", + "actors": ["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"crash"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"baseline-before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"baseline-prepared"}, + {"do":"confirmCheckout","actor":"a","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]}}, + {"do":"dbExpectCheckout","before":"baseline-before","prepared":"baseline-prepared","quantity":1}, + {"do":"reload","actor":"a","settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"prepared"} + ], + "criteria": [ + { + "id": "9803a", + "category": "production", + "points": 0, + "desc": "after a verified process crash and reconnection, recovered state retains acknowledged checkout effects and contains no partial checkout", + "steps": [ + {"do":"crashCheckout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"actor":"a","before":"before","prepared":"prepared","quantity":1,"requests":16,"offsetMs":0,"target":"database"} + ] + }, + { + "id": "9803b", + "category": "production", + "points": 0, + "desc": "a fresh customer can complete checkout after recovery", + "steps": [ + {"do":"signUp","actor":"b","name":"recovered"}, + {"do":"dbRecordCheckout","account":"{user:recovered}","item":"Keyboard","as":"recovered-before"}, + {"do":"click", "awaitWrites": true,"actor":"b","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:recovered}","item":"Keyboard","as":"recovered-prepared"}, + {"do":"click","actor":"b","testid":"cart-toggle"}, + {"do":"click","actor":"b","testid":"checkout-submit"}, + {"do":"expectNumber","actor":"b","testid":"cart-count","equals":0,"within":10000}, + {"do":"click","actor":"b","testid":"overlay-close","ifAvailable":true}, + {"do":"dbExpectCheckout","before":"recovered-before","prepared":"recovered-prepared","quantity":1} + ] + } + ] + }, + { + "id": 9804, + "name": "database crash, 16 checkout calls, 5ms offset", + "actors": ["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"crash"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"baseline-before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"baseline-prepared"}, + {"do":"confirmCheckout","actor":"a","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]}}, + {"do":"dbExpectCheckout","before":"baseline-before","prepared":"baseline-prepared","quantity":1}, + {"do":"reload","actor":"a","settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"prepared"} + ], + "criteria": [ + { + "id": "9804a", + "category": "production", + "points": 0, + "desc": "after a verified process crash and reconnection, recovered state retains acknowledged checkout effects and contains no partial checkout", + "steps": [ + {"do":"crashCheckout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"actor":"a","before":"before","prepared":"prepared","quantity":1,"requests":16,"offsetMs":5,"target":"database"} + ] + }, + { + "id": "9804b", + "category": "production", + "points": 0, + "desc": "a fresh customer can complete checkout after recovery", + "steps": [ + {"do":"signUp","actor":"b","name":"recovered"}, + {"do":"dbRecordCheckout","account":"{user:recovered}","item":"Keyboard","as":"recovered-before"}, + {"do":"click", "awaitWrites": true,"actor":"b","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:recovered}","item":"Keyboard","as":"recovered-prepared"}, + {"do":"click","actor":"b","testid":"cart-toggle"}, + {"do":"click","actor":"b","testid":"checkout-submit"}, + {"do":"expectNumber","actor":"b","testid":"cart-count","equals":0,"within":10000}, + {"do":"click","actor":"b","testid":"overlay-close","ifAvailable":true}, + {"do":"dbExpectCheckout","before":"recovered-before","prepared":"recovered-prepared","quantity":1} + ] + } + ] + }, + { + "id": 9805, + "name": "database crash, 16 checkout calls, 20ms offset", + "actors": ["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"crash"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"baseline-before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"baseline-prepared"}, + {"do":"confirmCheckout","actor":"a","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]}}, + {"do":"dbExpectCheckout","before":"baseline-before","prepared":"baseline-prepared","quantity":1}, + {"do":"reload","actor":"a","settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"before"}, + {"do":"click", "awaitWrites": true,"actor":"a","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:crash}","item":"Keyboard","as":"prepared"} + ], + "criteria": [ + { + "id": "9805a", + "category": "production", + "points": 0, + "desc": "after a verified process crash and reconnection, recovered state retains acknowledged checkout effects and contains no partial checkout", + "steps": [ + {"do":"crashCheckout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"actor":"a","before":"before","prepared":"prepared","quantity":1,"requests":16,"offsetMs":20,"target":"database"} + ] + }, + { + "id": "9805b", + "category": "production", + "points": 0, + "desc": "a fresh customer can complete checkout after recovery", + "steps": [ + {"do":"signUp","actor":"b","name":"recovered"}, + {"do":"dbRecordCheckout","account":"{user:recovered}","item":"Keyboard","as":"recovered-before"}, + {"do":"click", "awaitWrites": true,"actor":"b","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"dbRecordCheckout","account":"{user:recovered}","item":"Keyboard","as":"recovered-prepared"}, + {"do":"click","actor":"b","testid":"cart-toggle"}, + {"do":"click","actor":"b","testid":"checkout-submit"}, + {"do":"expectNumber","actor":"b","testid":"cart-count","equals":0,"within":10000}, + {"do":"click","actor":"b","testid":"overlay-close","ifAvailable":true}, + {"do":"dbExpectCheckout","before":"recovered-before","prepared":"recovered-prepared","quantity":1} + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/diagnostic-checkout-order-crash.json b/tools/stack-bench/tracks/ecommerce/scenarios/diagnostic-checkout-order-crash.json new file mode 100644 index 00000000000..d55a7cbe6dd --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/diagnostic-checkout-order-crash.json @@ -0,0 +1,447 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Native order crash verdict qualification", + "note": "Draft zero-point controls. One fixed 16-call, zero-offset trial per applicable process boundary. PostgreSQL and MongoDB use both boundaries; SpacetimeDB uses its combined database boundary once. Each trial shares one retained crash observation across atomicity and durability assertions. The reader observes cart and order data, not unrequested warehouse or payment features. Valid reconstruction from durable data is allowed. Fault-window misses and incomplete observation remain inconclusive. Fresh-customer recovery is a separate zero-point prerequisite control, not a third proposed scored invariant. This single-item trial is not general multi-line, power-loss, or network-fault coverage.", + "features": [ + { + "id": 9900, + "name": "Native order checkout: application crash", + "actors": [ + "a", + "b" + ], + "setup": [ + { + "do": "signUp", + "actor": "a", + "name": "crash" + }, + { + "do": "dbRecordCheckout", + "account": "{user:crash}", + "item": "Keyboard", + "as": "baseline-before", + "storage": { + "kind": "order-data", + "cart": true, + "warehouses": false + } + }, + { + "do": "click", + "awaitWrites": true, + "actor": "a", + "testid": "add-to-cart", + "in": { + "testid": "item-card", + "contains": "Keyboard" + } + }, + { + "do": "dbRecordCheckout", + "account": "{user:crash}", + "item": "Keyboard", + "as": "baseline-prepared", + "storage": { + "kind": "order-data", + "cart": true, + "warehouses": false + } + }, + { + "do": "confirmCheckout", + "actor": "a", + "namedAction": { + "id": "checkout", + "path": "/api/checkout", + "reducer": "checkout", + "args": [] + } + }, + { + "do": "dbExpectCheckout", + "before": "baseline-before", + "prepared": "baseline-prepared", + "quantity": 1 + }, + { + "do": "reload", + "actor": "a", + "settleMs": 0 + }, + { + "do": "dbRecordCheckout", + "account": "{user:crash}", + "item": "Keyboard", + "as": "before", + "storage": { + "kind": "order-data", + "cart": true, + "warehouses": false + } + }, + { + "do": "click", + "awaitWrites": true, + "actor": "a", + "testid": "add-to-cart", + "in": { + "testid": "item-card", + "contains": "Keyboard" + } + }, + { + "do": "dbRecordCheckout", + "account": "{user:crash}", + "item": "Keyboard", + "as": "prepared", + "storage": { + "kind": "order-data", + "cart": true, + "warehouses": false + } + }, + { + "do": "crashCheckout", + "namedAction": { + "id": "checkout", + "path": "/api/checkout", + "reducer": "checkout", + "args": [] + }, + "actor": "a", + "before": "before", + "prepared": "prepared", + "quantity": 1, + "requests": 16, + "offsetMs": 0, + "target": "application", + "as": "crash-observation" + } + ], + "criteria": [ + { + "id": "9900a", + "category": "production", + "points": 0, + "desc": "interrupted checkout leaves either the prepared cart or one complete order with the cart cleared", + "steps": [ + { + "do": "expectCrashCheckout", + "from": "crash-observation", + "verdict": "atomicity" + } + ] + }, + { + "id": "9900b", + "category": "production", + "points": 0, + "desc": "acknowledged checkout is not rolled back and previously recorded orders remain unchanged", + "steps": [ + { + "do": "expectCrashCheckout", + "from": "crash-observation", + "verdict": "durability" + } + ] + }, + { + "id": "9900c", + "category": "production", + "points": 0, + "desc": "a fresh customer can complete checkout after recovery", + "steps": [ + { + "do": "signUp", + "actor": "b", + "name": "recovered" + }, + { + "do": "dbRecordCheckout", + "account": "{user:recovered}", + "item": "Keyboard", + "as": "recovered-before", + "storage": { + "kind": "order-data", + "cart": true, + "warehouses": false + } + }, + { + "do": "click", + "awaitWrites": true, + "actor": "b", + "testid": "add-to-cart", + "in": { + "testid": "item-card", + "contains": "Keyboard" + } + }, + { + "do": "dbRecordCheckout", + "account": "{user:recovered}", + "item": "Keyboard", + "as": "recovered-prepared", + "storage": { + "kind": "order-data", + "cart": true, + "warehouses": false + } + }, + { + "do": "click", + "actor": "b", + "testid": "cart-toggle" + }, + { + "do": "click", + "actor": "b", + "testid": "checkout-submit" + }, + { + "do": "expectNumber", + "actor": "b", + "testid": "cart-count", + "equals": 0, + "within": 10000 + }, + { + "do": "click", + "actor": "b", + "testid": "overlay-close", + "ifAvailable": true + }, + { + "do": "dbExpectCheckout", + "before": "recovered-before", + "prepared": "recovered-prepared", + "quantity": 1 + } + ] + } + ] + }, + { + "id": 9901, + "name": "Native order checkout: database crash", + "actors": [ + "a", + "b" + ], + "setup": [ + { + "do": "signUp", + "actor": "a", + "name": "crash" + }, + { + "do": "dbRecordCheckout", + "account": "{user:crash}", + "item": "Keyboard", + "as": "baseline-before", + "storage": { + "kind": "order-data", + "cart": true, + "warehouses": false + } + }, + { + "do": "click", + "awaitWrites": true, + "actor": "a", + "testid": "add-to-cart", + "in": { + "testid": "item-card", + "contains": "Keyboard" + } + }, + { + "do": "dbRecordCheckout", + "account": "{user:crash}", + "item": "Keyboard", + "as": "baseline-prepared", + "storage": { + "kind": "order-data", + "cart": true, + "warehouses": false + } + }, + { + "do": "confirmCheckout", + "actor": "a", + "namedAction": { + "id": "checkout", + "path": "/api/checkout", + "reducer": "checkout", + "args": [] + } + }, + { + "do": "dbExpectCheckout", + "before": "baseline-before", + "prepared": "baseline-prepared", + "quantity": 1 + }, + { + "do": "reload", + "actor": "a", + "settleMs": 0 + }, + { + "do": "dbRecordCheckout", + "account": "{user:crash}", + "item": "Keyboard", + "as": "before", + "storage": { + "kind": "order-data", + "cart": true, + "warehouses": false + } + }, + { + "do": "click", + "awaitWrites": true, + "actor": "a", + "testid": "add-to-cart", + "in": { + "testid": "item-card", + "contains": "Keyboard" + } + }, + { + "do": "dbRecordCheckout", + "account": "{user:crash}", + "item": "Keyboard", + "as": "prepared", + "storage": { + "kind": "order-data", + "cart": true, + "warehouses": false + } + }, + { + "do": "crashCheckout", + "namedAction": { + "id": "checkout", + "path": "/api/checkout", + "reducer": "checkout", + "args": [] + }, + "actor": "a", + "before": "before", + "prepared": "prepared", + "quantity": 1, + "requests": 16, + "offsetMs": 0, + "target": "database", + "as": "crash-observation" + } + ], + "criteria": [ + { + "id": "9901a", + "category": "production", + "points": 0, + "desc": "interrupted checkout leaves either the prepared cart or one complete order with the cart cleared", + "steps": [ + { + "do": "expectCrashCheckout", + "from": "crash-observation", + "verdict": "atomicity" + } + ] + }, + { + "id": "9901b", + "category": "production", + "points": 0, + "desc": "acknowledged checkout is not rolled back and previously recorded orders remain unchanged", + "steps": [ + { + "do": "expectCrashCheckout", + "from": "crash-observation", + "verdict": "durability" + } + ] + }, + { + "id": "9901c", + "category": "production", + "points": 0, + "desc": "a fresh customer can complete checkout after recovery", + "steps": [ + { + "do": "signUp", + "actor": "b", + "name": "recovered" + }, + { + "do": "dbRecordCheckout", + "account": "{user:recovered}", + "item": "Keyboard", + "as": "recovered-before", + "storage": { + "kind": "order-data", + "cart": true, + "warehouses": false + } + }, + { + "do": "click", + "awaitWrites": true, + "actor": "b", + "testid": "add-to-cart", + "in": { + "testid": "item-card", + "contains": "Keyboard" + } + }, + { + "do": "dbRecordCheckout", + "account": "{user:recovered}", + "item": "Keyboard", + "as": "recovered-prepared", + "storage": { + "kind": "order-data", + "cart": true, + "warehouses": false + } + }, + { + "do": "click", + "actor": "b", + "testid": "cart-toggle" + }, + { + "do": "click", + "actor": "b", + "testid": "checkout-submit" + }, + { + "do": "expectNumber", + "actor": "b", + "testid": "cart-count", + "equals": 0, + "within": 10000 + }, + { + "do": "click", + "actor": "b", + "testid": "overlay-close", + "ifAvailable": true + }, + { + "do": "dbExpectCheckout", + "before": "recovered-before", + "prepared": "recovered-prepared", + "quantity": 1 + } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/diagnostic-purchase-contention.json b/tools/stack-bench/tracks/ecommerce/scenarios/diagnostic-purchase-contention.json new file mode 100644 index 00000000000..fc120a98785 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/diagnostic-purchase-contention.json @@ -0,0 +1,199 @@ +{ + "schemaVersion": 1, "track": "ecommerce", "level": 2, + "name": "ample stock purchase contention", + "note": "Draft reference diagnostic; not selected by scored campaigns. Three fresh two-account cohorts at each width 1/4/16/64. Requests are distinct one-unit purchase intentions; these are not distinct client populations. Compare complete responses with per-buyer orders/payments and per-warehouse stock. No per-request durable identity, throughput or server-overlap claim. Correct and defective reference qualification required.", + "features": [ + {"id":9200,"name":"ample: 1 calls, cohort 1","actors":["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"ample-1-1-a"}, + {"do":"signUp","actor":"b","name":"ample-1-1-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:ample-1-1-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:ample-1-1-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9200a","category":"production","points":0,"desc":"all affordable purchases create matching orders, payments and stock effects", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":1,"settleMs":0}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":1}, + {"do":"expectCallOutcomes","accepted":1} + ]}] + }, + {"id":9201,"name":"ample: 1 calls, cohort 2","actors":["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"ample-1-2-a"}, + {"do":"signUp","actor":"b","name":"ample-1-2-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:ample-1-2-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:ample-1-2-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9201a","category":"production","points":0,"desc":"all affordable purchases create matching orders, payments and stock effects", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":1,"settleMs":0}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":1}, + {"do":"expectCallOutcomes","accepted":1} + ]}] + }, + {"id":9202,"name":"ample: 1 calls, cohort 3","actors":["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"ample-1-3-a"}, + {"do":"signUp","actor":"b","name":"ample-1-3-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:ample-1-3-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:ample-1-3-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9202a","category":"production","points":0,"desc":"all affordable purchases create matching orders, payments and stock effects", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":1,"settleMs":0}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":1}, + {"do":"expectCallOutcomes","accepted":1} + ]}] + }, + {"id":9203,"name":"ample: 4 calls, cohort 1","actors":["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"ample-4-1-a"}, + {"do":"signUp","actor":"b","name":"ample-4-1-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:ample-4-1-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:ample-4-1-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9203a","category":"production","points":0,"desc":"all affordable purchases create matching orders, payments and stock effects", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":4,"settleMs":0}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":4}, + {"do":"expectCallOutcomes","accepted":4} + ]}] + }, + {"id":9204,"name":"ample: 4 calls, cohort 2","actors":["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"ample-4-2-a"}, + {"do":"signUp","actor":"b","name":"ample-4-2-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:ample-4-2-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:ample-4-2-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9204a","category":"production","points":0,"desc":"all affordable purchases create matching orders, payments and stock effects", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":4,"settleMs":0}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":4}, + {"do":"expectCallOutcomes","accepted":4} + ]}] + }, + {"id":9205,"name":"ample: 4 calls, cohort 3","actors":["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"ample-4-3-a"}, + {"do":"signUp","actor":"b","name":"ample-4-3-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:ample-4-3-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:ample-4-3-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9205a","category":"production","points":0,"desc":"all affordable purchases create matching orders, payments and stock effects", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":4,"settleMs":0}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":4}, + {"do":"expectCallOutcomes","accepted":4} + ]}] + }, + {"id":9206,"name":"ample: 16 calls, cohort 1","actors":["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"ample-16-1-a"}, + {"do":"signUp","actor":"b","name":"ample-16-1-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:ample-16-1-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:ample-16-1-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9206a","category":"production","points":0,"desc":"all affordable purchases create matching orders, payments and stock effects", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":16,"settleMs":0}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":16}, + {"do":"expectCallOutcomes","accepted":16} + ]}] + }, + {"id":9207,"name":"ample: 16 calls, cohort 2","actors":["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"ample-16-2-a"}, + {"do":"signUp","actor":"b","name":"ample-16-2-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:ample-16-2-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:ample-16-2-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9207a","category":"production","points":0,"desc":"all affordable purchases create matching orders, payments and stock effects", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":16,"settleMs":0}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":16}, + {"do":"expectCallOutcomes","accepted":16} + ]}] + }, + {"id":9208,"name":"ample: 16 calls, cohort 3","actors":["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"ample-16-3-a"}, + {"do":"signUp","actor":"b","name":"ample-16-3-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:ample-16-3-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:ample-16-3-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9208a","category":"production","points":0,"desc":"all affordable purchases create matching orders, payments and stock effects", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":16,"settleMs":0}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":16}, + {"do":"expectCallOutcomes","accepted":16} + ]}] + }, + {"id":9209,"name":"ample: 64 calls, cohort 1","actors":["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"ample-64-1-a"}, + {"do":"signUp","actor":"b","name":"ample-64-1-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:ample-64-1-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:ample-64-1-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9209a","category":"production","points":0,"desc":"all affordable purchases create matching orders, payments and stock effects", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":64,"settleMs":0}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":64}, + {"do":"expectCallOutcomes","accepted":64} + ]}] + }, + {"id":9210,"name":"ample: 64 calls, cohort 2","actors":["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"ample-64-2-a"}, + {"do":"signUp","actor":"b","name":"ample-64-2-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:ample-64-2-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:ample-64-2-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9210a","category":"production","points":0,"desc":"all affordable purchases create matching orders, payments and stock effects", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":64,"settleMs":0}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":64}, + {"do":"expectCallOutcomes","accepted":64} + ]}] + }, + {"id":9211,"name":"ample: 64 calls, cohort 3","actors":["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"ample-64-3-a"}, + {"do":"signUp","actor":"b","name":"ample-64-3-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:ample-64-3-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:ample-64-3-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9211a","category":"production","points":0,"desc":"all affordable purchases create matching orders, payments and stock effects", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":64,"settleMs":0}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":64}, + {"do":"expectCallOutcomes","accepted":64} + ]}] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/diagnostic-restock-contention.json b/tools/stack-bench/tracks/ecommerce/scenarios/diagnostic-restock-contention.json new file mode 100644 index 00000000000..c4a47065dde --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/diagnostic-restock-contention.json @@ -0,0 +1,1447 @@ +{ + "schemaVersion": 1, "track": "ecommerce", "level": 2, + "name": "Restock versus purchase contention", + "note": "Draft reference diagnostic; not selected by scored campaigns. 80 fresh cohorts, 1000 calls per app. Half buy one unit and half restock one unit through the declared default action input. Starting stock 128 makes every purchase affordable regardless of ordering. Groups are fully prepared before release with 0/-20/-5/+5/+20ms restock offsets. Final snapshots verify net per-warehouse conservation and buyer order/payment counts; compensating errors can remain invisible. Histories record actual client dispatch; no server-overlap or throughput claim. Correct and defective reference qualification required.", + "features": [ + {"id":9300,"name":"restock: 2 calls, cohort 0-1, restock offset 0ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-2-0-1-a"}, + {"do":"signUp","actor":"b","name":"restock-2-0-1-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-2-0-1-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-2-0-1-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9300a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":1,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":1,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":1}, + {"do":"expectCallOutcomes","accepted":2} + ]}] + }, + {"id":9301,"name":"restock: 2 calls, cohort 0-2, restock offset 0ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-2-0-2-a"}, + {"do":"signUp","actor":"b","name":"restock-2-0-2-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-2-0-2-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-2-0-2-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9301a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":1,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":1,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":1}, + {"do":"expectCallOutcomes","accepted":2} + ]}] + }, + {"id":9302,"name":"restock: 2 calls, cohort 0-3, restock offset 0ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-2-0-3-a"}, + {"do":"signUp","actor":"b","name":"restock-2-0-3-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-2-0-3-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-2-0-3-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9302a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":1,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":1,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":1}, + {"do":"expectCallOutcomes","accepted":2} + ]}] + }, + {"id":9303,"name":"restock: 2 calls, cohort 0-4, restock offset 0ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-2-0-4-a"}, + {"do":"signUp","actor":"b","name":"restock-2-0-4-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-2-0-4-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-2-0-4-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9303a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":1,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":1,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":1}, + {"do":"expectCallOutcomes","accepted":2} + ]}] + }, + {"id":9304,"name":"restock: 2 calls, cohort -20-1, restock offset -20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-2--20-1-a"}, + {"do":"signUp","actor":"b","name":"restock-2--20-1-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-2--20-1-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-2--20-1-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9304a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":1,"settleMs":0,"delayMs":20,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":1,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":1}, + {"do":"expectCallOutcomes","accepted":2} + ]}] + }, + {"id":9305,"name":"restock: 2 calls, cohort -20-2, restock offset -20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-2--20-2-a"}, + {"do":"signUp","actor":"b","name":"restock-2--20-2-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-2--20-2-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-2--20-2-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9305a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":1,"settleMs":0,"delayMs":20,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":1,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":1}, + {"do":"expectCallOutcomes","accepted":2} + ]}] + }, + {"id":9306,"name":"restock: 2 calls, cohort -20-3, restock offset -20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-2--20-3-a"}, + {"do":"signUp","actor":"b","name":"restock-2--20-3-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-2--20-3-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-2--20-3-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9306a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":1,"settleMs":0,"delayMs":20,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":1,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":1}, + {"do":"expectCallOutcomes","accepted":2} + ]}] + }, + {"id":9307,"name":"restock: 2 calls, cohort -20-4, restock offset -20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-2--20-4-a"}, + {"do":"signUp","actor":"b","name":"restock-2--20-4-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-2--20-4-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-2--20-4-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9307a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":1,"settleMs":0,"delayMs":20,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":1,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":1}, + {"do":"expectCallOutcomes","accepted":2} + ]}] + }, + {"id":9308,"name":"restock: 2 calls, cohort -5-1, restock offset -5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-2--5-1-a"}, + {"do":"signUp","actor":"b","name":"restock-2--5-1-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-2--5-1-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-2--5-1-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9308a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":1,"settleMs":0,"delayMs":5,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":1,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":1}, + {"do":"expectCallOutcomes","accepted":2} + ]}] + }, + {"id":9309,"name":"restock: 2 calls, cohort -5-2, restock offset -5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-2--5-2-a"}, + {"do":"signUp","actor":"b","name":"restock-2--5-2-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-2--5-2-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-2--5-2-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9309a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":1,"settleMs":0,"delayMs":5,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":1,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":1}, + {"do":"expectCallOutcomes","accepted":2} + ]}] + }, + {"id":9310,"name":"restock: 2 calls, cohort -5-3, restock offset -5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-2--5-3-a"}, + {"do":"signUp","actor":"b","name":"restock-2--5-3-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-2--5-3-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-2--5-3-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9310a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":1,"settleMs":0,"delayMs":5,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":1,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":1}, + {"do":"expectCallOutcomes","accepted":2} + ]}] + }, + {"id":9311,"name":"restock: 2 calls, cohort -5-4, restock offset -5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-2--5-4-a"}, + {"do":"signUp","actor":"b","name":"restock-2--5-4-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-2--5-4-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-2--5-4-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9311a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":1,"settleMs":0,"delayMs":5,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":1,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":1}, + {"do":"expectCallOutcomes","accepted":2} + ]}] + }, + {"id":9312,"name":"restock: 2 calls, cohort 5-1, restock offset 5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-2-5-1-a"}, + {"do":"signUp","actor":"b","name":"restock-2-5-1-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-2-5-1-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-2-5-1-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9312a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":1,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":1,"delayMs":5}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":1}, + {"do":"expectCallOutcomes","accepted":2} + ]}] + }, + {"id":9313,"name":"restock: 2 calls, cohort 5-2, restock offset 5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-2-5-2-a"}, + {"do":"signUp","actor":"b","name":"restock-2-5-2-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-2-5-2-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-2-5-2-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9313a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":1,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":1,"delayMs":5}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":1}, + {"do":"expectCallOutcomes","accepted":2} + ]}] + }, + {"id":9314,"name":"restock: 2 calls, cohort 5-3, restock offset 5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-2-5-3-a"}, + {"do":"signUp","actor":"b","name":"restock-2-5-3-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-2-5-3-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-2-5-3-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9314a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":1,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":1,"delayMs":5}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":1}, + {"do":"expectCallOutcomes","accepted":2} + ]}] + }, + {"id":9315,"name":"restock: 2 calls, cohort 5-4, restock offset 5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-2-5-4-a"}, + {"do":"signUp","actor":"b","name":"restock-2-5-4-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-2-5-4-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-2-5-4-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9315a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":1,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":1,"delayMs":5}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":1}, + {"do":"expectCallOutcomes","accepted":2} + ]}] + }, + {"id":9316,"name":"restock: 2 calls, cohort 20-1, restock offset 20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-2-20-1-a"}, + {"do":"signUp","actor":"b","name":"restock-2-20-1-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-2-20-1-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-2-20-1-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9316a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":1,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":1,"delayMs":20}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":1}, + {"do":"expectCallOutcomes","accepted":2} + ]}] + }, + {"id":9317,"name":"restock: 2 calls, cohort 20-2, restock offset 20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-2-20-2-a"}, + {"do":"signUp","actor":"b","name":"restock-2-20-2-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-2-20-2-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-2-20-2-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9317a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":1,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":1,"delayMs":20}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":1}, + {"do":"expectCallOutcomes","accepted":2} + ]}] + }, + {"id":9318,"name":"restock: 2 calls, cohort 20-3, restock offset 20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-2-20-3-a"}, + {"do":"signUp","actor":"b","name":"restock-2-20-3-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-2-20-3-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-2-20-3-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9318a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":1,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":1,"delayMs":20}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":1}, + {"do":"expectCallOutcomes","accepted":2} + ]}] + }, + {"id":9319,"name":"restock: 2 calls, cohort 20-4, restock offset 20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-2-20-4-a"}, + {"do":"signUp","actor":"b","name":"restock-2-20-4-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-2-20-4-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-2-20-4-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9319a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":1,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":1,"delayMs":20}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":1}, + {"do":"expectCallOutcomes","accepted":2} + ]}] + }, + {"id":9320,"name":"restock: 8 calls, cohort 0-1, restock offset 0ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-8-0-1-a"}, + {"do":"signUp","actor":"b","name":"restock-8-0-1-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-8-0-1-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-8-0-1-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9320a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":4,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":4,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":4}, + {"do":"expectCallOutcomes","accepted":8} + ]}] + }, + {"id":9321,"name":"restock: 8 calls, cohort 0-2, restock offset 0ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-8-0-2-a"}, + {"do":"signUp","actor":"b","name":"restock-8-0-2-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-8-0-2-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-8-0-2-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9321a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":4,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":4,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":4}, + {"do":"expectCallOutcomes","accepted":8} + ]}] + }, + {"id":9322,"name":"restock: 8 calls, cohort 0-3, restock offset 0ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-8-0-3-a"}, + {"do":"signUp","actor":"b","name":"restock-8-0-3-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-8-0-3-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-8-0-3-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9322a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":4,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":4,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":4}, + {"do":"expectCallOutcomes","accepted":8} + ]}] + }, + {"id":9323,"name":"restock: 8 calls, cohort 0-4, restock offset 0ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-8-0-4-a"}, + {"do":"signUp","actor":"b","name":"restock-8-0-4-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-8-0-4-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-8-0-4-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9323a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":4,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":4,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":4}, + {"do":"expectCallOutcomes","accepted":8} + ]}] + }, + {"id":9324,"name":"restock: 8 calls, cohort -20-1, restock offset -20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-8--20-1-a"}, + {"do":"signUp","actor":"b","name":"restock-8--20-1-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-8--20-1-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-8--20-1-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9324a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":4,"settleMs":0,"delayMs":20,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":4,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":4}, + {"do":"expectCallOutcomes","accepted":8} + ]}] + }, + {"id":9325,"name":"restock: 8 calls, cohort -20-2, restock offset -20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-8--20-2-a"}, + {"do":"signUp","actor":"b","name":"restock-8--20-2-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-8--20-2-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-8--20-2-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9325a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":4,"settleMs":0,"delayMs":20,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":4,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":4}, + {"do":"expectCallOutcomes","accepted":8} + ]}] + }, + {"id":9326,"name":"restock: 8 calls, cohort -20-3, restock offset -20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-8--20-3-a"}, + {"do":"signUp","actor":"b","name":"restock-8--20-3-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-8--20-3-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-8--20-3-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9326a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":4,"settleMs":0,"delayMs":20,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":4,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":4}, + {"do":"expectCallOutcomes","accepted":8} + ]}] + }, + {"id":9327,"name":"restock: 8 calls, cohort -20-4, restock offset -20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-8--20-4-a"}, + {"do":"signUp","actor":"b","name":"restock-8--20-4-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-8--20-4-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-8--20-4-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9327a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":4,"settleMs":0,"delayMs":20,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":4,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":4}, + {"do":"expectCallOutcomes","accepted":8} + ]}] + }, + {"id":9328,"name":"restock: 8 calls, cohort -5-1, restock offset -5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-8--5-1-a"}, + {"do":"signUp","actor":"b","name":"restock-8--5-1-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-8--5-1-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-8--5-1-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9328a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":4,"settleMs":0,"delayMs":5,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":4,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":4}, + {"do":"expectCallOutcomes","accepted":8} + ]}] + }, + {"id":9329,"name":"restock: 8 calls, cohort -5-2, restock offset -5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-8--5-2-a"}, + {"do":"signUp","actor":"b","name":"restock-8--5-2-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-8--5-2-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-8--5-2-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9329a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":4,"settleMs":0,"delayMs":5,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":4,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":4}, + {"do":"expectCallOutcomes","accepted":8} + ]}] + }, + {"id":9330,"name":"restock: 8 calls, cohort -5-3, restock offset -5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-8--5-3-a"}, + {"do":"signUp","actor":"b","name":"restock-8--5-3-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-8--5-3-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-8--5-3-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9330a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":4,"settleMs":0,"delayMs":5,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":4,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":4}, + {"do":"expectCallOutcomes","accepted":8} + ]}] + }, + {"id":9331,"name":"restock: 8 calls, cohort -5-4, restock offset -5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-8--5-4-a"}, + {"do":"signUp","actor":"b","name":"restock-8--5-4-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-8--5-4-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-8--5-4-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9331a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":4,"settleMs":0,"delayMs":5,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":4,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":4}, + {"do":"expectCallOutcomes","accepted":8} + ]}] + }, + {"id":9332,"name":"restock: 8 calls, cohort 5-1, restock offset 5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-8-5-1-a"}, + {"do":"signUp","actor":"b","name":"restock-8-5-1-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-8-5-1-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-8-5-1-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9332a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":4,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":4,"delayMs":5}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":4}, + {"do":"expectCallOutcomes","accepted":8} + ]}] + }, + {"id":9333,"name":"restock: 8 calls, cohort 5-2, restock offset 5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-8-5-2-a"}, + {"do":"signUp","actor":"b","name":"restock-8-5-2-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-8-5-2-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-8-5-2-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9333a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":4,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":4,"delayMs":5}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":4}, + {"do":"expectCallOutcomes","accepted":8} + ]}] + }, + {"id":9334,"name":"restock: 8 calls, cohort 5-3, restock offset 5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-8-5-3-a"}, + {"do":"signUp","actor":"b","name":"restock-8-5-3-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-8-5-3-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-8-5-3-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9334a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":4,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":4,"delayMs":5}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":4}, + {"do":"expectCallOutcomes","accepted":8} + ]}] + }, + {"id":9335,"name":"restock: 8 calls, cohort 5-4, restock offset 5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-8-5-4-a"}, + {"do":"signUp","actor":"b","name":"restock-8-5-4-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-8-5-4-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-8-5-4-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9335a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":4,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":4,"delayMs":5}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":4}, + {"do":"expectCallOutcomes","accepted":8} + ]}] + }, + {"id":9336,"name":"restock: 8 calls, cohort 20-1, restock offset 20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-8-20-1-a"}, + {"do":"signUp","actor":"b","name":"restock-8-20-1-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-8-20-1-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-8-20-1-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9336a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":4,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":4,"delayMs":20}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":4}, + {"do":"expectCallOutcomes","accepted":8} + ]}] + }, + {"id":9337,"name":"restock: 8 calls, cohort 20-2, restock offset 20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-8-20-2-a"}, + {"do":"signUp","actor":"b","name":"restock-8-20-2-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-8-20-2-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-8-20-2-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9337a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":4,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":4,"delayMs":20}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":4}, + {"do":"expectCallOutcomes","accepted":8} + ]}] + }, + {"id":9338,"name":"restock: 8 calls, cohort 20-3, restock offset 20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-8-20-3-a"}, + {"do":"signUp","actor":"b","name":"restock-8-20-3-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-8-20-3-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-8-20-3-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9338a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":4,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":4,"delayMs":20}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":4}, + {"do":"expectCallOutcomes","accepted":8} + ]}] + }, + {"id":9339,"name":"restock: 8 calls, cohort 20-4, restock offset 20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-8-20-4-a"}, + {"do":"signUp","actor":"b","name":"restock-8-20-4-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-8-20-4-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-8-20-4-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9339a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":4,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":4,"delayMs":20}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":4}, + {"do":"expectCallOutcomes","accepted":8} + ]}] + }, + {"id":9340,"name":"restock: 16 calls, cohort 0-1, restock offset 0ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-16-0-1-a"}, + {"do":"signUp","actor":"b","name":"restock-16-0-1-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-16-0-1-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-16-0-1-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9340a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":8,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":8,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":8}, + {"do":"expectCallOutcomes","accepted":16} + ]}] + }, + {"id":9341,"name":"restock: 16 calls, cohort 0-2, restock offset 0ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-16-0-2-a"}, + {"do":"signUp","actor":"b","name":"restock-16-0-2-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-16-0-2-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-16-0-2-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9341a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":8,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":8,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":8}, + {"do":"expectCallOutcomes","accepted":16} + ]}] + }, + {"id":9342,"name":"restock: 16 calls, cohort 0-3, restock offset 0ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-16-0-3-a"}, + {"do":"signUp","actor":"b","name":"restock-16-0-3-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-16-0-3-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-16-0-3-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9342a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":8,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":8,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":8}, + {"do":"expectCallOutcomes","accepted":16} + ]}] + }, + {"id":9343,"name":"restock: 16 calls, cohort 0-4, restock offset 0ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-16-0-4-a"}, + {"do":"signUp","actor":"b","name":"restock-16-0-4-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-16-0-4-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-16-0-4-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9343a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":8,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":8,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":8}, + {"do":"expectCallOutcomes","accepted":16} + ]}] + }, + {"id":9344,"name":"restock: 16 calls, cohort -20-1, restock offset -20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-16--20-1-a"}, + {"do":"signUp","actor":"b","name":"restock-16--20-1-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-16--20-1-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-16--20-1-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9344a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":8,"settleMs":0,"delayMs":20,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":8,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":8}, + {"do":"expectCallOutcomes","accepted":16} + ]}] + }, + {"id":9345,"name":"restock: 16 calls, cohort -20-2, restock offset -20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-16--20-2-a"}, + {"do":"signUp","actor":"b","name":"restock-16--20-2-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-16--20-2-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-16--20-2-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9345a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":8,"settleMs":0,"delayMs":20,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":8,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":8}, + {"do":"expectCallOutcomes","accepted":16} + ]}] + }, + {"id":9346,"name":"restock: 16 calls, cohort -20-3, restock offset -20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-16--20-3-a"}, + {"do":"signUp","actor":"b","name":"restock-16--20-3-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-16--20-3-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-16--20-3-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9346a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":8,"settleMs":0,"delayMs":20,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":8,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":8}, + {"do":"expectCallOutcomes","accepted":16} + ]}] + }, + {"id":9347,"name":"restock: 16 calls, cohort -20-4, restock offset -20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-16--20-4-a"}, + {"do":"signUp","actor":"b","name":"restock-16--20-4-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-16--20-4-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-16--20-4-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9347a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":8,"settleMs":0,"delayMs":20,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":8,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":8}, + {"do":"expectCallOutcomes","accepted":16} + ]}] + }, + {"id":9348,"name":"restock: 16 calls, cohort -5-1, restock offset -5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-16--5-1-a"}, + {"do":"signUp","actor":"b","name":"restock-16--5-1-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-16--5-1-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-16--5-1-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9348a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":8,"settleMs":0,"delayMs":5,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":8,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":8}, + {"do":"expectCallOutcomes","accepted":16} + ]}] + }, + {"id":9349,"name":"restock: 16 calls, cohort -5-2, restock offset -5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-16--5-2-a"}, + {"do":"signUp","actor":"b","name":"restock-16--5-2-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-16--5-2-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-16--5-2-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9349a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":8,"settleMs":0,"delayMs":5,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":8,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":8}, + {"do":"expectCallOutcomes","accepted":16} + ]}] + }, + {"id":9350,"name":"restock: 16 calls, cohort -5-3, restock offset -5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-16--5-3-a"}, + {"do":"signUp","actor":"b","name":"restock-16--5-3-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-16--5-3-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-16--5-3-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9350a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":8,"settleMs":0,"delayMs":5,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":8,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":8}, + {"do":"expectCallOutcomes","accepted":16} + ]}] + }, + {"id":9351,"name":"restock: 16 calls, cohort -5-4, restock offset -5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-16--5-4-a"}, + {"do":"signUp","actor":"b","name":"restock-16--5-4-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-16--5-4-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-16--5-4-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9351a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":8,"settleMs":0,"delayMs":5,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":8,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":8}, + {"do":"expectCallOutcomes","accepted":16} + ]}] + }, + {"id":9352,"name":"restock: 16 calls, cohort 5-1, restock offset 5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-16-5-1-a"}, + {"do":"signUp","actor":"b","name":"restock-16-5-1-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-16-5-1-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-16-5-1-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9352a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":8,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":8,"delayMs":5}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":8}, + {"do":"expectCallOutcomes","accepted":16} + ]}] + }, + {"id":9353,"name":"restock: 16 calls, cohort 5-2, restock offset 5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-16-5-2-a"}, + {"do":"signUp","actor":"b","name":"restock-16-5-2-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-16-5-2-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-16-5-2-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9353a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":8,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":8,"delayMs":5}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":8}, + {"do":"expectCallOutcomes","accepted":16} + ]}] + }, + {"id":9354,"name":"restock: 16 calls, cohort 5-3, restock offset 5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-16-5-3-a"}, + {"do":"signUp","actor":"b","name":"restock-16-5-3-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-16-5-3-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-16-5-3-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9354a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":8,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":8,"delayMs":5}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":8}, + {"do":"expectCallOutcomes","accepted":16} + ]}] + }, + {"id":9355,"name":"restock: 16 calls, cohort 5-4, restock offset 5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-16-5-4-a"}, + {"do":"signUp","actor":"b","name":"restock-16-5-4-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-16-5-4-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-16-5-4-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9355a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":8,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":8,"delayMs":5}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":8}, + {"do":"expectCallOutcomes","accepted":16} + ]}] + }, + {"id":9356,"name":"restock: 16 calls, cohort 20-1, restock offset 20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-16-20-1-a"}, + {"do":"signUp","actor":"b","name":"restock-16-20-1-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-16-20-1-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-16-20-1-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9356a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":8,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":8,"delayMs":20}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":8}, + {"do":"expectCallOutcomes","accepted":16} + ]}] + }, + {"id":9357,"name":"restock: 16 calls, cohort 20-2, restock offset 20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-16-20-2-a"}, + {"do":"signUp","actor":"b","name":"restock-16-20-2-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-16-20-2-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-16-20-2-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9357a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":8,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":8,"delayMs":20}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":8}, + {"do":"expectCallOutcomes","accepted":16} + ]}] + }, + {"id":9358,"name":"restock: 16 calls, cohort 20-3, restock offset 20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-16-20-3-a"}, + {"do":"signUp","actor":"b","name":"restock-16-20-3-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-16-20-3-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-16-20-3-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9358a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":8,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":8,"delayMs":20}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":8}, + {"do":"expectCallOutcomes","accepted":16} + ]}] + }, + {"id":9359,"name":"restock: 16 calls, cohort 20-4, restock offset 20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-16-20-4-a"}, + {"do":"signUp","actor":"b","name":"restock-16-20-4-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-16-20-4-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-16-20-4-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9359a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":8,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":8,"delayMs":20}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":8}, + {"do":"expectCallOutcomes","accepted":16} + ]}] + }, + {"id":9360,"name":"restock: 24 calls, cohort 0-1, restock offset 0ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-24-0-1-a"}, + {"do":"signUp","actor":"b","name":"restock-24-0-1-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-24-0-1-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-24-0-1-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9360a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":12,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":12,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":12}, + {"do":"expectCallOutcomes","accepted":24} + ]}] + }, + {"id":9361,"name":"restock: 24 calls, cohort 0-2, restock offset 0ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-24-0-2-a"}, + {"do":"signUp","actor":"b","name":"restock-24-0-2-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-24-0-2-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-24-0-2-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9361a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":12,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":12,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":12}, + {"do":"expectCallOutcomes","accepted":24} + ]}] + }, + {"id":9362,"name":"restock: 24 calls, cohort 0-3, restock offset 0ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-24-0-3-a"}, + {"do":"signUp","actor":"b","name":"restock-24-0-3-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-24-0-3-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-24-0-3-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9362a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":12,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":12,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":12}, + {"do":"expectCallOutcomes","accepted":24} + ]}] + }, + {"id":9363,"name":"restock: 24 calls, cohort 0-4, restock offset 0ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-24-0-4-a"}, + {"do":"signUp","actor":"b","name":"restock-24-0-4-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-24-0-4-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-24-0-4-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9363a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":12,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":12,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":12}, + {"do":"expectCallOutcomes","accepted":24} + ]}] + }, + {"id":9364,"name":"restock: 24 calls, cohort -20-1, restock offset -20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-24--20-1-a"}, + {"do":"signUp","actor":"b","name":"restock-24--20-1-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-24--20-1-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-24--20-1-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9364a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":12,"settleMs":0,"delayMs":20,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":12,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":12}, + {"do":"expectCallOutcomes","accepted":24} + ]}] + }, + {"id":9365,"name":"restock: 24 calls, cohort -20-2, restock offset -20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-24--20-2-a"}, + {"do":"signUp","actor":"b","name":"restock-24--20-2-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-24--20-2-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-24--20-2-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9365a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":12,"settleMs":0,"delayMs":20,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":12,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":12}, + {"do":"expectCallOutcomes","accepted":24} + ]}] + }, + {"id":9366,"name":"restock: 24 calls, cohort -20-3, restock offset -20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-24--20-3-a"}, + {"do":"signUp","actor":"b","name":"restock-24--20-3-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-24--20-3-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-24--20-3-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9366a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":12,"settleMs":0,"delayMs":20,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":12,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":12}, + {"do":"expectCallOutcomes","accepted":24} + ]}] + }, + {"id":9367,"name":"restock: 24 calls, cohort -20-4, restock offset -20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-24--20-4-a"}, + {"do":"signUp","actor":"b","name":"restock-24--20-4-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-24--20-4-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-24--20-4-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9367a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":12,"settleMs":0,"delayMs":20,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":12,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":12}, + {"do":"expectCallOutcomes","accepted":24} + ]}] + }, + {"id":9368,"name":"restock: 24 calls, cohort -5-1, restock offset -5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-24--5-1-a"}, + {"do":"signUp","actor":"b","name":"restock-24--5-1-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-24--5-1-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-24--5-1-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9368a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":12,"settleMs":0,"delayMs":5,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":12,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":12}, + {"do":"expectCallOutcomes","accepted":24} + ]}] + }, + {"id":9369,"name":"restock: 24 calls, cohort -5-2, restock offset -5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-24--5-2-a"}, + {"do":"signUp","actor":"b","name":"restock-24--5-2-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-24--5-2-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-24--5-2-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9369a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":12,"settleMs":0,"delayMs":5,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":12,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":12}, + {"do":"expectCallOutcomes","accepted":24} + ]}] + }, + {"id":9370,"name":"restock: 24 calls, cohort -5-3, restock offset -5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-24--5-3-a"}, + {"do":"signUp","actor":"b","name":"restock-24--5-3-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-24--5-3-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-24--5-3-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9370a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":12,"settleMs":0,"delayMs":5,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":12,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":12}, + {"do":"expectCallOutcomes","accepted":24} + ]}] + }, + {"id":9371,"name":"restock: 24 calls, cohort -5-4, restock offset -5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-24--5-4-a"}, + {"do":"signUp","actor":"b","name":"restock-24--5-4-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-24--5-4-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-24--5-4-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9371a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":12,"settleMs":0,"delayMs":5,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":12,"delayMs":0}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":12}, + {"do":"expectCallOutcomes","accepted":24} + ]}] + }, + {"id":9372,"name":"restock: 24 calls, cohort 5-1, restock offset 5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-24-5-1-a"}, + {"do":"signUp","actor":"b","name":"restock-24-5-1-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-24-5-1-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-24-5-1-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9372a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":12,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":12,"delayMs":5}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":12}, + {"do":"expectCallOutcomes","accepted":24} + ]}] + }, + {"id":9373,"name":"restock: 24 calls, cohort 5-2, restock offset 5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-24-5-2-a"}, + {"do":"signUp","actor":"b","name":"restock-24-5-2-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-24-5-2-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-24-5-2-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9373a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":12,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":12,"delayMs":5}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":12}, + {"do":"expectCallOutcomes","accepted":24} + ]}] + }, + {"id":9374,"name":"restock: 24 calls, cohort 5-3, restock offset 5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-24-5-3-a"}, + {"do":"signUp","actor":"b","name":"restock-24-5-3-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-24-5-3-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-24-5-3-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9374a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":12,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":12,"delayMs":5}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":12}, + {"do":"expectCallOutcomes","accepted":24} + ]}] + }, + {"id":9375,"name":"restock: 24 calls, cohort 5-4, restock offset 5ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-24-5-4-a"}, + {"do":"signUp","actor":"b","name":"restock-24-5-4-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-24-5-4-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-24-5-4-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9375a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":12,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":12,"delayMs":5}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":12}, + {"do":"expectCallOutcomes","accepted":24} + ]}] + }, + {"id":9376,"name":"restock: 24 calls, cohort 20-1, restock offset 20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-24-20-1-a"}, + {"do":"signUp","actor":"b","name":"restock-24-20-1-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-24-20-1-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-24-20-1-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9376a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":12,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":12,"delayMs":20}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":12}, + {"do":"expectCallOutcomes","accepted":24} + ]}] + }, + {"id":9377,"name":"restock: 24 calls, cohort 20-2, restock offset 20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-24-20-2-a"}, + {"do":"signUp","actor":"b","name":"restock-24-20-2-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-24-20-2-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-24-20-2-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9377a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":12,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":12,"delayMs":20}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":12}, + {"do":"expectCallOutcomes","accepted":24} + ]}] + }, + {"id":9378,"name":"restock: 24 calls, cohort 20-3, restock offset 20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-24-20-3-a"}, + {"do":"signUp","actor":"b","name":"restock-24-20-3-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-24-20-3-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-24-20-3-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9378a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":12,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":12,"delayMs":20}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":12}, + {"do":"expectCallOutcomes","accepted":24} + ]}] + }, + {"id":9379,"name":"restock: 24 calls, cohort 20-4, restock offset 20ms","actors":["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"restock-24-20-4-a"}, + {"do":"signUp","actor":"b","name":"restock-24-20-4-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":128,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"dbRecordCheckout","account":"{user:restock-24-20-4-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:restock-24-20-4-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9379a","category":"production","points":0,"desc":"buyer orders and payments match responses; warehouse stock has the expected net change", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":12,"settleMs":0,"delayMs":0,"alongside":[{"action":"restock","actors":["admin"],"namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Keyboard","attribute":"data-restock-input"},"requestTimeoutMs":30000,"requests":12,"delayMs":20}]}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":12}, + {"do":"expectCallOutcomes","accepted":24} + ]}] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/diagnostic-scarce-stock.json b/tools/stack-bench/tracks/ecommerce/scenarios/diagnostic-scarce-stock.json new file mode 100644 index 00000000000..4eec329140f --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/diagnostic-scarce-stock.json @@ -0,0 +1,199 @@ +{ + "schemaVersion": 1, "track": "ecommerce", "level": 2, + "name": "scarce stock purchase contention", + "note": "Draft reference diagnostic; not selected by scored campaigns. Three fresh two-account cohorts at each width 1/4/16/64. Requests are distinct one-unit purchase intentions; these are not distinct client populations. Compare complete responses with per-buyer orders/payments and per-warehouse stock. No per-request durable identity, throughput or server-overlap claim. Correct and defective reference qualification required.", + "features": [ + {"id":9200,"name":"scarce: 1 calls, cohort 1","actors":["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"scarce-1-1-a"}, + {"do":"signUp","actor":"b","name":"scarce-1-1-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":3,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:scarce-1-1-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:scarce-1-1-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9200a","category":"production","points":0,"desc":"only available units can sell; orders, payments and stock agree", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":1,"settleMs":0}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":1}, + {"do":"expectCallOutcomes","accepted":1} + ]}] + }, + {"id":9201,"name":"scarce: 1 calls, cohort 2","actors":["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"scarce-1-2-a"}, + {"do":"signUp","actor":"b","name":"scarce-1-2-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":3,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:scarce-1-2-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:scarce-1-2-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9201a","category":"production","points":0,"desc":"only available units can sell; orders, payments and stock agree", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":1,"settleMs":0}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":1}, + {"do":"expectCallOutcomes","accepted":1} + ]}] + }, + {"id":9202,"name":"scarce: 1 calls, cohort 3","actors":["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"scarce-1-3-a"}, + {"do":"signUp","actor":"b","name":"scarce-1-3-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":3,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:scarce-1-3-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:scarce-1-3-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9202a","category":"production","points":0,"desc":"only available units can sell; orders, payments and stock agree", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":1,"settleMs":0}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":1}, + {"do":"expectCallOutcomes","accepted":1} + ]}] + }, + {"id":9203,"name":"scarce: 4 calls, cohort 1","actors":["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"scarce-4-1-a"}, + {"do":"signUp","actor":"b","name":"scarce-4-1-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":3,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:scarce-4-1-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:scarce-4-1-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9203a","category":"production","points":0,"desc":"only available units can sell; orders, payments and stock agree", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":4,"settleMs":0}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":3}, + {"do":"expectCallOutcomes","accepted":3} + ]}] + }, + {"id":9204,"name":"scarce: 4 calls, cohort 2","actors":["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"scarce-4-2-a"}, + {"do":"signUp","actor":"b","name":"scarce-4-2-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":3,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:scarce-4-2-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:scarce-4-2-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9204a","category":"production","points":0,"desc":"only available units can sell; orders, payments and stock agree", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":4,"settleMs":0}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":3}, + {"do":"expectCallOutcomes","accepted":3} + ]}] + }, + {"id":9205,"name":"scarce: 4 calls, cohort 3","actors":["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"scarce-4-3-a"}, + {"do":"signUp","actor":"b","name":"scarce-4-3-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":3,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:scarce-4-3-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:scarce-4-3-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9205a","category":"production","points":0,"desc":"only available units can sell; orders, payments and stock agree", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":4,"settleMs":0}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":3}, + {"do":"expectCallOutcomes","accepted":3} + ]}] + }, + {"id":9206,"name":"scarce: 16 calls, cohort 1","actors":["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"scarce-16-1-a"}, + {"do":"signUp","actor":"b","name":"scarce-16-1-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":3,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:scarce-16-1-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:scarce-16-1-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9206a","category":"production","points":0,"desc":"only available units can sell; orders, payments and stock agree", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":16,"settleMs":0}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":3}, + {"do":"expectCallOutcomes","accepted":3} + ]}] + }, + {"id":9207,"name":"scarce: 16 calls, cohort 2","actors":["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"scarce-16-2-a"}, + {"do":"signUp","actor":"b","name":"scarce-16-2-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":3,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:scarce-16-2-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:scarce-16-2-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9207a","category":"production","points":0,"desc":"only available units can sell; orders, payments and stock agree", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":16,"settleMs":0}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":3}, + {"do":"expectCallOutcomes","accepted":3} + ]}] + }, + {"id":9208,"name":"scarce: 16 calls, cohort 3","actors":["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"scarce-16-3-a"}, + {"do":"signUp","actor":"b","name":"scarce-16-3-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":3,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:scarce-16-3-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:scarce-16-3-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9208a","category":"production","points":0,"desc":"only available units can sell; orders, payments and stock agree", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":16,"settleMs":0}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":3}, + {"do":"expectCallOutcomes","accepted":3} + ]}] + }, + {"id":9209,"name":"scarce: 64 calls, cohort 1","actors":["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"scarce-64-1-a"}, + {"do":"signUp","actor":"b","name":"scarce-64-1-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":3,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:scarce-64-1-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:scarce-64-1-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9209a","category":"production","points":0,"desc":"only available units can sell; orders, payments and stock agree", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":64,"settleMs":0}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":3}, + {"do":"expectCallOutcomes","accepted":3} + ]}] + }, + {"id":9210,"name":"scarce: 64 calls, cohort 2","actors":["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"scarce-64-2-a"}, + {"do":"signUp","actor":"b","name":"scarce-64-2-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":3,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:scarce-64-2-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:scarce-64-2-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9210a","category":"production","points":0,"desc":"only available units can sell; orders, payments and stock agree", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":64,"settleMs":0}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":3}, + {"do":"expectCallOutcomes","accepted":3} + ]}] + }, + {"id":9211,"name":"scarce: 64 calls, cohort 3","actors":["a","b"], + "setup": [ + {"do":"signUp","actor":"a","name":"scarce-64-3-a"}, + {"do":"signUp","actor":"b","name":"scarce-64-3-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":3,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbRecordCheckout","account":"{user:scarce-64-3-a}","item":"Keyboard","as":"before-a"}, + {"do":"dbRecordCheckout","account":"{user:scarce-64-3-b}","item":"Keyboard","as":"before-b"} + ], + "criteria": [{"id":"9211a","category":"production","points":0,"desc":"only available units can sell; orders, payments and stock agree", + "steps": [ + {"do":"callConcurrently","action":"buy","actors":["a","b"],"namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"requestTimeoutMs":30000,"requests":64,"settleMs":0}, + {"do":"dbExpectPurchases","before":{"a":"before-a","b":"before-b"},"purchases":3}, + {"do":"expectCallOutcomes","accepted":3} + ]}] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/diagnostic-shipping-cancellation.json b/tools/stack-bench/tracks/ecommerce/scenarios/diagnostic-shipping-cancellation.json new file mode 100644 index 00000000000..16d50352c85 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/diagnostic-shipping-cancellation.json @@ -0,0 +1,118 @@ +{ + "schemaVersion": 1, "track": "ecommerce", "level": 3, + "name": "Shipping versus cancellation", + "note": "Draft, unscored. Two serial controls and three independent one-pair races. Each trial starts with one independently verified order. Admit shipped with no refund/restoration, or cancelled with exactly one refund/restoration. Both refusals with pending state fail. Request dispatch overlap is not proof of overlapping server execution; no launch-order requirement. Retain every outcome and native stored-state snapshot. No cart, payment-provider or delivery timing claim.", + "features": [ + { + "id": 9250, "name": "ship-first", "actors": ["customer", "staff"], + "setup": [ + {"do":"signUp","actor":"customer","name":"ship-first"}, + {"do":"signIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true}, + {"do":"dbRecordCheckout","account":"{user:ship-first}","item":"Coffee Grinder","as":"before","storage":{"kind":"order-data","cart":false,"warehouses":true}}, + {"do":"callConcurrently","actors":["customer"],"action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Coffee Grinder","attribute":"data-buy-input"},"requests":1,"settleMs":0}, + {"do":"expectCallOutcomes","accepted":1}, + {"do":"dbExpectPurchases","before":{"customer":"before"},"purchases":1}, + {"do":"dbRecordCheckout","account":"{user:ship-first}","item":"Coffee Grinder","as":"pending","storage":{"kind":"order-data","cart":false,"warehouses":true}}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"within":1000,"unlessVisible":"order-item"}, + {"do":"click","actor":"customer","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"customer","testid":"order-status","in":{"testid":"order-item","contains":"Coffee Grinder"},"value":"pending","ignoreCase":true} + ], + "criteria": [{"id": "9250a", "category": "production", "points": 0, + "desc": "shipping and cancellation produce one legal stored outcome without double effects", "steps": [ + {"do":"callAction","actor":"staff","from":"customer","action":"ship","namedAction":{"id":"ship","path":"/api/fulfilment/ship","reducer":"ship_order","args":[0],"params":[{"name":"orderId","in":"body","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Coffee Grinder","attribute":"data-ship-input"},"settleMs":0}, + {"do":"expectActionOutcome","actor":"staff","outcome":"accepted"}, + {"do":"dbExpectCancellation","before":"pending","shipping":"wins"}, + {"do":"callAction","actor":"customer","from":"customer","action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Coffee Grinder","attribute":"data-cancel-input"},"settleMs":0}, + {"do":"expectActionOutcome","actor":"customer","outcome":"completed"}, + {"do":"dbExpectCancellation","before":"pending","shipping":"wins"} + ]}] + }, + { + "id": 9251, "name": "cancel-first", "actors": ["customer", "staff"], + "setup": [ + {"do":"signUp","actor":"customer","name":"cancel-first"}, + {"do":"signIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true}, + {"do":"dbRecordCheckout","account":"{user:cancel-first}","item":"Coffee Grinder","as":"before","storage":{"kind":"order-data","cart":false,"warehouses":true}}, + {"do":"callConcurrently","actors":["customer"],"action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Coffee Grinder","attribute":"data-buy-input"},"requests":1,"settleMs":0}, + {"do":"expectCallOutcomes","accepted":1}, + {"do":"dbExpectPurchases","before":{"customer":"before"},"purchases":1}, + {"do":"dbRecordCheckout","account":"{user:cancel-first}","item":"Coffee Grinder","as":"pending","storage":{"kind":"order-data","cart":false,"warehouses":true}}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"within":1000,"unlessVisible":"order-item"}, + {"do":"click","actor":"customer","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"customer","testid":"order-status","in":{"testid":"order-item","contains":"Coffee Grinder"},"value":"pending","ignoreCase":true} + ], + "criteria": [{"id": "9251a", "category": "production", "points": 0, + "desc": "shipping and cancellation produce one legal stored outcome without double effects", "steps": [ + {"do":"callAction","actor":"customer","from":"customer","action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Coffee Grinder","attribute":"data-cancel-input"},"settleMs":0}, + {"do":"expectActionOutcome","actor":"customer","outcome":"accepted"}, + {"do":"dbExpectCancellation","before":"pending"}, + {"do":"callAction","actor":"staff","from":"customer","action":"ship","namedAction":{"id":"ship","path":"/api/fulfilment/ship","reducer":"ship_order","args":[0],"params":[{"name":"orderId","in":"body","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Coffee Grinder","attribute":"data-ship-input"},"settleMs":0}, + {"do":"expectActionOutcome","actor":"staff","outcome":"completed"}, + {"do":"dbExpectCancellation","before":"pending"} + ]}] + }, + { + "id": 9252, "name": "race-1", "actors": ["customer", "staff"], + "setup": [ + {"do":"signUp","actor":"customer","name":"race-1"}, + {"do":"signIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true}, + {"do":"dbRecordCheckout","account":"{user:race-1}","item":"Coffee Grinder","as":"before","storage":{"kind":"order-data","cart":false,"warehouses":true}}, + {"do":"callConcurrently","actors":["customer"],"action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Coffee Grinder","attribute":"data-buy-input"},"requests":1,"settleMs":0}, + {"do":"expectCallOutcomes","accepted":1}, + {"do":"dbExpectPurchases","before":{"customer":"before"},"purchases":1}, + {"do":"dbRecordCheckout","account":"{user:race-1}","item":"Coffee Grinder","as":"pending","storage":{"kind":"order-data","cart":false,"warehouses":true}}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"within":1000,"unlessVisible":"order-item"}, + {"do":"click","actor":"customer","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"customer","testid":"order-status","in":{"testid":"order-item","contains":"Coffee Grinder"},"value":"pending","ignoreCase":true} + ], + "criteria": [{"id": "9252a", "category": "production", "points": 0, + "desc": "shipping and cancellation produce one legal stored outcome without double effects", "steps": [ + {"do":"callConcurrently","actors":["customer"],"from":"customer","action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Coffee Grinder","attribute":"data-cancel-input"},"requests":1,"settleMs":0,"alongside":[{"actors":["staff"],"from":"customer","action":"ship","namedAction":{"id":"ship","path":"/api/fulfilment/ship","reducer":"ship_order","args":[0],"params":[{"name":"orderId","in":"body","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Coffee Grinder","attribute":"data-ship-input"},"requests":1}]}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCancellation","before":"pending","shipping":"competes"} + ]}] + }, + { + "id": 9253, "name": "race-2", "actors": ["customer", "staff"], + "setup": [ + {"do":"signUp","actor":"customer","name":"race-2"}, + {"do":"signIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true}, + {"do":"dbRecordCheckout","account":"{user:race-2}","item":"Coffee Grinder","as":"before","storage":{"kind":"order-data","cart":false,"warehouses":true}}, + {"do":"callConcurrently","actors":["customer"],"action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Coffee Grinder","attribute":"data-buy-input"},"requests":1,"settleMs":0}, + {"do":"expectCallOutcomes","accepted":1}, + {"do":"dbExpectPurchases","before":{"customer":"before"},"purchases":1}, + {"do":"dbRecordCheckout","account":"{user:race-2}","item":"Coffee Grinder","as":"pending","storage":{"kind":"order-data","cart":false,"warehouses":true}}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"within":1000,"unlessVisible":"order-item"}, + {"do":"click","actor":"customer","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"customer","testid":"order-status","in":{"testid":"order-item","contains":"Coffee Grinder"},"value":"pending","ignoreCase":true} + ], + "criteria": [{"id": "9253a", "category": "production", "points": 0, + "desc": "shipping and cancellation produce one legal stored outcome without double effects", "steps": [ + {"do":"callConcurrently","actors":["customer"],"from":"customer","action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Coffee Grinder","attribute":"data-cancel-input"},"requests":1,"settleMs":0,"alongside":[{"actors":["staff"],"from":"customer","action":"ship","namedAction":{"id":"ship","path":"/api/fulfilment/ship","reducer":"ship_order","args":[0],"params":[{"name":"orderId","in":"body","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Coffee Grinder","attribute":"data-ship-input"},"requests":1}]}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCancellation","before":"pending","shipping":"competes"} + ]}] + }, + { + "id": 9254, "name": "race-3", "actors": ["customer", "staff"], + "setup": [ + {"do":"signUp","actor":"customer","name":"race-3"}, + {"do":"signIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true}, + {"do":"dbRecordCheckout","account":"{user:race-3}","item":"Coffee Grinder","as":"before","storage":{"kind":"order-data","cart":false,"warehouses":true}}, + {"do":"callConcurrently","actors":["customer"],"action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Coffee Grinder","attribute":"data-buy-input"},"requests":1,"settleMs":0}, + {"do":"expectCallOutcomes","accepted":1}, + {"do":"dbExpectPurchases","before":{"customer":"before"},"purchases":1}, + {"do":"dbRecordCheckout","account":"{user:race-3}","item":"Coffee Grinder","as":"pending","storage":{"kind":"order-data","cart":false,"warehouses":true}}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"within":1000,"unlessVisible":"order-item"}, + {"do":"click","actor":"customer","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"customer","testid":"order-status","in":{"testid":"order-item","contains":"Coffee Grinder"},"value":"pending","ignoreCase":true} + ], + "criteria": [{"id": "9254a", "category": "production", "points": 0, + "desc": "shipping and cancellation produce one legal stored outcome without double effects", "steps": [ + {"do":"callConcurrently","actors":["customer"],"from":"customer","action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Coffee Grinder","attribute":"data-cancel-input"},"requests":1,"settleMs":0,"alongside":[{"actors":["staff"],"from":"customer","action":"ship","namedAction":{"id":"ship","path":"/api/fulfilment/ship","reducer":"ship_order","args":[0],"params":[{"name":"orderId","in":"body","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Coffee Grinder","attribute":"data-ship-input"},"requests":1}]}, + {"do":"expectCallOutcomes"}, + {"do":"dbExpectCancellation","before":"pending","shipping":"competes"} + ]}] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/mixed-operation-history.json b/tools/stack-bench/tracks/ecommerce/scenarios/mixed-operation-history.json new file mode 100644 index 00000000000..3520c6ce5b8 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/mixed-operation-history.json @@ -0,0 +1,229 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 3, + "name": "Mixed operation history", + "note": "One fixed 30-operation sequence. Reconcile both customers after every accepted operation. This measures sequential composition, not throughput or general linearizability. Additional qualification schedules do not multiply scoring points.", + "features": [{ + "id": 202, "name": "Mixed operations preserve stock and orders", "actors": ["a","b","admin"], + "setup": [ + {"do":"signUp","actor":"a","name":"history-17-a"}, + {"do":"signUp","actor":"b","name":"history-17-b"}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + {"do":"click","actor":"admin","testid":"admin-link", "unlessVisible": "admin-panel"}, + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"East","quantity":200,"settleMs":0}, + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"West","quantity":200,"settleMs":0}, + {"do":"dbSetStock","item":"Coffee Grinder","warehouse":"East","quantity":200,"settleMs":0}, + {"do":"dbSetStock","item":"Coffee Grinder","warehouse":"West","quantity":200,"settleMs":0} + ], + "criteria": [{ + "id": "202f", "category": "production", "points": 1, + "desc": "each completed operation preserves order ownership, booked totals, carts and stock, with cancellation restored once", + "statedBy": "Completed operations preserve prior orders, account ownership and exact stock; cancelling the same order restores stock once.", + "steps": [ + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"a","testid":"catalog-link","ifAvailable":true,"within":1000}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-0-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-0-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"callConcurrently","actors":["a"],"action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Desk Lamp","attribute":"data-buy-input"},"requests":1,"requestTimeoutMs":10000,"settleMs":0}, + {"do":"dbExpectOperation","before":"history-0-0","otherBefore":"history-0-1","actor":"a","operation":"buy"}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-1-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-1-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"callConcurrently","actors":["a"],"action":"cart-add","namedAction":{"id":"cart-add","path":"/api/cart","reducer":"add_to_cart","args":[0],"params":[{"name":"itemId","in":"body","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Coffee Grinder","attribute":"data-buy-input"},"requests":1,"requestTimeoutMs":10000,"settleMs":0}, + {"do":"dbExpectOperation","before":"history-1-0","otherBefore":"history-1-1","actor":"a","operation":"cart-add"}, + {"do":"fill","actor":"admin","testid":"transfer-from","text":"East","in":{"testid":"admin-item-row","contains":"Coffee Grinder"}}, + {"do":"fill","actor":"admin","testid":"transfer-to","text":"West","in":{"testid":"admin-item-row","contains":"Coffee Grinder"}}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-2-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-2-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"callConcurrently","actors":["admin"],"action":"transfer","namedAction":{"id":"transfer","path":"/api/admin/transfer","reducer":"admin_transfer_stock","args":[0,0,0,3],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"fromWarehouseId","in":"body","wireType":"u64"},{"name":"toWarehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-item-row","contains":"Coffee Grinder","attribute":"data-transfer-input","fields":["itemId","fromWarehouseId","toWarehouseId"]},"requests":1,"requestTimeoutMs":10000,"settleMs":0}, + {"do":"dbExpectOperation","before":"history-2-0","otherBefore":"history-2-1","actor":"admin","operation":"transfer"}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-3-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-3-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"callConcurrently","actors":["admin"],"action":"restock","namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Desk Lamp","attribute":"data-restock-input"},"requests":1,"requestTimeoutMs":10000,"settleMs":0}, + {"do":"dbExpectOperation","before":"history-3-0","otherBefore":"history-3-1","actor":"admin","operation":"restock"}, + {"do":"reload","actor":"a","settleMs":0}, + {"do":"ensureSignedIn","actor":"a","name":"history-17-a","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"unlessVisible":"cart-total","within":1000}, + {"do": "click", "actor": "a", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["cart-toggle", "cart-total"]}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"a","testid":"cart-quantity","in":{"testid":"cart-item","contains":"Coffee Grinder"},"equals":1,"within":10000}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-4-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-4-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"callConcurrently","actors":["a"],"action":"cart-update","namedAction":{"id":"cart-update","path":"/api/cart/:itemId","method":"PATCH","reducer":"update_cart_quantity","args":[0,3],"params":[{"name":"itemId","in":"path","placeholder":":itemId","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"cart-item","contains":"Coffee Grinder","attribute":"data-cart-input","fields":["itemId"]},"requests":1,"requestTimeoutMs":10000,"settleMs":0}, + {"do":"dbExpectOperation","before":"history-4-0","otherBefore":"history-4-1","actor":"a","operation":"cart-update"}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-5-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-5-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"callConcurrently","actors":["a"],"action":"checkout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"requests":1,"requestTimeoutMs":10000,"settleMs":0}, + {"do":"dbExpectOperation","before":"history-5-0","otherBefore":"history-5-1","actor":"a","operation":"checkout"}, + {"do":"reload","actor":"a","settleMs":0}, + {"do":"ensureSignedIn","actor":"a","name":"history-17-a","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"within":1000}, + { "do": "click", "actor": "a", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "a", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + {"do":"click","actor":"a","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"a","testid":"order-item","contains":"pending","within":10000}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-6-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-6-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"callConcurrently","actors":["a"],"action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"pending","attribute":"data-cancel-input"},"requests":1,"requestTimeoutMs":10000,"settleMs":0}, + {"do":"dbExpectOperation","before":"history-6-0","otherBefore":"history-6-1","actor":"a","operation":"cancel"}, + {"do":"reload","actor":"a","settleMs":0}, + {"do":"ensureSignedIn","actor":"a","name":"history-17-a","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"within":1000}, + { "do": "click", "actor": "a", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "a", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + {"do":"click","actor":"a","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"a","testid":"order-item","contains":"cancelled","within":10000}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-7-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-7-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"callConcurrently","actors":["a"],"action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"cancelled","attribute":"data-cancel-input"},"requests":1,"requestTimeoutMs":10000,"settleMs":0}, + {"do":"dbExpectOperation","before":"history-7-0","otherBefore":"history-7-1","actor":"a","operation":"cancel"}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-8-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-8-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"reload","actor":"a","settleMs":0}, + {"do":"ensureSignedIn","actor":"a","name":"history-17-a","readyTestid":"current-user"}, + {"do":"dbExpectOperation","before":"history-8-0","otherBefore":"history-8-1","actor":"a","operation":"reconnect"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"a","testid":"catalog-link","ifAvailable":true,"within":1000}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-9-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-9-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"callConcurrently","actors":["a"],"action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Desk Lamp","attribute":"data-buy-input"},"requests":1,"requestTimeoutMs":10000,"settleMs":0}, + {"do":"dbExpectOperation","before":"history-9-0","otherBefore":"history-9-1","actor":"a","operation":"buy"}, + {"do":"click","actor":"b","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"b","testid":"catalog-link","ifAvailable":true,"within":1000}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-10-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-10-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"callConcurrently","actors":["b"],"action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Desk Lamp","attribute":"data-buy-input"},"requests":1,"requestTimeoutMs":10000,"settleMs":0}, + {"do":"dbExpectOperation","before":"history-10-0","otherBefore":"history-10-1","actor":"b","operation":"buy"}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-11-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-11-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"callConcurrently","actors":["b"],"action":"cart-add","namedAction":{"id":"cart-add","path":"/api/cart","reducer":"add_to_cart","args":[0],"params":[{"name":"itemId","in":"body","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Coffee Grinder","attribute":"data-buy-input"},"requests":1,"requestTimeoutMs":10000,"settleMs":0}, + {"do":"dbExpectOperation","before":"history-11-0","otherBefore":"history-11-1","actor":"b","operation":"cart-add"}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-12-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-12-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"callConcurrently","actors":["admin"],"action":"restock","namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Desk Lamp","attribute":"data-restock-input"},"requests":1,"requestTimeoutMs":10000,"settleMs":0}, + {"do":"dbExpectOperation","before":"history-12-0","otherBefore":"history-12-1","actor":"admin","operation":"restock"}, + {"do":"fill","actor":"admin","testid":"transfer-from","text":"West","in":{"testid":"admin-item-row","contains":"Coffee Grinder"}}, + {"do":"fill","actor":"admin","testid":"transfer-to","text":"East","in":{"testid":"admin-item-row","contains":"Coffee Grinder"}}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-13-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-13-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"callConcurrently","actors":["admin"],"action":"transfer","namedAction":{"id":"transfer","path":"/api/admin/transfer","reducer":"admin_transfer_stock","args":[0,0,0,4],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"fromWarehouseId","in":"body","wireType":"u64"},{"name":"toWarehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-item-row","contains":"Coffee Grinder","attribute":"data-transfer-input","fields":["itemId","fromWarehouseId","toWarehouseId"]},"requests":1,"requestTimeoutMs":10000,"settleMs":0}, + {"do":"dbExpectOperation","before":"history-13-0","otherBefore":"history-13-1","actor":"admin","operation":"transfer"}, + {"do":"reload","actor":"b","settleMs":0}, + {"do":"ensureSignedIn","actor":"b","name":"history-17-b","readyTestid":"current-user"}, + {"do":"click","actor":"b","testid":"overlay-close","ifAvailable":true,"unlessVisible":"cart-total","within":1000}, + {"do": "click", "actor": "b", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["cart-toggle", "cart-total"]}, + {"do":"click","actor":"b","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"b","testid":"cart-quantity","in":{"testid":"cart-item","contains":"Coffee Grinder"},"equals":1,"within":10000}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-14-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-14-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"callConcurrently","actors":["b"],"action":"cart-update","namedAction":{"id":"cart-update","path":"/api/cart/:itemId","method":"PATCH","reducer":"update_cart_quantity","args":[0,4],"params":[{"name":"itemId","in":"path","placeholder":":itemId","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"cart-item","contains":"Coffee Grinder","attribute":"data-cart-input","fields":["itemId"]},"requests":1,"requestTimeoutMs":10000,"settleMs":0}, + {"do":"dbExpectOperation","before":"history-14-0","otherBefore":"history-14-1","actor":"b","operation":"cart-update"}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-15-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-15-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"callConcurrently","actors":["b"],"action":"checkout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"requests":1,"requestTimeoutMs":10000,"settleMs":0}, + {"do":"dbExpectOperation","before":"history-15-0","otherBefore":"history-15-1","actor":"b","operation":"checkout"}, + {"do":"reload","actor":"b","settleMs":0}, + {"do":"ensureSignedIn","actor":"b","name":"history-17-b","readyTestid":"current-user"}, + {"do":"click","actor":"b","testid":"overlay-close","ifAvailable":true,"within":1000}, + { "do": "click", "actor": "b", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "b", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + {"do":"click","actor":"b","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"b","testid":"order-item","contains":"pending","within":10000}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-16-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-16-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"callConcurrently","actors":["b"],"action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"pending","attribute":"data-cancel-input"},"requests":1,"requestTimeoutMs":10000,"settleMs":0}, + {"do":"dbExpectOperation","before":"history-16-0","otherBefore":"history-16-1","actor":"b","operation":"cancel"}, + {"do":"reload","actor":"b","settleMs":0}, + {"do":"ensureSignedIn","actor":"b","name":"history-17-b","readyTestid":"current-user"}, + {"do":"click","actor":"b","testid":"overlay-close","ifAvailable":true,"within":1000}, + { "do": "click", "actor": "b", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "b", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + {"do":"click","actor":"b","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"b","testid":"order-item","contains":"cancelled","within":10000}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-17-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-17-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"callConcurrently","actors":["b"],"action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"cancelled","attribute":"data-cancel-input"},"requests":1,"requestTimeoutMs":10000,"settleMs":0}, + {"do":"dbExpectOperation","before":"history-17-0","otherBefore":"history-17-1","actor":"b","operation":"cancel"}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-18-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-18-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"reload","actor":"b","settleMs":0}, + {"do":"ensureSignedIn","actor":"b","name":"history-17-b","readyTestid":"current-user"}, + {"do":"dbExpectOperation","before":"history-18-0","otherBefore":"history-18-1","actor":"b","operation":"reconnect"}, + {"do":"click","actor":"b","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"b","testid":"catalog-link","ifAvailable":true,"within":1000}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-19-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-19-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"callConcurrently","actors":["b"],"action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Desk Lamp","attribute":"data-buy-input"},"requests":1,"requestTimeoutMs":10000,"settleMs":0}, + {"do":"dbExpectOperation","before":"history-19-0","otherBefore":"history-19-1","actor":"b","operation":"buy"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"a","testid":"catalog-link","ifAvailable":true,"within":1000}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-20-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-20-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"callConcurrently","actors":["a"],"action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Desk Lamp","attribute":"data-buy-input"},"requests":1,"requestTimeoutMs":10000,"settleMs":0}, + {"do":"dbExpectOperation","before":"history-20-0","otherBefore":"history-20-1","actor":"a","operation":"buy"}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-21-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-21-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"callConcurrently","actors":["a"],"action":"cart-add","namedAction":{"id":"cart-add","path":"/api/cart","reducer":"add_to_cart","args":[0],"params":[{"name":"itemId","in":"body","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Coffee Grinder","attribute":"data-buy-input"},"requests":1,"requestTimeoutMs":10000,"settleMs":0}, + {"do":"dbExpectOperation","before":"history-21-0","otherBefore":"history-21-1","actor":"a","operation":"cart-add"}, + {"do":"fill","actor":"admin","testid":"transfer-from","text":"East","in":{"testid":"admin-item-row","contains":"Coffee Grinder"}}, + {"do":"fill","actor":"admin","testid":"transfer-to","text":"West","in":{"testid":"admin-item-row","contains":"Coffee Grinder"}}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-22-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-22-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"callConcurrently","actors":["admin"],"action":"transfer","namedAction":{"id":"transfer","path":"/api/admin/transfer","reducer":"admin_transfer_stock","args":[0,0,0,5],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"fromWarehouseId","in":"body","wireType":"u64"},{"name":"toWarehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-item-row","contains":"Coffee Grinder","attribute":"data-transfer-input","fields":["itemId","fromWarehouseId","toWarehouseId"]},"requests":1,"requestTimeoutMs":10000,"settleMs":0}, + {"do":"dbExpectOperation","before":"history-22-0","otherBefore":"history-22-1","actor":"admin","operation":"transfer"}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-23-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-23-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"callConcurrently","actors":["admin"],"action":"restock","namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Desk Lamp","attribute":"data-restock-input"},"requests":1,"requestTimeoutMs":10000,"settleMs":0}, + {"do":"dbExpectOperation","before":"history-23-0","otherBefore":"history-23-1","actor":"admin","operation":"restock"}, + {"do":"reload","actor":"a","settleMs":0}, + {"do":"ensureSignedIn","actor":"a","name":"history-17-a","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"unlessVisible":"cart-total","within":1000}, + {"do": "click", "actor": "a", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["cart-toggle", "cart-total"]}, + {"do":"click","actor":"a","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"a","testid":"cart-quantity","in":{"testid":"cart-item","contains":"Coffee Grinder"},"equals":1,"within":10000}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-24-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-24-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"callConcurrently","actors":["a"],"action":"cart-update","namedAction":{"id":"cart-update","path":"/api/cart/:itemId","method":"PATCH","reducer":"update_cart_quantity","args":[0,5],"params":[{"name":"itemId","in":"path","placeholder":":itemId","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"cart-item","contains":"Coffee Grinder","attribute":"data-cart-input","fields":["itemId"]},"requests":1,"requestTimeoutMs":10000,"settleMs":0}, + {"do":"dbExpectOperation","before":"history-24-0","otherBefore":"history-24-1","actor":"a","operation":"cart-update"}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-25-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-25-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"callConcurrently","actors":["a"],"action":"checkout","namedAction":{"id":"checkout","path":"/api/checkout","reducer":"checkout","args":[]},"requests":1,"requestTimeoutMs":10000,"settleMs":0}, + {"do":"dbExpectOperation","before":"history-25-0","otherBefore":"history-25-1","actor":"a","operation":"checkout"}, + {"do":"reload","actor":"a","settleMs":0}, + {"do":"ensureSignedIn","actor":"a","name":"history-17-a","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"within":1000}, + { "do": "click", "actor": "a", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "a", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + {"do":"click","actor":"a","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"a","testid":"order-item","contains":"pending","within":10000}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-26-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-26-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"callConcurrently","actors":["a"],"action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"pending","attribute":"data-cancel-input"},"requests":1,"requestTimeoutMs":10000,"settleMs":0}, + {"do":"dbExpectOperation","before":"history-26-0","otherBefore":"history-26-1","actor":"a","operation":"cancel"}, + {"do":"reload","actor":"a","settleMs":0}, + {"do":"ensureSignedIn","actor":"a","name":"history-17-a","readyTestid":"current-user"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"within":1000}, + { "do": "click", "actor": "a", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "a", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + {"do":"click","actor":"a","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"a","testid":"order-item","contains":"cancelled","within":10000}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-27-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-27-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"callConcurrently","actors":["a"],"action":"cancel","namedAction":{"id":"cancel","path":"/api/orders/:id/cancel","reducer":"cancel_order","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"order-item","contains":"cancelled","attribute":"data-cancel-input"},"requests":1,"requestTimeoutMs":10000,"settleMs":0}, + {"do":"dbExpectOperation","before":"history-27-0","otherBefore":"history-27-1","actor":"a","operation":"cancel"}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-28-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-28-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"reload","actor":"a","settleMs":0}, + {"do":"ensureSignedIn","actor":"a","name":"history-17-a","readyTestid":"current-user"}, + {"do":"dbExpectOperation","before":"history-28-0","otherBefore":"history-28-1","actor":"a","operation":"reconnect"}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"a","testid":"catalog-link","ifAvailable":true,"within":1000}, + {"do":"dbRecordCheckout","account":"{user:history-17-a}","item":"Desk Lamp","as":"history-29-0","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"dbRecordCheckout","account":"{user:history-17-b}","item":"Desk Lamp","as":"history-29-1","storage":{"kind":"order-data","cart":true,"warehouses":true}}, + {"do":"callConcurrently","actors":["a"],"action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Desk Lamp","attribute":"data-buy-input"},"requests":1,"requestTimeoutMs":10000,"settleMs":0}, + {"do":"dbExpectOperation","before":"history-29-0","otherBefore":"history-29-1","actor":"a","operation":"buy"} + ] + }] + }] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-account-state-reconnect.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-account-state-reconnect.json new file mode 100644 index 00000000000..8570e98d49d --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-account-state-reconnect.json @@ -0,0 +1,42 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Progression account state after reconnect", + "features": [ + { + "id": 105, + "name": "An account keeps what belongs to it", + "actors": ["shopper", "peer"], + "setup": [ + { "do": "signUp", "actor": "shopper", "name": "pat" }, + { "do": "signIn", "actor": "peer", "name": "pat" }, + { "do": "click", "actor": "shopper", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "click", "actor": "shopper", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Keyboard" } }, + {"do": "click", "actor": "shopper", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["cart-toggle", "cart-total"]}, + { "do": "click", "actor": "shopper", "testid": "cart-toggle", "unlessVisible": "cart-total" }, + { "do": "expect", "actor": "shopper", "testid": "cart-item", "contains": "Keyboard" }, + { "do": "click", "actor": "peer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "105b", + "category": "production", + "desc": "the same account and cart survive the connection dropping and coming back", + "points": 1, + "statedBy": "the signed-in account and its current data remain available after reconnecting", + "steps": [ + { "do": "setOffline", "actor": "shopper", "settleMs": 3000 }, + { "do": "click", "actor": "peer", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Headphones" } }, + { "do": "setOffline", "actor": "shopper", "offline": false, "settleMs": 6000 }, + { "do": "expect", "actor": "shopper", "testid": "current-user", "contains": "pat" }, + {"do": "click", "actor": "shopper", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["cart-toggle", "cart-total"]}, + { "do": "click", "actor": "shopper", "testid": "cart-toggle", "unlessVisible": "cart-total" }, + { "do": "expect", "actor": "shopper", "testid": "cart-item", "contains": "Keyboard" }, + { "do": "expect", "actor": "shopper", "testid": "cart-item", "contains": "Headphones" } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-account-state-reload.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-account-state-reload.json new file mode 100644 index 00000000000..283c1e148e8 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-account-state-reload.json @@ -0,0 +1,49 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Progression account state after reload", + "features": [ + { + "id": 105, + "name": "An account keeps what belongs to it", + "actors": ["shopper"], + "setup": [ + { "do": "signUp", "actor": "shopper", "name": "pat" }, + { "do": "click", "actor": "shopper", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "105a", + "category": "production", + "desc": "cart and order history survive a backend restart, including a fresh account login", + "points": 1, + "statedBy": "the signed-in account, cart, and orders persist across a page reload", + "steps": [ + { "do": "click", "actor": "shopper", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Headphones" } }, + {"do": "click", "actor": "shopper", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["cart-toggle", "cart-total"]}, + { "do": "click", "actor": "shopper", "testid": "cart-toggle", "unlessVisible": "cart-total" }, + { "do": "click", "actor": "shopper", "testid": "checkout-submit", "settleMs": 1500 }, + { "do": "click", "actor": "shopper", "testid": "overlay-close", "ifAvailable": true }, + { "do": "click", "actor": "shopper", "testid": "catalog-link", "settleMs": 2500, "ifAvailable": true }, + { "do": "click", "actor": "shopper", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Keyboard" } }, + {"do": "click", "actor": "shopper", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["cart-toggle", "cart-total"]}, + { "do": "click", "actor": "shopper", "testid": "cart-toggle", "unlessVisible": "cart-total" }, + { "do": "expect", "actor": "shopper", "testid": "cart-item", "contains": "Keyboard", "within": 10000 }, + { "do": "restartBackend", "settleMs": 1000 }, + { "do": "freshClient", "actor": "shopper" }, + { "do": "signIn", "actor": "shopper-fresh", "name": "pat" }, + { "do": "click", "actor": "shopper-fresh", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "shopper-fresh", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["cart-toggle", "cart-total"]}, + { "do": "click", "actor": "shopper-fresh", "testid": "cart-toggle", "unlessVisible": "cart-total" }, + { "do": "expect", "actor": "shopper-fresh", "testid": "cart-item", "contains": "Keyboard", "count": 1 }, + { "do": "click", "actor": "shopper-fresh", "testid": "overlay-close", "ifAvailable": true }, + {"do": "click", "actor": "shopper-fresh", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + { "do": "click", "actor": "shopper-fresh", "testid": "orders-toggle", "unlessVisible": "order-item" }, + { "do": "expect", "actor": "shopper-fresh", "testid": "order-item", "contains": "Headphones", "count": 1 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-automatic-reorder-access.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-automatic-reorder-access.json new file mode 100644 index 00000000000..71941b9bf34 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-automatic-reorder-access.json @@ -0,0 +1,55 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 5, + "name": "Automatic reorder access boundary", + "features": [ + { + "id": 502, + "name": "Warehouse staff manage automatic reorder rules", + "actors": [ + "staff", + "customer" + ], + "setup": [ + {"do":"signIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true}, + {"do":"expect","actor":"staff","testid":"reorder-link","within":6000}, + {"do":"click","actor":"staff","testid":"reorder-link"}, + {"do":"fill","actor":"staff","testid":"reorder-item","text":"Desk Lamp"}, + {"do":"fill","actor":"staff","testid":"reorder-threshold","text":"2"}, + {"do":"fill","actor":"staff","testid":"reorder-quantity","text":"5"}, + {"do":"callAction","actor":"staff","action":"saveReorderRule","namedAction":{"id":"saveReorderRule","path":"/api/reorders/{itemId}","method":"PUT","reducer":"save_reorder_rule","args":[0,2,5],"params":[{"name":"itemId","in":"path","placeholder":"{itemId}","wireType":"u64"},{"name":"threshold","in":"body"},{"name":"quantity","in":"body"}]},"input":{"testid":"reorder-submit","attribute":"data-action-input"}}, + {"do":"expectActionOutcome","actor":"staff","outcome":"accepted"}, + {"do":"reload","actor":"staff","settleMs":1000}, + {"do":"ensureSignedIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true,"readyTestid":"current-user"}, + {"do":"click","actor":"staff","testid":"reorder-link","unlessVisible":"reorder-submit"}, + {"do":"expectElementCount","actor":"staff","testid":"reorder-rule-item","contains":"Desk Lamp","equals":1,"within":10000}, + {"do":"signUp","actor":"customer","name":"reorder-customer"} + ], + "criteria": [ + { + "id": "502c", + "category": "production", + "desc": "a customer cannot see or replay automatic reorder management", + "points": 2, + "statedBy": "Only warehouse staff can manage automatic reorder rules.", + "steps": [ + {"do":"expect","actor":"customer","testid":"reorder-link","absent":true}, + {"do":"fill","actor":"staff","testid":"reorder-item","text":"Desk Lamp"}, + {"do":"fill","actor":"staff","testid":"reorder-threshold","text":"1"}, + {"do":"fill","actor":"staff","testid":"reorder-quantity","text":"9"}, + {"do":"callAction","actor":"customer","from":"staff","action":"saveReorderRule","namedAction":{"id":"saveReorderRule","path":"/api/reorders/{itemId}","method":"PUT","reducer":"save_reorder_rule","args":[0,1,9],"params":[{"name":"itemId","in":"path","placeholder":"{itemId}","wireType":"u64"},{"name":"threshold","in":"body"},{"name":"quantity","in":"body"}]},"input":{"testid":"reorder-submit","attribute":"data-action-input"}}, + {"do":"expectActionOutcome","actor":"customer","outcome":"refused","routeProvenBy":"staff"}, + {"do":"reload","actor":"staff","settleMs":1000}, + {"do":"ensureSignedIn","actor":"staff","name":"staff","readyTestid":"current-user","password":"stackbench-staff-2026","exact":true}, + {"do":"click","actor":"staff","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"staff","testid":"reorder-link","unlessVisible":"reorder-submit"}, + {"do":"expect","actor":"staff","testid":"reorder-rule-item","contains":"Desk Lamp","count":1,"attribute":"data-threshold","value":"2"}, + {"do":"expect","actor":"staff","testid":"reorder-rule-item","contains":"Desk Lamp","attribute":"data-quantity","value":"5"} + ] + } + ] + } + ], + "note": "Draft: revised observations require matching reference and defect-control qualification." +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-automatic-reorder-duplicate.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-automatic-reorder-duplicate.json new file mode 100644 index 00000000000..b4469d410cb --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-automatic-reorder-duplicate.json @@ -0,0 +1,84 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 5, + "name": "Automatic reorder duplicate boundary", + "features": [ + { + "id": 502, + "name": "Warehouse staff manage automatic reorder rules", + "actors": [ + "staff", + "admin", + "buyer-a", + "buyer-b", + "buyer-c" + ], + "setup": [ + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"East","quantity":2,"settleMs":2000}, + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"West","quantity":1,"settleMs":2000}, + {"do":"signIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true}, + {"do":"expect","actor":"staff","testid":"reorder-link","within":6000}, + {"do":"click","actor":"staff","testid":"reorder-link"}, + {"do":"fill","actor":"staff","testid":"reorder-item","text":"Desk Lamp"}, + {"do":"fill","actor":"staff","testid":"reorder-threshold","text":"2"}, + {"do":"fill","actor":"staff","testid":"reorder-quantity","text":"5"}, + {"do":"callAction","actor":"staff","action":"saveReorderRule","namedAction":{"id":"saveReorderRule","path":"/api/reorders/{itemId}","method":"PUT","reducer":"save_reorder_rule","args":[0,2,5],"params":[{"name":"itemId","in":"path","placeholder":"{itemId}","wireType":"u64"},{"name":"threshold","in":"body"},{"name":"quantity","in":"body"}]},"input":{"testid":"reorder-submit","attribute":"data-action-input"}}, + {"do":"expectActionOutcome","actor":"staff","outcome":"accepted"}, + {"do":"reload","actor":"staff","settleMs":1000}, + {"do":"ensureSignedIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true,"readyTestid":"current-user"}, + {"do":"click","actor":"staff","testid":"reorder-link","unlessVisible":"reorder-submit"}, + {"do":"expectElementCount","actor":"staff","testid":"reorder-rule-item","contains":"Desk Lamp","equals":1,"within":10000}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"click","actor":"admin","testid":"restocks-link","ifAvailable":true,"within":1000}, + {"do":"signUp","actor":"buyer-a","name":"reorder-a"}, + {"do":"signUp","actor":"buyer-b","name":"reorder-b"}, + {"do":"signUp","actor":"buyer-c","name":"reorder-c"}, + {"do":"reload","actor":"admin","settleMs":1000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","readyTestid":"current-user","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":"admin-revenue"}, + {"do":"click","actor":"admin","testid":"restocks-link","ifAvailable":true,"within":1000}, + {"do":"expect","actor":"admin","testid":"schedule-restock-submit"}, + {"do":"expectElementCount","actor":"admin","testid":"pending-restock-item","equals":0}, + {"do":"callAction","actor":"buyer-a","action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Desk Lamp","attribute":"data-buy-input"}}, + {"do":"expectActionOutcome","actor":"buyer-a","outcome":"accepted"}, + {"do":"dbExpectStock","item":"Desk Lamp","equals":2}, + {"do":"reload","actor":"admin","settleMs":1000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","readyTestid":"current-user","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":"admin-revenue"}, + {"do":"click","actor":"admin","testid":"restocks-link","ifAvailable":true,"within":1000}, + {"do":"expect","actor":"admin","testid":"schedule-restock-submit"}, + {"do":"expectElementCount","actor":"admin","testid":"pending-restock-item","equals":1,"within":10000}, + {"do":"expect","actor":"admin","testid":"pending-restock-item","contains":"Desk Lamp","attribute":"data-quantity","value":"5","count":1} + ], + "criteria": [ + { + "id": "502b", + "category": "production", + "desc": "more sales do not duplicate a pending restock", + "points": 2, + "statedBy": "A pending automatic restock is not scheduled twice.", + "steps": [ + {"do":"callAction","actor":"buyer-b","action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Desk Lamp","attribute":"data-buy-input"}}, + {"do":"expectActionOutcome","actor":"buyer-b","outcome":"accepted"}, + {"do":"callAction","actor":"buyer-c","action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Desk Lamp","attribute":"data-buy-input"}}, + {"do":"expectActionOutcome","actor":"buyer-c","outcome":"accepted"}, + {"do":"dbExpectStock","item":"Desk Lamp","equals":0}, + {"do":"reload","actor":"admin","settleMs":1000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","readyTestid":"current-user","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":"admin-revenue"}, + {"do":"click","actor":"admin","testid":"restocks-link","ifAvailable":true,"within":1000}, + {"do":"expect","actor":"admin","testid":"schedule-restock-submit"}, + {"do":"expectElementCount","actor":"admin","testid":"pending-restock-item","equals":1,"within":10000}, + {"do":"expect","actor":"admin","testid":"pending-restock-item","contains":"Desk Lamp","attribute":"data-quantity","value":"5","count":1} + ] + } + ] + } + ], + "note": "Draft: revised observations require matching reference and defect-control qualification." +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-automatic-reorder.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-automatic-reorder.json new file mode 100644 index 00000000000..6d0dc2434bd --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-automatic-reorder.json @@ -0,0 +1,67 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 5, + "name": "Automatic reorder rules", + "features": [ + { + "id": 502, + "name": "Warehouse staff manage automatic reorder rules", + "actors": [ + "staff", + "admin", + "buyer-a" + ], + "setup": [ + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"East","quantity":2,"settleMs":2000}, + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"West","quantity":1,"settleMs":2000}, + {"do":"signIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true}, + {"do":"expect","actor":"staff","testid":"reorder-link","within":6000}, + {"do":"click","actor":"staff","testid":"reorder-link"}, + {"do":"fill","actor":"staff","testid":"reorder-item","text":"Desk Lamp"}, + {"do":"fill","actor":"staff","testid":"reorder-threshold","text":"2"}, + {"do":"fill","actor":"staff","testid":"reorder-quantity","text":"5"}, + {"do":"callAction","actor":"staff","action":"saveReorderRule","namedAction":{"id":"saveReorderRule","path":"/api/reorders/{itemId}","method":"PUT","reducer":"save_reorder_rule","args":[0,2,5],"params":[{"name":"itemId","in":"path","placeholder":"{itemId}","wireType":"u64"},{"name":"threshold","in":"body"},{"name":"quantity","in":"body"}]},"input":{"testid":"reorder-submit","attribute":"data-action-input"}}, + {"do":"expectActionOutcome","actor":"staff","outcome":"accepted"}, + {"do":"reload","actor":"staff","settleMs":1000}, + {"do":"ensureSignedIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true,"readyTestid":"current-user"}, + {"do":"click","actor":"staff","testid":"reorder-link","unlessVisible":"reorder-submit"}, + {"do":"expectElementCount","actor":"staff","testid":"reorder-rule-item","contains":"Desk Lamp","equals":1,"within":10000}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"click","actor":"admin","testid":"restocks-link","ifAvailable":true,"within":1000}, + {"do":"signUp","actor":"buyer-a","name":"reorder-a"} + ], + "criteria": [ + { + "id": "502a", + "category": "feature", + "desc": "crossing the threshold creates one pending restock", + "points": 3, + "statedBy": "Crossing a reorder threshold schedules a restock.", + "steps": [ + {"do":"reload","actor":"admin","settleMs":1000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","readyTestid":"current-user","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":"admin-revenue"}, + {"do":"click","actor":"admin","testid":"restocks-link","ifAvailable":true,"within":1000}, + {"do":"expect","actor":"admin","testid":"schedule-restock-submit"}, + {"do":"expectElementCount","actor":"admin","testid":"pending-restock-item","equals":0}, + {"do":"callAction","actor":"buyer-a","action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Desk Lamp","attribute":"data-buy-input"}}, + {"do":"expectActionOutcome","actor":"buyer-a","outcome":"accepted"}, + {"do":"dbExpectStock","item":"Desk Lamp","equals":2}, + {"do":"reload","actor":"admin","settleMs":1000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","readyTestid":"current-user","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":"admin-revenue"}, + {"do":"click","actor":"admin","testid":"restocks-link","ifAvailable":true,"within":1000}, + {"do":"expect","actor":"admin","testid":"schedule-restock-submit"}, + {"do":"expectElementCount","actor":"admin","testid":"pending-restock-item","equals":1,"within":10000}, + {"do":"expect","actor":"admin","testid":"pending-restock-item","contains":"Desk Lamp","attribute":"data-quantity","value":"5","count":1} + ] + } + ] + } + ], + "note": "Draft: revised observations require matching reference and defect-control qualification." +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-books-balance.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-books-balance.json new file mode 100644 index 00000000000..0d34bb44e3d --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-books-balance.json @@ -0,0 +1,80 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Progression books balance", + "note": "Draft: 1,000 serial accepted purchases, split equally between two customers. Native order and warehouse snapshots qualify the seeded population; revenue and fresh stock retain their existing scoring owners. Each selected check runs the same setup. No latency or throughput SLO is scored.", + "features": [ + { + "id": 107, + "name": "The books balance", + "actors": [ + "buyer", + "other", + "admin" + ], + "setup": [ + {"do": "signUp", "actor": "buyer", "name": "sam"}, + {"do": "signUp", "actor": "other", "name": "tess"}, + {"do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true}, + {"do": "dbSetStock", "item": "Laptop Stand", "warehouse": "East", "quantity": 1020, "settleMs": 0}, + {"do": "dbSetStock", "item": "Laptop Stand", "warehouse": "West", "quantity": 0, "settleMs": 0}, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + {"do": "click", "actor": "admin", "testid": "admin-link", "unlessVisible": "admin-panel"}, + {"do": "recordNumber", "actor": "admin", "testid": "admin-revenue", "as": "revenue-before"}, + {"do": "dbRecordStock", "item": "Laptop Stand", "as": "stand-before"}, + {"do": "dbRecordCheckout", "account": "{user:sam}", "item": "Laptop Stand", "as": "orders-buyer", "storage": {"kind": "order-data", "cart": false, "warehouses": false}}, + {"do": "dbRecordCheckout", "account": "{user:tess}", "item": "Laptop Stand", "as": "orders-other", "storage": {"kind": "order-data", "cart": false, "warehouses": false}}, + {"do": "dbRecordCheckout", "account": "{user:sam}", "item": "Laptop Stand", "as": "stock-buyer", "storage": {"kind": "order-data", "cart": false, "warehouses": true}}, + {"do": "dbRecordCheckout", "account": "{user:tess}", "item": "Laptop Stand", "as": "stock-other", "storage": {"kind": "order-data", "cart": false, "warehouses": true}}, + { "do": "click", "actor": "buyer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "click", "actor": "other", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { + "repeat": 500, + "steps": [ + {"do": "callAction", "actor": "buyer", "action": "buy", "namedAction": {"id": "buy", "path": "/api/items/:id/buy", "reducer": "buy_now", "args": [0], "params": [{"name": "itemId", "in": "path", "placeholder": ":id", "wireType": "u64"}]}, "input": {"testid": "item-card", "contains": "Laptop Stand", "attribute": "data-buy-input"}, "settleMs": 0}, + {"do": "expectActionOutcome", "actor": "buyer", "outcome": "accepted"}, + {"do": "callAction", "actor": "other", "action": "buy", "namedAction": {"id": "buy", "path": "/api/items/:id/buy", "reducer": "buy_now", "args": [0], "params": [{"name": "itemId", "in": "path", "placeholder": ":id", "wireType": "u64"}]}, "input": {"testid": "item-card", "contains": "Laptop Stand", "attribute": "data-buy-input"}, "settleMs": 0}, + {"do": "expectActionOutcome", "actor": "other", "outcome": "accepted"} + ] + } + ], + "criteria": [ + { + "id": "107a", + "category": "production", + "desc": "revenue includes all 1,000 accepted purchases at their stored prices", + "points": 1, + "statedBy": "total revenue equals the sum of order totals", + "steps": [ + {"do": "dbExpectPurchaseCount", "before": ["orders-buyer", "orders-other"], "purchasesEach": 500}, + {"do": "reload", "actor": "admin", "settleMs": 1000}, + {"do": "ensureSignedIn", "actor": "admin", "name": "admin", "readyTestid": "current-user", "password": "stackbench-admin-2026", "exact": true}, + {"do": "click", "actor": "admin", "testid": "overlay-close", "ifAvailable": true, "within": 1000}, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-revenue"]}, + {"do": "click", "actor": "admin", "testid": "admin-link", "unlessVisible": "admin-revenue"}, + {"do": "expectNumber", "actor": "admin", "testid": "admin-revenue", "relativeTo": "revenue-before", "plus": 29000, "within": 10000} + ] + }, + { + "id": "107b", + "category": "production", + "desc": "all 1,000 purchases have matching warehouse effects and fresh stock", + "points": 1, + "statedBy": "warehouse totals and a fresh storefront agree with completed sales", + "steps": [ + {"do": "dbExpectPurchaseCount", "before": ["stock-buyer", "stock-other"], "purchasesEach": 500}, + {"do": "reload", "actor": "admin", "settleMs": 2000}, + {"do": "ensureSignedIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true, "readyTestid": "current-user"}, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + {"do": "click", "actor": "admin", "testid": "admin-link", "ifAvailable": true, "unlessVisible": "admin-panel"}, + {"do": "expectNumber", "actor": "admin", "testid": "admin-stock", "in": {"testid": "admin-item-row", "contains": "Laptop Stand"}, "relativeTo": "stand-before", "plus": -1000}, + {"do": "freshClient", "actor": "buyer"}, + { "do": "click", "actor": "buyer-fresh", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "expectNumber", "actor": "buyer-fresh", "testid": "item-stock", "in": {"testid": "item-card", "contains": "Laptop Stand"}, "relativeTo": "stand-before", "plus": -1000} + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-bundle-checkout.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-bundle-checkout.json new file mode 100644 index 00000000000..6ee35cf7246 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-bundle-checkout.json @@ -0,0 +1,236 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 5, + "name": "Bundle reservations and checkout", + "features": [ + { + "id": 741, + "name": "Bundle checkout", + "actors": [ + "admin", + "buyer" + ], + "setup": [ + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"bundles-link","unlessVisible":"bundle-name-input"}, + {"do":"signUp","actor":"buyer","name":"bundle-checkout"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":6,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"East","quantity":4,"settleMs":0}, + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"fill","actor":"admin","testid":"bundle-name-input","text":"Checkout bundle"}, + {"do":"fill","actor":"admin","testid":"bundle-price-input","text":"75.00"}, + {"do":"fill","actor":"admin","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"}, + {"do":"click","actor":"admin","testid":"bundle-save"}, + {"do":"expect","actor":"admin","testid":"bundle-card","within":10000,"contains":"Checkout bundle"}, + {"do":"click","actor":"buyer","testid":"bundles-link","unlessVisible":"bundle-card"} + ], + "criteria": [ + { + "id": "741a", + "category": "feature", + "desc": "adding a bundle reserves its components and checkout records the bundle price once", + "points": 2, + "steps": [ + {"do":"click","actor":"buyer","testid":"bundle-add-to-cart","in":{"testid":"bundle-card","contains":"Checkout bundle"},"settleMs":500}, + {"do":"dbExpectStock","item":"Keyboard","warehouse":"East","equals":4}, + {"do":"dbExpectStock","item":"Desk Lamp","warehouse":"East","equals":3}, + {"do":"click","actor":"buyer","testid":"cart-toggle"}, + {"do":"click","actor":"buyer","testid":"checkout-submit"}, + {"do":"click","actor":"buyer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"buyer","testid":"orders-toggle"}, + {"do":"expect","actor":"buyer","testid":"order-item","within":10000,"contains":"Checkout bundle","count":1}, + {"do":"expectNumber","actor":"buyer","testid":"payment-amount","equals":75,"within":10000,"in":{"testid":"order-item","contains":"Checkout bundle"}}, + {"do":"dbExpectStock","item":"Keyboard","warehouse":"East","equals":4}, + {"do":"dbExpectStock","item":"Desk Lamp","warehouse":"East","equals":3}, + {"do":"dbExpectStock","item":"Keyboard","warehouse":"West","equals":0}, + {"do":"dbExpectStock","item":"Desk Lamp","warehouse":"West","equals":0} + ] + } + ] + }, + { + "id": 744, + "name": "Competing bundle reservations", + "actors": [ + "admin", + "a", + "b" + ], + "setup": [ + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"bundles-link","unlessVisible":"bundle-name-input"}, + {"do":"signUp","actor":"a","name":"bundle-race-a"}, + {"do":"signUp","actor":"b","name":"bundle-race-b"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":2,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"East","quantity":1,"settleMs":0}, + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"fill","actor":"admin","testid":"bundle-name-input","text":"Scarce bundle"}, + {"do":"fill","actor":"admin","testid":"bundle-price-input","text":"75.00"}, + {"do":"fill","actor":"admin","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"}, + {"do":"click","actor":"admin","testid":"bundle-save"}, + {"do":"expect","actor":"admin","testid":"bundle-card","within":10000,"contains":"Scarce bundle"}, + {"do":"click","actor":"a","testid":"bundles-link","unlessVisible":"bundle-card"}, + {"do":"click","actor":"b","testid":"bundles-link","unlessVisible":"bundle-card"} + ], + "criteria": [ + { + "id": "744a", + "category": "production", + "desc": "two competing reservations accept exactly one whole bundle without consuming extra components", + "points": 3, + "steps": [ + {"do":"callConcurrently","actors":["a","b"],"action":"addBundleToCart","namedAction":{"id":"addBundleToCart","path":"/api/cart/bundles","reducer":"add_bundle_to_cart","args":[0],"params":[{"name":"bundleId","in":"body","wireType":"u64"}]},"input":{"testid":"bundle-card","contains":"Scarce bundle","attribute":"data-bundle-input"},"settleMs":1000}, + {"do":"expectCallOutcomes","accepted":1}, + {"do":"dbExpectStock","item":"Keyboard","warehouse":"East","equals":0}, + {"do":"dbExpectStock","item":"Desk Lamp","warehouse":"East","equals":0}, + {"do":"reload","actor":"a","settleMs":1000}, + {"do":"reload","actor":"b","settleMs":1000}, + {"do":"click","actor":"a","testid":"cart-toggle"}, + {"do":"click","actor":"b","testid":"cart-toggle"}, + {"do":"expectActorsWith","actors":["a","b"],"testid":"cart-item","contains":"Scarce bundle","equals":1,"maxEach":1}, + {"do":"click","actor":"a","testid":"bundle-remove","in":{"testid":"cart-item","contains":"Scarce bundle"},"ifAvailable":true,"settleMs":500}, + {"do":"click","actor":"b","testid":"bundle-remove","in":{"testid":"cart-item","contains":"Scarce bundle"},"ifAvailable":true,"settleMs":500}, + {"do":"dbExpectStock","item":"Keyboard","warehouse":"East","equals":2}, + {"do":"dbExpectStock","item":"Desk Lamp","warehouse":"East","equals":1} + ] + } + ] + }, + { + "id": 745, + "name": "Incomplete bundle stock", + "actors": [ + "admin", + "buyer" + ], + "setup": [ + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"bundles-link","unlessVisible":"bundle-name-input"}, + {"do":"signUp","actor":"buyer","name":"bundle-partial"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":2,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"East","quantity":0,"settleMs":0}, + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"fill","actor":"admin","testid":"bundle-name-input","text":"Unavailable bundle"}, + {"do":"fill","actor":"admin","testid":"bundle-price-input","text":"75.00"}, + {"do":"fill","actor":"admin","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"}, + {"do":"click","actor":"admin","testid":"bundle-save"}, + {"do":"expect","actor":"admin","testid":"bundle-card","within":10000,"contains":"Unavailable bundle"}, + {"do":"click","actor":"buyer","testid":"bundles-link","unlessVisible":"bundle-card"} + ], + "criteria": [ + { + "id": "745a", + "category": "production", + "desc": "a missing component refuses the reservation without taking available stock or adding a cart line", + "points": 3, + "steps": [ + {"do":"callAction","actor":"buyer","action":"addBundleToCart","namedAction":{"id":"addBundleToCart","path":"/api/cart/bundles","reducer":"add_bundle_to_cart","args":[0],"params":[{"name":"bundleId","in":"body","wireType":"u64"}]},"input":{"testid":"bundle-card","contains":"Unavailable bundle","attribute":"data-bundle-input"},"settleMs":1000}, + {"do":"expectActionOutcome","actor":"buyer","outcome":"application-refused"}, + {"do":"dbExpectStock","item":"Keyboard","warehouse":"East","equals":2}, + {"do":"dbExpectStock","item":"Desk Lamp","warehouse":"East","equals":0}, + {"do":"click","actor":"buyer","testid":"cart-toggle"}, + {"do":"expect","actor":"buyer","testid":"cart-item","within":10000,"contains":"Unavailable bundle","absent":true} + ] + } + ] + }, + { + "id": 746, + "name": "Bundle reservation restart", + "actors": [ + "admin", + "buyer" + ], + "setup": [ + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"bundles-link","unlessVisible":"bundle-name-input"}, + {"do":"signUp","actor":"buyer","name":"bundle-expiry"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":6,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"East","quantity":4,"settleMs":0}, + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"fill","actor":"admin","testid":"bundle-name-input","text":"Expiring bundle"}, + {"do":"fill","actor":"admin","testid":"bundle-price-input","text":"75.00"}, + {"do":"fill","actor":"admin","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"}, + {"do":"click","actor":"admin","testid":"bundle-save"}, + {"do":"expect","actor":"admin","testid":"bundle-card","within":10000,"contains":"Expiring bundle"}, + {"do":"click","actor":"buyer","testid":"bundles-link","unlessVisible":"bundle-card"} + ], + "criteria": [ + { + "id": "746a", + "category": "production", + "desc": "an expired reservation releases each component once across a backend restart", + "points": 3, + "steps": [ + {"do":"click","actor":"buyer","testid":"bundle-add-to-cart","in":{"testid":"bundle-card","contains":"Expiring bundle"},"settleMs":500}, + {"do":"dbExpectStock","item":"Keyboard","warehouse":"East","equals":4}, + {"do":"dbExpectStock","item":"Desk Lamp","warehouse":"East","equals":3}, + {"do":"restartBackend","settleMs":1500}, + {"do":"wait","actor":"buyer","ms":92000}, + {"do":"reload","actor":"buyer","settleMs":1000}, + {"do":"ensureSignedIn","actor":"buyer","name":"bundle-expiry","readyTestid":"current-user"}, + {"do":"click","actor":"buyer","testid":"cart-toggle"}, + {"do":"expect","actor":"buyer","testid":"cart-item-expired","within":10000,"in":{"testid":"cart-item","contains":"Expiring bundle"}}, + {"do":"dbExpectStock","item":"Keyboard","warehouse":"East","equals":6}, + {"do":"dbExpectStock","item":"Desk Lamp","warehouse":"East","equals":4}, + {"do":"restartBackend","settleMs":1500}, + {"do":"dbExpectStock","item":"Keyboard","warehouse":"East","equals":6}, + {"do":"dbExpectStock","item":"Desk Lamp","warehouse":"East","equals":4} + ] + } + ] + }, + { + "id": 747, + "name": "Repeated bundle checkout", + "actors": [ + "admin", + "a", + "b" + ], + "setup": [ + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"bundles-link","unlessVisible":"bundle-name-input"}, + {"do":"signUp","actor":"a","name":"bundle-repeat"}, + {"do":"signIn","actor":"b","name":"bundle-repeat"}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":6,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"East","quantity":4,"settleMs":0}, + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"fill","actor":"admin","testid":"bundle-name-input","text":"Repeated bundle"}, + {"do":"fill","actor":"admin","testid":"bundle-price-input","text":"75.00"}, + {"do":"fill","actor":"admin","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"}, + {"do":"click","actor":"admin","testid":"bundle-save"}, + {"do":"expect","actor":"admin","testid":"bundle-card","within":10000,"contains":"Repeated bundle"}, + {"do":"click","actor":"a","testid":"bundles-link","unlessVisible":"bundle-card"}, + {"do":"click","actor":"a","testid":"bundle-add-to-cart","in":{"testid":"bundle-card","contains":"Repeated bundle"},"settleMs":500}, + {"do":"dbExpectStock","item":"Keyboard","warehouse":"East","equals":4}, + {"do":"dbExpectStock","item":"Desk Lamp","warehouse":"East","equals":3} + ], + "criteria": [ + { + "id": "747a", + "category": "production", + "desc": "two checkout requests consume one reservation and create one paid bundle", + "points": 3, + "steps": [ + {"do":"callConcurrently","actors":["a","b"],"action":"checkout","settleMs":1000}, + {"do":"expectCallOutcomes","accepted":1}, + {"do":"reload","actor":"a","settleMs":1000}, + {"do":"click","actor":"a","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"a","testid":"orders-toggle"}, + {"do":"expect","actor":"a","testid":"order-item","within":10000,"contains":"Repeated bundle","count":1}, + {"do":"expectNumber","actor":"a","testid":"payment-amount","equals":75,"within":10000,"in":{"testid":"order-item","contains":"Repeated bundle"}}, + {"do":"dbExpectStock","item":"Keyboard","warehouse":"East","equals":4}, + {"do":"dbExpectStock","item":"Desk Lamp","warehouse":"East","equals":3} + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-bundle-returns.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-bundle-returns.json new file mode 100644 index 00000000000..29f758d6cda --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-bundle-returns.json @@ -0,0 +1,150 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 6, + "name": "Bundle returns", + "features": [ + { + "id": 742, + "name": "Historical bundle return", + "actors": [ + "admin", + "buyer", + "staff" + ], + "setup": [ + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"bundles-link","unlessVisible":"bundle-name-input"}, + {"do":"signUp","actor":"buyer","name":"Historical bundle"}, + {"do":"signIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":6,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"East","quantity":4,"settleMs":0}, + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"fill","actor":"admin","testid":"bundle-name-input","text":"Historical bundle"}, + {"do":"fill","actor":"admin","testid":"bundle-price-input","text":"75.00"}, + {"do":"fill","actor":"admin","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"}, + {"do":"click","actor":"admin","testid":"bundle-save"}, + {"do":"expect","actor":"admin","testid":"bundle-card","within":10000,"contains":"Historical bundle"}, + {"do":"click","actor":"buyer","testid":"bundles-link","unlessVisible":"bundle-card"}, + {"do":"click","actor":"buyer","testid":"bundle-add-to-cart","in":{"testid":"bundle-card","contains":"Historical bundle"}}, + {"do":"click","actor":"buyer","testid":"cart-toggle"}, + {"do":"click","actor":"buyer","testid":"checkout-submit"}, + {"do":"click","actor":"buyer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"buyer","testid":"orders-toggle"}, + {"do":"expect","actor":"buyer","testid":"order-item","within":10000,"contains":"Historical bundle","count":1}, + {"do":"expectNumber","actor":"buyer","testid":"payment-amount","equals":75,"within":10000,"in":{"testid":"order-item","contains":"Historical bundle"}}, + {"do":"dbExpectStock","item":"Keyboard","warehouse":"East","equals":4}, + {"do":"dbExpectStock","item":"Desk Lamp","warehouse":"East","equals":3}, + {"do":"click","actor":"staff","testid":"staff-link"}, + {"do":"expect","actor":"staff","testid":"queue-item","within":10000,"contains":"Historical bundle"}, + {"do":"click","actor":"staff","testid":"ship-submit","in":{"testid":"queue-item","contains":"Historical bundle"}}, + {"do":"expect","actor":"buyer","testid":"order-status","within":10000,"in":{"testid":"order-item","contains":"Historical bundle"},"statusText":"shipped","ignoreCase":true} + ], + "criteria": [ + { + "id": "742a", + "category": "feature", + "desc": "returning a shipped bundle refunds the paid price and restores original components after its definition changes", + "points": 2, + "steps": [ + {"do":"fill","actor":"admin","testid":"bundle-name-input","text":"Historical bundle"}, + {"do":"fill","actor":"admin","testid":"bundle-price-input","text":"9.00"}, + {"do":"fill","actor":"admin","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":1},{\"item\":\"Desk Lamp\",\"quantity\":3}]"}, + {"do":"click","actor":"admin","testid":"bundle-save"}, + {"do":"expect","actor":"admin","testid":"bundle-card","within":10000,"contains":"Historical bundle"}, + {"do":"click","actor":"buyer","testid":"return-bundle","in":{"testid":"order-item","contains":"Historical bundle"}}, + {"do":"expect","actor":"buyer","testid":"order-status","within":10000,"in":{"testid":"order-item","contains":"Historical bundle"},"statusText":"returned","ignoreCase":true}, + {"do":"expectNumber","actor":"buyer","testid":"bundle-refund-amount","equals":75,"within":10000,"in":{"testid":"order-item","contains":"Historical bundle"}}, + {"do":"dbExpectStock","item":"Keyboard","warehouse":"East","equals":6}, + {"do":"dbExpectStock","item":"Desk Lamp","warehouse":"East","equals":4}, + {"do":"dbExpectStock","item":"Keyboard","warehouse":"West","equals":0}, + {"do":"dbExpectStock","item":"Desk Lamp","warehouse":"West","equals":0} + ] + }, + { + "id": "742b", + "category": "production", + "desc": "replaying a completed bundle return after restart does not refund or restock it twice", + "points": 3, + "steps": [ + {"do":"restartBackend","settleMs":1500}, + {"do":"reload","actor":"buyer","settleMs":1000}, + {"do":"ensureSignedIn","actor":"buyer","name":"Historical bundle","readyTestid":"current-user"}, + {"do":"click","actor":"buyer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"buyer","testid":"orders-toggle"}, + {"do":"callAction","actor":"buyer","action":"returnBundle","namedAction":{"id":"returnBundle","path":"/api/bundle-orders/:orderId/return","reducer":"return_bundle","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":orderId","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Historical bundle","attribute":"data-bundle-return-input"},"settleMs":1000}, + {"do":"reload","actor":"buyer","settleMs":1000}, + {"do":"click","actor":"buyer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"buyer","testid":"orders-toggle"}, + {"do":"expectNumber","actor":"buyer","testid":"bundle-refund-amount","equals":75,"within":10000,"in":{"testid":"order-item","contains":"Historical bundle"}}, + {"do":"dbExpectStock","item":"Keyboard","warehouse":"East","equals":6}, + {"do":"dbExpectStock","item":"Desk Lamp","warehouse":"East","equals":4} + ] + } + ] + }, + { + "id": 748, + "name": "Bundle return ownership", + "actors": [ + "admin", + "buyer", + "staff", + "other" + ], + "setup": [ + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"bundles-link","unlessVisible":"bundle-name-input"}, + {"do":"signUp","actor":"buyer","name":"Private bundle"}, + {"do":"signIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"East","quantity":6,"settleMs":0}, + {"do":"dbSetStock","item":"Keyboard","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"East","quantity":4,"settleMs":0}, + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"fill","actor":"admin","testid":"bundle-name-input","text":"Private bundle"}, + {"do":"fill","actor":"admin","testid":"bundle-price-input","text":"75.00"}, + {"do":"fill","actor":"admin","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"}, + {"do":"click","actor":"admin","testid":"bundle-save"}, + {"do":"expect","actor":"admin","testid":"bundle-card","within":10000,"contains":"Private bundle"}, + {"do":"click","actor":"buyer","testid":"bundles-link","unlessVisible":"bundle-card"}, + {"do":"click","actor":"buyer","testid":"bundle-add-to-cart","in":{"testid":"bundle-card","contains":"Private bundle"}}, + {"do":"click","actor":"buyer","testid":"cart-toggle"}, + {"do":"click","actor":"buyer","testid":"checkout-submit"}, + {"do":"click","actor":"buyer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"buyer","testid":"orders-toggle"}, + {"do":"expect","actor":"buyer","testid":"order-item","within":10000,"contains":"Private bundle","count":1}, + {"do":"expectNumber","actor":"buyer","testid":"payment-amount","equals":75,"within":10000,"in":{"testid":"order-item","contains":"Private bundle"}}, + {"do":"dbExpectStock","item":"Keyboard","warehouse":"East","equals":4}, + {"do":"dbExpectStock","item":"Desk Lamp","warehouse":"East","equals":3}, + {"do":"click","actor":"staff","testid":"staff-link"}, + {"do":"expect","actor":"staff","testid":"queue-item","within":10000,"contains":"Private bundle"}, + {"do":"click","actor":"staff","testid":"ship-submit","in":{"testid":"queue-item","contains":"Private bundle"}}, + {"do":"expect","actor":"buyer","testid":"order-status","within":10000,"in":{"testid":"order-item","contains":"Private bundle"},"statusText":"shipped","ignoreCase":true}, + {"do":"signUp","actor":"other","name":"bundle-other"} + ], + "criteria": [ + { + "id": "748a", + "category": "production", + "desc": "another customer cannot return a paid bundle by submitting its order ID", + "points": 3, + "steps": [ + {"do":"callAction","actor":"other","from":"buyer","action":"returnBundle","namedAction":{"id":"returnBundle","path":"/api/bundle-orders/:orderId/return","reducer":"return_bundle","args":[0],"params":[{"name":"orderId","in":"path","placeholder":":orderId","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Private bundle","attribute":"data-bundle-return-input"},"settleMs":1000}, + {"do":"expectActionOutcome","actor":"other","outcome":"application-refused"}, + {"do":"reload","actor":"buyer","settleMs":1000}, + {"do":"click","actor":"buyer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"buyer","testid":"orders-toggle"}, + {"do":"expect","actor":"buyer","testid":"order-status","within":10000,"in":{"testid":"order-item","contains":"Private bundle"},"statusText":"shipped","ignoreCase":true}, + {"do":"dbExpectStock","item":"Keyboard","warehouse":"East","equals":4}, + {"do":"dbExpectStock","item":"Desk Lamp","warehouse":"East","equals":3}, + {"do":"click","actor":"buyer","testid":"return-bundle","in":{"testid":"order-item","contains":"Private bundle"}}, + {"do":"expectNumber","actor":"buyer","testid":"bundle-refund-amount","equals":75,"within":10000,"in":{"testid":"order-item","contains":"Private bundle"}}, + {"do":"dbExpectStock","item":"Keyboard","warehouse":"East","equals":6}, + {"do":"dbExpectStock","item":"Desk Lamp","warehouse":"East","equals":4} + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-cart-checkout.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-cart-checkout.json new file mode 100644 index 00000000000..f0ec4a1c01a --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-cart-checkout.json @@ -0,0 +1,103 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Cart and checkout", + "note": "Draft: checkout must book every item at its stored price in one order and empty the whole cart. A subsequent two-item request includes a false price; it may be refused with orders and cart unchanged or accepted at stored prices. Independent order-data reads reconcile both outcomes. Stock is checked on the normal UI path only; this does not claim multi-item crash atomicity or independent attack-stock reconciliation. An independent session then adds a second product during checkout. Both calls must succeed; the extra unit belongs either to that order or to the remaining cart, with exact stock reconciliation when warehouse access is part of the delivered task.", + "features": [ + { + "id": 4, + "name": "Account cart and checkout", + "actors": ["quantity", "checkout", "raceCheckout", "raceAdd"], + "setup": [ + { "do": "signUp", "actor": "quantity", "name": "cart-quantity" }, + { "do": "signUp", "actor": "checkout", "name": "cart-checkout" }, + { "do": "click", "actor": "quantity", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "click", "actor": "checkout", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "4a", + "category": "feature", + "desc": "adding the same item twice raises its quantity instead of adding a second line", + "points": 1, + "statedBy": "Adding an existing cart item increases its quantity.", + "steps": [ + { "do": "click", "actor": "quantity", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Headphones" } }, + { "do": "wait", "actor": "quantity", "ms": 800 }, + { "do": "click", "actor": "quantity", "testid": "overlay-close", "ifAvailable": true }, + { "do": "click", "actor": "quantity", "testid": "catalog-link", "ifAvailable": true }, + { "do": "click", "actor": "quantity", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Headphones" } }, + {"do": "click", "actor": "quantity", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["cart-toggle", "cart-total"]}, + { "do": "click", "actor": "quantity", "testid": "cart-toggle", "unlessVisible": "cart-total" }, + { "do": "expect", "actor": "quantity", "testid": "cart-item", "contains": "Headphones", "count": 1 }, + { "do": "expectNumber", "actor": "quantity", "testid": "cart-quantity", "in": { "testid": "cart-item", "contains": "Headphones" }, "equals": 2 } + ] + }, + { + "id": "4d", + "category": "feature", + "desc": "checkout creates one order, reduces stock, and empties the cart", + "points": 2, + "statedBy": "Checkout creates one order, reduces stock, and empties the cart.", + "steps": [ + { "do": "expectNumber", "actor": "checkout", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Desk Lamp" }, "equals": 100 }, + { "do": "dbRecordCheckout", "account": "{user:cart-checkout}", "item": "Desk Lamp", "as": "normal-before", "storage": { "kind": "order-data", "cart": true, "warehouses": false } }, + { "do": "click", "awaitWrites": true, "actor": "checkout", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Desk Lamp" } }, + { "do": "click", "actor": "checkout", "testid": "overlay-close", "ifAvailable": true }, + { "do": "click", "actor": "checkout", "testid": "catalog-link", "ifAvailable": true }, + { "do": "click", "awaitWrites": true, "actor": "checkout", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Coffee Grinder" } }, + {"do": "click", "actor": "checkout", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["cart-toggle", "cart-total"]}, + { "do": "click", "actor": "checkout", "testid": "cart-toggle", "unlessVisible": "cart-total" }, + { "do": "expect", "actor": "checkout", "testid": "cart-item", "count": 2 }, + { "do": "dbRecordCheckout", "account": "{user:cart-checkout}", "item": "Desk Lamp", "as": "normal-prepared", "storage": { "kind": "order-data", "cart": true, "warehouses": false } }, + { "do": "click", "actor": "checkout", "testid": "checkout-submit" }, + { "do": "dbExpectCheckout", "before": "normal-before", "prepared": "normal-prepared", "within": 10000, "quantity": [{ "item": "Desk Lamp", "quantity": 1 }, { "item": "Coffee Grinder", "quantity": 1 }] }, + { "do": "reload", "actor": "checkout", "settleMs": 2500 }, + { "do": "ensureSignedIn", "actor": "checkout", "name": "cart-checkout", "readyTestid": "current-user" }, + { "do": "click", "actor": "checkout", "testid": "overlay-close", "ifAvailable": true, "within": 1000 }, + { "do": "click", "actor": "checkout", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "checkout", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["cart-toggle", "cart-total"]}, + { "do": "click", "actor": "checkout", "testid": "cart-toggle", "unlessVisible": "cart-total" }, + { "do": "expectNumber", "actor": "checkout", "testid": "cart-count", "equals": 0 }, + { "do": "click", "actor": "checkout", "testid": "overlay-close", "ifAvailable": true }, + { "do": "click", "actor": "checkout", "testid": "catalog-link", "ifAvailable": true }, + { "do": "expectNumber", "actor": "checkout", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Desk Lamp" }, "equals": 99 }, + { "do": "expectNumber", "actor": "checkout", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Coffee Grinder" }, "equals": 99 }, + {"do":"click","actor":"checkout","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do": "click", "actor": "checkout", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + { "do": "click", "actor": "checkout", "testid": "orders-toggle", "unlessVisible": "order-item" }, + { "do": "expect", "actor": "checkout", "testid": "order-item", "contains": "Desk Lamp", "count": 1 }, + { "do": "click", "actor": "checkout", "testid": "overlay-close", "ifAvailable": true }, + { "do": "click", "actor": "checkout", "testid": "catalog-link", "ifAvailable": true }, + { "do": "dbRecordCheckout", "account": "{user:cart-checkout}", "item": "Desk Lamp", "as": "claim-before", "storage": { "kind": "order-data", "cart": true, "warehouses": false } }, + { "do": "click", "awaitWrites": true, "actor": "checkout", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Desk Lamp" } }, + { "do": "click", "actor": "checkout", "testid": "overlay-close", "ifAvailable": true }, + { "do": "click", "actor": "checkout", "testid": "catalog-link", "ifAvailable": true }, + { "do": "click", "awaitWrites": true, "actor": "checkout", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Coffee Grinder" } }, + {"do": "click", "actor": "checkout", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["cart-toggle", "cart-total"]}, + { "do": "click", "actor": "checkout", "testid": "cart-toggle", "unlessVisible": "cart-total" }, + { "do": "expect", "actor": "checkout", "testid": "cart-item", "count": 2 }, + { "do": "dbRecordCheckout", "account": "{user:cart-checkout}", "item": "Desk Lamp", "as": "claim-prepared", "storage": { "kind": "order-data", "cart": true, "warehouses": false } }, + { "do": "callAction", "actor": "checkout", "action": "checkout", "namedAction": { "id": "checkout", "path": "/api/checkout", "method": "POST", "reducer": "checkout", "args": [1], "params": [{ "name": "price", "in": "body" }] }, "settleMs": 0 }, + { "do": "expectActionOutcome", "actor": "checkout", "outcome": "completed" }, + { "do": "dbExpectCheckout", "actor": "checkout", "before": "claim-before", "prepared": "claim-prepared", "quantity": [{ "item": "Desk Lamp", "quantity": 1 }, { "item": "Coffee Grinder", "quantity": 1 }] }, + {"do":"signUp","actor":"raceCheckout","name":"checkout-overlap"}, + {"do":"signIn","actor":"raceAdd","name":"checkout-overlap"}, + {"do":"dbRecordCheckout","account":"{user:checkout-overlap}","item":"Desk Lamp","as":"overlap-before","storage":{"kind":"order-data","cart":true,"warehouses":"if-requested"}}, + { "do": "click", "actor": "raceCheckout", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do":"click", "awaitWrites": true,"actor":"raceCheckout","testid":"add-to-cart","in":{"testid":"item-card","contains":"Desk Lamp"}}, + {"do": "click", "actor": "raceCheckout", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["cart-toggle", "cart-total"]}, + {"do":"click","actor":"raceCheckout","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expect","actor":"raceCheckout","testid":"cart-item","contains":"Desk Lamp","count":1}, + {"do":"dbRecordCheckout","account":"{user:checkout-overlap}","item":"Desk Lamp","as":"overlap-prepared","storage":{"kind":"order-data","cart":true,"warehouses":"if-requested"}}, + { "do": "click", "actor": "raceAdd", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do":"callConcurrently","actors":["raceCheckout"],"action":"checkout","namedAction":{"id":"checkout","path":"/api/checkout","method":"POST","reducer":"checkout","args":[]},"requests":1,"settleMs":0,"alongside":[{"actors":["raceAdd"],"action":"cart-add","namedAction":{"id":"cart-add","path":"/api/cart","method":"POST","reducer":"add_to_cart","args":[0],"params":[{"name":"itemId","in":"body","wireType":"u64"}]},"requests":1,"input":{"testid":"item-card","contains":"Coffee Grinder","attribute":"data-buy-input"}}]}, + {"do":"expectCallOutcomes","accepted":2}, + {"do":"dbExpectCheckout","before":"overlap-before","prepared":"overlap-prepared","quantity":[{"item":"Desk Lamp","quantity":1}],"alongsideAdd":"Coffee Grinder"} + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-cart-recovery.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-cart-recovery.json new file mode 100644 index 00000000000..44d53d9fd6e --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-cart-recovery.json @@ -0,0 +1,78 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 5, + "name": "Cart recovery", + "features": [ + { + "id": 503, + "name": "Expired carts restore only available items", + "actors": [ + "available-shopper", + "partial-shopper" + ], + "setup": [ + {"do":"signUp","actor":"available-shopper","name":"cart-recovery-available"}, + {"do":"click","actor":"available-shopper","testid":"add-to-cart","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"signUp","actor":"partial-shopper","name":"cart-recovery-partial"}, + {"do":"click","actor":"partial-shopper","testid":"add-to-cart","in":{"testid":"item-card","contains":"Gaming Mouse"}}, + {"do":"click","actor":"partial-shopper","testid":"add-to-cart","in":{"testid":"item-card","contains":"Desk Lamp"}}, + {"do":"wait","actor":"available-shopper","ms":310000} + ], + "criteria": [ + { + "id": "503a", + "category": "feature", + "desc": "restoring an expired cart reserves available items again", + "points": 3, + "statedBy": "A customer can restore the available items from an expired cart.", + "steps": [ + {"do":"reload","actor":"available-shopper","settleMs":1000}, + {"do":"ensureSignedIn","actor":"available-shopper","name":"cart-recovery-available","readyTestid":"current-user"}, + {"do":"expect","actor":"available-shopper","testid":"expired-cart"}, + {"do":"expectNumber","actor":"available-shopper","testid":"cart-count","equals":0}, + {"do":"recordNumber","actor":"available-shopper","testid":"item-stock","in":{"testid":"item-card","contains":"Keyboard"},"as":"restore-available-shopper"}, + {"do":"click","actor":"available-shopper","testid":"restore-cart","in":{"testid":"expired-cart"}}, + {"do":"click","actor":"available-shopper","testid":"cart-toggle"}, + {"do":"expect","actor":"available-shopper","testid":"cart-item","contains":"Keyboard"}, + {"do":"expect","actor":"available-shopper","testid":"cart-restore-warning","absent":true}, + {"do":"expectNumber","actor":"available-shopper","testid":"cart-quantity","in":{"testid":"cart-item","contains":"Keyboard"},"equals":1}, + {"do":"reload","actor":"available-shopper","settleMs":1000}, + {"do":"ensureSignedIn","actor":"available-shopper","name":"cart-recovery-available","readyTestid":"current-user"}, + {"do":"click","actor":"available-shopper","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"available-shopper","testid":"catalog-link","unlessVisible":"item-card"}, + {"do":"expectNumber","actor":"available-shopper","testid":"item-stock","in":{"testid":"item-card","contains":"Keyboard"},"relativeTo":"restore-available-shopper","plus":-1} + ] + }, + { + "id": "503b", + "category": "feature", + "desc": "a partial restore keeps available items and names each unavailable item", + "points": 3, + "statedBy": "Restore the available items and list each item that could not be restored.", + "steps": [ + {"do":"reload","actor":"partial-shopper","settleMs":1000}, + {"do":"ensureSignedIn","actor":"partial-shopper","name":"cart-recovery-partial","readyTestid":"current-user"}, + {"do":"expect","actor":"partial-shopper","testid":"expired-cart"}, + {"do":"expectNumber","actor":"partial-shopper","testid":"cart-count","equals":0}, + {"do":"recordNumber","actor":"partial-shopper","testid":"item-stock","in":{"testid":"item-card","contains":"Gaming Mouse"},"as":"restore-partial-shopper"}, + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"East","quantity":0,"settleMs":2000}, + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"West","quantity":0,"settleMs":2000}, + {"do":"click","actor":"partial-shopper","testid":"restore-cart","in":{"testid":"expired-cart"}}, + {"do":"click","actor":"partial-shopper","testid":"cart-toggle"}, + {"do":"expect","actor":"partial-shopper","testid":"cart-item","contains":"Gaming Mouse"}, + {"do":"expect","actor":"partial-shopper","testid":"cart-item","contains":"Desk Lamp","absent":true}, + {"do":"expect","actor":"partial-shopper","testid":"cart-restore-warning","contains":"Desk Lamp"}, + {"do":"expectNumber","actor":"partial-shopper","testid":"cart-quantity","in":{"testid":"cart-item","contains":"Gaming Mouse"},"equals":1}, + {"do":"reload","actor":"partial-shopper","settleMs":1000}, + {"do":"ensureSignedIn","actor":"partial-shopper","name":"cart-recovery-partial","readyTestid":"current-user"}, + {"do":"click","actor":"partial-shopper","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"partial-shopper","testid":"catalog-link","unlessVisible":"item-card"}, + {"do":"expectNumber","actor":"partial-shopper","testid":"item-stock","in":{"testid":"item-card","contains":"Gaming Mouse"},"relativeTo":"restore-partial-shopper","plus":-1} + ] + } + ] + } + ], + "note": "Draft: revised observations require matching reference and defect-control qualification." +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-catalog-management.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-catalog-management.json new file mode 100644 index 00000000000..3d5d695ac75 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-catalog-management.json @@ -0,0 +1,52 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 5, + "name": "Catalog management", + "features": [ + { + "id": 622, + "name": "Catalog management", + "actors": ["admin", "visitor"], + "setup": [ + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + {"do":"click","actor":"admin","testid":"current-user","ifAvailable":true,"unlessVisible":["admin-link","catalog-management-link","catalog-name"]}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":["catalog-management-link","catalog-name"]}, + {"do":"click","actor":"admin","testid":"catalog-management-link","ifAvailable":true,"unlessVisible":"catalog-name"}, + { "do": "fill", "actor": "admin", "testid": "catalog-name", "text": "Travel Mug" }, + { "do": "fill", "actor": "admin", "testid": "catalog-category", "text": "Kitchen" }, + { "do": "fill", "actor": "admin", "testid": "catalog-price", "text": "24.00" }, + { "do": "fill", "actor": "admin", "testid": "catalog-variants", "text": "Black, Silver" }, + { "do": "click", "awaitWrites": true, "actor": "admin", "testid": "catalog-save" }, + { "do": "reload", "actor": "visitor", "settleMs": 2500 }, + { "do": "click", "actor": "visitor", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "search-input" }, + { "do": "fill", "actor": "visitor", "testid": "search-input", "text": "Travel Mug", "enter": true } + ], + "criteria": [ + { + "id": "622a", + "category": "feature", + "desc": "a new product reaches the public catalog", + "points": 2, + "statedBy": "Authorized staff can add products.", + "steps": [ + { "do": "expect", "actor": "visitor", "testid": "item-card", "contains": "Travel Mug", "within": 10000 } + ] + }, + { + "id": "622b", + "category": "feature", + "desc": "the product exposes its named variants", + "points": 2, + "statedBy": "Products can have named variants.", + "steps": [ + { "do": "expect", "actor": "visitor", "testid": "item-card", "contains": "Travel Mug", "within": 10000 }, + { "do": "openItem", "actor": "visitor", "item": "Travel Mug", "unlessVisible": "item-variant" }, + { "do": "expectElementCount", "actor": "visitor", "testid": "item-variant", "contains": "Black", "equals": 1, "within": 10000 }, + { "do": "expectElementCount", "actor": "visitor", "testid": "item-variant", "contains": "Silver", "equals": 1, "within": 10000 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-catalog-volume.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-catalog-volume.json new file mode 100644 index 00000000000..bb598d20837 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-catalog-volume.json @@ -0,0 +1,114 @@ +{ + "schemaVersion": 1, "track": "ecommerce", "level": 3, + "name": "Created catalog records remain reachable", + "features": [{ + "id": 622, "name": "Catalog management", "actors": ["admin","visitor"], + "setup": [ + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"current-user","ifAvailable":true,"unlessVisible":["admin-link","catalog-management-link","catalog-name"]}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":["catalog-management-link","catalog-name"]}, + {"do":"click","actor":"admin","testid":"catalog-management-link","ifAvailable":true,"unlessVisible":"catalog-name"} + ], + "criteria": [{ + "id": "622c", + "category": "production", + "desc": "All 300 committed products remain reachable once each across sorted result pages", + "points": 1, + "statedBy": "Authorized staff can add products; moving between pages must not omit or repeat an item.", + "note": "Draft setup optimization: compare two confirmed UI writes; replay only when they differ solely in product name. Otherwise use the form. Keep the final creation through UI, every independent committed-write check and every page assertion. This check measures completeness, not throughput. Affected qualification is pending.", + "steps": [ + {"do":"dbRecordCheckout","account":"admin","item":"Bluetooth Speaker","as":"catalog-before","storage":{"kind":"order-data","cart":false,"warehouses":false}}, + {"do":"fill","actor":"admin","testid":"catalog-name","text":"Volume Item 0299"}, + {"do":"fill","actor":"admin","testid":"catalog-category","text":"Volume"}, + {"do":"fill","actor":"admin","testid":"catalog-price","text":"1.25"}, + {"do":"fill","actor":"admin","testid":"catalog-variants","text":"Standard"}, + {"do":"click","actor":"admin","testid":"catalog-save","settleMs":100}, + {"do":"dbExpectCatalogItem","before":"catalog-before","name":"Volume Item 0299","priceMinor":125,"within":10000}, + {"do":"click","actor":"admin","testid":"current-user","ifAvailable":true,"unlessVisible":["admin-link","catalog-management-link","catalog-name"]}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":["catalog-management-link","catalog-name"]}, + {"do":"click","actor":"admin","testid":"catalog-management-link","ifAvailable":true,"unlessVisible":"catalog-name"}, + {"do":"fill","actor":"admin","testid":"catalog-name","text":"Volume Item 0298"}, + {"do":"fill","actor":"admin","testid":"catalog-category","text":"Volume"}, + {"do":"fill","actor":"admin","testid":"catalog-price","text":"1.25"}, + {"do":"fill","actor":"admin","testid":"catalog-variants","text":"Standard"}, + {"do":"click","actor":"admin","testid":"catalog-save","settleMs":100}, + {"do":"dbExpectCatalogItem","before":"catalog-before","name":"Volume Item 0298","priceMinor":125,"within":10000}, + {"forEach":["Volume Item 0297","Volume Item 0296","Volume Item 0295","Volume Item 0294","Volume Item 0293","Volume Item 0292","Volume Item 0291","Volume Item 0290","Volume Item 0289","Volume Item 0288","Volume Item 0287","Volume Item 0286","Volume Item 0285","Volume Item 0284","Volume Item 0283","Volume Item 0282","Volume Item 0281","Volume Item 0280","Volume Item 0279","Volume Item 0278","Volume Item 0277","Volume Item 0276","Volume Item 0275","Volume Item 0274","Volume Item 0273","Volume Item 0272","Volume Item 0271","Volume Item 0270","Volume Item 0269","Volume Item 0268","Volume Item 0267","Volume Item 0266","Volume Item 0265","Volume Item 0264","Volume Item 0263","Volume Item 0262","Volume Item 0261","Volume Item 0260","Volume Item 0259","Volume Item 0258","Volume Item 0257","Volume Item 0256","Volume Item 0255","Volume Item 0254","Volume Item 0253","Volume Item 0252","Volume Item 0251","Volume Item 0250","Volume Item 0249","Volume Item 0248","Volume Item 0247","Volume Item 0246","Volume Item 0245","Volume Item 0244","Volume Item 0243","Volume Item 0242","Volume Item 0241","Volume Item 0240","Volume Item 0239","Volume Item 0238","Volume Item 0237","Volume Item 0236","Volume Item 0235","Volume Item 0234","Volume Item 0233","Volume Item 0232","Volume Item 0231","Volume Item 0230","Volume Item 0229","Volume Item 0228","Volume Item 0227","Volume Item 0226","Volume Item 0225","Volume Item 0224","Volume Item 0223","Volume Item 0222","Volume Item 0221","Volume Item 0220","Volume Item 0219","Volume Item 0218","Volume Item 0217","Volume Item 0216","Volume Item 0215","Volume Item 0214","Volume Item 0213","Volume Item 0212","Volume Item 0211","Volume Item 0210","Volume Item 0209","Volume Item 0208","Volume Item 0207","Volume Item 0206","Volume Item 0205","Volume Item 0204","Volume Item 0203","Volume Item 0202","Volume Item 0201","Volume Item 0200","Volume Item 0199","Volume Item 0198","Volume Item 0197","Volume Item 0196","Volume Item 0195","Volume Item 0194","Volume Item 0193","Volume Item 0192","Volume Item 0191","Volume Item 0190","Volume Item 0189","Volume Item 0188","Volume Item 0187","Volume Item 0186","Volume Item 0185","Volume Item 0184","Volume Item 0183","Volume Item 0182","Volume Item 0181","Volume Item 0180","Volume Item 0179","Volume Item 0178","Volume Item 0177","Volume Item 0176","Volume Item 0175","Volume Item 0174","Volume Item 0173","Volume Item 0172","Volume Item 0171","Volume Item 0170","Volume Item 0169","Volume Item 0168","Volume Item 0167","Volume Item 0166","Volume Item 0165","Volume Item 0164","Volume Item 0163","Volume Item 0162","Volume Item 0161","Volume Item 0160","Volume Item 0159","Volume Item 0158","Volume Item 0157","Volume Item 0156","Volume Item 0155","Volume Item 0154","Volume Item 0153","Volume Item 0152","Volume Item 0151","Volume Item 0150","Volume Item 0149","Volume Item 0148","Volume Item 0147","Volume Item 0146","Volume Item 0145","Volume Item 0144","Volume Item 0143","Volume Item 0142","Volume Item 0141","Volume Item 0140","Volume Item 0139","Volume Item 0138","Volume Item 0137","Volume Item 0136","Volume Item 0135","Volume Item 0134","Volume Item 0133","Volume Item 0132","Volume Item 0131","Volume Item 0130","Volume Item 0129","Volume Item 0128","Volume Item 0127","Volume Item 0126","Volume Item 0125","Volume Item 0124","Volume Item 0123","Volume Item 0122","Volume Item 0121","Volume Item 0120","Volume Item 0119","Volume Item 0118","Volume Item 0117","Volume Item 0116","Volume Item 0115","Volume Item 0114","Volume Item 0113","Volume Item 0112","Volume Item 0111","Volume Item 0110","Volume Item 0109","Volume Item 0108","Volume Item 0107","Volume Item 0106","Volume Item 0105","Volume Item 0104","Volume Item 0103","Volume Item 0102","Volume Item 0101","Volume Item 0100","Volume Item 0099","Volume Item 0098","Volume Item 0097","Volume Item 0096","Volume Item 0095","Volume Item 0094","Volume Item 0093","Volume Item 0092","Volume Item 0091","Volume Item 0090","Volume Item 0089","Volume Item 0088","Volume Item 0087","Volume Item 0086","Volume Item 0085","Volume Item 0084","Volume Item 0083","Volume Item 0082","Volume Item 0081","Volume Item 0080","Volume Item 0079","Volume Item 0078","Volume Item 0077","Volume Item 0076","Volume Item 0075","Volume Item 0074","Volume Item 0073","Volume Item 0072","Volume Item 0071","Volume Item 0070","Volume Item 0069","Volume Item 0068","Volume Item 0067","Volume Item 0066","Volume Item 0065","Volume Item 0064","Volume Item 0063","Volume Item 0062","Volume Item 0061","Volume Item 0060","Volume Item 0059","Volume Item 0058","Volume Item 0057","Volume Item 0056","Volume Item 0055","Volume Item 0054","Volume Item 0053","Volume Item 0052","Volume Item 0051","Volume Item 0050","Volume Item 0049","Volume Item 0048","Volume Item 0047","Volume Item 0046","Volume Item 0045","Volume Item 0044","Volume Item 0043","Volume Item 0042","Volume Item 0041","Volume Item 0040","Volume Item 0039","Volume Item 0038","Volume Item 0037","Volume Item 0036","Volume Item 0035","Volume Item 0034","Volume Item 0033","Volume Item 0032","Volume Item 0031","Volume Item 0030","Volume Item 0029","Volume Item 0028","Volume Item 0027","Volume Item 0026","Volume Item 0025","Volume Item 0024","Volume Item 0023","Volume Item 0022","Volume Item 0021","Volume Item 0020","Volume Item 0019","Volume Item 0018","Volume Item 0017","Volume Item 0016","Volume Item 0015","Volume Item 0014","Volume Item 0013","Volume Item 0012","Volume Item 0011","Volume Item 0010","Volume Item 0009","Volume Item 0008","Volume Item 0007","Volume Item 0006","Volume Item 0005","Volume Item 0004","Volume Item 0003","Volume Item 0002","Volume Item 0001"],"steps":[{"do":"click","actor":"admin","testid":"current-user","ifAvailable":true,"unlessVisible":["admin-link","catalog-management-link","catalog-name"]},{"do":"click","actor":"admin","testid":"admin-link","unlessVisible":["catalog-management-link","catalog-name"]},{"do":"click","actor":"admin","testid":"catalog-management-link","ifAvailable":true,"unlessVisible":"catalog-name"},{"do":"repeatFormWrite","actor":"admin","match":"Volume Item 0299","control":"Volume Item 0298","replacement":"{value}","fields":[{"testid":"catalog-name","text":"{value}"},{"testid":"catalog-category","text":"Volume"},{"testid":"catalog-price","text":"1.25"},{"testid":"catalog-variants","text":"Standard"}],"submit":"catalog-save","settleMs":100},{"do":"dbExpectCatalogItem","before":"catalog-before","name":"{value}","priceMinor":125,"within":10000}]}, + {"do":"click","actor":"admin","testid":"current-user","ifAvailable":true,"unlessVisible":["admin-link","catalog-management-link","catalog-name"]}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":["catalog-management-link","catalog-name"]}, + {"do":"click","actor":"admin","testid":"catalog-management-link","ifAvailable":true,"unlessVisible":"catalog-name"}, + {"do":"fill","actor":"admin","testid":"catalog-name","text":"Volume Item 0000"}, + {"do":"fill","actor":"admin","testid":"catalog-category","text":"Volume"}, + {"do":"fill","actor":"admin","testid":"catalog-price","text":"1.25"}, + {"do":"fill","actor":"admin","testid":"catalog-variants","text":"Standard"}, + {"do":"click","actor":"admin","testid":"catalog-save","settleMs":100}, + {"do":"dbExpectCatalogItem","before":"catalog-before","name":"Volume Item 0000","priceMinor":125,"within":10000}, + {"do":"reload","actor":"visitor","settleMs":500}, + {"do":"click","actor":"visitor","testid":"catalog-link","ifAvailable":true,"unlessVisible":"search-input"}, + {"do":"fill","actor":"visitor","testid":"search-input","text":"Volume Item","enter":true}, + {"do":"dbRecordCheckout","account":"admin","item":"Bluetooth Speaker","as":"seeded-catalog","storage":{"kind":"order-data","cart":false,"warehouses":false}}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0000","Volume Item 0001","Volume Item 0002","Volume Item 0003","Volume Item 0004","Volume Item 0005","Volume Item 0006","Volume Item 0007","Volume Item 0008","Volume Item 0009"]}, + {"do":"click","actor":"visitor","testid":"search-next-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0010","Volume Item 0011","Volume Item 0012","Volume Item 0013","Volume Item 0014","Volume Item 0015","Volume Item 0016","Volume Item 0017","Volume Item 0018","Volume Item 0019"]}, + {"do":"click","actor":"visitor","testid":"search-next-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0020","Volume Item 0021","Volume Item 0022","Volume Item 0023","Volume Item 0024","Volume Item 0025","Volume Item 0026","Volume Item 0027","Volume Item 0028","Volume Item 0029"]}, + {"do":"click","actor":"visitor","testid":"search-next-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0030","Volume Item 0031","Volume Item 0032","Volume Item 0033","Volume Item 0034","Volume Item 0035","Volume Item 0036","Volume Item 0037","Volume Item 0038","Volume Item 0039"]}, + {"do":"click","actor":"visitor","testid":"search-next-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0040","Volume Item 0041","Volume Item 0042","Volume Item 0043","Volume Item 0044","Volume Item 0045","Volume Item 0046","Volume Item 0047","Volume Item 0048","Volume Item 0049"]}, + {"do":"click","actor":"visitor","testid":"search-next-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0050","Volume Item 0051","Volume Item 0052","Volume Item 0053","Volume Item 0054","Volume Item 0055","Volume Item 0056","Volume Item 0057","Volume Item 0058","Volume Item 0059"]}, + {"do":"click","actor":"visitor","testid":"search-next-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0060","Volume Item 0061","Volume Item 0062","Volume Item 0063","Volume Item 0064","Volume Item 0065","Volume Item 0066","Volume Item 0067","Volume Item 0068","Volume Item 0069"]}, + {"do":"click","actor":"visitor","testid":"search-next-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0070","Volume Item 0071","Volume Item 0072","Volume Item 0073","Volume Item 0074","Volume Item 0075","Volume Item 0076","Volume Item 0077","Volume Item 0078","Volume Item 0079"]}, + {"do":"click","actor":"visitor","testid":"search-next-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0080","Volume Item 0081","Volume Item 0082","Volume Item 0083","Volume Item 0084","Volume Item 0085","Volume Item 0086","Volume Item 0087","Volume Item 0088","Volume Item 0089"]}, + {"do":"click","actor":"visitor","testid":"search-next-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0090","Volume Item 0091","Volume Item 0092","Volume Item 0093","Volume Item 0094","Volume Item 0095","Volume Item 0096","Volume Item 0097","Volume Item 0098","Volume Item 0099"]}, + {"do":"click","actor":"visitor","testid":"search-next-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0100","Volume Item 0101","Volume Item 0102","Volume Item 0103","Volume Item 0104","Volume Item 0105","Volume Item 0106","Volume Item 0107","Volume Item 0108","Volume Item 0109"]}, + {"do":"click","actor":"visitor","testid":"search-next-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0110","Volume Item 0111","Volume Item 0112","Volume Item 0113","Volume Item 0114","Volume Item 0115","Volume Item 0116","Volume Item 0117","Volume Item 0118","Volume Item 0119"]}, + {"do":"click","actor":"visitor","testid":"search-next-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0120","Volume Item 0121","Volume Item 0122","Volume Item 0123","Volume Item 0124","Volume Item 0125","Volume Item 0126","Volume Item 0127","Volume Item 0128","Volume Item 0129"]}, + {"do":"click","actor":"visitor","testid":"search-next-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0130","Volume Item 0131","Volume Item 0132","Volume Item 0133","Volume Item 0134","Volume Item 0135","Volume Item 0136","Volume Item 0137","Volume Item 0138","Volume Item 0139"]}, + {"do":"click","actor":"visitor","testid":"search-next-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0140","Volume Item 0141","Volume Item 0142","Volume Item 0143","Volume Item 0144","Volume Item 0145","Volume Item 0146","Volume Item 0147","Volume Item 0148","Volume Item 0149"]}, + {"do":"click","actor":"visitor","testid":"search-next-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0150","Volume Item 0151","Volume Item 0152","Volume Item 0153","Volume Item 0154","Volume Item 0155","Volume Item 0156","Volume Item 0157","Volume Item 0158","Volume Item 0159"]}, + {"do":"click","actor":"visitor","testid":"search-next-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0160","Volume Item 0161","Volume Item 0162","Volume Item 0163","Volume Item 0164","Volume Item 0165","Volume Item 0166","Volume Item 0167","Volume Item 0168","Volume Item 0169"]}, + {"do":"click","actor":"visitor","testid":"search-next-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0170","Volume Item 0171","Volume Item 0172","Volume Item 0173","Volume Item 0174","Volume Item 0175","Volume Item 0176","Volume Item 0177","Volume Item 0178","Volume Item 0179"]}, + {"do":"click","actor":"visitor","testid":"search-next-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0180","Volume Item 0181","Volume Item 0182","Volume Item 0183","Volume Item 0184","Volume Item 0185","Volume Item 0186","Volume Item 0187","Volume Item 0188","Volume Item 0189"]}, + {"do":"click","actor":"visitor","testid":"search-next-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0190","Volume Item 0191","Volume Item 0192","Volume Item 0193","Volume Item 0194","Volume Item 0195","Volume Item 0196","Volume Item 0197","Volume Item 0198","Volume Item 0199"]}, + {"do":"click","actor":"visitor","testid":"search-next-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0200","Volume Item 0201","Volume Item 0202","Volume Item 0203","Volume Item 0204","Volume Item 0205","Volume Item 0206","Volume Item 0207","Volume Item 0208","Volume Item 0209"]}, + {"do":"click","actor":"visitor","testid":"search-next-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0210","Volume Item 0211","Volume Item 0212","Volume Item 0213","Volume Item 0214","Volume Item 0215","Volume Item 0216","Volume Item 0217","Volume Item 0218","Volume Item 0219"]}, + {"do":"click","actor":"visitor","testid":"search-next-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0220","Volume Item 0221","Volume Item 0222","Volume Item 0223","Volume Item 0224","Volume Item 0225","Volume Item 0226","Volume Item 0227","Volume Item 0228","Volume Item 0229"]}, + {"do":"click","actor":"visitor","testid":"search-next-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0230","Volume Item 0231","Volume Item 0232","Volume Item 0233","Volume Item 0234","Volume Item 0235","Volume Item 0236","Volume Item 0237","Volume Item 0238","Volume Item 0239"]}, + {"do":"click","actor":"visitor","testid":"search-next-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0240","Volume Item 0241","Volume Item 0242","Volume Item 0243","Volume Item 0244","Volume Item 0245","Volume Item 0246","Volume Item 0247","Volume Item 0248","Volume Item 0249"]}, + {"do":"click","actor":"visitor","testid":"search-next-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0250","Volume Item 0251","Volume Item 0252","Volume Item 0253","Volume Item 0254","Volume Item 0255","Volume Item 0256","Volume Item 0257","Volume Item 0258","Volume Item 0259"]}, + {"do":"click","actor":"visitor","testid":"search-next-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0260","Volume Item 0261","Volume Item 0262","Volume Item 0263","Volume Item 0264","Volume Item 0265","Volume Item 0266","Volume Item 0267","Volume Item 0268","Volume Item 0269"]}, + {"do":"click","actor":"visitor","testid":"search-next-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0270","Volume Item 0271","Volume Item 0272","Volume Item 0273","Volume Item 0274","Volume Item 0275","Volume Item 0276","Volume Item 0277","Volume Item 0278","Volume Item 0279"]}, + {"do":"click","actor":"visitor","testid":"search-next-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0280","Volume Item 0281","Volume Item 0282","Volume Item 0283","Volume Item 0284","Volume Item 0285","Volume Item 0286","Volume Item 0287","Volume Item 0288","Volume Item 0289"]}, + {"do":"click","actor":"visitor","testid":"search-next-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0290","Volume Item 0291","Volume Item 0292","Volume Item 0293","Volume Item 0294","Volume Item 0295","Volume Item 0296","Volume Item 0297","Volume Item 0298","Volume Item 0299"]}, + {"do":"click","actor":"visitor","testid":"search-previous-page"}, + {"do":"expectSequence","actor":"visitor","testid":"item-name","in":{"testid":"search-results"},"equals":["Volume Item 0280","Volume Item 0281","Volume Item 0282","Volume Item 0283","Volume Item 0284","Volume Item 0285","Volume Item 0286","Volume Item 0287","Volume Item 0288","Volume Item 0289"]} + ] + }] + }] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-checkout-crash.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-checkout-crash.json new file mode 100644 index 00000000000..28bd5b00624 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-checkout-crash.json @@ -0,0 +1,619 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Checkout recovery after process crashes", + "note": "One fixed 16-call, zero-offset trial per independent process boundary. Database then application; combined SpacetimeDB and Convex boundaries reuse their measured database crash once. Both assertions use both retained observations. Unknown outcomes, missed windows and incomplete recovery remain unmeasured. Each baseline, interrupted cart and fresh-customer progress checkout contains Keyboard and Coffee Grinder. The test observes recovered order/cart correctness across both products, including stock/reservation/allocation conservation when the delivered task declares warehouse access. It does not test physical rows, optional payment features, power loss or storage-device durability. After each retained crash observation, remove any remaining cart lines through the declared UI before the next baseline. This prevents earlier reservations expiring into a later trial; it never changes the saved fault verdict.", + "features": [ + { + "id": 910, + "name": "Checkout crash recovery", + "actors": [ + "a", + "b", + "c" + ], + "setup": [ + { + "do": "signUp", + "actor": "a", + "name": "database" + }, + { + "do": "dbRecordCheckout", + "account": "{user:database}", + "item": "Keyboard", + "as": "database-baseline-before", + "storage": { + "kind": "order-data", + "cart": true, + "warehouses": "if-requested" + } + }, + { "do": "click", "actor": "a", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { + "do": "click", + "awaitWrites": true, + "actor": "a", + "testid": "add-to-cart", + "in": { + "testid": "item-card", + "contains": "Keyboard" + } + }, + { + "do": "click", + "actor": "a", + "testid": "overlay-close", + "ifAvailable": true, + "within": 1000 + }, + { + "do": "click", + "actor": "a", + "testid": "catalog-link", + "ifAvailable": true, + "within": 1000 + }, + { + "do": "click", + "awaitWrites": true, + "actor": "a", + "testid": "add-to-cart", + "in": { + "testid": "item-card", + "contains": "Coffee Grinder" + } + }, + { + "do": "dbRecordCheckout", + "account": "{user:database}", + "item": "Keyboard", + "as": "database-baseline-prepared", + "storage": { + "kind": "order-data", + "cart": true, + "warehouses": "if-requested" + } + }, + { + "do": "confirmCheckout", + "actor": "a", + "namedAction": { + "id": "checkout", + "reducer": "checkout", + "path": "/api/checkout", + "args": [] + } + }, + { + "do": "dbExpectCheckout", + "before": "database-baseline-before", + "prepared": "database-baseline-prepared", + "quantity": [ + { + "item": "Keyboard", + "quantity": 1 + }, + { + "item": "Coffee Grinder", + "quantity": 1 + } + ] + }, + { + "do": "reload", + "actor": "a", + "settleMs": 0 + }, + { + "do": "dbRecordCheckout", + "account": "{user:database}", + "item": "Keyboard", + "as": "database-before", + "storage": { + "kind": "order-data", + "cart": true, + "warehouses": "if-requested" + } + }, + { "do": "click", "actor": "a", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { + "do": "click", + "awaitWrites": true, + "actor": "a", + "testid": "add-to-cart", + "in": { + "testid": "item-card", + "contains": "Keyboard" + } + }, + { + "do": "click", + "actor": "a", + "testid": "overlay-close", + "ifAvailable": true, + "within": 1000 + }, + { + "do": "click", + "actor": "a", + "testid": "catalog-link", + "ifAvailable": true, + "within": 1000 + }, + { + "do": "click", + "awaitWrites": true, + "actor": "a", + "testid": "add-to-cart", + "in": { + "testid": "item-card", + "contains": "Coffee Grinder" + } + }, + { + "do": "dbRecordCheckout", + "account": "{user:database}", + "item": "Keyboard", + "as": "database-prepared", + "storage": { + "kind": "order-data", + "cart": true, + "warehouses": "if-requested" + } + }, + { + "do": "crashCheckout", + "actor": "a", + "before": "database-before", + "prepared": "database-prepared", + "quantity": [ + { + "item": "Keyboard", + "quantity": 1 + }, + { + "item": "Coffee Grinder", + "quantity": 1 + } + ], + "requests": 16, + "offsetMs": 0, + "target": "database", + "as": "database-crash-observation", + "namedAction": { + "id": "checkout", + "reducer": "checkout", + "path": "/api/checkout", + "args": [] + } + }, + { + "do": "reload", + "actor": "a", + "settleMs": 0 + }, + { + "do": "ensureSignedIn", + "actor": "a", + "name": "database", + "readyTestid": "current-user" + }, + { + "do": "click", + "actor": "a", + "testid": "overlay-close", + "ifAvailable": true, + "within": 1000 + }, + { "do": "click", "actor": "a", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "a", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["cart-toggle", "cart-total"]}, + { + "do": "click", + "actor": "a", + "testid": "cart-toggle", + "unlessVisible": "cart-total" + }, + { + "do": "click", + "awaitWrites": true, + "actor": "a", + "testid": "cart-remove", + "in": { + "testid": "cart-item", + "contains": "Keyboard" + }, + "ifAvailable": true, + "within": 1000 + }, + { + "do": "click", + "awaitWrites": true, + "actor": "a", + "testid": "cart-remove", + "in": { + "testid": "cart-item", + "contains": "Coffee Grinder" + }, + "ifAvailable": true, + "within": 1000 + }, + { + "do": "expectNumber", + "actor": "a", + "testid": "cart-count", + "equals": 0, + "within": 10000 + }, + { + "do": "signUp", + "actor": "b", + "name": "application" + }, + { + "do": "dbRecordCheckout", + "account": "{user:application}", + "item": "Keyboard", + "as": "application-baseline-before", + "storage": { + "kind": "order-data", + "cart": true, + "warehouses": "if-requested" + } + }, + { "do": "click", "actor": "b", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { + "do": "click", + "awaitWrites": true, + "actor": "b", + "testid": "add-to-cart", + "in": { + "testid": "item-card", + "contains": "Keyboard" + } + }, + { + "do": "click", + "actor": "b", + "testid": "overlay-close", + "ifAvailable": true, + "within": 1000 + }, + { + "do": "click", + "actor": "b", + "testid": "catalog-link", + "ifAvailable": true, + "within": 1000 + }, + { + "do": "click", + "awaitWrites": true, + "actor": "b", + "testid": "add-to-cart", + "in": { + "testid": "item-card", + "contains": "Coffee Grinder" + } + }, + { + "do": "dbRecordCheckout", + "account": "{user:application}", + "item": "Keyboard", + "as": "application-baseline-prepared", + "storage": { + "kind": "order-data", + "cart": true, + "warehouses": "if-requested" + } + }, + { + "do": "confirmCheckout", + "actor": "b", + "namedAction": { + "id": "checkout", + "reducer": "checkout", + "path": "/api/checkout", + "args": [] + } + }, + { + "do": "dbExpectCheckout", + "before": "application-baseline-before", + "prepared": "application-baseline-prepared", + "quantity": [ + { + "item": "Keyboard", + "quantity": 1 + }, + { + "item": "Coffee Grinder", + "quantity": 1 + } + ] + }, + { + "do": "reload", + "actor": "b", + "settleMs": 0 + }, + { + "do": "dbRecordCheckout", + "account": "{user:application}", + "item": "Keyboard", + "as": "application-before", + "storage": { + "kind": "order-data", + "cart": true, + "warehouses": "if-requested" + } + }, + { "do": "click", "actor": "b", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { + "do": "click", + "awaitWrites": true, + "actor": "b", + "testid": "add-to-cart", + "in": { + "testid": "item-card", + "contains": "Keyboard" + } + }, + { + "do": "click", + "actor": "b", + "testid": "overlay-close", + "ifAvailable": true, + "within": 1000 + }, + { + "do": "click", + "actor": "b", + "testid": "catalog-link", + "ifAvailable": true, + "within": 1000 + }, + { + "do": "click", + "awaitWrites": true, + "actor": "b", + "testid": "add-to-cart", + "in": { + "testid": "item-card", + "contains": "Coffee Grinder" + } + }, + { + "do": "dbRecordCheckout", + "account": "{user:application}", + "item": "Keyboard", + "as": "application-prepared", + "storage": { + "kind": "order-data", + "cart": true, + "warehouses": "if-requested" + } + }, + { + "do": "crashCheckout", + "actor": "b", + "before": "application-before", + "prepared": "application-prepared", + "quantity": [ + { + "item": "Keyboard", + "quantity": 1 + }, + { + "item": "Coffee Grinder", + "quantity": 1 + } + ], + "requests": 16, + "offsetMs": 0, + "target": "application", + "as": "application-crash-observation", + "reuseCombinedFrom": "database-crash-observation", + "namedAction": { + "id": "checkout", + "reducer": "checkout", + "path": "/api/checkout", + "args": [] + } + }, + { + "do": "reload", + "actor": "b", + "settleMs": 0 + }, + { + "do": "ensureSignedIn", + "actor": "b", + "name": "application", + "readyTestid": "current-user" + }, + { + "do": "click", + "actor": "b", + "testid": "overlay-close", + "ifAvailable": true, + "within": 1000 + }, + { "do": "click", "actor": "b", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "b", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["cart-toggle", "cart-total"]}, + { + "do": "click", + "actor": "b", + "testid": "cart-toggle", + "unlessVisible": "cart-total" + }, + { + "do": "click", + "awaitWrites": true, + "actor": "b", + "testid": "cart-remove", + "in": { + "testid": "cart-item", + "contains": "Keyboard" + }, + "ifAvailable": true, + "within": 1000 + }, + { + "do": "click", + "awaitWrites": true, + "actor": "b", + "testid": "cart-remove", + "in": { + "testid": "cart-item", + "contains": "Coffee Grinder" + }, + "ifAvailable": true, + "within": 1000 + }, + { + "do": "expectNumber", + "actor": "b", + "testid": "cart-count", + "equals": 0, + "within": 10000 + }, + { + "do": "signUp", + "actor": "c", + "name": "recovered" + }, + { + "do": "dbRecordCheckout", + "account": "{user:recovered}", + "item": "Keyboard", + "as": "recovered-before", + "storage": { + "kind": "order-data", + "cart": true, + "warehouses": "if-requested" + } + }, + { "do": "click", "actor": "c", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { + "do": "click", + "awaitWrites": true, + "actor": "c", + "testid": "add-to-cart", + "in": { + "testid": "item-card", + "contains": "Keyboard" + } + }, + { + "do": "click", + "actor": "c", + "testid": "overlay-close", + "ifAvailable": true, + "within": 1000 + }, + { + "do": "click", + "actor": "c", + "testid": "catalog-link", + "ifAvailable": true, + "within": 1000 + }, + { + "do": "click", + "awaitWrites": true, + "actor": "c", + "testid": "add-to-cart", + "in": { + "testid": "item-card", + "contains": "Coffee Grinder" + } + }, + { + "do": "dbRecordCheckout", + "account": "{user:recovered}", + "item": "Keyboard", + "as": "recovered-prepared", + "storage": { + "kind": "order-data", + "cart": true, + "warehouses": "if-requested" + } + }, + {"do": "click", "actor": "c", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["cart-toggle", "cart-total"]}, + { + "do": "click", + "actor": "c", + "testid": "cart-toggle", + "unlessVisible": "cart-total" + }, + { + "do": "click", + "actor": "c", + "testid": "checkout-submit" + }, + { + "do": "dbExpectCheckout", + "before": "recovered-before", + "prepared": "recovered-prepared", + "within": 10000, + "quantity": [ + { + "item": "Keyboard", + "quantity": 1 + }, + { + "item": "Coffee Grinder", + "quantity": 1 + } + ] + }, + {"do":"reload","actor":"c","settleMs":0}, + {"do":"ensureSignedIn","actor":"c","name":"recovered","readyTestid":"current-user"}, + {"do":"click","actor":"c","testid":"overlay-close","ifAvailable":true,"unlessVisible":"cart-total","within":1000}, + {"do":"click","actor":"c","testid":"current-user","ifAvailable":true,"unlessVisible":["cart-toggle","cart-total"]}, + {"do":"click","actor":"c","testid":"cart-toggle","unlessVisible":"cart-total"}, + {"do":"expectNumber","actor":"c","testid":"cart-count","equals":0,"within":10000} + ], + "criteria": [ + { + "id": "910a", + "category": "production", + "points": 1, + "desc": "interrupted checkout recovers to the prepared cart or one complete order with the cart cleared after each independent process crash", + "steps": [ + { + "do": "expectCrashCheckout", + "from": "database-crash-observation", + "verdict": "atomicity" + }, + { + "do": "expectCrashCheckout", + "from": "application-crash-observation", + "verdict": "atomicity" + } + ] + }, + { + "id": "910b", + "category": "production", + "points": 1, + "desc": "acknowledged checkout is not rolled back and earlier orders remain unchanged after each independent process crash", + "steps": [ + { + "do": "expectCrashCheckout", + "from": "database-crash-observation", + "verdict": "durability" + }, + { + "do": "expectCrashCheckout", + "from": "application-crash-observation", + "verdict": "durability" + } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-core-business.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-core-business.json new file mode 100644 index 00000000000..32417361e7e --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-core-business.json @@ -0,0 +1,207 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 5, + "name": "Core business features", + "features": [ + { + "id": 620, + "name": "Customer profile", + "actors": ["owner", "other"], + "setup": [], + "criteria": [ + { + "id": "620a", + "desc": "the owner sees the saved profile after reload", + "points": 2, + "statedBy": "A customer profile persists.", + "steps": [ + { "do": "signUp", "actor": "owner", "name": "profile-owner" }, + { "do": "click", "actor": "owner", "testid": "profile-link" }, + { "do": "fill", "actor": "owner", "testid": "profile-name", "text": "Avery Stone" }, + { "do": "fill", "actor": "owner", "testid": "profile-address", "text": "14 Market Street" }, + { "do": "click", "actor": "owner", "testid": "profile-save" }, + { "do": "reload", "actor": "owner", "settleMs": 2500 }, + { "do": "ensureSignedIn", "actor": "owner", "name": "profile-owner", "readyTestid": "current-user" }, + { "do": "click", "actor": "owner", "testid": "profile-link" }, + { "do": "expect", "actor": "owner", "testid": "profile-address-summary", "contains": "14 Market Street" } + ] + }, + { + "id": "620b", + "desc": "another customer cannot see the owner's profile", + "points": 2, + "statedBy": "Customer profiles are private.", + "steps": [ + { "do": "signUp", "actor": "owner", "name": "profile-private-owner" }, + { "do": "click", "actor": "owner", "testid": "profile-link" }, + { "do": "fill", "actor": "owner", "testid": "profile-name", "text": "Avery Stone" }, + { "do": "fill", "actor": "owner", "testid": "profile-address", "text": "14 Market Street" }, + { "do": "click", "actor": "owner", "testid": "profile-save" }, + { "do": "signUp", "actor": "other", "name": "profile-other" }, + { "do": "click", "actor": "other", "testid": "profile-link" }, + { "do": "expect", "actor": "other", "testid": "profile-address-summary", "contains": "14 Market Street", "absent": true } + ] + } + ] + }, + { + "id": 621, + "name": "Staff roles", + "actors": ["admin", "staff"], + "setup": [], + "criteria": [ + { + "id": "621a", + "desc": "an administrator can assign a staff role", + "points": 2, + "statedBy": "Administrators can assign staff roles.", + "steps": [ + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + { "do": "expect", "actor": "admin", "testid": "admin-link", "within": 6000 }, + { "do": "click", "actor": "admin", "testid": "admin-link" }, + { "do": "fill", "actor": "admin", "testid": "staff-role-select", "in": { "testid": "staff-role-account-staff" }, "text": "inventory" }, + { "do": "click", "actor": "admin", "testid": "staff-role-save", "in": { "testid": "staff-role-account-staff" } }, + { "do": "expect", "actor": "admin", "testid": "staff-role-select", "in": { "testid": "staff-role-account-staff" }, "value": "inventory", "within": 10000 } + ] + }, + { + "id": "621b", + "desc": "a staff member cannot assign roles through the UI or a replayed request", + "points": 2, + "statedBy": "Staff members cannot assign roles.", + "steps": [ + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + { "do": "expect", "actor": "admin", "testid": "admin-link", "within": 6000 }, + { "do": "click", "actor": "admin", "testid": "admin-link" }, + { "do": "fill", "actor": "admin", "testid": "staff-role-select", "in": { "testid": "staff-role-account-staff" }, "text": "inventory" }, + { "do": "click", "actor": "admin", "testid": "staff-role-save", "in": { "testid": "staff-role-account-staff" } }, + { "do": "signIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true }, + { "do": "click", "actor": "staff", "testid": "staff-link" }, + { "do": "expect", "actor": "staff", "testid": "staff-role-save", "absent": true }, + { "do": "replayAs", "actor": "staff", "from": "admin", "match": "role", "settleMs": 1500 }, + { "do": "expectReplayRejected", "actor": "staff" } + ] + } + ] + }, + { + "id": 622, + "name": "Catalog management", + "actors": ["admin", "visitor"], + "setup": [ + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + { "do": "click", "actor": "admin", "testid": "admin-link" }, + { "do": "fill", "actor": "admin", "testid": "catalog-name", "text": "Travel Mug" }, + { "do": "fill", "actor": "admin", "testid": "catalog-category", "text": "Kitchen" }, + { "do": "fill", "actor": "admin", "testid": "catalog-price", "text": "24.00" }, + { "do": "fill", "actor": "admin", "testid": "catalog-variants", "text": "Black, Silver" }, + { "do": "click", "awaitWrites": true, "actor": "admin", "testid": "catalog-save" }, + { "do": "reload", "actor": "visitor", "settleMs": 2500 } + ], + "criteria": [ + { + "id": "622a", + "desc": "a new product reaches the public catalog", + "points": 2, + "statedBy": "Authorized staff can add products.", + "steps": [ + { "do": "expect", "actor": "visitor", "testid": "item-card", "contains": "Travel Mug", "within": 10000 } + ] + }, + { + "id": "622b", + "desc": "the product exposes its named variants", + "points": 2, + "statedBy": "Products can have named variants.", + "steps": [ + { "do": "expectElementCount", "actor": "visitor", "testid": "item-variant", "contains": "Black", "equals": 1, "within": 10000 }, + { "do": "expectElementCount", "actor": "visitor", "testid": "item-variant", "contains": "Silver", "equals": 1, "within": 10000 } + ] + } + ] + }, + { + "id": 623, + "name": "Payment records", + "actors": ["tab1", "tab2", "owner"], + "setup": [ + { "do": "signUp", "actor": "tab1", "name": "payment-record" }, + { "do": "signIn", "actor": "tab2", "name": "payment-record" }, + { "do": "signIn", "actor": "owner", "name": "payment-record" }, + { "do": "click", "actor": "owner", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Desk Lamp" } }, + { "do": "click", "actor": "owner", "testid": "cart-toggle" }, + { "do": "recordNumber", "actor": "owner", "testid": "cart-total", "as": "payment-total" }, + { "do": "callConcurrently", "actors": ["tab1", "tab2"], "action": "checkout", "settleMs": 5000 }, + { "do": "expectCallOutcomes" }, + { "do": "freshClient", "actor": "owner" }, + { "do": "signIn", "actor": "owner-fresh", "name": "payment-record" }, + {"do":"click","actor":"owner-fresh","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { "do": "click", "actor": "owner-fresh", "testid": "orders-toggle" }, + { "do": "expectElementCount", "actor": "owner-fresh", "testid": "order-item", "contains": "Desk Lamp", "equals": 1, "within": 10000 } + ], + "criteria": [ + { + "id": "623a", + "category": "production", + "desc": "checkout records the exact paid amount", + "points": 3, + "statedBy": "Checkout records the amount paid on the order.", + "steps": [ + { "do": "expect", "actor": "owner-fresh", "testid": "payment-status", "in": { "testid": "order-item", "contains": "Desk Lamp" }, "value": "paid", "ignoreCase": true, "within": 10000 }, + { "do": "expectNumber", "actor": "owner-fresh", "testid": "payment-amount", "in": { "testid": "order-item", "contains": "Desk Lamp" }, "relativeTo": "payment-total", "plus": 0 } + ] + }, + { + "id": "623b", + "category": "production", + "desc": "one checkout has one payment record", + "points": 2, + "statedBy": "A checkout does not create duplicate payments.", + "steps": [ + { "do": "expectElementCount", "actor": "owner-fresh", "testid": "payment-record", "in": { "testid": "order-item", "contains": "Desk Lamp" }, "equals": 1, "within": 10000 } + ] + } + ] + }, + { + "id": 624, + "name": "Staff activity history", + "actors": ["admin", "staff", "customer"], + "setup": [ + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + { "do": "signIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true }, + { "do": "signIn", "actor": "customer", "name": "customer", "password": "stackbench-customer-2026", "exact": true }, + { "do": "click", "actor": "admin", "testid": "admin-link" }, + { "do": "fill", "actor": "admin", "testid": "catalog-name", "text": "Activity Mug" }, + { "do": "fill", "actor": "admin", "testid": "catalog-category", "text": "Kitchen" }, + { "do": "fill", "actor": "admin", "testid": "catalog-price", "text": "43.00" }, + { "do": "fill", "actor": "admin", "testid": "catalog-variants", "text": "Green" }, + { "do": "click", "actor": "admin", "testid": "catalog-save" } + ], + "criteria": [ + { + "id": "624a", + "desc": "an administrative change records its actor and subject", + "points": 3, + "statedBy": "Administrative changes have an activity history.", + "steps": [ + { "do": "click", "actor": "staff", "testid": "staff-link" }, + { "do": "click", "actor": "staff", "testid": "activity-link" }, + { "do": "expect", "actor": "staff", "testid": "activity-entry", "contains": "Activity Mug", "within": 10000 }, + { "do": "expect", "actor": "staff", "testid": "activity-actor", "in": { "testid": "activity-entry", "contains": "Activity Mug" }, "contains": "admin", "within": 10000 } + ] + }, + { + "id": "624b", + "desc": "customers cannot open staff activity history", + "points": 2, + "statedBy": "Staff activity history is not available to customers.", + "steps": [ + { "do": "expect", "actor": "customer", "testid": "activity-link", "absent": true } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-customer-profile.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-customer-profile.json new file mode 100644 index 00000000000..03fde77e56e --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-customer-profile.json @@ -0,0 +1,108 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 2, + "name": "Customer profile", + "note": "Draft: updated observations require matching reference and defect-control qualification.", + "features": [ + { + "id": 620, + "name": "Customer profile", + "actors": ["owner", "privateOwner", "other"], + "setup": [ + { "do": "signUp", "actor": "owner", "name": "profile-owner" }, + { "do": "click", "actor": "owner", "testid": "overlay-close", "ifAvailable": true, "within": 1000 }, + { "do": "click", "actor": "owner", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "owner", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["profile-link"]}, + { "do": "click", "actor": "owner", "testid": "profile-link" }, + { "do": "fill", "actor": "owner", "testid": "profile-name", "text": "Avery Stone" }, + { "do": "fill", "actor": "owner", "testid": "profile-address", "text": "14 Market Street {user:profilemarker}" }, + { "do": "click", "actor": "owner", "testid": "profile-save" }, + { "do": "expect", "actor": "owner", "testid": "profile-save-state", "attribute": "data-submit-state", "value": "succeeded", "within": 10000 } + ], + "criteria": [ + { + "id": "620c", + "category": "feature", + "desc": "the owner can save and view a customer profile", + "points": 1, + "statedBy": "A signed-in customer can save and view their name and shipping address.", + "steps": [ + { "do": "click", "actor": "owner", "testid": "overlay-close", "ifAvailable": true }, + { "do": "click", "actor": "owner", "testid": "catalog-link", "ifAvailable": true }, + {"do": "click", "actor": "owner", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["profile-link", "profile-address-summary"]}, + { "do": "click", "actor": "owner", "testid": "profile-link", "unlessVisible": "profile-address-summary" }, + { "do": "expect", "actor": "owner", "testid": "profile-address-summary", "contains": "14 Market Street {user:profilemarker}" } + ] + }, + { + "id": "620a", + "category": "production", + "desc": "the saved profile survives reload and backend restart in a fresh browser", + "points": 2, + "statedBy": "The saved profile remains after a reload.", + "steps": [ + { "do": "reload", "actor": "owner", "settleMs": 2500 }, + { "do": "ensureSignedIn", "actor": "owner", "name": "profile-owner", "readyTestid": "current-user" }, + { "do": "click", "actor": "owner", "testid": "overlay-close", "ifAvailable": true, "within": 1000 }, + { "do": "click", "actor": "owner", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "owner", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["profile-link"]}, + { "do": "click", "actor": "owner", "testid": "profile-link" }, + { "do": "expect", "actor": "owner", "testid": "profile-address-summary", "contains": "14 Market Street {user:profilemarker}" }, + { "do": "restartBackend", "settleMs": 1000 }, + { "do": "freshClient", "actor": "owner" }, + { "do": "signIn", "actor": "owner-fresh", "name": "profile-owner" }, + { "do": "click", "actor": "owner-fresh", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "owner-fresh", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["profile-link"]}, + { "do": "click", "actor": "owner-fresh", "testid": "profile-link" }, + { "do": "expect", "actor": "owner-fresh", "testid": "profile-address-summary", "contains": "14 Market Street {user:profilemarker}" } + ] + }, + { + "id": "620b", + "category": "production", + "desc": "another customer neither sees nor receives the owner's private address", + "points": 2, + "statedBy": "Each customer can see only their own profile.", + "steps": [ + { "do": "reload", "actor": "privateOwner", "settleMs": 0 }, + { "do": "signUp", "actor": "privateOwner", "name": "profile-private-owner" }, + { "do": "click", "actor": "privateOwner", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "privateOwner", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["profile-link"]}, + { "do": "click", "actor": "privateOwner", "testid": "profile-link" }, + { "do": "fill", "actor": "privateOwner", "testid": "profile-name", "text": "Avery Stone" }, + { "do": "fill", "actor": "privateOwner", "testid": "profile-address", "text": "14 Market Street {user:profilemarker}" }, + { "do": "click", "actor": "privateOwner", "testid": "profile-save" }, + { "do": "expect", "actor": "privateOwner", "testid": "profile-save-state", "attribute": "data-submit-state", "value": "succeeded", "within": 10000 }, + { "do": "reload", "actor": "privateOwner", "settleMs": 1000 }, + { "do": "ensureSignedIn", "actor": "privateOwner", "name": "profile-private-owner", "readyTestid": "current-user" }, + { "do": "click", "actor": "privateOwner", "testid": "overlay-close", "ifAvailable": true, "within": 1000 }, + { "do": "click", "actor": "privateOwner", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "privateOwner", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["profile-link"]}, + { "do": "click", "actor": "privateOwner", "testid": "profile-link" }, + { "do": "expect", "actor": "privateOwner", "testid": "profile-address-summary", "contains": "14 Market Street {user:profilemarker}" }, + { "do": "expectReceived", "actor": "privateOwner", "contains": "14 Market Street {user:profilemarker}", "within": 10000 }, + { "do": "reload", "actor": "other", "settleMs": 0 }, + { "do": "signUp", "actor": "other", "name": "profile-other" }, + { "do": "click", "actor": "other", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "other", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["profile-link"]}, + { "do": "click", "actor": "other", "testid": "profile-link" }, + { "do": "fill", "actor": "other", "testid": "profile-name", "text": "Blake Reed" }, + { "do": "fill", "actor": "other", "testid": "profile-address", "text": "9 Harbor Road" }, + { "do": "click", "actor": "other", "testid": "profile-save" }, + { "do": "expect", "actor": "other", "testid": "profile-save-state", "attribute": "data-submit-state", "value": "succeeded", "within": 10000 }, + { "do": "reload", "actor": "other", "settleMs": 1000 }, + { "do": "ensureSignedIn", "actor": "other", "name": "profile-other", "readyTestid": "current-user" }, + { "do": "click", "actor": "other", "testid": "overlay-close", "ifAvailable": true, "within": 1000 }, + { "do": "click", "actor": "other", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "other", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["profile-link"]}, + { "do": "click", "actor": "other", "testid": "profile-link" }, + { "do": "expect", "actor": "other", "testid": "profile-address-summary", "contains": "9 Harbor Road", "within": 10000 }, + { "do": "expect", "actor": "other", "testid": "profile-address-summary", "contains": "14 Market Street {user:profilemarker}", "absent": true }, + { "do": "expectNotReceived", "actor": "other", "contains": "14 Market Street {user:profilemarker}" } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-delivery-notifications.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-delivery-notifications.json new file mode 100644 index 00000000000..496dc198cfd --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-delivery-notifications.json @@ -0,0 +1,97 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 5, + "name": "Delivery notifications", + "features": [ + { + "id": 501, + "name": "Delivery creates one private notification", + "actors": [ + "owner", + "other", + "staff" + ], + "setup": [ + {"do":"signUp","actor":"owner","name":"delivery-owner"}, + {"do":"signUp","actor":"other","name":"delivery-other"}, + {"do":"click","actor":"owner","testid":"notification-settings","unlessVisible":"notification-order"}, + {"do":"click","actor":"owner","testid":"notification-order"}, + {"do":"click","actor":"owner","testid":"notification-save"}, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"owner","testid":"catalog-link"}, + {"do":"signIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true}, + {"do":"callAction","actor":"owner","action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Desk Lamp","attribute":"data-buy-input"}}, + {"do":"expectActionOutcome","actor":"owner","outcome":"accepted"}, + {"do":"reload","actor":"owner","settleMs":1000}, + {"do":"ensureSignedIn","actor":"owner","name":"delivery-owner","readyTestid":"current-user"}, + {"do":"click","actor":"owner","testid":"catalog-link","ifAvailable":true,"unlessVisible":["notifications-panel","notifications-toggle","notification-settings"]}, + {"do":"click","actor":"owner","testid":"notification-settings","ifAvailable":true,"unlessVisible":["notifications-panel","notifications-toggle"]}, + {"do":"click","actor":"owner","testid":"notifications-toggle","unlessVisible":"notifications-panel"}, + {"do":"expect","actor":"owner","testid":"notifications-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"expectElementCount","actor":"owner","testid":"notification-item","contains":"deliver","equals":0}, + {"do":"reload","actor":"owner","settleMs":1000}, + {"do":"ensureSignedIn","actor":"owner","name":"delivery-owner","readyTestid":"current-user"}, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"owner","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"recordTime","as":"notification-shipped"}, + {"do":"callAction","actor":"staff","from":"owner","action":"ship","namedAction":{"id":"ship","path":"/api/fulfilment/ship","reducer":"ship_order","args":[0],"params":[{"name":"orderId","in":"body","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Desk Lamp","attribute":"data-ship-input"}}, + {"do":"expectActionOutcome","actor":"staff","outcome":"accepted"}, + {"do":"recordTime","as":"notification-shipped-accepted"}, + {"do":"reload","actor":"owner","settleMs":1000}, + {"do":"ensureSignedIn","actor":"owner","name":"delivery-owner","readyTestid":"current-user"}, + {"do":"click","actor":"owner","testid":"catalog-link","ifAvailable":true,"unlessVisible":["notifications-panel","notifications-toggle","notification-settings"]}, + {"do":"click","actor":"owner","testid":"notification-settings","ifAvailable":true,"unlessVisible":["notifications-panel","notifications-toggle"]}, + {"do":"click","actor":"owner","testid":"notifications-toggle","unlessVisible":"notifications-panel"}, + {"do":"expect","actor":"owner","testid":"notifications-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"expectElapsed","since":"notification-shipped","atMost":40000}, + {"do":"expectElementCount","actor":"owner","testid":"notification-item","contains":"deliver","equals":0}, + {"do":"wait","actor":"owner","ms":75000,"since":"notification-shipped-accepted"}, + {"do":"reload","actor":"owner","settleMs":1000}, + {"do":"ensureSignedIn","actor":"owner","name":"delivery-owner","readyTestid":"current-user"}, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"owner","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"owner","testid":"order-status","in":{"testid":"order-item","contains":"Desk Lamp"},"statusText":"delivered","ignoreCase":true} + ], + "criteria": [ + { + "id": "501a", + "category": "feature", + "desc": "the order owner receives one delivery notification", + "points": 3, + "statedBy": "A delivered order creates one notification for its owner.", + "steps": [ + {"do":"reload","actor":"owner","settleMs":1000}, + {"do":"ensureSignedIn","actor":"owner","name":"delivery-owner","readyTestid":"current-user"}, + {"do":"click","actor":"owner","testid":"catalog-link","ifAvailable":true,"unlessVisible":["notifications-panel","notifications-toggle","notification-settings"]}, + {"do":"click","actor":"owner","testid":"notification-settings","ifAvailable":true,"unlessVisible":["notifications-panel","notifications-toggle"]}, + {"do":"click","actor":"owner","testid":"notifications-toggle","unlessVisible":"notifications-panel"}, + {"do":"expect","actor":"owner","testid":"notifications-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"expectElementCount","actor":"owner","testid":"notification-item","contains":"deliver","equals":1,"within":10000} + ] + }, + { + "id": "501b", + "category": "production", + "desc": "another customer cannot see the delivery notification", + "points": 2, + "statedBy": "Delivery notifications are private to the order owner.", + "steps": [ + {"do":"reload","actor":"owner","settleMs":1000}, + {"do":"ensureSignedIn","actor":"owner","name":"delivery-owner","readyTestid":"current-user"}, + {"do":"click","actor":"owner","testid":"catalog-link","ifAvailable":true,"unlessVisible":["notifications-panel","notifications-toggle","notification-settings"]}, + {"do":"click","actor":"owner","testid":"notification-settings","ifAvailable":true,"unlessVisible":["notifications-panel","notifications-toggle"]}, + {"do":"click","actor":"owner","testid":"notifications-toggle","unlessVisible":"notifications-panel"}, + {"do":"expectElementCount","actor":"owner","testid":"notification-item","contains":"deliver","equals":1,"within":10000}, + {"do":"click","actor":"other","testid":"catalog-link","ifAvailable":true,"unlessVisible":["notifications-panel","notifications-toggle","notification-settings"]}, + {"do":"click","actor":"other","testid":"notification-settings","ifAvailable":true,"unlessVisible":["notifications-panel","notifications-toggle"]}, + {"do":"click","actor":"other","testid":"notifications-toggle","unlessVisible":"notifications-panel"}, + {"do":"expect","actor":"other","testid":"notifications-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"expect","actor":"other","testid":"notification-item","contains":"deliver","absent":true} + ] + } + ] + } + ], + "note": "Draft: revised observations require matching reference and defect-control qualification." +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-faceted-filters.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-faceted-filters.json new file mode 100644 index 00000000000..4c7fbc838e0 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-faceted-filters.json @@ -0,0 +1,45 @@ +{ + "schemaVersion": 1, + "note": "Draft: repaired probe behavior needs matching qualification evidence.", + "track": "ecommerce", + "level": 3, + "name": "Faceted search filters", + "features": [ + { + "id": 401, + "name": "Filters compose", + "actors": ["visitor"], + "setup": [ + { "do": "dbSetStock", "item": "Coffee Grinder", "warehouse": "East", "quantity": 0, "settleMs": 0 }, + { "do": "dbSetStock", "item": "Coffee Grinder", "warehouse": "West", "quantity": 0, "settleMs": 1000 }, + { "do": "reload", "actor": "visitor", "settleMs": 1000 }, + { "do": "click", "actor": "visitor", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "category-filter" }, + { "do": "fill", "actor": "visitor", "testid": "category-filter", "text": "Home" }, + { "do": "fill", "actor": "visitor", "testid": "minimum-price", "text": "50" }, + { "do": "fill", "actor": "visitor", "testid": "maximum-price", "text": "200" }, + { "do": "pressKey", "actor": "visitor", "key": "Tab", "settleMs": 0 }, + { "do": "click", "actor": "visitor", "testid": "filter-apply", "ifAvailable": true } + ], + "criteria": [ + { + "id": "401a", + "category": "feature", + "desc": "category, price, and availability filters apply together", + "points": 3, + "statedBy": "Apply all selected filters together", + "steps": [ + { "do": "expectElementCount", "actor": "visitor", "testid": "item-card", "in": { "testid": "search-results" }, "contains": "Coffee Grinder", "equals": 1, "within": 10000 }, + { "do": "click", "actor": "visitor", "testid": "in-stock-filter" }, + { "do": "click", "actor": "visitor", "testid": "filter-apply", "ifAvailable": true }, + { "do": "waitUntilAbsent", "actor": "visitor", "testid": "item-card", "in": { "testid": "search-results" }, "contains": "Coffee Grinder", "within": 10000 }, + { "do": "expectElementCount", "actor": "visitor", "testid": "item-card", "in": { "testid": "search-results" }, "contains": "Air Purifier", "equals": 1 }, + { "do": "expect", "actor": "visitor", "testid": "item-card", "in": { "testid": "search-results" }, "contains": "USB Cable", "absent": true }, + { "do": "expect", "actor": "visitor", "testid": "item-card", "in": { "testid": "search-results" }, "contains": "Desk Lamp", "absent": true }, + { "do": "expect", "actor": "visitor", "testid": "item-card", "in": { "testid": "search-results" }, "contains": "Espresso Machine", "absent": true }, + { "do": "expect", "actor": "visitor", "testid": "item-card", "in": { "testid": "search-results" }, "contains": "Gaming Mouse", "absent": true } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-faceted-pagination.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-faceted-pagination.json new file mode 100644 index 00000000000..a2b34d33849 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-faceted-pagination.json @@ -0,0 +1,35 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 3, + "name": "Faceted search pagination", + "features": [ + { + "id": 402, + "name": "Pages are stable", + "actors": ["visitor"], + "setup": [ + { "do": "click", "actor": "visitor", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "minimum-price" }, + { "do": "fill", "actor": "visitor", "testid": "minimum-price", "text": "1" }, + { "do": "pressKey", "actor": "visitor", "key": "Tab", "settleMs": 0 }, + { "do": "click", "actor": "visitor", "testid": "filter-apply", "ifAvailable": true } + ], + "criteria": [ + { + "id": "402a", + "category": "feature", + "desc": "moving between pages returns the same ordered items without duplicates", + "points": 3, + "statedBy": "Moving between pages must not omit or repeat an item", + "steps": [ + { "do": "expectSequence", "actor": "visitor", "testid": "item-name", "in": { "testid": "search-results" }, "equals": ["Air Purifier", "Bluetooth Speaker", "Coffee Grinder", "Desk Lamp", "Espresso Machine", "Gaming Mouse", "Headphones", "Induction Cooktop", "Keyboard", "Laptop Stand"] }, + { "do": "click", "actor": "visitor", "testid": "search-next-page" }, + { "do": "expectSequence", "actor": "visitor", "testid": "item-name", "in": { "testid": "search-results" }, "equals": ["Mirrorless Camera", "USB Cable", "Webcam"] }, + { "do": "click", "actor": "visitor", "testid": "search-previous-page" }, + { "do": "expectSequence", "actor": "visitor", "testid": "item-name", "in": { "testid": "search-results" }, "equals": ["Air Purifier", "Bluetooth Speaker", "Coffee Grinder", "Desk Lamp", "Espresso Machine", "Gaming Mouse", "Headphones", "Induction Cooktop", "Keyboard", "Laptop Stand"] } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-managed-support-privacy.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-managed-support-privacy.json new file mode 100644 index 00000000000..25ca14670e4 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-managed-support-privacy.json @@ -0,0 +1,84 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 3, + "name": "Managed support privacy", + "note": "Draft: updated observations require matching reference and defect-control qualification.", + "features": [ + { + "id": 613, + "name": "Managed support privacy", + "actors": ["owner", "other", "staff"], + "setup": [ + { "do": "signUp", "actor": "owner", "name": "managed-private-owner" }, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"owner","testid":"current-user","ifAvailable":true,"unlessVisible":["support-link","support-email"]}, + {"do":"click","actor":"owner","testid":"support-link","unlessVisible":"support-email"}, + { "do": "fill", "actor": "owner", "testid": "support-email", "text": "managed-private@example.com" }, + { "do": "fill", "actor": "owner", "testid": "support-subject", "text": "Private managed case {user:casemarker}" }, + { "do": "fill", "actor": "owner", "testid": "support-message", "text": "Private case details." }, + { "do": "click", "actor": "owner", "testid": "support-submit" }, + { "do": "expect", "actor": "owner", "testid": "support-reference", "nonEmpty": true }, + { "do": "signUp", "actor": "other", "name": "managed-private-other" }, + {"do":"click","actor":"other","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"other","testid":"current-user","ifAvailable":true,"unlessVisible":["support-link","support-email"]}, + {"do":"click","actor":"other","testid":"support-link","unlessVisible":"support-email"}, + { "do": "signIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true }, + {"do": "click", "actor": "staff", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["staff-link", "support-assignee"]}, + {"do":"click","actor":"staff","testid":"staff-link","unlessVisible":"support-assignee"}, + {"do":"click","actor":"staff","testid":"support-queue-link","unlessVisible":"support-assignee","ifAvailable":true} + ], + "criteria": [ + { + "id": "613b", + "category": "production", + "desc": "another customer cannot read or reply to the managed case", + "points": 1, + "statedBy": "A customer can access only cases that belong to their account.", + "steps": [ + { "do": "reload", "actor": "owner", "settleMs": 1000 }, + { "do": "ensureSignedIn", "actor": "owner", "name": "managed-private-owner", "readyTestid": "current-user" }, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"owner","testid":"current-user","ifAvailable":true,"unlessVisible":["support-link","support-ticket"]}, + {"do":"click","actor":"owner","testid":"support-link","unlessVisible":"support-ticket"}, + { "do": "expect", "actor": "owner", "testid": "support-ticket", "contains": "Private managed case {user:casemarker}" }, + { "do": "expectReceived", "actor": "owner", "contains": "Private managed case {user:casemarker}", "within": 10000 }, + { "do": "reload", "actor": "other", "settleMs": 1000 }, + { "do": "ensureSignedIn", "actor": "other", "name": "managed-private-other", "readyTestid": "current-user" }, + {"do":"click","actor":"other","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"other","testid":"current-user","ifAvailable":true,"unlessVisible":["support-link","support-email"]}, + {"do":"click","actor":"other","testid":"support-link","unlessVisible":"support-email"}, + { "do": "fill", "actor": "other", "testid": "support-email", "text": "managed-other@example.com" }, + { "do": "fill", "actor": "other", "testid": "support-subject", "text": "Other managed case" }, + { "do": "fill", "actor": "other", "testid": "support-message", "text": "Separate account issue." }, + { "do": "click", "actor": "other", "testid": "support-submit" }, + { "do": "expect", "actor": "other", "testid": "support-reference", "nonEmpty": true }, + { "do": "reload", "actor": "other", "settleMs": 1000 }, + { "do": "ensureSignedIn", "actor": "other", "name": "managed-private-other", "readyTestid": "current-user" }, + {"do":"click","actor":"other","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"other","testid":"current-user","ifAvailable":true,"unlessVisible":["support-link","support-ticket"]}, + {"do":"click","actor":"other","testid":"support-link","unlessVisible":"support-ticket"}, + { "do": "expect", "actor": "other", "testid": "support-ticket", "contains": "Other managed case", "within": 10000 }, + { "do": "expect", "actor": "other", "testid": "support-ticket", "contains": "Private managed case {user:casemarker}", "absent": true }, + { "do": "expectNotReceived", "actor": "other", "contains": "Private managed case {user:casemarker}" }, + { "do": "fill", "actor": "owner", "testid": "support-reply", "in": { "testid": "support-ticket", "contains": "Private managed case {user:casemarker}" }, "text": "Owner-only update" }, + { "do": "click", "actor": "owner", "testid": "support-reply-submit", "in": { "testid": "support-ticket", "contains": "Private managed case {user:casemarker}" } }, + { "do": "replayAs", "actor": "owner", "from": "owner", "match": "__613b-authorized-reply__", "namedAction": { "id": "replySupport", "path": "/api/support/:id/replies", "method": "POST", "reducer": "reply_support", "args": [0, "Owner confirmed reply"], "params": [{ "name": "ticketId", "in": "path", "placeholder": ":id", "wireType": "u64" }, { "name": "body", "in": "body" }] }, "namedTarget": { "testid": "support-ticket", "contains": "Private managed case {user:casemarker}", "attribute": "data-entity-id", "valueType": "string" }, "settleMs": 1500 }, + { "do": "expectReplayCompleted", "actor": "owner", "requireAccepted": true }, + { "do": "replayAs", "actor": "other", "from": "owner", "match": "Owner-only update", "namedAction": { "id": "replySupport", "path": "/api/support/:id/replies", "method": "POST", "reducer": "reply_support", "args": [0, "Owner-only update"], "params": [{ "name": "ticketId", "in": "path", "placeholder": ":id", "wireType": "u64" }, { "name": "body", "in": "body" }] }, "namedTarget": { "testid": "support-ticket", "contains": "Private managed case {user:casemarker}", "attribute": "data-entity-id", "valueType": "string" }, "settleMs": 1500 }, + { "do": "expectReplayRejected", "actor": "other", "allowNotFound": true }, + { "do": "reload", "actor": "staff", "settleMs": 2000 }, + { "do": "ensureSignedIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true, "readyTestid": "current-user" }, + {"do": "click", "actor": "staff", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["staff-link", "support-assignee"]}, + {"do":"click","actor":"staff","testid":"staff-link","ifAvailable":true,"unlessVisible":"support-assignee"}, + {"do":"click","actor":"staff","testid":"support-queue-link","unlessVisible":"support-assignee","ifAvailable":true}, + { "do": "expect", "actor": "staff", "testid": "support-ticket", "contains": "Private managed case {user:casemarker}" }, + { "do": "expectElementCount", "actor": "staff", "testid": "support-reply-item", "contains": "Owner-only update", "equals": 1, "within": 10000 }, + { "do": "expectReceived", "actor": "staff", "contains": "Owner-only update", "within": 10000 }, + { "do": "expectNotReceived", "actor": "other", "contains": "Owner-only update" } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-managed-support-shared.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-managed-support-shared.json new file mode 100644 index 00000000000..858ce617216 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-managed-support-shared.json @@ -0,0 +1,86 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 3, + "name": "Shared managed support case", + "note": "Draft: updated observations require matching reference and defect-control qualification.", + "features": [ + { + "id": 613, + "name": "Shared managed support case", + "actors": ["owner", "staff"], + "setup": [ + { "do": "signUp", "actor": "owner", "name": "managed-shared-owner" }, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"owner","testid":"current-user","ifAvailable":true,"unlessVisible":["support-link","support-email"]}, + {"do":"click","actor":"owner","testid":"support-link","unlessVisible":"support-email"}, + { "do": "fill", "actor": "owner", "testid": "support-email", "text": "managed-shared@example.com" }, + { "do": "fill", "actor": "owner", "testid": "support-subject", "text": "Shared managed case" }, + { "do": "fill", "actor": "owner", "testid": "support-message", "text": "Please investigate this case." }, + { "do": "click", "actor": "owner", "testid": "support-submit" }, + { "do": "expect", "actor": "owner", "testid": "support-reference", "nonEmpty": true }, + { "do": "signIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true }, + {"do": "click", "actor": "staff", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["staff-link", "support-assignee"]}, + {"do":"click","actor":"staff","testid":"staff-link","unlessVisible":"support-assignee"}, + {"do":"click","actor":"staff","testid":"support-queue-link","unlessVisible":"support-assignee","ifAvailable":true} + ], + "criteria": [ + { + "id": "613c", + "category": "feature", + "desc": "staff can update a support case and the customer can reply", + "points": 1, + "statedBy": "Customers and staff can exchange replies and update the status of a support case.", + "steps": [ + { "do": "fill", "actor": "staff", "testid": "support-status-input", "in": { "testid": "support-ticket", "contains": "Shared managed case" }, "text": "in progress" }, + { "do": "click", "actor": "staff", "testid": "support-update", "in": { "testid": "support-ticket", "contains": "Shared managed case" }, "settleMs": 1500 }, + { "do": "fill", "actor": "staff", "testid": "support-reply", "in": { "testid": "support-ticket", "contains": "Shared managed case" }, "text": "Case received." }, + { "do": "click", "awaitWrites": true, "actor": "staff", "testid": "support-reply-submit", "in": { "testid": "support-ticket", "contains": "Shared managed case" } }, + { "do": "reload", "actor": "owner", "settleMs": 2000 }, + { "do": "ensureSignedIn", "actor": "owner", "name": "managed-shared-owner", "readyTestid": "current-user" }, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"owner","testid":"current-user","ifAvailable":true,"unlessVisible":["support-link","support-ticket"]}, + {"do":"click","actor":"owner","testid":"support-link","unlessVisible":"support-ticket"}, + { "do": "expect", "actor": "owner", "testid": "support-status", "in": { "testid": "support-ticket", "contains": "Shared managed case" }, "contains": "in progress" }, + { "do": "expect", "actor": "owner", "testid": "support-reply-item", "in": { "testid": "support-ticket", "contains": "Shared managed case" }, "contains": "Case received." }, + { "do": "fill", "actor": "owner", "testid": "support-reply", "in": { "testid": "support-ticket", "contains": "Shared managed case" }, "text": "Thank you." }, + { "do": "click", "awaitWrites": true, "actor": "owner", "testid": "support-reply-submit", "in": { "testid": "support-ticket", "contains": "Shared managed case" } }, + { "do": "reload", "actor": "staff", "settleMs": 2000 }, + { "do": "ensureSignedIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true, "readyTestid": "current-user" }, + {"do": "click", "actor": "staff", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["staff-link", "support-assignee"]}, + {"do":"click","actor":"staff","testid":"staff-link","unlessVisible":"support-assignee"}, + {"do":"click","actor":"staff","testid":"support-queue-link","unlessVisible":"support-assignee","ifAvailable":true}, + { "do": "expect", "actor": "staff", "testid": "support-reply-item", "in": { "testid": "support-ticket", "contains": "Shared managed case" }, "contains": "Thank you." } + ] + }, + { + "id": "613a", + "category": "production", + "desc": "the customer and staff see the same replies and status live", + "points": 2, + "statedBy": "Customers and authorized staff use one shared support case.", + "steps": [ + { "do": "fill", "actor": "staff", "testid": "support-status-input", "in": { "testid": "support-ticket", "contains": "Shared managed case" }, "text": "open" }, + { "do": "click", "actor": "staff", "testid": "support-update", "in": { "testid": "support-ticket", "contains": "Shared managed case" } }, + { "do": "expect", "actor": "staff", "testid": "support-status", "in": { "testid": "support-ticket", "contains": "Shared managed case" }, "contains": "open" }, + { "do": "reload", "actor": "owner", "settleMs": 1000 }, + { "do": "ensureSignedIn", "actor": "owner", "name": "managed-shared-owner", "readyTestid": "current-user" }, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"owner","testid":"current-user","ifAvailable":true,"unlessVisible":["support-link","support-ticket"]}, + {"do":"click","actor":"owner","testid":"support-link","unlessVisible":"support-ticket"}, + { "do": "expect", "actor": "owner", "testid": "support-status", "in": { "testid": "support-ticket", "contains": "Shared managed case" }, "contains": "open" }, + { "do": "fill", "actor": "staff", "testid": "support-status-input", "in": { "testid": "support-ticket", "contains": "Shared managed case" }, "text": "in progress" }, + { "do": "click", "actor": "staff", "testid": "support-update", "in": { "testid": "support-ticket", "contains": "Shared managed case" }, "settleMs": 1500 }, + { "do": "fill", "actor": "staff", "testid": "support-reply", "in": { "testid": "support-ticket", "contains": "Shared managed case" }, "text": "We are investigating." }, + { "do": "click", "actor": "staff", "testid": "support-reply-submit", "in": { "testid": "support-ticket", "contains": "Shared managed case" } }, + { "do": "expect", "actor": "owner", "testid": "support-status", "in": { "testid": "support-ticket", "contains": "Shared managed case" }, "contains": "in progress", "within": 10000 }, + { "do": "expect", "actor": "owner", "testid": "support-reply-item", "in": { "testid": "support-ticket", "contains": "Shared managed case" }, "contains": "We are investigating.", "within": 10000 }, + { "do": "fill", "actor": "owner", "testid": "support-reply", "in": { "testid": "support-ticket", "contains": "Shared managed case" }, "text": "Thank you for the update." }, + { "do": "click", "actor": "owner", "testid": "support-reply-submit", "in": { "testid": "support-ticket", "contains": "Shared managed case" } }, + { "do": "expect", "actor": "staff", "testid": "support-reply-item", "in": { "testid": "support-ticket", "contains": "Shared managed case" }, "contains": "Thank you for the update.", "within": 10000 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-notification-preferences.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-notification-preferences.json new file mode 100644 index 00000000000..e542371b643 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-notification-preferences.json @@ -0,0 +1,85 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 5, + "name": "Notification preferences", + "features": [ + { + "id": 630, + "name": "Account notification preferences", + "actors": ["owner", "other"], + "setup": [ + { "do": "signUp", "actor": "owner", "name": "notification-owner" }, + { "do": "signUp", "actor": "other", "name": "notification-other" }, + { "do": "click", "actor": "owner", "testid": "overlay-close", "ifAvailable": true, "within": 1000 }, + { "do": "click", "actor": "owner", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "owner", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["notification-settings", "notification-order"]}, + { "do": "click", "actor": "owner", "testid": "notification-settings", "unlessVisible": "notification-order" }, + { "do": "click", "actor": "owner", "testid": "notification-order" }, + { "do": "click", "actor": "owner", "testid": "notification-save" }, + { "do": "expect", "actor": "owner", "testid": "notification-save-state", "attribute": "data-submit-state", "value": "succeeded", "within": 10000 } + ], + "criteria": [ + { + "id": "630c", + "category": "feature", + "desc": "the customer can save a notification choice", + "points": 1, + "statedBy": "Signed-in customers can turn order and stock notifications on or off.", + "steps": [ + { "do": "click", "actor": "owner", "testid": "overlay-close", "ifAvailable": true }, + { "do": "click", "actor": "owner", "testid": "catalog-link", "ifAvailable": true }, + {"do": "click", "actor": "owner", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["notification-settings", "notification-order"]}, + { "do": "click", "actor": "owner", "testid": "notification-settings", "unlessVisible": "notification-order" }, + { "do": "expect", "actor": "owner", "testid": "notification-order", "attribute": "data-state", "value": "on" } + ] + }, + { + "id": "630a", + "category": "production", + "desc": "notification choices survive reload and backend restart in a fresh browser", + "points": 2, + "statedBy": "Notification choices persist for the account.", + "steps": [ + { "do": "reload", "actor": "owner", "settleMs": 3000 }, + { "do": "ensureSignedIn", "actor": "owner", "name": "notification-owner", "readyTestid": "current-user" }, + { "do": "click", "actor": "owner", "testid": "overlay-close", "ifAvailable": true, "within": 1000 }, + { "do": "click", "actor": "owner", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "owner", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["notification-settings", "notification-order"]}, + { "do": "click", "actor": "owner", "testid": "notification-settings", "unlessVisible": "notification-order" }, + { "do": "expect", "actor": "owner", "testid": "notification-order", "attribute": "data-state", "value": "on" }, + { "do": "restartBackend", "settleMs": 1000 }, + { "do": "freshClient", "actor": "owner" }, + { "do": "signIn", "actor": "owner-fresh", "name": "notification-owner" }, + { "do": "click", "actor": "owner-fresh", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "owner-fresh", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["notification-settings", "notification-order"]}, + { "do": "click", "actor": "owner-fresh", "testid": "notification-settings", "unlessVisible": "notification-order" }, + { "do": "expect", "actor": "owner-fresh", "testid": "notification-order", "attribute": "data-state", "value": "on" } + ] + }, + { + "id": "630b", + "category": "production", + "desc": "the owner's choice does not change another account", + "points": 2, + "statedBy": "One customer's notification choices do not affect another customer.", + "steps": [ + { "do": "reload", "actor": "owner", "settleMs": 3000 }, + { "do": "ensureSignedIn", "actor": "owner", "name": "notification-owner", "readyTestid": "current-user" }, + { "do": "click", "actor": "owner", "testid": "overlay-close", "ifAvailable": true, "within": 1000 }, + { "do": "click", "actor": "owner", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "owner", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["notification-settings", "notification-order"]}, + { "do": "click", "actor": "owner", "testid": "notification-settings", "unlessVisible": "notification-order" }, + { "do": "expect", "actor": "owner", "testid": "notification-order", "attribute": "data-state", "value": "on" }, + { "do": "reload", "actor": "other", "settleMs": 3000 }, + { "do": "ensureSignedIn", "actor": "other", "name": "notification-other", "readyTestid": "current-user" }, + { "do": "click", "actor": "other", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "other", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["notification-settings", "notification-order"]}, + { "do": "click", "actor": "other", "testid": "notification-settings", "unlessVisible": "notification-order" }, + { "do": "expect", "actor": "other", "testid": "notification-order", "attribute": "data-state", "value": "off" } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-open-list-live.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-open-list-live.json new file mode 100644 index 00000000000..e285364f480 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-open-list-live.json @@ -0,0 +1,49 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Progression open review list", + "features": [ + { + "id": 902, + "name": "An open list stays current", + "actors": ["reader", "reviewer"], + "setup": [ + { "do": "signUp", "actor": "reviewer", "name": "raceR" }, + { "do": "signUp", "actor": "reader", "name": "raceL" }, + { "do": "click", "actor": "reviewer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do":"callAction","actor":"reviewer","action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"}}, + {"do":"expectActionOutcome","actor":"reviewer","outcome":"accepted"}, + {"do":"click","actor":"reviewer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do": "click", "actor": "reviewer", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + {"do":"click","actor":"reviewer","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"reviewer","testid":"order-item","contains":"Keyboard","within":10000}, + {"do":"click","actor":"reviewer","testid":"overlay-close","ifAvailable":true}, + {"do":"click","actor":"reviewer","testid":"catalog-link","ifAvailable":true}, + {"do": "openItem", "actor": "reviewer", "item": "Keyboard"}, + { "do": "expect", "actor": "reviewer", "testid": "item-detail", "within": 10000 }, + {"do":"click","actor":"reviewer","testid":"review-toggle","unlessVisible":"review-rating","ifAvailable":true}, + { "do": "fill", "actor": "reviewer", "testid": "review-rating", "text": "5" }, + { "do": "click", "actor": "reader", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "902a", + "category": "production", + "desc": "one customer has the Keyboard's reviews open before another posts one; the already-open view shows that review exactly once", + "points": 1, + "statedBy": "A view opened while a review is submitted converges to the current review list.", + "steps": [ + {"do": "openItem", "actor": "reader", "item": "Keyboard"}, + { "do": "expect", "actor": "reader", "testid": "item-detail", "within": 10000 }, + {"do":"click","actor":"reviewer","testid":"review-toggle","unlessVisible":"review-rating","ifAvailable":true}, + { "do": "fill", "actor": "reviewer", "testid": "review-input", "text": "live-review-kbd" }, + { "do": "click", "actor": "reviewer", "testid": "review-submit" }, + { "do": "expectElementCount", "actor": "reviewer", "testid": "review-item", "contains": "live-review-kbd", "equals": 1, "within": 10000 }, + { "do": "expectElementCount", "actor": "reader", "testid": "review-item", "contains": "live-review-kbd", "equals": 1, "within": 10000 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-order-return-boundary.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-order-return-boundary.json new file mode 100644 index 00000000000..b123306b790 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-order-return-boundary.json @@ -0,0 +1,113 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 3, + "name": "Pending order return boundary", + "features": [ + { + "id": 331, + "name": "Pending order return boundary", + "actors": [ + "customer" + ], + "setup": [ + {"do":"signUp","actor":"customer","name":"return-boundary"}, + {"do":"callAction","actor":"customer","action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Desk Lamp","attribute":"data-buy-input"}}, + {"do":"expectActionOutcome","actor":"customer","outcome":"accepted"}, + {"do":"reload","actor":"customer","settleMs":1000}, + {"do":"ensureSignedIn","actor":"customer","name":"return-boundary","readyTestid":"current-user"}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"customer","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"customer","testid":"order-status","in":{"testid":"order-item","contains":"Desk Lamp"},"statusText":"pending","ignoreCase":true} + ], + "criteria": [ + { + "id": "3e", + "category": "interface", + "desc": "a pending order does not offer a return button", + "points": 1, + "statedBy": "An item from a pending order cannot be returned.", + "steps": [ + {"do":"expect","actor":"customer","testid":"return-item","in":{"testid":"order-item","contains":"Desk Lamp"},"absent":true} + ] + } + ] + }, + { + "id": 332, + "name": "Pending return server boundary", + "actors": [ + "customer", + "staff", + "admin" + ], + "setup": [ + {"do":"signUp","actor":"customer","name":"return-boundary"}, + {"do":"callAction","actor":"customer","action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Desk Lamp","attribute":"data-buy-input"}}, + {"do":"expectActionOutcome","actor":"customer","outcome":"accepted"}, + {"do":"reload","actor":"customer","settleMs":1000}, + {"do":"ensureSignedIn","actor":"customer","name":"return-boundary","readyTestid":"current-user"}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"customer","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"customer","testid":"order-status","in":{"testid":"order-item","contains":"Desk Lamp"},"statusText":"pending","ignoreCase":true}, + {"do":"signIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true}, + {"do":"reload","actor":"customer","settleMs":1000}, + {"do":"ensureSignedIn","actor":"customer","name":"return-boundary","readyTestid":"current-user"}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"customer","testid":"catalog-link","unlessVisible":"item-card"}, + {"do":"callAction","actor":"customer","action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"}}, + {"do":"expectActionOutcome","actor":"customer","outcome":"accepted"}, + {"do":"reload","actor":"customer","settleMs":1000}, + {"do":"ensureSignedIn","actor":"customer","name":"return-boundary","readyTestid":"current-user"}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"customer","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"callAction","actor":"staff","from":"customer","action":"ship","namedAction":{"id":"ship","path":"/api/fulfilment/ship","reducer":"ship_order","args":[0],"params":[{"name":"orderId","in":"body","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Keyboard","attribute":"data-ship-input"}}, + {"do":"expectActionOutcome","actor":"staff","outcome":"accepted"}, + {"do":"reload","actor":"customer","settleMs":1000}, + {"do":"ensureSignedIn","actor":"customer","name":"return-boundary","readyTestid":"current-user"}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"customer","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"customer","testid":"order-status","in":{"testid":"order-item","contains":"Keyboard"},"statusText":"shipped","ignoreCase":true}, + {"do":"callAction","actor":"customer","action":"returnItem","namedAction":{"id":"returnItem","path":"/api/orders/{orderId}/items/{itemId}/return","method":"POST","reducer":"return_order_item","args":[0,0],"params":[{"name":"orderId","in":"path","placeholder":"{orderId}","wireType":"u64"},{"name":"itemId","in":"path","placeholder":"{itemId}","wireType":"u64"}]},"input":{"testid":"order-line","contains":"Keyboard","attribute":"data-return-input"}}, + {"do":"expectActionOutcome","actor":"customer","outcome":"accepted"}, + {"do":"reload","actor":"customer","settleMs":1000}, + {"do":"ensureSignedIn","actor":"customer","name":"return-boundary","readyTestid":"current-user"}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"customer","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"customer","testid":"order-status","in":{"testid":"order-item","contains":"Keyboard"},"statusText":"returned","ignoreCase":true}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"recordNumber","actor":"admin","testid":"admin-revenue","as":"pending-revenue"}, + {"do":"dbRecordStock","item":"Desk Lamp","warehouse":"East","as":"pending-East"}, + {"do":"dbRecordStock","item":"Desk Lamp","warehouse":"West","as":"pending-West"}, + {"do":"expect","actor":"customer","testid":"order-status","in":{"testid":"order-item","contains":"Desk Lamp"},"statusText":"pending","ignoreCase":true} + ], + "criteria": [ + { + "id": "3f", + "category": "production", + "desc": "the server refuses a pending return without changing stock or revenue", + "points": 2, + "statedBy": "Only shipped items can be returned.", + "steps": [ + {"do":"callAction","actor":"customer","action":"returnItem","namedAction":{"id":"returnItem","path":"/api/orders/{orderId}/items/{itemId}/return","method":"POST","reducer":"return_order_item","args":[0,0],"params":[{"name":"orderId","in":"path","placeholder":"{orderId}","wireType":"u64"},{"name":"itemId","in":"path","placeholder":"{itemId}","wireType":"u64"}]},"input":{"testid":"order-line","contains":"Desk Lamp","attribute":"data-return-input"}}, + {"do":"expectActionOutcome","actor":"customer","outcome":"validation-refused"}, + {"do":"reload","actor":"customer","settleMs":1000}, + {"do":"ensureSignedIn","actor":"customer","name":"return-boundary","readyTestid":"current-user"}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"customer","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"customer","testid":"order-status","in":{"testid":"order-item","contains":"Desk Lamp"},"statusText":"pending","ignoreCase":true}, + {"do":"dbExpectStock","item":"Desk Lamp","warehouse":"East","relativeTo":"pending-East","plus":0}, + {"do":"dbExpectStock","item":"Desk Lamp","warehouse":"West","relativeTo":"pending-West","plus":0}, + {"do":"reload","actor":"admin","settleMs":1000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","readyTestid":"current-user","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":"admin-revenue"}, + {"do":"expectNumber","actor":"admin","testid":"admin-revenue","relativeTo":"pending-revenue","plus":0} + ] + } + ] + } + ], + "note": "Draft: revised observations require matching reference and defect-control qualification." +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-order-return-complete.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-order-return-complete.json new file mode 100644 index 00000000000..bcbf45a9669 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-order-return-complete.json @@ -0,0 +1,71 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 3, + "name": "Completed order return", + "note": "Draft: revised observations require matching reference and defect-control qualification.", + "features": [ + { + "id": 330, + "name": "Completed order return", + "actors": [ + "customer", + "staff", + "admin" + ], + "setup": [ + {"do":"signUp","actor":"customer","name":"return-complete"}, + {"do":"signIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"staff","testid":"staff-link"}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"expectNumber","actor":"customer","testid":"item-price","in":{"testid":"item-card","contains":"Keyboard"},"equals":89}, + {"do":"dbRecordStock","item":"Keyboard","as":"return-stock-before"}, + {"do":"recordNumber","actor":"admin","testid":"admin-revenue","as":"return-revenue-before"}, + {"do":"click","actor":"customer","testid":"buy-now","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"reload","actor":"staff","settleMs":1000}, + {"do":"ensureSignedIn","actor":"staff","name":"staff","readyTestid":"current-user","password":"stackbench-staff-2026","exact":true}, + {"do":"click","actor":"staff","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"staff","testid":"staff-link","unlessVisible":"queue-item"}, + {"do":"expect","actor":"staff","testid":"queue-item","contains":"Keyboard","within":10000}, + {"do":"dbExpectStock","item":"Keyboard","relativeTo":"return-stock-before","plus":-1}, + {"do":"reload","actor":"admin","settleMs":1000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","readyTestid":"current-user","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":"admin-revenue"}, + {"do":"expectNumber","actor":"admin","testid":"admin-revenue","relativeTo":"return-revenue-before","plus":89,"within":10000}, + {"do":"click","actor":"staff","testid":"ship-submit","in":{"testid":"queue-item","contains":"Keyboard"}}, + {"do":"reload","actor":"customer","settleMs":1000}, + {"do":"ensureSignedIn","actor":"customer","name":"return-complete","readyTestid":"current-user"}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"customer","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"customer","testid":"order-status","in":{"testid":"order-item","contains":"Keyboard"},"statusText":"shipped","ignoreCase":true,"within":10000} + ], + "criteria": [ + { + "id": "3c", + "category": "production", + "desc": "returning a shipped item restores stock and revenue and marks the item returned", + "points": 3, + "statedBy": "An accepted return marks the item as returned, restores stock, and reduces revenue by the price paid.", + "steps": [ + {"do":"click","actor":"customer","testid":"return-item","in":{"testid":"order-item","contains":"Keyboard"}}, + {"do":"expect","actor":"customer","testid":"order-item","ignoreCase":true,"within":10000,"contains":"Keyboard","containsText":"returned"}, + {"do":"freshClient","actor":"customer"}, + {"do":"signIn","actor":"customer-fresh","name":"return-complete"}, + {"do":"click","actor":"customer-fresh","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"customer-fresh","testid":"orders-toggle"}, + {"do":"expect","actor":"customer-fresh","testid":"order-item","ignoreCase":true,"within":10000,"contains":"Keyboard","containsText":"returned"}, + {"do":"reload","actor":"admin","settleMs":1000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true,"readyTestid":"current-user"}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":"admin-revenue"}, + {"do":"expectNumber","actor":"admin","testid":"admin-revenue","relativeTo":"return-revenue-before","plus":0,"within":10000}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"customer","testid":"catalog-link","ifAvailable":true,"within":1000}, + {"do":"dbExpectStock","item":"Keyboard","relativeTo":"return-stock-before","plus":0} + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-order-support-boundary.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-order-support-boundary.json new file mode 100644 index 00000000000..cc8b336c27d --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-order-support-boundary.json @@ -0,0 +1,49 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 4, + "name": "Order support ownership boundary", + "features": [ + { + "id": 614, + "name": "Order support ownership boundary", + "actors": ["owner", "other"], + "setup": [ + { "do": "signUp", "actor": "owner", "name": "order-boundary-owner" }, + { "do": "click", "actor": "owner", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Desk Lamp" } }, + { "do": "click", "actor": "owner", "testid": "support-link" }, + { "do": "fill", "actor": "owner", "testid": "support-subject", "text": "Owner order case" }, + { "do": "fill", "actor": "owner", "testid": "support-message", "text": "This case belongs to the order owner." }, + { "do": "click", "actor": "owner", "testid": "support-submit" }, + { "do": "click", "actor": "owner", "testid": "support-order-option", "contains": "Desk Lamp", "in": { "testid": "support-ticket", "contains": "Owner order case" } }, + { "do": "signUp", "actor": "other", "name": "order-boundary-other" }, + { "do": "click", "actor": "other", "testid": "support-link" }, + { "do": "fill", "actor": "other", "testid": "support-subject", "text": "Other order case" }, + { "do": "fill", "actor": "other", "testid": "support-message", "text": "This is a separate case." }, + { "do": "click", "actor": "other", "testid": "support-submit" } + ], + "criteria": [ + { + "id": "614b", + "category": "production", + "desc": "another customer cannot attach or inspect the owner's order", + "points": 2, + "statedBy": "A customer cannot attach or inspect another customer's order.", + "steps": [ + { "do": "expect", "actor": "other", "testid": "support-order-option", "contains": "Desk Lamp", "absent": true }, + { "do": "callAction", "actor": "other", "from": "owner", "action": "linkSupportOrder", "authentication": "actor", "input": { "testid": "support-link-order", "attribute": "data-action-input", "overrides": { "caseId": { "actor": "other", "testid": "support-ticket", "contains": "Other order case", "attribute": "data-entity-id" } } }, "namedAction": { "id": "linkSupportOrder", "path": "/api/support/cases/{caseId}/order", "method": "POST", "reducer": "link_support_order", "args": [0, 0], "params": [{ "name": "caseId", "in": "path", "placeholder": "{caseId}", "wireType": "u64" }, { "name": "orderId", "in": "body", "wireType": "u64" }] }, "settleMs": 1500 }, + { "do": "callAction", "actor": "owner", "action": "linkSupportOrder", "authentication": "actor", "input": { "testid": "support-link-order", "attribute": "data-action-input" }, "namedAction": { "id": "linkSupportOrder", "path": "/api/support/cases/{caseId}/order", "method": "POST", "reducer": "link_support_order", "args": [0, 0], "params": [{ "name": "caseId", "in": "path", "placeholder": "{caseId}", "wireType": "u64" }, { "name": "orderId", "in": "body", "wireType": "u64" }] }, "settleMs": 1500 }, + { "do": "expectActionOutcome", "actor": "owner", "outcome": "accepted" }, + { "do": "expectElementCount", "actor": "owner", "testid": "support-order", "contains": "Desk Lamp", "equals": 1, "within": 10000 }, + { "do": "expectActionOutcome", "actor": "other", "outcome": "refused", "routeProvenBy": "owner" }, + { "do": "freshClient", "actor": "other" }, + { "do": "signIn", "actor": "other-fresh", "name": "order-boundary-other" }, + { "do": "click", "actor": "other-fresh", "testid": "support-link" }, + { "do": "expect", "actor": "other-fresh", "testid": "support-order", "in": { "testid": "support-ticket", "contains": "Other order case" }, "contains": "Desk Lamp", "absent": true }, + { "do": "expectElementCount", "actor": "owner", "testid": "support-order", "contains": "Desk Lamp", "equals": 1, "within": 10000 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-order-support-owned.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-order-support-owned.json new file mode 100644 index 00000000000..9a9c4440356 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-order-support-owned.json @@ -0,0 +1,38 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 4, + "name": "Owned order support link", + "features": [ + { + "id": 614, + "name": "Owned order support link", + "actors": ["owner", "staff"], + "setup": [ + { "do": "signUp", "actor": "owner", "name": "order-support-owner" }, + { "do": "click", "actor": "owner", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Desk Lamp" } }, + { "do": "click", "actor": "owner", "testid": "support-link" }, + { "do": "fill", "actor": "owner", "testid": "support-subject", "text": "Owned order case" }, + { "do": "fill", "actor": "owner", "testid": "support-message", "text": "Question about my Desk Lamp order." }, + { "do": "click", "actor": "owner", "testid": "support-submit" } + ], + "criteria": [ + { + "id": "614a", + "category": "feature", + "desc": "the customer can link their order and staff can inspect it", + "points": 3, + "statedBy": "A customer can attach one of their orders to a support case.", + "steps": [ + { "do": "click", "actor": "owner", "testid": "support-order-option", "contains": "Desk Lamp", "in": { "testid": "support-ticket", "contains": "Owned order case" } }, + { "do": "click", "actor": "owner", "testid": "support-link-order", "in": { "testid": "support-ticket", "contains": "Owned order case" } }, + { "do": "signIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true }, + {"do":"click","actor":"staff","testid":"staff-link","unlessVisible":"support-assignee"}, + {"do":"click","actor":"staff","testid":"support-queue-link","unlessVisible":"support-assignee","ifAvailable":true}, + { "do": "expect", "actor": "staff", "testid": "support-order", "in": { "testid": "support-ticket", "contains": "Owned order case" }, "contains": "Desk Lamp", "within": 10000 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-personalized-recommendations.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-personalized-recommendations.json new file mode 100644 index 00000000000..5c01f29412b --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-personalized-recommendations.json @@ -0,0 +1,253 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 4, + "name": "Personalized recommendations", + "features": [ + { + "id": 403, + "name": "Recommendations use customer activity", + "actors": [ + "sales-helper", + "audio-customer", + "home-customer", + "computing-customer" + ], + "setup": [ + { + "do": "signUp", + "actor": "sales-helper", + "name": "recommend-sales" + }, + { + "do": "click", + "actor": "sales-helper", + "testid": "buy-now", + "in": { + "testid": "item-card", + "contains": "Desk Lamp" + }, + "settleMs": 500 + }, + { + "do": "click", + "actor": "sales-helper", + "testid": "buy-now", + "in": { + "testid": "item-card", + "contains": "Desk Lamp" + }, + "settleMs": 500 + }, + { + "do": "click", + "actor": "sales-helper", + "testid": "buy-now", + "in": { + "testid": "item-card", + "contains": "Desk Lamp" + }, + "settleMs": 500 + }, + { + "do": "signUp", + "actor": "audio-customer", + "name": "recommend-audio" + }, + { + "do": "click", + "actor": "audio-customer", + "testid": "buy-now", + "in": { + "testid": "item-card", + "contains": "Bluetooth Speaker" + } + }, + { + "do": "signUp", + "actor": "home-customer", + "name": "recommend-home" + }, + { + "do": "click", + "actor": "home-customer", + "testid": "buy-now", + "in": { + "testid": "item-card", + "contains": "Coffee Grinder" + } + }, + { + "do": "signUp", + "actor": "computing-customer", + "name": "recommend-computing" + }, + { + "do": "click", + "actor": "computing-customer", + "testid": "buy-now", + "in": { + "testid": "item-card", + "contains": "Keyboard" + } + } + ], + "criteria": [ + { + "id": "403a", + "category": "feature", + "desc": "recommendations follow the customer's categories, global sales, and name tie-break", + "points": 4, + "statedBy": "Order the remaining items by global units sold, highest first, then by item name.", + "steps": [ + { + "do": "expect", + "actor": "audio-customer", + "testid": "recommended-item", + "in": { + "testid": "recommendations" + }, + "contains": "Headphones" + }, + { + "do": "expectNumber", + "actor": "home-customer", + "testid": "recommendation-rank", + "in": { + "testid": "recommended-item", + "contains": "Desk Lamp" + }, + "equals": 1, + "within": 10000 + }, + { + "do": "expectElementCount", + "actor": "computing-customer", + "testid": "recommended-item", + "in": { + "testid": "recommendations" + }, + "equals": 3, + "within": 10000 + }, + { + "do": "expectNumber", + "actor": "computing-customer", + "testid": "recommendation-rank", + "in": { + "testid": "recommended-item", + "contains": "Gaming Mouse" + }, + "equals": 1, + "within": 10000 + }, + { + "do": "expectNumber", + "actor": "computing-customer", + "testid": "recommendation-rank", + "in": { + "testid": "recommended-item", + "contains": "Laptop Stand" + }, + "equals": 2, + "within": 10000 + }, + { + "do": "expectNumber", + "actor": "computing-customer", + "testid": "recommendation-rank", + "in": { + "testid": "recommended-item", + "contains": "Webcam" + }, + "equals": 3, + "within": 10000 + } + ] + }, + { + "id": "403b", + "category": "production", + "desc": "one customer's activity does not replace another customer's recommendations", + "points": 2, + "statedBy": "Customer recommendation profiles are isolated.", + "steps": [ + { + "do": "click", + "actor": "audio-customer", + "testid": "buy-now", + "in": { + "testid": "item-card", + "contains": "Headphones" + } + }, + { + "do": "waitUntilAbsent", + "actor": "audio-customer", + "testid": "recommended-item", + "in": { + "testid": "recommendations" + }, + "contains": "Headphones", + "within": 10000 + }, + { + "do": "expectNumber", + "actor": "home-customer", + "testid": "recommendation-rank", + "in": { + "testid": "recommended-item", + "contains": "Desk Lamp" + }, + "equals": 1, + "within": 10000 + }, + { + "do": "expectElementCount", + "actor": "computing-customer", + "testid": "recommended-item", + "in": { + "testid": "recommendations" + }, + "equals": 3, + "within": 10000 + }, + { + "do": "expectNumber", + "actor": "computing-customer", + "testid": "recommendation-rank", + "in": { + "testid": "recommended-item", + "contains": "Gaming Mouse" + }, + "equals": 1, + "within": 10000 + }, + { + "do": "expectNumber", + "actor": "computing-customer", + "testid": "recommendation-rank", + "in": { + "testid": "recommended-item", + "contains": "Laptop Stand" + }, + "equals": 2, + "within": 10000 + }, + { + "do": "expectNumber", + "actor": "computing-customer", + "testid": "recommendation-rank", + "in": { + "testid": "recommended-item", + "contains": "Webcam" + }, + "equals": 3, + "within": 10000 + } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-price-cart-checkout.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-price-cart-checkout.json new file mode 100644 index 00000000000..0ecda5a342e --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-price-cart-checkout.json @@ -0,0 +1,44 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 2, + "name": "Open cart price changes", + "features": [ + { + "id": 420, + "name": "Open cart price changes", + "actors": ["admin", "customer"], + "setup": [ + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + { "do": "signUp", "actor": "customer", "name": "price-cart" }, + { "do": "click", "actor": "admin", "testid": "admin-link" } + ], + "criteria": [ + { + "id": "4c", + "category": "production", + "desc": "a price change updates an open cart and direct checkout persists the new total", + "points": 3, + "statedBy": "The public catalog and every open cart show the new price without a reload. Checkout uses the current price.", + "steps": [ + { "do": "fill", "actor": "customer", "testid": "search-input", "text": "Desk Lamp", "enter": true, "settleMs": 1500 }, + { "do": "click", "actor": "customer", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Desk Lamp" } }, + { "do": "click", "actor": "customer", "testid": "cart-toggle" }, + { "do": "expectNumber", "actor": "customer", "testid": "cart-total", "equals": 42 }, + { "do": "fill", "actor": "admin", "testid": "price-input", "text": "52.00", "in": { "testid": "admin-item-row", "contains": "Desk Lamp" } }, + { "do": "click", "actor": "admin", "testid": "price-submit", "in": { "testid": "admin-item-row", "contains": "Desk Lamp" } }, + { "do": "expectNumber", "actor": "customer", "testid": "cart-total", "equals": 52, "within": 10000 }, + { "do": "callAction", "actor": "customer", "action": "checkout", "namedAction": { "id": "checkout", "path": "/api/checkout", "method": "POST", "reducer": "checkout", "args": [] }, "settleMs": 1500 }, + { "do": "expectActionOutcome", "actor": "customer", "outcome": "accepted" }, + { "do": "freshClient", "actor": "customer" }, + { "do": "signIn", "actor": "customer-fresh", "name": "price-cart" }, + {"do":"click","actor":"customer-fresh","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { "do": "click", "actor": "customer-fresh", "testid": "orders-toggle" }, + { "do": "expect", "actor": "customer-fresh", "testid": "order-item", "contains": "Desk Lamp", "count": 1 }, + { "do": "expectNumber", "actor": "customer-fresh", "testid": "order-total", "in": { "testid": "order-item", "contains": "Desk Lamp" }, "equals": 52 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-product-bundles.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-product-bundles.json new file mode 100644 index 00000000000..ff8e202749b --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-product-bundles.json @@ -0,0 +1,76 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 4, + "name": "Product bundles", + "features": [ + { + "id": 740, + "name": "Bundle definitions", + "actors": [ + "admin", + "visitor" + ], + "setup": [ + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"bundles-link","unlessVisible":"bundle-name-input"} + ], + "criteria": [ + { + "id": "740a", + "category": "feature", + "desc": "a saved bundle shows the exact price and component quantities after reopening the application", + "points": 2, + "steps": [ + {"do":"fill","actor":"admin","testid":"bundle-name-input","text":"Office bundle"}, + {"do":"fill","actor":"admin","testid":"bundle-price-input","text":"75.00"}, + {"do":"fill","actor":"admin","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"}, + {"do":"click","actor":"admin","testid":"bundle-save"}, + {"do":"expect","actor":"admin","testid":"bundle-card","within":10000,"contains":"Office bundle"}, + {"do":"reload","actor":"visitor","settleMs":1000}, + {"do":"click","actor":"visitor","testid":"bundles-link","unlessVisible":"bundle-card"}, + {"do":"expectNumber","actor":"visitor","testid":"bundle-price","equals":75,"within":10000,"in":{"testid":"bundle-card","contains":"Office bundle"}}, + {"do":"expectElementCount","actor":"visitor","testid":"bundle-component","in":{"testid":"bundle-card","contains":"Office bundle"},"equals":2}, + {"do":"expect","actor":"visitor","testid":"bundle-component","contains":"Keyboard","in":{"testid":"bundle-card","contains":"Office bundle"},"attribute":"data-quantity","value":"2","within":10000}, + {"do":"expect","actor":"visitor","testid":"bundle-component","contains":"Desk Lamp","in":{"testid":"bundle-card","contains":"Office bundle"},"attribute":"data-quantity","value":"1","within":10000} + ] + } + ] + }, + { + "id": 743, + "name": "Bundle management authorization", + "actors": [ + "admin", + "customer" + ], + "setup": [ + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"click","actor":"admin","testid":"bundles-link","unlessVisible":"bundle-name-input"}, + {"do":"signUp","actor":"customer","name":"bundle-management"}, + {"do":"fill","actor":"admin","testid":"bundle-name-input","text":"Protected bundle"}, + {"do":"fill","actor":"admin","testid":"bundle-price-input","text":"75.00"}, + {"do":"fill","actor":"admin","testid":"bundle-components-input","text":"[{\"item\":\"Keyboard\",\"quantity\":2},{\"item\":\"Desk Lamp\",\"quantity\":1}]"}, + {"do":"click","actor":"admin","testid":"bundle-save"}, + {"do":"expect","actor":"admin","testid":"bundle-card","within":10000,"contains":"Protected bundle"} + ], + "criteria": [ + { + "id": "743a", + "category": "production", + "desc": "a customer cannot replace a staff-created bundle through the application write", + "points": 3, + "steps": [ + {"do":"fill","actor":"admin","testid":"bundle-price-input","text":"1.00"}, + {"do":"callAction","actor":"customer","from":"admin","input":{"testid":"bundle-save","attribute":"data-bundle-save-input"},"action":"saveBundle","namedAction":{"id":"saveBundle","path":"/api/bundles","reducer":"save_bundle","args":["Protected bundle",1,"[{\"item\":\"Keyboard\",\"quantity\":1}]"],"params":[{"name":"name","in":"body"},{"name":"price","in":"body"},{"name":"componentsJson","in":"body"}]},"settleMs":1000}, + {"do":"expectActionOutcome","actor":"customer","outcome":"application-refused"}, + {"do":"reload","actor":"admin","settleMs":1000}, + {"do":"click","actor":"admin","testid":"bundles-link","unlessVisible":"bundle-card"}, + {"do":"expectNumber","actor":"admin","testid":"bundle-price","equals":75,"within":10000,"in":{"testid":"bundle-card","contains":"Protected bundle"}}, + {"do":"expect","actor":"admin","testid":"bundle-component","contains":"Keyboard","in":{"testid":"bundle-card","contains":"Protected bundle"},"attribute":"data-quantity","value":"2","within":10000} + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-promotion-checkout.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-promotion-checkout.json new file mode 100644 index 00000000000..21ba2ee5bfd --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-promotion-checkout.json @@ -0,0 +1,92 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 5, + "name": "Promotion checkout", + "features": [ + { + "id": 621, + "name": "Bounded promotions at checkout", + "actors": ["staff", "activeBuyer", "expiredBuyer", "firstBuyer", "secondBuyer"], + "setup": [ + { "do": "signIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true }, + {"do":"click","actor":"staff","testid":"staff-link","unlessVisible":"promotions-link"}, + {"do":"click","actor":"staff","testid":"promotions-link","unlessVisible":"promotion-code"}, + { "do": "fill", "actor": "staff", "testid": "promotion-code", "text": "LIVE10" }, + { "do": "fill", "actor": "staff", "testid": "promotion-discount", "text": "10" }, + { "do": "fill", "actor": "staff", "testid": "promotion-start", "text": "2020-01-01" }, + { "do": "fill", "actor": "staff", "testid": "promotion-end", "text": "2099-12-31" }, + { "do": "fill", "actor": "staff", "testid": "promotion-limit", "text": "10" }, + { "do": "click", "actor": "staff", "testid": "promotion-submit" }, + { "do": "fill", "actor": "staff", "testid": "promotion-code", "text": "OLD10" }, + { "do": "fill", "actor": "staff", "testid": "promotion-discount", "text": "10" }, + { "do": "fill", "actor": "staff", "testid": "promotion-start", "text": "2000-01-01" }, + { "do": "fill", "actor": "staff", "testid": "promotion-end", "text": "2000-12-31" }, + { "do": "fill", "actor": "staff", "testid": "promotion-limit", "text": "10" }, + { "do": "click", "actor": "staff", "testid": "promotion-submit" }, + { "do": "fill", "actor": "staff", "testid": "promotion-code", "text": "ONCE10" }, + { "do": "fill", "actor": "staff", "testid": "promotion-discount", "text": "10" }, + { "do": "fill", "actor": "staff", "testid": "promotion-start", "text": "2020-01-01" }, + { "do": "fill", "actor": "staff", "testid": "promotion-end", "text": "2099-12-31" }, + { "do": "fill", "actor": "staff", "testid": "promotion-limit", "text": "1" }, + { "do": "click", "actor": "staff", "testid": "promotion-submit" }, + { "do": "signUp", "actor": "activeBuyer", "name": "promotion-active" }, + { "do": "signUp", "actor": "expiredBuyer", "name": "promotion-expired" }, + { "do": "signUp", "actor": "firstBuyer", "name": "promotion-first" }, + { "do": "signUp", "actor": "secondBuyer", "name": "promotion-second" } + ], + "criteria": [ + { + "id": "621a", + "category": "feature", + "desc": "an active promotion changes checkout and is recorded on the order", + "points": 3, + "statedBy": "An active promotion applies to the cart and the order records its discount.", + "steps": [ + { "do": "click", "actor": "activeBuyer", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Keyboard" } }, + { "do": "click", "actor": "activeBuyer", "testid": "cart-toggle" }, + { "do": "fill", "actor": "activeBuyer", "testid": "cart-promotion", "text": "LIVE10" }, + { "do": "click", "actor": "activeBuyer", "testid": "apply-promotion" }, + { "do": "click", "actor": "activeBuyer", "testid": "checkout-submit" }, + {"do":"click","actor":"activeBuyer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { "do": "click", "actor": "activeBuyer", "testid": "orders-toggle" }, + { "do": "expectNumber", "actor": "activeBuyer", "testid": "order-discount", "in": { "testid": "order-item", "contains": "Keyboard" }, "equals": 8.9 } + ] + }, + { + "id": "621b", + "category": "feature", + "desc": "an expired promotion is refused", + "points": 2, + "statedBy": "Expired promotions cannot be applied.", + "steps": [ + { "do": "click", "actor": "expiredBuyer", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Keyboard" } }, + { "do": "click", "actor": "expiredBuyer", "testid": "cart-toggle" }, + { "do": "fill", "actor": "expiredBuyer", "testid": "cart-promotion", "text": "OLD10" }, + { "do": "click", "actor": "expiredBuyer", "testid": "apply-promotion" }, + { "do": "expect", "actor": "expiredBuyer", "testid": "promotion-error" } + ] + }, + { + "id": "621c", + "category": "feature", + "desc": "a fully redeemed promotion is refused", + "points": 2, + "statedBy": "A promotion cannot be used after its redemption limit is reached.", + "steps": [ + { "do": "click", "actor": "firstBuyer", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Keyboard" } }, + { "do": "click", "actor": "firstBuyer", "testid": "cart-toggle" }, + { "do": "fill", "actor": "firstBuyer", "testid": "cart-promotion", "text": "ONCE10" }, + { "do": "click", "actor": "firstBuyer", "testid": "apply-promotion" }, + { "do": "click", "actor": "firstBuyer", "testid": "checkout-submit" }, + { "do": "click", "actor": "secondBuyer", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Keyboard" } }, + { "do": "click", "actor": "secondBuyer", "testid": "cart-toggle" }, + { "do": "fill", "actor": "secondBuyer", "testid": "cart-promotion", "text": "ONCE10" }, + { "do": "click", "actor": "secondBuyer", "testid": "apply-promotion" }, + { "do": "expect", "actor": "secondBuyer", "testid": "promotion-error" } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-promotion-reporting.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-promotion-reporting.json new file mode 100644 index 00000000000..5bd9818e0e6 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-promotion-reporting.json @@ -0,0 +1,52 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 5, + "name": "Promotion reporting", + "features": [ + { + "id": 622, + "name": "Exact promotion totals", + "actors": ["staff", "buyer"], + "setup": [ + { "do": "signIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true }, + {"do":"click","actor":"staff","testid":"staff-link","unlessVisible":"promotions-link"}, + {"do":"click","actor":"staff","testid":"promotions-link","unlessVisible":"promotion-code"}, + { "do": "fill", "actor": "staff", "testid": "promotion-code", "text": "REPORT10" }, + { "do": "fill", "actor": "staff", "testid": "promotion-discount", "text": "10" }, + { "do": "fill", "actor": "staff", "testid": "promotion-start", "text": "2020-01-01" }, + { "do": "fill", "actor": "staff", "testid": "promotion-end", "text": "2099-12-31" }, + { "do": "fill", "actor": "staff", "testid": "promotion-limit", "text": "2" }, + { "do": "click", "actor": "staff", "testid": "promotion-submit" }, + { "do": "signUp", "actor": "buyer", "name": "promotion-report-buyer" }, + { "do": "click", "actor": "buyer", "testid": "add-to-cart", "in": { "testid": "item-card", "contains": "Keyboard" } }, + { "do": "click", "actor": "buyer", "testid": "cart-toggle" }, + { "do": "fill", "actor": "buyer", "testid": "cart-promotion", "text": "REPORT10" }, + { "do": "click", "actor": "buyer", "testid": "apply-promotion" }, + { "do": "click", "actor": "buyer", "testid": "checkout-submit" } + ], + "criteria": [ + { + "id": "622a", + "category": "feature", + "desc": "the promotion report has the exact redemption count", + "points": 1, + "statedBy": "Promotion redemption counts match orders that used the promotion.", + "steps": [ + { "do": "expectNumber", "actor": "staff", "testid": "promotion-redemptions", "in": { "testid": "promotion-report", "contains": "REPORT10" }, "equals": 1 } + ] + }, + { + "id": "622b", + "category": "feature", + "desc": "the promotion report has the exact discounted revenue", + "points": 2, + "statedBy": "Promotion revenue after discounts matches orders that used the promotion.", + "steps": [ + { "do": "expectNumber", "actor": "staff", "testid": "promotion-revenue", "in": { "testid": "promotion-report", "contains": "REPORT10" }, "equals": 80.1 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-promotion-rules.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-promotion-rules.json new file mode 100644 index 00000000000..00d5aa4de7a --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-promotion-rules.json @@ -0,0 +1,211 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 5, + "name": "Promotion rules", + "features": [ + { + "id": 620, + "name": "Staff-managed promotion rules", + "actors": [ + "staff", + "customer" + ], + "setup": [ + { + "do": "signIn", + "actor": "staff", + "name": "staff", + "password": "stackbench-staff-2026", + "exact": true + }, + { + "do": "signUp", + "actor": "customer", + "name": "promotion-customer" + }, + { "do": "click", "actor": "staff", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["staff-link", "promotions-link", "promotion-code"] }, + { "do": "click", "actor": "staff", "testid": "staff-link", "unlessVisible": ["promotions-link", "promotion-code"] } + ], + "criteria": [ + { + "id": "620a", + "category": "feature", + "desc": "staff can save every bounded promotion value", + "points": 2, + "statedBy": "Staff can create a promotion code with a discount, active period, and redemption limit.", + "steps": [ + { + "do": "click", + "actor": "staff", + "testid": "promotions-link", + "unlessVisible": "promotion-code" + }, + { + "do": "fill", + "actor": "staff", + "testid": "promotion-code", + "text": "SAVE10" + }, + { + "do": "fill", + "actor": "staff", + "testid": "promotion-discount", + "text": "10" + }, + { + "do": "fill", + "actor": "staff", + "testid": "promotion-start", + "text": "2099-01-01" + }, + { + "do": "fill", + "actor": "staff", + "testid": "promotion-end", + "text": "2099-12-31" + }, + { + "do": "fill", + "actor": "staff", + "testid": "promotion-limit", + "text": "2" + }, + { + "do": "click", + "actor": "staff", + "testid": "promotion-submit" + }, + { + "do": "expectNumber", + "actor": "staff", + "testid": "promotion-discount", + "in": { + "testid": "promotion-item", + "contains": "SAVE10" + }, + "equals": 10 + }, + { + "do": "expect", + "actor": "staff", + "testid": "promotion-start", + "in": { + "testid": "promotion-item", + "contains": "SAVE10" + }, + "contains": "2099-01-01" + }, + { + "do": "expect", + "actor": "staff", + "testid": "promotion-end", + "in": { + "testid": "promotion-item", + "contains": "SAVE10" + }, + "contains": "2099-12-31" + }, + { + "do": "expectNumber", + "actor": "staff", + "testid": "promotion-limit", + "in": { + "testid": "promotion-item", + "contains": "SAVE10" + }, + "equals": 2 + } + ] + }, + { + "id": "620b", + "category": "production", + "desc": "customers cannot create promotion rules", + "points": 2, + "statedBy": "Customers cannot manage promotion rules.", + "steps": [ + { "do": "click", "actor": "staff", "testid": "promotions-link", "unlessVisible": "promotion-code" }, + { "do": "fill", "actor": "staff", "testid": "promotion-code", "text": "ACCESS10" }, + { "do": "fill", "actor": "staff", "testid": "promotion-discount", "text": "10" }, + { "do": "fill", "actor": "staff", "testid": "promotion-start", "text": "2099-01-01" }, + { "do": "fill", "actor": "staff", "testid": "promotion-end", "text": "2099-12-31" }, + { "do": "fill", "actor": "staff", "testid": "promotion-limit", "text": "2" }, + { "do": "click", "actor": "staff", "testid": "promotion-submit" }, + { "do": "expect", "actor": "staff", "testid": "promotion-item", "contains": "ACCESS10", "within": 10000 }, + { "do": "reload", "actor": "staff", "settleMs": 1000 }, + { "do": "ensureSignedIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true, "readyTestid": "current-user" }, + { "do": "click", "actor": "staff", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["staff-link", "promotions-link", "promotion-code"] }, + { "do": "click", "actor": "staff", "testid": "staff-link", "ifAvailable": true, "unlessVisible": ["promotions-link", "promotion-code"] }, + { "do": "click", "actor": "staff", "testid": "promotions-link", "unlessVisible": "promotion-code" }, + { "do": "expect", "actor": "staff", "testid": "promotion-item", "contains": "ACCESS10", "within": 10000 }, + { "do": "replayAs", "actor": "staff", "from": "staff", "match": "__620b-authorized-create__", "namedAction": { "id": "createPromotion", "path": "/api/promotions", "reducer": "create_promotion", "args": ["PROOF10", 10, 4070908800000000, 4102444740000000, 2], "params": [{ "name": "code", "in": "body" }, { "name": "discountPercent", "in": "body" }, { "name": "startMicros", "in": "body" }, { "name": "endMicros", "in": "body" }, { "name": "usageLimit", "in": "body" }] }, "settleMs": 1500 }, + { "do": "expectReplayCompleted", "actor": "staff", "requireAccepted": true }, + { + "do": "replayAs", + "actor": "customer", + "from": "staff", + "match": "ACCESS10", + "swap": { + "find": "ACCESS10", + "with": "HACK10" + }, + "namedAction": { + "id": "createPromotion", + "path": "/api/promotions", + "reducer": "create_promotion", + "args": [ + "ACCESS10", + 10, + 4070908800000000, + 4102444740000000, + 2 + ], + "params": [ + { + "name": "code", + "in": "body" + }, + { + "name": "discountPercent", + "in": "body" + }, + { + "name": "startMicros", + "in": "body" + }, + { + "name": "endMicros", + "in": "body" + }, + { + "name": "usageLimit", + "in": "body" + } + ] + }, + "settleMs": 1500 + }, + { + "do": "expectReplayRejected", + "actor": "customer" + }, + { "do": "reload", "actor": "staff", "settleMs": 1000 }, + { "do": "ensureSignedIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true, "readyTestid": "current-user" }, + { "do": "click", "actor": "staff", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["staff-link", "promotions-link", "promotion-code"] }, + { "do": "click", "actor": "staff", "testid": "staff-link", "ifAvailable": true, "unlessVisible": ["promotions-link", "promotion-code"] }, + { "do": "click", "actor": "staff", "testid": "promotions-link", "unlessVisible": "promotion-code" }, + { "do": "expect", "actor": "staff", "testid": "promotion-item", "contains": "ACCESS10", "within": 10000 }, + { + "do": "expect", + "actor": "staff", + "testid": "promotion-item", + "contains": "HACK10", + "absent": true + } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-purchasing.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-purchasing.json new file mode 100644 index 00000000000..36c0500158f --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-purchasing.json @@ -0,0 +1,34 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Purchasing and orders", + "features": [ + { + "id": 3, + "name": "Purchase order history", + "actors": ["buyer"], + "setup": [ + { "do": "signUp", "actor": "buyer", "name": "purchase-buyer" }, + { "do": "click", "actor": "buyer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "3c", + "category": "feature", + "desc": "the purchase is recorded in the buyer's order history at the price paid", + "points": 1, + "statedBy": "A purchase creates an order for the customer at the price paid.", + "steps": [ + { "do": "click", "actor": "buyer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Coffee Grinder" } }, + {"do":"click","actor":"buyer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do": "click", "actor": "buyer", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + { "do": "click", "actor": "buyer", "testid": "orders-toggle", "unlessVisible": "order-item" }, + { "do": "expect", "actor": "buyer", "testid": "order-item", "contains": "Coffee Grinder" }, + { "do": "expectNumber", "actor": "buyer", "testid": "order-total", "in": { "testid": "order-item", "contains": "Coffee Grinder" }, "equals": 64 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-recommendation-feedback.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-recommendation-feedback.json new file mode 100644 index 00000000000..bd5bfdfe1f5 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-recommendation-feedback.json @@ -0,0 +1,116 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 5, + "name": "Recommendation feedback", + "features": [ + { + "id": 504, + "name": "Recommendation dismissal", + "actors": ["customer", "other"], + "setup": [ + { "do": "signUp", "actor": "customer", "name": "feedback-owner" }, + { "do": "click", "actor": "customer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Bluetooth Speaker" } }, + { "do": "signUp", "actor": "other", "name": "feedback-other" }, + { "do": "click", "actor": "other", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Bluetooth Speaker" } }, + { "do": "expect", "actor": "customer", "testid": "recommended-item", "contains": "Headphones" }, + { "do": "expect", "actor": "other", "testid": "recommended-item", "contains": "Headphones" }, + { "do": "click", "actor": "customer", "testid": "dismiss-recommendation", "in": { "testid": "recommended-item", "contains": "Headphones" } } + ], + "criteria": [ + { + "id": "504a", + "category": "feature", + "desc": "dismissing a recommendation removes it from the customer view", + "points": 2, + "statedBy": "Customers can dismiss a recommendation.", + "steps": [ + { + "do": "expect", + "actor": "customer", + "testid": "recommended-item", + "contains": "Headphones", + "absent": true, + "within": 10000 + } + ] + }, + { + "id": "504b", + "category": "production", + "desc": "one customer's dismissal does not hide another customer's result", + "points": 2, + "statedBy": "Recommendation feedback belongs to one customer.", + "steps": [ + { + "do": "waitUntilAbsent", + "actor": "customer", + "testid": "recommended-item", + "contains": "Headphones", + "within": 10000 + }, + { + "do": "expect", + "actor": "other", + "testid": "recommended-item", + "contains": "Headphones" + } + ] + }, + { + "id": "504c", + "category": "production", + "desc": "a dismissed recommendation stays absent after reload and backend restart in a fresh browser", + "points": 2, + "statedBy": "A dismissed recommendation stays dismissed for that customer.", + "steps": [ + { + "do": "reload", + "actor": "customer", + "settleMs": 3000 + }, + { + "do": "ensureSignedIn", + "actor": "customer", + "name": "feedback-owner", + "readyTestid": "current-user" + }, + { + "do": "expect", + "actor": "customer", + "testid": "recommended-item", + "contains": "Headphones", + "absent": true + }, + { + "do": "restartBackend", + "settleMs": 1000 + }, + { + "do": "freshClient", + "actor": "customer" + }, + { + "do": "signIn", + "actor": "customer-fresh", + "name": "feedback-owner" + }, + { + "do": "expect", + "actor": "customer-fresh", + "testid": "recommendations", + "within": 10000 + }, + { + "do": "expect", + "actor": "customer-fresh", + "testid": "recommended-item", + "contains": "Headphones", + "absent": true + } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-review-access.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-review-access.json new file mode 100644 index 00000000000..f4768f20776 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-review-access.json @@ -0,0 +1,48 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 3, + "name": "Review access", + "note": "Review eligibility includes a nonbuyer claiming the username of a buyer. The normal nonbuyer call and a call with an extra username must both be refused without changing the buyer's review. HTTP sends username in the body; the native reducer receives an extra positional argument. Native argument rejection is a valid defense, not evidence that an application ownership branch ran. This does not cover arbitrary account-ID fields or other identity formats.", + "features": [{ + "id": 618, + "name": "Review access", + "actors": ["owner", "stranger"], + "setup": [ + {"do":"signUp","actor":"owner","name":"claim-review-owner","exact":true}, + {"do":"signUp","actor":"stranger","name":"claim-review-stranger","exact":true}, + { "do": "click", "actor": "owner", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do":"callAction","actor":"owner","action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"},"settleMs":0}, + {"do":"expectActionOutcome","actor":"owner","outcome":"accepted"}, + {"do":"callAction","actor":"owner","action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Desk Lamp","attribute":"data-buy-input"},"settleMs":0}, + {"do":"expectActionOutcome","actor":"owner","outcome":"accepted"}, + { "do": "click", "actor": "stranger", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [{ + "id":"618a", "category":"production", "points":2, + "desc":"a nonbuyer cannot submit or replace a review by claiming a buyer's username", + "statedBy":"Only a customer who bought an item can review it.", + "steps":[ + {"do":"callAction","actor":"owner","action":"submitReview","namedAction":{"id":"submitReview","path":"/api/items/:id/reviews","method":"POST","reducer":"submit_review","args":[0,5,"Original buyer review"],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"},{"name":"rating","in":"body"},{"name":"comment","in":"body"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input","fields":["itemId"]},"settleMs":0}, + {"do":"expectActionOutcome","actor":"owner","outcome":"accepted"}, + {"do":"callAction","actor":"stranger","action":"submitReview","namedAction":{"id":"submitReview","path":"/api/items/:id/reviews","method":"POST","reducer":"submit_review","args":[0,5,"Nonbuyer review"],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"},{"name":"rating","in":"body"},{"name":"comment","in":"body"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input","fields":["itemId"]},"settleMs":0}, + {"do":"expectActionOutcome","actor":"stranger","outcome":"application-refused","routeProvenBy":"owner"}, + {"do":"callAction","actor":"stranger","action":"submitReview","namedAction":{"id":"submitReview","path":"/api/items/:id/reviews","method":"POST","reducer":"submit_review","args":[0,5,"Forged buyer review","claim-review-owner"],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"},{"name":"rating","in":"body"},{"name":"comment","in":"body"},{"name":"username","in":"body"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input","fields":["itemId"]},"settleMs":0}, + {"do":"expectActionOutcome","actor":"stranger","outcome":"application-refused","routeProvenBy":"owner"}, + {"do":"callAction","actor":"stranger","action":"submitReview","namedAction":{"id":"submitReview","path":"/api/items/:id/reviews","method":"POST","reducer":"submit_review","args":[0,5,"Forged unreviewed purchase","claim-review-owner"],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"},{"name":"rating","in":"body"},{"name":"comment","in":"body"},{"name":"username","in":"body"}]},"input":{"testid":"item-card","contains":"Desk Lamp","attribute":"data-buy-input","fields":["itemId"]},"settleMs":0}, + {"do":"expectActionOutcome","actor":"stranger","outcome":"application-refused","routeProvenBy":"owner"}, + {"do":"freshClient","actor":"owner"}, + {"do":"signIn","actor":"owner-fresh","name":"claim-review-owner","exact":true}, + { "do": "click", "actor": "owner-fresh", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do":"openItem","actor":"owner-fresh","item":"Keyboard"}, + {"do":"expect","actor":"owner-fresh","testid":"review-item","contains":"Original buyer review","within":10000}, + {"do":"expect","actor":"owner-fresh","testid":"review-item","contains":"Nonbuyer review","absent":true,"within":3000}, + {"do":"expect","actor":"owner-fresh","testid":"review-item","contains":"Forged buyer review","absent":true,"within":3000}, + { "do": "click", "actor": "owner-fresh", "testid": "overlay-close", "ifAvailable": true }, + { "do": "click", "actor": "owner-fresh", "testid": "catalog-link", "ifAvailable": true }, + {"do":"openItem","actor":"owner-fresh","item":"Desk Lamp"}, + {"do":"expect","actor":"owner-fresh","testid":"review-item","contains":"Forged unreviewed purchase","absent":true,"within":3000} + ] + }] + }] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-review-script.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-review-script.json new file mode 100644 index 00000000000..47c7e89e58d --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-review-script.json @@ -0,0 +1,46 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 3, + "name": "Stored review script safety", + "note": "A separate eligible buyer must store benign content and existing and fresh readers must display it. The attack may be rejected or safely displayed. Server errors and unknown outcomes cannot pass. The grader-owned browser binding records script execution outside the page. Observe one second after rendering; this is a bounded stored-review script check, not comprehensive XSS coverage. Missing observer is unmeasured.", + "features": [{ + "id": 9180, + "name": "Stored review content", + "actors": ["owner","writer","reader"], + "setup": [ + {"do":"signUp","actor":"owner","name":"script-review-owner"}, + {"do":"signUp","actor":"reader","name":"script-review-reader"}, + {"do":"signUp","actor":"writer","name":"script-review-writer"}, + { "do": "click", "actor": "owner", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do":"callAction","actor":"owner","action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"}}, + {"do":"expectActionOutcome","actor":"owner","outcome":"accepted"}, + { "do": "click", "actor": "writer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do":"callAction","actor":"writer","action":"buy","namedAction":{"id":"buy","path":"/api/items/:id/buy","reducer":"buy_now","args":[0],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input"}}, + {"do":"expectActionOutcome","actor":"writer","outcome":"accepted"}, + {"do":"armScriptCanary","actor":"owner"} + ], + "criteria": [{ + "id": "9180a", "category": "production", "points": 1, + "desc": "review input is rejected or displayed without executing script in another customer session", + "steps": [ + {"do":"callAction","actor":"owner","action":"submitReview","namedAction":{"id":"submitReview","path":"/api/items/:id/reviews","method":"POST","reducer":"submit_review","args":[0,5,"Benign stored review"],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"},{"name":"rating","in":"body"},{"name":"comment","in":"body"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input","fields":["itemId"]},"settleMs":0}, + {"do":"expectActionOutcome","actor":"owner","outcome":"accepted"}, + {"do":"callAction","actor":"writer","action":"submitReview","namedAction":{"id":"submitReview","path":"/api/items/:id/reviews","method":"POST","reducer":"submit_review","args":[0,5,"Stored review marker "],"params":[{"name":"itemId","in":"path","placeholder":":id","wireType":"u64"},{"name":"rating","in":"body"},{"name":"comment","in":"body"}]},"input":{"testid":"item-card","contains":"Keyboard","attribute":"data-buy-input","fields":["itemId"]},"settleMs":0}, + {"do":"expectActionOutcome","actor":"writer","outcome":"completed","routeProvenBy":"owner"}, + {"do":"openItem","actor":"owner","item":"Keyboard"}, + {"do":"expect","actor":"owner","testid":"review-item","contains":"Benign stored review","within":10000}, + {"do":"wait","actor":"owner","ms":1000}, + {"do":"expectNoScriptExecution","actor":"owner"}, + {"do":"freshClient","actor":"reader"}, + {"do":"signIn","actor":"reader-fresh","name":"script-review-reader"}, + {"do":"armScriptCanary","actor":"reader-fresh"}, + { "do": "click", "actor": "reader-fresh", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do":"openItem","actor":"reader-fresh","item":"Keyboard"}, + {"do":"expect","actor":"reader-fresh","testid":"review-item","contains":"Benign stored review","within":10000}, + {"do":"wait","actor":"reader-fresh","ms":1000}, + {"do":"expectNoScriptExecution","actor":"reader-fresh"} + ] + }] + }] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-search-ordering.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-search-ordering.json new file mode 100644 index 00000000000..f6f1d69e2e3 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-search-ordering.json @@ -0,0 +1,162 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 3, + "name": "Search ordering after purchases", + "note": "Draft: requires matching reference and defect-control qualification.", + "features": [ + { + "id": 402, + "name": "Purchases preserve search ordering", + "actors": [ + "visitor", + "buyer" + ], + "setup": [ + { + "do": "signUp", + "actor": "buyer", + "name": "search-order-buyer" + }, + { + "do": "dbRecordStock", + "item": "Headphones", + "as": "before-search-purchase" + }, + { "do": "click", "actor": "buyer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { + "do": "click", + "actor": "buyer", + "testid": "buy-now", + "in": { + "testid": "item-card", + "contains": "Headphones" + } + }, + { + "do": "dbExpectStock", + "item": "Headphones", + "relativeTo": "before-search-purchase", + "plus": -1 + }, + { + "do": "reload", + "actor": "visitor", + "settleMs": 0 + }, + { "do": "click", "actor": "visitor", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "minimum-price" }, + { + "do": "fill", + "actor": "visitor", + "testid": "minimum-price", + "text": "1" + }, + { "do": "pressKey", "actor": "visitor", "key": "Tab", "settleMs": 0 }, + { + "do": "click", + "actor": "visitor", + "testid": "filter-apply", + "ifAvailable": true + } + ], + "criteria": [ + { + "id": "402b", + "category": "production", + "desc": "active filters and text searches remain alphabetical after purchases; clearing them restores purchase ranking", + "points": 1, + "statedBy": "Active search and filter results use item-name order; clearing them restores purchase ranking", + "steps": [ + { + "do": "expectSequence", + "actor": "visitor", + "testid": "item-name", + "in": { + "testid": "search-results" + }, + "equals": [ + "Air Purifier", + "Bluetooth Speaker", + "Coffee Grinder", + "Desk Lamp", + "Espresso Machine", + "Gaming Mouse", + "Headphones", + "Induction Cooktop", + "Keyboard", + "Laptop Stand" + ] + }, + { + "do": "fill", + "actor": "visitor", + "testid": "search-input", + "text": "e", + "enter": true + }, + { + "do": "fill", + "actor": "visitor", + "testid": "minimum-price", + "text": "" + }, + { "do": "pressKey", "actor": "visitor", "key": "Tab", "settleMs": 0 }, + { + "do": "click", + "actor": "visitor", + "testid": "filter-apply", + "ifAvailable": true + }, + { + "do": "expectSequence", + "actor": "visitor", + "testid": "item-name", + "in": { + "testid": "search-results" + }, + "equals": [ + "Air Purifier", + "Bluetooth Speaker", + "Coffee Grinder", + "Desk Lamp", + "Espresso Machine", + "Gaming Mouse", + "Headphones", + "Keyboard", + "Mirrorless Camera", + "USB Cable" + ] + }, + { + "do": "fill", + "actor": "visitor", + "testid": "search-input", + "text": "", + "enter": true + }, + { + "do": "expectSequence", + "actor": "visitor", + "testid": "item-name", + "in": { + "testid": "item-list" + }, + "equals": [ + "Headphones", + "Air Purifier", + "Bluetooth Speaker", + "Coffee Grinder", + "Desk Lamp", + "Espresso Machine", + "Gaming Mouse", + "Induction Cooktop", + "Keyboard", + "Laptop Stand" + ] + } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-shipping-accounting.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-shipping-accounting.json new file mode 100644 index 00000000000..51f0c55fcb0 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-shipping-accounting.json @@ -0,0 +1,74 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 3, + "name": "Shipping accounting", + "features": [ + { + "id": 202, + "name": "Shipping preserves completed purchase accounting", + "actors": [ + "customer", + "admin", + "staff" + ], + "setup": [ + {"do":"signUp","actor":"customer","name":"shipping-accounting"}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + {"do":"click","actor":"admin","testid":"admin-link", "unlessVisible": "admin-panel"}, + {"do":"signIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true}, + {"do": "click", "actor": "staff", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["staff-link", "fulfilment-panel"]}, + {"do":"click","actor":"staff","testid":"staff-link", "unlessVisible": "fulfilment-panel"}, + { "do": "click", "actor": "customer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "202e", + "category": "production", + "desc": "shipping a purchased order does not deduct stock or add revenue again", + "points": 2, + "statedBy": "Shipping preserves the stock and revenue recorded for the purchase.", + "steps": [ + {"do":"dbRecordStock","item":"Keyboard","as":"stock-before-purchase"}, + {"do":"recordNumber","actor":"admin","testid":"admin-revenue","as":"revenue-before-purchase"}, + {"do":"click","actor":"customer","testid":"buy-now","in":{"testid":"item-card","contains":"Keyboard"}}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do": "click", "actor": "customer", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + {"do":"click","actor":"customer","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"customer","testid":"order-status","in":{"testid":"order-item","contains":"Keyboard"},"statusText":"pending", "ignoreCase": true,"within":10000}, + {"do":"dbExpectStock","item":"Keyboard","relativeTo":"stock-before-purchase","plus":-1}, + {"do":"reload","actor":"admin","settleMs":2000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true,"readyTestid":"current-user"}, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + {"do":"click","actor":"admin","testid":"admin-link","ifAvailable":true, "unlessVisible": "admin-panel"}, + {"do":"expectNumber","actor":"admin","testid":"admin-revenue","relativeTo":"revenue-before-purchase","plus":89,"within":10000}, + {"do":"recordNumber","actor":"admin","testid":"admin-revenue","as":"revenue-before-ship"}, + {"do":"dbRecordStock","item":"Keyboard","warehouse":"East","as":"East-before-ship"}, + {"do":"dbRecordStock","item":"Keyboard","warehouse":"West","as":"West-before-ship"}, + {"do":"reload","actor":"staff","settleMs":2000}, + {"do":"ensureSignedIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true,"readyTestid":"current-user"}, + {"do": "click", "actor": "staff", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["staff-link", "fulfilment-panel"]}, + {"do":"click","actor":"staff","testid":"staff-link","ifAvailable":true, "unlessVisible": "fulfilment-panel"}, + {"do":"callAction","actor":"staff","action":"ship","namedAction":{"id":"ship","path":"/api/fulfilment/ship","reducer":"ship_order","args":[0],"params":[{"name":"orderId","in":"body","wireType":"u64"}]},"input":{"testid":"order-item","contains":"Keyboard","attribute":"data-ship-input"},"from":"customer","settleMs":0}, + {"do":"expectActionOutcome","actor":"staff","outcome":"accepted"}, + {"do":"reload","actor":"customer","settleMs":2000}, + {"do":"ensureSignedIn","actor":"customer","name":"shipping-accounting","readyTestid":"current-user"}, + {"do":"click","actor":"customer","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { "do": "click", "actor": "customer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": ["item-list", "order-item"] }, + {"do": "click", "actor": "customer", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["orders-toggle", "order-item"]}, + {"do":"click","actor":"customer","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expect","actor":"customer","testid":"order-status","in":{"testid":"order-item","contains":"Keyboard"},"statusText":"shipped", "ignoreCase": true,"within":10000}, + {"do":"dbExpectStock","item":"Keyboard","warehouse":"East","relativeTo":"East-before-ship","plus":0}, + {"do":"dbExpectStock","item":"Keyboard","warehouse":"West","relativeTo":"West-before-ship","plus":0}, + {"do":"reload","actor":"admin","settleMs":2000}, + {"do":"ensureSignedIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true,"readyTestid":"current-user"}, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + {"do":"click","actor":"admin","testid":"admin-link","ifAvailable":true, "unlessVisible": "admin-panel"}, + {"do":"expectNumber","actor":"admin","testid":"admin-revenue","relativeTo":"revenue-before-ship","plus":0,"within":10000} + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-signed-out-purchase.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-signed-out-purchase.json new file mode 100644 index 00000000000..69d89a5ecdf --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-signed-out-purchase.json @@ -0,0 +1,45 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Progression signed-out purchase access", + "note": "Draft: wait for the signed-in native order and drain writes captured during the guest click and its existing settle interval before reload. Affected qualification is pending; first dispatch after capture closes is outside this bounded observation.", + "features": [ + { + "id": 3, + "name": "Buying", + "actors": ["buyer", "visitor"], + "setup": [ + { "do": "signUp", "actor": "buyer", "name": "signed-in-buyer" }, + { "do": "click", "actor": "visitor", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + { "do": "expect", "actor": "visitor", "testid": "item-card", "contains": "Keyboard" }, + { "do": "click", "actor": "buyer", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" } + ], + "criteria": [ + { + "id": "3a", + "category": "production", + "desc": "using the purchase control while signed out does not buy an item", + "points": 1, + "statedBy": "Unauthenticated callers cannot purchase.", + "steps": [ + { "do": "recordNumber", "actor": "visitor", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Keyboard" }, "as": "keyboard-before-buyer" }, + { "do": "dbRecordCheckout", "account": "{user:signed-in-buyer}", "item": "Keyboard", "as": "buyer-orders", "storage": { "kind": "order-data", "cart": false, "warehouses": false } }, + { "do": "click", "actor": "buyer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Keyboard" } }, + { "do": "dbExpectPurchaseCount", "before": ["buyer-orders"], "purchasesEach": 1, "within": 10000 }, + { "do": "reload", "actor": "visitor", "settleMs": 2000 }, + { "do": "click", "actor": "visitor", "testid": "overlay-close", "ifAvailable": true }, + { "do": "click", "actor": "visitor", "testid": "catalog-link", "ifAvailable": true }, + { "do": "expectNumber", "actor": "visitor", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Keyboard" }, "relativeTo": "keyboard-before-buyer", "plus": -1 }, + { "do": "recordNumber", "actor": "visitor", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Keyboard" }, "as": "keyboard-before-guest" }, + { "do": "click", "actor": "visitor", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Keyboard" }, "ifAvailable": true, "settleMs": 1500, "awaitWrites": true }, + { "do": "reload", "actor": "visitor", "settleMs": 2000 }, + { "do": "click", "actor": "visitor", "testid": "overlay-close", "ifAvailable": true }, + { "do": "click", "actor": "visitor", "testid": "catalog-link", "ifAvailable": true }, + { "do": "expectNumber", "actor": "visitor", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Keyboard" }, "relativeTo": "keyboard-before-guest", "plus": 0 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-split-tender-refunds.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-split-tender-refunds.json new file mode 100644 index 00000000000..c44ec569deb --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-split-tender-refunds.json @@ -0,0 +1,152 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 6, + "name": "Refund allocation", + "note": "Draft: revised observations require matching reference and defect-control qualification.", + "features": [ + { + "id": 751, + "name": "Full refund restores each original payment portion", + "actors": [ + "owner", + "staff" + ], + "setup": [ + {"do":"signUp","actor":"owner","name":"credit-owner-751"}, + {"do":"click","actor":"owner","testid":"credit-link","unlessVisible":"credit-panel"}, + {"do":"expect","actor":"owner","testid":"credit-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"signIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true}, + {"do":"click","actor":"staff","testid":"credit-link","unlessVisible":"credit-panel"}, + {"do":"expect","actor":"staff","testid":"credit-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"fill","actor":"staff","testid":"credit-amount-input","text":"10.00"}, + {"do":"fill","actor":"staff","testid":"credit-reference-input","text":"credit-751"}, + {"do":"callAction","actor":"staff","from":"staff","action":"grantCredit","namedAction":{"id":"grantCredit","path":"/api/staff/credit","method":"POST","reducer":"grant_credit","args":[0,1000,""],"params":[{"name":"accountId","in":"body","wireType":"u64"},{"name":"amountMinor","in":"body"},{"name":"reference","in":"body"}]},"input":{"testid":"credit-grant","attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","testid":"credit-panel","attribute":"data-account-id"}}},"settleMs":0}, + {"do":"expectActionOutcome","actor":"staff","outcome":"accepted"}, + {"do":"reload","actor":"owner","settleMs":1000}, + {"do":"ensureSignedIn","actor":"owner","name":"credit-owner-751","readyTestid":"current-user"}, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"owner","testid":"credit-link","unlessVisible":"credit-balance"}, + {"do":"expectNumber","actor":"owner","testid":"credit-balance","equals":10,"within":10000}, + {"do":"click","actor":"owner","testid":"catalog-link"}, + {"do":"click","actor":"owner","testid":"add-to-cart","in":{"testid":"item-card","contains":"Desk Lamp"}}, + {"do":"click","actor":"owner","testid":"cart-toggle"}, + {"do":"expectNumber","actor":"owner","testid":"cart-total","equals":42}, + {"do":"click","actor":"owner","testid":"credit-checkout"}, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"owner","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expectNumber","actor":"owner","testid":"payment-credit-amount","equals":10,"in":{"testid":"order-item","contains":"Desk Lamp"},"within":10000}, + {"do":"expectNumber","actor":"owner","testid":"payment-external-amount","equals":32,"in":{"testid":"order-item","contains":"Desk Lamp"}}, + {"do":"click","actor":"owner","testid":"support-link"}, + {"do":"fill","actor":"owner","testid":"support-subject","text":"Split refund 751"}, + {"do":"fill","actor":"owner","testid":"support-message","text":"Please refund this order."}, + {"do":"click","actor":"owner","testid":"support-submit"}, + {"do":"click","actor":"owner","testid":"support-order-option","contains":"Desk Lamp","in":{"testid":"support-ticket","contains":"Split refund 751"}}, + {"do":"click","actor":"owner","testid":"support-link-order","in":{"testid":"support-ticket","contains":"Split refund 751"}}, + {"do":"click","actor":"staff","testid":"staff-link","unlessVisible":"support-assignee"}, + {"do":"click","actor":"staff","testid":"support-queue-link","unlessVisible":"support-assignee","ifAvailable":true,"within":1000} + ], + "criteria": [ + { + "id": "751a", + "category": "feature", + "desc": "Full refund restores each original payment portion", + "points": 2, + "steps": [ + {"do":"click","awaitWrites":true,"actor":"staff","testid":"support-refund","in":{"testid":"support-ticket","contains":"Split refund 751"}}, + {"do":"freshClient","actor":"owner"}, + {"do":"signIn","actor":"owner-fresh","name":"credit-owner-751"}, + {"do":"click","actor":"owner-fresh","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"owner-fresh","testid":"orders-toggle"}, + {"do":"expectElementCount","actor":"owner-fresh","testid":"refund-entry","contains":"Desk Lamp","equals":1,"within":10000}, + {"do":"expectNumber","actor":"owner-fresh","testid":"order-refund-total","equals":42,"in":{"testid":"order-item","contains":"Desk Lamp"}}, + {"do":"expectNumber","actor":"owner-fresh","testid":"refund-credit-amount","equals":10,"in":{"testid":"refund-entry","contains":"Desk Lamp"}}, + {"do":"expectNumber","actor":"owner-fresh","testid":"refund-external-amount","equals":32,"in":{"testid":"refund-entry","contains":"Desk Lamp"}}, + {"do":"click","actor":"owner-fresh","testid":"credit-link"}, + {"do":"expect","actor":"owner-fresh","testid":"credit-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"expectNumber","actor":"owner-fresh","testid":"credit-balance","equals":10} + ] + } + ] + }, + { + "id": 756, + "name": "Concurrent refunds do not duplicate credit after restart", + "actors": [ + "owner", + "staff", + "staff2" + ], + "setup": [ + {"do":"signUp","actor":"owner","name":"credit-owner-756"}, + {"do":"click","actor":"owner","testid":"credit-link","unlessVisible":"credit-panel"}, + {"do":"expect","actor":"owner","testid":"credit-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"signIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true}, + {"do":"click","actor":"staff","testid":"credit-link","unlessVisible":"credit-panel"}, + {"do":"expect","actor":"staff","testid":"credit-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"fill","actor":"staff","testid":"credit-amount-input","text":"10.00"}, + {"do":"fill","actor":"staff","testid":"credit-reference-input","text":"credit-756"}, + {"do":"callAction","actor":"staff","from":"staff","action":"grantCredit","namedAction":{"id":"grantCredit","path":"/api/staff/credit","method":"POST","reducer":"grant_credit","args":[0,1000,""],"params":[{"name":"accountId","in":"body","wireType":"u64"},{"name":"amountMinor","in":"body"},{"name":"reference","in":"body"}]},"input":{"testid":"credit-grant","attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","testid":"credit-panel","attribute":"data-account-id"}}},"settleMs":0}, + {"do":"expectActionOutcome","actor":"staff","outcome":"accepted"}, + {"do":"reload","actor":"owner","settleMs":1000}, + {"do":"ensureSignedIn","actor":"owner","name":"credit-owner-756","readyTestid":"current-user"}, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"owner","testid":"credit-link","unlessVisible":"credit-balance"}, + {"do":"expectNumber","actor":"owner","testid":"credit-balance","equals":10,"within":10000}, + {"do":"click","actor":"owner","testid":"catalog-link"}, + {"do":"click","actor":"owner","testid":"add-to-cart","in":{"testid":"item-card","contains":"Desk Lamp"}}, + {"do":"click","actor":"owner","testid":"cart-toggle"}, + {"do":"expectNumber","actor":"owner","testid":"cart-total","equals":42}, + {"do":"click","actor":"owner","testid":"credit-checkout"}, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"owner","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expectNumber","actor":"owner","testid":"payment-credit-amount","equals":10,"in":{"testid":"order-item","contains":"Desk Lamp"},"within":10000}, + {"do":"expectNumber","actor":"owner","testid":"payment-external-amount","equals":32,"in":{"testid":"order-item","contains":"Desk Lamp"}}, + {"do":"click","actor":"owner","testid":"support-link"}, + {"do":"fill","actor":"owner","testid":"support-subject","text":"Split refund 756"}, + {"do":"fill","actor":"owner","testid":"support-message","text":"Please refund this order."}, + {"do":"click","actor":"owner","testid":"support-submit"}, + {"do":"click","actor":"owner","testid":"support-order-option","contains":"Desk Lamp","in":{"testid":"support-ticket","contains":"Split refund 756"}}, + {"do":"click","actor":"owner","testid":"support-link-order","in":{"testid":"support-ticket","contains":"Split refund 756"}}, + {"do":"click","actor":"staff","testid":"staff-link","unlessVisible":"support-assignee"}, + {"do":"click","actor":"staff","testid":"support-queue-link","unlessVisible":"support-assignee","ifAvailable":true,"within":1000}, + {"do":"signIn","actor":"staff2","name":"staff","password":"stackbench-staff-2026","exact":true} + ], + "criteria": [ + { + "id": "756a", + "category": "production", + "desc": "Concurrent refunds restore the original credit and external amounts once, including after restart", + "points": 2, + "steps": [ + {"do":"callConcurrently","actors":["staff","staff2"],"action":"supportRefund","namedAction":{"id":"supportRefund","path":"/api/support/cases/{caseId}/refund","method":"POST","reducer":"support_refund","args":[0],"params":[{"name":"caseId","in":"path","placeholder":"{caseId}","wireType":"u64"}]},"from":"staff","input":{"testid":"support-ticket","contains":"Split refund 756","attribute":"data-refund-input"},"settleMs":0}, + {"do":"expectCallOutcomes"}, + {"do":"freshClient","actor":"owner"}, + {"do":"signIn","actor":"owner-fresh","name":"credit-owner-756"}, + {"do":"click","actor":"owner-fresh","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"owner-fresh","testid":"orders-toggle"}, + {"do":"expectElementCount","actor":"owner-fresh","testid":"refund-entry","contains":"Desk Lamp","equals":1,"within":10000}, + {"do":"expectNumber","actor":"owner-fresh","testid":"order-refund-total","equals":42,"in":{"testid":"order-item","contains":"Desk Lamp"}}, + {"do":"expectNumber","actor":"owner-fresh","testid":"refund-credit-amount","equals":10,"in":{"testid":"refund-entry","contains":"Desk Lamp"}}, + {"do":"expectNumber","actor":"owner-fresh","testid":"refund-external-amount","equals":32,"in":{"testid":"refund-entry","contains":"Desk Lamp"}}, + {"do":"click","actor":"owner-fresh","testid":"credit-link"}, + {"do":"expect","actor":"owner-fresh","testid":"credit-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"expectNumber","actor":"owner-fresh","testid":"credit-balance","equals":10}, + {"do":"restartBackend","settleMs":1000}, + {"do":"freshClient","actor":"owner"}, + {"do":"signIn","actor":"owner-fresh","name":"credit-owner-756"}, + {"do":"click","actor":"owner-fresh","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"owner-fresh","testid":"orders-toggle"}, + {"do":"expectElementCount","actor":"owner-fresh","testid":"refund-entry","contains":"Desk Lamp","equals":1,"within":10000}, + {"do":"expectNumber","actor":"owner-fresh","testid":"order-refund-total","equals":42,"in":{"testid":"order-item","contains":"Desk Lamp"}}, + {"do":"expectNumber","actor":"owner-fresh","testid":"refund-credit-amount","equals":10,"in":{"testid":"refund-entry","contains":"Desk Lamp"}}, + {"do":"expectNumber","actor":"owner-fresh","testid":"refund-external-amount","equals":32,"in":{"testid":"refund-entry","contains":"Desk Lamp"}}, + {"do":"click","actor":"owner-fresh","testid":"credit-link"}, + {"do":"expect","actor":"owner-fresh","testid":"credit-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"expectNumber","actor":"owner-fresh","testid":"credit-balance","equals":10} + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-staff-access.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-staff-access.json new file mode 100644 index 00000000000..161d38037f8 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-staff-access.json @@ -0,0 +1,50 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Staff access", + "features": [ + { + "id": 601, + "name": "Staff access", + "actors": ["customer", "staff", "admin", "authorized"], + "setup": [], + "criteria": [ + { + "id": "601a", + "category": "feature", + "desc": "staff and administrators can sign in and open staff tools", + "points": 2, + "statedBy": "Staff and administrators can sign in and use staff areas.", + "steps": [ + { "do": "signIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true }, + { "do": "click", "actor": "staff", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["staff-link", "staff-area"] }, + { "do": "click", "actor": "staff", "testid": "staff-link", "ifAvailable": true, "unlessVisible": "staff-area" }, + { "do": "expect", "actor": "staff", "testid": "staff-area", "within": 6000 }, + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + { "do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["staff-link", "staff-area"] }, + { "do": "click", "actor": "admin", "testid": "staff-link", "ifAvailable": true, "unlessVisible": "staff-area" }, + { "do": "expect", "actor": "admin", "testid": "staff-area", "within": 6000 } + ] + }, + { + "id": "601b", + "category": "production", + "desc": "customers cannot open staff tools", + "points": 2, + "statedBy": "Customers cannot open staff tools.", + "steps": [ + { "do": "signIn", "actor": "authorized", "name": "staff", "password": "stackbench-staff-2026", "exact": true }, + { "do": "click", "actor": "authorized", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["staff-link", "staff-area"] }, + { "do": "click", "actor": "authorized", "testid": "staff-link", "unlessVisible": "staff-area" }, + { "do": "expect", "actor": "authorized", "testid": "staff-area" }, + { "do": "signIn", "actor": "customer", "name": "customer", "password": "stackbench-customer-2026", "exact": true }, + { "do": "click", "actor": "customer", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["staff-link", "staff-area"] }, + { "do": "click", "actor": "customer", "testid": "staff-link", "ifAvailable": true, "settleMs": 1500, "unlessVisible": "staff-area" }, + { "do": "expect", "actor": "customer", "testid": "staff-area", "absent": true } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-staff-activity.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-staff-activity.json new file mode 100644 index 00000000000..6c9fa5a29d9 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-staff-activity.json @@ -0,0 +1,52 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 5, + "name": "Staff activity history", + "features": [ + { + "id": 624, + "name": "Attributable staff activity", + "actors": ["admin", "staff", "customer"], + "setup": [ + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + { "do": "signIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true }, + { "do": "signIn", "actor": "customer", "name": "customer", "password": "stackbench-customer-2026", "exact": true }, + { "do": "click", "actor": "admin", "testid": "admin-link" }, + { "do": "fill", "actor": "admin", "testid": "catalog-name", "text": "Activity Mug" }, + { "do": "fill", "actor": "admin", "testid": "catalog-category", "text": "Kitchen" }, + { "do": "fill", "actor": "admin", "testid": "catalog-price", "text": "43.00" }, + { "do": "fill", "actor": "admin", "testid": "catalog-variants", "text": "Green" }, + { "do": "click", "actor": "admin", "testid": "catalog-save" } + ], + "criteria": [ + { + "id": "624a", + "category": "feature", + "desc": "an administrative change records its actor, action, subject, and time", + "points": 3, + "statedBy": "Each administrative change identifies its actor, action, subject, and time.", + "steps": [ + { "do": "click", "actor": "staff", "testid": "staff-link" }, + { "do": "click", "actor": "staff", "testid": "activity-link" }, + { "do": "expect", "actor": "staff", "testid": "activity-entry", "contains": "Activity Mug", "within": 10000 }, + { "do": "expect", "actor": "staff", "testid": "activity-actor", "in": { "testid": "activity-entry", "contains": "Activity Mug" }, "contains": "admin" }, + { "do": "expect", "actor": "staff", "testid": "activity-action", "in": { "testid": "activity-entry", "contains": "Activity Mug" }, "nonEmpty": true }, + { "do": "expect", "actor": "staff", "testid": "activity-subject", "in": { "testid": "activity-entry", "contains": "Activity Mug" }, "contains": "Activity Mug" }, + { "do": "expect", "actor": "staff", "testid": "activity-time", "in": { "testid": "activity-entry", "contains": "Activity Mug" } } + ] + }, + { + "id": "624b", + "category": "production", + "desc": "customers cannot open staff activity history", + "points": 2, + "statedBy": "Customers cannot open staff activity history.", + "steps": [ + { "do": "expect", "actor": "customer", "testid": "activity-link", "absent": true } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-staff-roles.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-staff-roles.json new file mode 100644 index 00000000000..170673b7373 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-staff-roles.json @@ -0,0 +1,163 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 2, + "name": "Staff roles", + "note": "Draft: updated observations require matching reference and defect-control qualification.", + "features": [ + { + "id": 621, + "name": "Staff roles", + "actors": ["admin", "replayAdmin", "staff", "roleAdmin", "promotedStaff"], + "setup": [ + { "do": "signIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + {"do":"click","actor":"admin","testid":"current-user","ifAvailable":true,"unlessVisible":["admin-link","staff-roles-link","staff-role-account-staff"]}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":["staff-roles-link","staff-role-account-staff"]}, + {"do":"click","actor":"admin","testid":"staff-roles-link","ifAvailable":true,"unlessVisible":"staff-role-account-staff"}, + { "do": "fill", "actor": "admin", "testid": "staff-role-select", "in": { "testid": "staff-role-account-staff" }, "text": "inventory" }, + { "do": "click", "actor": "admin", "testid": "staff-role-save", "in": { "testid": "staff-role-account-staff" } }, + { "do": "expect", "actor": "admin", "testid": "staff-role-account-staff", "attribute": "data-submit-state", "value": "succeeded", "within": 10000 } + ], + "criteria": [ + { + "id": "621c", + "category": "feature", + "desc": "an administrator can assign a staff role", + "points": 1, + "statedBy": "An administrator can assign a role to an existing staff account.", + "steps": [ + { "do": "expect", "actor": "admin", "testid": "staff-role-select", "in": { "testid": "staff-role-account-staff" }, "value": "inventory", "within": 10000 } + ] + }, + { + "id": "621a", + "category": "production", + "desc": "an assigned staff role survives reload and backend restart in a fresh browser", + "points": 2, + "statedBy": "Assigned staff roles persist.", + "steps": [ + { "do": "reload", "actor": "admin", "settleMs": 2500 }, + { "do": "ensureSignedIn", "actor": "admin", "name": "admin", "password": "stackbench-admin-2026", "exact": true, "readyTestid": "current-user" }, + {"do":"click","actor":"admin","testid":"current-user","ifAvailable":true,"unlessVisible":["admin-link","staff-roles-link","staff-role-account-staff"]}, + {"do":"click","actor":"admin","testid":"admin-link","unlessVisible":["staff-roles-link","staff-role-account-staff"],"ifAvailable":true}, + {"do":"click","actor":"admin","testid":"staff-roles-link","ifAvailable":true,"unlessVisible":"staff-role-account-staff"}, + { "do": "expect", "actor": "admin", "testid": "staff-role-select", "in": { "testid": "staff-role-account-staff" }, "value": "inventory", "within": 10000 }, + { "do": "restartBackend", "settleMs": 1000 }, + { "do": "freshClient", "actor": "admin" }, + { "do": "signIn", "actor": "admin-fresh", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + {"do":"click","actor":"admin-fresh","testid":"current-user","ifAvailable":true,"unlessVisible":["admin-link","staff-roles-link","staff-role-account-staff"]}, + {"do":"click","actor":"admin-fresh","testid":"admin-link","unlessVisible":["staff-roles-link","staff-role-account-staff"]}, + {"do":"click","actor":"admin-fresh","testid":"staff-roles-link","ifAvailable":true,"unlessVisible":"staff-role-account-staff"}, + { "do": "expect", "actor": "admin-fresh", "testid": "staff-role-select", "in": { "testid": "staff-role-account-staff" }, "value": "inventory", "within": 10000 } + ] + }, + { + "id": "621b", + "category": "production", + "desc": "a staff member cannot assign or change roles", + "points": 2, + "statedBy": "A staff member cannot assign or change roles.", + "steps": [ + { "do": "reload", "actor": "replayAdmin", "settleMs": 0 }, + { "do": "signIn", "actor": "replayAdmin", "name": "admin", "password": "stackbench-admin-2026", "exact": true }, + {"do":"click","actor":"replayAdmin","testid":"current-user","ifAvailable":true,"unlessVisible":["admin-link","staff-roles-link","staff-role-account-staff"]}, + {"do":"click","actor":"replayAdmin","testid":"admin-link","unlessVisible":["staff-roles-link","staff-role-account-staff"]}, + {"do":"click","actor":"replayAdmin","testid":"staff-roles-link","ifAvailable":true,"unlessVisible":"staff-role-account-staff"}, + {"do":"replayAs","actor":"replayAdmin","from":"replayAdmin","match":"__role-authorization-named-action__","namedAction":{"id":"assignStaffRole","path":"/api/staff/:id/role","method":"PUT","reducer":"assign_staff_role","args":[0,"staff"],"params":[{"name":"accountId","in":"path","placeholder":":id","wireType":"u64"},{"name":"role","in":"body"}]},"namedTarget":{"testid":"staff-role-account-staff","attribute":"data-account-id","valueType":"string"},"settleMs":0}, + {"do":"expectReplayCompleted","actor":"replayAdmin","requireAccepted":true}, + { "do": "reload", "actor": "replayAdmin", "settleMs": 1000 }, + { "do": "ensureSignedIn", "actor": "replayAdmin", "name": "admin", "password": "stackbench-admin-2026", "exact": true, "readyTestid": "current-user" }, + {"do":"click","actor":"replayAdmin","testid":"current-user","ifAvailable":true,"unlessVisible":["admin-link","staff-roles-link","staff-role-account-staff"]}, + {"do":"click","actor":"replayAdmin","testid":"admin-link","unlessVisible":["staff-roles-link","staff-role-account-staff"],"ifAvailable":true}, + {"do":"click","actor":"replayAdmin","testid":"staff-roles-link","ifAvailable":true,"unlessVisible":"staff-role-account-staff"}, + { "do": "expect", "actor": "replayAdmin", "testid": "staff-role-select", "in": { "testid": "staff-role-account-staff" }, "value": "staff", "within": 10000 }, + {"do":"replayAs","actor":"replayAdmin","from":"replayAdmin","match":"__role-authorization-named-action__","namedAction":{"id":"assignStaffRole","path":"/api/staff/:id/role","method":"PUT","reducer":"assign_staff_role","args":[0,"inventory"],"params":[{"name":"accountId","in":"path","placeholder":":id","wireType":"u64"},{"name":"role","in":"body"}]},"namedTarget":{"testid":"staff-role-account-staff2","attribute":"data-account-id","valueType":"string"},"settleMs":0}, + {"do":"expectReplayCompleted","actor":"replayAdmin","requireAccepted":true}, + { "do": "reload", "actor": "replayAdmin", "settleMs": 0 }, + { "do": "ensureSignedIn", "actor": "replayAdmin", "name": "admin", "password": "stackbench-admin-2026", "exact": true, "readyTestid": "current-user" }, + {"do":"click","actor":"replayAdmin","testid":"current-user","ifAvailable":true,"unlessVisible":["admin-link","staff-roles-link","staff-role-account-staff2"]}, + {"do":"click","actor":"replayAdmin","testid":"admin-link","ifAvailable":true,"unlessVisible":["staff-roles-link","staff-role-account-staff2"]}, + {"do":"click","actor":"replayAdmin","testid":"staff-roles-link","ifAvailable":true,"unlessVisible":"staff-role-account-staff2"}, + {"do":"expect","actor":"replayAdmin","testid":"staff-role-select","in":{"testid":"staff-role-account-staff2"},"value":"inventory","within":10000}, + {"do":"replayAs","actor":"replayAdmin","from":"replayAdmin","match":"__role-authorization-named-action__","namedAction":{"id":"assignStaffRole","path":"/api/staff/:id/role","method":"PUT","reducer":"assign_staff_role","args":[0,"staff"],"params":[{"name":"accountId","in":"path","placeholder":":id","wireType":"u64"},{"name":"role","in":"body"}]},"namedTarget":{"testid":"staff-role-account-staff2","attribute":"data-account-id","valueType":"string"},"settleMs":0}, + {"do":"expectReplayCompleted","actor":"replayAdmin","requireAccepted":true}, + { "do": "reload", "actor": "replayAdmin", "settleMs": 0 }, + { "do": "ensureSignedIn", "actor": "replayAdmin", "name": "admin", "password": "stackbench-admin-2026", "exact": true, "readyTestid": "current-user" }, + {"do":"click","actor":"replayAdmin","testid":"current-user","ifAvailable":true,"unlessVisible":["admin-link","staff-roles-link","staff-role-account-staff2"]}, + {"do":"click","actor":"replayAdmin","testid":"admin-link","ifAvailable":true,"unlessVisible":["staff-roles-link","staff-role-account-staff2"]}, + {"do":"click","actor":"replayAdmin","testid":"staff-roles-link","ifAvailable":true,"unlessVisible":"staff-role-account-staff2"}, + {"do":"expect","actor":"replayAdmin","testid":"staff-role-select","in":{"testid":"staff-role-account-staff2"},"value":"staff","within":10000}, + { "do": "reload", "actor": "staff", "settleMs": 0 }, + { "do": "signIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true }, + {"do":"click","actor":"staff","testid":"current-user","ifAvailable":true,"unlessVisible":["staff-link","staff-area"]}, + {"do":"click","actor":"staff","testid":"staff-link","ifAvailable":true,"unlessVisible":"staff-area"}, + {"do":"replayAs","actor":"staff","from":"replayAdmin","match":"__role-authorization-named-action__","namedAction":{"id":"assignStaffRole","path":"/api/staff/:id/role","method":"PUT","reducer":"assign_staff_role","args":[0,"inventory"],"params":[{"name":"accountId","in":"path","placeholder":":id","wireType":"u64"},{"name":"role","in":"body"}]},"namedTarget":{"testid":"staff-role-account-staff2","attribute":"data-account-id","valueType":"string"},"settleMs":0}, + { "do": "expectReplayRejected", "actor": "staff" }, + { "do": "reload", "actor": "replayAdmin", "settleMs": 0 }, + { "do": "ensureSignedIn", "actor": "replayAdmin", "name": "admin", "password": "stackbench-admin-2026", "exact": true, "readyTestid": "current-user" }, + {"do":"click","actor":"replayAdmin","testid":"current-user","ifAvailable":true,"unlessVisible":["admin-link","staff-roles-link","staff-role-account-staff2"]}, + {"do":"click","actor":"replayAdmin","testid":"admin-link","ifAvailable":true,"unlessVisible":["staff-roles-link","staff-role-account-staff2"]}, + {"do":"click","actor":"replayAdmin","testid":"staff-roles-link","ifAvailable":true,"unlessVisible":"staff-role-account-staff2"}, + {"do":"expect","actor":"replayAdmin","testid":"staff-role-select","in":{"testid":"staff-role-account-staff2"},"value":"staff","within":10000} + ] + }, + { + "id": "621d", + "category": "production", + "desc": "removing administrator access blocks a previously authorized session without changing the target role", + "points": 2, + "statedBy": "Only the admin role grants administrator access.", + "steps": [ + {"do": "reload", "actor": "roleAdmin", "settleMs": 0}, + {"do": "signIn", "actor": "roleAdmin", "name": "admin", "password": "stackbench-admin-2026", "exact": true}, + {"do":"click","actor":"roleAdmin","testid":"current-user","ifAvailable":true,"unlessVisible":["admin-link","staff-roles-link","staff-role-account-staff"]}, + {"do":"click","actor":"roleAdmin","testid":"admin-link","unlessVisible":["staff-roles-link","staff-role-account-staff"]}, + {"do":"click","actor":"roleAdmin","testid":"staff-roles-link","ifAvailable":true,"unlessVisible":"staff-role-account-staff"}, + {"do": "replayAs", "actor": "roleAdmin", "from": "roleAdmin", "match": "__role-revocation-named-action__", "namedAction": {"id": "assignStaffRole", "path": "/api/staff/:id/role", "method": "PUT", "reducer": "assign_staff_role", "args": [0, "admin"], "params": [{"name": "accountId", "in": "path", "placeholder": ":id", "wireType": "u64"}, {"name": "role", "in": "body"}]}, "namedTarget": {"testid": "staff-role-account-staff", "attribute": "data-account-id", "valueType": "string"}, "settleMs": 0}, + {"do": "expectReplayCompleted", "actor": "roleAdmin", "requireAccepted": true}, + {"do": "reload", "actor": "roleAdmin", "settleMs": 0}, + {"do": "ensureSignedIn", "actor": "roleAdmin", "name": "admin", "password": "stackbench-admin-2026", "exact": true, "readyTestid": "current-user"}, + {"do":"click","actor":"roleAdmin","testid":"current-user","ifAvailable":true,"unlessVisible":["admin-link","staff-roles-link","staff-role-account-staff"]}, + {"do":"click","actor":"roleAdmin","testid":"admin-link","unlessVisible":["staff-roles-link","staff-role-account-staff"],"ifAvailable":true}, + {"do":"click","actor":"roleAdmin","testid":"staff-roles-link","ifAvailable":true,"unlessVisible":"staff-role-account-staff"}, + {"do": "expect", "actor": "roleAdmin", "testid": "staff-role-select", "in": {"testid": "staff-role-account-staff"}, "value": "admin", "within": 10000}, + { "do": "reload", "actor": "promotedStaff", "settleMs": 0 }, + {"do": "signIn", "actor": "promotedStaff", "name": "staff", "password": "stackbench-staff-2026", "exact": true}, + {"do":"replayAs","actor":"promotedStaff","from":"roleAdmin","match":"__role-authorization-named-action__","namedAction":{"id":"assignStaffRole","path":"/api/staff/:id/role","method":"PUT","reducer":"assign_staff_role","args":[0,"inventory"],"params":[{"name":"accountId","in":"path","placeholder":":id","wireType":"u64"},{"name":"role","in":"body"}]},"namedTarget":{"testid":"staff-role-account-staff2","attribute":"data-account-id","valueType":"string"},"settleMs":0}, + {"do": "expectReplayCompleted", "actor": "promotedStaff", "requireAccepted": true}, + {"do": "reload", "actor": "roleAdmin", "settleMs": 0}, + {"do": "ensureSignedIn", "actor": "roleAdmin", "name": "admin", "password": "stackbench-admin-2026", "exact": true, "readyTestid": "current-user"}, + {"do":"click","actor":"roleAdmin","testid":"current-user","ifAvailable":true,"unlessVisible":["admin-link","staff-roles-link","staff-role-account-staff2"]}, + {"do":"click","actor":"roleAdmin","testid":"admin-link","ifAvailable":true,"unlessVisible":["staff-roles-link","staff-role-account-staff2"]}, + {"do":"click","actor":"roleAdmin","testid":"staff-roles-link","ifAvailable":true,"unlessVisible":"staff-role-account-staff2"}, + {"do":"expect","actor":"roleAdmin","testid":"staff-role-select","in":{"testid":"staff-role-account-staff2"},"value":"inventory","within":10000}, + {"do":"replayAs","actor":"roleAdmin","from":"roleAdmin","match":"__role-authorization-named-action__","namedAction":{"id":"assignStaffRole","path":"/api/staff/:id/role","method":"PUT","reducer":"assign_staff_role","args":[0,"staff"],"params":[{"name":"accountId","in":"path","placeholder":":id","wireType":"u64"},{"name":"role","in":"body"}]},"namedTarget":{"testid":"staff-role-account-staff2","attribute":"data-account-id","valueType":"string"},"settleMs":0}, + {"do": "expectReplayCompleted", "actor": "roleAdmin", "requireAccepted": true}, + {"do": "reload", "actor": "roleAdmin", "settleMs": 0}, + {"do": "ensureSignedIn", "actor": "roleAdmin", "name": "admin", "password": "stackbench-admin-2026", "exact": true, "readyTestid": "current-user"}, + {"do":"click","actor":"roleAdmin","testid":"current-user","ifAvailable":true,"unlessVisible":["admin-link","staff-roles-link","staff-role-account-staff2"]}, + {"do":"click","actor":"roleAdmin","testid":"admin-link","ifAvailable":true,"unlessVisible":["staff-roles-link","staff-role-account-staff2"]}, + {"do":"click","actor":"roleAdmin","testid":"staff-roles-link","ifAvailable":true,"unlessVisible":"staff-role-account-staff2"}, + {"do":"expect","actor":"roleAdmin","testid":"staff-role-select","in":{"testid":"staff-role-account-staff2"},"value":"staff","within":10000}, + {"do": "replayAs", "actor": "roleAdmin", "from": "roleAdmin", "match": "__role-revocation-named-action__", "namedAction": {"id": "assignStaffRole", "path": "/api/staff/:id/role", "method": "PUT", "reducer": "assign_staff_role", "args": [0, "staff"], "params": [{"name": "accountId", "in": "path", "placeholder": ":id", "wireType": "u64"}, {"name": "role", "in": "body"}]}, "namedTarget": {"testid": "staff-role-account-staff", "attribute": "data-account-id", "valueType": "string"}, "settleMs": 0}, + {"do": "expectReplayCompleted", "actor": "roleAdmin", "requireAccepted": true}, + {"do": "reload", "actor": "roleAdmin", "settleMs": 0}, + {"do": "ensureSignedIn", "actor": "roleAdmin", "name": "admin", "password": "stackbench-admin-2026", "exact": true, "readyTestid": "current-user"}, + {"do":"click","actor":"roleAdmin","testid":"current-user","ifAvailable":true,"unlessVisible":["admin-link","staff-roles-link","staff-role-account-staff"]}, + {"do":"click","actor":"roleAdmin","testid":"admin-link","unlessVisible":["staff-roles-link","staff-role-account-staff"],"ifAvailable":true}, + {"do":"click","actor":"roleAdmin","testid":"staff-roles-link","ifAvailable":true,"unlessVisible":"staff-role-account-staff"}, + {"do": "expect", "actor": "roleAdmin", "testid": "staff-role-select", "in": {"testid": "staff-role-account-staff"}, "value": "staff", "within": 10000}, + {"do":"replayAs","actor":"promotedStaff","from":"roleAdmin","match":"__role-authorization-named-action__","namedAction":{"id":"assignStaffRole","path":"/api/staff/:id/role","method":"PUT","reducer":"assign_staff_role","args":[0,"inventory"],"params":[{"name":"accountId","in":"path","placeholder":":id","wireType":"u64"},{"name":"role","in":"body"}]},"namedTarget":{"testid":"staff-role-account-staff2","attribute":"data-account-id","valueType":"string"},"settleMs":0}, + {"do": "expectReplayRejected", "actor": "promotedStaff"}, + {"do": "reload", "actor": "roleAdmin", "settleMs": 0}, + {"do": "ensureSignedIn", "actor": "roleAdmin", "name": "admin", "password": "stackbench-admin-2026", "exact": true, "readyTestid": "current-user"}, + {"do":"click","actor":"roleAdmin","testid":"current-user","ifAvailable":true,"unlessVisible":["admin-link","staff-roles-link","staff-role-account-staff2"]}, + {"do":"click","actor":"roleAdmin","testid":"admin-link","ifAvailable":true,"unlessVisible":["staff-roles-link","staff-role-account-staff2"]}, + {"do":"click","actor":"roleAdmin","testid":"staff-roles-link","ifAvailable":true,"unlessVisible":"staff-role-account-staff2"}, + {"do":"expect","actor":"roleAdmin","testid":"staff-role-select","in":{"testid":"staff-role-account-staff2"},"value":"staff","within":10000} + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-stock-alert-delivery.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-stock-alert-delivery.json new file mode 100644 index 00000000000..82865e84760 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-stock-alert-delivery.json @@ -0,0 +1,68 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 5, + "name": "Stock alert delivery", + "features": [ + { + "id": 631, + "name": "Stock alert delivery", + "actors": ["subscriber", "admin"], + "setup": [ + {"do":"signUp","actor":"subscriber","name":"stock-subscriber"}, + { "do": "click", "actor": "subscriber", "testid": "overlay-close", "ifAvailable": true, "within": 1000 }, + { "do": "click", "actor": "subscriber", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "subscriber", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["notification-settings", "notification-order"]}, + {"do":"click","actor":"subscriber","testid":"notification-settings","unlessVisible":"notification-order"}, + {"do":"click","actor":"subscriber","testid":"notification-stock"}, + {"do":"expect","actor":"subscriber","testid":"notification-stock","attribute":"data-state","value":"on"}, + {"do":"click","actor":"subscriber","testid":"notification-save"}, + {"do":"expect","actor":"subscriber","testid":"notification-save-state","attribute":"data-submit-state","value":"succeeded","within":10000}, + {"do":"click","actor":"subscriber","testid":"overlay-close","ifAvailable":true}, + {"do":"click","actor":"subscriber","testid":"catalog-link","ifAvailable":true}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"dbSetStock","item":"Air Purifier","warehouse":"East","quantity":0,"settleMs":1000}, + {"do":"dbSetStock","item":"Air Purifier","warehouse":"West","quantity":0,"settleMs":1000}, + {"do":"reload","actor":"subscriber","settleMs":1000}, + {"do":"ensureSignedIn","actor":"subscriber","name":"stock-subscriber","readyTestid":"current-user"}, + { "do": "click", "actor": "subscriber", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do":"click","actor":"subscriber","testid":"stock-alert","in":{"testid":"item-card","contains":"Air Purifier"}}, + {"do":"expect","actor":"subscriber","testid":"item-card","contains":"Air Purifier","attribute":"data-submit-state","value":"succeeded","within":10000} + ], + "criteria": [ + { + "id": "631c", + "category": "feature", + "desc": "restored stock sends the requested alert", + "points": 1, + "statedBy": "Show an alert when stock returns.", + "steps": [ + {"do":"freshClient","actor":"subscriber"}, + {"do":"signIn","actor":"subscriber-fresh","name":"stock-subscriber"}, + {"do":"click","actor":"subscriber-fresh","testid":"catalog-link","ifAvailable":true,"unlessVisible":["notifications-panel","notifications-toggle","notification-settings"]}, + {"do": "click", "actor": "subscriber-fresh", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["notification-settings", "notifications-panel", "notifications-toggle"]}, + {"do":"click","actor":"subscriber-fresh","testid":"notification-settings","ifAvailable":true,"unlessVisible":["notifications-panel","notifications-toggle"]}, + {"do": "click", "actor": "subscriber-fresh", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["notifications-toggle", "notifications-panel"]}, + {"do":"click","actor":"subscriber-fresh","testid":"notifications-toggle","unlessVisible":"notifications-panel"}, + {"do":"expect","actor":"subscriber-fresh","testid":"notifications-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"expectElementCount","actor":"subscriber-fresh","testid":"stock-alert-delivery","contains":"Air Purifier","equals":0}, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link"]}, + {"do":"click","actor":"admin","testid":"admin-link"}, + {"do":"callAction","actor":"admin","action":"restock","namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Air Purifier","attribute":"data-restock-input"},"settleMs":0}, + {"do":"expectActionOutcome","actor":"admin","outcome":"accepted"}, + {"do":"wait","actor":"admin","ms":75000}, + {"do":"freshClient","actor":"subscriber"}, + {"do":"signIn","actor":"subscriber-fresh","name":"stock-subscriber"}, + {"do":"click","actor":"subscriber-fresh","testid":"catalog-link","ifAvailable":true,"unlessVisible":["notifications-panel","notifications-toggle","notification-settings"]}, + {"do": "click", "actor": "subscriber-fresh", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["notification-settings", "notifications-panel", "notifications-toggle"]}, + {"do":"click","actor":"subscriber-fresh","testid":"notification-settings","ifAvailable":true,"unlessVisible":["notifications-panel","notifications-toggle"]}, + {"do": "click", "actor": "subscriber-fresh", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["notifications-toggle", "notifications-panel"]}, + {"do":"click","actor":"subscriber-fresh","testid":"notifications-toggle","unlessVisible":"notifications-panel"}, + {"do":"expect","actor":"subscriber-fresh","testid":"notifications-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"expect","actor":"subscriber-fresh","testid":"stock-alert-delivery","contains":"Air Purifier","within":15000} + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-stock-alert-direct-restore.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-stock-alert-direct-restore.json new file mode 100644 index 00000000000..613ecc38b37 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-stock-alert-direct-restore.json @@ -0,0 +1,65 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 5, + "name": "Stock alert delivery after a direct stock write", + "features": [ + { + "id": 631, + "name": "Stock alert delivery after a direct stock write", + "actors": ["subscriber"], + "setup": [ + {"do":"signUp","actor":"subscriber","name":"stock-subscriber"}, + { "do": "click", "actor": "subscriber", "testid": "overlay-close", "ifAvailable": true, "within": 1000 }, + { "do": "click", "actor": "subscriber", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "subscriber", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["notification-settings", "notification-order"]}, + {"do":"click","actor":"subscriber","testid":"notification-settings","unlessVisible":"notification-order"}, + {"do":"click","actor":"subscriber","testid":"notification-stock"}, + {"do":"expect","actor":"subscriber","testid":"notification-stock","attribute":"data-state","value":"on"}, + {"do":"click","actor":"subscriber","testid":"notification-save"}, + {"do":"expect","actor":"subscriber","testid":"notification-save-state","attribute":"data-submit-state","value":"succeeded","within":10000}, + {"do":"click","actor":"subscriber","testid":"overlay-close","ifAvailable":true}, + {"do":"click","actor":"subscriber","testid":"catalog-link","ifAvailable":true}, + {"do":"dbSetStock","item":"Air Purifier","warehouse":"East","quantity":0,"settleMs":1000}, + {"do":"dbSetStock","item":"Air Purifier","warehouse":"West","quantity":0,"settleMs":1000}, + {"do":"reload","actor":"subscriber","settleMs":1000}, + {"do":"ensureSignedIn","actor":"subscriber","name":"stock-subscriber","readyTestid":"current-user"}, + { "do": "click", "actor": "subscriber", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do":"click","actor":"subscriber","testid":"stock-alert","in":{"testid":"item-card","contains":"Air Purifier"}}, + {"do":"expect","actor":"subscriber","testid":"item-card","contains":"Air Purifier","attribute":"data-submit-state","value":"succeeded","within":10000} + ], + "criteria": [ + { + "id": "631d", + "category": "production", + "desc": "stock restored by a direct database write sends the requested alert", + "points": 1, + "statedBy": "A stock request creates an alert when stock returns, including stock written outside the application.", + "note": "No delivery latency is stated, so a sweep that runs once a minute must pass: the alert has 75 seconds after the write plus 15 seconds on a fresh sign-in.", + "steps": [ + {"do":"freshClient","actor":"subscriber"}, + {"do":"signIn","actor":"subscriber-fresh","name":"stock-subscriber"}, + {"do":"click","actor":"subscriber-fresh","testid":"catalog-link","ifAvailable":true,"unlessVisible":["notifications-panel","notifications-toggle","notification-settings"]}, + {"do": "click", "actor": "subscriber-fresh", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["notification-settings", "notifications-panel", "notifications-toggle"]}, + {"do":"click","actor":"subscriber-fresh","testid":"notification-settings","ifAvailable":true,"unlessVisible":["notifications-panel","notifications-toggle"]}, + {"do": "click", "actor": "subscriber-fresh", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["notifications-toggle", "notifications-panel"]}, + {"do":"click","actor":"subscriber-fresh","testid":"notifications-toggle","unlessVisible":"notifications-panel"}, + {"do":"expect","actor":"subscriber-fresh","testid":"notifications-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"expectElementCount","actor":"subscriber-fresh","testid":"stock-alert-delivery","contains":"Air Purifier","equals":0}, + {"do":"dbSetStock","item":"Air Purifier","warehouse":"East","quantity":1,"settleMs":0}, + {"do":"wait","actor":"subscriber","ms":75000}, + {"do":"freshClient","actor":"subscriber"}, + {"do":"signIn","actor":"subscriber-fresh","name":"stock-subscriber"}, + {"do":"click","actor":"subscriber-fresh","testid":"catalog-link","ifAvailable":true,"unlessVisible":["notifications-panel","notifications-toggle","notification-settings"]}, + {"do": "click", "actor": "subscriber-fresh", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["notification-settings", "notifications-panel", "notifications-toggle"]}, + {"do":"click","actor":"subscriber-fresh","testid":"notification-settings","ifAvailable":true,"unlessVisible":["notifications-panel","notifications-toggle"]}, + {"do": "click", "actor": "subscriber-fresh", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["notifications-toggle", "notifications-panel"]}, + {"do":"click","actor":"subscriber-fresh","testid":"notifications-toggle","unlessVisible":"notifications-panel"}, + {"do":"expect","actor":"subscriber-fresh","testid":"notifications-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"expect","actor":"subscriber-fresh","testid":"stock-alert-delivery","contains":"Air Purifier","within":15000} + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-stock-alerts.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-stock-alerts.json new file mode 100644 index 00000000000..d9a92e0463a --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-stock-alerts.json @@ -0,0 +1,114 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 5, + "name": "Stock alerts", + "features": [ + { + "id": 631, + "name": "Private one-time stock alerts", + "actors": ["subscriber", "other", "admin"], + "setup": [ + {"do":"signUp","actor":"subscriber","name":"stock-subscriber"}, + {"do":"signUp","actor":"other","name":"stock-other"}, + { "do": "click", "actor": "subscriber", "testid": "overlay-close", "ifAvailable": true, "within": 1000 }, + { "do": "click", "actor": "subscriber", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "subscriber", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["notification-settings", "notification-order"]}, + {"do":"click","actor":"subscriber","testid":"notification-settings","unlessVisible":"notification-order"}, + {"do":"click","actor":"subscriber","testid":"notification-stock"}, + {"do":"expect","actor":"subscriber","testid":"notification-stock","attribute":"data-state","value":"on"}, + {"do":"click","actor":"subscriber","testid":"notification-save"}, + {"do":"expect","actor":"subscriber","testid":"notification-save-state","attribute":"data-submit-state","value":"succeeded","within":10000}, + {"do":"click","actor":"subscriber","testid":"overlay-close","ifAvailable":true}, + {"do":"click","actor":"subscriber","testid":"catalog-link","ifAvailable":true}, + { "do": "click", "actor": "other", "testid": "overlay-close", "ifAvailable": true, "within": 1000 }, + { "do": "click", "actor": "other", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do": "click", "actor": "other", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["notification-settings", "notification-order"]}, + {"do":"click","actor":"other","testid":"notification-settings","unlessVisible":"notification-order"}, + {"do":"click","actor":"other","testid":"notification-stock"}, + {"do":"expect","actor":"other","testid":"notification-stock","attribute":"data-state","value":"on"}, + {"do":"click","actor":"other","testid":"notification-save"}, + {"do":"expect","actor":"other","testid":"notification-save-state","attribute":"data-submit-state","value":"succeeded","within":10000}, + {"do":"click","actor":"other","testid":"overlay-close","ifAvailable":true}, + {"do":"click","actor":"other","testid":"catalog-link","ifAvailable":true}, + {"do":"signIn","actor":"admin","name":"admin","password":"stackbench-admin-2026","exact":true}, + {"do":"dbSetStock","item":"Air Purifier","warehouse":"East","quantity":0,"settleMs":1000}, + {"do":"dbSetStock","item":"Air Purifier","warehouse":"West","quantity":0,"settleMs":1000}, + {"do":"reload","actor":"subscriber","settleMs":1000}, + {"do":"ensureSignedIn","actor":"subscriber","name":"stock-subscriber","readyTestid":"current-user"}, + { "do": "click", "actor": "subscriber", "testid": "catalog-link", "ifAvailable": true, "unlessVisible": "item-list" }, + {"do":"markDelivery","actor":"subscriber","as":"stock-alert-request","phase":"start"}, + {"do":"click","actor":"subscriber","testid":"stock-alert","in":{"testid":"item-card","contains":"Air Purifier"}}, + {"do":"expect","actor":"subscriber","testid":"item-card","contains":"Air Purifier","attribute":"data-submit-state","value":"succeeded","within":10000}, + {"do":"markDelivery","actor":"subscriber","as":"stock-alert-request","phase":"end"}, + {"do": "click", "actor": "admin", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["admin-link", "admin-panel"]}, + {"do":"click","actor":"admin","testid":"admin-link", "unlessVisible": "admin-panel"}, + {"do":"callAction","actor":"admin","action":"restock","namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Air Purifier","attribute":"data-restock-input"},"settleMs":0}, + {"do":"expectActionOutcome","actor":"admin","outcome":"accepted"}, + {"do":"wait","actor":"admin","ms":75000} + ], + "criteria": [ + { + "id": "631a", + "category": "production", + "desc": "restored stock sends one alert and later restocks do not duplicate it", + "points": 2, + "statedBy": "A stock request creates one alert when stock returns.", + "note": "Sample a fresh account view 75 seconds after the second accepted restock. This catches persistent duplicates delivered by that sample, not all later or transient duplicates.", + "steps": [ + {"do":"freshClient","actor":"subscriber"}, + {"do":"signIn","actor":"subscriber-fresh","name":"stock-subscriber"}, + {"do":"click","actor":"subscriber-fresh","testid":"catalog-link","ifAvailable":true,"unlessVisible":["notifications-panel","notifications-toggle","notification-settings"]}, + {"do": "click", "actor": "subscriber-fresh", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["notification-settings", "notifications-panel", "notifications-toggle"]}, + {"do":"click","actor":"subscriber-fresh","testid":"notification-settings","ifAvailable":true,"unlessVisible":["notifications-panel","notifications-toggle"]}, + {"do": "click", "actor": "subscriber-fresh", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["notifications-toggle", "notifications-panel"]}, + {"do":"click","actor":"subscriber-fresh","testid":"notifications-toggle","unlessVisible":"notifications-panel"}, + {"do":"expect","actor":"subscriber-fresh","testid":"notifications-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"expectElementCount","actor":"subscriber-fresh","testid":"stock-alert-delivery","contains":"Air Purifier","equals":1,"within":15000}, + {"do":"callAction","actor":"admin","action":"restock","namedAction":{"id":"restock","path":"/api/admin/restock","reducer":"admin_restock","args":[0,0,1],"params":[{"name":"itemId","in":"body","wireType":"u64"},{"name":"warehouseId","in":"body","wireType":"u64"},{"name":"quantity","in":"body"}]},"input":{"testid":"admin-location-row","contains":"Air Purifier","attribute":"data-restock-input"},"settleMs":0}, + {"do":"expectActionOutcome","actor":"admin","outcome":"accepted"}, + {"do":"wait","actor":"admin","ms":75000}, + {"do":"freshClient","actor":"subscriber"}, + {"do":"signIn","actor":"subscriber-fresh","name":"stock-subscriber"}, + {"do":"click","actor":"subscriber-fresh","testid":"catalog-link","ifAvailable":true,"unlessVisible":["notifications-panel","notifications-toggle","notification-settings"]}, + {"do": "click", "actor": "subscriber-fresh", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["notification-settings", "notifications-panel", "notifications-toggle"]}, + {"do":"click","actor":"subscriber-fresh","testid":"notification-settings","ifAvailable":true,"unlessVisible":["notifications-panel","notifications-toggle"]}, + {"do": "click", "actor": "subscriber-fresh", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["notifications-toggle", "notifications-panel"]}, + {"do":"click","actor":"subscriber-fresh","testid":"notifications-toggle","unlessVisible":"notifications-panel"}, + {"do":"expect","actor":"subscriber-fresh","testid":"notifications-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"expectElementCount","actor":"subscriber-fresh","testid":"stock-alert-delivery","contains":"Air Purifier","equals":1,"within":15000} + ] + }, + { + "id": "631b", + "category": "production", + "desc": "a customer who did not request the alert cannot see it", + "points": 1, + "statedBy": "Stock alerts are private to the requesting customer.", + "steps": [ + {"do":"freshClient","actor":"subscriber"}, + {"do":"signIn","actor":"subscriber-fresh","name":"stock-subscriber"}, + {"do":"click","actor":"subscriber-fresh","testid":"catalog-link","ifAvailable":true,"unlessVisible":["notifications-panel","notifications-toggle","notification-settings"]}, + {"do": "click", "actor": "subscriber-fresh", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["notification-settings", "notifications-panel", "notifications-toggle"]}, + {"do":"click","actor":"subscriber-fresh","testid":"notification-settings","ifAvailable":true,"unlessVisible":["notifications-panel","notifications-toggle"]}, + {"do": "click", "actor": "subscriber-fresh", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["notifications-toggle", "notifications-panel"]}, + {"do":"click","actor":"subscriber-fresh","testid":"notifications-toggle","unlessVisible":"notifications-panel"}, + {"do":"expect","actor":"subscriber-fresh","testid":"notifications-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"expect","actor":"subscriber-fresh","testid":"stock-alert-delivery","contains":"Air Purifier","within":15000}, + {"do":"expect","actor":"subscriber-fresh","testid":"notification-item","contains":"Air Purifier","within":15000}, + {"do":"freshClient","actor":"other"}, + {"do":"signIn","actor":"other-fresh","name":"stock-other"}, + {"do":"click","actor":"other-fresh","testid":"catalog-link","ifAvailable":true,"unlessVisible":["notifications-panel","notifications-toggle","notification-settings"]}, + {"do": "click", "actor": "other-fresh", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["notification-settings", "notifications-panel", "notifications-toggle"]}, + {"do":"click","actor":"other-fresh","testid":"notification-settings","ifAvailable":true,"unlessVisible":["notifications-panel","notifications-toggle"]}, + {"do": "click", "actor": "other-fresh", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["notifications-toggle", "notifications-panel"]}, + {"do":"click","actor":"other-fresh","testid":"notifications-toggle","unlessVisible":"notifications-panel"}, + {"do":"expect","actor":"other-fresh","testid":"notifications-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"expect","actor":"other-fresh","testid":"notification-item","contains":"Air Purifier","absent":true}, + {"do":"expectNotDelivered","actor":"other-fresh","from":"subscriber","as":"stock-alert-request"} + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-stock-limit.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-stock-limit.json new file mode 100644 index 00000000000..bcb760c99e5 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-stock-limit.json @@ -0,0 +1,42 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Progression stock limit", + "features": [ + { + "id": 3, + "name": "Buying", + "actors": ["buyer", "watcher", "visitor"], + "setup": [ + { "do": "dbSetStock", "item": "Desk Lamp", "warehouse": "East", "quantity": 2, "settleMs": 2000 }, + { "do": "dbSetStock", "item": "Desk Lamp", "warehouse": "West", "quantity": 1, "settleMs": 2000 }, + { "do": "signUp", "actor": "buyer", "name": "eli" }, + { "do": "signUp", "actor": "watcher", "name": "fay" } + ], + "criteria": [ + { + "id": "3d", + "category": "production", + "desc": "an item sells out visibly, and a further purchase is refused without changing stock", + "points": 2, + "statedBy": "an item with zero stock cannot be bought", + "steps": [ + { "do": "click", "actor": "buyer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Desk Lamp" } }, + { "do": "wait", "actor": "buyer", "ms": 800 }, + { "do": "click", "actor": "buyer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Desk Lamp" } }, + { "do": "wait", "actor": "buyer", "ms": 800 }, + { "do": "click", "actor": "buyer", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Desk Lamp" } }, + { "do": "expectNumber", "actor": "buyer", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Desk Lamp" }, "equals": 0 }, + { "do": "expect", "actor": "buyer", "testid": "out-of-stock", "in": { "testid": "item-card", "contains": "Desk Lamp" } }, + { "do": "callAction", "actor": "buyer", "action": "buy", "namedAction": { "id": "buy", "path": "/api/items/:id/buy", "reducer": "buy_now", "args": [0], "params": [{ "name": "itemId", "in": "path", "placeholder": ":id", "wireType": "u64" }] }, "input": { "testid": "item-card", "contains": "Desk Lamp", "attribute": "data-buy-input" }, "settleMs": 2000 }, + { "do": "expectActionOutcome", "actor": "buyer", "outcome": "validation-refused" }, + { "do": "reload", "actor": "buyer", "settleMs": 3000 }, + { "do": "ensureSignedIn", "actor": "buyer", "name": "eli", "readyTestid": "current-user" }, + { "do": "expectNumber", "actor": "buyer", "testid": "item-stock", "in": { "testid": "item-card", "contains": "Desk Lamp" }, "equals": 0 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-store-credit.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-store-credit.json new file mode 100644 index 00000000000..4bbbb35c68d --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-store-credit.json @@ -0,0 +1,244 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 5, + "name": "Store credit", + "note": "Draft: revised observations require matching reference and defect-control qualification.", + "features": [ + { + "id": 750, + "name": "Credit checkout records both payment portions", + "actors": [ + "owner", + "staff", + "tab" + ], + "setup": [ + {"do":"signUp","actor":"owner","name":"credit-owner-750"}, + {"do":"click","actor":"owner","testid":"credit-link","unlessVisible":"credit-panel"}, + {"do":"expect","actor":"owner","testid":"credit-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"signIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true}, + {"do":"click","actor":"staff","testid":"credit-link","unlessVisible":"credit-panel"}, + {"do":"expect","actor":"staff","testid":"credit-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"fill","actor":"staff","testid":"credit-amount-input","text":"10.00"}, + {"do":"fill","actor":"staff","testid":"credit-reference-input","text":"credit-750"}, + {"do":"callAction","actor":"staff","from":"staff","action":"grantCredit","namedAction":{"id":"grantCredit","path":"/api/staff/credit","method":"POST","reducer":"grant_credit","args":[0,1000,""],"params":[{"name":"accountId","in":"body","wireType":"u64"},{"name":"amountMinor","in":"body"},{"name":"reference","in":"body"}]},"input":{"testid":"credit-grant","attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","testid":"credit-panel","attribute":"data-account-id"}}},"settleMs":0}, + {"do":"expectActionOutcome","actor":"staff","outcome":"accepted"}, + {"do":"reload","actor":"owner","settleMs":1000}, + {"do":"ensureSignedIn","actor":"owner","name":"credit-owner-750","readyTestid":"current-user"}, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"owner","testid":"credit-link","unlessVisible":"credit-balance"}, + {"do":"expectNumber","actor":"owner","testid":"credit-balance","equals":10,"within":10000}, + {"do":"click","actor":"owner","testid":"catalog-link"}, + {"do":"click","actor":"owner","testid":"add-to-cart","in":{"testid":"item-card","contains":"Desk Lamp"}}, + {"do":"click","actor":"owner","testid":"cart-toggle"}, + {"do":"expectNumber","actor":"owner","testid":"cart-total","equals":42} + ], + "criteria": [ + { + "id": "750a", + "category": "feature", + "desc": "Credit checkout records both payment portions", + "points": 2, + "steps": [ + {"do":"click","actor":"owner","testid":"credit-checkout"}, + {"do":"freshClient","actor":"owner"}, + {"do":"signIn","actor":"owner-fresh","name":"credit-owner-750"}, + {"do":"click","actor":"owner-fresh","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"owner-fresh","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expectElementCount","actor":"owner-fresh","testid":"order-item","contains":"Desk Lamp","equals":1,"within":10000}, + {"do":"expectNumber","actor":"owner-fresh","testid":"payment-credit-amount","equals":10,"in":{"testid":"order-item","contains":"Desk Lamp"}}, + {"do":"expectNumber","actor":"owner-fresh","testid":"payment-external-amount","equals":32,"in":{"testid":"order-item","contains":"Desk Lamp"}}, + {"do":"expectNumber","actor":"owner-fresh","testid":"payment-amount","equals":42,"in":{"testid":"order-item","contains":"Desk Lamp"}}, + {"do":"click","actor":"owner-fresh","testid":"credit-link","unlessVisible":"credit-panel"}, + {"do":"expect","actor":"owner-fresh","testid":"credit-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"expectNumber","actor":"owner-fresh","testid":"credit-balance","equals":0} + ] + } + ] + }, + { + "id": 752, + "name": "Repeating a grant reference does not increase the balance twice", + "actors": [ + "owner", + "staff", + "tab" + ], + "setup": [ + {"do":"signUp","actor":"owner","name":"credit-owner-752"}, + {"do":"click","actor":"owner","testid":"credit-link","unlessVisible":"credit-panel"}, + {"do":"expect","actor":"owner","testid":"credit-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"signIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true}, + {"do":"click","actor":"staff","testid":"credit-link","unlessVisible":"credit-panel"}, + {"do":"expect","actor":"staff","testid":"credit-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"fill","actor":"staff","testid":"credit-amount-input","text":"10.00"}, + {"do":"fill","actor":"staff","testid":"credit-reference-input","text":"credit-752"}, + {"do":"callAction","actor":"staff","from":"staff","action":"grantCredit","namedAction":{"id":"grantCredit","path":"/api/staff/credit","method":"POST","reducer":"grant_credit","args":[0,1000,""],"params":[{"name":"accountId","in":"body","wireType":"u64"},{"name":"amountMinor","in":"body"},{"name":"reference","in":"body"}]},"input":{"testid":"credit-grant","attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","testid":"credit-panel","attribute":"data-account-id"}}},"settleMs":0}, + {"do":"expectActionOutcome","actor":"staff","outcome":"accepted"}, + {"do":"reload","actor":"owner","settleMs":1000}, + {"do":"ensureSignedIn","actor":"owner","name":"credit-owner-752","readyTestid":"current-user"}, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"owner","testid":"credit-link","unlessVisible":"credit-balance"}, + {"do":"expectNumber","actor":"owner","testid":"credit-balance","equals":10,"within":10000} + ], + "criteria": [ + { + "id": "752a", + "category": "production", + "desc": "Repeating a grant reference does not increase the balance twice", + "points": 2, + "steps": [ + {"do":"callAction","actor":"staff","from":"staff","action":"grantCredit","namedAction":{"id":"grantCredit","path":"/api/staff/credit","method":"POST","reducer":"grant_credit","args":[0,1000,""],"params":[{"name":"accountId","in":"body","wireType":"u64"},{"name":"amountMinor","in":"body"},{"name":"reference","in":"body"}]},"input":{"testid":"credit-grant","attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","testid":"credit-panel","attribute":"data-account-id"}}},"settleMs":0}, + {"do":"expectActionOutcome","actor":"staff","outcome":"completed"}, + {"do":"freshClient","actor":"owner"}, + {"do":"signIn","actor":"owner-fresh","name":"credit-owner-752"}, + {"do":"click","actor":"owner-fresh","testid":"credit-link","unlessVisible":"credit-panel"}, + {"do":"expect","actor":"owner-fresh","testid":"credit-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"expectNumber","actor":"owner-fresh","testid":"credit-balance","equals":10} + ] + } + ] + }, + { + "id": 753, + "name": "A customer cannot grant credit", + "actors": [ + "owner", + "staff", + "tab" + ], + "setup": [ + {"do":"signUp","actor":"owner","name":"credit-owner-753"}, + {"do":"click","actor":"owner","testid":"credit-link","unlessVisible":"credit-panel"}, + {"do":"expect","actor":"owner","testid":"credit-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"signIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true}, + {"do":"click","actor":"staff","testid":"credit-link","unlessVisible":"credit-panel"}, + {"do":"expect","actor":"staff","testid":"credit-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"fill","actor":"staff","testid":"credit-amount-input","text":"10.00"}, + {"do":"fill","actor":"staff","testid":"credit-reference-input","text":"credit-753"}, + {"do":"callAction","actor":"staff","from":"staff","action":"grantCredit","namedAction":{"id":"grantCredit","path":"/api/staff/credit","method":"POST","reducer":"grant_credit","args":[0,1000,""],"params":[{"name":"accountId","in":"body","wireType":"u64"},{"name":"amountMinor","in":"body"},{"name":"reference","in":"body"}]},"input":{"testid":"credit-grant","attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","testid":"credit-panel","attribute":"data-account-id"}}},"settleMs":0}, + {"do":"expectActionOutcome","actor":"staff","outcome":"accepted"}, + {"do":"reload","actor":"owner","settleMs":1000}, + {"do":"ensureSignedIn","actor":"owner","name":"credit-owner-753","readyTestid":"current-user"}, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"owner","testid":"credit-link","unlessVisible":"credit-balance"}, + {"do":"expectNumber","actor":"owner","testid":"credit-balance","equals":10,"within":10000} + ], + "criteria": [ + { + "id": "753a", + "category": "production", + "desc": "A customer cannot grant credit", + "points": 2, + "steps": [ + {"do":"fill","actor":"staff","testid":"credit-reference-input","text":"unauthorized-credit-753"}, + {"do":"callAction","actor":"owner","from":"staff","action":"grantCredit","namedAction":{"id":"grantCredit","path":"/api/staff/credit","method":"POST","reducer":"grant_credit","args":[0,1000,""],"params":[{"name":"accountId","in":"body","wireType":"u64"},{"name":"amountMinor","in":"body"},{"name":"reference","in":"body"}]},"input":{"testid":"credit-grant","attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","testid":"credit-panel","attribute":"data-account-id"}}},"settleMs":0}, + {"do":"expectActionOutcome","actor":"owner","outcome":"refused"}, + {"do":"freshClient","actor":"owner"}, + {"do":"signIn","actor":"owner-fresh","name":"credit-owner-753"}, + {"do":"click","actor":"owner-fresh","testid":"credit-link","unlessVisible":"credit-panel"}, + {"do":"expect","actor":"owner-fresh","testid":"credit-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"expectNumber","actor":"owner-fresh","testid":"credit-balance","equals":10} + ] + } + ] + }, + { + "id": 754, + "name": "Concurrent checkout consumes one cart and one credit allocation", + "actors": [ + "owner", + "staff", + "tab" + ], + "setup": [ + {"do":"signUp","actor":"owner","name":"credit-owner-754"}, + {"do":"click","actor":"owner","testid":"credit-link","unlessVisible":"credit-panel"}, + {"do":"expect","actor":"owner","testid":"credit-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"signIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true}, + {"do":"click","actor":"staff","testid":"credit-link","unlessVisible":"credit-panel"}, + {"do":"expect","actor":"staff","testid":"credit-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"fill","actor":"staff","testid":"credit-amount-input","text":"10.00"}, + {"do":"fill","actor":"staff","testid":"credit-reference-input","text":"credit-754"}, + {"do":"callAction","actor":"staff","from":"staff","action":"grantCredit","namedAction":{"id":"grantCredit","path":"/api/staff/credit","method":"POST","reducer":"grant_credit","args":[0,1000,""],"params":[{"name":"accountId","in":"body","wireType":"u64"},{"name":"amountMinor","in":"body"},{"name":"reference","in":"body"}]},"input":{"testid":"credit-grant","attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","testid":"credit-panel","attribute":"data-account-id"}}},"settleMs":0}, + {"do":"expectActionOutcome","actor":"staff","outcome":"accepted"}, + {"do":"reload","actor":"owner","settleMs":1000}, + {"do":"ensureSignedIn","actor":"owner","name":"credit-owner-754","readyTestid":"current-user"}, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"owner","testid":"credit-link","unlessVisible":"credit-balance"}, + {"do":"expectNumber","actor":"owner","testid":"credit-balance","equals":10,"within":10000}, + {"do":"click","actor":"owner","testid":"catalog-link"}, + {"do":"click","actor":"owner","testid":"add-to-cart","in":{"testid":"item-card","contains":"Desk Lamp"}}, + {"do":"click","actor":"owner","testid":"cart-toggle"}, + {"do":"expectNumber","actor":"owner","testid":"cart-total","equals":42}, + {"do":"signIn","actor":"tab","name":"credit-owner-754"} + ], + "criteria": [ + { + "id": "754a", + "category": "production", + "desc": "Concurrent checkout consumes one cart and one credit allocation", + "points": 2, + "steps": [ + {"do":"callConcurrently","actors":["owner","tab"],"action":"checkoutCredit","namedAction":{"id":"checkoutCredit","path":"/api/checkout/credit","method":"POST","reducer":"checkout_credit","args":[]},"settleMs":0}, + {"do":"expectCallOutcomes"}, + {"do":"freshClient","actor":"owner"}, + {"do":"signIn","actor":"owner-fresh","name":"credit-owner-754"}, + {"do":"click","actor":"owner-fresh","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"owner-fresh","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expectElementCount","actor":"owner-fresh","testid":"order-item","contains":"Desk Lamp","equals":1,"within":10000}, + {"do":"expectNumber","actor":"owner-fresh","testid":"payment-credit-amount","equals":10,"in":{"testid":"order-item","contains":"Desk Lamp"}}, + {"do":"expectNumber","actor":"owner-fresh","testid":"payment-external-amount","equals":32,"in":{"testid":"order-item","contains":"Desk Lamp"}}, + {"do":"expectNumber","actor":"owner-fresh","testid":"payment-amount","equals":42,"in":{"testid":"order-item","contains":"Desk Lamp"}}, + {"do":"click","actor":"owner-fresh","testid":"credit-link","unlessVisible":"credit-panel"}, + {"do":"expect","actor":"owner-fresh","testid":"credit-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"expectNumber","actor":"owner-fresh","testid":"credit-balance","equals":0} + ] + } + ] + }, + { + "id": 755, + "name": "Issued credit survives a backend restart", + "actors": [ + "owner", + "staff", + "tab" + ], + "setup": [ + {"do":"signUp","actor":"owner","name":"credit-owner-755"}, + {"do":"click","actor":"owner","testid":"credit-link","unlessVisible":"credit-panel"}, + {"do":"expect","actor":"owner","testid":"credit-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"signIn","actor":"staff","name":"staff","password":"stackbench-staff-2026","exact":true}, + {"do":"click","actor":"staff","testid":"credit-link","unlessVisible":"credit-panel"}, + {"do":"expect","actor":"staff","testid":"credit-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"fill","actor":"staff","testid":"credit-amount-input","text":"10.00"}, + {"do":"fill","actor":"staff","testid":"credit-reference-input","text":"credit-755"}, + {"do":"callAction","actor":"staff","from":"staff","action":"grantCredit","namedAction":{"id":"grantCredit","path":"/api/staff/credit","method":"POST","reducer":"grant_credit","args":[0,1000,""],"params":[{"name":"accountId","in":"body","wireType":"u64"},{"name":"amountMinor","in":"body"},{"name":"reference","in":"body"}]},"input":{"testid":"credit-grant","attribute":"data-action-input","overrides":{"accountId":{"actor":"owner","testid":"credit-panel","attribute":"data-account-id"}}},"settleMs":0}, + {"do":"expectActionOutcome","actor":"staff","outcome":"accepted"}, + {"do":"reload","actor":"owner","settleMs":1000}, + {"do":"ensureSignedIn","actor":"owner","name":"credit-owner-755","readyTestid":"current-user"}, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"owner","testid":"credit-link","unlessVisible":"credit-balance"}, + {"do":"expectNumber","actor":"owner","testid":"credit-balance","equals":10,"within":10000} + ], + "criteria": [ + { + "id": "755a", + "category": "production", + "desc": "Issued credit survives a backend restart", + "points": 2, + "steps": [ + {"do":"restartBackend","settleMs":1000}, + {"do":"freshClient","actor":"owner"}, + {"do":"signIn","actor":"owner-fresh","name":"credit-owner-755"}, + {"do":"click","actor":"owner-fresh","testid":"credit-link","unlessVisible":"credit-panel"}, + {"do":"expect","actor":"owner-fresh","testid":"credit-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"expectNumber","actor":"owner-fresh","testid":"credit-balance","equals":10} + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-subscriptions.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-subscriptions.json new file mode 100644 index 00000000000..18c47027e5b --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-subscriptions.json @@ -0,0 +1,215 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 5, + "name": "Scheduled purchases", + "features": [ + { + "id": 760, + "name": "A subscription creates exactly its requested deliveries and payments", + "actors": [ + "owner", + "other" + ], + "setup": [ + {"do":"signUp","actor":"owner","name":"subscription-760"}, + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"East","quantity":10,"settleMs":0}, + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"click","actor":"owner","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"}, + {"do":"expect","actor":"owner","testid":"subscriptions-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"fill","actor":"owner","testid":"subscription-item-input","text":"Desk Lamp"}, + {"do":"fill","actor":"owner","testid":"subscription-quantity-input","text":"1"}, + {"do":"fill","actor":"owner","testid":"subscription-interval-input","text":"30"}, + {"do":"fill","actor":"owner","testid":"subscription-deliveries-input","text":"2"}, + {"do":"click","actor":"owner","testid":"subscription-create"}, + {"do":"expect","actor":"owner","testid":"subscription-status","in":{"testid":"subscription-row","contains":"Desk Lamp"},"value":"active","within":10000} + ], + "criteria": [ + { + "id": "760a", + "category": "feature", + "desc": "A subscription creates exactly its requested deliveries and payments", + "points": 2, + "steps": [ + {"do":"freshClient","actor":"owner"}, + {"do":"signIn","actor":"owner-fresh","name":"subscription-760"}, + {"do":"click","actor":"owner-fresh","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"}, + {"do":"expect","actor":"owner-fresh","testid":"subscriptions-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"expect","actor":"owner-fresh","testid":"subscription-status","in":{"testid":"subscription-row","contains":"Desk Lamp"},"value":"complete","within":90000}, + {"do":"expectElementCount","actor":"owner-fresh","testid":"subscription-delivery","in":{"testid":"subscription-row","contains":"Desk Lamp"},"equals":2}, + {"do":"expectSequence","actor":"owner-fresh","testid":"subscription-delivery-status","in":{"testid":"subscription-row","contains":"Desk Lamp"},"equals":["paid","paid"]}, + {"do":"expectNumber","actor":"owner-fresh","testid":"subscription-total","in":{"testid":"subscription-row","contains":"Desk Lamp"},"equals":84}, + {"do":"dbExpectStock","item":"Desk Lamp","equals":8}, + {"do":"click","actor":"owner-fresh","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"owner-fresh","testid":"orders-toggle"}, + {"do":"expectElementCount","actor":"owner-fresh","testid":"order-item","contains":"Desk Lamp","equals":2,"within":10000}, + {"do":"expectElementCount","actor":"owner-fresh","testid":"payment-record","equals":2} + ] + } + ] + }, + { + "id": 761, + "name": "A pending subscription continues after backend restart without duplicate deliveries", + "actors": [ + "owner", + "other" + ], + "setup": [ + {"do":"signUp","actor":"owner","name":"subscription-761"}, + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"East","quantity":10,"settleMs":0}, + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"click","actor":"owner","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"}, + {"do":"expect","actor":"owner","testid":"subscriptions-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"fill","actor":"owner","testid":"subscription-item-input","text":"Desk Lamp"}, + {"do":"fill","actor":"owner","testid":"subscription-quantity-input","text":"1"}, + {"do":"fill","actor":"owner","testid":"subscription-interval-input","text":"30"}, + {"do":"fill","actor":"owner","testid":"subscription-deliveries-input","text":"2"}, + {"do":"click","actor":"owner","testid":"subscription-create"}, + {"do":"expect","actor":"owner","testid":"subscription-status","in":{"testid":"subscription-row","contains":"Desk Lamp"},"value":"active","within":10000} + ], + "criteria": [ + { + "id": "761a", + "category": "production", + "desc": "A pending subscription continues after backend restart without duplicate deliveries", + "points": 2, + "steps": [ + {"do":"restartBackend","settleMs":1000}, + {"do":"freshClient","actor":"owner"}, + {"do":"signIn","actor":"owner-fresh","name":"subscription-761"}, + {"do":"click","actor":"owner-fresh","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"}, + {"do":"expect","actor":"owner-fresh","testid":"subscriptions-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"expect","actor":"owner-fresh","testid":"subscription-status","in":{"testid":"subscription-row","contains":"Desk Lamp"},"value":"complete","within":90000}, + {"do":"expectElementCount","actor":"owner-fresh","testid":"subscription-delivery","in":{"testid":"subscription-row","contains":"Desk Lamp"},"equals":2}, + {"do":"expectSequence","actor":"owner-fresh","testid":"subscription-delivery-status","in":{"testid":"subscription-row","contains":"Desk Lamp"},"equals":["paid","paid"]}, + {"do":"expectNumber","actor":"owner-fresh","testid":"subscription-total","in":{"testid":"subscription-row","contains":"Desk Lamp"},"equals":84}, + {"do":"dbExpectStock","item":"Desk Lamp","equals":8}, + {"do":"click","actor":"owner-fresh","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"owner-fresh","testid":"orders-toggle"}, + {"do":"expectElementCount","actor":"owner-fresh","testid":"order-item","contains":"Desk Lamp","equals":2,"within":10000}, + {"do":"expectElementCount","actor":"owner-fresh","testid":"payment-record","equals":2}, + {"do":"restartBackend","settleMs":1000}, + {"do":"freshClient","actor":"owner"}, + {"do":"signIn","actor":"owner-fresh","name":"subscription-761"}, + {"do":"click","actor":"owner-fresh","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"}, + {"do":"expect","actor":"owner-fresh","testid":"subscriptions-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"expect","actor":"owner-fresh","testid":"subscription-status","in":{"testid":"subscription-row","contains":"Desk Lamp"},"value":"complete","within":90000}, + {"do":"expectElementCount","actor":"owner-fresh","testid":"subscription-delivery","in":{"testid":"subscription-row","contains":"Desk Lamp"},"equals":2}, + {"do":"expectSequence","actor":"owner-fresh","testid":"subscription-delivery-status","in":{"testid":"subscription-row","contains":"Desk Lamp"},"equals":["paid","paid"]}, + {"do":"expectNumber","actor":"owner-fresh","testid":"subscription-total","in":{"testid":"subscription-row","contains":"Desk Lamp"},"equals":84}, + {"do":"dbExpectStock","item":"Desk Lamp","equals":8}, + {"do":"click","actor":"owner-fresh","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"owner-fresh","testid":"orders-toggle"}, + {"do":"expectElementCount","actor":"owner-fresh","testid":"order-item","contains":"Desk Lamp","equals":2,"within":10000}, + {"do":"expectElementCount","actor":"owner-fresh","testid":"payment-record","equals":2} + ] + } + ] + }, + { + "id": 762, + "name": "Another customer cannot cancel an active subscription", + "actors": [ + "owner", + "other" + ], + "setup": [ + {"do":"signUp","actor":"other","name":"subscription-other-762"}, + {"do":"signUp","actor":"owner","name":"subscription-762"}, + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"East","quantity":10,"settleMs":0}, + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"click","actor":"owner","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"}, + {"do":"expect","actor":"owner","testid":"subscriptions-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"fill","actor":"owner","testid":"subscription-item-input","text":"Desk Lamp"}, + {"do":"fill","actor":"owner","testid":"subscription-quantity-input","text":"1"}, + {"do":"fill","actor":"owner","testid":"subscription-interval-input","text":"30"}, + {"do":"fill","actor":"owner","testid":"subscription-deliveries-input","text":"2"}, + {"do":"click","actor":"owner","testid":"subscription-create"}, + {"do":"expect","actor":"owner","testid":"subscription-status","in":{"testid":"subscription-row","contains":"Desk Lamp"},"value":"active","within":10000} + ], + "criteria": [ + { + "id": "762a", + "category": "production", + "desc": "Another customer cannot cancel an active subscription", + "points": 2, + "steps": [ + {"do":"callAction","actor":"other","from":"owner","action":"cancelSubscription","namedAction":{"id":"cancelSubscription","path":"/api/subscriptions/{subscriptionId}/cancel","method":"POST","reducer":"cancel_subscription","args":[0],"params":[{"name":"subscriptionId","in":"path","placeholder":"{subscriptionId}","wireType":"u64"}]},"input":{"testid":"subscription-cancel","attribute":"data-action-input"},"settleMs":0}, + {"do":"expectActionOutcome","actor":"other","outcome":"refused"}, + {"do":"expect","actor":"owner","testid":"subscription-status","in":{"testid":"subscription-row","contains":"Desk Lamp"},"value":"active","within":10000}, + {"do":"click","actor":"owner","testid":"subscription-cancel","in":{"testid":"subscription-row","contains":"Desk Lamp"}}, + {"do":"freshClient","actor":"owner"}, + {"do":"signIn","actor":"owner-fresh","name":"subscription-762"}, + {"do":"click","actor":"owner-fresh","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"}, + {"do":"expect","actor":"owner-fresh","testid":"subscriptions-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"expect","actor":"owner-fresh","testid":"subscription-status","in":{"testid":"subscription-row","contains":"Desk Lamp"},"value":"cancelled","within":10000}, + {"do":"recordNumber","actor":"owner-fresh","testid":"subscription-delivery","in":{"testid":"subscription-row","contains":"Desk Lamp"},"count":true,"as":"deliveries-at-cancel"}, + {"do":"dbRecordStock","item":"Desk Lamp","as":"stock-at-cancel"}, + {"do":"wait","actor":"owner-fresh","ms":65000}, + {"do":"expectElementCount","actor":"owner-fresh","testid":"subscription-delivery","in":{"testid":"subscription-row","contains":"Desk Lamp"},"relativeTo":"deliveries-at-cancel","plus":0}, + {"do":"dbExpectStock","item":"Desk Lamp","relativeTo":"stock-at-cancel","plus":0} + ] + } + ] + }, + { + "id": 763, + "name": "Pause survives a restart and resume completes the remaining deliveries", + "actors": [ + "owner", + "other" + ], + "setup": [ + {"do":"signUp","actor":"owner","name":"subscription-763"}, + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"East","quantity":10,"settleMs":0}, + {"do":"dbSetStock","item":"Desk Lamp","warehouse":"West","quantity":0,"settleMs":0}, + {"do":"click","actor":"owner","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"}, + {"do":"expect","actor":"owner","testid":"subscriptions-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"fill","actor":"owner","testid":"subscription-item-input","text":"Desk Lamp"}, + {"do":"fill","actor":"owner","testid":"subscription-quantity-input","text":"1"}, + {"do":"fill","actor":"owner","testid":"subscription-interval-input","text":"30"}, + {"do":"fill","actor":"owner","testid":"subscription-deliveries-input","text":"2"}, + {"do":"click","actor":"owner","testid":"subscription-create"}, + {"do":"expect","actor":"owner","testid":"subscription-status","in":{"testid":"subscription-row","contains":"Desk Lamp"},"value":"active","within":10000} + ], + "criteria": [ + { + "id": "763a", + "category": "production", + "desc": "Pause survives a restart and resume completes the remaining deliveries", + "points": 2, + "steps": [ + {"do":"click","actor":"owner","testid":"subscription-pause","in":{"testid":"subscription-row","contains":"Desk Lamp"}}, + {"do":"expect","actor":"owner","testid":"subscription-status","in":{"testid":"subscription-row","contains":"Desk Lamp"},"value":"paused","within":10000}, + {"do":"restartBackend","settleMs":1000}, + {"do":"freshClient","actor":"owner"}, + {"do":"signIn","actor":"owner-fresh","name":"subscription-763"}, + {"do":"click","actor":"owner-fresh","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"}, + {"do":"expect","actor":"owner-fresh","testid":"subscriptions-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"expect","actor":"owner-fresh","testid":"subscription-status","in":{"testid":"subscription-row","contains":"Desk Lamp"},"value":"paused","within":10000}, + {"do":"recordNumber","actor":"owner-fresh","testid":"subscription-delivery","in":{"testid":"subscription-row","contains":"Desk Lamp"},"count":true,"as":"deliveries-at-pause"}, + {"do":"dbRecordStock","item":"Desk Lamp","as":"stock-at-pause"}, + {"do":"wait","actor":"owner-fresh","ms":35000}, + {"do":"expectElementCount","actor":"owner-fresh","testid":"subscription-delivery","in":{"testid":"subscription-row","contains":"Desk Lamp"},"relativeTo":"deliveries-at-pause","plus":0}, + {"do":"dbExpectStock","item":"Desk Lamp","relativeTo":"stock-at-pause","plus":0}, + {"do":"click","awaitWrites":true,"actor":"owner-fresh","testid":"subscription-resume","in":{"testid":"subscription-row","contains":"Desk Lamp"}}, + {"do":"freshClient","actor":"owner"}, + {"do":"signIn","actor":"owner-fresh","name":"subscription-763"}, + {"do":"click","actor":"owner-fresh","testid":"subscriptions-link","unlessVisible":"subscriptions-panel"}, + {"do":"expect","actor":"owner-fresh","testid":"subscriptions-panel","attribute":"aria-busy","value":"false","within":10000}, + {"do":"expect","actor":"owner-fresh","testid":"subscription-status","in":{"testid":"subscription-row","contains":"Desk Lamp"},"value":"complete","within":90000}, + {"do":"expectElementCount","actor":"owner-fresh","testid":"subscription-delivery","in":{"testid":"subscription-row","contains":"Desk Lamp"},"equals":2}, + {"do":"expectSequence","actor":"owner-fresh","testid":"subscription-delivery-status","in":{"testid":"subscription-row","contains":"Desk Lamp"},"equals":["paid","paid"]}, + {"do":"expectNumber","actor":"owner-fresh","testid":"subscription-total","in":{"testid":"subscription-row","contains":"Desk Lamp"},"equals":84}, + {"do":"dbExpectStock","item":"Desk Lamp","equals":8}, + {"do":"click","actor":"owner-fresh","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"owner-fresh","testid":"orders-toggle"}, + {"do":"expectElementCount","actor":"owner-fresh","testid":"order-item","contains":"Desk Lamp","equals":2,"within":10000}, + {"do":"expectElementCount","actor":"owner-fresh","testid":"payment-record","equals":2} + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-support-history.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-support-history.json new file mode 100644 index 00000000000..ff2ad9a4b5a --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-support-history.json @@ -0,0 +1,148 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 2, + "name": "Customer support history", + "note": "Draft: updated observations require matching reference and defect-control qualification.", + "features": [ + { + "id": 612, + "name": "Customer support history", + "actors": ["owner", "other"], + "setup": [ + { "do": "signUp", "actor": "owner", "name": "support-owner" }, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"owner","testid":"current-user","ifAvailable":true,"unlessVisible":["support-link","support-email"]}, + {"do":"click","actor":"owner","testid":"support-link","unlessVisible":"support-email"}, + { "do": "fill", "actor": "owner", "testid": "support-email", "text": "owner@example.com" }, + { "do": "fill", "actor": "owner", "testid": "support-subject", "text": "Owner ticket {user:ticketmarker}" }, + { "do": "fill", "actor": "owner", "testid": "support-message", "text": "Private account issue." }, + { "do": "click", "actor": "owner", "testid": "support-submit" }, + { "do": "expect", "actor": "owner", "testid": "support-reference", "nonEmpty": true }, + { "do": "signUp", "actor": "other", "name": "support-other" } + ], + "criteria": [ + { + "id": "612c", + "category": "feature", + "desc": "the customer can view their support ticket history", + "points": 1, + "statedBy": "Signed-in customers can see their support ticket history.", + "steps": [ + { "do": "reload", "actor": "owner", "settleMs": 1000 }, + { "do": "ensureSignedIn", "actor": "owner", "name": "support-owner", "readyTestid": "current-user" }, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"owner","testid":"current-user","ifAvailable":true,"unlessVisible":["support-link","support-ticket"]}, + {"do":"click","actor":"owner","testid":"support-link","unlessVisible":"support-ticket"}, + { "do": "expect", "actor": "owner", "testid": "support-ticket", "contains": "Owner ticket {user:ticketmarker}" } + ] + }, + { + "id": "612a", + "category": "production", + "desc": "support history survives reload and backend restart in a fresh browser", + "points": 2, + "statedBy": "A customer's support history persists.", + "steps": [ + { "do": "reload", "actor": "owner", "settleMs": 3000 }, + { "do": "ensureSignedIn", "actor": "owner", "name": "support-owner", "readyTestid": "current-user" }, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"owner","testid":"current-user","ifAvailable":true,"unlessVisible":["support-link","support-ticket"]}, + {"do":"click","actor":"owner","testid":"support-link","unlessVisible":"support-ticket"}, + { "do": "expect", "actor": "owner", "testid": "support-ticket", "contains": "Owner ticket {user:ticketmarker}" }, + { "do": "restartBackend", "settleMs": 1000 }, + { "do": "freshClient", "actor": "owner" }, + { "do": "signIn", "actor": "owner-fresh", "name": "support-owner" }, + {"do":"click","actor":"owner-fresh","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"owner-fresh","testid":"current-user","ifAvailable":true,"unlessVisible":["support-link","support-ticket"]}, + {"do":"click","actor":"owner-fresh","testid":"support-link","unlessVisible":"support-ticket"}, + { "do": "expect", "actor": "owner-fresh", "testid": "support-ticket", "contains": "Owner ticket {user:ticketmarker}" } + ] + }, + { + "id": "612b", + "category": "production", + "desc": "another customer neither sees nor receives the private ticket", + "points": 2, + "statedBy": "Customers cannot see another customer's support tickets.", + "steps": [ + { "do": "reload", "actor": "owner", "settleMs": 1000 }, + { "do": "ensureSignedIn", "actor": "owner", "name": "support-owner", "readyTestid": "current-user" }, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"owner","testid":"current-user","ifAvailable":true,"unlessVisible":["support-link","support-email"]}, + {"do":"click","actor":"owner","testid":"support-link","unlessVisible":"support-email"}, + { "do": "fill", "actor": "owner", "testid": "support-email", "text": "owner@example.com" }, + { "do": "fill", "actor": "owner", "testid": "support-subject", "text": "Private ticket {user:privateticketmarker}" }, + { "do": "fill", "actor": "owner", "testid": "support-message", "text": "Private account issue." }, + { "do": "click", "actor": "owner", "testid": "support-submit" }, + { "do": "expect", "actor": "owner", "testid": "support-reference", "nonEmpty": true }, + { "do": "reload", "actor": "owner", "settleMs": 1000 }, + { "do": "ensureSignedIn", "actor": "owner", "name": "support-owner", "readyTestid": "current-user" }, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"owner","testid":"current-user","ifAvailable":true,"unlessVisible":["support-link","support-ticket"]}, + {"do":"click","actor":"owner","testid":"support-link","unlessVisible":"support-ticket"}, + { "do": "expect", "actor": "owner", "testid": "support-ticket", "contains": "Private ticket {user:privateticketmarker}" }, + { "do": "expectReceived", "actor": "owner", "contains": "Private ticket {user:privateticketmarker}", "within": 10000 }, + { "do": "reload", "actor": "other", "settleMs": 0 }, + { "do": "ensureSignedIn", "actor": "other", "name": "support-other", "readyTestid": "current-user" }, + {"do":"click","actor":"other","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"other","testid":"current-user","ifAvailable":true,"unlessVisible":["support-link","support-email"]}, + {"do":"click","actor":"other","testid":"support-link","unlessVisible":"support-email"}, + { "do": "fill", "actor": "other", "testid": "support-email", "text": "other@example.com" }, + { "do": "fill", "actor": "other", "testid": "support-subject", "text": "Other ticket" }, + { "do": "fill", "actor": "other", "testid": "support-message", "text": "Separate account issue." }, + { "do": "click", "actor": "other", "testid": "support-submit" }, + { "do": "expect", "actor": "other", "testid": "support-reference", "nonEmpty": true }, + { "do": "reload", "actor": "other", "settleMs": 1000 }, + { "do": "ensureSignedIn", "actor": "other", "name": "support-other", "readyTestid": "current-user" }, + {"do":"click","actor":"other","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"other","testid":"current-user","ifAvailable":true,"unlessVisible":["support-link","support-ticket"]}, + {"do":"click","actor":"other","testid":"support-link","unlessVisible":"support-ticket"}, + { "do": "expect", "actor": "other", "testid": "support-ticket", "contains": "Other ticket", "within": 10000 }, + { "do": "expect", "actor": "other", "testid": "support-ticket", "contains": "Private ticket {user:privateticketmarker}", "absent": true }, + { "do": "expectNotReceived", "actor": "other", "contains": "Private ticket {user:privateticketmarker}" } + ] + }, + { + "id": "612d", + "category": "production", + "desc": "after logout the same browser storage no longer grants access to private support history", + "points": 2, + "statedBy": "Signed-out visitors cannot access private support tickets.", + "steps": [ + {"do":"reload","actor":"owner","settleMs":1000}, + {"do":"ensureSignedIn","actor":"owner","name":"support-owner","readyTestid":"current-user"}, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"owner","testid":"current-user","ifAvailable":true,"unlessVisible":["support-link","support-email"]}, + {"do":"click","actor":"owner","testid":"support-link","unlessVisible":"support-email"}, + {"do":"fill","actor":"owner","testid":"support-email","text":"owner@example.com"}, + {"do":"fill","actor":"owner","testid":"support-subject","text":"Logout ticket {user:logoutticketmarker}"}, + {"do":"fill","actor":"owner","testid":"support-message","text":"Private account issue."}, + {"do":"click","actor":"owner","testid":"support-submit"}, + {"do":"expect","actor":"owner","testid":"support-reference","nonEmpty":true}, + {"do":"reload","actor":"owner","settleMs":1000}, + {"do":"ensureSignedIn","actor":"owner","name":"support-owner","readyTestid":"current-user"}, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"owner","testid":"current-user","ifAvailable":true,"unlessVisible":["support-link","support-ticket"]}, + {"do":"click","actor":"owner","testid":"support-link","unlessVisible":"support-ticket"}, + {"do":"expect","actor":"owner","testid":"support-ticket","contains":"Logout ticket {user:logoutticketmarker}"}, + {"do":"expectReceived","actor":"owner","contains":"Logout ticket {user:logoutticketmarker}","within":10000}, + {"do":"reload","actor":"owner","settleMs":1000}, + {"do":"ensureSignedIn","actor":"owner","name":"support-owner","readyTestid":"current-user"}, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"owner","testid":"current-user","unlessVisible":"signout"}, + {"do":"click","actor":"owner","testid":"signout"}, + {"do":"waitUntilAbsent","actor":"owner","testid":"current-user","within":6000}, + {"do":"freshClient","actor":"owner","preserveStorage":true}, + {"do":"expect","actor":"owner-fresh","testid":"current-user","absent":true}, + {"do":"click","actor":"owner-fresh","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"owner-fresh","testid":"support-link","ifAvailable":true,"unlessVisible":"support-ticket"}, + {"do":"expect","actor":"owner-fresh","testid":"support-email"}, + {"do":"expect","actor":"owner-fresh","testid":"support-ticket","contains":"Logout ticket {user:logoutticketmarker}","absent":true}, + {"do":"expectNotReceived","actor":"owner-fresh","contains":"Logout ticket {user:logoutticketmarker}"} + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-support-intake.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-support-intake.json new file mode 100644 index 00000000000..34e62c8bccd --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-support-intake.json @@ -0,0 +1,31 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 1, + "name": "Support intake", + "features": [ + { + "id": 610, + "name": "Support intake", + "actors": ["visitor"], + "setup": [], + "criteria": [ + { + "id": "610a", + "category": "feature", + "desc": "a visitor can submit a support ticket and receives a reference", + "points": 2, + "statedBy": "Anyone can open a support ticket and receives a reference.", + "steps": [ + { "do": "click", "actor": "visitor", "testid": "support-link" }, + { "do": "fill", "actor": "visitor", "testid": "support-email", "text": "visitor@example.com" }, + { "do": "fill", "actor": "visitor", "testid": "support-subject", "text": "Damaged package" }, + { "do": "fill", "actor": "visitor", "testid": "support-message", "text": "The package arrived damaged." }, + { "do": "click", "actor": "visitor", "testid": "support-submit" }, + { "do": "expect", "actor": "visitor", "testid": "support-reference", "nonEmpty": true } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-support-refunds-access.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-support-refunds-access.json new file mode 100644 index 00000000000..09a77c0d751 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-support-refunds-access.json @@ -0,0 +1,61 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 5, + "name": "Support refund access", + "note": "Draft: revised observations require matching reference and defect-control qualification.", + "features": [ + { + "id": 615, + "name": "Support refund access", + "actors": ["owner", "witness", "staff"], + "setup": [ + { "do": "signUp", "actor": "owner", "name": "refund-access-owner" }, + { "do": "click", "actor": "owner", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Keyboard" } }, + { "do": "click", "actor": "owner", "testid": "support-link" }, + { "do": "fill", "actor": "owner", "testid": "support-subject", "text": "Access refund case" }, + { "do": "fill", "actor": "owner", "testid": "support-message", "text": "This refund requires staff approval." }, + { "do": "click", "actor": "owner", "testid": "support-submit" }, + { "do": "click", "actor": "owner", "testid": "support-order-option", "contains": "Keyboard", "in": { "testid": "support-ticket", "contains": "Access refund case" } }, + { "do": "click", "actor": "owner", "testid": "support-link-order", "in": { "testid": "support-ticket", "contains": "Access refund case" } }, + { "do": "signUp", "actor": "witness", "name": "refund-access-witness" }, + { "do": "click", "actor": "witness", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Mouse" } }, + { "do": "click", "actor": "witness", "testid": "overlay-close", "ifAvailable": true, "within": 1000 }, + { "do": "click", "actor": "witness", "testid": "support-link" }, + { "do": "fill", "actor": "witness", "testid": "support-subject", "text": "Witness refund case" }, + { "do": "fill", "actor": "witness", "testid": "support-message", "text": "Refund this order." }, + { "do": "click", "actor": "witness", "testid": "support-submit" }, + { "do": "click", "actor": "witness", "testid": "support-order-option", "contains": "Mouse", "in": { "testid": "support-ticket", "contains": "Witness refund case" } }, + { "do": "click", "actor": "witness", "testid": "support-link-order", "in": { "testid": "support-ticket", "contains": "Witness refund case" } }, + { "do": "signIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true }, + {"do":"click","actor":"staff","testid":"staff-link","unlessVisible":"support-assignee"}, + {"do":"click","actor":"staff","testid":"support-queue-link","unlessVisible":"support-assignee","ifAvailable":true,"within":1000}, + {"do":"callAction","actor":"staff","action":"supportRefund","input":{"testid":"support-refund","attribute":"data-action-input","in":{"testid":"support-ticket","contains":"Witness refund case"}},"namedAction":{"id":"supportRefund","path":"/api/support/cases/{caseId}/refund","method":"POST","reducer":"support_refund","args":[0],"params":[{"name":"caseId","in":"path","placeholder":"{caseId}","wireType":"u64"}]},"settleMs":1500}, + {"do":"expectActionOutcome","actor":"staff","outcome":"accepted"} + ], + "criteria": [ + { + "id": "615c", + "category": "production", + "desc": "a customer cannot issue a refund or change its records", + "points": 1, + "statedBy": "Only authorized staff can refund an order from a support case.", + "steps": [ + {"do":"expect","actor":"owner","testid":"support-refund","absent":true}, + {"do":"callAction","actor":"owner","from":"staff","action":"supportRefund","authentication":"actor","input":{"testid":"support-refund","attribute":"data-action-input","in":{"testid":"support-ticket","contains":"Access refund case"}},"namedAction":{"id":"supportRefund","path":"/api/support/cases/{caseId}/refund","method":"POST","reducer":"support_refund","args":[0],"params":[{"name":"caseId","in":"path","placeholder":"{caseId}","wireType":"u64"}]},"settleMs":1500}, + {"do":"expectActionOutcome","actor":"owner","outcome":"refused","routeProvenBy":"staff"}, + {"do":"reload","actor":"owner","settleMs":1000}, + {"do":"ensureSignedIn","actor":"owner","name":"refund-access-owner","readyTestid":"current-user"}, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"owner","testid":"support-link","unlessVisible":"support-subject"}, + {"do":"expect","actor":"owner","testid":"support-status","in":{"testid":"support-ticket","contains":"Access refund case"},"notContains":"resolved"}, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"owner","testid":"orders-toggle"}, + {"do":"expect","actor":"owner","testid":"order-status","in":{"testid":"order-item","contains":"Keyboard"},"statusText":"pending","ignoreCase":true}, + {"do":"expect","actor":"owner","testid":"refund-entry","contains":"Keyboard","absent":true} + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-support-refunds-accounting.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-support-refunds-accounting.json new file mode 100644 index 00000000000..1712dee26fa --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-support-refunds-accounting.json @@ -0,0 +1,64 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 5, + "name": "Support refund accounting", + "note": "Draft: revised observations require matching reference and defect-control qualification.", + "features": [ + { + "id": 615, + "name": "Support refund accounting", + "actors": ["owner", "staff"], + "setup": [ + { "do": "signUp", "actor": "owner", "name": "refund-accounting-owner" }, + { "do": "click", "actor": "owner", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Mouse" } }, + { "do": "click", "actor": "owner", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Keyboard" } }, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { "do": "click", "actor": "owner", "testid": "orders-toggle", "unlessVisible": "order-item" }, + { "do": "recordNumber", "actor": "owner", "testid": "order-total", "in": { "testid": "order-item", "contains": "Keyboard" }, "as": "paid-total" }, + { "do": "click", "actor": "owner", "testid": "overlay-close", "ifAvailable": true, "within": 1000 }, + { "do": "click", "actor": "owner", "testid": "support-link" }, + { "do": "fill", "actor": "owner", "testid": "support-subject", "text": "Accounting refund case" }, + { "do": "fill", "actor": "owner", "testid": "support-message", "text": "Refund the exact amount paid." }, + { "do": "click", "actor": "owner", "testid": "support-submit" }, + { "do": "click", "actor": "owner", "testid": "support-order-option", "contains": "Keyboard", "in": { "testid": "support-ticket", "contains": "Accounting refund case" } }, + { "do": "click", "actor": "owner", "testid": "support-link-order", "in": { "testid": "support-ticket", "contains": "Accounting refund case" } }, + { "do": "signIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true }, + {"do":"click","actor":"staff","testid":"staff-link","unlessVisible":"support-assignee"}, + {"do":"click","actor":"staff","testid":"support-queue-link","unlessVisible":"support-assignee","ifAvailable":true,"within":1000} + ], + "criteria": [ + { + "id": "615b", + "category": "production", + "desc": "the refund equals the paid total, cannot be applied twice, and leaves another order unrefunded", + "points": 1, + "statedBy": "The recorded refund equals the amount paid and is applied only once.", + "steps": [ + {"do":"click","awaitWrites":true,"actor":"staff","testid":"support-refund","in":{"testid":"support-ticket","contains":"Accounting refund case"}}, + {"do":"reload","actor":"owner","settleMs":1000}, + {"do":"ensureSignedIn","actor":"owner","name":"refund-accounting-owner","readyTestid":"current-user"}, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"within":1000}, + {"do":"click","actor":"owner","testid":"support-link","unlessVisible":"support-subject"}, + {"do":"expectNumber","actor":"owner","testid":"support-refund-total","in":{"testid":"support-ticket","contains":"Accounting refund case"},"relativeTo":"paid-total","plus":0,"within":10000}, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"owner","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expectNumber","actor":"owner","testid":"order-refund-total","in":{"testid":"order-item","contains":"Keyboard"},"relativeTo":"paid-total","plus":0,"within":10000}, + {"do":"expectElementCount","actor":"owner","testid":"refund-entry","in":{"testid":"order-item","contains":"Keyboard"},"contains":"Keyboard","equals":1,"within":10000}, + {"do":"replayAs","actor":"staff","from":"staff","match":"refund","namedAction":{"id":"supportRefund","path":"/api/support/cases/{caseId}/refund","method":"POST","reducer":"support_refund","args":[0],"params":[{"name":"caseId","in":"path","placeholder":"{caseId}","wireType":"u64"}]},"namedTarget":{"testid":"support-ticket","contains":"Accounting refund case","attribute":"data-entity-id","valueType":"string"},"settleMs":1500}, + {"do":"expectReplayCompleted","actor":"staff"}, + {"do":"freshClient","actor":"owner"}, + {"do":"signIn","actor":"owner-fresh","name":"refund-accounting-owner"}, + {"do":"click","actor":"owner-fresh","testid":"support-link"}, + {"do":"expectNumber","actor":"owner-fresh","testid":"support-refund-total","in":{"testid":"support-ticket","contains":"Accounting refund case"},"relativeTo":"paid-total","plus":0,"within":10000}, + {"do":"click","actor":"owner-fresh","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + {"do":"click","actor":"owner-fresh","testid":"orders-toggle","unlessVisible":"order-item"}, + {"do":"expectNumber","actor":"owner-fresh","testid":"order-refund-total","in":{"testid":"order-item","contains":"Keyboard"},"relativeTo":"paid-total","plus":0,"within":10000}, + {"do":"expectElementCount","actor":"owner-fresh","testid":"refund-entry","in":{"testid":"order-item","contains":"Keyboard"},"contains":"Keyboard","equals":1,"within":10000}, + {"do":"expectElementCount","actor":"owner-fresh","testid":"refund-entry","in":{"testid":"order-item","contains":"Mouse"},"contains":"Mouse","equals":0,"within":10000} + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-support-refunds-resolution.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-support-refunds-resolution.json new file mode 100644 index 00000000000..c33a9b33e31 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-support-refunds-resolution.json @@ -0,0 +1,42 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 5, + "name": "Support refund resolution", + "features": [ + { + "id": 615, + "name": "Support refund resolution", + "actors": ["owner", "staff"], + "setup": [ + { "do": "signUp", "actor": "owner", "name": "refund-resolution-owner" }, + { "do": "click", "actor": "owner", "testid": "buy-now", "in": { "testid": "item-card", "contains": "Keyboard" } }, + { "do": "click", "actor": "owner", "testid": "support-link" }, + { "do": "fill", "actor": "owner", "testid": "support-subject", "text": "Resolution refund case" }, + { "do": "fill", "actor": "owner", "testid": "support-message", "text": "Please refund my Keyboard order." }, + { "do": "click", "actor": "owner", "testid": "support-submit" }, + { "do": "click", "actor": "owner", "testid": "support-order-option", "contains": "Keyboard", "in": { "testid": "support-ticket", "contains": "Resolution refund case" } }, + { "do": "click", "actor": "owner", "testid": "support-link-order", "in": { "testid": "support-ticket", "contains": "Resolution refund case" } }, + { "do": "signIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true }, + {"do":"click","actor":"staff","testid":"staff-link","unlessVisible":"support-assignee"}, + {"do":"click","actor":"staff","testid":"support-queue-link","unlessVisible":"support-assignee","ifAvailable":true} + ], + "criteria": [ + { + "id": "615a", + "category": "feature", + "desc": "an authorized refund resolves the case and updates the order", + "points": 2, + "statedBy": "Authorized staff can refund an order from its support case.", + "steps": [ + { "do": "click", "actor": "staff", "testid": "support-refund", "in": { "testid": "support-ticket", "contains": "Resolution refund case" } }, + { "do": "expect", "actor": "owner", "testid": "support-status", "in": { "testid": "support-ticket", "contains": "Resolution refund case" }, "contains": "resolved", "within": 10000 }, + {"do":"click","actor":"owner","testid":"overlay-close","ifAvailable":true,"unlessVisible":"order-item","within":1000}, + { "do": "click", "actor": "owner", "testid": "orders-toggle" }, + { "do": "expect", "actor": "owner", "testid": "order-status", "in": { "testid": "order-item", "contains": "Keyboard" }, "contains": "refunded", "within": 10000 } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-support-return-interaction.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-support-return-interaction.json new file mode 100644 index 00000000000..8e7e22adc2f --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-support-return-interaction.json @@ -0,0 +1,792 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 6, + "name": "Return and support refund accounting", + "features": [ + { + "id": 757, + "name": "Return and support refund accounting", + "actors": [ + "owner", + "staff", + "admin" + ], + "setup": [ + { + "do": "signUp", + "actor": "owner", + "name": "return-refund-owner" + }, + { + "do": "signIn", + "actor": "staff", + "name": "staff", + "password": "stackbench-staff-2026", + "exact": true + }, + { + "do": "signIn", + "actor": "admin", + "name": "admin", + "password": "stackbench-admin-2026", + "exact": true + }, + { + "do": "click", + "actor": "staff", + "testid": "staff-link", + "unlessVisible": "support-assignee" + }, + { + "do": "click", + "actor": "staff", + "testid": "support-queue-link", + "unlessVisible": "support-assignee", + "ifAvailable": true + }, + { + "do": "click", + "actor": "admin", + "testid": "admin-link" + } + ], + "criteria": [ + { + "id": "757a", + "category": "production", + "desc": "a support refund followed by physical return restores each warehouse once and refunds only the price paid", + "points": 2, + "steps": [ + { + "do": "reload", + "actor": "owner", + "settleMs": 0 + }, + { + "do": "ensureSignedIn", + "actor": "owner", + "name": "return-refund-owner", + "readyTestid": "current-user", + "settleMs": 0 + }, + { + "do": "click", + "actor": "owner", + "testid": "catalog-link", + "ifAvailable": true + }, + { + "do": "dbRecordStock", + "item": "Keyboard", + "as": "757atotal" + }, + { + "do": "dbRecordStock", + "item": "Keyboard", + "warehouse": "East", + "as": "757aEast" + }, + { + "do": "dbRecordStock", + "item": "Keyboard", + "warehouse": "West", + "as": "757aWest" + }, + { + "do": "recordNumber", + "actor": "admin", + "testid": "admin-revenue", + "as": "757arevenue" + }, + { + "do": "click", + "actor": "owner", + "testid": "buy-now", + "in": { + "testid": "item-card", + "contains": "Keyboard" + } + }, + { + "do": "click", + "actor": "owner", + "testid": "overlay-close", + "ifAvailable": true, + "unlessVisible": "order-item", + "within": 1000 + }, + { + "do": "click", + "actor": "owner", + "testid": "orders-toggle", + "unlessVisible": "order-item" + }, + { + "do": "expect", + "actor": "owner", + "testid": "order-status", + "within": 10000, + "in": { + "testid": "order-item", + "contains": "Keyboard" + }, + "statusText": "pending", + "ignoreCase": true + }, + { + "do": "recordNumber", + "actor": "owner", + "testid": "order-total", + "in": { + "testid": "order-item", + "contains": "Keyboard" + }, + "as": "757apaid" + }, + { + "do": "callAction", + "actor": "staff", + "action": "ship", + "namedAction": { + "id": "ship", + "path": "/api/fulfilment/ship", + "reducer": "ship_order", + "args": [ + 0 + ], + "params": [ + { + "name": "orderId", + "in": "body", + "wireType": "u64" + } + ] + }, + "input": { + "testid": "order-item", + "contains": "Keyboard", + "attribute": "data-ship-input" + }, + "from": "owner", + "settleMs": 0 + }, + { + "do": "expectActionOutcome", + "actor": "staff", + "outcome": "accepted" + }, + { + "do": "expect", + "actor": "owner", + "testid": "order-status", + "within": 10000, + "in": { + "testid": "order-item", + "contains": "Keyboard" + }, + "statusText": "shipped", + "ignoreCase": true + }, + { + "do": "dbExpectStock", + "item": "Keyboard", + "relativeTo": "757atotal", + "plus": -1 + }, + { + "do": "reload", + "actor": "owner", + "settleMs": 0 + }, + { + "do": "ensureSignedIn", + "actor": "owner", + "name": "return-refund-owner", + "readyTestid": "current-user", + "settleMs": 0 + }, + { + "do": "click", + "actor": "owner", + "testid": "support-link" + }, + { + "do": "fill", + "actor": "owner", + "testid": "support-subject", + "text": "Return refund 757a" + }, + { + "do": "fill", + "actor": "owner", + "testid": "support-message", + "text": "Please refund this order." + }, + { + "do": "click", + "actor": "owner", + "testid": "support-submit" + }, + { + "do": "expect", + "actor": "owner", + "testid": "support-ticket", + "within": 10000, + "contains": "Return refund 757a" + }, + { + "do": "click", + "actor": "owner", + "testid": "support-order-option", + "contains": "Keyboard", + "in": { + "testid": "support-ticket", + "contains": "Return refund 757a" + } + }, + { + "do": "click", + "actor": "owner", + "testid": "support-link-order", + "in": { + "testid": "support-ticket", + "contains": "Return refund 757a" + } + }, + { + "do": "expect", + "actor": "owner", + "testid": "support-order", + "within": 10000, + "in": { + "testid": "support-ticket", + "contains": "Return refund 757a" + } + }, + { + "do": "callAction", + "actor": "staff", + "action": "supportRefund", + "namedAction": { + "id": "supportRefund", + "path": "/api/support/cases/{caseId}/refund", + "method": "POST", + "reducer": "support_refund", + "args": [ + 0 + ], + "params": [ + { + "name": "caseId", + "in": "path", + "placeholder": "{caseId}", + "wireType": "u64" + } + ] + }, + "input": { + "testid": "support-ticket", + "contains": "Return refund 757a", + "attribute": "data-refund-input" + }, + "from": "owner", + "settleMs": 0 + }, + { + "do": "expectActionOutcome", + "actor": "staff", + "outcome": "accepted" + }, + { + "do": "dbExpectStock", + "item": "Keyboard", + "relativeTo": "757atotal", + "plus": -1 + }, + { + "do": "click", + "actor": "owner", + "testid": "overlay-close", + "ifAvailable": true, + "unlessVisible": "order-item", + "within": 1000 + }, + { + "do": "click", + "actor": "owner", + "testid": "orders-toggle", + "unlessVisible": "order-item" + }, + { + "do": "click", + "actor": "owner", + "testid": "return-item", + "in": { + "testid": "order-item", + "contains": "Keyboard" + } + }, + { + "do": "expect", + "actor": "owner", + "testid": "order-item", + "within": 10000, + "contains": "Keyboard", + "ignoreCase": true, + "containsText": "returned" + }, + { + "do": "freshClient", + "actor": "owner" + }, + { + "do": "signIn", + "actor": "owner-fresh", + "name": "return-refund-owner" + }, + { + "do": "click", + "actor": "owner-fresh", + "testid": "overlay-close", + "ifAvailable": true, + "unlessVisible": "order-item", + "within": 1000 + }, + { + "do": "click", + "actor": "owner-fresh", + "testid": "orders-toggle" + }, + { + "do": "expect", + "actor": "owner-fresh", + "testid": "order-item", + "within": 10000, + "contains": "Keyboard", + "ignoreCase": true, + "containsText": "returned" + }, + { + "do": "expectNumber", + "actor": "owner-fresh", + "testid": "order-refund-total", + "in": { + "testid": "order-item", + "contains": "Keyboard" + }, + "within": 10000, + "relativeTo": "757apaid", + "plus": 0 + }, + { + "do": "dbExpectStock", + "item": "Keyboard", + "warehouse": "East", + "relativeTo": "757aEast", + "plus": 0 + }, + { + "do": "dbExpectStock", + "item": "Keyboard", + "warehouse": "West", + "relativeTo": "757aWest", + "plus": 0 + }, + { + "do": "reload", + "actor": "admin", + "settleMs": 0 + }, + { + "do": "ensureSignedIn", + "actor": "admin", + "name": "admin", + "password": "stackbench-admin-2026", + "exact": true, + "readyTestid": "current-user" + }, + { + "do": "click", + "actor": "admin", + "testid": "admin-link", + "unlessVisible": "admin-revenue" + }, + { + "do": "expectNumber", + "actor": "admin", + "testid": "admin-revenue", + "relativeTo": "757arevenue", + "plus": 0, + "within": 10000 + } + ] + }, + { + "id": "757b", + "category": "production", + "desc": "a physical return followed by support refund restores each warehouse once and refunds only the price paid", + "points": 2, + "steps": [ + { + "do": "reload", + "actor": "owner", + "settleMs": 0 + }, + { + "do": "ensureSignedIn", + "actor": "owner", + "name": "return-refund-owner", + "readyTestid": "current-user", + "settleMs": 0 + }, + { + "do": "click", + "actor": "owner", + "testid": "catalog-link", + "ifAvailable": true + }, + { + "do": "dbRecordStock", + "item": "Desk Lamp", + "as": "757btotal" + }, + { + "do": "dbRecordStock", + "item": "Desk Lamp", + "warehouse": "East", + "as": "757bEast" + }, + { + "do": "dbRecordStock", + "item": "Desk Lamp", + "warehouse": "West", + "as": "757bWest" + }, + { + "do": "recordNumber", + "actor": "admin", + "testid": "admin-revenue", + "as": "757brevenue" + }, + { + "do": "click", + "actor": "owner", + "testid": "buy-now", + "in": { + "testid": "item-card", + "contains": "Desk Lamp" + } + }, + { + "do": "click", + "actor": "owner", + "testid": "overlay-close", + "ifAvailable": true, + "unlessVisible": "order-item", + "within": 1000 + }, + { + "do": "click", + "actor": "owner", + "testid": "orders-toggle", + "unlessVisible": "order-item" + }, + { + "do": "expect", + "actor": "owner", + "testid": "order-status", + "within": 10000, + "in": { + "testid": "order-item", + "contains": "Desk Lamp" + }, + "statusText": "pending", + "ignoreCase": true + }, + { + "do": "recordNumber", + "actor": "owner", + "testid": "order-total", + "in": { + "testid": "order-item", + "contains": "Desk Lamp" + }, + "as": "757bpaid" + }, + { + "do": "callAction", + "actor": "staff", + "action": "ship", + "namedAction": { + "id": "ship", + "path": "/api/fulfilment/ship", + "reducer": "ship_order", + "args": [ + 0 + ], + "params": [ + { + "name": "orderId", + "in": "body", + "wireType": "u64" + } + ] + }, + "input": { + "testid": "order-item", + "contains": "Desk Lamp", + "attribute": "data-ship-input" + }, + "from": "owner", + "settleMs": 0 + }, + { + "do": "expectActionOutcome", + "actor": "staff", + "outcome": "accepted" + }, + { + "do": "expect", + "actor": "owner", + "testid": "order-status", + "within": 10000, + "in": { + "testid": "order-item", + "contains": "Desk Lamp" + }, + "statusText": "shipped", + "ignoreCase": true + }, + { + "do": "dbExpectStock", + "item": "Desk Lamp", + "relativeTo": "757btotal", + "plus": -1 + }, + { + "do": "reload", + "actor": "owner", + "settleMs": 0 + }, + { + "do": "ensureSignedIn", + "actor": "owner", + "name": "return-refund-owner", + "readyTestid": "current-user", + "settleMs": 0 + }, + { + "do": "click", + "actor": "owner", + "testid": "support-link" + }, + { + "do": "fill", + "actor": "owner", + "testid": "support-subject", + "text": "Return refund 757b" + }, + { + "do": "fill", + "actor": "owner", + "testid": "support-message", + "text": "Please refund this order." + }, + { + "do": "click", + "actor": "owner", + "testid": "support-submit" + }, + { + "do": "expect", + "actor": "owner", + "testid": "support-ticket", + "within": 10000, + "contains": "Return refund 757b" + }, + { + "do": "click", + "actor": "owner", + "testid": "support-order-option", + "contains": "Desk Lamp", + "in": { + "testid": "support-ticket", + "contains": "Return refund 757b" + } + }, + { + "do": "click", + "actor": "owner", + "testid": "support-link-order", + "in": { + "testid": "support-ticket", + "contains": "Return refund 757b" + } + }, + { + "do": "expect", + "actor": "owner", + "testid": "support-order", + "within": 10000, + "in": { + "testid": "support-ticket", + "contains": "Return refund 757b" + } + }, + { + "do": "click", + "actor": "owner", + "testid": "overlay-close", + "ifAvailable": true, + "unlessVisible": "order-item", + "within": 1000 + }, + { + "do": "click", + "actor": "owner", + "testid": "orders-toggle", + "unlessVisible": "order-item" + }, + { + "do": "click", + "actor": "owner", + "testid": "return-item", + "in": { + "testid": "order-item", + "contains": "Desk Lamp" + } + }, + { + "do": "expect", + "actor": "owner", + "testid": "order-item", + "within": 10000, + "contains": "Desk Lamp", + "ignoreCase": true, + "containsText": "returned" + }, + { + "do": "callAction", + "actor": "staff", + "action": "supportRefund", + "namedAction": { + "id": "supportRefund", + "path": "/api/support/cases/{caseId}/refund", + "method": "POST", + "reducer": "support_refund", + "args": [ + 0 + ], + "params": [ + { + "name": "caseId", + "in": "path", + "placeholder": "{caseId}", + "wireType": "u64" + } + ] + }, + "input": { + "testid": "support-ticket", + "contains": "Return refund 757b", + "attribute": "data-refund-input" + }, + "from": "owner", + "settleMs": 0 + }, + { + "do": "expectActionOutcome", + "actor": "staff", + "outcome": "accepted" + }, + { + "do": "freshClient", + "actor": "owner" + }, + { + "do": "signIn", + "actor": "owner-fresh", + "name": "return-refund-owner" + }, + { + "do": "click", + "actor": "owner-fresh", + "testid": "overlay-close", + "ifAvailable": true, + "unlessVisible": "order-item", + "within": 1000 + }, + { + "do": "click", + "actor": "owner-fresh", + "testid": "orders-toggle" + }, + { + "do": "expect", + "actor": "owner-fresh", + "testid": "order-item", + "within": 10000, + "contains": "Desk Lamp", + "ignoreCase": true, + "containsText": "returned" + }, + { + "do": "expectNumber", + "actor": "owner-fresh", + "testid": "order-refund-total", + "in": { + "testid": "order-item", + "contains": "Desk Lamp" + }, + "within": 10000, + "relativeTo": "757bpaid", + "plus": 0 + }, + { + "do": "dbExpectStock", + "item": "Desk Lamp", + "warehouse": "East", + "relativeTo": "757bEast", + "plus": 0 + }, + { + "do": "dbExpectStock", + "item": "Desk Lamp", + "warehouse": "West", + "relativeTo": "757bWest", + "plus": 0 + }, + { + "do": "reload", + "actor": "admin", + "settleMs": 0 + }, + { + "do": "ensureSignedIn", + "actor": "admin", + "name": "admin", + "password": "stackbench-admin-2026", + "exact": true, + "readyTestid": "current-user" + }, + { + "do": "click", + "actor": "admin", + "testid": "admin-link", + "unlessVisible": "admin-revenue" + }, + { + "do": "expectNumber", + "actor": "admin", + "testid": "admin-revenue", + "relativeTo": "757brevenue", + "plus": 0, + "within": 10000 + } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/scenarios/progression-support-triage.json b/tools/stack-bench/tracks/ecommerce/scenarios/progression-support-triage.json new file mode 100644 index 00000000000..98ec3823b02 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/scenarios/progression-support-triage.json @@ -0,0 +1,76 @@ +{ + "schemaVersion": 1, + "track": "ecommerce", + "level": 2, + "name": "Support triage", + "features": [ + { + "id": 611, + "name": "Support triage", + "actors": ["visitor", "staff"], + "setup": [ + { "do": "click", "actor": "visitor", "testid": "support-link" }, + { "do": "fill", "actor": "visitor", "testid": "support-email", "text": "triage@example.com" }, + { "do": "fill", "actor": "visitor", "testid": "support-subject", "text": "Missing item" }, + { "do": "fill", "actor": "visitor", "testid": "support-message", "text": "One item is missing." }, + { "do": "click", "actor": "visitor", "testid": "support-submit" }, + { "do": "expect", "actor": "visitor", "testid": "support-reference", "nonEmpty": true }, + { "do": "signIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true }, + {"do": "click", "actor": "staff", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["staff-link", "support-assignee"]}, + {"do":"click","actor":"staff","testid":"staff-link","unlessVisible":"support-assignee"}, + {"do":"click","actor":"staff","testid":"support-queue-link","unlessVisible":"support-assignee","ifAvailable":true} + ], + "criteria": [ + { + "id": "611a", + "category": "feature", + "desc": "staff can assign a new ticket", + "points": 1, + "statedBy": "Staff can assign support tickets.", + "steps": [ + { "do": "fill", "actor": "staff", "testid": "support-assignee", "in": { "testid": "support-ticket", "contains": "Missing item" }, "text": "staff" }, + { "do": "click", "actor": "staff", "testid": "support-update", "in": { "testid": "support-ticket", "contains": "Missing item" } }, + { "do": "expect", "actor": "staff", "testid": "support-ticket", "contains": "Missing item", "attribute": "data-submit-state", "value": "succeeded" }, + { "do": "reload", "actor": "staff", "settleMs": 2500 }, + { "do": "ensureSignedIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true, "readyTestid": "current-user" }, + {"do": "click", "actor": "staff", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["staff-link", "support-assignee"]}, + {"do":"click","actor":"staff","testid":"staff-link","ifAvailable":true,"unlessVisible":"support-assignee"}, + {"do":"click","actor":"staff","testid":"support-queue-link","unlessVisible":"support-assignee","ifAvailable":true}, + { "do": "expect", "actor": "staff", "testid": "support-assignee", "in": { "testid": "support-ticket", "contains": "Missing item" }, "value": "staff" } + ] + }, + { + "id": "611b", + "category": "feature", + "desc": "staff can set a ticket priority", + "points": 1, + "statedBy": "Staff can set support ticket priority.", + "steps": [ + { "do": "fill", "actor": "staff", "testid": "support-priority", "in": { "testid": "support-ticket", "contains": "Missing item" }, "text": "high" }, + { "do": "click", "actor": "staff", "testid": "support-update", "in": { "testid": "support-ticket", "contains": "Missing item" } }, + { "do": "expect", "actor": "staff", "testid": "support-ticket", "contains": "Missing item", "attribute": "data-submit-state", "value": "succeeded" }, + { "do": "reload", "actor": "staff", "settleMs": 2500 }, + { "do": "ensureSignedIn", "actor": "staff", "name": "staff", "password": "stackbench-staff-2026", "exact": true, "readyTestid": "current-user" }, + {"do": "click", "actor": "staff", "testid": "current-user", "ifAvailable": true, "unlessVisible": ["staff-link", "support-assignee"]}, + {"do":"click","actor":"staff","testid":"staff-link","ifAvailable":true,"unlessVisible":"support-assignee"}, + {"do":"click","actor":"staff","testid":"support-queue-link","unlessVisible":"support-assignee","ifAvailable":true}, + { "do": "expect", "actor": "staff", "testid": "support-priority", "in": { "testid": "support-ticket", "contains": "Missing item" }, "value": "high" } + ] + }, + { + "id": "611c", + "category": "feature", + "desc": "staff can change a ticket status", + "points": 1, + "statedBy": "Staff can change support ticket status.", + "steps": [ + { "do": "fill", "actor": "staff", "testid": "support-status-input", "in": { "testid": "support-ticket", "contains": "Missing item" }, "text": "in progress" }, + { "do": "click", "actor": "staff", "testid": "support-update", "in": { "testid": "support-ticket", "contains": "Missing item" } }, + { "do": "expect", "actor": "staff", "testid": "support-ticket", "contains": "Missing item", "attribute": "data-submit-state", "value": "succeeded" }, + { "do": "expect", "actor": "staff", "testid": "support-status", "in": { "testid": "support-ticket", "contains": "Missing item" }, "contains": "in progress" } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/track.json b/tools/stack-bench/tracks/ecommerce/track.json new file mode 100644 index 00000000000..769919e43b5 --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/track.json @@ -0,0 +1,168 @@ +{ + "schemaVersion": 1, + "title": "Shop", + "slug": "ecom", + "validatedThrough": 0, + "plannedThrough": 6, + "portOffset": 300, + "restartProbe": "/api/items", + "reseedOnReset": true, + "databaseProvenance": { + "browserAction": "signUp" + }, + "suites": { + "1": [ + { + "id": "features", + "inherit": "none", + "spec": "scenarios/01-features.json" + }, + { + "id": "invariants", + "inherit": "all-higher-levels", + "spec": "scenarios/01-invariants.json" + }, + { + "id": "contention", + "inherit": "all-higher-levels", + "spec": "scenarios/01-contention.json" + }, + { + "id": "systems", + "inherit": "all-higher-levels", + "spec": "scenarios/01-systems.json" + } + ], + "2": [ + { + "id": "features", + "inherit": "none", + "spec": "scenarios/02-features.json" + }, + { + "id": "invariants", + "inherit": "all-higher-levels", + "spec": "scenarios/02-invariants.json" + }, + { + "id": "contention", + "inherit": "all-higher-levels", + "spec": "scenarios/01-contention.json" + } + ], + "3": [ + { + "id": "features", + "inherit": "none", + "spec": "scenarios/03-features.json" + }, + { + "id": "invariants", + "inherit": "all-higher-levels", + "spec": "scenarios/03-invariants.json" + }, + { + "id": "systems", + "inherit": "all-higher-levels", + "spec": "scenarios/01-systems.json" + } + ] + }, + "actions": [ + { + "id": "buy", + "path": "/api/items/00000000-0000-0000-0000-000000000000/buy", + "reducer": "buy_now", + "args": [ + 0 + ], + "params": [ + { + "name": "itemId", + "in": "body", + "wireType": "u64" + } + ] + }, + { + "id": "addToCart", + "path": "/api/cart", + "reducer": "add_to_cart", + "args": [ + 0, + 1 + ], + "params": [ + { + "name": "itemId", + "in": "body", + "wireType": "u64" + }, + { + "name": "quantity", + "in": "body" + } + ] + }, + { + "id": "checkout", + "path": "/api/checkout", + "reducer": "checkout", + "args": [] + }, + { + "id": "restock", + "path": "/api/admin/restock", + "reducer": "admin_restock", + "args": [ + 0, + 0, + 1 + ], + "params": [ + { + "name": "itemId", + "in": "body", + "wireType": "u64" + }, + { + "name": "warehouseId", + "in": "body", + "wireType": "u64" + }, + { + "name": "quantity", + "in": "body" + } + ] + }, + { + "id": "signUp", + "path": "/api/auth/signup", + "reducer": "sign_up", + "args": [ + "", + "" + ], + "params": [ + { + "name": "username", + "in": "body" + }, + { + "name": "password", + "in": "body" + } + ] + }, + { + "id": "signIn", + "path": "/api/auth/signin", + "reducer": "sign_in", + "args": [ + "", + "" + ] + } + ] +} diff --git a/tools/stack-bench/tracks/ecommerce/walk.ts b/tools/stack-bench/tracks/ecommerce/walk.ts new file mode 100644 index 00000000000..0521a441d4d --- /dev/null +++ b/tools/stack-bench/tracks/ecommerce/walk.ts @@ -0,0 +1,202 @@ +// This contract walk uses one browser and one customer; scenarios own multi-actor behavior. + +import { applyCredentialAliases } from '../../src/composition/credential-aliases.js'; +import type { Locator } from 'playwright'; +import type { LintHook, LintWalkContext } from '../../linter/lint.js'; + +const ADMIN_USER = 'admin'; +const ADMIN_PASS = 'stackbench-admin-2026'; + +export const seededAdminPassword = (credentialAliases: unknown): string => + applyCredentialAliases(ADMIN_PASS, credentialAliases); + +// SEARCH_ONLY is outside the opening list so search must cover the full catalog. +const CART_ITEM = 'Headphones'; +const REVIEW_ITEM = 'Air Purifier'; +const SEARCH_ONLY = 'Webcam'; + +export async function walk({ page, args, byStage, blocked, checkHook, results, uniq, tid, CHECK_TIMEOUT }: LintWalkContext): Promise { + const fail = (h: LintHook, detail: string): void => { results.push({ id: h.id, status: 'FAIL', detail }); }; + const pass = (h: LintHook): void => { results.push({ id: h.id, status: 'PASS' }); }; + const click = async (locator: Locator, label: string): Promise => { + try { + await locator.click({ timeout: CHECK_TIMEOUT }); + } catch { + // A live UI can replace a button while state arrives. Resolve the locator + // again once before declaring that the product cannot perform the action. + await page.waitForTimeout(200); + try { + await locator.click({ timeout: CHECK_TIMEOUT }); + } catch (error: unknown) { + throw new Error(`${label}: ${error instanceof Error ? error.message : String(error)}`, { cause: error }); + } + } + }; + + await page.goto(args.url ?? '', { waitUntil: 'domcontentloaded', timeout: 15000 }); + const hasStage = (...stages: string[]): boolean => stages.some(stage => byStage(stage).length > 0); + + // Stage: landing — the storefront and its prices are public, so everything + // here must resolve before anyone signs in. + let ok = true; + const needsAccount = hasStage('storefront', 'item', 'item-after-review', 'cart', + 'after-checkout', 'admin', 'operations', 'fulfilment'); + for (const h of byStage('landing')) ok = (await checkHook(page, h, results)) && ok; + + if (ok && needsAccount) { + // Use alphanumeric credentials because username character rules are not part of the contract. + await page.locator(tid('signup-username')).first().fill(`lint${uniq}`); + await page.locator(tid('signup-password')).first().fill(`pwlint${uniq}`); + await click(page.locator(tid('signup-submit')).first(), 'submit sign-up'); + } + + // Stage: storefront — signing up turns the buying controls on. + if (ok && byStage('storefront').length) { + for (const h of byStage('storefront')) ok = (await checkHook(page, h, results)) && ok; + } else blocked('storefront'); + + // Reviews are gated on having bought the item, so the walk earns the right + // before it exercises the form — otherwise a spec-compliant refusal would + // read as a missing hook. + if (ok && byStage('item-after-review').length) { + await click(page.locator(tid('item-card'), { hasText: REVIEW_ITEM }).first() + .locator(tid('buy-now')).first(), `buy ${REVIEW_ITEM}`); + await page.waitForTimeout(1200); + } + + // Stage: search-results — search for an item the storefront is not showing. + if (ok && byStage('search-results').length) { + const search = page.locator(tid('search-input')).first(); + await search.fill(SEARCH_ONLY); + await search.press('Enter').catch(() => {}); + for (const h of byStage('search-results')) { + if (h.id === 'search-results') { + const hit = page.locator(tid('search-results')).first(); + try { + await hit.waitFor({ state: 'attached', timeout: CHECK_TIMEOUT }); + const card = hit.locator(tid('item-card'), { hasText: SEARCH_ONLY }).first(); + await card.waitFor({ state: 'visible', timeout: CHECK_TIMEOUT }); + pass(h); + } catch { + fail(h, `searched for "${SEARCH_ONLY}" but no ${tid('item-card')} containing it appeared inside ${tid(h.id)} — search must cover the whole catalogue, not just the items on the storefront`); + ok = false; + } + } else ok = (await checkHook(page, h, results)) && ok; + } + await search.fill(''); + await search.press('Enter').catch(() => {}); + } else blocked('search-results'); + + // Stage: item — open one item's detail view from its card. + if (ok && byStage('item').length) { + const card = page.locator(tid('item-card'), { hasText: REVIEW_ITEM }).first(); + try { + await card.waitFor({ state: 'visible', timeout: CHECK_TIMEOUT }); + await card.locator(tid('item-name')).first().click(); + } catch { + await card.click().catch(() => {}); + } + for (const h of byStage('item')) ok = (await checkHook(page, h, results)) && ok; + } else blocked('item'); + + // Stage: item-after-review — the review must arrive without a reload. + const probe = `lint review ${uniq}`; + if (ok && byStage('item-after-review').length) { + const rating = page.locator(tid('review-rating')).first(); + // A rating control can be a select, a number input or a row of buttons. + await rating.selectOption('5') + .catch(async () => { await rating.fill('5').catch(async () => { await rating.click().catch(() => {}); }); }); + await page.locator(tid('review-input')).first().fill(probe); + await click(page.locator(tid('review-submit')).first(), 'submit review'); + for (const h of byStage('item-after-review')) { + const loc = page.locator(tid(h.id), { hasText: probe }).first(); + try { + await loc.waitFor({ state: 'visible', timeout: CHECK_TIMEOUT }); + pass(h); + } catch { + fail(h, `submitted review "${probe}" but no ${tid(h.id)} containing it appeared without a reload — expected: ${h.element}`); + ok = false; + } + } + } else blocked('item-after-review'); + + // Stage: cart — add a known item, then open the cart. + if (ok && byStage('cart').length) { + // Search reaches the product regardless of ranking or page size. + const search = page.locator(tid('search-input')).first(); + await search.fill(CART_ITEM); + await search.press('Enter').catch(() => {}); + const card = page.locator(tid('search-results')).first() + .locator(tid('item-card'), { hasText: CART_ITEM }).first(); + await card.waitFor({ state: 'visible', timeout: CHECK_TIMEOUT }); + await click(card.locator(tid('add-to-cart')).first(), `add ${CART_ITEM} to cart`); + await search.fill(''); + await search.press('Enter').catch(() => {}); + await click(page.locator(tid('cart-toggle')).first(), 'open cart'); + for (const h of byStage('cart')) ok = (await checkHook(page, h, results)) && ok; + } else blocked('cart'); + + // Stage: after-checkout — the order must show the item that was bought. + if (ok && byStage('after-checkout').length) { + await click(page.locator(tid('checkout-submit')).first(), 'submit checkout'); + await page.waitForTimeout(1500); + const orders = page.locator(tid('orders-toggle')).first(); + if (await orders.isVisible().catch(() => false)) await orders.click(); + for (const h of byStage('after-checkout')) { + if (h.id === 'order-item') { + const loc = page.locator(tid(h.id), { hasText: CART_ITEM }).first(); + try { + await loc.waitFor({ state: 'visible', timeout: CHECK_TIMEOUT }); + pass(h); + } catch { + fail(h, `checked out a cart containing "${CART_ITEM}" but no ${tid(h.id)} naming it appeared — expected: ${h.element}`); + ok = false; + } + } else ok = (await checkHook(page, h, results)) && ok; + } + } else blocked('after-checkout'); + + // Stage: admin — a separate account with its own area. Signing out and back + // in as the seeded admin is the only way to reach it. + if (ok && hasStage('admin', 'operations', 'fulfilment')) { + if (!await page.locator(tid('signout')).first().isVisible()) { + await click(page.locator(tid('current-user')).first(), 'open account menu'); + } + await click(page.locator(tid('signout')).first(), 'sign out'); + await page.waitForTimeout(1000); + const toggle = page.locator(tid('signin-toggle')).first(); + if (await toggle.count()) await toggle.click().catch(() => {}); + await page.locator(tid('signin-username')).first().fill(ADMIN_USER); + await page.locator(tid('signin-password')).first().fill(seededAdminPassword(args.credentialAliases)); + await click(page.locator(tid('signin-submit')).first(), 'submit sign-in'); + await page.waitForTimeout(1500); + const link = page.locator(tid('admin-link')).first(); + if (await link.isVisible().catch(() => false)) await link.click(); + for (const h of byStage('admin')) await checkHook(page, h, results); // check all, non-blocking + + // L2 adds controls to the admin panel itself. Check all of them before + // navigating to fulfilment so a missing control cannot be hidden by the + // later page transition. + for (const h of byStage('operations')) await checkHook(page, h, results); + + // The pending order makes fulfillment controls observable. + const fulfilmentHooks = byStage('fulfilment'); + if (fulfilmentHooks.length) { + const staffLink = page.locator(tid('staff-link')).first(); + if (await staffLink.isVisible().catch(() => false)) { + await staffLink.click(); + for (const h of fulfilmentHooks) await checkHook(page, h, results); + } else { + blocked('fulfilment'); + } + } + } else { + blocked('admin'); + blocked('operations'); + blocked('fulfilment'); + } + + for (const h of byStage('scenario')) { + results.push({ id: h.id, status: 'SCENARIO', detail: h.note }); + } +} diff --git a/tools/stack-bench/tracks/loop/contracts/01-loop.json b/tools/stack-bench/tracks/loop/contracts/01-loop.json new file mode 100644 index 00000000000..231aa09452d --- /dev/null +++ b/tools/stack-bench/tracks/loop/contracts/01-loop.json @@ -0,0 +1,11 @@ +{ + "level": 1, + "hooks": [ + { + "id": "app-title", + "element": "the visible application title", + "stage": "landing", + "check": "visible" + } + ] +} diff --git a/tools/stack-bench/tracks/loop/prompts/01-loop.md b/tools/stack-bench/tracks/loop/prompts/01-loop.md new file mode 100644 index 00000000000..43ad76d91a5 --- /dev/null +++ b/tools/stack-bench/tracks/loop/prompts/01-loop.md @@ -0,0 +1,4 @@ +# Loop fixture + +Expose a visible heading with `id="app-title"` whose text is +`Fixture Chat`. diff --git a/tools/stack-bench/tracks/loop/scenarios/01-loop.json b/tools/stack-bench/tracks/loop/scenarios/01-loop.json new file mode 100644 index 00000000000..855ced9f9a4 --- /dev/null +++ b/tools/stack-bench/tracks/loop/scenarios/01-loop.json @@ -0,0 +1,29 @@ +{ + "schemaVersion": 1, + "level": 1, + "features": [ + { + "id": 1, + "name": "Application title", + "actors": ["viewer"], + "setup": [], + "criteria": [ + { + "id": "title-visible", + "desc": "The application title is visible", + "points": 1, + "statedBy": "Fixture Chat", + "steps": [ + { + "do": "expect", + "actor": "viewer", + "testid": "app-title", + "contains": "Fixture Chat", + "within": 1000 + } + ] + } + ] + } + ] +} diff --git a/tools/stack-bench/tracks/loop/track.json b/tools/stack-bench/tracks/loop/track.json new file mode 100644 index 00000000000..d51e37f4080 --- /dev/null +++ b/tools/stack-bench/tracks/loop/track.json @@ -0,0 +1,19 @@ +{ + "schemaVersion": 1, + "title": "Loop Fixture", + "slug": "loop", + "internal": true, + "validatedThrough": 1, + "plannedThrough": 1, + "portOffset": 600, + "restartProbe": "/", + "suites": { + "1": [ + { + "id": "features", + "inherit": "none", + "spec": "scenarios/01-loop.json" + } + ] + } +} diff --git a/tools/stack-bench/tracks/loop/walk.ts b/tools/stack-bench/tracks/loop/walk.ts new file mode 100644 index 00000000000..8972ece9b76 --- /dev/null +++ b/tools/stack-bench/tracks/loop/walk.ts @@ -0,0 +1,6 @@ +import type { LintWalkContext } from '../../linter/lint.js'; + +export async function walk({ page, args, byStage, checkHook, results }: LintWalkContext): Promise { + await page.goto(args.url ?? ''); + for (const hook of byStage('landing')) await checkHook(page, hook, results); +} diff --git a/tools/stack-bench/tsconfig.build.json b/tools/stack-bench/tsconfig.build.json new file mode 100644 index 00000000000..182363d1fbb --- /dev/null +++ b/tools/stack-bench/tsconfig.build.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": false, + "rootDir": ".", + "outDir": "dist", + "sourceMap": true, + "noEmitOnError": true + } +} diff --git a/tools/stack-bench/tsconfig.json b/tools/stack-bench/tsconfig.json new file mode 100644 index 00000000000..28321a6f1e6 --- /dev/null +++ b/tools/stack-bench/tsconfig.json @@ -0,0 +1,32 @@ +{ + "compilerOptions": { + "target": "ES2023", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2023"], + "types": ["node"], + "strict": true, + "noUncheckedIndexedAccess": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "forceConsistentCasingInFileNames": true, + "verbatimModuleSyntax": true, + "skipLibCheck": false, + "noEmit": true + }, + "include": [ + "appliance/**/*.ts", + "commands/**/*.ts", + "container/**/*.ts", + "dashboard/**/*.ts", + "grader/**/*.ts", + "linter/**/*.ts", + "scripts/**/*.ts", + "src/**/*.ts", + "tests/**/*.ts", + "tracks/**/*.ts" + ], + "exclude": [ + "tests/fixtures/spacetime-module/**" + ] +}